public static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency) {
        ArrayList<Set<NGram>> features = new ArrayList<>(maxNGramSize + 1);
        Set<NGram> feature = new HashSet<>();
        features.add(feature);

        EnglishPunctuations punctuations = EnglishPunctuations.getInstance();
        for (int n = 1; n <= maxNGramSize; n++) {
            Map<smile.nlp.NGram, MutableInt> candidates = new HashMap<>();

            for (String[] sentence : sentences) {
                for (int i = 0; i <= sentence.length - n; i++) {
                    smile.nlp.NGram ngram = new smile.nlp.NGram(Arrays.copyOfRange(sentence, i, i+n));

                    boolean add = false;
                    if (n == 1) {
                        add = true;
                    } else {
                        NGram initialGram  = new NGram(Arrays.copyOfRange(sentence, i, i+n-1), 0);
                        NGram finalGram  = new NGram(Arrays.copyOfRange(sentence, i+1, i+n), 0);
                        if (feature.contains(initialGram) && feature.contains(finalGram)) {
                            add = true;
                        }
                    }

                    if (add) {
                        MutableInt count = candidates.get(ngram);
                        if (count == null) {
                            candidates.put(ngram, new MutableInt(1));
                        } else {
                            count.increment();
                        }
                    }
                }
            }

            feature = new HashSet<>();
            features.add(feature);
            for (Map.Entry<smile.nlp.NGram, MutableInt> entry : candidates.entrySet()) {
                MutableInt count = entry.getValue();
                if (count.value >= minFrequency) {
                    smile.nlp.NGram ngram = entry.getKey();
                    if (ngram.words.length == 1 && punctuations.contains(ngram.words[0])) {
                        continue;
                    }

                    feature.add(new NGram(ngram.words, count.value));
                }
            }
        }

        // filter out stop words
        EnglishStopWords stopWords = EnglishStopWords.DEFAULT;
        return features.stream().map(ngrams -> {
            NGram[] collocations = ngrams.stream().filter(ngram -> {
                boolean stopWord = true;
                String[] words = ngram.words;
                if (!stopWords.contains(words[0]) && !stopWords.contains(words[words.length - 1])) {
                    for (String word : words) {
                        if (!stopWords.contains(word)) {
                            stopWord = false;
                            break;
                        }
                    }
                }
                return !stopWord;
            }).toArray(NGram[]::new);

            Arrays.sort(collocations, Collections.reverseOrder());
            return collocations;
        }).toArray(NGram[][]::new);
    }
@Test
    public void testExtract() throws IOException {
        System.out.println("n-gram extraction");
        String text = new String(Files.readAllBytes(smile.util.Paths.getTestData("text/turing.txt")));

        PorterStemmer stemmer = new PorterStemmer();
        SimpleTokenizer tokenizer = new SimpleTokenizer();
        ArrayList<String[]> sentences = new ArrayList<>();
        for (String paragraph : SimpleParagraphSplitter.getInstance().split(text)) {
            for (String s : SimpleSentenceSplitter.getInstance().split(paragraph)) {
                String[] sentence = tokenizer.split(s);
                for (int i = 0; i < sentence.length; i++) {
                    sentence[i] = stemmer.stripPluralParticiple(sentence[i]).toLowerCase();
                }
                sentences.add(sentence);
            }
        }

        NGram[][] result = NGram.of(sentences, 4, 4);

        assertEquals(5, result.length);
        for (NGram[] ngrams : result) {
            for (NGram ngram : ngrams) {
                System.out.println(ngram);
            }
            System.out.println();
        }

        assertEquals(0, result[0].length);
        assertEquals(331, result[1].length);
        assertEquals(16, result[2].length);
        assertEquals(7, result[3].length);
        assertEquals(0, result[4].length);
    }