private void execute(Path commonTokensFile, List<Path> inputFiles) throws Exception {
        Path luceneDir = Files.createTempDirectory("tika-eval-lucene-");
        AbstractTokenTFDFPriorityQueue queue = new TokenDFPriorityQueue(TOP_N);
        long totalDocs = -1;
        long sumDocFreqs = -1;
        long sumTotalTermFreqs = -1;
        long uniqueTerms = -1;
        try (Directory directory = FSDirectory.open(luceneDir)) {

            AnalyzerManager analyzerManager = AnalyzerManager.newInstance(-1);

            Analyzer analyzer = analyzerManager.getCommonTokensAnalyzer();
            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
            int maxLen = 1000000;
            int len = 0;
            try (IndexWriter writer = new IndexWriter(directory, indexWriterConfig)) {
                List<Document> docs = new ArrayList<>();
                for (Path inputFile : inputFiles) {
                    //total hack
                    boolean isLeipzig = inputFile
                            .getFileName()
                            .toString()
                            .contains("-sentences.txt");
                    int lines = 0;
                    try (BufferedReader reader = getReader(inputFile)) {
                        String line = reader.readLine();
                        while (line != null) {
                            if (isLeipzig) {
                                int tab = line.indexOf("\t");
                                if (tab > -1) {
                                    line = line.substring(tab + 1);
                                }
                            }
                            len += line.length();
                            Document document = new Document();
                            document.add(new TextField(FIELD, line, Field.Store.NO));
                            docs.add(document);
                            if (len > maxLen) {
                                writer.addDocuments(docs);
                                docs.clear();
                                len = 0;
                            }
                            line = reader.readLine();
                            if (++lines % 100000 == 0) {
                                System.out.println("processed " + lines + " for " + inputFile.getFileName() + " :: " + commonTokensFile.toAbsolutePath());
                            }
                        }
                    }
                }
                if (docs.size() > 0) {
                    writer.addDocuments(docs);
                }
                writer.commit();
                writer.flush();
            }

            try (IndexReader reader = DirectoryReader.open(directory)) {
                LeafReader wrappedReader = SlowCompositeReaderWrapper.wrap(reader);
                totalDocs = wrappedReader.getDocCount(FIELD);
                sumDocFreqs = wrappedReader.getSumDocFreq(FIELD);
                sumTotalTermFreqs = wrappedReader.getSumTotalTermFreq(FIELD);

                Terms terms = wrappedReader.terms(FIELD);
                TermsEnum termsEnum = terms.iterator();
                BytesRef bytesRef = termsEnum.next();
                int docsWThisField = wrappedReader.getDocCount(FIELD);
                while (bytesRef != null) {
                    uniqueTerms++;
                    int df = termsEnum.docFreq();
                    long tf = termsEnum.totalTermFreq();
                    if (MIN_DOC_FREQ > -1 && df < MIN_DOC_FREQ) {
                        bytesRef = termsEnum.next();
                        continue;
                    }

                    if (queue.top() == null || queue.size() < TOP_N || df >= queue.top().df) {
                        String t = bytesRef.utf8ToString();
                        if (!SKIP_LIST.contains(t)) {
                            queue.insertWithOverflow(new TokenDFTF(t, df, tf));
                        }

                    }
                    bytesRef = termsEnum.next();
                }
            }
        } finally {
            FileUtils.deleteDirectory(luceneDir.toFile());
        }

        writeTopN(commonTokensFile, totalDocs, sumDocFreqs, sumTotalTermFreqs, uniqueTerms, queue);


    }
Utility class that reads in a UTF-8 input file with one document per row
and outputs the 20000 tokens with the highest document frequencies.
<p>
The CommmonTokensAnalyzer intentionally drops tokens shorter than 4 characters,
but includes bigrams for cjk.
<p>
It also has a include list for __email__ and __url__ and a skip list
for common html markup terms.
tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/tools/TopCommonTokenCounter.java