void count(String field) throws IOException {
        long tokenCount = leafReader.getSumTotalTermFreq(field);
        if (tokenCount > Integer.MAX_VALUE) {
            throw new IllegalArgumentException("can't handle longs");
        }
        int tokenCountInt = (int) tokenCount;
        int uniqueTokenCount = 0;
        SummaryStatistics summStats = new SummaryStatistics();
        double ent = 0.0d;
        double p = 0.0d;
        double base = 2.0;

        Terms terms = leafReader.terms(field);
        if (terms == null) {
            //if there were no terms
            fieldStats.put(field,
                    new TokenStatistics(uniqueTokenCount, tokenCountInt, new TokenIntPair[0], ent,
                            summStats));
            return;

        }
        TermsEnum termsEnum = terms.iterator();
        BytesRef bytesRef = termsEnum.next();
        TokenCountPriorityQueue queue = new TokenCountPriorityQueue(topN);

        while (bytesRef != null) {

            long termFreq = termsEnum.totalTermFreq();
            if (termFreq > Integer.MAX_VALUE) {
                throw new IllegalArgumentException("Sorry can't handle longs yet");
            }
            int tf = (int) termFreq;
            //TODO: figure out how to avoid Stringifying this
            //to get codepoint count
            String t = bytesRef.utf8ToString();
            int len = t.codePointCount(0, t.length());
            for (int i = 0; i < tf; i++) {
                summStats.addValue(len);
            }
            p = (double) tf / (double) tokenCount;
            ent += p * FastMath.log(base, p);

            if (queue.top() == null || queue.size() < topN || tf >= queue.top().getValue()) {
                queue.insertWithOverflow(new TokenIntPair(t, tf));
            }

            uniqueTokenCount++;
            bytesRef = termsEnum.next();
        }
        if (tokenCountInt > 0) {
            ent = (-1.0d / (double) tokenCountInt) * ent;
        }

        fieldStats.put(field,
                new TokenStatistics(uniqueTokenCount, tokenCountInt, queue.getArray(), ent,
                        summStats));
    }
Experimental class uses Lucene's MemoryIndex to effectively build the
token info.
tika-eval/tika-eval-core/src/test/java/org/apache/tika/eval/core/tokens/LuceneTokenCounter.java