private void _add(String field, Analyzer analyzer, String content) throws IOException {
        int totalTokens = 0;

        TokenStream ts = analyzer.tokenStream(field, content);
        CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
        ts.reset();
        Map<String, MutableInt> tokenMap = map.computeIfAbsent(field, k -> new HashMap<>());
        while (ts.incrementToken()) {
            String token = termAtt.toString();
            MutableInt cnt = tokenMap.get(token);
            if (cnt == null) {
                cnt = new MutableInt(1);
                tokenMap.put(token, cnt);
            } else {
                cnt.increment();
            }
            totalTokens++;
        }
        ts.close();
        ts.end();

        int totalUniqueTokens = tokenMap.size();

        double ent = 0.0d;
        double p = 0.0d;
        double base = 2.0;

        org.apache.tika.eval.core.textstats.TokenCountPriorityQueue queue =
                new TokenCountPriorityQueue(topN);

        SummaryStatistics summaryStatistics = new SummaryStatistics();
        for (Map.Entry<String, MutableInt> e : tokenMap.entrySet()) {
            String token = e.getKey();
            int termFreq = e.getValue().intValue();

            p = (double) termFreq / (double) totalTokens;
            ent += p * FastMath.log(base, p);
            int len = token.codePointCount(0, token.length());
            for (int i = 0; i < e.getValue().intValue(); i++) {
                summaryStatistics.addValue(len);
            }
            if (queue.top() == null || queue.size() < topN || termFreq >= queue.top().getValue()) {
                queue.insertWithOverflow(new TokenIntPair(token, termFreq));
            }

        }
        if (totalTokens > 0) {
            ent = (-1.0d / (double) totalTokens) * ent;
        }

/*            Collections.sort(allTokens);
            List<TokenIntPair> topNList = new ArrayList<>(topN);
            for (int i = 0; i < topN && i < allTokens.size(); i++) {
                topNList.add(allTokens.get(i));
            }*/

        tokenStatistics.put(field,
                new TokenStatistics(totalUniqueTokens, totalTokens, queue.getArray(), ent,
                        summaryStatistics));

    }
@deprecated use {@link CompositeTextStatsCalculator}
with {@link TokenEntropy},
{@link TokenLengths}
and {@link TopNTokens}.
tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/TokenCounter.java