private synchronized void tryToLoad(String langCode) {
        if (alreadyTriedToLoad.contains(langCode)) {
            return;
        }
        //check once more now that we're in a
        //synchronized block
        if (commonTokenMap.get(langCode) != null) {
            return;
        }
        InputStream is = null;
        Path p = null;
        if (commonTokensDir != null) {
            p = commonTokensDir.resolve(langCode);
        }

        try {
            if (p == null || !Files.isRegularFile(p)) {
                is = this.getClass().getResourceAsStream("/common_tokens/" + langCode);
            } else {
                is = Files.newInputStream(p);
            }


            if (is == null) {
                String path = (p == null) ? "resource on class path: /common_tokens/" + langCode :
                        p.toAbsolutePath().toString();
                LOG.warn("Couldn't find common tokens file for: '" + langCode + "' tried here: " +
                        path);
                alreadyTriedToLoad.add(langCode);
                return;
            }

            LangModel model = null;
            try (BufferedReader reader = new BufferedReader(
                    new InputStreamReader(is, COMMON_TOKENS_CHARSET))) {
                alreadyTriedToLoad.add(langCode);
                String line = reader.readLine();
                while (line != null) {
                    line = line.trim();
                    if (line.startsWith("#")) {
                        if (line.startsWith(TERM_FREQS)) {
                            digitsMatcher.reset(line);
                            if (digitsMatcher.find()) {
                                model = new LangModel(Long.parseLong(digitsMatcher.group(1)));
                            }
                        }
                        line = reader.readLine();
                        continue;
                    }
                    //allow language models with, e.g. tab-delimited counts after the term
                    String[] cols = line.split("\t");
                    String t = cols[0].trim();
                    if (t.length() > 0 && cols.length > 2) {
                        if (model == null) {
                            throw new IllegalArgumentException(
                                    "Common tokens file must have included comment line " +
                                            " with " + TERM_FREQS);
                        }
                        //document frequency
                        String df = cols[1];
                        //token frequency
                        long tf = Long.parseLong(cols[2]);
                        model.add(t, tf);
                    }

                    line = reader.readLine();
                }
            }
            commonTokenMap.put(langCode, model);
        } catch (IOException e) {
            LOG.warn("IOException trying to read: '" + langCode + "'");
        } finally {
            IOUtils.closeQuietly(is);
        }
    }
@param lang
@return pair of actual language code used and a set of common
tokens for that language
tika-eval/tika-eval-core/src/main/java/org/apache/tika/eval/core/tokens/CommonTokenCountManager.java