static TokenizerConfig loadTokenizer(Path tokenizerPath) throws IOException {
        ObjectMapper mapper = new ObjectMapper(new JsonFactory());
        JsonNode rootNode = mapper.readTree(tokenizerPath.toFile());
        // The tokenizer file is a JSON object with the following schema
        /*
         * {
         *   "version": "1.0",
         *   "truncation": null,
         *   "padding": null,
         *   "added_tokens": [
         *     {
         *       "id": 0,
         *       "special": true,
         *       "content": "[PAD]",
         *       "single_word": false,
         *       "lstrip": false,
         *       "rstrip": false,
         *       "normalized": false
         *     }
         *   ],
         *   "normalizer": {
         *     "type": "BertNormalizer",
         *     "clean_text": true,
         *     "handle_chinese_chars": true,
         *     "strip_accents": null,
         *     "lowercase": false
         *   },
         *   "pre_tokenizer": {
         *     "type": "BertPreTokenizer"
         *   },
         *   "post_processor": {
         *     "type": "TemplateProcessing",
         *     "single": [
         *       {
         *         "SpecialToken": {
         *           "id": "[CLS]",
         *           "type_id": 0
         *         }
         *       },
         *       {
         *         "Sequence": {
         *           "id": "A",
         *           "type_id": 0
         *         }
         *       },
         *       {
         *         "SpecialToken": {
         *           "id": "[SEP]",
         *           "type_id": 0
         *         }
         *       }
         *     ],
         *     "pair": [
         *       {
         *         "SpecialToken": {
         *           "id": "[CLS]",
         *           "type_id": 0
         *         }
         *       },
         *       {
         *         "Sequence": {
         *           "id": "A",
         *           "type_id": 0
         *         }
         *       },
         *       {
         *         "SpecialToken": {
         *           "id": "[SEP]",
         *           "type_id": 0
         *         }
         *       },
         *       {
         *         "Sequence": {
         *           "id": "B",
         *           "type_id": 1
         *         }
         *       },
         *       {
         *         "SpecialToken": {
         *           "id": "[SEP]",
         *           "type_id": 1
         *         }
         *       }
         *     ],
         *     "special_tokens": {
         *       "[SEP]": {
         *         "id": "[SEP]",
         *         "ids": [
         *           102
         *         ],
         *         "tokens": [
         *           "[SEP]"
         *         ]
         *       },
         *       "[CLS]": {
         *         "id": "[CLS]",
         *         "ids": [
         *           101
         *         ],
         *         "tokens": [
         *           "[CLS]"
         *         ]
         *       }
         *     }
         *   },
         *   "decoder": {
         *     "type": "WordPiece",
         *     "prefix": "##",
         *     "cleanup": true
         *   },
         *   "model": {
         *     "unk_token": "[UNK]",
         *     "continuing_subword_prefix": "##",
         *     "max_input_chars_per_word": 100,
         *     "vocab": {
         *       "[PAD]": 0,
         *       ...
         *       }
         *   }
         * }
         */

        Map<String,Integer> vocabMap = new HashMap<>();
        String unknownToken;
        String classificationToken;
        String separatorToken;
        boolean lowercase = false;
        boolean stripAccents = false;
        int maxInputCharsPerWord = 100;

        // Parse out token normalization settings
        JsonNode normalizer = rootNode.get("normalizer");
        if (normalizer != null) {
            lowercase = normalizer.get("lowercase").asBoolean();
            stripAccents = normalizer.get("strip_accents").asBoolean();
        } else {
            throw new IllegalStateException("Failed to parse tokenizer json, did not find the normalizer");
        }

        // Parse out classification and separator tokens
        JsonNode postProcessor = rootNode.get("post_processor");
        if (postProcessor != null) {
            String processorType = postProcessor.get("type").asText();
            if (processorType != null && processorType.equals("TemplateProcessing")) {
                JsonNode specialTokens = postProcessor.get("special_tokens");
                if (specialTokens != null) {
                    JsonNode sepNode = specialTokens.get(SEPARATOR_TOKEN);
                    if (sepNode != null) {
                        separatorToken = sepNode.get("tokens").get(0).asText();
                    } else {
                        throw new IllegalStateException("Failed to parse tokenizer json, did not find separator token.");
                    }
                    JsonNode classificationNode = specialTokens.get(CLASSIFICATION_TOKEN);
                    if (classificationNode != null) {
                        classificationToken = classificationNode.get("tokens").get(0).asText();
                    } else {
                        throw new IllegalStateException("Failed to parse tokenizer json, did not find classification token.");
                    }
                } else {
                    throw new IllegalStateException("Failed to parse tokenizer json, did not find the special tokens.");
                }
            } else if (processorType != null && processorType.equals("BertProcessing")) {
                JsonNode sepNode = postProcessor.get("sep");
                if (sepNode != null) {
                    separatorToken = sepNode.get(0).asText();
                } else {
                    throw new IllegalStateException("Failed to parse tokenizer json, did not find separator token.");
                }
                JsonNode clsNode = postProcessor.get("cls");
                if (clsNode != null) {
                    classificationToken = clsNode.get(0).asText();
                } else {
                    throw new IllegalStateException("Failed to parse tokenizer json, did not find classification token.");
                }
            } else {
                throw new IllegalStateException("Failed to parse tokenizer json, did not recognise post_processor:type " + processorType);
            }
        } else {
            throw new IllegalStateException("Failed to parse tokenizer json, did not find the post processor");
        }

        // Parse out tokens and ids
        JsonNode model = rootNode.get("model");
        if (model != null) {
            unknownToken = model.get("unk_token").asText();
            if (unknownToken == null || unknownToken.isEmpty()) {
                throw new IllegalStateException("Failed to parse tokenizer json, did not extract unknown token");
            }
            maxInputCharsPerWord = model.get("max_input_chars_per_word").asInt();
            if (maxInputCharsPerWord == 0) {
                throw new IllegalStateException("Failed to parse tokenizer json, did not extract max_input_chars_per_word");
            }
            JsonNode vocab = model.get("vocab");
            if (vocab != null) {
                for (Iterator<Map.Entry<String,JsonNode>> termItr = vocab.fields(); termItr.hasNext();) {
                    Map.Entry<String,JsonNode> term = termItr.next();
                    int value = term.getValue().asInt(-1);

                    if (value == -1) {
                        throw new IllegalStateException("Failed to parse tokenizer json, could not extract vocab item '" + term.getKey() + "'");
                    } else {
                        vocabMap.put(term.getKey(),value);
                    }
                }
            } else {
                throw new IllegalStateException("Failed to parse tokenizer json, did not extract vocab");
            }
        } else {
            throw new IllegalStateException("Failed to parse tokenizer json, did not find the model");
        }
        return new TokenizerConfig(vocabMap,unknownToken,classificationToken,separatorToken,lowercase,stripAccents,maxInputCharsPerWord);
    }
@Test
    public void testTokenizerLoading() throws URISyntaxException, IOException {
        Path vocabPath = Paths.get(BERTFeatureExtractorTest.class.getResource("bert-base-cased-vocab.txt").toURI());
        Path tokenizerPath = Paths.get(BERTFeatureExtractorTest.class.getResource("bert-base-cased-tokenizer.json").toURI());
        BERTFeatureExtractor.TokenizerConfig config = BERTFeatureExtractor.loadTokenizer(tokenizerPath);

        List<String> vocabList = Files.readAllLines(vocabPath, StandardCharsets.UTF_8);

        Assertions.assertEquals(config.tokenIDs.size(),vocabList.size());

        for (String vocabElement : vocabList) {
            Assertions.assertTrue(config.tokenIDs.containsKey(vocabElement));
        }

        Assertions.assertEquals(100, config.maxInputCharsPerWord);
        Assertions.assertEquals(false, config.lowercase);
        Assertions.assertEquals(false, config.stripAccents);
        Assertions.assertEquals("[UNK]",config.unknownToken);
        Assertions.assertEquals("[CLS]",config.classificationToken);
        Assertions.assertEquals("[SEP]",config.separatorToken);

        tokenizerPath = Paths.get(BERTFeatureExtractorTest.class.getResource("tinybert-tokenizer.json").toURI());
        config = BERTFeatureExtractor.loadTokenizer(tokenizerPath);
        Assertions.assertEquals(100, config.maxInputCharsPerWord);
        Assertions.assertEquals(true, config.lowercase);
        Assertions.assertEquals(true, config.stripAccents);
        Assertions.assertEquals("[UNK]",config.unknownToken);
        Assertions.assertEquals("[CLS]",config.classificationToken);
        Assertions.assertEquals("[SEP]",config.separatorToken);
    }