Use chat_template.jinja as the single source: drop the {% include %} chat_template field from tokenizer_config.json
Browse files- tokenizer_config.json +2 -3
tokenizer_config.json
CHANGED
|
@@ -571,6 +571,5 @@
|
|
| 571 |
"pad_token": "〈|PAD|〉",
|
| 572 |
"sep_token": "〈|SEP|〉",
|
| 573 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 574 |
-
"unk_token": "〈|UNK|〉"
|
| 575 |
-
|
| 576 |
-
}
|
|
|
|
| 571 |
"pad_token": "〈|PAD|〉",
|
| 572 |
"sep_token": "〈|SEP|〉",
|
| 573 |
"tokenizer_class": "PreTrainedTokenizerFast",
|
| 574 |
+
"unk_token": "〈|UNK|〉"
|
| 575 |
+
}
|
|
|