Upload tokenizer files (vocab, config, README)
Browse files- tokenizer.json +1 -2
tokenizer.json
CHANGED
|
@@ -59,8 +59,7 @@
|
|
| 59 |
]
|
| 60 |
},
|
| 61 |
"decoder": {
|
| 62 |
-
"type": "
|
| 63 |
-
"unk_token": "<unk>"
|
| 64 |
},
|
| 65 |
"special": [
|
| 66 |
0,
|
|
|
|
| 59 |
]
|
| 60 |
},
|
| 61 |
"decoder": {
|
| 62 |
+
"type": "WordLevel"
|
|
|
|
| 63 |
},
|
| 64 |
"special": [
|
| 65 |
0,
|