Upload tokenizer files (vocab, config, README)

tokenizer.json

@@ -1,43 +1,5 @@
 {
   "version": "1.0",
-  "added_tokens": [
-    {
-      "id": 0,
-      "content": "<pad>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 1,
-      "content": "<unk>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 2,
-      "content": "<bos>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 3,
-      "content": "<eos>",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    }
-  ],
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
@@ -72,26 +34,6 @@
       "special": true
     }
   },
-  "normalizer": {
-    "type": "Sequence",
-    "normalizers": [
-      {
-        "type": "NFC"
-      }
-    ]
-  },
-  "pre_tokenizer": {
-    "type": "Sequence",
-    "pretokenizers": [
-      {
-        "type": "Whitespace"
-      },
-      {
-        "type": "Punctuation",
-        "behavior": "Isolated"
-      }
-    ]
-  },
   "decoder": {
     "type": "WordPiece",
     "unk_token": "<unk>"
@@ -102,12 +44,6 @@
     2,
     3
   ],
-  "special_tokens": {
-    "pad_token": 0,
-    "unk_token": 1,
-    "bos_token": 2,
-    "eos_token": 3
-  },
   "model": {
     "type": "WordLevel",
     "vocab": {
@@ -1119,7 +1055,7 @@
     "single": [
       {
         "SpecialToken": {
-          "id": "<bos>",
+          "id": 2,
           "type_id": 0
         }
       },
@@ -1131,7 +1067,7 @@
       },
       {
         "SpecialToken": {
-          "id": "<eos>",
+          "id": 3,
           "type_id": 0
         }
       }
@@ -1139,7 +1075,7 @@
     "pair": [
       {
         "SpecialToken": {
-          "id": "<bos>",
+          "id": 2,
           "type_id": 0
         }
       },
@@ -1157,14 +1093,14 @@
       },
       {
         "SpecialToken": {
-          "id": "<eos>",
+          "id": 3,
           "type_id": 0
         }
       }
     ],
     "special_tokens": {
       "<pad>": {
-        "id": "<pad>",
+        "id": 0,
         "ids": [
           0
         ],
@@ -1173,7 +1109,7 @@
         ]
       },
       "<unk>": {
-        "id": "<unk>",
+        "id": 1,
         "ids": [
           1
         ],
@@ -1182,7 +1118,7 @@
         ]
       },
       "<bos>": {
-        "id": "<bos>",
+        "id": 2,
         "ids": [
           2
         ],
@@ -1191,7 +1127,7 @@
         ]
       },
       "<eos>": {
-        "id": "<eos>",
+        "id": 3,
         "ids": [
           3
         ],