gwlms
/

roberta-tokenizer

Model card Files Files and versions

stefan-it commited on Nov 26, 2024

Commit

7d5882d

·

verified ·

1 Parent(s): cadb455

tokenizer: add fast version

Files changed (1) hide show

tokenizer.json +6 -6

tokenizer.json CHANGED Viewed

@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "[UNK]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,7 +23,7 @@
     },
     {
       "id": 2,
-      "content": "[CLS]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -32,7 +32,7 @@
     },
     {
       "id": 3,
-      "content": "[SEP]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -79,9 +79,9 @@
     "ignore_merges": false,
     "vocab": {
       "[PAD]": 0,
-      "[UNK]": 1,
-      "[CLS]": 2,
-      "[SEP]": 3,
       "[MASK]": 4,
       "!": 5,
       "\"": 6,

     },
     {
       "id": 1,
+      "content": "[CLS]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     },
     {
       "id": 2,
+      "content": "[SEP]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     },
     {
       "id": 3,
+      "content": "[UNK]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
     "ignore_merges": false,
     "vocab": {
       "[PAD]": 0,
+      "[CLS]": 1,
+      "[SEP]": 2,
+      "[UNK]": 3,
       "[MASK]": 4,
       "!": 5,
       "\"": 6,