Fix non-consecutive token indices new

.ipynb_checkpoints/added_tokens-checkpoint.json

@@ -1,6 +1,5 @@
 {
-  "<|0.00|>": 50365,
-  "<|0.02|>": 50366,
+  "<|0.02|>": 50365,
   "<|0.04|>": 50367,
   "<|0.06|>": 50368,
   "<|0.08|>": 50369,

.ipynb_checkpoints/tokenizer_config-checkpoint.json

@@ -866,14 +866,6 @@
       "special": true
     },
     "50365": {
-      "content": "<|0.00|>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50366": {
       "content": "<|0.02|>",
       "lstrip": false,
       "normalized": true,

added_tokens.json

@@ -1,6 +1,5 @@
 {
-  "<|0.00|>": 50365,
-  "<|0.02|>": 50366,
+  "<|0.02|>": 50365,
   "<|0.04|>": 50367,
   "<|0.06|>": 50368,
   "<|0.08|>": 50369,

tokenizer_config.json

@@ -866,14 +866,6 @@
       "special": true
     },
     "50365": {
-      "content": "<|0.00|>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50366": {
       "content": "<|0.02|>",
       "lstrip": false,
       "normalized": true,