goabonga commited on
Commit
7f2d321
·
verified ·
1 Parent(s): d699bee

Upload tokenizer files (vocab, config, README)

Browse files
Files changed (1) hide show
  1. tokenizer.json +6 -6
tokenizer.json CHANGED
@@ -1057,11 +1057,11 @@
1057
  "Australia": 999
1058
  },
1059
  "unk_token": "<unk>",
1060
- "special": {
1061
- "pad_token": "<pad>",
1062
- "unk_token": "<unk>",
1063
- "bos_token": "<bos>",
1064
- "eos_token": "<eos>"
1065
- }
1066
  }
1067
  }
 
1057
  "Australia": 999
1058
  },
1059
  "unk_token": "<unk>",
1060
+ "special": [
1061
+ 0,
1062
+ 1,
1063
+ 2,
1064
+ 3
1065
+ ]
1066
  }
1067
  }