YanSte
/

t5_large_fine_tuning_question_answering_hc3_chatgpt_prompts

Text Generation

text2text-generation

Model card Files Files and versions

Upload tokenizer

#6

by asma-j-alali - opened 17 days ago

base: refs/heads/main

←

from: refs/pr/6

Discussion Files changed

Files changed (2) hide show

tokenizer.json +4 -4
tokenizer_config.json +3 -1

tokenizer.json CHANGED Viewed

@@ -950,8 +950,8 @@
   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
   },
   "post_processor": {
     "type": "TemplateProcessing",
@@ -1010,8 +1010,8 @@
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
-    "add_prefix_space": true,
-    "prepend_scheme": "always"
   },
   "model": {
     "type": "Unigram",

   "pre_tokenizer": {
     "type": "Metaspace",
     "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
   },
   "post_processor": {
     "type": "TemplateProcessing",
   "decoder": {
     "type": "Metaspace",
     "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
   },
   "model": {
     "type": "Unigram",

tokenizer_config.json CHANGED Viewed

@@ -1,4 +1,5 @@
 {
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
@@ -927,9 +928,10 @@
     "<extra_id_98>",
     "<extra_id_99>"
   ],
-  "clean_up_tokenization_spaces": true,
   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},

 {
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<pad>",
     "<extra_id_98>",
     "<extra_id_99>"
   ],
+  "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_ids": 100,
+  "extra_special_tokens": {},
   "model_max_length": 512,
   "pad_token": "<pad>",
   "sp_model_kwargs": {},