readme

README.md

@@ -8,35 +8,7 @@ datasets:
 ---
 # 🇳🇴 Norwegian T5 Base model Trained on the NCC🇳🇴  
 
-This is a Norwegian T5-base model trained on the Norwegian Colossal Corpus (NCC) on a TPU v3-8. It needs to be finetuned on a specific task before being used for anything.
+This is a Norwegian T5-base model trained on the Norwegian Colossal Corpus (NCC) on a TPU v3-8. 
 
-Currently the model is training. It is expected that it should be finished by the end of August 2021.
-
- The following setting were used in training:
-```bash
-./run_t5_mlm_flax_streaming.py \
-    --output_dir="./" \
-    --model_type="t5" \
-    --config_name="./" \
-    --tokenizer_name="./" \
-    --dataset_name="pere/norwegian_colossal_corpus_v2_short100k" \
-    --max_seq_length="512" \
-    --weight_decay="0.01" \
-    --per_device_train_batch_size="32" \
-    --per_device_eval_batch_size="32" \
-    --learning_rate="8e-3" \
-    --warmup_steps="5000" \
-    --overwrite_output_dir \
-    --cache_dir /mnt/disks/flaxdisk/cache/ \
-    --num_train_epochs="5" \
-    --adam_beta1="0.9" \
-    --adam_beta2="0.98" \
-    --logging_steps="500" \
-    --num_train_steps="1000000" \
-    --num_eval_samples="5000" \
-    --save_steps="5000" \
-    --eval_steps="5000" \
-    --preprocessing_num_workers 96 \
-    --adafactor \
-    --push_to_hub
+This model is currently training. It will finish in January 2022. Please do not use yet..
  ```

events.out.tfevents.1639128562.t1v-n-358ff5d1-w-0.686981.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dfce941b39945a6d6d07c566fd3bc9b6fac8b6004009113d225a917c03538e7
+size 40

events.out.tfevents.1639128677.t1v-n-358ff5d1-w-0.688351.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8932ca5995a80bb77d76b41daf7a8a4d63d987473564afc7e00a9d10158be496
+size 40

events.out.tfevents.1639128772.t1v-n-358ff5d1-w-0.689734.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92f6bb44edb94beb130d22c68fed5473bfea890e3f0770e698d0786b3ae1bb84
+size 40

events.out.tfevents.1639128961.t1v-n-358ff5d1-w-0.691429.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce6c72f09124acb4634a23ea7eb63aec00eba13595fe0887a7ef8116c5361376
+size 40

events.out.tfevents.1639129025.t1v-n-358ff5d1-w-0.692738.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46c8eace8366230b32c94f9c7986a751c6faeb2da07e46805c33cf50072349ba
+size 40

events.out.tfevents.1639129541.t1v-n-358ff5d1-w-0.694454.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbabad7dcae1ed7eeaf7462fd304baeae84d3fe85bf4eb6371738109d9654c7a
+size 40

run.sh

@@ -1,8 +1,8 @@
 ./run_t5_mlm_flax_streaming.py \
-    --output_dir="./" \
+    --output_dir="." \
     --model_type="t5" \
-    --config_name="./" \
-    --tokenizer_name="./" \
+    --config_name="./config.json" \
+    --tokenizer_name="." \
     --dataset_name="NbAiLab/nbailab_extended"  \
     --max_seq_length="512" \
     --weight_decay="0.01" \

run_t5_mlm_flax.py

@@ -599,7 +599,7 @@ if __name__ == "__main__":
     dropout_rngs = jax.random.split(rng, jax.local_device_count())
 
     if model_args.model_name_or_path:
-        model = FlaxT5ForConditionalGeneration.from_pretrained(
+        model = FlaxT5ForConditionalGeneration.from_pretrainedu
             model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
         )
     else:

run_t5_mlm_flax_streaming.py

@@ -554,17 +554,13 @@ if __name__ == "__main__":
     rng = jax.random.PRNGKey(training_args.seed)
     dropout_rngs = jax.random.split(rng, jax.local_device_count())
 
-    #Pere changed 13 august
-    #model = FlaxT5ForConditionalGeneration(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype))
 
     if model_args.model_name_or_path:
         model = FlaxT5ForConditionalGeneration.from_pretrained(
             model_args.model_name_or_path, config=config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
         )
     else:
-        model = FlaxT5ForConditionalGeneration.from_pretrained(
-            config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype)
-        )
+        model = FlaxT5ForConditionalGeneration(config, seed=training_args.seed, dtype=getattr(jnp, model_args.dtype))
 
 
     # Data collator