Add files using upload-large-folder tool

checkpoints/steps_30000_pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d87b99f1d617645b4d1cf2dacb09448ba576d45d1ecef78ebda4bf6f08957abc
+size 8935435793

config.json

@@ -0,0 +1,115 @@
+{
+  "run_id": "0903_libero_goal_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm",
+  "run_root_dir": "./playground/Checkpoints",
+  "seed": 42,
+  "trackers": [
+    "jsonl",
+    "wandb"
+  ],
+  "wandb_entity": "michaelyu-1101-fudanuniversity",
+  "wandb_project": "Internvla",
+  "is_debug": false,
+  "framework": {
+    "framework_py": "DinoQFormerACT",
+    "qwenvl": {
+      "base_vlm": "/mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000",
+      "attn_implementation": "flash_attention_2",
+      "vl_hidden_dim": 2048
+    },
+    "dino": {
+      "dino_backbone": "dinov2_vitl14"
+    },
+    "layer_qformer": {
+      "qformer_end_layer": 37,
+      "qformer_start_layer": 36,
+      "num_query_tokens": 64,
+      "grad_scale": 0.5
+    },
+    "action_model": {
+      "action_model_type": "DiT-B",
+      "action_hidden_dim": 768,
+      "action_dim": 7,
+      "use_ema": false,
+      "future_action_window_size": 7,
+      "past_action_window_size": 0,
+      "repeated_diffusion_steps": 8
+    },
+    "reduce_in_full_precision": true
+  },
+  "datasets": {
+    "vlm_data": {
+      "dataformat": "llava_json",
+      "dataset_use": "asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en",
+      "eval_dataset": "aokvqa_cauldron_llava_format",
+      "data_flatten": false,
+      "base_interval": 2,
+      "max_pixels": 50176,
+      "min_pixels": 784,
+      "fix_image_size": [
+        224,
+        224
+      ],
+      "model_max_length": 1024,
+      "model_type": "qwen2.5vl",
+      "per_device_batch_size": 4
+    },
+    "vla_data": {
+      "dataset_py": "lerobot_libero",
+      "data_root_dir": "playground/Datasets/LEROBOT_LIBERO_DATA",
+      "data_mix": "libero_goal",
+      "action_type": "delta_qpos",
+      "CoT_prompt": "Your task is {instruction}. To identify the key objects for your task. Locate their bounding boxes in [x1,y1,x2,y2] format.",
+      "CoT_answer": "bbox",
+      "default_image_resolution": [
+        3,
+        224,
+        224
+      ],
+      "per_device_batch_size": 16,
+      "load_all_data_for_training": true,
+      "obs": [
+        "image_0"
+      ]
+    }
+  },
+  "trainer": {
+    "epochs": 100,
+    "max_train_steps": 100000,
+    "num_warmup_steps": 5000,
+    "save_interval": 10000,
+    "eval_interval": 1000,
+    "learning_rate": {
+      "base": 2.5e-05
+    },
+    "lr_scheduler_type": "cosine_with_min_lr",
+    "scheduler_specific_kwargs": {
+      "min_lr": 1e-06
+    },
+    "freeze_modules": "",
+    "loss_scale": {
+      "vla": 1.0,
+      "vlm": 0.1
+    },
+    "max_grad_norm": 1.0,
+    "warmup_ratio": 0.1,
+    "weight_decay": 0.0,
+    "logging_frequency": 10,
+    "gradient_clipping": 1.0,
+    "gradient_accumulation_steps": 1,
+    "optimizer": {
+      "name": "AdamW",
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "eps": 1e-08,
+      "weight_decay": 1e-08
+    },
+    "is_resume": false,
+    "resume_epoch": null,
+    "resume_step": null,
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true
+  },
+  "output_dir": "./playground/Checkpoints/0903_libero_goal_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm"
+}

config.yaml

@@ -0,0 +1,96 @@
+run_id: 0903_libero_goal_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm
+run_root_dir: ./playground/Checkpoints
+seed: 42
+trackers:
+- jsonl
+- wandb
+wandb_entity: michaelyu-1101-fudanuniversity
+wandb_project: Internvla
+is_debug: false
+framework:
+  framework_py: DinoQFormerACT
+  qwenvl:
+    base_vlm: /mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000
+    attn_implementation: flash_attention_2
+    vl_hidden_dim: 2048
+  dino:
+    dino_backbone: dinov2_vitl14
+  layer_qformer:
+    qformer_end_layer: 37
+    qformer_start_layer: 36
+    num_query_tokens: 64
+    grad_scale: 0.5
+  action_model:
+    action_model_type: DiT-B
+    action_hidden_dim: 768
+    action_dim: 7
+    use_ema: false
+    future_action_window_size: 7
+    past_action_window_size: 0
+    repeated_diffusion_steps: 8
+  reduce_in_full_precision: true
+datasets:
+  vlm_data:
+    dataformat: llava_json
+    dataset_use: asv2_conversation_en,asv2_detailed_description_en,asv2_region_captioning_en,coco_internvl_longcap_en,coco_karpathy_train_567_en,coco_negative_gpt4o_en,coco_poetry_zh,coco_rem_en_zh,cocorem_exist_yorn_en,cocotextv2_en,cocotextv2_gpt4o_en,okvqa_en,refcoco_grounding_aug_en,refcoco_grounding_en,tallyqa_coco_en,toloka_grounding_aug_en,vqav2_en,vsr_en
+    eval_dataset: aokvqa_cauldron_llava_format
+    data_flatten: false
+    base_interval: 2
+    max_pixels: 50176
+    min_pixels: 784
+    fix_image_size:
+    - 224
+    - 224
+    model_max_length: 1024
+    model_type: qwen2.5vl
+    per_device_batch_size: 4
+  vla_data:
+    dataset_py: lerobot_libero
+    data_root_dir: playground/Datasets/LEROBOT_LIBERO_DATA
+    data_mix: libero_goal
+    action_type: delta_qpos
+    CoT_prompt: Your task is {instruction}. To identify the key objects for your task.
+      Locate their bounding boxes in [x1,y1,x2,y2] format.
+    CoT_answer: bbox
+    default_image_resolution:
+    - 3
+    - 224
+    - 224
+    per_device_batch_size: 16
+    load_all_data_for_training: true
+    obs:
+    - image_0
+trainer:
+  epochs: 100
+  max_train_steps: 100000
+  num_warmup_steps: 5000
+  save_interval: 10000
+  eval_interval: 1000
+  learning_rate:
+    base: 2.5e-05
+  lr_scheduler_type: cosine_with_min_lr
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+  freeze_modules: ''
+  loss_scale:
+    vla: 1.0
+    vlm: 0.1
+  max_grad_norm: 1.0
+  warmup_ratio: 0.1
+  weight_decay: 0.0
+  logging_frequency: 10
+  gradient_clipping: 1.0
+  gradient_accumulation_steps: 1
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 1.0e-08
+  is_resume: false
+  resume_epoch: null
+  resume_step: null
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+output_dir: ./playground/Checkpoints/0903_libero_goal_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm

dataset_statistics.json

@@ -0,0 +1,133 @@
+{
+  "franka": {
+    "action": {
+      "mean": [
+        0.04721052572131157,
+        0.028835246339440346,
+        -0.1485840231180191,
+        -0.0025010062381625175,
+        0.026408178731799126,
+        0.027379808947443962,
+        0.6299911737442017
+      ],
+      "std": [
+        0.3968801498413086,
+        0.3473387360572815,
+        0.49239858984947205,
+        0.055331431329250336,
+        0.07844757288694382,
+        0.10008802264928818,
+        0.48270025849342346
+      ],
+      "max": [
+        0.9375,
+        0.9375,
+        0.9375,
+        0.3557142913341522,
+        0.375,
+        0.375,
+        1.0
+      ],
+      "min": [
+        -0.9375,
+        -0.9375,
+        -0.9375,
+        -0.2582142949104309,
+        -0.375,
+        -0.2871428430080414,
+        0.0
+      ],
+      "q01": [
+        -0.8785714507102966,
+        -0.7553571462631226,
+        -0.9375,
+        -0.1510714292526245,
+        -0.1639285683631897,
+        -0.13777500048279764,
+        0.0
+      ],
+      "q99": [
+        0.9375,
+        0.9107142686843872,
+        0.9375,
+        0.20357142388820648,
+        0.26357144117355347,
+        0.375,
+        1.0
+      ],
+      "mask": [
+        true,
+        true,
+        true,
+        true,
+        true,
+        true,
+        false
+      ]
+    },
+    "state": {
+      "mean": [
+        -0.09923473745584488,
+        0.013597904704511166,
+        1.0694637298583984,
+        2.82898211479187,
+        0.30799180269241333,
+        -0.274286687374115,
+        0.028092455118894577,
+        -0.027339335530996323
+      ],
+      "std": [
+        0.11653962731361389,
+        0.11478105187416077,
+        0.10487838834524128,
+        0.5570293664932251,
+        0.7221656441688538,
+        0.36479514837265015,
+        0.01507475133985281,
+        0.014990941621363163
+      ],
+      "max": [
+        0.13579000532627106,
+        0.33316105604171753,
+        1.3660105466842651,
+        3.473310708999634,
+        2.6688623428344727,
+        0.8255361318588257,
+        0.04233968257904053,
+        0.0010111660230904818
+      ],
+      "min": [
+        -0.46141114830970764,
+        -0.30129560828208923,
+        0.9083037972450256,
+        0.35277295112609863,
+        -1.4858465194702148,
+        -1.5227035284042358,
+        -0.0013586411951109767,
+        -0.042040832340717316
+      ],
+      "q01": [
+        -0.42401049643754957,
+        -0.27338370531797407,
+        0.911226047873497,
+        1.3085840785503386,
+        -0.691297555565834,
+        -1.130668159723282,
+        0.0016738151130266487,
+        -0.040336399003863335
+      ],
+      "q99": [
+        0.08990443304181095,
+        0.26473945528268716,
+        1.2910678112506866,
+        3.2425890421867365,
+        2.3376442337036116,
+        0.4659483411908149,
+        0.040610933862626555,
+        -0.0015016929572448147
+      ]
+    },
+    "num_transitions": 52042,
+    "num_trajectories": 428
+  }
+}

run_lerobot_datasets.sh

@@ -0,0 +1,64 @@
+export HF_HOME=/mnt/petrelfs/share/yejinhui/Models/huggingface_cache
+
+export NCCL_SOCKET_IFNAME=bond0
+export NCCL_IB_HCA=mlx5_2,mlx5_3
+
+# 用于check save 的时候的通信
+export NCCL_BLOCKING_WAIT=1
+export NCCL_ASYNC_ERROR_HANDLING=1
+export NCCL_TIMEOUT=1000  # 超时时间设为 1 小时（单位：秒）
+
+cd /mnt/petrelfs/yujunqiu/code/vla-baseline/llavavla-00hf1
+
+# MODEL_PATH=/mnt/petrelfs/yejinhui/Projects/llavavla/playground/Pretrained_models/Qwen2.5-VL-3B-Instruct # must be a local path, due to simpler will run in other where
+# data_root_dir=./playground/Datasets/OXE_LEROBOT_DATASET
+run_root_dir=./playground/Checkpoints
+task_name=libero_goal
+run_id=0903_${task_name}_augsteps_0_wo_flash_attention_wo_augsteps_two_view_action_chunk_16_pretrained_vlm
+
+
+export WANDB_MODE=disabled
+
+output_dir=${run_root_dir}/${run_id}
+mkdir -p ${output_dir}
+# mv this script to the output dir
+cp $0 ${output_dir}/
+
+  # --pretrained_checkpoint ${MODEL_PATH} \
+# export CUDA_VISIBLE_DEVICES=4,5,6,7
+
+  # --datasets.vla_data.data_mix libero_goal \
+  # --framework.framework_py qwenpi \
+
+DEBUG=False
+# DEBUG=True
+
+if [ "$DEBUG" = True ]; then
+  num_processes=1
+  run_id=debug
+else
+  num_processes=8
+fi
+
+
+accelerate launch \
+  --config_file scripts/run_scripts/deepspeed_zero2.yaml \
+  --num_processes ${num_processes} \
+  llavavla/training/train_qwenvla.py \
+  --config_yaml ./llavavla/config/lerobot_data/qwenvla_cotrain_libero.yaml \
+  --datasets.vla_data.per_device_batch_size 16 \
+  --datasets.vla_data.data_mix ${task_name} \
+  --framework.action_model.future_action_window_size 7 \
+  --trainer.max_train_steps 100_000 \
+  --trainer.save_interval 10_000 \
+  --run_root_dir ${run_root_dir} \
+  --run_id ${run_id} \
+  --wandb_project Internvla \
+  --wandb_entity michaelyu-1101-fudanuniversity \
+  --is_debug ${DEBUG} \
+  --framework.qwenvl.base_vlm /mnt/phwfile/efm_t/zhuyangkun_tmp_need_del/exp/exp_08_09/manip_sys2_qwen25_3b_onevision_molmo_a0all_refsp20/checkpoint-20000
+
+#   --framework.qwenvl.base_vlm ${MODEL_PATH} \
+#   --data_root_dir ${data_root_dir} \
+
+  # --is_debug True

summary.jsonl

@@ -0,0 +1,3 @@
+{"steps": 10000}
+{"steps": 20000}
+{"steps": 30000}