add r2r models

r2r/fine_tuned/cma/config.json

@@ -0,0 +1,134 @@
+{
+  "architectures": [
+    "CMANet"
+  ],
+  "model_cfg": {
+    "checkpoint_folder": "data/checkpoints/20250619_cma_train/ckpts",
+    "eval": {
+      "action": null,
+      "ckpt_to_load": "",
+      "len_traj_act": null,
+      "max_steps": 195,
+      "num_sample": null,
+      "pm_threshold": null,
+      "rotation_threshold": null,
+      "sample": false,
+      "save_results": true,
+      "split": [
+        "val_seen"
+      ],
+      "start_eval_epoch": -1,
+      "step_interval": 50,
+      "stop_mode": null,
+      "success_distance": 3.0,
+      "use_ckpt_config": false
+    },
+    "il": {
+      "batch_size": 2,
+      "camera_name": "pano_camera_0",
+      "ckpt_to_load": "",
+      "dataset_3dgs_root_dir": "data/datasets/3dgs",
+      "dataset_grutopia10_root_dir": "data/datasets/grutopia10",
+      "dataset_r2r_root_dir": "data/datasets/R2R_VLNCE_v1-3_preprocessed",
+      "epochs": 80,
+      "filter_failure": {
+        "min_rgb_nums": 15,
+        "use": true
+      },
+      "inflection_weight_coef": 3.2,
+      "lmdb_features_dir": "data/sample_episodes/20250211_sample_origin/sample_data.lmdb",
+      "lmdb_map_size": 1000000000000.0,
+      "load_from_ckpt": false,
+      "load_from_pretrain": false,
+      "loss": {
+        "alpha": 0.0001,
+        "dist_scale": 1
+      },
+      "lr": 0.0001,
+      "num_workers": 8,
+      "report_to": "wandb",
+      "save_filter_frozen_weights": false,
+      "save_interval_epochs": 5,
+      "save_interval_steps": null,
+      "use_descrete_dataset": null,
+      "use_iw": true,
+      "warmup_ratio": 0.05,
+      "weight_decay": 1e-05
+    },
+    "local_rank": 0,
+    "log_dir": "data/checkpoints/20250619_cma_train/logs",
+    "model": {
+      "ablate_depth": null,
+      "ablate_instruction": null,
+      "ablate_rgb": null,
+      "cross_modal_encoder": null,
+      "depth_encoder": {
+        "backbone": "resnet50",
+        "cnn_type": "VlnResnetDepthEncoder",
+        "ddppo_checkpoint": "data/ddppo-models/gibson-4plus-mp3d-train-val-test-resnet50.pth",
+        "output_size": 128,
+        "trainable": false
+      },
+      "diffusion_policy": null,
+      "distance_predictor": null,
+      "image_encoder": null,
+      "imu_encoder": null,
+      "instruction_encoder": {
+        "bidirectional": true,
+        "dataset_vocab": "data/datasets/R2R_VLNCE_v1-3_preprocessed/train/train.json.gz",
+        "embedding_file": "data/datasets/R2R_VLNCE_v1-3_preprocessed/embeddings.json.gz",
+        "embedding_size": 50,
+        "final_state_only": true,
+        "fine_tune_embeddings": false,
+        "hidden_size": 128,
+        "load_model": null,
+        "max_length": null,
+        "rnn_type": "LSTM",
+        "sensor_uuid": "instruction",
+        "use_pretrained_embeddings": true,
+        "vocab_size": 2504
+      },
+      "learn_angle": null,
+      "len_traj_act": 4,
+      "max_step": 200,
+      "normalize_rgb": null,
+      "policy_name": "CMA_Policy",
+      "prev_action_encoder": null,
+      "progress_monitor": {
+        "alpha": 1.0,
+        "concat_state_txt": null,
+        "use": true
+      },
+      "rgb_encoder": {
+        "cnn_type": "TorchVisionResNet50",
+        "output_size": 256,
+        "trainable": false
+      },
+      "seq2seq": null,
+      "state_encoder": {
+        "dropout_rate": null,
+        "hidden_size": 512,
+        "num_recurrent_layers": 2,
+        "rgb_depth_embed_method": null,
+        "rnn_type": "GRU",
+        "use_dropout": null
+      },
+      "stop_progress_predictor": null,
+      "text_encoder": null
+    },
+    "model_name": "cma",
+    "name": "20250619_cma_train",
+    "num_gpus": 1,
+    "output_dir": "data/checkpoints/20250619_cma_train/ckpts",
+    "seed": 0,
+    "tensorboard_dir": "data/checkpoints/20250619_cma_train/tensorboard",
+    "torch_gpu_id": 0,
+    "torch_gpu_ids": [
+      0
+    ],
+    "world_size": 1
+  },
+  "model_type": "cma",
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1"
+}

r2r/fine_tuned/cma/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:572c625623155f301a114485987128d1317541940fe94432fa92e62e532d66d3
+size 49260090

r2r/fine_tuned/cma/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:686f32f754bd0382831574ba3a29090b5e1d54edc0cbd48d89f5a816a67a4b69
+size 147809914

r2r/fine_tuned/cma/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91a9d8b0c5f7466cc65f08b3acfeb448ddb0262da4161bb9d8e89a0ac4f17caa
+size 14244

r2r/fine_tuned/cma/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f081d2d07b19d3825f13778a499dadd1508d8cabea562ed222a9c7c64094e28
+size 1064

r2r/fine_tuned/cma_plus/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91737ae7fa4a190a53efa6c843c9cbe0319db1d024c9da55eaea84fbef4e96b3
+size 147803318

r2r/fine_tuned/rdp/config.json

@@ -0,0 +1,253 @@
+{
+  "architectures": [
+    "RDPNet"
+  ],
+  "model_cfg": {
+    "checkpoint_folder": "data/checkpoints/20250619_rdp_train_new/ckpts",
+    "eval": {
+      "action": "descrete",
+      "ckpt_to_load": null,
+      "len_traj_act": 2,
+      "max_steps": 195,
+      "num_sample": 1,
+      "pm_threshold": 0.9,
+      "rotation_threshold": 0.01,
+      "sample": true,
+      "save_results": true,
+      "split": [
+        "val_unseen"
+      ],
+      "start_eval_epoch": -1,
+      "step_interval": 80,
+      "stop_mode": "stop_progress",
+      "success_distance": 3.0,
+      "use_ckpt_config": false
+    },
+    "il": {
+      "batch_size": 4,
+      "camera_name": "pano_camera_0",
+      "ckpt_to_load": "",
+      "dataset_3dgs_root_dir": "data/datasets/3dgs",
+      "dataset_grutopia10_root_dir": "data/datasets/grutopia10",
+      "dataset_r2r_root_dir": "data/datasets/R2R_VLNCE_v1-3_corrected",
+      "epochs": 50,
+      "filter_failure": {
+        "min_rgb_nums": 15,
+        "use": true
+      },
+      "inflection_weight_coef": null,
+      "lmdb_features_dir": "data/sample_episodes/20250211_sample_origin/sample_data.lmdb",
+      "load_from_ckpt": false,
+      "load_from_pretrain": true,
+      "loss": {
+        "alpha": 0.0001,
+        "dist_scale": 1
+      },
+      "lr": 0.0001,
+      "num_workers": 8,
+      "report_to": "wandb",
+      "save_filter_frozen_weights": true,
+      "save_interval_epochs": 5,
+      "save_interval_steps": null,
+      "use_descrete_dataset": true,
+      "use_iw": null,
+      "warmup_ratio": 0.1,
+      "weight_decay": 0.0001
+    },
+    "local_rank": 0,
+    "log_dir": "data/checkpoints/20250619_rdp_train_new/logs",
+    "model": {
+      "ablate_depth": null,
+      "ablate_instruction": null,
+      "ablate_rgb": null,
+      "cross_modal_encoder": {
+        "hidden_size": 512,
+        "input_type": 3,
+        "load_model": false,
+        "num_attention_heads": 8,
+        "num_x_layers": 2,
+        "txt_to_img": true,
+        "txt_to_img_layer": 2
+      },
+      "depth_encoder": null,
+      "diffusion_policy": {
+        "action_stats": {
+          "max": [
+            0.25,
+            0.25,
+            0.27
+          ],
+          "min": [
+            -0.25,
+            -0.25,
+            -0.27
+          ]
+        },
+        "clip_sample": true,
+        "cls_free_guidance_scale": 1.5,
+        "cls_mask_method": "mask_token",
+        "cls_mask_ratio": 0.25,
+        "len_traj_pred": 8,
+        "metric_waypoint_spacing": 1,
+        "num_diffusion_iters": 20,
+        "pred_type": "epsilon",
+        "random_mask_instr": true,
+        "random_mask_rgb": true,
+        "scheduler": "DDPM",
+        "stop_weight": 1,
+        "transformer_encoding_size": 512,
+        "transformer_n_cond_layers": 1,
+        "transformer_n_layers": 3,
+        "transformer_p_drop_emb": 0.2,
+        "txt_len": 80,
+        "type": "transformer",
+        "use": true,
+        "use_cls_free_guidance": true,
+        "waypoint_spacing": 1
+      },
+      "distance_predictor": {
+        "normalize": false,
+        "use": false
+      },
+      "eval": {
+        "action": "descrete",
+        "auto_remove": false,
+        "ckpt_to_load": null,
+        "distance_threshold": 1.5,
+        "episode_count": -1,
+        "len_traj_act": 4,
+        "load_eval_subset": true,
+        "max_len_traj_act": 8,
+        "max_steps": 195,
+        "min_displacement": 0.15,
+        "min_len_traj_act": 3,
+        "num_sample": 1,
+        "pm_threshold": 0.9,
+        "re_eval": false,
+        "rotation_threshold": 0.01,
+        "sample": null,
+        "save_results": true,
+        "sim_cfg_file": "vln/configs/sim_cfg_policy_h1_eval.yaml",
+        "split": [
+          "val_unseen"
+        ],
+        "start_eval_epoch": -1,
+        "step_interval": 80,
+        "stop_mode": "stop_progress",
+        "stop_progress_threshold": 0.85,
+        "stop_x_threshold": 0.015,
+        "stop_y_threshold": 0.015,
+        "stop_yaw_threshold": 0.05,
+        "success_distance": 3.0,
+        "train_eval_interval": 100,
+        "use_ckpt_config": false,
+        "use_dynamic_len_traj_act": false,
+        "vln_cfg_file": "vln/configs/vln_cfg_policy_eval.yaml"
+      },
+      "image_encoder": {
+        "depth": {
+          "backbone": "resnet50",
+          "bottleneck": "resnet",
+          "cnn_type": "VlnResnetDepthEncoder",
+          "ddppo_checkpoint": "data/ddppo-models/gibson-4plus-mp3d-train-val-test-resnet50.pth",
+          "feature_dim": 768,
+          "load_model": true,
+          "output_size": 128,
+          "projection_dim": 512,
+          "update_depth_encoder": false
+        },
+        "dropout": 0.1,
+        "env_drop": 0.3,
+        "img_stack_nums": 4,
+        "rgb": {
+          "feature_dim": 768,
+          "img_mod": "multi_patches_avg_pooling",
+          "load_model": true,
+          "model_name": "clip-long",
+          "model_path": "data/pretrained/clip-long/longclip-B.pt",
+          "multi_patches_num": 5,
+          "projection_dim": 512,
+          "rgb_proj": false,
+          "update_rgb_encoder": false
+        },
+        "use_env_drop": true,
+        "use_stack": false
+      },
+      "imu_encoder": {
+        "encoding_size": 64,
+        "input_size": 3,
+        "to_local_coords": true,
+        "use": true
+      },
+      "instruction_encoder": null,
+      "learn_angle": true,
+      "len_traj_act": 4,
+      "max_step": 200,
+      "normalize_rgb": null,
+      "policy_name": "RDP_Policy",
+      "prev_action_encoder": {
+        "encoding_size": 64,
+        "input_size": null,
+        "to_local_coords": null,
+        "type": "continuous",
+        "use": null
+      },
+      "progress_monitor": {
+        "alpha": null,
+        "concat_state_txt": true,
+        "use": true
+      },
+      "rgb_encoder": null,
+      "seq2seq": null,
+      "state_encoder": {
+        "dropout_rate": 0.2,
+        "hidden_size": 512,
+        "num_recurrent_layers": 1,
+        "rgb_depth_embed_method": "flat",
+        "rnn_type": "GRU",
+        "use_dropout": false
+      },
+      "stop_progress_predictor": {
+        "concat_state_txt": true,
+        "loss_alpha": 10,
+        "type": "continuous",
+        "use": true
+      },
+      "text_encoder": {
+        "ablate": null,
+        "embedding_size": 512,
+        "eot_token": 49407,
+        "final_state_only": null,
+        "hidden_size": 512,
+        "load_model": true,
+        "max_length": 248,
+        "model_name": "clip-long",
+        "model_path": "data/pretrained/clip-long/longclip-B.pt",
+        "num_l_layers": 6,
+        "pad_token": 0,
+        "sot_token": 49406,
+        "type": "clip-long",
+        "update_text_encoder": false,
+        "vocab_size": 50265
+      },
+      "use_iw": false
+    },
+    "model_name": "rdp",
+    "name": "20250619_rdp_train_new",
+    "num_gpus": 4,
+    "output_dir": "data/checkpoints/20250619_rdp_train_new/ckpts",
+    "seed": 0,
+    "tensorboard_dir": "data/checkpoints/20250619_rdp_train_new/tensorboard",
+    "torch_gpu_id": 0,
+    "torch_gpu_ids": [
+      0,
+      1,
+      2,
+      3
+    ],
+    "world_size": 4
+  },
+  "model_type": "rdp",
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1"
+}

r2r/fine_tuned/rdp/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d7fab707312afd3ea12a65c84edbeb1fa50c668d0036ff93e21a3919ff06b1
+size 368891962

r2r/fine_tuned/rdp/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0536d140be240a0fd5fb5894b179d453a06fa1c7f96a8c987b35543b01d00599
+size 565168086

r2r/fine_tuned/rdp/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbbdc54615092c7ae7b6703cfcbf9188e8913651c406ec7bc482f02df67dcb9d
+size 15006

r2r/fine_tuned/rdp/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4640b4a7e0c9ff2f9b7bead4a5038207678e9cbe45e41030074a0f2ed9b42ee5
+size 1064

r2r/fine_tuned/seq2seq/config.json

@@ -0,0 +1,136 @@
+{
+  "architectures": [
+    "Seq2SeqNet"
+  ],
+  "model_cfg": {
+    "checkpoint_folder": "data/checkpoints/20250619_seq2seq_train/ckpts",
+    "eval": {
+      "action": null,
+      "ckpt_to_load": null,
+      "len_traj_act": null,
+      "max_steps": 195,
+      "num_sample": null,
+      "pm_threshold": null,
+      "rotation_threshold": 0.01,
+      "sample": false,
+      "save_results": true,
+      "split": [
+        "val_seen"
+      ],
+      "start_eval_epoch": -1,
+      "step_interval": 50,
+      "stop_mode": null,
+      "success_distance": 3.0,
+      "use_ckpt_config": false
+    },
+    "il": {
+      "batch_size": 2,
+      "camera_name": "pano_camera_0",
+      "ckpt_to_load": "",
+      "dataset_3dgs_root_dir": "data/datasets/3dgs",
+      "dataset_grutopia10_root_dir": "data/datasets/grutopia10",
+      "dataset_r2r_root_dir": "data/datasets/R2R_VLNCE_v1-3_preprocessed",
+      "epochs": 100,
+      "filter_failure": {
+        "min_rgb_nums": 15,
+        "use": true
+      },
+      "inflection_weight_coef": 3.2,
+      "lmdb_features_dir": "data/sample_episodes/20250211_sample_origin/sample_data.lmdb",
+      "lmdb_map_size": 1000000000000.0,
+      "load_from_ckpt": false,
+      "load_from_pretrain": true,
+      "loss": {
+        "alpha": 0.0001,
+        "dist_scale": 1
+      },
+      "lr": 0.0001,
+      "num_workers": 8,
+      "report_to": "wandb",
+      "save_filter_frozen_weights": false,
+      "save_interval_epochs": 5,
+      "save_interval_steps": null,
+      "use_descrete_dataset": null,
+      "use_iw": true,
+      "warmup_ratio": 0.1,
+      "weight_decay": 0.0001
+    },
+    "local_rank": 0,
+    "log_dir": "data/checkpoints/20250619_seq2seq_train/logs",
+    "model": {
+      "ablate_depth": null,
+      "ablate_instruction": null,
+      "ablate_rgb": null,
+      "cross_modal_encoder": null,
+      "depth_encoder": {
+        "backbone": "resnet50",
+        "cnn_type": "VlnResnetDepthEncoder",
+        "ddppo_checkpoint": "data/ddppo-models/gibson-4plus-mp3d-train-val-test-resnet50.pth",
+        "output_size": 128,
+        "trainable": false
+      },
+      "diffusion_policy": null,
+      "distance_predictor": null,
+      "image_encoder": null,
+      "imu_encoder": null,
+      "instruction_encoder": {
+        "bidirectional": false,
+        "dataset_vocab": "data/datasets/R2R_VLNCE_v1-3_preprocessed/train/train.json.gz",
+        "embedding_file": "data/datasets/R2R_VLNCE_v1-3_preprocessed/embeddings.json.gz",
+        "embedding_size": 50,
+        "final_state_only": true,
+        "fine_tune_embeddings": false,
+        "hidden_size": 128,
+        "load_model": null,
+        "max_length": null,
+        "rnn_type": "LSTM",
+        "sensor_uuid": "instruction",
+        "use_pretrained_embeddings": true,
+        "vocab_size": 2504
+      },
+      "learn_angle": null,
+      "len_traj_act": 4,
+      "max_step": 200,
+      "normalize_rgb": null,
+      "policy_name": "Seq2Seq_Policy",
+      "prev_action_encoder": null,
+      "progress_monitor": {
+        "alpha": 1.0,
+        "concat_state_txt": null,
+        "use": true
+      },
+      "rgb_encoder": {
+        "cnn_type": "TorchVisionResNet50",
+        "output_size": 256,
+        "trainable": false
+      },
+      "seq2seq": {
+        "use_prev_action": false
+      },
+      "state_encoder": {
+        "dropout_rate": null,
+        "hidden_size": 512,
+        "num_recurrent_layers": 1,
+        "rgb_depth_embed_method": null,
+        "rnn_type": "GRU",
+        "use_dropout": null
+      },
+      "stop_progress_predictor": null,
+      "text_encoder": null
+    },
+    "model_name": "seq2seq",
+    "name": "20250619_seq2seq_train",
+    "num_gpus": 1,
+    "output_dir": "data/checkpoints/20250619_seq2seq_train/ckpts",
+    "seed": 0,
+    "tensorboard_dir": "data/checkpoints/20250619_seq2seq_train/tensorboard",
+    "torch_gpu_id": 0,
+    "torch_gpu_ids": [
+      0
+    ],
+    "world_size": 1
+  },
+  "model_type": "seq2seq",
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1"
+}

r2r/fine_tuned/seq2seq/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1a9c011abecf2958a513a5269d1f650ff54dd3ce72ba9c813005b629604c771
+size 19673710

r2r/fine_tuned/seq2seq/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5570070c8697664f46668bdc96727381124c4c4d1f0e20e75c8940d4274d1c4
+size 133019958

r2r/fine_tuned/seq2seq/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1c1483b2f971b5b4fb5c7742b5ae188a8eafdb358c870fb90a320a187afabd1
+size 14244

r2r/fine_tuned/seq2seq/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0508238d3474230955f9d0198dbf096ccc53817711c80525217e2d61170a4af
+size 1064

r2r/fine_tuned/seq2seq_plus/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b510fc3e18a1f57cc714af9fc975dcb60eddfcf76aa51daae582bd864dbec4a
+size 133013046

r2r/zero_shot/cma/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693259a1e8d426c64573bbc838e430dba0e57cbe2e2b8d9a14b3b80e8c4fd448
+size 147809576

r2r/zero_shot/seq2seq/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81284a6a74998db7c91f4bf9cac6865715a7ced3300ff12386675d861aa66faf
+size 133012722