Upload 16 files

added_tokens.json

@@ -0,0 +1,6 @@
+{
+  "<image>": 49155,
+  "<|end_of_role|>": 49153,
+  "<|start_of_role|>": 49152,
+  "<|tool_call|>": 49154
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{%- if tools %}\n    {{- '<|start_of_role|>available_tools<|end_of_role|>\n' }}\n    {%- for tool in tools %}\n    {{- tool | tojson(indent=4) }}\n    {%- if not loop.last %}\n        {{- '\n\n' }}\n    {%- endif %}\n    {%- endfor %}\n    {{- '<|end_of_text|>\n' }}\n{%- endif %}\n{%- for message in messages if message['role'] == 'system'%}{% else %}<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n{% endfor %}{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n    {{- '<|system|>\n' + message['content'][0]['text'] + '\n' }}\n    {%- elif message['role'] == 'user' %}<|user|>\n {# Render all images first #}{% for content in message['content'] | selectattr('type', 'equalto', 'image') %}{{ '<image>\n' }}{% endfor %}{# Render all text next #}{% for content in message['content'] | selectattr('type', 'equalto', 'text') %}{{ content['text'] + '\n' }}{% endfor %}\n{%- elif message['role'] == 'assistant' %}\n    {{- '<|assistant|>\n'  + message['content'][0]['text']  + '<|end_of_text|>' }}\n    {%- elif message['role'] == 'assistant_tool_call' %}\n    {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'][0]['text']  + '<|end_of_text|>\n' }}\n    {%- elif message['role'] == 'tool_response' %}\n    {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'][0]['text'] + '<|end_of_text|>\n' }}\n    {%- endif %}\n    {%- if loop.last and add_generation_prompt %}\n    {{- '<|assistant|>\n' }}\n    {%- endif %}\n{%- endfor %}"
+}

config.json

@@ -0,0 +1,172 @@
+{
+  "_name_or_path": "ibm-granite/granite-vision-3.1-2b-preview",
+  "auto_map": 
+  {
+  "AutoModel": "modeling_colgranitevision.ColGraniteVisionModel",
+  "AutoProcessor" : "processing_colgranitevision.ColGraniteVisionProcessor"
+
+},
+  "architectures": [
+    "ColGraniteVision"
+  ],
+  "base_model": null,
+  "emb_dim_doc": 128,
+  "emb_dim_query": 128,
+  "image_grid_pinpoints": [
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      384,
+      2688
+    ],
+    [
+      384,
+      3072
+    ],
+    [
+      384,
+      3456
+    ],
+    [
+      384,
+      3840
+    ],
+    [
+      768,
+      384
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      1152,
+      384
+    ],
+    [
+      1152,
+      768
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      1536,
+      384
+    ],
+    [
+      1536,
+      768
+    ],
+    [
+      1920,
+      384
+    ],
+    [
+      1920,
+      768
+    ],
+    [
+      2304,
+      384
+    ],
+    [
+      2688,
+      384
+    ],
+    [
+      3072,
+      384
+    ],
+    [
+      3456,
+      384
+    ],
+    [
+      3840,
+      384
+    ]
+  ],
+  "image_seq_length": 576,
+  "image_token_index": 49155,
+  "model_type": "colgranitevision",
+  "multimodal_projector_bias": true,
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "architectures": [
+      "GraniteForCausalLM"
+    ],
+    "attention_dropout": 0.1,
+    "attention_multiplier": 0.015625,
+    "bos_token_id": 0,
+    "embedding_multiplier": 12.0,
+    "eos_token_id": 0,
+    "hidden_size": 2048,
+    "intermediate_size": 8192,
+    "logits_scaling": 8.0,
+    "max_position_embeddings": 16384,
+    "model_type": "granite",
+    "num_hidden_layers": 40,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "residual_multiplier": 0.22,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 300000,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "vocab_size": 49156
+  },
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.50.0.dev0",
+  "use_image_newline_parameter": true,
+  "vision_config": {
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 384,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 27,
+    "patch_size": 14
+  },
+  "vision_feature_layer": [
+    -24,
+    -20,
+    -12,
+    -1
+  ],
+  "vision_feature_select_strategy": "full"
+}

custom_llava_next.py

@@ -0,0 +1,122 @@
+from transformers.models.llava_next.modeling_llava_next import LlavaNextForConditionalGeneration
+import torch
+from transformers.models.llava_next.modeling_llava_next import unpad_image, get_anyres_image_grid_shape
+import numpy as np
+
+class LlavaNextWithCustomPacking(LlavaNextForConditionalGeneration):
+    def pack_image_features(
+        self,
+        image_features,
+        image_sizes,
+        vision_feature_select_strategy,
+        image_newline=None,
+        base_image_feature_location="last",
+    ):
+        """
+        Reshape, unpad and then pack each image_feature into a single image_features tensor containing all visual vectors.
+
+        Args:
+            image_features (`List[torch.Tensor]` of length num_images, each of shape `(num_patches, image_length, embed_dim)`)
+                List of image feature tensor, each contains all the visual feature of all patches.
+            image_sizes (`torch.Tensor` of shape `(num_images, 2)`)
+                Actual image size of each images (H, W).
+            vision_feature_select_strategy (`str`)
+                The feature selection strategy used to select the vision feature from the vision backbone.
+            image_newline (`torch.Tensor` of shape `(embed_dim)`)
+                New line embedding vector.
+        Returns:
+            image_features (`torch.Tensor` of shape `(all_feat_len, embed_dim)`)
+            feature_lens (`List[int]`)
+                token length of each image in image_features
+        """
+
+
+        new_image_features = []
+        feature_lens = []
+        for image_idx, image_feature in enumerate(image_features):
+            if image_feature.shape[0] > 1:
+                base_image_feature = image_feature[0]
+                image_feature = image_feature[1:]
+                height = width = self.config.vision_config.image_size // self.config.vision_config.patch_size
+
+                num_patch_height, num_patch_width = get_anyres_image_grid_shape(
+                    image_sizes[image_idx],
+                    self.config.image_grid_pinpoints,
+                    self.config.vision_config.image_size,
+                )
+
+                if (
+                    np.prod(image_feature.shape) % (num_patch_height * num_patch_width * height * width) != 0
+                    and vision_feature_select_strategy == "default"
+                ):
+                    logger.warning_once(
+                        "Image feature shape does not line up with the provided patch size. "
+                        "You may be using the `default` vision_feature_select_strategy with a"
+                        " visual encoder that does not have CLS."
+                    )
+
+                image_feature = image_feature.view(num_patch_height, num_patch_width, height, width, -1)
+                image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()
+                image_feature = image_feature.flatten(1, 2).flatten(2, 3)
+                image_feature = unpad_image(image_feature, image_sizes[image_idx])
+                if image_newline is not None:
+                    image_feature = torch.cat(
+                        (
+                            image_feature,
+                            image_newline[:, None, None]
+                            .expand(*image_feature.shape[:-1], 1)
+                            .to(image_feature.device, image_feature.dtype),
+                        ),
+                        dim=-1,
+                    )
+                image_feature = image_feature.flatten(1, 2).transpose(0, 1)
+                if base_image_feature_location == "last":
+                    image_feature = torch.cat((image_feature, base_image_feature), dim=0)
+                else:
+                    image_feature = torch.cat((base_image_feature, image_feature), dim=0)
+
+            else:
+                image_feature = image_feature[0]
+                if image_newline is not None:
+                    image_feature = torch.cat((image_feature, image_newline[None].to(image_feature)), dim=0)
+            new_image_features.append(image_feature)
+            feature_lens.append(image_feature.size(0))
+        image_features = torch.cat(new_image_features, dim=0)
+        feature_lens = torch.tensor(feature_lens, dtype=torch.long, device=image_features.device)
+        return image_features, feature_lens
+
+
+
+def main():
+    import torch
+    from transformers import AutoConfig
+
+    # Load config and model
+    model_id = "ibm-granite/granite-vision-3.1-2b-preview"
+    config = AutoConfig.from_pretrained(model_id)
+    model = LlavaNextWithCustomPacking.from_pretrained(model_id, config=config)
+
+    # Dummy image features for 2 images (1 base + 3x3 patch grid flattened)
+    B = 2  # batch size
+    num_views = 3
+    num_patches = 729
+    embed_dim = model.config.text_config.hidden_size
+    image_features = [
+        torch.randn(num_views, num_patches, embed_dim) for _ in range(B)
+    ]
+    image_sizes = torch.tensor([[384, 384], [384, 384]])  # H, W for each image
+
+    # Call overridden pack_image_features
+    packed_feats, lengths = model.pack_image_features(
+        image_features=image_features,
+        image_sizes=image_sizes,
+        vision_feature_select_strategy="default",
+        image_newline=model.image_newline,
+        base_image_feature_location="last",
+    )
+
+    print("Packed features shape:", packed_feats.shape)
+    print("Feature lengths:", lengths)
+
+if __name__ == "__main__":
+    main()

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec8a694db663b30616ff06812d60256bb474c52051df2003faaec47c42b9a556
+size 4955415688

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9df92d92a0d79465e4ee5eb57a51ee1630b159dc5833e26af9ca7bc9b3788d24
+size 4999979448

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42ebe0fe87507de69b86074af24513756fbcc205e83ccb2ee7bbe9238a751f29
+size 1947355456

model.safetensors.index.json

@@ -0,0 +1,824 @@
+{
+  "metadata": {
+    "total_size": 11902636800
+  },
+  "weight_map": {
+    "custom_text_proj.bias": "model-00003-of-00003.safetensors",
+    "custom_text_proj.weight": "model-00003-of-00003.safetensors",
+    "model.image_newline": "model-00001-of-00003.safetensors",
+    "model.language_model.model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.language_model.model.layers.32.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.32.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.33.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.34.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.35.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.36.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.37.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.38.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.39.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.language_model.model.norm.weight": "model-00003-of-00003.safetensors",
+    "model.multi_modal_projector.linear_1.bias": "model-00001-of-00003.safetensors",
+    "model.multi_modal_projector.linear_1.weight": "model-00001-of-00003.safetensors",
+    "model.multi_modal_projector.linear_2.bias": "model-00001-of-00003.safetensors",
+    "model.multi_modal_projector.linear_2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.embeddings.patch_embedding.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.26.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.attention.in_proj_bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.attention.in_proj_weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.attention.out_proj.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.attention.out_proj.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.head.probe": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.post_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.vision_tower.vision_model.post_layernorm.weight": "model-00001-of-00003.safetensors"
+  }
+}

modeling_colgranitevision.py

@@ -0,0 +1,115 @@
+from typing import ClassVar, Optional
+
+import torch
+from torch import nn
+from transformers import LlavaNextConfig, \
+    LlavaNextPreTrainedModel
+
+from custom_llava_next import LlavaNextWithCustomPacking as LlavaNextForConditionalGeneration
+from colgranitevision_config import  ColGraniteVisionConfig
+
+# from transformers.models.paligemma.modeling_paligemma import (
+#     PaliGemmaConfig,
+#     PaliGemmaForConditionalGeneration,
+#     PaliGemmaPreTrainedModel,
+# )
+
+
+class ColGraniteVision(LlavaNextPreTrainedModel):
+    """
+    ColGraniteVision model implementation.
+    """
+
+    main_input_name: ClassVar[str] = "doc_input_ids"  # transformers-related
+    config_class = ColGraniteVisionConfig
+
+    def __init__(self, config: ColGraniteVisionConfig):
+        super().__init__(config=config)
+
+        model = LlavaNextForConditionalGeneration(config=config)
+        if model.language_model._tied_weights_keys is not None:
+            self._tied_weights_keys = [f"model.language_model.{k}" for k in model.language_model._tied_weights_keys]
+        self.model = model
+
+        # TODO: Wait for ColPali2 to create a ColPaliConfig to allow specifying the embedding dimension.
+        # We could do it now but it would break all the models trying to load the model from the checkpoint.
+        self.dim = 128
+        self.custom_text_proj = nn.Linear(self.model.config.text_config.hidden_size, self.dim)
+
+        self.post_init()
+
+    def forward(self, *args, **kwargs) -> torch.Tensor:
+        # Delete output_hidden_states from kwargs
+        kwargs.pop("output_hidden_states", None)
+        if "pixel_values" in kwargs:
+            kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype=self.dtype)
+
+        outputs = self.model(*args, output_hidden_states=True, **kwargs)  # (batch_size, sequence_length, hidden_size)
+        last_hidden_states = outputs.hidden_states[-1]  # (batch_size, sequence_length, hidden_size)
+
+        attention_mask = kwargs["attention_mask"]
+        if "pixel_values" in kwargs:
+            input_ids = kwargs['input_ids']
+            image_mask = (input_ids == self.config.image_token_index)
+            # inputs_embeds = last_hidden_states.masked_scatter(image_mask)
+            N, M = image_mask.shape
+            # Create an index matrix: each row is 0, 1, ..., M-1
+            idx = torch.arange(M, device=image_mask.device).expand(N, M)
+            # Replace False positions with -1 so they are ignored by topk (since all valid indices are >=0)
+            masked_idx = torch.where(image_mask, idx, torch.tensor(-1, device=image_mask.device))
+            topk_values, _ = torch.topk(masked_idx, k=729, dim=1)
+            last_k_indices, _ = torch.sort(topk_values, dim=1)
+            last_k_indices_exp = last_k_indices.unsqueeze(-1).expand(-1, -1, last_hidden_states.size(-1))
+            last_hidden_states = torch.gather(last_hidden_states, 1, last_k_indices_exp)
+            attention_mask = torch.gather(attention_mask, 1, last_k_indices)
+
+        attention_mask = attention_mask.unsqueeze(-1)
+
+        proj = self.custom_text_proj(last_hidden_states)  # (batch_size, sequence_length, dim)
+
+        # L2 normalization
+        proj = proj / (proj.norm(dim=-1, keepdim=True) + 1e-8)
+
+        # proj = proj * kwargs["attention_mask"].unsqueeze(-1)  # (batch_size, sequence_length, dim)
+        proj = proj * attention_mask  # (batch_size, sequence_length, dim)
+
+        return proj
+
+    def get_input_embeddings(self):
+        return self.model.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.model.language_model.get_output_embeddings()
+
+    def set_output_embeddings(self, new_embeddings):
+        self.model.language_model.set_output_embeddings(new_embeddings)
+
+    def set_decoder(self, decoder):
+        self.model.language_model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.model.language_model.get_decoder()
+
+    def tie_weights(self):
+        return self.model.language_model.tie_weights()
+
+    def resize_token_embeddings(
+            self,
+            new_num_tokens: Optional[int] = None,
+            pad_to_multiple_of=None,
+    ) -> nn.Embedding:
+        model_embeds = self.model.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+
+        # Update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.config.vocab_size = model_embeds.num_embeddings
+        self.model.vocab_size = model_embeds.num_embeddings
+
+        return model_embeds
+
+    @property
+    def patch_size(self) -> int:
+        return self.model.vision_tower.config.patch_size

preprocessor_config.json

@@ -0,0 +1,136 @@
+{
+  "crop_size": {
+    "height": 384,
+    "width": 384
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_grid_pinpoints": [
+    [
+      384,
+      768
+    ],
+    [
+      384,
+      1152
+    ],
+    [
+      384,
+      1536
+    ],
+    [
+      384,
+      1920
+    ],
+    [
+      384,
+      2304
+    ],
+    [
+      384,
+      2688
+    ],
+    [
+      384,
+      3072
+    ],
+    [
+      384,
+      3456
+    ],
+    [
+      384,
+      3840
+    ],
+    [
+      768,
+      384
+    ],
+    [
+      768,
+      768
+    ],
+    [
+      768,
+      1152
+    ],
+    [
+      768,
+      1536
+    ],
+    [
+      768,
+      1920
+    ],
+    [
+      1152,
+      384
+    ],
+    [
+      1152,
+      768
+    ],
+    [
+      1152,
+      1152
+    ],
+    [
+      1536,
+      384
+    ],
+    [
+      1536,
+      768
+    ],
+    [
+      1920,
+      384
+    ],
+    [
+      1920,
+      768
+    ],
+    [
+      2304,
+      384
+    ],
+    [
+      2688,
+      384
+    ],
+    [
+      3072,
+      384
+    ],
+    [
+      3456,
+      384
+    ],
+    [
+      3840,
+      384
+    ]
+  ],
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "LlavaNextImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "ColGraniteVisionProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 384,
+    "width": 384
+  }
+}

processing_colgranitevision.py

@@ -0,0 +1,396 @@
+import math
+from typing import ClassVar, List, Optional, Tuple, Union
+
+import torch
+from PIL import Image, ImageOps
+from transformers import BatchFeature, LlavaNextProcessor
+
+
+def round_by_factor(number: float, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: float, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: float, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+class ColGraniteVisionProcessor(LlavaNextProcessor):
+    """
+    Processor for ColPali.
+    """
+
+    visual_prompt_prefix: ClassVar[str] = "<|user|>\n<image>\nDescribe the image.\n"
+    system_message: ClassVar[
+        str] = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
+    query_prefix: ClassVar[str] = "Query: "
+    query_start: ClassVar[str] = "<|user|>\n"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.factor = 14
+        self.min_size = 384
+        self.max_size = 384 * 2
+        self.suffix_len = 10
+        self.patch_size = 14
+
+    @property
+    def query_augmentation_token(self) -> str:
+        """
+        Return the query augmentation token.
+        Query augmentation buffers are used as reasoning buffers during inference.
+        """
+        return self.tokenizer.pad_token
+
+    @staticmethod
+    def smart_resize_helper(
+            width: int,
+            height: int,
+            factor: int,
+            min_size: int,
+            max_size: int
+    ) -> Tuple[int, int]:
+        """
+        Returns the resized image dimensions such that:
+        1. The smaller dimension is set to 'min_size'.
+        2. The larger dimension is scaled proportionally to maintain aspect ratio.
+        3. If the larger dimension exceeds 'max_size', it is clipped to 'max_size',
+        and the smaller dimension is adjusted accordingly to maintain aspect ratio.
+        4. Both dimensions are divisible by 'factor'.
+        """
+
+        # Determine scale factor based on min_size
+        if height < width:
+            scale_factor = min_size / height
+        else:
+            scale_factor = min_size / width
+
+        new_width = round(width * scale_factor)
+        new_height = round(height * scale_factor)
+
+        # If the longer dimension exceeds max_size, adjust accordingly
+        if max(new_width, new_height) > max_size:
+            clip_factor = max_size / max(new_width, new_height)
+            new_width = round(new_width * clip_factor)
+            new_height = round(new_height * clip_factor)
+
+        # Ensure dimensions are divisible by factor
+        # new_width = round_by_factor(new_width, factor)
+        # new_height = round_by_factor(new_height, factor)
+
+        return new_width, new_height
+
+    @staticmethod
+    def pad_image_center(image: Image.Image,
+                         target_width: int,
+                         target_height: int,
+                         fill_color=(0, 0, 0)) -> Image.Image:
+        """
+        Pads the given image to be centered within the target dimensions.
+
+        :param image: PIL Image to be padded.
+        :param target_width: The desired width after padding.
+        :param target_height: The desired height after padding.
+        :param fill_color: Background color (default is black).
+        :return: Padded image with centered content.
+        """
+
+        # Get original image size
+        img_width, img_height = image.size
+
+        # Compute padding values
+        pad_left = (target_width - img_width) // 2
+        pad_top = (target_height - img_height) // 2
+        pad_right = target_width - img_width - pad_left
+        pad_bottom = target_height - img_height - pad_top
+
+        # Apply padding
+        padded_image = ImageOps.expand(image, (pad_left, pad_top, pad_right, pad_bottom), fill_color).convert("RGB")
+
+        return padded_image
+
+    def smart_resize(self, image: Image.Image) -> Image.Image:
+        """
+        Resize and convert the image to the required format.
+        """
+        image_size = image.size
+        resized_height, resized_width = self.smart_resize_helper(
+            width=image_size[0],
+            height=image_size[1],
+            factor=self.factor,
+            min_size=self.min_size,
+            max_size=self.max_size
+        )
+        return image.convert("RGB").resize((resized_width, resized_height))
+
+    def smart_resize_and_pad(self, image: Image.Image) -> Image.Image:
+        """
+        Resize and pad the image to the required format.
+        """
+        return self.resize_and_pad_centered(
+            image=image,
+            factor=self.factor,
+            min_size=self.min_size,
+            max_size=self.max_size,
+            fill_color=0
+        )
+
+    def resize_and_pad_centered(self,
+                                image: Image.Image,
+                                factor: int,
+                                min_size: int,
+                                max_size: int,
+                                fill_color=0
+                                ) -> Image.Image:
+        """
+        Resizes and pads an image such that:
+        - The short side is set to `min_size`.
+        - The long side is scaled proportionally but clipped to `max_size`.
+        - The image is centered within the final padded area.
+
+        :param image: PIL Image
+        :param factor: Factor to make dimensions divisible by
+        :param min_size: Minimum size for the short side
+        :param max_size: Maximum allowed size for the long side
+        :param fill_color: Background padding color (default black)
+        :return: Resized and padded image
+        """
+
+        # Get original size
+        width, height = image.size
+
+        if min_size == -1 or max_size == -1:
+            return image.convert("RGB")
+
+        # Determine scale factor based on the short side (min_size)
+        if width < height:
+            scale_factor = min_size / width
+            target_width = min_size
+            max_scale_factor = min(max_size / height, scale_factor)
+            target_height = round(height * max_scale_factor)
+        else:
+            scale_factor = min_size / height
+            target_height = min_size
+            max_scale_factor = min(max_size / width, scale_factor)
+            target_width = round(width * max_scale_factor)
+
+        # Ensure the longer side does not exceed max_size
+        # if max(target_width, target_height) > max_size:
+        #     clip_factor = max_size / max(target_width, target_height)
+        #     target_width = round(target_width * clip_factor)
+        #     target_height = round(target_height * clip_factor)
+
+        # Ensure dimensions are divisible by factor
+        # target_width = round_by_factor(target_width, factor)
+        # target_height = round_by_factor(target_height, factor)
+
+        # Resize the image
+        resized_image = image.resize((target_width, target_height), Image.LANCZOS)
+
+        # Determine final padded dimensions (aligned to short side)
+        if width < height:
+            final_width, final_height = min_size, max_size
+        else:
+            final_width, final_height = max_size, min_size
+
+        # Compute padding to center the image
+        pad_left = (final_width - target_width) // 2
+        pad_top = (final_height - target_height) // 2
+        pad_right = final_width - target_width - pad_left
+        pad_bottom = final_height - target_height - pad_top
+
+        # Apply centered padding
+        # final_image = ImageOps.expand(resized_image, (pad_left, pad_top, pad_right, pad_bottom), fill_color).convert("RGB")
+        final_image = resized_image.convert("RGB")
+
+        return final_image
+
+    def format_data(self, question, image):
+        return [
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": self.system_message}],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image",
+                        "image": image,
+                    },
+                    {
+                        "type": "text",
+                        "text": question,
+                    },
+                ],
+            }
+        ]
+
+    def format_data_wo_role(self, question, image=None):
+        return [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image",
+                        "image": image,
+                    },
+                    {
+                        "type": "text",
+                        "text": question,
+                    },
+                ],
+            }
+        ]
+
+    def process_images(
+            self,
+            images: List[Image.Image],
+    ) -> BatchFeature:
+        """
+        Process images for ColPali.
+        """
+        # texts_doc = [self.apply_chat_template(self.format_data_wo_role(self.visual_prompt_prefix, img),tokenize=False ) for img in images]
+        texts_doc = [self.visual_prompt_prefix for _ in images]
+        images = [self.smart_resize_and_pad(image) for image in images]
+
+        batch_doc = self(
+            text=texts_doc,
+            images=images,
+            return_tensors="pt",
+            padding="longest",
+        )
+        return batch_doc
+
+    def process_queries(self, queries, max_length=2048, suffix=None):
+        if suffix is None:
+            suffix = self.query_augmentation_token * self.suffix_len
+
+        processed = []
+        for q in queries:
+            q = self.query_start + self.query_prefix + q
+            # truncate before it eats actual query content
+            if len(q) + len(suffix) > max_length:
+                q = q[: max_length - len(suffix) - 1]
+            q += suffix + "\n"
+            processed.append(q)
+
+        return self(
+            text=processed,
+            images=None,
+            return_tensors="pt",
+            padding="longest",
+            truncation=True,
+            max_length=max_length,
+        )
+
+    def score(
+            self,
+            qs: List[torch.Tensor],
+            ps: List[torch.Tensor],
+            device: Optional[Union[str, torch.device]] = None,
+            **kwargs,
+    ) -> torch.Tensor:
+        """
+        Compute the MaxSim score (ColBERT-like) for the given multi-vector query and passage embeddings.
+        """
+        return self.score_multi_vector(qs, ps, device=device, **kwargs)
+
+    def get_n_patches(
+            self,
+            image_size: Tuple[int, int],
+            patch_size: int,
+    ) -> Tuple[int, int]:
+        n_patches_x = self.image_processor.size["width"] // patch_size
+        n_patches_y = self.image_processor.size["height"] // patch_size
+
+        return n_patches_x, n_patches_y
+
+    def get_image_mask(self, batch_images: BatchFeature) -> torch.Tensor:
+        return batch_images.input_ids == self.image_token_id
+
+    @staticmethod
+    def score_single_vector(
+            qs: List[torch.Tensor],
+            ps: List[torch.Tensor],
+            device: Optional[Union[str, torch.device]] = None,
+    ) -> torch.Tensor:
+        """
+        Compute the dot product score for the given single-vector query and passage embeddings.
+        """
+
+        if len(qs) == 0:
+            raise ValueError("No queries provided")
+        if len(ps) == 0:
+            raise ValueError("No passages provided")
+
+        qs_stacked = torch.stack(qs).to(device)
+        ps_stacked = torch.stack(ps).to(device)
+
+        scores = torch.einsum("bd,cd->bc", qs_stacked, ps_stacked)
+        assert scores.shape[0] == len(qs), f"Expected {len(qs)} scores, got {scores.shape[0]}"
+
+        scores = scores.to(torch.float32)
+        return scores
+
+    @staticmethod
+    def score_multi_vector(
+            qs: Union[torch.Tensor, List[torch.Tensor]],
+            ps: Union[torch.Tensor, List[torch.Tensor]],
+            batch_size: int = 128,
+            device: Optional[Union[str, torch.device]] = None,
+    ) -> torch.Tensor:
+        """
+        Compute the late-interaction/MaxSim score (ColBERT-like) for the given multi-vector
+        query embeddings (`qs`) and passage embeddings (`ps`). For ColPali, a passage is the
+        image of a document page.
+
+        Because the embedding tensors are multi-vector and can thus have different shapes, they
+        should be fed as:
+        (1) a list of tensors, where the i-th tensor is of shape (sequence_length_i, embedding_dim)
+        (2) a single tensor of shape (n_passages, max_sequence_length, embedding_dim) -> usually
+            obtained by padding the list of tensors.
+
+        Args:
+            qs (`Union[torch.Tensor, List[torch.Tensor]`): Query embeddings.
+            ps (`Union[torch.Tensor, List[torch.Tensor]`): Passage embeddings.
+            batch_size (`int`, *optional*, defaults to 128): Batch size for computing scores.
+            device (`Union[str, torch.device]`, *optional*): Device to use for computation. If not
+                provided, uses `get_torch_device("auto")`.
+
+        Returns:
+            `torch.Tensor`: A tensor of shape `(n_queries, n_passages)` containing the scores. The score
+            tensor is saved on the "cpu" device.
+        """
+
+        if len(qs) == 0:
+            raise ValueError("No queries provided")
+        if len(ps) == 0:
+            raise ValueError("No passages provided")
+
+        scores_list: List[torch.Tensor] = []
+
+        for i in range(0, len(qs), batch_size):
+            scores_batch = []
+            qs_batch = torch.nn.utils.rnn.pad_sequence(qs[i: i + batch_size], batch_first=True, padding_value=0).to(
+                device
+            )
+            for j in range(0, len(ps), batch_size):
+                ps_batch = torch.nn.utils.rnn.pad_sequence(
+                    ps[j: j + batch_size], batch_first=True, padding_value=0
+                ).to(device)
+                scores_batch.append(torch.einsum("bnd,csd->bcns", qs_batch, ps_batch).max(dim=3)[0].sum(dim=2))
+            scores_batch = torch.cat(scores_batch, dim=1).cpu()
+            scores_list.append(scores_batch)
+
+        scores = torch.cat(scores_list, dim=0)
+        assert scores.shape[0] == len(qs), f"Expected {len(qs)} scores, got {scores.shape[0]}"
+
+        scores = scores.to(torch.float32)
+        return scores

special_tokens_map.json

@@ -0,0 +1,35 @@
+{
+  "additional_special_tokens": [
+    "<|start_of_role|>",
+    "<|end_of_role|>",
+    "<|tool_call|>"
+  ],
+  "bos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,208 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<fim_prefix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<fim_middle>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<fim_suffix>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<fim_pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<commit_before>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<commit_msg>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "17": {
+      "content": "<commit_after>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "18": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<|start_of_role|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<|end_of_role|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<|tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49155": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|start_of_role|>",
+    "<|end_of_role|>",
+    "<|tool_call|>"
+  ],
+  "bos_token": "<|end_of_text|>",
+  "chat_template": "{%- if tools %}\n    {{- '<|start_of_role|>available_tools<|end_of_role|>\n' }}\n    {%- for tool in tools %}\n    {{- tool | tojson(indent=4) }}\n    {%- if not loop.last %}\n        {{- '\n\n' }}\n    {%- endif %}\n    {%- endfor %}\n    {{- '<|end_of_text|>\n' }}\n{%- endif %}\n{%- for message in messages if message['role'] == 'system'%}{% else %}<|system|>\nA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n{% endfor %}{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n    {{- '<|system|>\n' + message['content'] + '\n' }}\n    {%- elif message['role'] == 'user' %}\n    {{- '<|user|>\n' + message['content'] + '\n' }}\n    {%- elif message['role'] == 'assistant' %}\n    {{- '<|assistant|>\n'  + message['content'] + '<|end_of_text|>' }}\n    {%- elif message['role'] == 'assistant_tool_call' %}\n    {{- '<|start_of_role|>assistant<|end_of_role|><|tool_call|>' + message['content'] + '<|end_of_text|>\n' }}\n    {%- elif message['role'] == 'tool_response' %}\n    {{- '<|start_of_role|>tool_response<|end_of_role|>' + message['content'] + '<|end_of_text|>\n' }}\n    {%- endif %}\n    {%- if loop.last and add_generation_prompt %}\n    {{- '<|assistant|>\n' }}\n    {%- endif %}\n{%- endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 16384,
+  "pad_token": "<|end_of_text|>",
+  "padding_side": "right",
+  "processor_class": "ColGraniteVisionProcessor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|end_of_text|>",
+  "vocab_size": 49152
+}