Upload videollama3_original_model

videollama3_original_model/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

videollama3_original_model/.huggingface/.gitignore

@@ -0,0 +1 @@
+*

videollama3_original_model/.huggingface/download/.gitattributes.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+a6344aac8c09253b3b630fb776ae94478aa0275b
+1763248024.9122677

videollama3_original_model/.huggingface/download/README.md.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+c3a6ec2b1e4a9b96d16fb9693d66448828c9ca28
+1763248024.9312024

videollama3_original_model/.huggingface/download/added_tokens.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+066b1421f76beca53bde9481638b4826054c122f
+1763248024.8180633

videollama3_original_model/.huggingface/download/chat_template.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+f67d31e44cc5c21bd8f1b7b26d27417e98691e6b
+1763248024.8308272

videollama3_original_model/.huggingface/download/config.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+a5b54001faaa818772e4ab6274e95383186c431a
+1763248024.7803109

videollama3_original_model/.huggingface/download/configuration_videollama3.py.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+70cc7c7620eb451ae84d4bb79a6e0845a6c9b4df
+1763248024.749131

videollama3_original_model/.huggingface/download/configuration_videollama3_encoder.py.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+9bd0decacc5c352b38c3249d1556dd91c6848fa8
+1763248024.796934

videollama3_original_model/.huggingface/download/generation_config.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+2e995d693e130e3e92b87ff760bbe99ccdc54ea0
+1763248024.7491724

videollama3_original_model/.huggingface/download/image_processing_videollama3.py.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+91099b890869d0a0b1834a313fc6d107bdd8ab3f
+1763248024.8219907

videollama3_original_model/.huggingface/download/merges.txt.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+31349551d90c7606f325fe0f11bbb8bd5fa0d7c7
+1763248024.8034952

videollama3_original_model/.huggingface/download/model-00001-of-00004.safetensors.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+cf34749d69dab7940984ef88be8a33e161e467809c0aacf8825ac7de0a841cf6
+1763248029.6650863

videollama3_original_model/.huggingface/download/model-00002-of-00004.safetensors.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+0ff2ad8aea2d46c9a2f215a781975144a09de6c1d9a8225d71da130913c34f4a
+1763248029.674504

videollama3_original_model/.huggingface/download/model-00003-of-00004.safetensors.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+bd3e1c5410fb5b59e26fab11ea36d594a8ff0e11ff9fa56853c8f50a9ec3f61f
+1763248029.780972

videollama3_original_model/.huggingface/download/model-00004-of-00004.safetensors.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+47e9f7d43163f99f20820ae8f6a95c8440cc66d195b75aa6a662372cacfc9bfb
+1763248026.0995622

videollama3_original_model/.huggingface/download/model.safetensors.index.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+596eaa3c7d781aeab36fc27a7f396827be8b4ca6
+1763248024.8861187

videollama3_original_model/.huggingface/download/modeling_videollama3.py.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+1d72a96961bef654b467a0bbfb9bab20e3bad338
+1763248024.8835113

videollama3_original_model/.huggingface/download/modeling_videollama3_encoder.py.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+8ce6f86eb9763de6ad86f66f94aea99a6d35412d
+1763248024.950885

videollama3_original_model/.huggingface/download/preprocessor_config.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+067f85616904a5313af095dfde820ee42654763a
+1763248024.9201517

videollama3_original_model/.huggingface/download/processing_videollama3.py.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+0f3b28b868e9616afa3ef47fc58994ce4c89d135
+1763248024.9481933

videollama3_original_model/.huggingface/download/processor_config.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+379794013adc490b089027c06a72663e4c0c6d2d
+1763248024.9543974

videollama3_original_model/.huggingface/download/special_tokens_map.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+ac23c0aaa2434523c494330aeb79c58395378103
+1763248024.9825387

videollama3_original_model/.huggingface/download/tokenizer_config.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+c765d2ec9fcd7008d21010e9354275da5312d710
+1763248024.977651

videollama3_original_model/.huggingface/download/vocab.json.metadata

@@ -0,0 +1,3 @@
+d5b763e368861e7f5096e7ff1b49f92fbccf8ae6
+6bce3a0a3866c4791a74d83d78f6824c3af64ec3
+1763248025.0156815

videollama3_original_model/README.md

@@ -0,0 +1,138 @@
+---
+library_name: transformers
+tags:
+- multi-modal
+- large-language-model
+- video-language-model
+license: apache-2.0
+datasets:
+- lmms-lab/LLaVA-OneVision-Data
+- allenai/pixmo-docs
+- HuggingFaceM4/Docmatix
+- lmms-lab/LLaVA-Video-178K
+- ShareGPT4Video/ShareGPT4Video
+language:
+- en
+metrics:
+- accuracy
+pipeline_tag: video-text-to-text
+base_model:
+- Qwen/Qwen2.5-7B-Instruct
+- DAMO-NLP-SG/VideoLLaMA3-7B-Image
+---
+
+
+<p align="center">
+    <img src="https://cdn-uploads.huggingface.co/production/uploads/626938b16f8f86ad21deb989/tt5KYnAUmQlHtfB1-Zisl.png" width="150" style="margin-bottom: 0.2;"/>
+<p>
+
+
+<h3 align="center"><a href="https://arxiv.org/abs/2501.13106">VideoLLaMA 3: Frontier Multimodal Foundation Models for Video Understanding</a></h3>
+
+
+<h5 align="center"> If you like our project, please give us a star ⭐ on <a href="https://github.com/DAMO-NLP-SG/VideoLLaMA3">Github</a> for the latest update.  </h5>
+
+
+## 📰 News
+<!-- * **[2024.01.23]**  👋👋 Update technical report. If you have works closely related to VideoLLaMA3 but not mentioned in the paper, feel free to let us know. -->
+* **[2024.01.24]**  🔥🔥 Online Demo is available: [VideoLLaMA3-Image-7B](https://huggingface.co/spaces/lixin4ever/VideoLLaMA3-Image), [VideoLLaMA3-7B](https://huggingface.co/spaces/lixin4ever/VideoLLaMA3).
+* **[2024.01.22]**  Release models and inference code of VideoLLaMA 3.
+
+## 🌟 Introduction
+VideoLLaMA 3 represents a state-of-the-art series of multimodal foundation models designed to excel in both image and video understanding tasks. Leveraging advanced architectures, VideoLLaMA 3 demonstrates exceptional capabilities in processing and interpreting visual content across various contexts. These models are specifically designed to address complex multimodal challenges, such as integrating textual and visual information, extracting insights from sequential video data, and performing high-level reasoning over both dynamic and static visual scenes.
+
+
+
+
+
+## 🌎 Model Zoo
+| Model                | Base Model   | HF Link                                                      |
+| -------------------- | ------------ | ------------------------------------------------------------ |
+| VideoLLaMA3-7B (**This Checkpoint**)       | Qwen2.5-7B   | [DAMO-NLP-SG/VideoLLaMA3-7B](https://huggingface.co/DAMO-NLP-SG/VideoLLaMA3-7B) |
+| VideoLLaMA3-2B       | Qwen2.5-1.5B | [DAMO-NLP-SG/VideoLLaMA3-2B](https://huggingface.co/DAMO-NLP-SG/VideoLLaMA3-2B) |
+| VideoLLaMA3-7B-Image | Qwen2.5-7B   | [DAMO-NLP-SG/VideoLLaMA3-7B-Image](https://huggingface.co/DAMO-NLP-SG/VideoLLaMA3-7B-Image) |
+| VideoLLaMA3-2B-Image | Qwen2.5-1.5B | [DAMO-NLP-SG/VideoLLaMA3-2B-Image](https://huggingface.co/DAMO-NLP-SG/VideoLLaMA3-2B-Image) |
+
+We also upload the tuned vision encoder of VideoLLaMA3-7B for wider application:
+
+| Model                         | Base Model                | HF Link                                                      |
+| ----------------------------- | ------------------------- | ------------------------------------------------------------ |
+| VideoLLaMA3-7B Vision Encoder | siglip-so400m-patch14-384 | [DAMO-NLP-SG/VL3-SigLIP-NaViT](https://huggingface.co/DAMO-NLP-SG/VL3-SigLIP-NaViT) |
+
+
+
+## 🚀 Main Results
+
+
+<img width="500" alt="image" src="https://cdn-uploads.huggingface.co/production/uploads/609115c79a8bcaa437b234a9/ggmVF_v68QsHuPFVZR3MR.png">
+
+* \* denotes the reproduced results.
+
+## 🤖 Quick Start
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoProcessor, AutoModel, AutoImageProcessor
+
+model_name = "DAMO-NLP-SG/VideoLLaMA3-7B"
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    trust_remote_code=True,
+    device_map="auto",
+    torch_dtype=torch.bfloat16,
+    attn_implementation="flash_attention_2",
+)
+processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
+video_path = "put your video path here"
+question = "Describe this video in detail."
+
+# Video conversation
+conversation = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {
+        "role": "user",
+        "content": [
+            {"type": "video", "video": {"video_path": video_path, "fps": 1, "max_frames": 128}},
+            {"type": "text", "text": question},
+        ]
+    },
+]
+
+inputs = processor(conversation=conversation, return_tensors="pt")
+inputs = {k: v.cuda() if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
+if "pixel_values" in inputs:
+    inputs["pixel_values"] = inputs["pixel_values"].to(torch.bfloat16)
+output_ids = model.generate(**inputs, max_new_tokens=128)
+response = processor.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+print(response)
+```
+
+
+## Citation
+
+If you find VideoLLaMA useful for your research and applications, please cite using this BibTeX:
+```bibtex
+@article{damonlpsg2025videollama3,
+  title={VideoLLaMA 3: Frontier Multimodal Foundation Models for Image and Video Understanding},
+  author={Boqiang Zhang, Kehan Li, Zesen Cheng, Zhiqiang Hu, Yuqian Yuan, Guanzheng Chen, Sicong Leng, Yuming Jiang, Hang Zhang, Xin Li, Peng Jin, Wenqi Zhang, Fan Wang, Lidong Bing, Deli Zhao},
+  journal={arXiv preprint arXiv:2501.13106},
+  year={2025},
+  url = {https://arxiv.org/abs/2501.13106}
+}
+
+@article{damonlpsg2024videollama2,
+  title={VideoLLaMA 2: Advancing Spatial-Temporal Modeling and Audio Understanding in Video-LLMs},
+  author={Cheng, Zesen and Leng, Sicong and Zhang, Hang and Xin, Yifei and Li, Xin and Chen, Guanzheng and Zhu, Yongxin and Zhang, Wenqi and Luo, Ziyang and Zhao, Deli and Bing, Lidong},
+  journal={arXiv preprint arXiv:2406.07476},
+  year={2024},
+  url = {https://arxiv.org/abs/2406.07476}
+}
+
+@article{damonlpsg2023videollama,
+  title = {Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding},
+  author = {Zhang, Hang and Li, Xin and Bing, Lidong},
+  journal = {arXiv preprint arXiv:2306.02858},
+  year = {2023},
+  url = {https://arxiv.org/abs/2306.02858}
+}
+```

videollama3_original_model/added_tokens.json

@@ -0,0 +1,27 @@
+{
+  "</tool_call>": 151658,
+  "<image>": 151665,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|stream_end|>": 151667,
+  "<|stream_start|>": 151666,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}