Initial upload of Phi-4-NVFP4

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 DeepSeek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

NOTICE

@@ -0,0 +1,6 @@
+NOTICE
+
+This artifact is a derivative work of microsoft/phi-4, distributed under the MIT License.
+The full license text is in the LICENSE file at the root of this repository.
+
+Source model: https://huggingface.co/microsoft/phi-4

README.md

@@ -0,0 +1,57 @@
+---
+license: mit
+license_link: https://opensource.org/license/mit
+base_model: microsoft/phi-4
+tags:
+  - quantized
+  - 4-bit
+  - int4
+  - awq
+language:
+  - en
+library_name: transformers
+pipeline_tag: text-generation
+---
+
+# Phi-4-NVFP4
+
+INT4 weight-only quantization of [`microsoft/phi-4`](https://huggingface.co/microsoft/phi-4).
+
+Microsoft Phi-4 in NVFP4 W4A4. Native vLLM compressed-tensors. About 8 GB on disk.
+
+| Property | Value |
+|---|---|
+| Base model | [microsoft/phi-4](https://huggingface.co/microsoft/phi-4) |
+| Quantization | INT4 weight-only |
+| Approx. on-disk size | ~9.7 GB |
+| License | MIT License |
+| Languages | English |
+
+## Load (vLLM)
+
+```bash
+vllm serve drawais/Phi-4-NVFP4 \
+  --max-model-len 32768 \
+  --gpu-memory-utilization 0.94
+```
+
+```python
+from vllm import LLM, SamplingParams
+llm = LLM(model="drawais/Phi-4-NVFP4", max_model_len=32768)
+print(llm.generate(["Hello!"], SamplingParams(max_tokens=128))[0].outputs[0].text)
+```
+
+## Footprint
+
+~9.7 GB on disk. Recommended VRAM: enough headroom for KV cache.
+
+## License & attribution
+
+This artifact is a derivative work of [`microsoft/phi-4`](https://huggingface.co/microsoft/phi-4),
+released by its original authors under the **MIT License**.
+
+This artifact is distributed under the same license. The full license text is
+included in [`LICENSE`](LICENSE), and required attribution is in [`NOTICE`](NOTICE).
+
+License text: https://opensource.org/license/mit
+Source model: https://huggingface.co/microsoft/phi-4

chat_template.jinja

@@ -0,0 +1 @@
+{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'assistant') %}{{'<|im_start|>assistant<|im_sep|>' + message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant<|im_sep|>' }}{% endif %}

config.json

@@ -0,0 +1,84 @@
+{
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 100257,
+  "dtype": "bfloat16",
+  "embd_pdrop": 0.0,
+  "eos_token_id": 100265,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 17920,
+  "max_position_embeddings": 16384,
+  "model_type": "phi3",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 10,
+  "original_max_position_embeddings": 16384,
+  "pad_token_id": 100349,
+  "quantization_config": {
+    "config_groups": {
+      "group_0": {
+        "format": "nvfp4-pack-quantized",
+        "input_activations": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": "local",
+          "group_size": 16,
+          "num_bits": 4,
+          "observer": "static_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": "torch.float8_e4m3fn",
+          "strategy": "tensor_group",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        },
+        "output_activations": null,
+        "targets": [
+          "Linear"
+        ],
+        "weights": {
+          "actorder": null,
+          "block_structure": null,
+          "dynamic": false,
+          "group_size": 16,
+          "num_bits": 4,
+          "observer": "memoryless_minmax",
+          "observer_kwargs": {},
+          "scale_dtype": "torch.float8_e4m3fn",
+          "strategy": "tensor_group",
+          "symmetric": true,
+          "type": "float",
+          "zp_dtype": null
+        }
+      }
+    },
+    "format": "nvfp4-pack-quantized",
+    "global_compression_ratio": null,
+    "ignore": [
+      "lm_head"
+    ],
+    "kv_cache_scheme": null,
+    "quant_method": "compressed-tensors",
+    "quantization_status": "compressed",
+    "sparsity_config": {},
+    "transform_config": {},
+    "version": "0.15.1.a20260428"
+  },
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "partial_rotary_factor": 1.0,
+    "rope_theta": 250000,
+    "rope_type": "default"
+  },
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "transformers_version": "5.8.0.dev0",
+  "use_cache": true,
+  "vocab_size": 100352
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": 100265,
+  "pad_token_id": 100349,
+  "transformers_version": "5.8.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56d99adbc01934fd87dcc0bb6ec336a3855f618d31b3105c365887ead5be81a6
+size 9723835440

recipe.yaml

@@ -0,0 +1,7 @@
+default_stage:
+  default_modifiers:
+    QuantizationModifier:
+      targets: [Linear]
+      ignore: [lm_head, 're:.*embed.*', 're:.*router.*']
+      scheme: NVFP4
+      bypass_divisibility_checks: false

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 16384,
+  "pad_token": "<|dummy_85|>",
+  "tokenizer_class": "TokenizersBackend"
+}