Update README.md
Browse files
README.md
CHANGED
|
@@ -118,7 +118,7 @@ transformers>=5.6.0
|
|
| 118 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 119 |
import torch
|
| 120 |
|
| 121 |
-
model_path = "tencent/Hy-MT2-
|
| 122 |
|
| 123 |
# Load tokenizer
|
| 124 |
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
|
@@ -161,7 +161,7 @@ uv pip install --editable . --torch-backend=auto
|
|
| 161 |
Start the vLLM server:
|
| 162 |
|
| 163 |
```bash
|
| 164 |
-
vllm serve tencent/Hy-MT2-
|
| 165 |
```
|
| 166 |
|
| 167 |
### sglang
|
|
@@ -178,7 +178,7 @@ pip3 install -e "python"
|
|
| 178 |
Launch SGLang server:
|
| 179 |
|
| 180 |
```bash
|
| 181 |
-
python3 -m sglang.launch_server --model tencent/Hy-MT2-
|
| 182 |
```
|
| 183 |
|
| 184 |
### llama_cpp
|
|
|
|
| 118 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 119 |
import torch
|
| 120 |
|
| 121 |
+
model_path = "tencent/Hy-MT2-7B-FP8"
|
| 122 |
|
| 123 |
# Load tokenizer
|
| 124 |
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
|
|
|
|
| 161 |
Start the vLLM server:
|
| 162 |
|
| 163 |
```bash
|
| 164 |
+
vllm serve tencent/Hy-MT2-7B-FP8 --tensor-parallel-size 1
|
| 165 |
```
|
| 166 |
|
| 167 |
### sglang
|
|
|
|
| 178 |
Launch SGLang server:
|
| 179 |
|
| 180 |
```bash
|
| 181 |
+
python3 -m sglang.launch_server --model tencent/Hy-MT2-7B-FP8 --tp 1
|
| 182 |
```
|
| 183 |
|
| 184 |
### llama_cpp
|