Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+hubert_kmeans/model.mdl filter=lfs diff=lfs merge=lfs -text
+llm/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+llm/tokenizer_config.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,31 @@
----
-license: cc-by-nc-4.0
----
+# SentiAvatar Model Checkpoints
+
+请从以下位置下载模型权重，并放置到此目录下：
+
+## 目录结构
+
+```
+checkpoints/
+├── llm/                              # Qwen2-0.5B SFT (Motion Token Planner)
+│   ├── config.json
+│   ├── model.safetensors
+│   ├── tokenizer.json
+│   └── ...
+├── mask_transformer/                 # Audio-Motion Mask Transformer
+│   ├── config.json
+│   └── model.safetensors
+├── rvqvae/                           # Residual VQ-VAE
+│   ├── opt.txt                       # 模型配置
+│   └── model/
+│       └── epoch_30.pth              # 模型权重
+├── face_vqvae/                       # Face VQVAE
+│   ├── pytorch_model_face_fad2cl_260116_codesize2048_codelength512.bin
+│   ├── mat_final.npy
+│   └── mat_final_R_I.npy
+├── chinese-hubert-base/              # Chinese HuBERT
+│   ├── config.json
+│   ├── preprocessor_config.json
+│   └── pytorch_model.bin
+└── eval_model/                       # ChronAccRet 评测模型
+    └── best_model.pt
+```

chinese-hubert-base/README.md

@@ -0,0 +1,48 @@
+---
+license: mit
+---
+Pretrained on 10k hours WenetSpeech L subset. More details in  [TencentGameMate/chinese_speech_pretrain](https://github.com/TencentGameMate/chinese_speech_pretrain)
+
+This model does not have a tokenizer as it was pretrained on audio alone. 
+In order to use this model speech recognition, a tokenizer should be created and the model should be fine-tuned on labeled text data.
+
+python package:
+transformers==4.16.2
+
+```python
+
+
+import torch
+import torch.nn.functional as F
+import soundfile as sf
+
+from transformers import (
+    Wav2Vec2FeatureExtractor,
+    HubertModel,
+)
+
+
+model_path=""
+wav_path=""
+
+feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(model_path)
+model = HubertModel.from_pretrained(model_path)
+
+# for pretrain: Wav2Vec2ForPreTraining
+# model = Wav2Vec2ForPreTraining.from_pretrained(model_path)
+
+model = model.to(device)
+model = model.half()
+model.eval()
+
+wav, sr = sf.read(wav_path)
+input_values = feature_extractor(wav, return_tensors="pt").input_values
+input_values = input_values.half()
+input_values = input_values.to(device)
+
+with torch.no_grad():
+    outputs = model(input_values)
+    last_hidden_state = outputs.last_hidden_state
+
+
+```

chinese-hubert-base/config.json

@@ -0,0 +1,71 @@
+{
+  "activation_dropout": 0.1,
+  "apply_spec_augment": true,
+  "architectures": [
+    "HubertModel"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.0,
+  "feat_proj_layer_norm": true,
+  "final_dropout": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_prob": 0.05,
+  "model_type": "hubert",
+  "num_attention_heads": 12,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.20.0.dev0",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 32
+}

chinese-hubert-base/preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0,
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

chinese-hubert-base/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fefccd26c2794a583b80f6f7210c721873cb7ebae2c1cde3baf9b27855e24d8
+size 377552987

eval_model/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:828fe6b931e1ca8cc8f092836290f998507be286f1ac5149ed503b49d65ddb01
+size 454859165

face_vqvae/mat_final.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f055de09c64182696499a26c2d6109349c627195bcd40c6adc3dd27f3922b34b
+size 21140

face_vqvae/mat_final_R_I.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67befad9e98e372995b5c5d6883bf98a4f6d993f09e139857d12fe16c7257242
+size 21140

face_vqvae/pytorch_model_face_fad2cl_260116_codesize2048_codelength512.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1353b1f67308a4ebe6a5c81a0c8a255963b806125717c0d5eb32165767c974f0
+size 51968811

hubert_kmeans/model.mdl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1faf1a70098f1853427347520475a802de9aaf7dfb955c3af2cd83b6ca3857cd
+size 1538989

llm/config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 225250
+}

llm/generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.50.0"
+}

llm/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:148ea91b5a9c20a6e388e4145b24228b178174c5d04cf2a24c8ee45d2a8426c7
+size 1119476616

llm/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llm/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77ea10323ffa96cd0baf4ad882dc2bced4d304f9fafd10ee05b4950d98c1179b
+size 25286251

llm/tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d454d38e6758b1e98515e8a7f5460aed3b0af55156b7dd1557cd0185114dd544
+size 13133512

mask_transformer/config.json

@@ -0,0 +1,22 @@
+{
+  "architectures": [
+    "AudioMotionTransformer"
+  ],
+  "audio_feat_dim": 768,
+  "codebook_size": 512,
+  "cond_drop_prob": 0.2,
+  "dropout": 0.2,
+  "dtype": "float32",
+  "hidden_act": "gelu",
+  "hidden_size": 512,
+  "intermediate_size": 1536,
+  "max_position_embeddings": 512,
+  "model_type": "audio_motion_transformer",
+  "num_frames": 5,
+  "num_heads": 16,
+  "num_layers": 8,
+  "num_tokens_per_frame": 4,
+  "rms_norm_eps": 1e-06,
+  "transformers_version": "4.57.1",
+  "vocab_size": 2049
+}

mask_transformer/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8ed946d32423cc7bf393f9967f7d3fbd11894b5f0b67ff55dbffdb5471358bb
+size 96170728

mask_transformer/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6113ea4928940205ec665c711dc80be83466909f2cbb7dcdc9a05479a1d5b970
+size 192411979

mask_transformer/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba6cdd9db0560aaf782233459fbc22e7b29251377516e638dc99072fc9275b68
+size 15429

mask_transformer/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f166488621222660ee59b49e91252a3d8629c4c55a1695f1ee697de614582c8c
+size 15429

mask_transformer/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abac5d9e6b4349d5e1569132ca1055d02315f27bf7dbc7383a633a787ac0de72
+size 15429

mask_transformer/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ea0c1e69201fb66e3dede3dfb473772b1a42c59a22368456e5dfc002b652e3
+size 15429

mask_transformer/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a68cc89bc205077640e629149b54fb06ae537dfdeb010a8c11a0804c012cda
+size 1465

mask_transformer/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:845ce5cf164cb40b2109e8c7665fb79c6e8ebf8e4aadeab38af2e24879078735
+size 5777

rvqvae/model/epoch_30.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92e3acf2b8a34a8705b24a8005b891d18b590d78fb2d37a11debe29389530a43
+size 790198921

rvqvae/opt.txt

@@ -0,0 +1,58 @@
+------------ Options -------------
+batch_size: 256
+body_dim: 153
+body_joints_num: 24
+body_parts: ['body', 'left', 'right', 'positions']
+checkpoints_dir: ./checkpoints
+code_dim: 512
+commit: 0.02
+data_root: /disk1/chuhao/dataset/mocap/mocap_susu_gen_demo/quat63nodes_v4_fix_pos
+dataset_name: quat63nodes_v2_0120
+debug: False
+depth: 3
+dilation_growth_rate: 3
+down_t: 1
+eval_every_e: 1
+feat_bias: 5
+fps: 20
+gamma: 0.05
+gpu_id: 0
+is_continue: False
+left_dim: 120
+left_joints_num: 20
+local_rank: 0
+log_dir: ./log/vq
+log_every: 10
+loss_vel: 50.0
+lr: 0.0001
+max_epoch: 100
+milestones: [50000, 1000000]
+mu: 0.99
+name: gqzV4
+nb_code: 512
+num_quantizers: 4
+num_workers: 4
+quantize_dropout_cutoff_index: 1
+quantize_dropout_prob: 0.8
+recons_loss: l1_smooth
+right_dim: 120
+right_joints_num: 20
+save_every_e: 2
+save_latest: 500
+seed: 3407
+shared_codebook: False
+start_positions_epoch: 0
+stride_t: 2
+total_joints_num: 63
+use_whole_encoder: False
+vq_act: relu
+vq_cnn_depth: 3
+vq_norm: None
+warm_up_iter: 2000
+weight_decay: 0.0
+weight_rec: 5.0
+which_epoch: latest
+whole_dim: 393
+width: 512
+window_size: 64
+-------------- End ----------------