Add SyncHuman model checkpoints

.gitattributes

@@ -0,0 +1,6 @@
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text

ckpts/OneStage/SyncHuman_2D3DCrossSpaceDiffusion/config.json

@@ -0,0 +1,108 @@
+{
+  "_class_name": "SyncHuman_2D3DCrossSpaceDiffusion",
+  "_diffusers_version": "0.29.1",
+  "act_fn": "silu",
+  "addition_channels": [
+    1280,
+    1280,
+    1280
+  ],
+  "addition_downsample": false,
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "cd_attention_last": false,
+  "cd_attention_mid": false,
+  "center_input_sample": false,
+  "class_embed_type": "projection",
+  "class_embeddings_concat": false,
+  "cond_channels": 1024,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 1024,
+  "cross_attention_norm": null,
+  "decay": 0.9999,
+  "down_block_types": [
+    "CrossAttnDownBlockMV2D",
+    "CrossAttnDownBlockMV2D",
+    "CrossAttnDownBlockMV2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 8,
+  "inv_gamma": 1.0,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlockMV2DCrossAttn",
+  "min_decay": 0.0,
+  "mlp_ratio": 4,
+  "model_channels": 1024,
+  "multiview_attention": true,
+  "mv_cond_channels": 640,
+  "mvcd_attention": true,
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_blocks": 24,
+  "num_class_embeds": null,
+  "num_dual_CrossAttblocks": 2,
+  "num_head_channels": 64,
+  "num_heads": 16,
+  "num_views": 5,
+  "only_cross_attention": false,
+  "optimization_step": 42500,
+  "out_channels": 4,
+  "patch_size": 1,
+  "pe_mode": "ape",
+  "power": 0.6666666666666666,
+  "projection_class_embeddings_input_dim": 2048,
+  "qk_rms_norm": true,
+  "qk_rms_norm_cross": false,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "resolution": 16,
+  "sample_size": 96,
+  "selfattn_block": "self_rowwise",
+  "share_mod": false,
+  "sparse_mv_attention": true,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": 1,
+  "trellis_in_channels": 8,
+  "trellis_out_channels": 8,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlockMV2D",
+    "CrossAttnUpBlockMV2D",
+    "CrossAttnUpBlockMV2D"
+  ],
+  "upcast_attention": true,
+  "update_after_step": 0,
+  "use_checkpoint": false,
+  "use_dino": false,
+  "use_ema_warmup": false,
+  "use_fp16": true,
+  "use_linear_projection": true
+}

ckpts/OneStage/SyncHuman_2D3DCrossSpaceDiffusion/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf8835494bf8afad7fe670a675431cee0dad0dbd05f63a4df23313da5fe08705
+size 6814820408

ckpts/OneStage/feature_extractor/preprocessor_config.json

@@ -0,0 +1,27 @@
+{
+  "crop_size": {
+    "height": 224,
+    "width": 224
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 224
+  }
+}

ckpts/OneStage/image_encoder/config.json

@@ -0,0 +1,23 @@
+{
+  "_name_or_path": "stabilityai/stable-diffusion-2-1-unclip",
+  "architectures": [
+    "CLIPVisionModelWithProjection"
+  ],
+  "attention_dropout": 0.0,
+  "dropout": 0.0,
+  "hidden_act": "gelu",
+  "hidden_size": 1280,
+  "image_size": 224,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 5120,
+  "layer_norm_eps": 1e-05,
+  "model_type": "clip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 32,
+  "patch_size": 14,
+  "projection_dim": 1024,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.0"
+}

ckpts/OneStage/image_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae616c24393dd1854372b0639e5541666f7521cbe219669255e865cb7f89466a
+size 1264217240

ckpts/OneStage/pipeline_config.json

@@ -0,0 +1,29 @@
+{
+  "num_views": 5,
+  "metadata": {
+    "feature_extractor": {
+      "class": "transformers.models.clip.image_processing_clip.CLIPImageProcessor",
+      "subdir": "feature_extractor"
+    },
+    "image_encoder": {
+      "class": "transformers.models.clip.modeling_clip.CLIPVisionModelWithProjection",
+      "subdir": "image_encoder"
+    },
+    "text_encoder": {
+      "class": "transformers.models.clip.modeling_clip.CLIPTextModel",
+      "subdir": "text_encoder"
+    },
+    "SyncHuman_2D3DCrossSpaceDiffusion": {
+      "class": "SyncHuman.models.OneStage.SyncHuman_2D3DCrossSpaceDiffusion.SyncHuman_2D3DCrossSpaceDiffusion",
+      "subdir": "SyncHuman_2D3DCrossSpaceDiffusion"
+    },
+    "vae": {
+      "class": "diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL",
+      "subdir": "vae"
+    },
+    "sparse_structure_decoder": {
+      "class": "SyncHuman.models.OneStage.sparse_structure_vae.SparseStructureDecoder",
+      "subdir": "sparse_structure_decoder"
+    }
+  }
+}

ckpts/OneStage/sparse_structure_decoder/config.json

@@ -0,0 +1,12 @@
+
+{
+    "name": "SparseStructureDecoder",
+    "args": {
+        "out_channels": 1,
+        "latent_channels": 8,
+        "num_res_blocks": 2,
+        "num_res_blocks_middle": 2,
+        "channels": [512, 128, 32],
+        "use_fp16": true
+    }
+}

ckpts/OneStage/sparse_structure_decoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70da369b1120a5a267be33ec133f8197f1647fa91fd5b81cd71b13c7031e619d
+size 147368034

ckpts/OneStage/text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "stabilityai/stable-diffusion-2-1-unclip",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1024,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 23,
+  "pad_token_id": 1,
+  "projection_dim": 512,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.0",
+  "vocab_size": 49408
+}

ckpts/OneStage/text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc1827c465450322616f06dea41596eac7d493f4e95904dcb51f0fc745c4e13f
+size 680820392

ckpts/OneStage/vae/config.json

@@ -0,0 +1,37 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.29.1",
+  "_name_or_path": "stabilityai/stable-diffusion-2-1-unclip",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "latents_mean": null,
+  "latents_std": null,
+  "layers_per_block": 2,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 768,
+  "scaling_factor": 0.18215,
+  "shift_factor": null,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ],
+  "use_post_quant_conv": true,
+  "use_quant_conv": true
+}

ckpts/OneStage/vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e4c08995484ee61270175e9e7a072b66a6e4eeb5f0c266667fe1f45b90daf9a
+size 167335342

ckpts/SecondStage/ckpts/decoder_GS/config.json

@@ -0,0 +1,34 @@
+{
+    "name": "SLatGaussianDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "mv_condition_mode":"4_view",
+        "use_multiscale": true,
+        "use_faceinfo":false,
+        "representation_config": {
+            "lr": {
+                "_xyz": 1.0,
+                "_features_dc": 1.0,
+                "_opacity": 1.0,
+                "_scaling": 1.0,
+                "_rotation": 0.1
+            },
+            "perturb_offset": true,
+            "voxel_size": 1.5,
+            "num_gaussians": 32,
+            "2d_filter_kernel_size": 0.1,
+            "3d_filter_kernel_size": 9e-4,
+            "scaling_bias": 4e-3,
+            "opacity_bias": 0.1,
+            "scaling_activation": "softplus"
+        }
+    }
+}

ckpts/SecondStage/ckpts/decoder_GS/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d6138d8e2f0756a67baba2c753019c224222c8ad5bec8f555a57ae8e0af2546
+size 1809690624

ckpts/SecondStage/ckpts/decoder_Mesh/config.json

@@ -0,0 +1,20 @@
+{
+    "name": "SLatMeshDecoder",
+    "args": {
+        "resolution": 64,
+        "model_channels": 768,
+        "latent_channels": 8,
+        "num_blocks": 12,
+        "num_heads": 12,
+        "mlp_ratio": 4,
+        "attn_mode": "swin",
+        "window_size": 8,
+        "use_fp16": true,
+        "mv_condition_mode":"4_view",
+        "use_multiscale": true,
+        "use_faceinfo":false,
+        "representation_config": {
+            "use_color": true
+        }
+    }
+}

ckpts/SecondStage/ckpts/decoder_Mesh/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8f1e2088ad3f4f87d2502744cc66c116d239b7dd1b74e5d850dad2aa56afd09
+size 1831922964

ckpts/SecondStage/ckpts/slat_flow/config.json

@@ -0,0 +1,19 @@
+{
+    "name": "SLatFlowModel",
+    "args": {
+        "resolution": 64,
+        "in_channels": 8,
+        "out_channels": 8,
+        "model_channels": 1024,
+        "cond_channels": 1024,
+        "num_blocks": 24,
+        "num_heads": 16,
+        "mlp_ratio": 4,
+        "patch_size": 2,
+        "num_io_res_blocks": 2,
+        "io_block_channels": [128],
+        "pe_mode": "ape",
+        "qk_rms_norm": true,
+        "use_fp16": true
+    }
+}

ckpts/SecondStage/ckpts/slat_flow/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:693fb2a58ad497bd222007301eeec49d14d60f8c12d2f2f00c221fa747b4c66c
+size 1203755136

ckpts/SecondStage/pipeline.json

@@ -0,0 +1,45 @@
+{
+    "name": "SyncHumanTwoStagePipeline",
+    "args": {
+        "models": {
+            "slat_decoder_gs": "ckpts/decoder_GS",
+            "slat_decoder_mesh": "ckpts/decoder_Mesh",
+            "slat_flow_model": "ckpts/slat_flow"
+        },
+        "slat_sampler": {
+            "name": "FlowEulerGuidanceIntervalSampler",
+            "args": {
+                "sigma_min": 1e-5
+            },
+            "params": {
+                "steps": 25,
+                "cfg_strength": 5.0,
+                "cfg_interval": [0.5, 1.0],
+                "rescale_t": 3.0
+            }
+        },
+        "slat_normalization": {
+            "mean": [
+                -2.1687545776367188,
+                -0.004347046371549368,
+                -0.13352349400520325,
+                -0.08418072760105133,
+                -0.5271206498146057,
+                0.7238689064979553,
+                -1.1414450407028198,
+                1.2039363384246826
+            ],
+            "std": [
+                2.377650737762451,
+                2.386378288269043,
+                2.124418020248413,
+                2.1748552322387695,
+                2.663944721221924,
+                2.371192216873169,
+                2.6217446327209473,
+                2.684523105621338
+            ]
+        },
+        "image_cond_model": "dinov2_vitl14_reg"
+    }
+}