| from transformers import PretrainedConfig |
| from typing import List |
|
|
| from transformers import Qwen2Config, CLIPVisionConfig |
|
|
| class InfMLLMUnifiedHDChatConfig(PretrainedConfig): |
| def __init__( |
| self, |
| vison_config=None, |
| lm_config=None, |
| lm_model="", |
| lm_tokenizer="", |
| lora_modules="", |
| lora_llm=False, |
| lora_r=128, |
| lora_alpha=256, |
| lora_dropout=0, |
| |
| encoder_img="", |
| image_size_img=336, |
| lora_encoder_img=False, |
| hd_num=9, |
| |
| encoder_video="", |
| |
| max_txt_len=4096, |
| conv_style='qwen-7b-chat', |
| precision="bf16", |
| **kwargs |
| ): |
| self.lm_model = lm_model |
| self.lm_tokenizer = lm_tokenizer |
| self.lora_modules = lora_modules |
| self.lora_llm = lora_llm |
| self.lora_r = lora_r |
| self.lora_alpha = lora_alpha |
| self.lora_dropout = lora_dropout |
|
|
| self.encoder_img = encoder_img |
| self.image_size_img = image_size_img |
| self.lora_encoder_img = lora_encoder_img |
| self.hd_num = hd_num |
|
|
| self.encoder_video = encoder_video |
| |
| self.max_txt_len = max_txt_len |
| self.conv_style = conv_style |
|
|
| self.precision = precision |
| |
| if type(vison_config) == dict: |
| self.vision_config = CLIPVisionConfig(**vison_config) |
| else: |
| self.vision_config = vison_config |
| |
| if type(lm_config) == dict: |
| self.lm_config = Qwen2Config(**lm_config) |
| else: |
| self.lm_config = lm_config |
| super().__init__(**kwargs) |
| |
