Add MIMIC-CXR FairLLaVA checkpoint

mimic-cxr/README.md

@@ -0,0 +1,26 @@
+---
+license: apache-2.0
+library_name: peft
+base_model: lmsys/vicuna-7b-v1.5
+tags:
+  - medical-imaging
+  - chest-xray
+  - mimic-cxr
+  - fairness
+  - lora
+---
+
+# FairLLaVA — MIMIC-CXR
+
+Fairness-aware LoRA adapter on top of LLaVA-Rad (Vicuna-7B + BiomedCLIP-CXR-518)
+for MIMIC-CXR chest-X-ray report generation. Trained with the FairLLaVA
+mutual-information regularizer on patient demographics (age, sex, race).
+
+## Files
+- `adapter_model.safetensors`, `adapter_config.json`, `non_lora_trainables.bin` — LoRA weights
+- `mm_projector.bin` — multimodal projector
+- `tokenizer.model`, `tokenizer_config.json`, `special_tokens_map.json`, `config.json` — tokenizer / config
+
+## Usage
+See https://github.com/bhosalems/FairLLaVA → `inference.py`. Point `model_path` at
+this directory and set `model_base="lmsys/vicuna-7b-v1.5"`.

mimic-cxr/adapter_config.json

@@ -0,0 +1,38 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "lmsys/vicuna-7b-v1.5",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "club_modules.age",
+    "club_modules.race",
+    "club_modules.gender"
+  ],
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

mimic-cxr/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8985de1c3d5e0be1617698338207fb9128f4586475cc0b429f5d1f0a6c6342
+size 319991916

mimic-cxr/config.json

@@ -0,0 +1,44 @@
+{
+  "_name_or_path": "lmsys/vicuna-7b-v1.5",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "freeze_mm_mlp_adapter": false,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_aspect_ratio": "square",
+  "image_grid_pinpoints": null,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mm_hidden_size": 768,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_use_im_patch_token": false,
+  "mm_use_im_start_end": false,
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -2,
+  "mm_vision_tower": "biomedclip_cxr_518",
+  "mm_vision_tower_checkpoint": "/home/csgrad/mbhosale/phd/MrFair/LLaVA-Rad/llava-rad_hf/biomedclipcxr_518_checkpoint.pt",
+  "mm_vision_tower_config": "llava/model/multimodal_encoder/open_clip_encoder/model_configs/biomedclip_cxr_518.json",
+  "model_type": "llava",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "output_hidden_states": true,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.39.3",
+  "tune_mm_mlp_adapter": false,
+  "use_cache": false,
+  "use_mm_proj": true,
+  "vocab_size": 32000
+}

mimic-cxr/mm_projector.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edc0c9123bca4edabc7f64c51772464917ba934a73ad7199ed26f3ac288d1c86
+size 302008633

mimic-cxr/non_lora_trainables.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afcc665b33a66ff25a10f091cb8c1de2edaa5ec4f2e2d60aef7e4cf826b39526
+size 39864496

mimic-cxr/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

mimic-cxr/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

mimic-cxr/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}