Upload folder using huggingface_hub

modeling_MMRet_CLIP.py

@@ -38,10 +38,10 @@ from transformers.utils import (
     replace_return_docstrings,
 )
 from transformers.models.clip.configuration_clip import CLIPConfig, CLIPTextConfig, CLIPVisionConfig
-
+from transformers import CLIPProcessor
 
 if is_flash_attn_2_available():
-    from ...modeling_flash_attention_utils import _flash_attention_forward
+    from transformers.modeling_flash_attention_utils import _flash_attention_forward
 
 
 logger = logging.get_logger(__name__)
@@ -50,7 +50,7 @@ logger = logging.get_logger(__name__)
 _CONFIG_FOR_DOC = "MMRet_CLIP"
 
 # Image classification docstring
-_IMAGE_CLASS_CHECKPOINT = "JUNJIE/MMRet-large"
+_IMAGE_CLASS_CHECKPOINT = "JUNJIE99/MMRet-base"
 _IMAGE_CLASS_EXPECTED_OUTPUT = "LABEL_0"
 
 
@@ -1160,6 +1160,9 @@ class CLIPModel(CLIPPreTrainedModel):
         # Initialize weights and apply final processing
         self.post_init()
 
+    def set_processor(self, model_name):
+        self.processor = CLIPProcessor.from_pretrained(model_name)
+
     @add_start_docstrings_to_model_forward(CLIP_TEXT_INPUTS_DOCSTRING)
     def get_text_features(
         self,
@@ -1258,18 +1261,18 @@ class CLIPModel(CLIPPreTrainedModel):
 
 
     def encode_image(self, images):
-        embeddings = self.model.get_image_features(images)
+        embeddings = self.get_image_features(images)
         embeddings = torch.nn.functional.normalize(embeddings, dim=-1)
         return embeddings
     
     def encode_text(self, text):
-        embeddings = self.model.get_text_features(**text)
+        embeddings = self.get_text_features(**text)
         embeddings = torch.nn.functional.normalize(embeddings, dim=-1)
         return embeddings
     
     def encode_multimodal(self, images, text):
-        text_embeddings = self.model.get_text_features(**text)
-        image_embeddings = self.model.get_image_features(images)
+        text_embeddings = self.get_text_features(**text)
+        image_embeddings = self.get_image_features(images)
         
         embeddings = text_embeddings + image_embeddings    
         embeddings = torch.nn.functional.normalize(embeddings, dim=-1)
@@ -1278,7 +1281,7 @@ class CLIPModel(CLIPPreTrainedModel):
 
     def data_process(self, images=None, text=None):
         if images is None and text is not None:
-            text = self.processor(text=text, return_tensors="pt", padding=True).to(self.model.device)
+            text = self.processor(text=text, return_tensors="pt", padding=True).to(self.device)
             
             return images, text, "text"
         elif images is not None and text is None:
@@ -1286,7 +1289,7 @@ class CLIPModel(CLIPPreTrainedModel):
                 images = Image.open(images).convert("RGB")
             elif isinstance(images, list):
                 images = [Image.open(image).convert("RGB") for image in images]
-            images = self.processor(images=images, return_tensors="pt").to(self.model.device)
+            images = self.processor(images=images, return_tensors="pt").to(self.device)
             images = images["pixel_values"]
             return images, text, "images"
         elif images is not None and text is not None:
@@ -1296,9 +1299,9 @@ class CLIPModel(CLIPPreTrainedModel):
             elif isinstance(images, list):
                 assert len(images) == len(text), "images and text must be lists of the same length when use list"
                 images = [Image.open(image).convert("RGB") for image in images]
-            images = self.processor(images=images, return_tensors="pt").to(self.model.device)
+            images = self.processor(images=images, return_tensors="pt").to(self.device)
             images = images["pixel_values"]
-            text = self.processor(text=text, return_tensors="pt", padding=True).to(self.model.device)
+            text = self.processor(text=text, return_tensors="pt", padding=True).to(self.device)
             return images, text, "multimodal"
         else:
             raise ValueError("images and text cannot both be None")