Upload 944 files

.gitattributes

@@ -33,3 +33,38 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/figure/framework.png filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/figure/visualization_examples.png filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_1/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_10/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_100000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_30000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_5/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_60000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_70000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_80000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train-change-bg/output/shot_90000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/figure/framework.png filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/figure/visualization_examples.png filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_1/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_10/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_10000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_100000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_20000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_30000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_40000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_5/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_50000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_60000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_70000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_80000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+LoCoOp-train/output/shot_90000/prompt_learner/model.pth.tar-50 filter=lfs diff=lfs merge=lfs -text
+zest_code/demo_assets/depths/n02824058_184.png filter=lfs diff=lfs merge=lfs -text
+zest_code/demo_assets/input_imgs/n02824058_184.png filter=lfs diff=lfs merge=lfs -text
+zest_code/demo_assets/material_exemplars/101001.png filter=lfs diff=lfs merge=lfs -text
+zest_code/demo_assets/output_images/result.png filter=lfs diff=lfs merge=lfs -text
+zest_code/demo_assets/temp_file/init_img.png filter=lfs diff=lfs merge=lfs -text
+zest_code/error1.txt filter=lfs diff=lfs merge=lfs -text
+zest_code/error2.txt filter=lfs diff=lfs merge=lfs -text
+zest_code/fig/gradio_demo.png filter=lfs diff=lfs merge=lfs -text
+zest_code/fig/method.jpg filter=lfs diff=lfs merge=lfs -text

ID-like-train-change-bg/README.md

@@ -0,0 +1,3 @@
+# ID-like Prompt Learning for Few-Shot Out-of-Distribution Detection
+
+This repository contains the code of our CVPR'2024 paper ID-like Prompt Learning for Few-Shot Out-of-Distribution Detection. We will gradually improve and enhance the code.

ID-like-train-change-bg/bash_allocation.slurm

@@ -0,0 +1,15 @@
+#!/bin/bash
+#SBATCH --job-name=zzzz1
+#SBATCH --output=output1.txt
+#SBATCH --error=error1.txt
+#SBATCH --cpus-per-task=5
+#SBATCH --ntasks=4
+#SBATCH --gres=gpu:4
+#SBATCH --mem=100000  
+#SBATCH -N 1
+
+
+python batch_file_deal.py 
+
+# 取消当前作业以释放节点
+scancel $SLURM_JOB_ID

ID-like-train-change-bg/batch_file_deal.py

@@ -0,0 +1,39 @@
+import subprocess
+import os
+
+# 设置 PYTHONPATH 环境变量
+pythonpath = '.'
+if 'PYTHONPATH' in os.environ:
+    pythonpath += ':' + os.environ['PYTHONPATH']
+os.environ['PYTHONPATH'] = pythonpath
+
+ROOT = "/home/zhourixin/OOD_Folder/CODE/other_methods/ID-like-train-change-bg"
+
+
+run_file = ROOT+"/eval_ood_detection.py"
+
+# subprocess.run(["python", run_file, "--n_shot=1", "--batch_size=1"])
+# subprocess.run(["python", run_file, "--n_shot=5", "--batch_size=1"])
+# subprocess.run(["python", run_file, "--n_shot=10", "--batch_size=1"])
+
+# subprocess.run(["python", run_file, "--n_shot=10000"])
+
+# subprocess.run(["python", run_file, "--n_shot=20000"])
+
+# subprocess.run(["python", run_file, "--n_shot=30000"])
+
+# subprocess.run(["python", run_file, "--n_shot=40000"])
+
+# subprocess.run(["python", run_file, "--n_shot=50000"])
+
+# subprocess.run(["python", run_file, "--n_shot=60000"])
+
+# subprocess.run(["python", run_file, "--n_shot=70000"])
+
+# subprocess.run(["python", run_file, "--n_shot=80000"])
+
+# subprocess.run(["python", run_file, "--n_shot=90000"])
+
+subprocess.run(["python", run_file, "--n_shot=100000"])
+
+

ID-like-train-change-bg/clip checkpoint path/ViT-B-16.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e213bf161ab676ffde04a98d171217cce89419f17ec4b3fd69552102861c01ca
+size 13434880

ID-like-train-change-bg/clip/__init__.py

@@ -0,0 +1 @@
+from .clip import *

ID-like-train-change-bg/clip/bpe_simple_vocab_16e6.txt.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a
+size 1356917

ID-like-train-change-bg/clip/clip.py

@@ -0,0 +1,232 @@
+import hashlib
+import os
+import urllib
+import warnings
+from typing import Any, Union, List
+# from pkg_resources import packaging
+
+import torch
+from PIL import Image
+from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
+from tqdm import tqdm
+
+from .model import build_model
+from .simple_tokenizer import SimpleTokenizer as _Tokenizer
+
+try:
+    from torchvision.transforms import InterpolationMode
+    BICUBIC = InterpolationMode.BICUBIC
+except ImportError:
+    BICUBIC = Image.BICUBIC
+
+
+# if packaging.version.parse(torch.__version__) < packaging.version.parse("1.7.1"):
+#     warnings.warn("PyTorch version 1.7.1 or higher is recommended")
+
+
+__all__ = ["available_models", "load", "tokenize"]
+_tokenizer = _Tokenizer()
+
+_MODELS = {
+    "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt",
+    "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt",
+    "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt",
+    "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt",
+    "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt",
+    "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt",
+    "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt",
+    "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt",
+}
+
+
+def _download(url: str, root: str):
+    os.makedirs(root, exist_ok=True)
+    filename = os.path.basename(url)
+
+    expected_sha256 = url.split("/")[-2]
+    download_target = os.path.join(root, filename)
+
+    if os.path.exists(download_target) and not os.path.isfile(download_target):
+        raise RuntimeError(f"{download_target} exists and is not a regular file")
+
+    if os.path.isfile(download_target):
+        if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256:
+            return download_target
+        else:
+            warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file")
+
+    with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:
+        with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop:
+            while True:
+                buffer = source.read(8192)
+                if not buffer:
+                    break
+
+                output.write(buffer)
+                loop.update(len(buffer))
+
+    if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256:
+        raise RuntimeError(f"Model has been downloaded but the SHA256 checksum does not not match")
+
+    return download_target
+
+
+def _convert_image_to_rgb(image):
+    return image.convert("RGB")
+
+
+def _transform(n_px):
+    return Compose([
+        Resize(n_px, interpolation=BICUBIC),
+        CenterCrop(n_px),
+        _convert_image_to_rgb,
+        ToTensor(),
+        Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)),
+    ])
+
+
+def available_models() -> List[str]:
+    """Returns the names of available CLIP models"""
+    return list(_MODELS.keys())
+
+
+def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None):
+    """Load a CLIP model
+
+    Parameters
+    ----------
+    name : str
+        A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict
+
+    device : Union[str, torch.device]
+        The device to put the loaded model
+
+    jit : bool
+        Whether to load the optimized JIT model or more hackable non-JIT model (default).
+
+    download_root: str
+        path to download the model files; by default, it uses "~/.cache/clip"
+
+    Returns
+    -------
+    model : torch.nn.Module
+        The CLIP model
+
+    preprocess : Callable[[PIL.Image], torch.Tensor]
+        A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input
+    """
+    if name in _MODELS:
+        model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip"))
+    elif os.path.isfile(name):
+        model_path = name
+    else:
+        raise RuntimeError(f"Model {name} not found; available models = {available_models()}")
+
+    try:
+        # loading JIT archive
+        model = torch.jit.load(model_path, map_location=device if jit else "cpu").eval()
+        state_dict = None
+    except RuntimeError:
+        # loading saved state dict
+        if jit:
+            warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead")
+            jit = False
+        state_dict = torch.load(model_path, map_location="cpu")
+
+    embed_dim = model.state_dict()["text_projection"].shape[1]
+    if not jit:
+        model = build_model(state_dict or model.state_dict()).to(device)
+        if str(device) == "cpu":
+            model.float()
+        return model, embed_dim, _transform(model.visual.input_resolution)
+
+    # patch the device names
+    device_holder = torch.jit.trace(lambda: torch.ones([]).to(torch.device(device)), example_inputs=[])
+    device_node = [n for n in device_holder.graph.findAllNodes("prim::Constant") if "Device" in repr(n)][-1]
+
+    def patch_device(module):
+        try:
+            graphs = [module.graph] if hasattr(module, "graph") else []
+        except RuntimeError:
+            graphs = []
+
+        if hasattr(module, "forward1"):
+            graphs.append(module.forward1.graph)
+
+        for graph in graphs:
+            for node in graph.findAllNodes("prim::Constant"):
+                if "value" in node.attributeNames() and str(node["value"]).startswith("cuda"):
+                    node.copyAttributes(device_node)
+
+    model.apply(patch_device)
+    patch_device(model.encode_image)
+    patch_device(model.encode_text)
+
+    # patch dtype to float32 on CPU
+    if str(device) == "cpu":
+        float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[])
+        float_input = list(float_holder.graph.findNode("aten::to").inputs())[1]
+        float_node = float_input.node()
+
+        def patch_float(module):
+            try:
+                graphs = [module.graph] if hasattr(module, "graph") else []
+            except RuntimeError:
+                graphs = []
+
+            if hasattr(module, "forward1"):
+                graphs.append(module.forward1.graph)
+
+            for graph in graphs:
+                for node in graph.findAllNodes("aten::to"):
+                    inputs = list(node.inputs())
+                    for i in [1, 2]:  # dtype can be the second or third argument to aten::to()
+                        if inputs[i].node()["value"] == 5:
+                            inputs[i].node().copyAttributes(float_node)
+
+        model.apply(patch_float)
+        patch_float(model.encode_image)
+        patch_float(model.encode_text)
+
+        model.float()
+
+    return model, embed_dim, _transform(model.input_resolution.item())
+
+
+def tokenize(texts: Union[str, List[str]], context_length: int = 77, truncate: bool = False) -> torch.LongTensor:
+    """
+    Returns the tokenized representation of given input string(s)
+
+    Parameters
+    ----------
+    texts : Union[str, List[str]]
+        An input string or a list of input strings to tokenize
+
+    context_length : int
+        The context length to use; all CLIP models use 77 as the context length
+
+    truncate: bool
+        Whether to truncate the text in case its encoding is longer than the context length
+
+    Returns
+    -------
+    A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]
+    """
+    if isinstance(texts, str):
+        texts = [texts]
+
+    sot_token = _tokenizer.encoder["<|startoftext|>"]
+    eot_token = _tokenizer.encoder["<|endoftext|>"]
+    all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts]
+    result = torch.zeros(len(all_tokens), context_length, dtype=torch.long)
+
+    for i, tokens in enumerate(all_tokens):
+        if len(tokens) > context_length:
+            if truncate:
+                tokens = tokens[:context_length]
+                tokens[-1] = eot_token
+            else:
+                raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}")
+        result[i, :len(tokens)] = torch.tensor(tokens)
+
+    return result

ID-like-train-change-bg/clip/model.py

@@ -0,0 +1,438 @@
+from collections import OrderedDict
+from typing import Tuple, Union
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1):
+        super().__init__()
+
+        # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1
+        self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+        self.relu1 = nn.ReLU(inplace=True)
+
+        self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+        self.relu2 = nn.ReLU(inplace=True)
+
+        self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity()
+
+        self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+        self.relu3 = nn.ReLU(inplace=True)
+
+        self.downsample = None
+        self.stride = stride
+
+        if stride > 1 or inplanes != planes * Bottleneck.expansion:
+            # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1
+            self.downsample = nn.Sequential(OrderedDict([
+                ("-1", nn.AvgPool2d(stride)),
+                ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)),
+                ("1", nn.BatchNorm2d(planes * self.expansion))
+            ]))
+
+    def forward(self, x: torch.Tensor):
+        identity = x
+
+        out = self.relu1(self.bn1(self.conv1(x)))
+        out = self.relu2(self.bn2(self.conv2(out)))
+        out = self.avgpool(out)
+        out = self.bn3(self.conv3(out))
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity
+        out = self.relu3(out)
+        return out
+
+
+class AttentionPool2d(nn.Module):
+    def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None):
+        super().__init__()
+        self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5)
+        self.k_proj = nn.Linear(embed_dim, embed_dim)
+        self.q_proj = nn.Linear(embed_dim, embed_dim)
+        self.v_proj = nn.Linear(embed_dim, embed_dim)
+        self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim)
+        self.num_heads = num_heads
+
+    def forward(self, x):
+        x = x.flatten(start_dim=2).permute(2, 0, 1)  # NCHW -> (HW)NC
+        x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0)  # (HW+1)NC
+        x = x + self.positional_embedding[:, None, :].to(x.dtype)  # (HW+1)NC
+        x, _ = F.multi_head_attention_forward(
+            query=x[:1], key=x, value=x,
+            embed_dim_to_check=x.shape[-1],
+            num_heads=self.num_heads,
+            q_proj_weight=self.q_proj.weight,
+            k_proj_weight=self.k_proj.weight,
+            v_proj_weight=self.v_proj.weight,
+            in_proj_weight=None,
+            in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]),
+            bias_k=None,
+            bias_v=None,
+            add_zero_attn=False,
+            dropout_p=0,
+            out_proj_weight=self.c_proj.weight,
+            out_proj_bias=self.c_proj.bias,
+            use_separate_proj_weight=True,
+            training=self.training,
+            need_weights=False
+        )
+        return x.squeeze(0)
+
+
+class ModifiedResNet(nn.Module):
+    """
+    A ResNet class that is similar to torchvision's but contains the following changes:
+    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
+    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
+    - The final pooling layer is a QKV attention instead of an average pool
+    """
+
+    def __init__(self, layers, output_dim, heads, input_resolution=224, width=64):
+        super().__init__()
+        self.output_dim = output_dim
+        self.input_resolution = input_resolution
+
+        # the 3-layer stem
+        self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(width // 2)
+        self.relu1 = nn.ReLU(inplace=True)
+        self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(width // 2)
+        self.relu2 = nn.ReLU(inplace=True)
+        self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False)
+        self.bn3 = nn.BatchNorm2d(width)
+        self.relu3 = nn.ReLU(inplace=True)
+        self.avgpool = nn.AvgPool2d(2)
+
+        # residual layers
+        self._inplanes = width  # this is a *mutable* variable used during construction
+        self.layer1 = self._make_layer(width, layers[0])
+        self.layer2 = self._make_layer(width * 2, layers[1], stride=2)
+        self.layer3 = self._make_layer(width * 4, layers[2], stride=2)
+        self.layer4 = self._make_layer(width * 8, layers[3], stride=2)
+
+        embed_dim = width * 32  # the ResNet feature dimension
+        self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim)
+
+    def _make_layer(self, planes, blocks, stride=1):
+        layers = [Bottleneck(self._inplanes, planes, stride)]
+
+        self._inplanes = planes * Bottleneck.expansion
+        for _ in range(1, blocks):
+            layers.append(Bottleneck(self._inplanes, planes))
+
+        return nn.Sequential(*layers)
+
+    def forward(self, x):
+        def stem(x):
+            x = self.relu1(self.bn1(self.conv1(x)))
+            x = self.relu2(self.bn2(self.conv2(x)))
+            x = self.relu3(self.bn3(self.conv3(x)))
+            x = self.avgpool(x)
+            return x
+
+        x = x.type(self.conv1.weight.dtype)
+        x = stem(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.attnpool(x)
+
+        return x
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class QuickGELU(nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x * torch.sigmoid(1.702 * x)
+
+
+class ResidualAttentionBlock(nn.Module):
+    def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None):
+        super().__init__()
+
+        self.attn = nn.MultiheadAttention(d_model, n_head)
+        self.ln_1 = LayerNorm(d_model)
+        self.mlp = nn.Sequential(OrderedDict([
+            ("c_fc", nn.Linear(d_model, d_model * 4)),
+            ("gelu", QuickGELU()),
+            ("c_proj", nn.Linear(d_model * 4, d_model))
+        ]))
+        self.ln_2 = LayerNorm(d_model)
+        self.attn_mask = attn_mask
+
+    def attention(self, x: torch.Tensor):
+        self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None
+        return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0]
+
+    def forward(self, x: torch.Tensor):
+        x = x + self.attention(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+
+class Transformer(nn.Module):
+    def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None):
+        super().__init__()
+        self.width = width
+        self.layers = layers
+        self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)])
+
+    def forward(self, x: torch.Tensor):
+        return self.resblocks(x)
+
+
+class VisionTransformer(nn.Module):
+    def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):
+        super().__init__()
+        self.input_resolution = input_resolution
+        self.output_dim = output_dim
+        self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)
+
+        scale = width ** -0.5
+        self.class_embedding = nn.Parameter(scale * torch.randn(width))
+        self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
+        self.ln_pre = LayerNorm(width)
+
+        self.transformer = Transformer(width, layers, heads)
+
+        self.ln_post = LayerNorm(width)
+        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))
+
+    def forward(self, x: torch.Tensor):
+        x = self.conv1(x)  # shape = [*, width, grid, grid]
+        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
+        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
+        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
+        x = x + self.positional_embedding.to(x.dtype)
+        x = self.ln_pre(x)
+
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+
+        x = self.ln_post(x[:, 0, :])
+
+        if self.proj is not None:
+            x = x @ self.proj
+
+        return x
+
+
+class CLIP(nn.Module):
+    def __init__(self,
+                 embed_dim: int,
+                 # vision
+                 image_resolution: int,
+                 vision_layers: Union[Tuple[int, int, int, int], int],
+                 vision_width: int,
+                 vision_patch_size: int,
+                 # text
+                 context_length: int,
+                 vocab_size: int,
+                 transformer_width: int,
+                 transformer_heads: int,
+                 transformer_layers: int
+                 ):
+        super().__init__()
+
+        self.context_length = context_length
+
+        if isinstance(vision_layers, (tuple, list)):
+            vision_heads = vision_width * 32 // 64
+            self.visual = ModifiedResNet(
+                layers=vision_layers,
+                output_dim=embed_dim,
+                heads=vision_heads,
+                input_resolution=image_resolution,
+                width=vision_width
+            )
+        else:
+            vision_heads = vision_width // 64
+            self.visual = VisionTransformer(
+                input_resolution=image_resolution,
+                patch_size=vision_patch_size,
+                width=vision_width,
+                layers=vision_layers,
+                heads=vision_heads,
+                output_dim=embed_dim
+            )
+
+        self.transformer = Transformer(
+            width=transformer_width,
+            layers=transformer_layers,
+            heads=transformer_heads,
+            attn_mask=self.build_attention_mask()
+        )
+
+        self.vocab_size = vocab_size
+        self.token_embedding = nn.Embedding(vocab_size, transformer_width)
+        self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width))
+        self.ln_final = LayerNorm(transformer_width)
+
+        self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim))
+        self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07))
+
+        self.initialize_parameters()
+
+    def initialize_parameters(self):
+        nn.init.normal_(self.token_embedding.weight, std=0.02)
+        nn.init.normal_(self.positional_embedding, std=0.01)
+
+        if isinstance(self.visual, ModifiedResNet):
+            if self.visual.attnpool is not None:
+                std = self.visual.attnpool.c_proj.in_features ** -0.5
+                nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std)
+                nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std)
+
+            for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]:
+                for name, param in resnet_block.named_parameters():
+                    if name.endswith("bn3.weight"):
+                        nn.init.zeros_(param)
+
+        proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5)
+        attn_std = self.transformer.width ** -0.5
+        fc_std = (2 * self.transformer.width) ** -0.5
+        for block in self.transformer.resblocks:
+            nn.init.normal_(block.attn.in_proj_weight, std=attn_std)
+            nn.init.normal_(block.attn.out_proj.weight, std=proj_std)
+            nn.init.normal_(block.mlp.c_fc.weight, std=fc_std)
+            nn.init.normal_(block.mlp.c_proj.weight, std=proj_std)
+
+        if self.text_projection is not None:
+            nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5)
+
+    def build_attention_mask(self):
+        # lazily create causal attention mask, with full attention between the vision tokens
+        # pytorch uses additive attention mask; fill with -inf
+        mask = torch.empty(self.context_length, self.context_length)
+        mask.fill_(float("-inf"))
+        mask.triu_(1)  # zero out the lower diagonal
+        return mask
+
+    @property
+    def dtype(self):
+        return self.visual.conv1.weight.dtype
+
+    def encode_image(self, image):
+        return self.visual(image.type(self.dtype))
+
+    def encode_text(self, text):
+        x = self.token_embedding(text).type(self.dtype)  # [batch_size, n_ctx, d_model]
+
+        x = x + self.positional_embedding.type(self.dtype)
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        x = self.ln_final(x).type(self.dtype)
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection
+
+        return x
+
+    def forward(self, image, text):
+        image_features = self.encode_image(image)
+        text_features = self.encode_text(text)
+
+        # normalized features
+        image_features = image_features / image_features.norm(dim=1, keepdim=True)
+        text_features = text_features / text_features.norm(dim=1, keepdim=True)
+
+        # cosine similarity as logits
+        logit_scale = self.logit_scale.exp()
+        logits_per_image = logit_scale * image_features @ text_features.t()
+        logits_per_text = logits_per_image.t()
+
+        # shape = [global_batch_size, global_batch_size]
+        return logits_per_image, logits_per_text
+
+
+def convert_weights(model: nn.Module):
+    """Convert applicable model parameters to fp16"""
+
+    def _convert_weights_to_fp16(l):
+        if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)):
+            l.weight.data = l.weight.data.half()
+            if l.bias is not None:
+                l.bias.data = l.bias.data.half()
+
+        if isinstance(l, nn.MultiheadAttention):
+            for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]:
+                tensor = getattr(l, attr)
+                if tensor is not None:
+                    tensor.data = tensor.data.half()
+
+        for name in ["text_projection", "proj"]:
+            if hasattr(l, name):
+                attr = getattr(l, name)
+                if attr is not None:
+                    attr.data = attr.data.half()
+
+    model.apply(_convert_weights_to_fp16)
+
+
+def build_model(state_dict: dict):
+    vit = "visual.proj" in state_dict
+
+    if vit:
+        vision_width = state_dict["visual.conv1.weight"].shape[0]
+        vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")])
+        vision_patch_size = state_dict["visual.conv1.weight"].shape[-1]
+        grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5)
+        image_resolution = vision_patch_size * grid_size
+    else:
+        counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]]
+        vision_layers = tuple(counts)
+        vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0]
+        output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5)
+        vision_patch_size = None
+        assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0]
+        image_resolution = output_width * 32
+
+    embed_dim = state_dict["text_projection"].shape[1]
+    context_length = state_dict["positional_embedding"].shape[0]
+    vocab_size = state_dict["token_embedding.weight"].shape[0]
+    transformer_width = state_dict["ln_final.weight"].shape[0]
+    transformer_heads = transformer_width // 64
+    transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith("transformer.resblocks")))
+
+    model = CLIP(
+        embed_dim,
+        image_resolution, vision_layers, vision_width, vision_patch_size,
+        context_length, vocab_size, transformer_width, transformer_heads, transformer_layers
+    )
+
+    for key in ["input_resolution", "context_length", "vocab_size"]:
+        if key in state_dict:
+            del state_dict[key]
+
+    # convert_weights(model)
+    model.load_state_dict(state_dict)
+    del state_dict
+    torch.cuda.empty_cache()
+    return model.eval()

ID-like-train-change-bg/clip/simple_tokenizer.py

@@ -0,0 +1,132 @@
+import gzip
+import html
+import os
+from functools import lru_cache
+
+import ftfy
+import regex as re
+
+
+@lru_cache()
+def default_bpe():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz")
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8+n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def basic_clean(text):
+    text = ftfy.fix_text(text)
+    text = html.unescape(html.unescape(text))
+    return text.strip()
+
+
+def whitespace_clean(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = text.strip()
+    return text
+
+
+class SimpleTokenizer(object):
+    def __init__(self, bpe_path: str = default_bpe()):
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        merges = gzip.open(bpe_path).read().decode("utf-8").split('\n')
+        merges = merges[1:49152-256-2+1]
+        merges = [tuple(merge.split()) for merge in merges]
+        vocab = list(bytes_to_unicode().values())
+        vocab = vocab + [v+'</w>' for v in vocab]
+        for merge in merges:
+            vocab.append(''.join(merge))
+        vocab.extend(['<|startoftext|>', '<|endoftext|>'])
+        self.encoder = dict(zip(vocab, range(len(vocab))))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.bpe_ranks = dict(zip(merges, range(len(merges))))
+        self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'}
+        self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE)
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token[:-1]) + ( token[-1] + '</w>',)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token+'</w>'
+
+        while True:
+            bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word)-1 and word[i+1] == second:
+                    new_word.append(first+second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def encode(self, text):
+        bpe_tokens = []
+        text = whitespace_clean(basic_clean(text)).lower()
+        for token in re.findall(self.pat, text):
+            token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8'))
+            bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('</w>', ' ')
+        return text

ID-like-train-change-bg/config.py

@@ -0,0 +1,248 @@
+data_path = 'your datasets path'
+
+DOWNLOAD_ROOT = '/home/zhourixin/OOD_Folder/CODE/other_methods/ID-like/clip checkpoint path'
+
+CLIP_ckpt = 'ViT-B/16'
+
+n_ctx = 16
+ctx_init = None
+ctx_position = 'end'
+learned_cls = False
+
+n_ex_ctx = 16
+ex_ctx_init = None
+ex_ctx_position = 'end'
+ex_learned_cls = True
+
+data_info = {
+    'ImageNet': {
+        'n_cls': 1000,
+        'labels': ['tench', 'goldfish', 'great white shark', 'tiger shark', 'hammerhead shark', 'electric ray',
+                   'stingray', 'rooster', 'hen', 'ostrich', 'brambling', 'goldfinch', 'house finch', 'junco',
+                   'indigo bunting', 'American robin', 'bulbul', 'jay', 'magpie', 'chickadee', 'American dipper',
+                   'kite (bird of prey)', 'bald eagle', 'vulture', 'great grey owl', 'fire salamander', 'smooth newt',
+                   'eft', 'spotted salamander', 'axolotl', 'American bullfrog', 'tree frog', 'tailed frog',
+                   'loggerhead sea turtle', 'leatherback sea turtle', 'mud turtle', 'terrapin', 'box turtle',
+                   'banded gecko', 'green iguana', 'Carolina anole', 'desert grassland whiptail lizard', 'agama',
+                   'frilled-necked lizard', 'alligator lizard', 'Gila monster', 'European green lizard', 'chameleon',
+                   'Komodo dragon', 'Nile crocodile', 'American alligator', 'triceratops', 'worm snake',
+                   'ring-necked snake', 'eastern hog-nosed snake', 'smooth green snake', 'kingsnake', 'garter snake',
+                   'water snake', 'vine snake', 'night snake', 'boa constrictor', 'African rock python', 'Indian cobra',
+                   'green mamba', 'sea snake', 'Saharan horned viper', 'eastern diamondback rattlesnake',
+                   'sidewinder rattlesnake', 'trilobite', 'harvestman', 'scorpion', 'yellow garden spider',
+                   'barn spider', 'European garden spider', 'southern black widow', 'tarantula', 'wolf spider', 'tick',
+                   'centipede', 'black grouse', 'ptarmigan', 'ruffed grouse', 'prairie grouse', 'peafowl', 'quail',
+                   'partridge', 'african grey parrot', 'macaw', 'sulphur-crested cockatoo', 'lorikeet', 'coucal',
+                   'bee eater', 'hornbill', 'hummingbird', 'jacamar', 'toucan', 'duck', 'red-breasted merganser',
+                   'goose', 'black swan', 'tusker', 'echidna', 'platypus', 'wallaby', 'koala', 'wombat', 'jellyfish',
+                   'sea anemone', 'brain coral', 'flatworm', 'nematode', 'conch', 'snail', 'slug', 'sea slug', 'chiton',
+                   'chambered nautilus', 'Dungeness crab', 'rock crab', 'fiddler crab', 'red king crab',
+                   'American lobster', 'spiny lobster', 'crayfish', 'hermit crab', 'isopod', 'white stork',
+                   'black stork', 'spoonbill', 'flamingo', 'little blue heron', 'great egret', 'bittern bird',
+                   'crane bird', 'limpkin', 'common gallinule', 'American coot', 'bustard', 'ruddy turnstone', 'dunlin',
+                   'common redshank', 'dowitcher', 'oystercatcher', 'pelican', 'king penguin', 'albatross',
+                   'grey whale', 'killer whale', 'dugong', 'sea lion', 'Chihuahua', 'Japanese Chin', 'Maltese',
+                   'Pekingese', 'Shih Tzu', 'King Charles Spaniel', 'Papillon', 'toy terrier', 'Rhodesian Ridgeback',
+                   'Afghan Hound', 'Basset Hound', 'Beagle', 'Bloodhound', 'Bluetick Coonhound',
+                   'Black and Tan Coonhound', 'Treeing Walker Coonhound', 'English foxhound', 'Redbone Coonhound',
+                   'borzoi', 'Irish Wolfhound', 'Italian Greyhound', 'Whippet', 'Ibizan Hound', 'Norwegian Elkhound',
+                   'Otterhound', 'Saluki', 'Scottish Deerhound', 'Weimaraner', 'Staffordshire Bull Terrier',
+                   'American Staffordshire Terrier', 'Bedlington Terrier', 'Border Terrier', 'Kerry Blue Terrier',
+                   'Irish Terrier', 'Norfolk Terrier', 'Norwich Terrier', 'Yorkshire Terrier', 'Wire Fox Terrier',
+                   'Lakeland Terrier', 'Sealyham Terrier', 'Airedale Terrier', 'Cairn Terrier', 'Australian Terrier',
+                   'Dandie Dinmont Terrier', 'Boston Terrier', 'Miniature Schnauzer', 'Giant Schnauzer',
+                   'Standard Schnauzer', 'Scottish Terrier', 'Tibetan Terrier', 'Australian Silky Terrier',
+                   'Soft-coated Wheaten Terrier', 'West Highland White Terrier', 'Lhasa Apso', 'Flat-Coated Retriever',
+                   'Curly-coated Retriever', 'Golden Retriever', 'Labrador Retriever', 'Chesapeake Bay Retriever',
+                   'German Shorthaired Pointer', 'Vizsla', 'English Setter', 'Irish Setter', 'Gordon Setter',
+                   'Brittany dog', 'Clumber Spaniel', 'English Springer Spaniel', 'Welsh Springer Spaniel',
+                   'Cocker Spaniel', 'Sussex Spaniel', 'Irish Water Spaniel', 'Kuvasz', 'Schipperke', 'Groenendael dog',
+                   'Malinois', 'Briard', 'Australian Kelpie', 'Komondor', 'Old English Sheepdog', 'Shetland Sheepdog',
+                   'collie', 'Border Collie', 'Bouvier des Flandres dog', 'Rottweiler', 'German Shepherd Dog',
+                   'Dobermann', 'Miniature Pinscher', 'Greater Swiss Mountain Dog', 'Bernese Mountain Dog',
+                   'Appenzeller Sennenhund', 'Entlebucher Sennenhund', 'Boxer', 'Bullmastiff', 'Tibetan Mastiff',
+                   'French Bulldog', 'Great Dane', 'St. Bernard', 'husky', 'Alaskan Malamute', 'Siberian Husky',
+                   'Dalmatian', 'Affenpinscher', 'Basenji', 'pug', 'Leonberger', 'Newfoundland dog',
+                   'Great Pyrenees dog', 'Samoyed', 'Pomeranian', 'Chow Chow', 'Keeshond', 'brussels griffon',
+                   'Pembroke Welsh Corgi', 'Cardigan Welsh Corgi', 'Toy Poodle', 'Miniature Poodle', 'Standard Poodle',
+                   'Mexican hairless dog (xoloitzcuintli)', 'grey wolf', 'Alaskan tundra wolf',
+                   'red wolf or maned wolf', 'coyote', 'dingo', 'dhole', 'African wild dog', 'hyena', 'red fox',
+                   'kit fox', 'Arctic fox', 'grey fox', 'tabby cat', 'tiger cat', 'Persian cat', 'Siamese cat',
+                   'Egyptian Mau', 'cougar', 'lynx', 'leopard', 'snow leopard', 'jaguar', 'lion', 'tiger', 'cheetah',
+                   'brown bear', 'American black bear', 'polar bear', 'sloth bear', 'mongoose', 'meerkat',
+                   'tiger beetle', 'ladybug', 'ground beetle', 'longhorn beetle', 'leaf beetle', 'dung beetle',
+                   'rhinoceros beetle', 'weevil', 'fly', 'bee', 'ant', 'grasshopper', 'cricket insect', 'stick insect',
+                   'cockroach', 'praying mantis', 'cicada', 'leafhopper', 'lacewing', 'dragonfly', 'damselfly',
+                   'red admiral butterfly', 'ringlet butterfly', 'monarch butterfly', 'small white butterfly',
+                   'sulphur butterfly', 'gossamer-winged butterfly', 'starfish', 'sea urchin', 'sea cucumber',
+                   'cottontail rabbit', 'hare', 'Angora rabbit', 'hamster', 'porcupine', 'fox squirrel', 'marmot',
+                   'beaver', 'guinea pig', 'common sorrel horse', 'zebra', 'pig', 'wild boar', 'warthog',
+                   'hippopotamus', 'ox', 'water buffalo', 'bison', 'ram (adult male sheep)', 'bighorn sheep',
+                   'Alpine ibex', 'hartebeest', 'impala (antelope)', 'gazelle', 'arabian camel', 'llama', 'weasel',
+                   'mink', 'European polecat', 'black-footed ferret', 'otter', 'skunk', 'badger', 'armadillo',
+                   'three-toed sloth', 'orangutan', 'gorilla', 'chimpanzee', 'gibbon', 'siamang', 'guenon',
+                   'patas monkey', 'baboon', 'macaque', 'langur', 'black-and-white colobus', 'proboscis monkey',
+                   'marmoset', 'white-headed capuchin', 'howler monkey', 'titi monkey', "Geoffroy's spider monkey",
+                   'common squirrel monkey', 'ring-tailed lemur', 'indri', 'Asian elephant', 'African bush elephant',
+                   'red panda', 'giant panda', 'snoek fish', 'eel', 'silver salmon', 'rock beauty fish', 'clownfish',
+                   'sturgeon', 'gar fish', 'lionfish', 'pufferfish', 'abacus', 'abaya', 'academic gown', 'accordion',
+                   'acoustic guitar', 'aircraft carrier', 'airliner', 'airship', 'altar', 'ambulance',
+                   'amphibious vehicle', 'analog clock', 'apiary', 'apron', 'trash can', 'assault rifle', 'backpack',
+                   'bakery', 'balance beam', 'balloon', 'ballpoint pen', 'Band-Aid', 'banjo', 'baluster / handrail',
+                   'barbell', 'barber chair', 'barbershop', 'barn', 'barometer', 'barrel', 'wheelbarrow', 'baseball',
+                   'basketball', 'bassinet', 'bassoon', 'swimming cap', 'bath towel', 'bathtub', 'station wagon',
+                   'lighthouse', 'beaker', 'military hat (bearskin or shako)', 'beer bottle', 'beer glass',
+                   'bell tower', 'baby bib', 'tandem bicycle', 'bikini', 'ring binder', 'binoculars', 'birdhouse',
+                   'boathouse', 'bobsleigh', 'bolo tie', 'poke bonnet', 'bookcase', 'bookstore', 'bottle cap',
+                   'hunting bow', 'bow tie', 'brass memorial plaque', 'bra', 'breakwater', 'breastplate', 'broom',
+                   'bucket', 'buckle', 'bulletproof vest', 'high-speed train', 'butcher shop', 'taxicab', 'cauldron',
+                   'candle', 'cannon', 'canoe', 'can opener', 'cardigan', 'car mirror', 'carousel', 'tool kit',
+                   'cardboard box / carton', 'car wheel', 'automated teller machine', 'cassette', 'cassette player',
+                   'castle', 'catamaran', 'CD player', 'cello', 'mobile phone', 'chain', 'chain-link fence',
+                   'chain mail', 'chainsaw', 'storage chest', 'chiffonier', 'bell or wind chime', 'china cabinet',
+                   'Christmas stocking', 'church', 'movie theater', 'cleaver', 'cliff dwelling', 'cloak', 'clogs',
+                   'cocktail shaker', 'coffee mug', 'coffeemaker', 'spiral or coil', 'combination lock',
+                   'computer keyboard', 'candy store', 'container ship', 'convertible', 'corkscrew', 'cornet',
+                   'cowboy boot', 'cowboy hat', 'cradle', 'construction crane', 'crash helmet', 'crate', 'infant bed',
+                   'Crock Pot', 'croquet ball', 'crutch', 'cuirass', 'dam', 'desk', 'desktop computer',
+                   'rotary dial telephone', 'diaper', 'digital clock', 'digital watch', 'dining table', 'dishcloth',
+                   'dishwasher', 'disc brake', 'dock', 'dog sled', 'dome', 'doormat', 'drilling rig', 'drum',
+                   'drumstick', 'dumbbell', 'Dutch oven', 'electric fan', 'electric guitar', 'electric locomotive',
+                   'entertainment center', 'envelope', 'espresso machine', 'face powder', 'feather boa',
+                   'filing cabinet', 'fireboat', 'fire truck', 'fire screen', 'flagpole', 'flute', 'folding chair',
+                   'football helmet', 'forklift', 'fountain', 'fountain pen', 'four-poster bed', 'freight car',
+                   'French horn', 'frying pan', 'fur coat', 'garbage truck', 'gas mask or respirator', 'gas pump',
+                   'goblet', 'go-kart', 'golf ball', 'golf cart', 'gondola', 'gong', 'gown', 'grand piano',
+                   'greenhouse', 'radiator grille', 'grocery store', 'guillotine', 'hair clip', 'hair spray',
+                   'half-track', 'hammer', 'hamper', 'hair dryer', 'hand-held computer', 'handkerchief',
+                   'hard disk drive', 'harmonica', 'harp', 'combine harvester', 'hatchet', 'holster', 'home theater',
+                   'honeycomb', 'hook', 'hoop skirt', 'gymnastic horizontal bar', 'horse-drawn vehicle', 'hourglass',
+                   'iPod', 'clothes iron', 'carved pumpkin', 'jeans', 'jeep', 'T-shirt', 'jigsaw puzzle', 'rickshaw',
+                   'joystick', 'kimono', 'knee pad', 'knot', 'lab coat', 'ladle', 'lampshade', 'laptop computer',
+                   'lawn mower', 'lens cap', 'letter opener', 'library', 'lifeboat', 'lighter', 'limousine',
+                   'ocean liner', 'lipstick', 'slip-on shoe', 'lotion', 'music speaker', 'loupe magnifying glass',
+                   'sawmill', 'magnetic compass', 'messenger bag', 'mailbox', 'maillot', 'one-piece bathing suit',
+                   'manhole cover', 'maraca', 'marimba', 'mask', 'matchstick', 'maypole', 'maze', 'measuring cup',
+                   'medicine cabinet', 'megalith', 'microphone', 'microwave oven', 'military uniform', 'milk can',
+                   'minibus', 'miniskirt', 'minivan', 'missile', 'mitten', 'mixing bowl', 'mobile home', 'ford model t',
+                   'modem', 'monastery', 'monitor', 'moped', 'mortar and pestle', 'graduation cap', 'mosque',
+                   'mosquito net', 'vespa', 'mountain bike', 'tent', 'computer mouse', 'mousetrap', 'moving van',
+                   'muzzle', 'metal nail', 'neck brace', 'necklace', 'baby pacifier', 'notebook computer', 'obelisk',
+                   'oboe', 'ocarina', 'odometer', 'oil filter', 'pipe organ', 'oscilloscope', 'overskirt',
+                   'bullock cart', 'oxygen mask', 'product packet / packaging', 'paddle', 'paddle wheel', 'padlock',
+                   'paintbrush', 'pajamas', 'palace', 'pan flute', 'paper towel', 'parachute', 'parallel bars',
+                   'park bench', 'parking meter', 'railroad car', 'patio', 'payphone', 'pedestal', 'pencil case',
+                   'pencil sharpener', 'perfume', 'Petri dish', 'photocopier', 'plectrum', 'Pickelhaube',
+                   'picket fence', 'pickup truck', 'pier', 'piggy bank', 'pill bottle', 'pillow', 'ping-pong ball',
+                   'pinwheel', 'pirate ship', 'drink pitcher', 'block plane', 'planetarium', 'plastic bag',
+                   'plate rack', 'farm plow', 'plunger', 'Polaroid camera', 'pole', 'police van', 'poncho',
+                   'pool table', 'soda bottle', 'plant pot', "potter's wheel", 'power drill', 'prayer rug', 'printer',
+                   'prison', 'projectile', 'projector', 'hockey puck', 'punching bag', 'purse', 'quill', 'quilt',
+                   'race car', 'racket', 'radiator', 'radio', 'radio telescope', 'rain barrel', 'recreational vehicle',
+                   'fishing casting reel', 'reflex camera', 'refrigerator', 'remote control', 'restaurant', 'revolver',
+                   'rifle', 'rocking chair', 'rotisserie', 'eraser', 'rugby ball', 'ruler measuring stick', 'sneaker',
+                   'safe', 'safety pin', 'salt shaker', 'sandal', 'sarong', 'saxophone', 'scabbard', 'weighing scale',
+                   'school bus', 'schooner', 'scoreboard', 'CRT monitor', 'screw', 'screwdriver', 'seat belt',
+                   'sewing machine', 'shield', 'shoe store', 'shoji screen / room divider', 'shopping basket',
+                   'shopping cart', 'shovel', 'shower cap', 'shower curtain', 'ski', 'balaclava ski mask',
+                   'sleeping bag', 'slide rule', 'sliding door', 'slot machine', 'snorkel', 'snowmobile', 'snowplow',
+                   'soap dispenser', 'soccer ball', 'sock', 'solar thermal collector', 'sombrero', 'soup bowl',
+                   'keyboard space bar', 'space heater', 'space shuttle', 'spatula', 'motorboat', 'spider web',
+                   'spindle', 'sports car', 'spotlight', 'stage', 'steam locomotive', 'through arch bridge',
+                   'steel drum', 'stethoscope', 'scarf', 'stone wall', 'stopwatch', 'stove', 'strainer', 'tram',
+                   'stretcher', 'couch', 'stupa', 'submarine', 'suit', 'sundial', 'sunglass', 'sunglasses', 'sunscreen',
+                   'suspension bridge', 'mop', 'sweatshirt', 'swim trunks / shorts', 'swing', 'electrical switch',
+                   'syringe', 'table lamp', 'tank', 'tape player', 'teapot', 'teddy bear', 'television', 'tennis ball',
+                   'thatched roof', 'front curtain', 'thimble', 'threshing machine', 'throne', 'tile roof', 'toaster',
+                   'tobacco shop', 'toilet seat', 'torch', 'totem pole', 'tow truck', 'toy store', 'tractor',
+                   'semi-trailer truck', 'tray', 'trench coat', 'tricycle', 'trimaran', 'tripod', 'triumphal arch',
+                   'trolleybus', 'trombone', 'hot tub', 'turnstile', 'typewriter keyboard', 'umbrella', 'unicycle',
+                   'upright piano', 'vacuum cleaner', 'vase', 'vaulted or arched ceiling', 'velvet fabric',
+                   'vending machine', 'vestment', 'viaduct', 'violin', 'volleyball', 'waffle iron', 'wall clock',
+                   'wallet', 'wardrobe', 'military aircraft', 'sink', 'washing machine', 'water bottle', 'water jug',
+                   'water tower', 'whiskey jug', 'whistle', 'hair wig', 'window screen', 'window shade', 'Windsor tie',
+                   'wine bottle', 'airplane wing', 'wok', 'wooden spoon', 'wool', 'split-rail fence', 'shipwreck',
+                   'sailboat', 'yurt', 'website', 'comic book', 'crossword', 'traffic or street sign', 'traffic light',
+                   'dust jacket', 'menu', 'plate', 'guacamole', 'consomme', 'hot pot', 'trifle', 'ice cream',
+                   'popsicle', 'baguette', 'bagel', 'pretzel', 'cheeseburger', 'hot dog', 'mashed potatoes', 'cabbage',
+                   'broccoli', 'cauliflower', 'zucchini', 'spaghetti squash', 'acorn squash', 'butternut squash',
+                   'cucumber', 'artichoke', 'bell pepper', 'cardoon', 'mushroom', 'Granny Smith apple', 'strawberry',
+                   'orange', 'lemon', 'fig', 'pineapple', 'banana', 'jackfruit', 'cherimoya (custard apple)',
+                   'pomegranate', 'hay', 'carbonara', 'chocolate syrup', 'dough', 'meatloaf', 'pizza', 'pot pie',
+                   'burrito', 'red wine', 'espresso', 'tea cup', 'eggnog', 'mountain', 'bubble', 'cliff', 'coral reef',
+                   'geyser', 'lakeshore', 'promontory', 'sandbar', 'beach', 'valley', 'volcano', 'baseball player',
+                   'bridegroom', 'scuba diver', 'rapeseed', 'daisy', "yellow lady's slipper", 'corn', 'acorn',
+                   'rose hip', 'horse chestnut seed', 'coral fungus', 'agaric', 'gyromitra', 'stinkhorn mushroom',
+                   'earth star fungus', 'hen of the woods mushroom', 'bolete', 'corn cob', 'toilet paper'],
+    },
+    'ImageNet100': {
+        'n_cls': 100,
+        'labels': ['stingray', 'ostrich', 'jay', 'American dipper', 'spotted salamander', 'alligator lizard',
+                   'Komodo dragon', 'wolf spider', 'african grey parrot', 'jacamar', 'red-breasted merganser', 'tusker',
+                   'jellyfish', 'brain coral', 'snail', 'white stork', 'dowitcher', 'albatross', 'Beagle', 'Otterhound',
+                   'Lakeland Terrier', 'Giant Schnauzer', 'Cocker Spaniel', 'Australian Kelpie', 'Miniature Pinscher',
+                   'Samoyed', 'Cardigan Welsh Corgi', 'Standard Poodle', 'Egyptian Mau', 'snow leopard', 'jaguar',
+                   'polar bear', 'cockroach', 'hare', 'orangutan', 'gibbon', 'guenon', 'black-and-white colobus',
+                   "Geoffroy's spider monkey", 'bath towel', 'bell tower', 'birdhouse', 'bookstore',
+                   'cardboard box / carton', 'chainsaw', 'chiffonier', 'cornet', 'cradle', 'crate', 'Crock Pot',
+                   'desktop computer', 'rotary dial telephone', 'dog sled', 'electric locomotive', 'flagpole',
+                   'four-poster bed', 'French horn', 'frying pan', 'fur coat', 'gas pump', 'gong', 'greenhouse', 'jeep',
+                   'ladle', 'lighter', 'one-piece bathing suit', 'marimba', 'ocarina', 'overskirt', 'palace',
+                   'paper towel', 'railroad car', 'pencil sharpener', 'Pickelhaube', 'pier', 'piggy bank', 'pool table',
+                   'power drill', 'race car', 'radio', 'rifle', 'sarong', 'schooner', 'sewing machine', 'sliding door',
+                   'sunglasses', 'swim trunks / shorts', 'syringe', 'front curtain', 'tow truck', 'trimaran',
+                   'wardrobe', 'water tower', 'shipwreck', 'crossword', 'ice cream', 'cabbage', 'promontory',
+                   'baseball player', 'hen of the woods mushroom'],
+    },
+    'ImageNet10': {
+        'n_cls': 10,
+        'labels': ['brambling', 'American bullfrog', 'Greater Swiss Mountain Dog', 'Siamese cat', 'common sorrel horse',
+                   'impala (antelope)', 'container ship', 'garbage truck', 'sports car', 'military aircraft'],
+    },
+    'ImageNet20': {
+        'n_cls': 20,
+        'labels': ['smooth newt', 'eft', 'spotted salamander', 'European green lizard', 'Nile crocodile', 'grey wolf',
+                   'Arctic fox', 'brown bear', 'starfish', 'zebra', 'balloon', 'high-speed train', 'canoe', 'missile',
+                   'moped', 'schooner', 'snowmobile', 'space shuttle', 'steam locomotive', 'tank'],
+    },
+    'car196': {
+        'n_cls': 196,
+        'labels': ['AM General Hummer SUV 2000', 'Acura RL Sedan 2012', 'Acura TL Sedan 2012', 'Acura TL Type-S 2008', 'Acura TSX Sedan 2012', 'Acura Integra Type R 2001', 'Acura ZDX Hatchback 2012', 'Aston Martin V8 Vantage Convertible 2012', 'Aston Martin V8 Vantage Coupe 2012', 'Aston Martin Virage Convertible 2012', 'Aston Martin Virage Coupe 2012', 'Audi RS 4 Convertible 2008', 'Audi A5 Coupe 2012', 'Audi TTS Coupe 2012', 'Audi R8 Coupe 2012', 'Audi V8 Sedan 1994', 'Audi 100 Sedan 1994', 'Audi 100 Wagon 1994', 'Audi TT Hatchback 2011', 'Audi S6 Sedan 2011', 'Audi S5 Convertible 2012', 'Audi S5 Coupe 2012', 'Audi S4 Sedan 2012', 'Audi S4 Sedan 2007', 'Audi TT RS Coupe 2012', 'BMW ActiveHybrid 5 Sedan 2012', 'BMW 1 Series Convertible 2012', 'BMW 1 Series Coupe 2012', 'BMW 3 Series Sedan 2012', 'BMW 3 Series Wagon 2012', 'BMW 6 Series Convertible 2007', 'BMW X5 SUV 2007', 'BMW X6 SUV 2012', 'BMW M3 Coupe 2012', 'BMW M5 Sedan 2010', 'BMW M6 Convertible 2010', 'BMW X3 SUV 2012', 'BMW Z4 Convertible 2012', 'Bentley Continental Supersports Conv. Convertible 2012', 'Bentley Arnage Sedan 2009', 'Bentley Mulsanne Sedan 2011', 'Bentley Continental GT Coupe 2012', 'Bentley Continental GT Coupe 2007', 'Bentley Continental Flying Spur Sedan 2007', 'Bugatti Veyron 16.4 Convertible 2009', 'Bugatti Veyron 16.4 Coupe 2009', 'Buick Regal GS 2012', 'Buick Rainier SUV 2007', 'Buick Verano Sedan 2012', 'Buick Enclave SUV 2012', 'Cadillac CTS-V Sedan 2012', 'Cadillac SRX SUV 2012', 'Cadillac Escalade EXT Crew Cab 2007', 'Chevrolet Silverado 1500 Hybrid Crew Cab 2012', 'Chevrolet Corvette Convertible 2012', 'Chevrolet Corvette ZR1 2012', 'Chevrolet Corvette Ron Fellows Edition Z06 2007', 'Chevrolet Traverse SUV 2012', 'Chevrolet Camaro Convertible 2012', 'Chevrolet HHR SS 2010', 'Chevrolet Impala Sedan 2007', 'Chevrolet Tahoe Hybrid SUV 2012', 'Chevrolet Sonic Sedan 2012', 'Chevrolet Express Cargo Van 2007', 'Chevrolet Avalanche Crew Cab 2012', 'Chevrolet Cobalt SS 2010', 'Chevrolet Malibu Hybrid Sedan 2010', 'Chevrolet TrailBlazer SS 2009', 'Chevrolet Silverado 2500HD Regular Cab 2012', 'Chevrolet Silverado 1500 Classic Extended Cab 2007', 'Chevrolet Express Van 2007', 'Chevrolet Monte Carlo Coupe 2007', 'Chevrolet Malibu Sedan 2007', 'Chevrolet Silverado 1500 Extended Cab 2012', 'Chevrolet Silverado 1500 Regular Cab 2012', 'Chrysler Aspen SUV 2009', 'Chrysler Sebring Convertible 2010', 'Chrysler Town and Country Minivan 2012', 'Chrysler 300 SRT-8 2010', 'Chrysler Crossfire Convertible 2008', 'Chrysler PT Cruiser Convertible 2008', 'Daewoo Nubira Wagon 2002', 'Dodge Caliber Wagon 2012', 'Dodge Caliber Wagon 2007', 'Dodge Caravan Minivan 1997', 'Dodge Ram Pickup 3500 Crew Cab 2010', 'Dodge Ram Pickup 3500 Quad Cab 2009', 'Dodge Sprinter Cargo Van 2009', 'Dodge Journey SUV 2012', 'Dodge Dakota Crew Cab 2010', 'Dodge Dakota Club Cab 2007', 'Dodge Magnum Wagon 2008', 'Dodge Challenger SRT8 2011', 'Dodge Durango SUV 2012', 'Dodge Durango SUV 2007', 'Dodge Charger Sedan 2012', 'Dodge Charger SRT-8 2009', 'Eagle Talon Hatchback 1998', 'FIAT 500 Abarth 2012', 'FIAT 500 Convertible 2012', 'Ferrari FF Coupe 2012', 'Ferrari California Convertible 2012', 'Ferrari 458 Italia Convertible 2012', 'Ferrari 458 Italia Coupe 2012', 'Fisker Karma Sedan 2012', 'Ford F-450 Super Duty Crew Cab 2012', 'Ford Mustang Convertible 2007', 'Ford Freestar Minivan 2007', 'Ford Expedition EL SUV 2009', 'Ford Edge SUV 2012', 'Ford Ranger SuperCab 2011', 'Ford GT Coupe 2006', 'Ford F-150 Regular Cab 2012', 'Ford F-150 Regular Cab 2007', 'Ford Focus Sedan 2007', 'Ford E-Series Wagon Van 2012', 'Ford Fiesta Sedan 2012', 'GMC Terrain SUV 2012', 'GMC Savana Van 2012', 'GMC Yukon Hybrid SUV 2012', 'GMC Acadia SUV 2012', 'GMC Canyon Extended Cab 2012', 'Geo Metro Convertible 1993', 'HUMMER H3T Crew Cab 2010', 'HUMMER H2 SUT Crew Cab 2009', 'Honda Odyssey Minivan 2012', 'Honda Odyssey Minivan 2007', 'Honda Accord Coupe 2012', 'Honda Accord Sedan 2012', 'Hyundai Veloster Hatchback 2012', 'Hyundai Santa Fe SUV 2012', 'Hyundai Tucson SUV 2012', 'Hyundai Veracruz SUV 2012', 'Hyundai Sonata Hybrid Sedan 2012', 'Hyundai Elantra Sedan 2007', 'Hyundai Accent Sedan 2012', 'Hyundai Genesis Sedan 2012', 'Hyundai Sonata Sedan 2012', 'Hyundai Elantra Touring Hatchback 2012', 'Hyundai Azera Sedan 2012', 'Infiniti G Coupe IPL 2012', 'Infiniti QX56 SUV 2011', 'Isuzu Ascender SUV 2008', 'Jaguar XK XKR 2012', 'Jeep Patriot SUV 2012', 'Jeep Wrangler SUV 2012', 'Jeep Liberty SUV 2012', 'Jeep Grand Cherokee SUV 2012', 'Jeep Compass SUV 2012', 'Lamborghini Reventon Coupe 2008', 'Lamborghini Aventador Coupe 2012', 'Lamborghini Gallardo LP 570-4 Superleggera 2012', 'Lamborghini Diablo Coupe 2001', 'Land Rover Range Rover SUV 2012', 'Land Rover LR2 SUV 2012', 'Lincoln Town Car Sedan 2011', 'MINI Cooper Roadster Convertible 2012', 'Maybach Landaulet Convertible 2012', 'Mazda Tribute SUV 2011', 'McLaren MP4-12C Coupe 2012', 'Mercedes-Benz 300-Class Convertible 1993', 'Mercedes-Benz C-Class Sedan 2012', 'Mercedes-Benz SL-Class Coupe 2009', 'Mercedes-Benz E-Class Sedan 2012', 'Mercedes-Benz S-Class Sedan 2012', 'Mercedes-Benz Sprinter Van 2012', 'Mitsubishi Lancer Sedan 2012', 'Nissan Leaf Hatchback 2012', 'Nissan NV Passenger Van 2012', 'Nissan Juke Hatchback 2012', 'Nissan 240SX Coupe 1998', 'Plymouth Neon Coupe 1999', 'Porsche Panamera Sedan 2012', 'Ram C/V Cargo Van Minivan 2012', 'Rolls-Royce Phantom Drophead Coupe Convertible 2012', 'Rolls-Royce Ghost Sedan 2012', 'Rolls-Royce Phantom Sedan 2012', 'Scion xD Hatchback 2012', 'Spyker C8 Convertible 2009', 'Spyker C8 Coupe 2009', 'Suzuki Aerio Sedan 2007', 'Suzuki Kizashi Sedan 2012', 'Suzuki SX4 Hatchback 2012', 'Suzuki SX4 Sedan 2012', 'Tesla Model S Sedan 2012', 'Toyota Sequoia SUV 2012', 'Toyota Camry Sedan 2012', 'Toyota Corolla Sedan 2012', 'Toyota 4Runner SUV 2012', 'Volkswagen Golf Hatchback 2012', 'Volkswagen Golf Hatchback 1991', 'Volkswagen Beetle Hatchback 2012', 'Volvo C30 Hatchback 2012', 'Volvo 240 Sedan 1993', 'Volvo XC90 SUV 2007', 'smart fortwo Convertible 2012'],
+    },
+    'food101': {
+        'n_cls': 101,
+        'labels': ['Apple pie', 'Baby back ribs', 'Baklava', 'Beef carpaccio', 'Beef tartare', 'Beet salad', 'Beignets', 'Bibimbap', 'Bread pudding', 'Breakfast burrito', 'Bruschetta', 'Caesar salad', 'Cannoli', 'Caprese salad', 'Carrot cake', 'Ceviche', 'Cheesecake', 'Cheese plate', 'Chicken curry', 'Chicken quesadilla', 'Chicken wings', 'Chocolate cake', 'Chocolate mousse', 'Churros', 'Clam chowder', 'Club sandwich', 'Crab cakes', 'Creme brulee', 'Croque madame', 'Cup cakes', 'Deviled eggs', 'Donuts', 'Dumplings', 'Edamame', 'Eggs benedict', 'Escargots', 'Falafel', 'Filet mignon', 'Fish and chips', 'Foie gras', 'French fries', 'French onion soup', 'French toast', 'Fried calamari', 'Fried rice', 'Frozen yogurt', 'Garlic bread', 'Gnocchi', 'Greek salad', 'Grilled cheese sandwich', 'Grilled salmon', 'Guacamole', 'Gyoza', 'Hamburger', 'Hot and sour soup', 'Hot dog', 'Huevos rancheros', 'Hummus', 'Ice cream', 'Lasagna', 'Lobster bisque', 'Lobster roll sandwich', 'Macaroni and cheese', 'Macarons', 'Miso soup', 'Mussels', 'Nachos', 'Omelette', 'Onion rings', 'Oysters', 'Pad thai', 'Paella', 'Pancakes', 'Panna cotta', 'Peking duck', 'Pho', 'Pizza', 'Pork chop', 'Poutine', 'Prime rib', 'Pulled pork sandwich', 'Ramen', 'Ravioli', 'Red velvet cake', 'Risotto', 'Samosa', 'Sashimi', 'Scallops', 'Seaweed salad', 'Shrimp and grits', 'Spaghetti bolognese', 'Spaghetti carbonara', 'Spring rolls', 'Steak', 'Strawberry shortcake', 'Sushi', 'Tacos', 'Takoyaki', 'Tiramisu', 'Tuna tartare', 'Waffles'],
+    },
+    'pet37': {
+        'n_cls': 37,
+        'labels': ['Abyssinian', 'American Bulldog', 'American Pit Bull Terrier', 'Basset Hound', 'Beagle', 'Bengal', 'Birman', 'Bombay', 'Boxer', 'British Shorthair', 'Chihuahua', 'Egyptian Mau', 'English Cocker Spaniel', 'English Setter', 'German Shorthaired', 'Great Pyrenees', 'Havanese', 'Japanese Chin', 'Keeshond', 'Leonberger', 'Maine Coon', 'Miniature Pinscher', 'Newfoundland', 'Persian', 'Pomeranian', 'Pug', 'Ragdoll', 'Russian Blue', 'Saint Bernard', 'Samoyed', 'Scottish Terrier', 'Shiba Inu', 'Siamese', 'Sphynx', 'Staffordshire Bull Terrier', 'Wheaten Terrier', 'Yorkshire Terrier'],
+    },
+    'bird200': {
+        'n_cls': 200,
+        'labels': ['Black footed Albatross', 'Laysan Albatross', 'Sooty Albatross', 'Groove billed Ani', 'Crested Auklet', 'Least Auklet', 'Parakeet Auklet', 'Rhinoceros Auklet', 'Brewer Blackbird', 'Red winged Blackbird', 'Rusty Blackbird', 'Yellow headed Blackbird', 'Bobolink', 'Indigo Bunting', 'Lazuli Bunting', 'Painted Bunting', 'Cardinal', 'Spotted Catbird', 'Gray Catbird', 'Yellow breasted Chat', 'Eastern Towhee', 'Chuck will Widow', 'Brandt Cormorant', 'Red faced Cormorant', 'Pelagic Cormorant', 'Bronzed Cowbird', 'Shiny Cowbird', 'Brown Creeper', 'American Crow', 'Fish Crow', 'Black billed Cuckoo', 'Mangrove Cuckoo', 'Yellow billed Cuckoo', 'Gray crowned Rosy Finch', 'Purple Finch', 'Northern Flicker', 'Acadian Flycatcher', 'Great Crested Flycatcher', 'Least Flycatcher', 'Olive sided Flycatcher', 'Scissor tailed Flycatcher', 'Vermilion Flycatcher', 'Yellow bellied Flycatcher', 'Frigatebird', 'Northern Fulmar', 'Gadwall', 'American Goldfinch', 'European Goldfinch', 'Boat tailed Grackle', 'Eared Grebe', 'Horned Grebe', 'Pied billed Grebe', 'Western Grebe', 'Blue Grosbeak', 'Evening Grosbeak', 'Pine Grosbeak', 'Rose breasted Grosbeak', 'Pigeon Guillemot', 'California Gull', 'Glaucous winged Gull', 'Heermann Gull', 'Herring Gull', 'Ivory Gull', 'Ring billed Gull', 'Slaty backed Gull', 'Western Gull', 'Anna Hummingbird', 'Ruby throated Hummingbird', 'Rufous Hummingbird', 'Green Violetear', 'Long tailed Jaeger', 'Pomarine Jaeger', 'Blue Jay', 'Florida Jay', 'Green Jay', 'Dark eyed Junco', 'Tropical Kingbird', 'Gray Kingbird', 'Belted Kingfisher', 'Green Kingfisher', 'Pied Kingfisher', 'Ringed Kingfisher', 'White breasted Kingfisher', 'Red legged Kittiwake', 'Horned Lark', 'Pacific Loon', 'Mallard', 'Western Meadowlark', 'Hooded Merganser', 'Red breasted Merganser', 'Mockingbird', 'Nighthawk', 'Clark Nutcracker', 'White breasted Nuthatch', 'Baltimore Oriole', 'Hooded Oriole', 'Orchard Oriole', 'Scott Oriole', 'Ovenbird', 'Brown Pelican', 'White Pelican', 'Western Wood Pewee', 'Sayornis', 'American Pipit', 'Whip poor Will', 'Horned Puffin', 'Common Raven', 'White necked Raven', 'American Redstart', 'Geococcyx', 'Loggerhead Shrike', 'Great Grey Shrike', 'Baird Sparrow', 'Black throated Sparrow', 'Brewer Sparrow', 'Chipping Sparrow', 'Clay colored Sparrow', 'House Sparrow', 'Field Sparrow', 'Fox Sparrow', 'Grasshopper Sparrow', 'Harris Sparrow', 'Henslow Sparrow', 'Le Conte Sparrow', 'Lincoln Sparrow', 'Nelson Sharp tailed Sparrow', 'Savannah Sparrow', 'Seaside Sparrow', 'Song Sparrow', 'Tree Sparrow', 'Vesper Sparrow', 'White crowned Sparrow', 'White throated Sparrow', 'Cape Glossy Starling', 'Bank Swallow', 'Barn Swallow', 'Cliff Swallow', 'Tree Swallow', 'Scarlet Tanager', 'Summer Tanager', 'Artic Tern', 'Black Tern', 'Caspian Tern', 'Common Tern', 'Elegant Tern', 'Forsters Tern', 'Least Tern', 'Green tailed Towhee', 'Brown Thrasher', 'Sage Thrasher', 'Black capped Vireo', 'Blue headed Vireo', 'Philadelphia Vireo', 'Red eyed Vireo', 'Warbling Vireo', 'White eyed Vireo', 'Yellow throated Vireo', 'Bay breasted Warbler', 'Black and white Warbler', 'Black throated Blue Warbler', 'Blue winged Warbler', 'Canada Warbler', 'Cape May Warbler', 'Cerulean Warbler', 'Chestnut sided Warbler', 'Golden winged Warbler', 'Hooded Warbler', 'Kentucky Warbler', 'Magnolia Warbler', 'Mourning Warbler', 'Myrtle Warbler', 'Nashville Warbler', 'Orange crowned Warbler', 'Palm Warbler', 'Pine Warbler', 'Prairie Warbler', 'Prothonotary Warbler', 'Swainson Warbler', 'Tennessee Warbler', 'Wilson Warbler', 'Worm eating Warbler', 'Yellow Warbler', 'Northern Waterthrush', 'Louisiana Waterthrush', 'Bohemian Waxwing', 'Cedar Waxwing', 'American Three toed Woodpecker', 'Pileated Woodpecker', 'Red bellied Woodpecker', 'Red cockaded Woodpecker', 'Red headed Woodpecker', 'Downy Woodpecker', 'Bewick Wren', 'Cactus Wren', 'Carolina Wren', 'House Wren', 'Marsh Wren', 'Rock Wren', 'Winter Wren', 'Common Yellowthroat'],
+    },
+    'cifar10': {
+        'n_cls': 10,
+        'labels': ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'],
+    },
+    'cifar100': {
+        'n_cls': 100,
+        'labels': ['apple', 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle', 'bicycle', 'bottle',
+                   'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel', 'can', 'castle', 'caterpillar', 'cattle',
+                   'chair', 'chimpanzee', 'clock', 'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur',
+                   'dolphin', 'elephant', 'flatfish', 'forest', 'fox', 'girl', 'hamster', 'house', 'kangaroo', 'keyboard',
+                   'lamp', 'lawn_mower', 'leopard', 'lion', 'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain',
+                   'mouse', 'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree', 'pear', 'pickup_truck', 'pine_tree',
+                   'plain', 'plate', 'poppy', 'porcupine', 'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket',
+                   'rose', 'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail', 'snake', 'spider',
+                   'squirrel', 'streetcar', 'sunflower', 'sweet_pepper', 'table', 'tank', 'telephone', 'television', 'tiger', 'tractor',
+                   'train', 'trout', 'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf', 'woman', 'worm'],
+    },
+    'bronze2NotLine': {
+        'n_cls': 11,
+        'labels': ['bronze ware of the early Shang age', 'bronze ware of the late Shang age', 
+        'bronze ware of the early Western Zhou age', 'bronze ware of the mid Western Zhou age', 'bronze ware of the late Western Zhou age', 
+        'bronze ware of the early Spring and Autumn age', 'bronze ware of the mid Spring and Autumn age', 'bronze ware of the late Spring and Autumn age', 
+        'bronze ware of the early Warring States age', 'bronze ware of the mid Warring States age', 'bronze ware of the late Warring States age'],
+    },
+}

ID-like-train-change-bg/dataloaders/__init__.py

@@ -0,0 +1,4 @@
+from .pet37 import OxfordIIITPet
+from .car196 import StanfordCars
+from .food101 import Food101
+from .bird200 import Cub2011

ID-like-train-change-bg/dataloaders/bird200.py

@@ -0,0 +1,64 @@
+import os
+import pandas as pd
+from torchvision.datasets.folder import default_loader
+from torchvision.datasets.utils import download_url
+from torch.utils.data import Dataset
+import torch
+
+class Cub2011(Dataset):
+    base_folder = 'CUB_200_2011/images'
+
+    def __init__(self, root, train=True, transform=None, loader=default_loader):
+        self.root = os.path.expanduser(root)
+        self.transform = transform
+        self.loader = default_loader
+        self.train = train
+
+        self._load_metadata()
+
+    def _load_metadata(self):
+        images = pd.read_csv(os.path.join(self.root, 'CUB_200_2011', 'images.txt'), sep=' ',
+                             names=['img_id', 'filepath'])
+        image_class_labels = pd.read_csv(os.path.join(self.root, 'CUB_200_2011', 'image_class_labels.txt'),
+                                         sep=' ', names=['img_id', 'target'])
+        train_test_split = pd.read_csv(os.path.join(self.root, 'CUB_200_2011', 'train_test_split.txt'),
+                                       sep=' ', names=['img_id', 'is_training_img'])
+
+        data = images.merge(image_class_labels, on='img_id')
+        self.data = data.merge(train_test_split, on='img_id')
+
+        if self.train:
+            self.data = self.data[self.data.is_training_img == 1]
+        else:
+            self.data = self.data[self.data.is_training_img == 0]
+
+        class_names = pd.read_csv(os.path.join(self.root, 'CUB_200_2011', 'classes.txt'),
+                                         sep=' ', names=['class_id', 'target'])
+        self.class_names_str = [name.split(".")[1].replace('_', ' ') for name in class_names.target]
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        sample = self.data.iloc[idx]
+        path = os.path.join(self.root, self.base_folder, sample.filepath)
+        target = sample.target - 1  # Targets start at 1 by default, so shift to 0
+        img = self.loader(path)
+
+        if self.transform is not None:
+            img = self.transform(img)
+
+        return img, target
+
+
+if __name__ == "__main__":
+    train_set = Cub2011(root = "/nobackup/dataset_myf", train = True)
+    val_set = Cub2011(root = "/nobackup/dataset_myf", train = False)
+    # idx = train_loader.dataset.data.target == 1
+
+
+    kwargs = {'num_workers': 4, 'pin_memory': True}
+    train_loader = torch.utils.data.DataLoader(train_set ,
+                    batch_size=16, shuffle=True, **kwargs)
+    val_loader = torch.utils.data.DataLoader(Cub2011(root = "/nobackup/dataset_myf", train = False),
+                    batch_size=16, shuffle=False, **kwargs)

ID-like-train-change-bg/dataloaders/car196.py

@@ -0,0 +1,149 @@
+import pathlib
+from typing import Callable, Optional, Any, Tuple
+
+from PIL import Image
+import torch
+
+from torchvision.datasets.utils import check_integrity,download_and_extract_archive, download_url, verify_str_arg
+from torchvision.datasets.vision import VisionDataset
+
+
+class StanfordCars(VisionDataset):
+    """`Stanford Cars <https://ai.stanford.edu/~jkrause/cars/car_dataset.html>`_ Dataset
+
+    The Cars dataset contains 16,185 images of 196 classes of cars. The data is
+    split into 8,144 training images and 8,041 testing images, where each class
+    has been split roughly in a 50-50 split
+
+    .. note::
+
+        This class needs `scipy <https://docs.scipy.org/doc/>`_ to load target files from `.mat` format.
+
+    Args:
+        root (string): Root directory of dataset
+        split (string, optional): The dataset split, supports ``"train"`` (default) or ``"test"``.
+        transform (callable, optional): A function/transform that  takes in an PIL image
+            and returns a transformed version. E.g, ``transforms.RandomCrop``
+        target_transform (callable, optional): A function/transform that takes in the
+            target and transforms it.
+        download (bool, optional): If True, downloads the dataset from the internet and
+            puts it in root directory. If dataset is already downloaded, it is not
+            downloaded again."""
+
+    def __init__(
+        self,
+        root: str,
+        split: str = "train",
+        transform: Optional[Callable] = None,
+        target_transform: Optional[Callable] = None,
+        download: bool = False,
+    ) -> None:
+
+        try:
+            import scipy.io as sio
+        except ImportError:
+            raise RuntimeError("Scipy is not found. This dataset needs to have scipy installed: pip install scipy")
+
+        super().__init__(root, transform=transform, target_transform=target_transform)
+
+        self._split = verify_str_arg(split, "split", ("train", "test"))
+        self._base_folder = pathlib.Path(root) / "stanford_cars"
+        devkit = self._base_folder / "devkit"
+
+        if self._split == "train":
+            self._annotations_mat_path = devkit / "cars_train_annos.mat"
+            self._images_base_path = self._base_folder / "cars_train"
+        else:
+            self._annotations_mat_path = self._base_folder / "cars_test_annos_withlabels.mat"
+            self._images_base_path = self._base_folder / "cars_test"
+
+        if download:
+            self.download()
+
+        if not self._check_exists():
+            raise RuntimeError("Dataset not found. You can use download=True to download it")
+
+        self._samples = [
+            (
+                str(self._images_base_path / annotation["fname"]),
+                annotation["class"] - 1,  # Original target mapping  starts from 1, hence -1
+            )
+            for annotation in sio.loadmat(self._annotations_mat_path, squeeze_me=True)["annotations"]
+        ]
+
+        self.classes = sio.loadmat(str(devkit / "cars_meta.mat"), squeeze_me=True)["class_names"].tolist()
+        self.class_to_idx = {cls: i for i, cls in enumerate(self.classes)}
+
+        self.class_names_str = self.classes
+
+    def __len__(self) -> int:
+        return len(self._samples)
+
+    def __getitem__(self, idx: int) -> Tuple[Any, Any]:
+        """Returns pil_image and class_id for given index"""
+        image_path, target = self._samples[idx]
+        pil_image = Image.open(image_path).convert("RGB")
+
+        if self.transform is not None:
+            pil_image = self.transform(pil_image)
+        if self.target_transform is not None:
+            target = self.target_transform(target)
+        return pil_image, target
+
+
+    def download(self) -> None:
+        if self._check_exists():
+            return
+
+        download_and_extract_archive(
+            url="https://ai.stanford.edu/~jkrause/cars/car_devkit.tgz",
+            download_root=str(self._base_folder),
+            md5="c3b158d763b6e2245038c8ad08e45376",
+        )
+        if self._split == "train":
+            download_and_extract_archive(
+                url="https://ai.stanford.edu/~jkrause/car196/cars_train.tgz",
+                download_root=str(self._base_folder),
+                md5="065e5b463ae28d29e77c1b4b166cfe61",
+            )
+        else:
+            download_and_extract_archive(
+                url="https://ai.stanford.edu/~jkrause/car196/cars_test.tgz",
+                download_root=str(self._base_folder),
+                md5="4ce7ebf6a94d07f1952d94dd34c4d501",
+            )
+            download_url(
+                url="https://ai.stanford.edu/~jkrause/car196/cars_test_annos_withlabels.mat",
+                root=str(self._base_folder),
+                md5="b0a2b23655a3edd16d84508592a98d10",
+            )
+
+    def _check_exists(self) -> bool:
+        if not (self._base_folder / "devkit").is_dir():
+            return False
+
+        return self._annotations_mat_path.exists() and self._images_base_path.is_dir()
+
+
+def examine_count(counter, name = "train"):
+    print(f"in the {name} set")
+    for label in counter:
+        print(label, counter[label])
+
+if __name__ == "__main__":
+
+    train_set = StanfordCars(root = "/nobackup/dataset_myf", split = "train", download = True)
+    test_set = StanfordCars(root = "/nobackup/dataset_myf", split = "test", download = True)
+    print(f"train set len {len(train_set)}")
+    print(f"test set len {len(test_set)}")
+    from collections import Counter
+    train_label_count = Counter([label for img, label in train_set._samples])
+    test_label_count = Counter([label for img, label in test_set._samples])
+    examine_count(train_label_count, name = "train")
+    examine_count(test_label_count, name = "test")
+
+    kwargs = {'num_workers': 4, 'pin_memory': True}
+    train_loader = torch.utils.data.DataLoader(train_set ,
+                    batch_size=16, shuffle=True, **kwargs)
+    val_loader = torch.utils.data.DataLoader(test_set,
+                    batch_size=16, shuffle=False, **kwargs)

ID-like-train-change-bg/dataloaders/food101.py

@@ -0,0 +1,123 @@
+from pathlib import Path
+import json
+from typing import Any, Tuple, Callable, Optional
+import torch
+import PIL.Image
+
+from torchvision.datasets.utils import check_integrity,download_and_extract_archive, download_url, verify_str_arg
+from torchvision.datasets.vision import VisionDataset
+
+class Food101(VisionDataset):
+    """`The Food-101 Data Set <https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101/>`_.
+
+    The Food-101 is a challenging data set of 101 food categories, with 101'000 images.
+    For each class, 250 manually reviewed test images are provided as well as 750 training images.
+    On purpose, the training images were not cleaned, and thus still contain some amount of noise.
+    This comes mostly in the form of intense colors and sometimes wrong labels. All images were
+    rescaled to have a maximum side length of 512 pixels.
+
+
+    Args:
+        root (string): Root directory of the dataset.
+        split (string, optional): The dataset split, supports ``"train"`` (default) and ``"test"``.
+        transform (callable, optional): A function/transform that  takes in an PIL image and returns a transformed
+            version. E.g, ``transforms.RandomCrop``.
+        target_transform (callable, optional): A function/transform that takes in the target and transforms it.
+        download (bool, optional): If True, downloads the dataset from the internet and
+            puts it in root directory. If dataset is already downloaded, it is not
+            downloaded again. Default is False.
+    """
+
+    _URL = "http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz"
+    _MD5 = "85eeb15f3717b99a5da872d97d918f87"
+
+    def __init__(
+        self,
+        root: str,
+        split: str = "train",
+        transform: Optional[Callable] = None,
+        target_transform: Optional[Callable] = None,
+        download: bool = False,
+    ) -> None:
+        super().__init__(root, transform=transform, target_transform=target_transform)
+        self._split = verify_str_arg(split, "split", ("train", "test"))
+        self._base_folder = Path(self.root) / "food-101"
+        self._meta_folder = self._base_folder / "meta"
+        self._images_folder = self._base_folder / "images"
+        self.class_names_str = ['Apple pie', 'Baby back ribs', 'Baklava', 'Beef carpaccio', 'Beef tartare', 'Beet salad', 'Beignets', 'Bibimbap', 'Bread pudding', 'Breakfast burrito', 'Bruschetta', 'Caesar salad', 'Cannoli', 'Caprese salad', 'Carrot cake', 'Ceviche', 'Cheesecake', 'Cheese plate', 'Chicken curry', 'Chicken quesadilla', 'Chicken wings', 'Chocolate cake', 'Chocolate mousse', 'Churros', 'Clam chowder', 'Club sandwich', 'Crab cakes', 'Creme brulee', 'Croque madame', 'Cup cakes', 'Deviled eggs', 'Donuts', 'Dumplings', 'Edamame', 'Eggs benedict', 'Escargots', 'Falafel', 'Filet mignon', 'Fish and chips', 'Foie gras', 'French fries', 'French onion soup', 'French toast', 'Fried calamari', 'Fried rice', 'Frozen yogurt', 'Garlic bread', 'Gnocchi', 'Greek salad', 'Grilled cheese sandwich', 'Grilled salmon', 'Guacamole', 'Gyoza', 'Hamburger', 'Hot and sour soup', 'Hot dog', 'Huevos rancheros', 'Hummus', 'Ice cream', 'Lasagna', 'Lobster bisque', 'Lobster roll sandwich', 'Macaroni and cheese', 'Macarons', 'Miso soup', 'Mussels', 'Nachos', 'Omelette', 'Onion rings', 'Oysters', 'Pad thai', 'Paella', 'Pancakes', 'Panna cotta', 'Peking duck', 'Pho', 'Pizza', 'Pork chop', 'Poutine', 'Prime rib', 'Pulled pork sandwich', 'Ramen', 'Ravioli', 'Red velvet cake', 'Risotto', 'Samosa', 'Sashimi', 'Scallops', 'Seaweed salad', 'Shrimp and grits', 'Spaghetti bolognese', 'Spaghetti carbonara', 'Spring rolls', 'Steak', 'Strawberry shortcake', 'Sushi', 'Tacos', 'Takoyaki', 'Tiramisu', 'Tuna tartare', 'Waffles']
+
+        if download:
+            self._download()
+
+        if not self._check_exists():
+            raise RuntimeError("Dataset not found. You can use download=True to download it")
+
+        self._labels = []
+        self._image_files = []
+        with open(self._meta_folder / f"{split}.json") as f:
+            metadata = json.loads(f.read())
+
+        self.classes = sorted(metadata.keys())
+        self.class_to_idx = dict(zip(self.classes, range(len(self.classes))))
+
+        for class_label, im_rel_paths in metadata.items():
+            self._labels += [self.class_to_idx[class_label]] * len(im_rel_paths)
+            self._image_files += [
+                self._images_folder.joinpath(*f"{im_rel_path}.jpg".split("/")) for im_rel_path in im_rel_paths
+            ]
+
+    def __len__(self) -> int:
+        return len(self._image_files)
+
+    def __getitem__(self, idx) -> Tuple[Any, Any]:
+        image_file, label = self._image_files[idx], self._labels[idx]
+        image = PIL.Image.open(image_file).convert("RGB")
+
+        if self.transform:
+            image = self.transform(image)
+
+        if self.target_transform:
+            label = self.target_transform(label)
+
+        return image, label
+
+
+    def extra_repr(self) -> str:
+        return f"split={self._split}"
+
+    def _check_exists(self) -> bool:
+        return all(folder.exists() and folder.is_dir() for folder in (self._meta_folder, self._images_folder))
+
+    def _download(self) -> None:
+        if self._check_exists():
+            return
+        download_and_extract_archive(self._URL, download_root=self.root, md5=self._MD5)
+
+def examine_count(counter, name = "train"):
+    print(f"in the {name} set")
+    for label in counter:
+        print(label, counter[label])
+
+if __name__ == "__main__":
+
+    label_names = []
+    with open('debug/food101_labels.txt') as f:
+        for name in f:
+            label_names.append(name.strip())
+    print(label_names)
+
+    train_set = Food101(root = "/nobackup/dataset_myf", split = "train", download = True)
+    test_set = Food101(root = "/nobackup/dataset_myf", split = "test")
+    print(f"train set len {len(train_set)}")
+    print(f"test set len {len(test_set)}")
+    from collections import Counter
+    train_label_count = Counter(train_set._labels)
+    test_label_count = Counter(test_set._labels)
+    # examine_count(train_label_count, name = "train")
+    # examine_count(test_label_count, name = "test")
+
+    kwargs = {'num_workers': 4, 'pin_memory': True}
+    train_loader = torch.utils.data.DataLoader(train_set ,
+                    batch_size=16, shuffle=True, **kwargs)
+    val_loader = torch.utils.data.DataLoader(test_set,
+                    batch_size=16, shuffle=False, **kwargs)

ID-like-train-change-bg/dataloaders/pet37.py

@@ -0,0 +1,152 @@
+import os
+import os.path
+
+from typing import Any, Tuple, Callable, Optional, Union, Sequence
+import torch
+from PIL import Image
+import pathlib
+from torchvision.datasets.utils import check_integrity,download_and_extract_archive, download_url, verify_str_arg
+from torchvision.datasets.vision import VisionDataset
+
+
+class OxfordIIITPet(VisionDataset):
+    """`Oxford-IIIT Pet Dataset   <https://www.robots.ox.ac.uk/~vgg/data/pets/>`_.
+
+    Args:
+        root (string): Root directory of the dataset.
+        split (string, optional): The dataset split, supports ``"trainval"`` (default) or ``"test"``.
+        target_types (string, sequence of strings, optional): Types of target to use. Can be ``category`` (default) or
+            ``segmentation``. Can also be a list to output a tuple with all specified target types. The types represent:
+
+                - ``category`` (int): Label for one of the 37 pet categories.
+                - ``segmentation`` (PIL image): Segmentation trimap of the image.
+
+            If empty, ``None`` will be returned as target.
+
+        transform (callable, optional): A function/transform that  takes in a PIL image and returns a transformed
+            version. E.g, ``transforms.RandomCrop``.
+        target_transform (callable, optional): A function/transform that takes in the target and transforms it.
+        download (bool, optional): If True, downloads the dataset from the internet and puts it into
+            ``root/oxford-iiit-pet``. If dataset is already downloaded, it is not downloaded again.
+    """
+
+    _RESOURCES = (
+        ("https://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz", "5c4f3ee8e5d25df40f4fd59a7f44e54c"),
+        ("https://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz", "95a8c909bbe2e81eed6a22bccdf3f68f"),
+    )
+    _VALID_TARGET_TYPES = ("category", "segmentation")
+
+    def __init__(
+        self,
+        root: str,
+        split: str = "trainval",
+        target_types: Union[Sequence[str], str] = "category",
+        transforms: Optional[Callable] = None,
+        transform: Optional[Callable] = None,
+        target_transform: Optional[Callable] = None,
+        download: bool = False,
+    ):
+        self._split = verify_str_arg(split, "split", ("trainval", "test"))
+        if isinstance(target_types, str):
+            target_types = [target_types]
+        self._target_types = [
+            verify_str_arg(target_type, "target_types", self._VALID_TARGET_TYPES) for target_type in target_types
+        ]
+
+        super().__init__(root, transforms=transforms, transform=transform, target_transform=target_transform)
+        self._base_folder = pathlib.Path(self.root) / "oxford-iiit-pet"
+        self._images_folder = self._base_folder / "images"
+        self._anns_folder = self._base_folder / "annotations"
+        self._segs_folder = self._anns_folder / "trimaps"
+
+        if download:
+            self._download()
+
+        if not self._check_exists():
+            raise RuntimeError("Dataset not found. You can use download=True to download it")
+
+        image_ids = []
+        self._labels = []
+        with open(self._anns_folder / f"{self._split}.txt") as file:
+            for line in file:
+                image_id, label, *_ = line.strip().split()
+                image_ids.append(image_id)
+                self._labels.append(int(label) - 1)
+
+        self.classes = [
+            " ".join(part.title() for part in raw_cls.split("_"))
+            for raw_cls, _ in sorted(
+                {(image_id.rsplit("_", 1)[0], label) for image_id, label in zip(image_ids, self._labels)},
+                key=lambda image_id_and_label: image_id_and_label[1],
+            )
+        ]
+        self.class_to_idx = dict(zip(self.classes, range(len(self.classes))))
+
+        self._images = [self._images_folder / f"{image_id}.jpg" for image_id in image_ids]
+        self._segs = [self._segs_folder / f"{image_id}.png" for image_id in image_ids]
+
+        self.class_names_str = self.classes
+
+    def __len__(self) -> int:
+        return len(self._images)
+
+    def __getitem__(self, idx: int) -> Tuple[Any, Any]:
+        image = Image.open(self._images[idx]).convert("RGB")
+
+        target: Any = []
+        for target_type in self._target_types:
+            if target_type == "category":
+                target.append(self._labels[idx])
+            else:  # target_type == "segmentation"
+                target.append(Image.open(self._segs[idx]))
+
+        if not target:
+            target = None
+        elif len(target) == 1:
+            target = target[0]
+        else:
+            target = tuple(target)
+
+        if self.transforms:
+            image, target = self.transforms(image, target)
+
+        return image, target
+
+
+    def _check_exists(self) -> bool:
+        for folder in (self._images_folder, self._anns_folder):
+            if not (os.path.exists(folder) and os.path.isdir(folder)):
+                return False
+        else:
+            return True
+
+    def _download(self) -> None:
+        if self._check_exists():
+            return
+
+        for url, md5 in self._RESOURCES:
+            download_and_extract_archive(url, download_root=str(self._base_folder), md5=md5)
+
+
+def examine_count(counter, name = "train"):
+    print(f"in the {name} set")
+    for label in counter:
+        print(label, counter[label])
+
+if __name__ == "__main__":
+
+    train_set = OxfordIIITPet(root = "/nobackup/dataset_myf", split = "trainval", download = True)
+    test_set = OxfordIIITPet(root = "/nobackup/dataset_myf", split = "test")
+    print(f"train set len {len(train_set)}")
+    print(f"test set len {len(test_set)}")
+    from collections import Counter
+    train_label_count = Counter(train_set._labels)
+    test_label_count = Counter(test_set._labels)
+    examine_count(train_label_count, name = "train")
+    examine_count(test_label_count, name = "test")
+
+    kwargs = {'num_workers': 4, 'pin_memory': True}
+    train_loader = torch.utils.data.DataLoader(train_set ,
+                    batch_size=16, shuffle=True, **kwargs)
+    val_loader = torch.utils.data.DataLoader(test_set,
+                    batch_size=16, shuffle=False, **kwargs)

ID-like-train-change-bg/eval_ood_detection.py

@@ -0,0 +1,123 @@
+import os
+import argparse
+import numpy as np
+import torch
+from scipy import stats
+import config
+from utils.common import setup_seed, get_and_print_results, print_measures
+from utils.file_ops import save_as_dataframe, setup_log
+from utils.plot_util import plot_distribution
+from utils.dataloaders_utils import set_few_shot_loader, set_val_loader, set_ood_loader_ImageNet
+from utils.id_like import get_prompts, get_result, load_model
+
+import pickle
+import collections
+
+def process_args():
+    parser = argparse.ArgumentParser(description='Evaluates OOD for CLIP',
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--root-dir', default="/home/zhourixin/OOD_Folder/CODE/other_methods/ID-like-train-change-bg/OODdata", type=str,
+                        help='root dir of datasets')
+    parser.add_argument('--in_dataset', default='bronze2NotLine', type=str, help='in-distribution dataset')
+    parser.add_argument('--seed', default=1, type=int, help="random seed")
+    parser.add_argument('--score', default='id-like', type=str)
+    parser.add_argument('--CLIP_ckpt', type=str, default='ViT-B/16',
+                        choices=['ViT-B/32', 'ViT-B/16', 'ViT-L/14'],
+                        help='which pretrained img encoder to use')
+    parser.add_argument('--n_shot', default=5, type=int,
+                        help="how many samples are used to estimate classwise mean and precision matrix")
+    parser.add_argument('--batch_size', default=10, type=int, help='mini-batch size')
+    parser.add_argument('--test_batch_size', default=512, type=int, help='mini-batch size')
+    parser.add_argument('--n_crop', default=256, type=int, help='crop num')
+    parser.add_argument('--n_selection', default=32, type=int, help='selection num')
+    # parser.add_argument('--selection_p', default=0.2, type=float, help='confidence selection percentile')
+    parser.add_argument('--n_ex_prompts', default=100, type=int, help='number of extra prompts')
+    parser.add_argument('--n_epoch', default=3, type=int, help='number of epoch')
+    parser.add_argument('--lr', '--learning-rate', default=5e-3, type=float, metavar='LR', help='initial learning rate',
+                        dest='lr')
+    parser.add_argument('--lam_in', default=1.0, type=float, help='lambda of id loss')
+    parser.add_argument('--lam_out', default=0.3, type=float, help='lambda of ood loss')
+    parser.add_argument('--lam_diff', default=0.2, type=float, help='lambda of difference')
+
+    args = parser.parse_args()
+
+    args.n_cls = config.data_info[args.in_dataset]['n_cls']
+
+    args.log_directory = f"/home/zhourixin/OOD_Folder/CODE/other_methods/ID-like-train-change-bg/results/{args.in_dataset}/id-like/{args.n_shot}shot/"
+
+    os.makedirs(args.log_directory, exist_ok=True)
+    setup_seed(args.seed)
+    return args
+
+def update(d, u):
+    for k, v in u.items():
+        if isinstance(v, collections.abc.Mapping):
+            d[k] = update(d.get(k, {}), v)
+        else:
+            d[k] = v
+    return d
+
+def train():
+    args = process_args()
+
+    log = setup_log(args)
+    # out_datasets = ['ssb_hard', 'ninco']
+    # out_datasets = ['imagenet22k_container', 'ssb_hard', 'ninco',
+    #  'inaturalist', 'textures', 'openimage_o']
+    out_datasets = ['imagenet22k_container_refine', 'bronzeS_containerM', 
+    'bronzeM_containerS', 'bronze_Line', 'ssb_hard', 'ninco',
+     'inaturalist', 'textures', 'openimage_o']
+    # out_datasets = ['imagenet22k_container_refine', 'bronzeS_containerM', 
+    # 'bronzeM_containerS', 'bronze_Line', 'ssb_hard', 'ninco',
+    #  'inaturalist', 'textures', 'openimage_o']
+    
+
+    test_labels = config.data_info[args.in_dataset]['labels']
+    ex_labels = ['X'] * args.n_ex_prompts
+
+    model_checkpoint_save_path = os.path.join(args.log_directory, 'model_checkpoint.pth')
+
+    if os.path.exists(model_checkpoint_save_path):
+        model = load_model(args, test_labels, ex_labels)
+    else:
+        few_shot_loader = set_few_shot_loader(args)
+        model = get_prompts(args, few_shot_loader, test_labels, ex_labels)
+
+    score_resulu_dic = {}
+
+    test_loader = set_val_loader(args)
+    result_in = get_result(args, model, test_loader, test_labels, ex_labels, if_acc=True)
+    score_in = result_in['scores']
+    acc = result_in['acc']
+    log.debug(f"Acc: {acc}")
+
+    update(score_resulu_dic,{"score_in":score_in})
+
+    auroc_list, aupr_list, fpr_list = [], [], []
+    for out_dataset in out_datasets:
+        log.debug(f"Evaluting OOD dataset {out_dataset}")
+        ood_loader = set_ood_loader_ImageNet(args, out_dataset)
+        result_out = get_result(args, model, ood_loader, test_labels, ex_labels)
+        score_out = result_out['scores']
+        log.debug(f"in scores: {stats.describe(score_in)}")
+        log.debug(f"out scores: {stats.describe(score_out)}")
+        plot_distribution(args, score_in, score_out, out_dataset)
+        get_and_print_results(args, log, score_in, score_out,
+                              auroc_list, aupr_list, fpr_list)
+        update(score_resulu_dic, {"out_score":{out_dataset:score_out}})
+
+    log.debug('\n\nMean Test Results')
+    print_measures(log, np.mean(auroc_list), np.mean(aupr_list),
+                   np.mean(fpr_list), method_name=args.score)
+    save_as_dataframe(args, out_datasets, fpr_list, auroc_list, aupr_list, acc)
+
+    with open(os.path.join(args.log_directory, 'score.pkl'),
+                'wb') as f:
+        pickle.dump(score_resulu_dic, f, pickle.HIGHEST_PROTOCOL)
+
+    
+
+
+if __name__ == "__main__":
+    train()
+    

ID-like-train-change-bg/utils/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import absolute_import
+from .common import *

ID-like-train-change-bg/utils/common.py

@@ -0,0 +1,164 @@
+import torch
+import torch.nn.functional as F
+import os
+import numpy as np
+import json
+import random
+import sklearn.metrics as sk
+
+
+def setup_seed(seed):
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+
+
+def accuracy(output, target, topk=(1,)):
+    """Computes the precision@k for the specified values of k"""
+    maxk = max(topk)
+    batch_size = target.size(0)
+    # values, indices = input.topk(k, dim=1, largest=True, sorted=True)
+    _, pred = output.topk(maxk, 1, True, True)
+    pred = pred.t()
+    correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+    res = []
+    for k in topk:
+        correct_k = correct[:k].flatten().float().sum(0)
+        res.append(correct_k.mul_(100.0 / batch_size))
+    return res
+
+
+def read_file(file_path, root='corpus'):
+    corpus = []
+    with open(os.path.join(root, file_path)) as f:
+        for line in f:
+            corpus.append(line[:-1])
+    return corpus
+
+
+def calculate_cosine_similarity(image_features, text_features):
+    image_features /= image_features.norm(dim=-1, keepdim=True)
+    text_features /= text_features.norm(dim=-1, keepdim=True)
+    similarity = text_features.cpu().numpy() @ image_features.cpu().numpy().T
+    return similarity
+
+
+class AverageMeter(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+
+def stable_cumsum(arr, rtol=1e-05, atol=1e-08):
+    """Use high precision for cumsum and check that final value matches sum
+    Parameters
+    ----------
+    arr : array-like
+        To be cumulatively summed as flat
+    rtol : float
+        Relative tolerance, see ``np.allclose``
+    atol : float
+        Absolute tolerance, see ``np.allclose``
+    """
+    out = np.cumsum(arr, dtype=np.float64)
+    expected = np.sum(arr, dtype=np.float64)
+    if not np.allclose(out[-1], expected, rtol=rtol, atol=atol):
+        raise RuntimeError('cumsum was found to be unstable: '
+                           'its last element does not correspond to sum')
+    return out
+
+
+def fpr_and_fdr_at_recall(y_true, y_score, recall_level=0.95, pos_label=None):
+    classes = np.unique(y_true)
+    if (pos_label is None and
+            not (np.array_equal(classes, [0, 1]) or
+                     np.array_equal(classes, [-1, 1]) or
+                     np.array_equal(classes, [0]) or
+                     np.array_equal(classes, [-1]) or
+                     np.array_equal(classes, [1]))):
+        raise ValueError("Data is not binary and pos_label is not specified")
+    elif pos_label is None:
+        pos_label = 1.
+
+    # make y_true a boolean vector
+    y_true = (y_true == pos_label)
+
+    # sort scores and corresponding truth values
+    desc_score_indices = np.argsort(y_score, kind="mergesort")[::-1]
+    y_score = y_score[desc_score_indices]
+    y_true = y_true[desc_score_indices]
+
+    # y_score typically has many tied values. Here we extract
+    # the indices associated with the distinct values. We also
+    # concatenate a value for the end of the curve.
+    distinct_value_indices = np.where(np.diff(y_score))[0]
+    threshold_idxs = np.r_[distinct_value_indices, y_true.size - 1]
+
+    # accumulate the true positives with decreasing threshold
+    tps = stable_cumsum(y_true)[threshold_idxs]
+    fps = 1 + threshold_idxs - tps      # add one because of zero-based indexing
+
+    thresholds = y_score[threshold_idxs]
+
+    recall = tps / tps[-1]
+
+    last_ind = tps.searchsorted(tps[-1])
+    sl = slice(last_ind, None, -1)      # [last_ind::-1]
+    recall, fps, tps, thresholds = np.r_[recall[sl], 1], np.r_[fps[sl], 0], np.r_[tps[sl], 0], thresholds[sl]
+
+    cutoff = np.argmin(np.abs(recall - recall_level))
+
+    return fps[cutoff] / (np.sum(np.logical_not(y_true)))   # , fps[cutoff]/(fps[cutoff] + tps[cutoff])
+
+
+def get_measures(_pos, _neg, recall_level=0.95):
+    pos = np.array(_pos[:]).reshape((-1, 1))
+    neg = np.array(_neg[:]).reshape((-1, 1))
+    examples = np.squeeze(np.vstack((pos, neg)))
+    labels = np.zeros(len(examples), dtype=np.int32)
+    labels[:len(pos)] += 1
+
+    auroc = sk.roc_auc_score(labels, examples)
+    aupr = sk.average_precision_score(labels, examples)
+    fpr = fpr_and_fdr_at_recall(labels, examples, recall_level)
+
+    return auroc, aupr, fpr
+
+
+def print_measures(log, auroc, aupr, fpr, method_name='Ours', recall_level=0.95):
+    if log == None:
+        print('FPR{:d}:\t\t\t{:.2f}'.format(int(100 * recall_level), 100 * fpr))
+        print('AUROC: \t\t\t{:.2f}'.format(100 * auroc))
+        print('AUPR:  \t\t\t{:.2f}'.format(100 * aupr))
+    else:
+        log.debug('\t\t\t\t' + method_name)
+        log.debug('  FPR{:d} AUROC AUPR'.format(int(100*recall_level)))
+        log.debug('& {:.2f} & {:.2f} & {:.2f}'.format(100*fpr, 100*auroc, 100*aupr))
+
+
+def get_and_print_results(args, log, in_score, out_score, auroc_list, aupr_list, fpr_list):
+    '''
+    1) evaluate detection performance for a given OOD test set (loader)
+    2) print results (FPR95, AUROC, AUPR)
+    '''
+    aurocs, auprs, fprs = [], [], []
+    measures = get_measures(-in_score, -out_score)
+    aurocs.append(measures[0]); auprs.append(measures[1]); fprs.append(measures[2])
+    print(f'in score samples (random sampled): {in_score[:3]}, out score samples: {out_score[:3]}')
+    # print(f'in score samples (min): {in_score[-3:]}, out score samples: {out_score[-3:]}')
+    auroc = np.mean(aurocs); aupr = np.mean(auprs); fpr = np.mean(fprs)
+    auroc_list.append(auroc); aupr_list.append(aupr); fpr_list.append(fpr) # used to calculate the avg over multiple OOD test sets
+    print_measures(log, auroc, aupr, fpr, args.score)

ID-like-train-change-bg/utils/dataloaders_utils.py

@@ -0,0 +1,462 @@
+import sys
+import os
+import numpy as np
+import torch
+import torchvision
+from torch import nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, Subset, DataLoader
+# from transformers import CLIPModel
+from torchvision import datasets, transforms
+import torchvision.transforms as transforms
+from dataloaders import StanfordCars, Food101, OxfordIIITPet, Cub2011
+from torchvision.datasets import CIFAR10, CIFAR100, SVHN
+from tqdm import tqdm
+import config
+from clip import load, tokenize
+from clip.simple_tokenizer import SimpleTokenizer as _Tokenizer
+
+_tokenizer = _Tokenizer()
+
+
+def update_class_to_idx(dataset, new_class_to_idx):
+    """
+    更新 dataset 实例的 class_to_idx 映射。
+    
+    :param dataset: ImageFolder 实例。
+    :param new_class_to_idx: 新的类别到索引的映射字典。
+    """
+    # 更新 class_to_idx 映射
+    dataset.class_to_idx = new_class_to_idx
+    
+    # 根据新的 class_to_idx 更新 idx_to_class 映射
+    dataset.idx_to_class = {idx: class_name for class_name, idx in new_class_to_idx.items()}
+    
+    # 重新构建样本列表，以确保它们与新的映射相匹配
+    dataset.samples = []
+    for class_name, idx in new_class_to_idx.items():
+        class_dir = os.path.join(dataset.root, class_name)
+        for entry in os.listdir(class_dir):
+            full_path = os.path.join(class_dir, entry)
+            if os.path.isfile(full_path):
+                dataset.samples.append((full_path, idx))
+
+
+def set_train_loader(args, subset=False, max_count=0):
+    root = args.root_dir
+    normalize = transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073),
+                                     std=(0.26862954, 0.26130258, 0.27577711))  # for CLIP
+    preprocess = transforms.Compose([
+        transforms.Resize(224),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        normalize
+    ])
+    kwargs = {'num_workers': 4, 'pin_memory': True}
+    batch_size = args.batch_size
+    batch_size = 256
+    shuffle = True
+    if args.in_dataset == "ImageNet":
+        path = os.path.join(root, 'ImageNet', 'train')
+    elif args.in_dataset == "ImageNet100":
+        path = os.path.join(root, "ImageNet100", 'train')
+    elif args.in_dataset == "ImageNet10":
+        path = os.path.join(root, "ImageNet10", 'train')
+    elif args.in_dataset == "ImageNet20":
+        path = os.path.join(root, "ImageNet20", 'train')
+    dataset = datasets.ImageFolder(path, transform=preprocess)
+    if subset:
+        from collections import defaultdict
+        classwise_count = defaultdict(int)
+        indices = []
+        for i, label in enumerate(dataset.targets):
+            if classwise_count[label] < max_count:
+                indices.append(i)
+                classwise_count[label] += 1
+        dataset = torch.utils.data.Subset(dataset, indices)
+    train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, **kwargs)
+
+    return train_loader
+
+
+def set_val_loader(args):
+    root = args.root_dir
+    normalize = transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073),
+                                     std=(0.26862954, 0.26130258, 0.27577711))  # for CLIP
+    preprocess = transforms.Compose([
+        transforms.Resize(224),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        normalize
+    ])
+    kwargs = {'num_workers': 4, 'pin_memory': True}
+    if args.in_dataset == "ImageNet":
+        path = os.path.join(root, 'ImageNet', 'val')
+        dataset = datasets.ImageFolder(path, transform=preprocess)
+
+    elif args.in_dataset == "bronze2NotLine":
+        path = os.path.join(root, "bronze_ID_and_OOD", "composite_split", "test")
+        dataset = datasets.ImageFolder(path, transform=preprocess)
+        new_class_to_idx = {'age_0':0, 'age_1':1, 'age_2':2, 'age_3':3, 
+        'age_4':4, 'age_5':5, 'age_6':6, 'age_7':7, 'age_8':8, 'age_9':9, 'age_10':10}
+        update_class_to_idx(dataset, new_class_to_idx)
+        
+
+
+    elif args.in_dataset == "ImageNet100":
+        path = os.path.join(root, "ImageNet100", 'val')
+        dataset = datasets.ImageFolder(path, transform=preprocess)
+    elif args.in_dataset == "ImageNet10":
+        path = os.path.join(root, "ImageNet10", 'train')
+        dataset = datasets.ImageFolder(path, transform=preprocess)
+    elif args.in_dataset == "ImageNet20":
+        path = os.path.join(root, "ImageNet20", 'train')
+        dataset = datasets.ImageFolder(path, transform=preprocess)
+    elif args.in_dataset == "car196":
+        path = root
+        dataset = StanfordCars(path, split="test", download=True, transform=preprocess)
+    elif args.in_dataset == "food101":
+        path = root
+        dataset = Food101(path, split="test", download=True, transform=preprocess)
+    elif args.in_dataset == "pet37":
+        path = root
+        dataset = OxfordIIITPet(path, split="test", download=True, transform=preprocess)
+    elif args.in_dataset == "bird200":
+        path = root
+        dataset = Cub2011(path, train=False, transform=preprocess)
+    elif args.in_dataset == "cifar10":
+        path = root
+        dataset = CIFAR10(path, train=False, transform=preprocess)
+    elif args.in_dataset == "cifar100":
+        path = root
+        dataset = CIFAR100(path, train=False, transform=preprocess)
+
+    val_loader = torch.utils.data.DataLoader(dataset, batch_size=args.test_batch_size, shuffle=False, **kwargs)
+
+    return val_loader
+
+
+def set_ood_loader_ImageNet(args, out_dataset):
+    '''
+    set OOD loader for ImageNet scale datasets
+    '''
+    # root = os.path.join(args.root_dir, 'ImageNet_OOD_dataset')
+    root = args.root_dir
+    # normalize = transforms.Normalize((0.5071, 0.4867, 0.4408), (0.2675, 0.2565, 0.2761))
+    normalize = transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073),
+                                     std=(0.26862954, 0.26130258, 0.27577711))  # for CLIP
+    preprocess = transforms.Compose([
+        transforms.Resize(224),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        normalize
+    ])
+    if out_dataset == 'imagenet22k_container':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'imagenet-21k-container', 'images'), transform=preprocess)
+    elif out_dataset == 'imagenet22k_container_refine':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'imagenet-21k-container-refine', 'images'), transform=preprocess)
+    elif out_dataset == 'bronzeS_containerM':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'transfer_dataset', 'bronze_structure_container_material', 'test'), transform=preprocess)
+    elif out_dataset == 'bronzeM_containerS':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'transfer_dataset', 'container_structure_bronze_material', 'test'), transform=preprocess)
+    elif out_dataset == 'bronze_Line':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'bronze_line'), transform=preprocess)
+    elif out_dataset == 'ssb_hard':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'ssb_hard'), transform=preprocess)
+    elif out_dataset == 'ninco':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'ninco'), transform=preprocess)
+    elif out_dataset == 'inaturalist':
+        # testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'iNaturalist'), transform=preprocess)
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'inaturalist'), transform=preprocess)
+    elif out_dataset == 'textures':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_classic', 'texture'), transform=preprocess)
+    elif out_dataset == 'openimage_o':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'images_largescale', 'openimage_o'), transform=preprocess)
+    
+
+
+    elif out_dataset == 'SUN':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'SUN'), transform=preprocess)
+    elif out_dataset == 'places365':  # filtered places
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'Places'), transform=preprocess)
+    elif out_dataset == 'placesbg':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'placesbg'), transform=preprocess)
+    elif out_dataset == 'dtd':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(root, 'dtd', 'images'), transform=preprocess)
+    elif out_dataset == 'svhn':
+        testsetout = SVHN(root=os.path.join(args.root_dir, 'svhn'), split='test', transform=preprocess)
+    elif out_dataset == "cifar10":
+        testsetout = CIFAR10(root=args.root_dir, train=False, transform=preprocess)
+    elif out_dataset == "cifar100":
+        testsetout = CIFAR100(root=args.root_dir, train=False, transform=preprocess)
+    elif out_dataset == 'ssb_hard':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(args.root_dir, 'ssb_hard'), transform=preprocess)
+    elif out_dataset == 'ninco':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(args.root_dir, 'ninco'), transform=preprocess)
+    elif out_dataset == 'openimage_o':
+        testsetout = torchvision.datasets.ImageFolder(root=os.path.join(args.root_dir, 'openimage_o'), transform=preprocess)
+    testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=args.test_batch_size, shuffle=False, num_workers=0)
+
+    return testloaderOut
+
+
+class RandomCrop(object):
+    def __init__(self, n_crop=2):
+        normalize = transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073),
+                                         std=(0.26862954, 0.26130258, 0.27577711))  # for CLIP
+        self.n_crop = n_crop
+        self.random_crop = transforms.Compose([
+            # transforms.RandomResizedCrop(224, scale=(0.2, 1.0)),
+            transforms.RandomResizedCrop(224),
+            transforms.RandomHorizontalFlip(),
+            transforms.ToTensor(),
+            normalize
+        ])
+
+    def __call__(self, x):
+        views = [self.random_crop(x).unsqueeze(dim=0) for _ in range(self.n_crop)]
+        views = torch.cat(views, dim=0)
+        return views
+
+
+def set_few_shot_loader(args):
+    root = args.root_dir
+    data_transform = RandomCrop(args.n_crop)
+    # data_transform = RandomCropAndMask(args.n_crop, args.n_crop)
+    shuffle = True
+    kwargs = {'num_workers': 0, 'pin_memory': True}
+
+    if args.in_dataset == "ImageNet":
+        path = os.path.join(root, 'images_largescale', 'imagenet_1k', 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "bronze2NotLine":
+        path = os.path.join(root, "bronze_ID_and_OOD", "composite_split", "train")
+        dataset = datasets.ImageFolder(path)
+        new_class_to_idx = {'age_0':0, 'age_1':1, 'age_2':2, 'age_3':3, 
+        'age_4':4, 'age_5':5, 'age_6':6, 'age_7':7, 'age_8':8, 'age_9':9, 'age_10':10}
+        update_class_to_idx(dataset, new_class_to_idx)
+        # A = dataset.class_to_idx
+
+    elif args.in_dataset == "ImageNet100":
+        path = os.path.join(root, "ImageNet100", 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "ImageNet10":
+        path = os.path.join(root, "ImageNet10", 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "ImageNet20":
+        path = os.path.join(root, "ImageNet20", 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "car196":
+        path = root
+        dataset = StanfordCars(path, split="train", download=True)
+        dataset.targets = [target for _, target in dataset]
+    elif args.in_dataset == "food101":
+        path = root
+        dataset = Food101(path, split="train", download=True)
+        dataset.targets = [target for _, target in dataset]
+    elif args.in_dataset == "pet37":
+        path = root
+        dataset = OxfordIIITPet(path, split="trainval", download=True)
+        dataset.targets = [target for _, target in dataset]
+    elif args.in_dataset == "bird200":
+        path = root
+        dataset = Cub2011(path, train=True)
+        dataset.targets = [dataset.data.iloc[idx].target - 1 for idx in range(len(dataset))]
+    elif args.in_dataset == "cifar10":
+        path = root
+        dataset = CIFAR10(path, train=True)
+    elif args.in_dataset == "cifar100":
+        path = root
+        dataset = CIFAR100(path, train=True)
+
+    indices = []
+    from collections import defaultdict
+    classwise_idx = defaultdict(list)
+    print('get dataset index')
+    for i, target in enumerate(tqdm(dataset.targets)):
+        classwise_idx[target].append(i)
+    print('sample few shot dataset')
+
+
+
+
+
+    from random import sample
+    for i in tqdm(range(args.n_cls)):
+        sample_length = len(classwise_idx[i])
+        if args.n_shot == 100000:
+            sl = sample(classwise_idx[i], int(sample_length*1))
+        elif args.n_shot == 90000:
+            sl = sample(classwise_idx[i], int(sample_length*0.9))
+        elif args.n_shot == 80000:
+            sl = sample(classwise_idx[i], int(sample_length*0.8))
+        elif args.n_shot == 70000:
+            sl = sample(classwise_idx[i], int(sample_length*0.7))
+        elif args.n_shot == 60000:
+            sl = sample(classwise_idx[i], int(sample_length*0.6))
+        elif args.n_shot == 50000:
+            sl = sample(classwise_idx[i], int(sample_length*0.5))
+        elif args.n_shot == 40000:
+            sl = sample(classwise_idx[i], int(sample_length*0.4))
+        elif args.n_shot == 30000:
+            sl = sample(classwise_idx[i], int(sample_length*0.3))
+        elif args.n_shot == 20000:
+            sl = sample(classwise_idx[i], int(sample_length*0.2))
+        elif args.n_shot == 10000:
+            sl = sample(classwise_idx[i], int(sample_length*0.1))
+        else:
+            sl = sample(classwise_idx[i], args.n_shot)
+        indices.extend(sl)
+
+    if args.in_dataset == "ImageNet":
+        path = os.path.join(root, 'images_largescale', 'imagenet_1k', 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+        # path = os.path.join(root, 'ImageNet', 'train')
+        # dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "bronze2NotLine":
+        path = os.path.join(root, "bronze_ID_and_OOD", "composite_split", "train")
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+        new_class_to_idx = {'age_0':0, 'age_1':1, 'age_2':2, 'age_3':3, 
+        'age_4':4, 'age_5':5, 'age_6':6, 'age_7':7, 'age_8':8, 'age_9':9, 'age_10':10}
+        update_class_to_idx(dataset, new_class_to_idx)
+    elif args.in_dataset == "ImageNet100":
+        path = os.path.join(root, "ImageNet100", 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "ImageNet10":
+        path = os.path.join(root, "ImageNet10", 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "ImageNet20":
+        path = os.path.join(root, "ImageNet20", 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "car196":
+        path = root
+        dataset = StanfordCars(path, split="train", download=True, transform=data_transform)
+    elif args.in_dataset == "food101":
+        path = root
+        dataset = Food101(path, split="train", download=True, transform=data_transform)
+    elif args.in_dataset == "pet37":
+        path = root
+        dataset = OxfordIIITPet(path, split="trainval", download=True, transform=data_transform)
+    elif args.in_dataset == "bird200":
+        path = root
+        dataset = Cub2011(path, train=True, transform=data_transform)
+    elif args.in_dataset == "cifar10":
+        path = root
+        dataset = CIFAR10(path, train=True, transform=data_transform)
+    elif args.in_dataset == "cifar100":
+        path = root
+        dataset = CIFAR100(path, train=True, transform=data_transform)
+    
+    dataset = torch.utils.data.Subset(dataset, indices)
+    few_shot_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=shuffle, **kwargs)
+
+    # from torch.utils.data.distributed import DistributedSampler
+    # sampler = DistributedSampler(dataset)
+    # few_shot_loader = torch.utils.data.DataLoader(dataset, sampler=sampler,
+    #                                               batch_size=args.batch_size,
+    #                                               shuffle=False, **kwargs)
+
+    return few_shot_loader
+
+
+def set_few_shot_loader_normal(args):
+    root = args.root_dir
+    normalize = transforms.Normalize(mean=(0.48145466, 0.4578275, 0.40821073),
+                                     std=(0.26862954, 0.26130258, 0.27577711))  # for CLIP
+    data_transform = transforms.Compose([
+        transforms.Resize(224),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        normalize
+    ])
+    # data_transform = RandomCropAndMask(args.n_crop, args.n_crop)
+    shuffle = True
+    kwargs = {'num_workers': 0, 'pin_memory': True}
+
+    if args.in_dataset == "ImageNet":
+        path = os.path.join(root, 'ImageNet', 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "ImageNet100":
+        path = os.path.join(root, "ImageNet100", 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "ImageNet10":
+        path = os.path.join(root, "ImageNet10", 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "ImageNet20":
+        path = os.path.join(root, "ImageNet20", 'train')
+        dataset = datasets.ImageFolder(path)
+    elif args.in_dataset == "car196":
+        path = root
+        dataset = StanfordCars(path, split="train", download=True)
+        dataset.targets = [target for _, target in dataset]
+    elif args.in_dataset == "food101":
+        path = root
+        dataset = Food101(path, split="train", download=True)
+        dataset.targets = [target for _, target in dataset]
+    elif args.in_dataset == "pet37":
+        path = root
+        dataset = OxfordIIITPet(path, split="trainval", download=True)
+        dataset.targets = [target for _, target in dataset]
+    elif args.in_dataset == "bird200":
+        path = root
+        dataset = Cub2011(path, train=True)
+        dataset.targets = [dataset.data.iloc[idx].target - 1 for idx in range(len(dataset))]
+    elif args.in_dataset == "cifar10":
+        path = root
+        dataset = CIFAR10(path, train=True)
+    elif args.in_dataset == "cifar100":
+        path = root
+        dataset = CIFAR100(path, train=True)
+
+    indices = []
+    from collections import defaultdict
+    classwise_idx = defaultdict(list)
+    print('get dataset index')
+    for i, target in enumerate(tqdm(dataset.targets)):
+        classwise_idx[target].append(i)
+    print('sample few shot dataset')
+    from random import sample
+    for i in tqdm(range(args.n_cls)):
+        sl = sample(classwise_idx[i], args.n_shot)
+        indices.extend(sl)
+
+    if args.in_dataset == "ImageNet":
+        path = os.path.join(root, 'ImageNet', 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "ImageNet100":
+        path = os.path.join(root, "ImageNet100", 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "ImageNet10":
+        path = os.path.join(root, "ImageNet10", 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "ImageNet20":
+        path = os.path.join(root, "ImageNet20", 'train')
+        dataset = datasets.ImageFolder(path, transform=data_transform)
+    elif args.in_dataset == "car196":
+        path = root
+        dataset = StanfordCars(path, split="train", download=True, transform=data_transform)
+    elif args.in_dataset == "food101":
+        path = root
+        dataset = Food101(path, split="train", download=True, transform=data_transform)
+    elif args.in_dataset == "pet37":
+        path = root
+        dataset = OxfordIIITPet(path, split="trainval", download=True, transform=data_transform)
+    elif args.in_dataset == "bird200":
+        path = root
+        dataset = Cub2011(path, train=True, transform=data_transform)
+    elif args.in_dataset == "cifar10":
+        path = root
+        dataset = CIFAR10(path, train=True, transform=data_transform)
+    elif args.in_dataset == "cifar100":
+        path = root
+        dataset = CIFAR100(path, train=True, transform=data_transform)
+    dataset = torch.utils.data.Subset(dataset, indices)
+    few_shot_loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=shuffle, **kwargs)
+
+    # from torch.utils.data.distributed import DistributedSampler
+    # sampler = DistributedSampler(dataset)
+    # few_shot_loader = torch.utils.data.DataLoader(dataset, sampler=sampler,
+    #                                               batch_size=args.batch_size,
+    #                                               shuffle=False, **kwargs)
+
+    return few_shot_loader

ID-like-train-change-bg/utils/file_ops.py

@@ -0,0 +1,68 @@
+import os
+import shutil
+import numpy as np
+import logging
+import pandas as pd
+
+
+def save_scores(args, scores, dataset_name):
+    with open(os.path.join(args.log_directory, f'{dataset_name}_scores.npy'), 'wb') as f:
+        np.save(f, scores)
+
+
+def load_scores(args, dataset_name):
+    with open(os.path.join(args.log_directory, f'{dataset_name}_scores.npy'), 'rb') as f:
+        scores = np.load(f)
+    return scores
+
+
+def setup_log(args):
+    log = logging.getLogger(__name__)
+    formatter = logging.Formatter('%(asctime)s : %(message)s')
+    fileHandler = logging.FileHandler(os.path.join(args.log_directory, "ood_eval_info.log"), mode='w')
+    fileHandler.setFormatter(formatter)
+    streamHandler = logging.StreamHandler()
+    streamHandler.setFormatter(formatter)
+    log.setLevel(logging.DEBUG)
+    log.addHandler(fileHandler)
+    log.addHandler(streamHandler)
+    # log.debug(f"#########{args.name}############")
+    return log
+
+
+def save_as_dataframe(args, out_datasets, fpr_list, auroc_list, aupr_list, acc_in):
+    fpr_list = [float('{:.2f}'.format(100 * fpr)) for fpr in fpr_list]
+    auroc_list = [float('{:.2f}'.format(100 * auroc)) for auroc in auroc_list]
+    aupr_list = [float('{:.2f}'.format(100 * aupr)) for aupr in aupr_list]
+    acc_in_list = [float('{:.2f}'.format(acc_in[0]))]*len(aupr_list)
+    import pandas as pd
+    data = {k: v for k, v in zip(out_datasets, zip(fpr_list, auroc_list, aupr_list, acc_in_list))}
+    data['AVG'] = [np.mean(fpr_list), np.mean(auroc_list), np.mean(aupr_list), np.mean(acc_in_list)]
+    data['AVG'] = [float('{:.2f}'.format(metric)) for metric in data['AVG']]
+    # Specify orient='index' to create the DataFrame using dictionary keys as rows
+    df = pd.DataFrame.from_dict(data, orient='index', columns=['FPR95', 'AUROC', 'AUPR', 'ACC_IN'])
+    df.to_csv(os.path.join(args.log_directory, f'result.csv'))
+
+
+def create_ImageNet_subset(src, dst, target_dirs):
+    assert (os.path.exists(src))
+    if not os.path.exists(dst):
+        os.makedirs(dst)
+    types = ['train', 'val']
+    for type in types:
+        for dir_name in os.listdir(os.path.join(src, type)):
+            if dir_name in target_dirs:
+                shutil.copytree(os.path.join(src, type, dir_name), os.path.join(dst, type, dir_name))
+
+
+def prepare_dataframe(captions_dir='gen_captions', dataset_name='imagenet_val', multiple=False):
+    # load caption file
+    captions_path = os.path.join(captions_dir, f'{dataset_name}_captions.tsv')
+    df = pd.read_csv(f"{captions_path}", sep='\t')
+    df.columns = ["image_id", "caption", "cls"]
+    if multiple:  # in case a single img has multiple captions
+        x = list(set(df['image_id'].values))
+        image_ids = np.arange(0, len(x))
+        train_images = [x[i] for i in image_ids]
+        df = df[df["image_id"].isin(train_images)].reset_index(drop=True)
+    return df

ID-like-train-change-bg/utils/id_like.py

@@ -0,0 +1,184 @@
+import os
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from tqdm import tqdm
+from utils.id_like_utils import ClipPromptLearner
+from utils.id_like_loss import get_loss
+from utils.common import AverageMeter, accuracy
+from utils import imagenet_templates
+from clip import load, tokenize
+from clip.simple_tokenizer import SimpleTokenizer as _Tokenizer
+_tokenizer = _Tokenizer()
+import config
+
+
+def select_in_out(args, image_features, sim):
+
+    idx_in = torch.topk(sim, dim=0, k=args.n_selection)[1].squeeze()
+    image_features_crop_in_temp = torch.index_select(image_features, index=idx_in, dim=0)
+    idx_out = torch.topk(-sim, dim=0, k=args.n_selection)[1].squeeze()
+    image_features_crop_out_temp = torch.index_select(image_features, index=idx_out, dim=0)
+    image_features_in_temp = image_features_crop_in_temp
+    image_features_out_temp = image_features_crop_out_temp
+
+    return image_features_in_temp, image_features_out_temp
+
+
+def get_in_out(args, clip, model, labels, images, targets):
+
+    image_features_in = []
+    image_features_out = []
+    targets_in = []
+    targets_out = []
+    with torch.no_grad():
+
+        for image_idx, (image, target) in enumerate(zip(images, targets)):
+            label = labels[target.item()]
+            # openai_imagenet_template = imagenet_templates.openai_imagenet_template
+            openai_imagenet_template = [lambda c: f'a photo of a {c}.']
+            select_prompts_in = [func(label) for func in openai_imagenet_template]
+            text_inputs = tokenize(select_prompts_in).cuda()
+            select_prompts_in = clip.encode_text(text_inputs)
+            select_prompts_in /= select_prompts_in.norm(dim=-1, keepdim=True)
+
+            image = image.cuda()
+            target = target.cuda()
+            image_features = model.get_image_features(image)
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+
+            sim = image_features @ select_prompts_in.t()
+            sim = torch.max(sim, dim=1, keepdim=True)[0]
+
+            image_features_in_temp, image_features_out_temp = select_in_out(args, image_features, sim)
+            image_features_in.append(image_features_in_temp)
+            image_features_out.append(image_features_out_temp)
+
+            # create in target
+            targets_in_temp = torch.tile(target, dims=(image_features_in_temp.size(0),))
+            targets_in.append(targets_in_temp)
+
+            # create out target
+            # no use
+            prompt_features = model.get_text_features()
+            prompt_features = prompt_features / prompt_features.norm(dim=-1, keepdim=True)
+            prompt_features_out = prompt_features[args.n_cls:, ...]
+            logit_out_temp = image_features_out_temp @ prompt_features_out.t()
+            targets_out_temp = torch.max(logit_out_temp, dim=1)[1] + args.n_cls
+            targets_out.append(targets_out_temp)
+
+        image_features_in = torch.cat(image_features_in, dim=0)
+        image_features_out = torch.cat(image_features_out, dim=0)
+        targets_in = torch.cat(targets_in, dim=0).cuda()
+        targets_out = torch.cat(targets_out, dim=0).cuda()
+    return image_features_in, image_features_out, targets_in, targets_out
+
+
+def get_prompts(args, loader, labels, ex_labels):
+    model = ClipPromptLearner(args,
+                              classnames=labels, ex_classnames=ex_labels, arch=args.CLIP_ckpt, device='cuda',
+                              n_ctx=config.n_ctx, ctx_init=config.ctx_init,
+                              ctx_position=config.ctx_position, learned_cls=config.learned_cls,
+                              n_ex_ctx=config.n_ex_ctx, ex_ctx_init=config.ex_ctx_init,
+                              ex_ctx_position=config.ex_ctx_position, ex_learned_cls=config.ex_learned_cls)
+
+    loss_meter = AverageMeter()
+    optimizer = torch.optim.AdamW([{'params': model.prompt_learner.parameters()},
+                                   {'params': model.ex_prompt_learner.parameters()}], args.lr)
+
+    clip, _, _ = load(args.CLIP_ckpt, device='cuda', download_root=config.DOWNLOAD_ROOT)
+
+    for epoch in range(args.n_epoch):
+
+        tqdm.write(f'Train epoch:{epoch + 1}/{args.n_epoch}')
+        for batch_idx, (images, targets) in enumerate(tqdm(loader)):
+            image_features_in, image_features_out, targets_in, targets_out = \
+                get_in_out(args, clip, model, labels, images, targets)
+
+            # train
+            # get prompts
+            logit_scale = model.logit_scale.exp()
+            prompt_features = model.get_text_features()
+            prompt_features = prompt_features / prompt_features.norm(dim=-1, keepdim=True)
+            loss, loss_str = get_loss(args, prompt_features,
+                                      image_features_in, image_features_out,
+                                      targets_in, targets_out, logit_scale)
+
+            # update
+            loss.backward()
+            optimizer.step()
+            optimizer.zero_grad()
+
+            loss_meter.update(loss.detach().cpu().item())
+            tqdm.write(f'Train epoch:{epoch + 1}/{args.n_epoch}\t'
+                       f'Loss_avg:{loss_meter.avg:.6f}\t' + loss_str)
+
+        if epoch+1 == args.n_epoch:
+            model_save_dir = args.log_directory
+            os.makedirs(model_save_dir, exist_ok=True)
+            model_checkpoint_save_path = os.path.join(model_save_dir, 'model_checkpoint.pth')
+            model_checkpoint = {
+                'prompt_learner_state_dict': model.prompt_learner.state_dict(),
+                'ex_prompt_learner_state_dict': model.ex_prompt_learner.state_dict(),
+            }
+            torch.save(model_checkpoint, model_checkpoint_save_path)
+
+    return model
+
+
+def get_result(args, model, loader, labels, ex_labels, if_acc=False):
+    tqdm_object = tqdm(loader, total=len(loader))
+    outputs = []
+    all_targets = []
+    result = {
+        'scores': None,
+        'acc': None,
+    }
+
+    with torch.no_grad():
+        text_features = model.get_text_features()
+        text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+
+    for batch_idx, (images, targets) in enumerate(tqdm_object):
+        with torch.no_grad():
+            images = images.cuda()
+            targets = targets.long().cuda()
+            image_features = model.image_encoder(images)
+            image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+            logit_scale = model.logit_scale.exp()
+            output = logit_scale * image_features @ text_features.t()
+
+            output = output.detach().cpu()
+            outputs.append(output)
+            all_targets.append(targets)
+    outputs = torch.cat(outputs, dim=0)
+    all_targets = torch.cat(all_targets, dim=0)
+
+    # scores
+
+    outputs_softmax = F.softmax(outputs, dim=1)
+    scores = torch.sum(outputs_softmax[:, args.n_cls:], dim=1).detach().cpu().squeeze().numpy() - 1
+
+    result['scores'] = scores
+    # acc
+    if if_acc:
+        res = accuracy(outputs[:, :args.n_cls], all_targets.detach().cpu())
+        result['acc'] = [acc.item() for acc in res]
+    return result
+
+
+def load_model(args, labels, ex_labels):
+    model = ClipPromptLearner(args,
+                              classnames=labels, ex_classnames=ex_labels, arch=args.CLIP_ckpt, device='cuda',
+                              n_ctx=config.n_ctx, ctx_init=config.ctx_init,
+                              ctx_position=config.ctx_position, learned_cls=config.learned_cls,
+                              n_ex_ctx=config.n_ex_ctx, ex_ctx_init=config.ex_ctx_init,
+                              ex_ctx_position=config.ex_ctx_position, ex_learned_cls=config.ex_learned_cls)
+    model_checkpoint_save_path = os.path.join(args.log_directory, 'model_checkpoint.pth')
+    model_checkpoint = torch.load(model_checkpoint_save_path)
+    model.prompt_learner.load_state_dict(model_checkpoint['prompt_learner_state_dict'])
+    model.ex_prompt_learner.load_state_dict(model_checkpoint['ex_prompt_learner_state_dict'])
+    return model.cuda()
+

ID-like-train-change-bg/utils/id_like_loss.py

@@ -0,0 +1,52 @@
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from tqdm import tqdm
+from utils.id_like_utils import ClipPromptLearner
+from utils.common import AverageMeter, accuracy
+import config
+
+
+def get_loss(args, prompt_features, image_features_in, image_features_out, targets_in, targets_out, logit_scale):
+    prompt_features_in = prompt_features[:args.n_cls, ...]
+    prompt_features_out = prompt_features[args.n_cls:, ...]
+    # loss_in
+    logit_in = logit_scale * image_features_in @ prompt_features.t()
+    # logit_in = logit_scale * image_features_in @ prompt_features_in.t()
+    loss_in = F.cross_entropy(logit_in, targets_in)
+
+    # loss_out
+    logit_out = logit_scale * image_features_out @ prompt_features.t()
+
+    # logit_out_softmax_probs = F.softmax(logit_out, dim=1)
+    # flag_out = torch.cat([torch.LongTensor([0] * args.n_cls + [1] * args.n_ex_prompts)], dim=0).cuda()
+    # logit_out_softmax_probs_in = torch.sum(logit_out_softmax_probs * (1 - flag_out), dim=1)
+    # logit_out_softmax_probs_in_log = -torch.log(1.-logit_out_softmax_probs_in)
+    # loss_out = torch.mean(logit_out_softmax_probs_in_log)
+
+    logit_out_softmax_probs = F.softmax(logit_out, dim=1)
+    flag_out = torch.cat([torch.LongTensor([0] * args.n_cls + [1] * args.n_ex_prompts)], dim=0).cuda()
+    logit_out_softmax_probs_in = torch.sum(logit_out_softmax_probs * (1 - flag_out), dim=1)
+    logit_out_softmax_probs_in_log = torch.log(logit_out_softmax_probs_in + 1e-16)
+    loss_out = torch.mean(logit_out_softmax_probs_in_log)
+
+    # loss_diff
+    loss_diff = torch.FloatTensor([0.]).cuda()
+    for p in range(prompt_features_out.size(0) - 1):
+        for q in range(p + 1, prompt_features_out.size(0)):
+            loss_diff += F.cosine_embedding_loss(input1=prompt_features_out[p].unsqueeze(dim=0),
+                                                 input2=prompt_features_out[q].unsqueeze(dim=0),
+                                                 target=torch.LongTensor([-1]).cuda())
+    if prompt_features_out.size(0) > 1:
+        loss_diff /= (prompt_features_out.size(0) * (prompt_features_out.size(0) - 1) / 2.)
+
+    # loss
+    loss = loss_in * args.lam_in + loss_out * args.lam_out + loss_diff * args.lam_diff
+    loss_str = f'Loss_now:{loss.detach().cpu().item():.6f}\t' \
+               f'Loss_in:{loss_in.detach().cpu().item():.6f}\t' \
+               f'Loss_out:{loss_out.detach().cpu().item():.6f}\t' \
+               f'Loss_diff:{loss_diff.detach().cpu().item():.6f}'
+
+    return loss, loss_str

ID-like-train-change-bg/utils/id_like_utils.py

@@ -0,0 +1,298 @@
+import math
+from typing import List, Tuple
+import os
+import json
+import time
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.cuda.amp import autocast, GradScaler
+import numpy as np
+
+# from transformers import CLIPTokenizer
+from tqdm import tqdm
+
+from clip import load, tokenize
+from clip.simple_tokenizer import SimpleTokenizer as _Tokenizer
+
+import config
+
+_tokenizer = _Tokenizer()
+DOWNLOAD_ROOT = config.DOWNLOAD_ROOT
+
+
+class TextEncoder(nn.Module):
+    def __init__(self, clip_model):
+        super().__init__()
+        self.transformer = clip_model.transformer
+        self.positional_embedding = clip_model.positional_embedding
+        self.ln_final = clip_model.ln_final
+        self.text_projection = clip_model.text_projection
+        self.dtype = clip_model.dtype
+
+    def forward(self, prompts, tokenized_prompts):
+        x = prompts + self.positional_embedding.type(self.dtype)
+        x = x.permute(1, 0, 2)  # NLD -> LND
+        x = self.transformer(x)
+        x = x.permute(1, 0, 2)  # LND -> NLD
+        x = self.ln_final(x).type(self.dtype)
+
+        # x.shape = [batch_size, n_ctx, transformer.width]
+        # take features from the eot embedding (eot_token is the highest number in each sequence)
+        x = x[torch.arange(x.shape[0]), tokenized_prompts.argmax(dim=-1)] @ self.text_projection
+
+        return x
+
+
+class PromptLearner(nn.Module):
+    def __init__(self, clip_model, classnames, n_ctx=16, ctx_init=None, ctx_position='end',
+                 learned_cls=False):
+        super().__init__()
+        n_cls = len(classnames)
+        self.learned_cls = learned_cls
+        dtype = clip_model.dtype
+        self.dtype = dtype
+        self.device = clip_model.visual.conv1.weight.device
+        ctx_dim = clip_model.ln_final.weight.shape[0]
+        self.ctx_dim = ctx_dim
+
+        if ctx_init:
+            print("Initializing the contect with given words: [{}]".format(ctx_init))
+            ctx_init = ctx_init.replace("_", " ")
+            if '[CLS]' in ctx_init:
+                ctx_list = ctx_init.split(" ")
+                split_idx = ctx_list.index("[CLS]")
+                ctx_init = ctx_init.replace("[CLS] ", "")
+                ctx_position = "middle"
+            else:
+                split_idx = None
+            self.split_idx = split_idx
+            n_ctx = len(ctx_init.split(" "))
+            prompt = tokenize(ctx_init).to(self.device)
+            with torch.no_grad():
+                embedding = clip_model.token_embedding(prompt).type(dtype)
+            ctx_vectors = embedding[0, 1: 1 + n_ctx, :]
+            prompt_prefix = ctx_init
+        else:
+            ctx_vectors = torch.empty(n_ctx, ctx_dim, dtype=dtype)
+            nn.init.normal_(ctx_vectors, std=0.02)
+            prompt_prefix = " ".join(["X"] * n_ctx)
+
+        self.prompt_prefix = prompt_prefix
+
+        print(f'Initial context: "{prompt_prefix}"')
+        print(f"Number of context words (tokens): {n_ctx}")
+
+        self.ctx_init_state = ctx_vectors.detach().clone()
+        self.ctx = nn.Parameter(ctx_vectors)  # to be optimized
+
+        if not self.learned_cls:
+            classnames = [name.replace("_", " ") for name in classnames]
+            name_lens = [len(_tokenizer.encode(name)) for name in classnames]
+            prompts = [prompt_prefix + " " + name + "." for name in classnames]
+        else:
+            cls_vectors = torch.empty(n_cls, 1, ctx_dim, dtype=dtype)  # assume each learnable cls_token is only 1 word
+            nn.init.normal_(cls_vectors, std=0.02)
+            cls_token = "X"
+            name_lens = [1 for _ in classnames]
+            prompts = [prompt_prefix + " " + cls_token + "." for _ in classnames]
+
+            self.cls_init_state = cls_vectors.detach().clone()
+            self.cls = nn.Parameter(cls_vectors)
+
+        tokenized_prompts = torch.cat([tokenize(p) for p in prompts]).to(self.device)
+        with torch.no_grad():
+            embedding = clip_model.token_embedding(tokenized_prompts).type(dtype)
+
+        self.register_buffer("token_prefix", embedding[:, :1, :])
+        if self.learned_cls:
+            self.register_buffer("token_suffix", embedding[:, 1 + n_ctx + 1:, :])
+        else:
+            self.register_buffer("token_suffix", embedding[:, 1 + n_ctx:, :])
+
+        self.ctx_init = ctx_init
+        self.tokenized_prompts = tokenized_prompts  # torch.Tensor
+        self.name_lens = name_lens
+        self.class_token_position = ctx_position
+        self.n_cls = n_cls
+        self.n_ctx = n_ctx
+        self.classnames = classnames
+
+    def reset(self):
+        ctx_vectors = self.ctx_init_state
+        self.ctx.copy_(ctx_vectors)
+        if self.learned_cls:
+            cls_vectors = self.cls_init_state
+            self.cls.copy_(cls_vectors)
+
+    def reset_classnames(self, classnames, arch):
+        self.n_cls = len(classnames)
+        if not self.learned_cls:
+            classnames = [name.replace("_", " ") for name in classnames]
+            name_lens = [len(_tokenizer.encode(name)) for name in classnames]
+            prompts = [self.prompt_prefix + " " + name + "." for name in classnames]
+        else:
+            cls_vectors = torch.empty(self.n_cls, 1, self.ctx_dim, dtype=self.dtype)
+            nn.init.normal_(cls_vectors, std=0.02)
+            cls_token = "X"
+            name_lens = [1 for _ in classnames]
+            prompts = [self.prompt_prefix + " " + cls_token + "." for _ in classnames]
+            # TODO: re-init the cls parameters
+            # self.cls = nn.Parameter(cls_vectors) # to be optimized
+            self.cls_init_state = cls_vectors.detach().clone()
+        tokenized_prompts = torch.cat([tokenize(p) for p in prompts]).to(self.device)
+
+        clip, _, _ = load(arch, device=self.device, download_root=DOWNLOAD_ROOT)
+
+        with torch.no_grad():
+            embedding = clip.token_embedding(tokenized_prompts).type(self.dtype)
+
+        self.token_prefix = embedding[:, :1, :]
+        self.token_suffix = embedding[:, 1 + self.n_ctx:, :]  # CLS, EOS
+
+        self.name_lens = name_lens
+        self.tokenized_prompts = tokenized_prompts
+        self.classnames = classnames
+
+    def forward(self, init=None):
+        if init is not None:
+            ctx = init
+        else:
+            ctx = self.ctx
+        if ctx.dim() == 2:
+            ctx = ctx.unsqueeze(0).expand(self.n_cls, -1, -1)
+        elif not ctx.size()[0] == self.n_cls:
+            ctx = ctx.unsqueeze(1).expand(-1, self.n_cls, -1, -1)
+
+        prefix = self.token_prefix
+        suffix = self.token_suffix
+
+        if self.learned_cls:
+            assert self.class_token_position == "end"
+        if self.class_token_position == "end":
+            if self.learned_cls:
+                cls = self.cls
+                prompts = torch.cat(
+                    [
+                        prefix,  # (n_cls, 1, dim)
+                        ctx.to(self.device),  # (n_cls, n_ctx, dim)
+                        cls.to(self.device),  # (n_cls, 1, dim)
+                        suffix,  # (n_cls, *, dim)
+                    ],
+                    dim=-2,
+                )
+            else:
+                prompts = torch.cat(
+                    [
+                        prefix,  # (n_cls, 1, dim)
+                        ctx.to(self.device),  # (n_cls, n_ctx, dim)
+                        suffix,  # (n_cls, *, dim)
+                    ],
+                    dim=-2,
+                )
+        elif self.class_token_position == "middle":
+            # TODO: to work with a batch of prompts
+            if self.split_idx is not None:
+                half_n_ctx = self.split_idx  # split the ctx at the position of [CLS] in `ctx_init`
+            else:
+                half_n_ctx = self.n_ctx // 2
+            prompts = []
+            for i in range(self.n_cls):
+                name_len = self.name_lens[i]
+                prefix_i = prefix[i: i + 1, :, :]
+                class_i = suffix[i: i + 1, :name_len, :]
+                suffix_i = suffix[i: i + 1, name_len:, :]
+                ctx_i_half1 = ctx[i: i + 1, :half_n_ctx, :]
+                ctx_i_half2 = ctx[i: i + 1, half_n_ctx:, :]
+                prompt = torch.cat(
+                    [
+                        prefix_i,  # (1, 1, dim)
+                        ctx_i_half1.to(self.device),  # (1, n_ctx//2, dim)
+                        class_i.to(self.device),  # (1, name_len, dim)
+                        ctx_i_half2.to(self.device),  # (1, n_ctx//2, dim)
+                        suffix_i,  # (1, *, dim)
+                    ],
+                    dim=1,
+                )
+                prompts.append(prompt)
+            prompts = torch.cat(prompts, dim=0)
+
+        elif self.class_token_position == "front":
+            prompts = []
+            for i in range(self.n_cls):
+                name_len = self.name_lens[i]
+                prefix_i = prefix[i: i + 1, :, :]
+                class_i = suffix[i: i + 1, :name_len, :]
+                suffix_i = suffix[i: i + 1, name_len:, :]
+                ctx_i = ctx[i: i + 1, :, :]
+                prompt = torch.cat(
+                    [
+                        prefix_i,  # (1, 1, dim)
+                        class_i.to(self.device),  # (1, name_len, dim)
+                        ctx_i.to(self.device),  # (1, n_ctx, dim)
+                        suffix_i,  # (1, *, dim)
+                    ],
+                    dim=1,
+                )
+                prompts.append(prompt)
+            prompts = torch.cat(prompts, dim=0)
+
+        else:
+            raise ValueError
+
+        return prompts
+
+
+class ClipPromptLearner(nn.Module):
+    def __init__(self, args,
+                 classnames, ex_classnames,
+                 criterion='cosine', arch="ViT-B/16", device='cuda',
+                 n_ctx=16, ctx_init=None, ctx_position='end', learned_cls=False,
+                 n_ex_ctx=16, ex_ctx_init=None, ex_ctx_position='end', ex_learned_cls=True):
+        super(ClipPromptLearner, self).__init__()
+        clip, _, _ = load(arch, device=device, download_root=DOWNLOAD_ROOT)
+
+        self.image_encoder = clip.visual
+
+        self.text_encoder = TextEncoder(clip)
+        self.text_encoder = nn.parallel.DataParallel(self.text_encoder).to(torch.device("cuda"))  # for mutil GPU
+        self.logit_scale = clip.logit_scale.data
+        # prompt
+        self.prompt_learner = PromptLearner(clip, classnames, n_ctx,
+                                            ctx_init, ctx_position, learned_cls=learned_cls)
+        self.ex_prompt_learner = PromptLearner(clip, ex_classnames, n_ex_ctx,
+                                               ex_ctx_init, ex_ctx_position, learned_cls=ex_learned_cls)
+
+        self.criterion = criterion
+
+    @property
+    def dtype(self):
+        return self.image_encoder.conv1.weight.dtype
+
+    def get_text_features(self):
+        prompts = torch.cat((self.prompt_learner(),
+                             self.ex_prompt_learner()), dim=0)
+        tokenized_prompts = torch.cat((self.prompt_learner.tokenized_prompts,
+                                       self.ex_prompt_learner.tokenized_prompts), dim=0)
+        prompts = prompts.cuda(non_blocking=True)  # for mutil GPU
+        tokenized_prompts = tokenized_prompts.cuda(non_blocking=True)  # for mutil GPU
+        text_features = []
+        t_features = self.text_encoder(prompts, tokenized_prompts)
+        return t_features
+
+    def get_image_features(self, image):
+        image_features = self.image_encoder(image.type(self.dtype))
+        return image_features
+
+    def forward(self, image):
+        with torch.no_grad():
+            image_features = self.image_encoder(image.type(self.dtype))
+
+        text_features = self.get_text_features()
+        image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+
+        logit_scale = self.logit_scale.exp()
+        logits = logit_scale * image_features @ text_features.t()
+
+        return logits