Upload 5 files

config_kwargs.json

@@ -0,0 +1,79 @@
+{
+  "monitor_metric": "eval_MRR@NM_0,eval_MRR@NM_1",
+  "trainer_kwargs": {
+    "class_name": "CROSS_MODAL_IR"
+  },
+  "trainee_kwargs": {
+    "class_name": "CLIP_Encoder",
+    "freeze_prefixes": [],
+    "use_attention": false,
+    "symmetric_CL": false,
+    "weighted_loss": false,
+    "image_type": false,
+    "mlm_type": false,
+    "tie_weights": false,
+    "loss": {
+      "class_name": "NLLLoss",
+      "align_uniform": true
+    }
+  },
+  "data_module_kwargs": {
+    "class_name": "cross_modal_kb_DataModule_article",
+    "data_processor": {
+      "class_name": "kb_viquae_data_processor",
+      "dataset_path": "../../all_data/kb_dataset_new/",
+      "kb_path": "../../all_data/passages/",
+      "entity_kb_path": "../../all_data/kb/",
+      "validation_dataset_path": "../../all_data/"
+    },
+    "input_key": "input",
+    "passage_key": "passage",
+    "relevant_indices_key": "BM25_provenance_indices",
+    "irrelevant_indices_key": "BM25_irrelevant_indices",
+    "use_image": true,
+    "add_positives": false,
+    "cross_modal_viquae_valid": true,
+    "use_mep": false,
+    "use_mlm": false,
+    "use_CL": true,
+    "augmented": false,
+    "random_mask": false,
+    "dataloader_kwargs": {
+      "num_workers": 6,
+      "prefetch_factor": 2
+    },
+    "tokenizer_kwargs": {
+      "class_name": "CLIPTokenizer",
+      "pretrained_model_name_or_path": "clip-vit-base-patch32_tokenizer"
+    },
+    "tokenization_kwargs": {
+      "max_length": 77,
+      "padding": "longest"
+    },
+    "image_processor_kwargs": {
+      "class_name": "ImageFormatter",
+      "feature_extractor_kwargs": {
+        "class_name": "CLIPFeatureExtractor",
+        "pretrained_model_name_or_path": "clip-vit-base-patch32_FE"
+      }
+    }
+  },
+  "text_encoder_kwargs": {
+    "class_name": "CLIP_Text_Encoder",
+    "checkpoint_name": "text_encoder",
+    "base_encoder_kwargs": {
+      "class_name": "CLIPModel",
+      "pretrained_model_name_or_path": "/gpfswork/rech/der/ufc61ee/my_transformers_cache/clip-vit-base-patch32"
+    },
+    "inference_path": "saved_models/CLIP_article/text_encoder"
+  },
+  "image_encoder_kwargs": {
+    "class_name": "CLIP_Image_Encoder",
+    "checkpoint_name": "image_encoder",
+    "base_encoder_kwargs": {
+      "class_name": "CLIPModel",
+      "pretrained_model_name_or_path": "/gpfswork/rech/der/ufc61ee/my_transformers_cache/clip-vit-base-patch32"
+    },
+    "inference_path": "saved_models/CLIP_article/image_encoder"
+  }
+}

experiment_params.json

@@ -0,0 +1,153 @@
+{
+  "subparser": null,
+  "max_seq_length": 256,
+  "question_max_seq_length": 256,
+  "RC": false,
+  "coco": false,
+  "flickr": false,
+  "aokvqa": false,
+  "grad_check": true,
+  "MJL": false,
+  "debug_run_time": false,
+  "only_neighbors": false,
+  "full_kb": false,
+  "image_eval": false,
+  "embedding_column": "my_DPR_few_shot",
+  "output_ds": null,
+  "output_kb": null,
+  "embed": false,
+  "data_path": null,
+  "train_batch_size": 1000,
+  "eval_batch_size": 1000,
+  "viquae_dev_batch_size": 1000,
+  "transformer_model_name": "bert-base-uncased",
+  "gradient_accumulation_steps": 1,
+  "warmup_proportion": 0.1,
+  "weight_decay": 0.01,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "warmup_steps": 4,
+  "adam_epsilon": 1e-08,
+  "num_train_epochs": 10,
+  "learning_rate": 2e-06,
+  "dropout": 0.5,
+  "bert_hidden_size": 768,
+  "MM": null,
+  "use_graph_P": null,
+  "use_graph_Q": null,
+  "syntactic_P": null,
+  "syntactic_Q": null,
+  "NS": null,
+  "IC_Q": false,
+  "IC_P": false,
+  "gcn_lr": 2e-05,
+  "head_lr": 2e-06,
+  "func": "cat",
+  "graph_pooling": "mean",
+  "max_nbr_nodes": 2,
+  "num_neib": 1,
+  "neib_depth": 1,
+  "undirect": false,
+  "node_hidden_size": 128,
+  "max_num_relations": null,
+  "max_num_syntactic_relations": null,
+  "map_wikidataRelCode_to_idx": null,
+  "graph_layout": "fr",
+  "residual": false,
+  "num_gcn": null,
+  "node_alias_max_len": 32,
+  "attention": false,
+  "num_paths": 1,
+  "kg_embed": false,
+  "graph_only": false,
+  "freeze": false,
+  "sanity_run": false,
+  "enhanced_question": false,
+  "use_entity_type": false,
+  "filter_edges": false,
+  "draw": false,
+  "layer_norm": false,
+  "filters_3": 768,
+  "tensor_neurons": 16,
+  "output_dir": "saved_models/CLIP_article",
+  "main_dir": "/gpfsdswork/projects/rech/der/ufc61ee/MEERQAT/ViQuAE/meerqat/GP-VQA",
+  "experiment_dir": "None",
+  "experiment_name": "CLIP_article",
+  "search": null,
+  "xlnet": false,
+  "tune_loss": false,
+  "tune_dev": true,
+  "tune_valid_loss": false,
+  "sanity_val_steps": 0,
+  "cpu": false,
+  "keep_in_memory": false,
+  "N": 0,
+  "nbr_workers": 1,
+  "num_proc": 1,
+  "local_cache": "/gpfswork/rech/der/ufc61ee/my_transformers_cache",
+  "IMAGE_PATH": null,
+  "transformer_path": null,
+  "resume_from": null,
+  "checkpoint": null,
+  "linear_MEP_checkpoint": null,
+  "linear_MLM_checkpoint": null,
+  "config": "experiments/ir/viquae_kb_full/clip/config_clip_article.json",
+  "context_checkpoint": null,
+  "grad_accum": 1,
+  "cls_token": "[CLS]",
+  "sep_token": "[SEP]",
+  "pad_token": "[PAD]",
+  "cls_token_at_end": false,
+  "mask_padding_with_zero": true,
+  "pad_on_left": false,
+  "sequence_a_segment_id": 0,
+  "pad_token_segment_id": 0,
+  "cls_token_segment_id": 0,
+  "pad_token_label_id": 0,
+  "pad_token_id": 0,
+  "past_index": -1,
+  "world_size": 1,
+  "use_lstm": false,
+  "use_entity": false,
+  "use_question_graph": false,
+  "use_question_objects": false,
+  "EFeat": null,
+  "NFeat": null,
+  "ltn": false,
+  "use_kelm": false,
+  "unshared": false,
+  "mlm": false,
+  "test": false,
+  "split_dpr": false,
+  "rename_model": false,
+  "mask_rate": 0.15,
+  "pretrained_triviaq": false,
+  "error_analysis": false,
+  "debug": false,
+  "stop_debug": false,
+  "entity_linking": false,
+  "entity_linking_split": false,
+  "build_entity_paths": null,
+  "object_detection": null,
+  "update_wikidata_ids": null,
+  "update_None_ids": null,
+  "dependency_parsing": null,
+  "image_captionning": null,
+  "build_graph": false,
+  "from_bert": false,
+  "fast": false,
+  "get_examples": false,
+  "get_statistics": null,
+  "word_vocab_size": 0,
+  "char_vocab_size": 0,
+  "max_word_len": 30,
+  "device": null,
+  "embedding_matrix": null,
+  "w2v_file": "word_vector_200d.vec",
+  "word_emb_dim": 200,
+  "char_lstm": false,
+  "char_cnn": false,
+  "seed": 32,
+  "tune_batch_size": false,
+  "model_names": null
+}

model-epoch=05-eval_MRR@NM_0=0.4627_sd=32.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c404255c2617efb880beb7494483787beae15d16948ff7e3c89746a7ca77eac4
+size 1815837916

results.txt

@@ -0,0 +1,2 @@
+-------- dev--------- 
+[0.46274155378341675]

validation_metrics.txt

@@ -0,0 +1,10 @@
+[{'MRR@NM': 0.3684380584589463, 'hits@1': 0.27678830164285234, 'avg_L_align': tensor(1.4444), 'avg_L_uniform': tensor(-2.1290)}, {'MRR@NM': 0.27367666416643144, 'hits@1': 0.18372319688109162}]
+[{'MRR@NM': 0.4243174365696836, 'hits@1': 0.3220614787900403, 'avg_L_align': tensor(1.4181), 'avg_L_uniform': tensor(-1.9166)}, {'MRR@NM': 0.32139479322781445, 'hits@1': 0.2309941520467836}]
+[{'MRR@NM': 0.4421935825064815, 'hits@1': 0.3378881433762065, 'avg_L_align': tensor(1.4131), 'avg_L_uniform': tensor(-1.9543)}, {'MRR@NM': 0.32120630824863855, 'hits@1': 0.22514619883040934}]
+[{'MRR@NM': 0.4545746362170726, 'hits@1': 0.3494795033548071, 'avg_L_align': tensor(1.4129), 'avg_L_uniform': tensor(-2.0277)}, {'MRR@NM': 0.3198178910308339, 'hits@1': 0.22660818713450293}]
+[{'MRR@NM': 0.46088537715427935, 'hits@1': 0.35416062950001115, 'avg_L_align': tensor(1.4164), 'avg_L_uniform': tensor(-2.1373)}, {'MRR@NM': 0.30702571062989, 'hits@1': 0.2149122807017544}]
+[{'MRR@NM': 0.46274156319615783, 'hits@1': 0.35592162457368315, 'avg_L_align': tensor(1.4235), 'avg_L_uniform': tensor(-2.2792)}, {'MRR@NM': 0.3126044940448983, 'hits@1': 0.2202729044834308}]
+[{'MRR@NM': 0.46134979593148656, 'hits@1': 0.3537593901161365, 'avg_L_align': tensor(1.4337), 'avg_L_uniform': tensor(-2.4167)}, {'MRR@NM': 0.3118104677421376, 'hits@1': 0.21978557504873295}]
+[{'MRR@NM': 0.45893284627342656, 'hits@1': 0.3517086110429995, 'avg_L_align': tensor(1.4450), 'avg_L_uniform': tensor(-2.5418)}, {'MRR@NM': 0.3164403163511111, 'hits@1': 0.22319688109161792}]
+[{'MRR@NM': 0.456976254547464, 'hits@1': 0.3501705267381467, 'avg_L_align': tensor(1.4546), 'avg_L_uniform': tensor(-2.6405)}, {'MRR@NM': 0.3106033856166459, 'hits@1': 0.22076023391812866}]
+[{'MRR@NM': 0.45557986387468385, 'hits@1': 0.3493234658166336, 'avg_L_align': tensor(1.4613), 'avg_L_uniform': tensor(-2.7086)}, {'MRR@NM': 0.31200438202869274, 'hits@1': 0.2168615984405458}]