Revert "astroBERT model ner model all_labeled_data_run01 checkpoint-169600"

This reverts commit e0c9e48487cc3c13021c03807116d83a86e44e35.

.ipynb_checkpoints/README-checkpoint.md

@@ -1,3 +0,0 @@
----
-license: mit
----

Tutorials/.ipynb_checkpoints/0_Embeddings-checkpoint.ipynb

@@ -1,295 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "274e6135-2d97-4244-9183-65bcb1d24c80",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Use the trained astroBERT model to generate embedings of text\n",
-    "# to be used for downstream tasks"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "2cc88ed3-6f52-49a2-99c0-344387758ab5",
-   "metadata": {},
-   "source": [
-    "# Tutorial 0: Loading astroBERT to produce text embeddings\n",
-    "This tutorial will show you how to load astroBERT and produce text embeddings that can be used on downstream tasks."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "9e65c041-9d66-4fb1-96b9-4937000da02e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 1 - load models and tokenizer"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "67d99e96-c532-49ef-8542-a48eef818956",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2022-10-20 16:07:24.705905: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
-     ]
-    }
-   ],
-   "source": [
-    "from transformers import AutoTokenizer, AutoModel"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "00e1d48e-9898-44ef-b00e-43e3ab7fed7d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# the model path can either be the name of the Huggingface repository\n",
-    "remote_model_path = 'adsabs/astroBERT'\n",
-    "# or the local path to the directory containing model weight and tokenizer vocab\n",
-    "local_model_path = '../'"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "9bcc6009-6009-463f-a7da-f010c5fae27e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# make sure you load the tokenier with do_lower_case=False\n",
-    "astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,\n",
-    "                                                    use_auth_token=True,\n",
-    "                                                    add_special_tokens=True,\n",
-    "                                                    do_lower_case=False,\n",
-    "                                                   )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "dbd144f0-6038-4917-94b0-aea9da72cac5",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "PreTrainedTokenizerFast(name_or_path='adsabs/astroBERT', vocab_size=30000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'})"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "astroBERT_tokenizer"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "dd9a9257-cbe4-4908-a9f4-8e1431dc375a",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
-      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
-     ]
-    }
-   ],
-   "source": [
-    "# automodels: defaults to BertModel\n",
-    "# it's normal to get warnings as a BertModel will not load the weights used for PreTraining\n",
-    "astroBERT_automodel = AutoModel.from_pretrained(remote_model_path, \n",
-    "                                                use_auth_token=True,\n",
-    "                                               )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "572ddd38-a0dc-4583-a5a6-c4f3b2cb2553",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 2 - make some inference, the outputs are the embeddings"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "32fc0b97-4a2d-42ab-aa83-f5d8b39672b1",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([3, 54])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# list of strings for which we want embeddings\n",
-    "strings = ['The Chandra X-ray Observatory (CXO), previously known as the Advanced X-ray Astrophysics Facility (AXAF), is a Flagship-class space telescope launched aboard the Space Shuttle Columbia during STS-93 by NASA on July 23, 1999.',\n",
-    "           'Independent lines of evidence from Type Ia supernovae and the CMB imply that the universe today is dominated by a mysterious form of energy known as dark energy, which appears to homogeneously permeate all of space.',\n",
-    "           'This work has been developed in the framework of the ‘Darklight’ programme, supported by the European Research Council through an Advanced Research Grant to LG (Project # 291521).'\n",
-    "          ]\n",
-    "\n",
-    "# tokenizer the strings, with padding (needed to process multiple strings efficiently)\n",
-    "inputs = astroBERT_tokenizer(strings, \n",
-    "                             padding=True, \n",
-    "                             return_tensors='pt'\n",
-    "                            )\n",
-    "\n",
-    "# check the shape of the inputs\n",
-    "print(inputs['input_ids'].shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "8b7c9456-573a-48e7-9bc2-839fcc25631d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# pass the inputs through astroBERT\n",
-    "import torch\n",
-    "# no need for gradients, since we are only doing inference\n",
-    "with torch.no_grad():\n",
-    "    output = astroBERT_automodel(**inputs, \n",
-    "                                 output_hidden_states=False\n",
-    "                                )  "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "116de57a-bb31-48d7-9556-64e01a16d56f",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "torch.Size([3, 54, 768])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# BertModel outputs two tensors: last_hidden_state (our embeddings) and pooler_output (to be discarded as it's not meaningful)\n",
-    "# see https://huggingface.co/docs/transformers/model_doc/bert#transformers.BertModel.forward\n",
-    "# embeddings will have shape = (# of strings, size of tokenized strings(padded), 768 (BERT embedding size))\n",
-    "embeddings = output[0]\n",
-    "print(embeddings.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "38e45291-6fd7-48cf-83df-e1cc5c8a699f",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "tensor([[ 0.5546,  0.9121,  0.6550,  ..., -0.1925,  0.7077, -0.2405],\n",
-      "        [ 0.6252,  0.3175,  1.0899,  ...,  0.0576,  0.0529,  0.0603],\n",
-      "        [ 0.1803, -0.4567,  1.2688,  ...,  0.6026, -0.5718, -0.2060],\n",
-      "        ...,\n",
-      "        [-0.4397, -0.5334,  1.1682,  ...,  0.9541,  0.4046, -0.4756],\n",
-      "        [-0.3911,  0.7793,  0.2432,  ...,  0.2268, -1.0489, -1.4864],\n",
-      "        [-0.4529, -0.7346,  0.0675,  ..., -0.3246, -0.2333, -0.6154]])\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(embeddings[0])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "26acf89f-b7fc-4872-ac81-0ee65030b465",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# If you wish to use the hidden states as additional embeddings, you can use output_hidden_states=True\n",
-    "\n",
-    "# no need for gradients, since we are only doing inference\n",
-    "with torch.no_grad():\n",
-    "    output = astroBERT_automodel(**inputs, \n",
-    "                                 output_hidden_states=True\n",
-    "                                )  "
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "a54314e9-5dcb-4c10-b0d2-219a93c7d16e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "13\n",
-      "torch.Size([3, 54, 768])\n"
-     ]
-    }
-   ],
-   "source": [
-    "# This will produce 13 embeddings, one for each hidden layer\n",
-    "embeddings = output[2]\n",
-    "print(len(embeddings))\n",
-    "print(embeddings[0].shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "76765dcb-8035-44b2-a5a3-db181b561095",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

Tutorials/.ipynb_checkpoints/1_Fill-Mask-checkpoint.ipynb

@@ -1,425 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "33df4373-a37b-4fd0-bc67-c297812871e4",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Use the trained astroBERT model with the fill-mask pipeline"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "164ee9bd-27f9-40a4-8461-3ce12fc928b0",
-   "metadata": {},
-   "source": [
-    "# Tutorial 1: using astroBERT with the fill-mask pipeline"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "59429414-f07e-45e5-8825-6fc6a8d26653",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 1 - load models and tokenizer"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "db8ee724-6a2a-4ea5-820e-5e2aa0a0f622",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2022-10-17 21:17:27.369794: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
-     ]
-    }
-   ],
-   "source": [
-    "from transformers import AutoTokenizer, BertForMaskedLM"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "id": "9a98fb63-0793-4684-a202-931cad17c7ca",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# the model path can either be the name of the Huggingface repository\n",
-    "remote_model_path = 'adsabs/astroBERT'\n",
-    "# or the local path to the directory containing model weight and tokenizer vocab\n",
-    "local_model_path = '../'"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "id": "25fedd16-283b-4817-9b19-2a5ff1c5ba88",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# make sure you load the tokenier with do_lower_case=False\n",
-    "astroBERT_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path=remote_model_path,\n",
-    "                                                    use_auth_token=True,\n",
-    "                                                    add_special_tokens=False,\n",
-    "                                                    do_lower_case=False,\n",
-    "                                                   )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "fb10db03-a5f0-44f7-8d41-0285f898a90d",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b6e0bf5ee71b4986a682adb43e994ede",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/666 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
-      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
-     ]
-    }
-   ],
-   "source": [
-    "astroBERT_automodel_for_mlm = BertForMaskedLM.from_pretrained(pretrained_model_name_or_path=remote_model_path, \n",
-    "                                                use_auth_token=True,\n",
-    "                                               )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 7,
-   "id": "e8b9b073-3876-4d0b-b8b2-e46fa25c76f0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# for pipeline to work you have to ensure that the model returns a dict\n",
-    "astroBERT_automodel_for_mlm.config.return_dict=True"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "id": "94338f6f-3467-4696-bf7d-f41a12eb889d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from transformers import FillMaskPipeline"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "id": "7b980d9f-4d86-4b54-9324-d57dd9b4b64f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "astroBERT_pipeline = FillMaskPipeline(model=astroBERT_automodel_for_mlm,\n",
-    "                                      tokenizer=astroBERT_tokenizer,\n",
-    "                                      task='fill-mask',\n",
-    "                                     )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "id": "5cb4d27b-ee3c-4ac7-ace2-4cc57ea9ce7a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "clean_sentences = ['M67 is one of the most studied open clusters.',\n",
-    "'A solar twin is a star with atmospheric parameters and chemical composition very similar to our Sun.',\n",
-    "'The dynamical evolution of planets close to their star is affected by tidal effects',\n",
-    "'The Kepler satellite collected high-precision long-term and continuous light curves for more than 100,000 solar-type stars',\n",
-    "'The Local Group is composed of the Milky Way, the Andromeda Galaxy, and numerous smaller satellite galaxies.',\n",
-    "'Cepheid variables are used to determine the distances to galaxies in the local universe.',\n",
-    "'Jets are created and sustained by accretion of matter onto a compact massive object.',\n",
-    "'A single star of one solar mass will evolve into a white dwarf.',\n",
-    "'The Very Large Array observes the sky at radio wavelengths.',\n",
-    "'Elements heavier than iron are generated in supernovae explosions.',\n",
-    "'Spitzer was the first spacecraft to fly in an Earth-trailing orbit.',\n",
-    "'Galaxy mergers can occur when two (or more) galaxies collide',\n",
-    "'Dark matter is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.',\n",
-    "'The Local Group of galaxies is pulled toward The Great Attractor.',\n",
-    "'The Moon is the only satellite of the Earth.',\n",
-    "'Galaxies are categorized according to their visual morphology as elliptical, spiral, or irregular.',\n",
-    "'Stars are made mostly of hydrogen.',\n",
-    "'Comet tails are created as comets approach the Sun.',\n",
-    "'Pluto is a dwarf planet in the Kuiper Belt.',\n",
-    "'The Milky Way has a supermassive black hole, Sagittarius A*, at its center.',\n",
-    "'Andromeda is the nearest large galaxy to the Milky Way and is roughly its equal in mass.',\n",
-    "'The interstellar medium is the gas and dust between stars.',\n",
-    "'The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the universe.',\n",
-    "'The Large and Small Magellanic Clouds are irregular dwarf galaxies and are two satellite galaxies of the Milky Way.']"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "9f3a6fdc-182f-4edb-8ef4-7e4253c2d4db",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "masked_sentences =  ['M67 is one of the most studied [MASK] clusters.',\n",
-    "'A solar twin is a star with [MASK] parameters and chemical composition very similar to our Sun.',\n",
-    "'The dynamical evolution of planets close to their star is affected by [MASK] effects',\n",
-    "'The Kepler satellite collected high-precision long-term and continuous light [MASK] for more than 100,000 solar-type stars',\n",
-    "'The Local Group is composed of the Milky Way, the [MASK] Galaxy, and numerous smaller satellite galaxies.',\n",
-    "'Cepheid variables are used to determine the [MASK] to galaxies in the local universe.',\n",
-    "'Jets are created and sustained by [MASK] of matter onto a compact massive object.',\n",
-    "'A single star of one solar mass will evolve into a [MASK] dwarf.',\n",
-    "'The Very Large Array observes the sky at [MASK] wavelengths.',\n",
-    "'Elements heavier than [MASK] are generated in supernovae explosions.',\n",
-    "'Spitzer was the first [MASK] to fly in an Earth-trailing orbit.',\n",
-    "'Galaxy [MASK] can occur when two (or more) galaxies collide',\n",
-    "'Dark [MASK] is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.',\n",
-    "'The Local Group of galaxies is pulled toward The Great [MASK] .',\n",
-    "'The Moon is the only [MASK] of the Earth.',\n",
-    "'Galaxies are categorized according to their visual morphology as [MASK] , spiral, or irregular.',\n",
-    "'Stars are made mostly of [MASK] .',\n",
-    "'Comet tails are created as comets approach the [MASK] .',\n",
-    "'Pluto is a dwarf [MASK] in the Kuiper Belt.',\n",
-    "'The Milky Way has a [MASK] black hole, Sagittarius A*, at its center.',\n",
-    "'Andromeda is the nearest large [MASK] to the Milky Way and is roughly its equal in mass.',\n",
-    "'The [MASK] medium is the gas and dust between stars.',\n",
-    "'The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the [MASK] .',\n",
-    "'The Large and Small Magellanic Clouds are irregular [MASK] galaxies and are two satellite galaxies of the Milky Way.',\n",
-    "]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "d4c729ad-89f4-4e70-b433-a65b6035c10b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "masked_words = [x for s1,s2 in zip(clean_sentences, masked_sentences) \n",
-    "              for x,y in zip(s1.split(), s2.split()) if y=='[MASK]']"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "id": "2a07a641-61a7-42dd-b70e-62eb97ad4e4b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "results = astroBERT_pipeline(inputs=masked_sentences, \n",
-    "                             top_k=3\n",
-    "                            )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 14,
-   "id": "ec2880d9-a8ad-4919-ab5b-732f3bcc21ae",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "M67 is one of the most studied [MASK] clusters.\n",
-      "original: open\n",
-      "\t  open 0.87\n",
-      "\t  globular 0.07\n",
-      "\t  star 0.03\n",
-      "\n",
-      "A solar twin is a star with [MASK] parameters and chemical composition very similar to our Sun.\n",
-      "original: atmospheric\n",
-      "\t  fundamental 0.56\n",
-      "\t  physical 0.25\n",
-      "\t  stellar 0.05\n",
-      "\n",
-      "The dynamical evolution of planets close to their star is affected by [MASK] effects\n",
-      "original: tidal\n",
-      "\t  tidal 0.07\n",
-      "\t  electromagnetic 0.05\n",
-      "\t  electrostatic 0.04\n",
-      "\n",
-      "The Kepler satellite collected high-precision long-term and continuous light [MASK] for more than 100,000 solar-type stars\n",
-      "original: curves\n",
-      "\t  curves 0.43\n",
-      "\t  ##s 0.04\n",
-      "\t  conditions 0.04\n",
-      "\n",
-      "The Local Group is composed of the Milky Way, the [MASK] Galaxy, and numerous smaller satellite galaxies.\n",
-      "original: Andromeda\n",
-      "\t  Andromeda 0.99\n",
-      "\t  M31 0.00\n",
-      "\t  Sagittarius 0.00\n",
-      "\n",
-      "Cepheid variables are used to determine the [MASK] to galaxies in the local universe.\n",
-      "original: distances\n",
-      "\t  distances 0.79\n",
-      "\t  distance 0.21\n",
-      "\t  redshifts 0.00\n",
-      "\n",
-      "Jets are created and sustained by [MASK] of matter onto a compact massive object.\n",
-      "original: accretion\n",
-      "\t  accretion 0.79\n",
-      "\t  infall 0.13\n",
-      "\t  fall 0.02\n",
-      "\n",
-      "A single star of one solar mass will evolve into a [MASK] dwarf.\n",
-      "original: white\n",
-      "\t  white 0.77\n",
-      "\t  brown 0.19\n",
-      "\t  red 0.02\n",
-      "\n",
-      "The Very Large Array observes the sky at [MASK] wavelengths.\n",
-      "original: radio\n",
-      "\t  radio 0.29\n",
-      "\t  centimeter 0.10\n",
-      "\t  all 0.09\n",
-      "\n",
-      "Elements heavier than [MASK] are generated in supernovae explosions.\n",
-      "original: iron\n",
-      "\t  iron 0.34\n",
-      "\t  helium 0.16\n",
-      "\t  oxygen 0.07\n",
-      "\n",
-      "Spitzer was the first [MASK] to fly in an Earth-trailing orbit.\n",
-      "original: spacecraft\n",
-      "\t  satellite 0.42\n",
-      "\t  spacecraft 0.20\n",
-      "\t  observatory 0.16\n",
-      "\n",
-      "Galaxy [MASK] can occur when two (or more) galaxies collide\n",
-      "original: mergers\n",
-      "\t  . 0.26\n",
-      "\t  A 0.05\n",
-      "\t  1 0.04\n",
-      "\n",
-      "Dark [MASK] is a hypothetical form of matter thought to account for approximately 85% of the matter in the universe.\n",
-      "original: matter\n",
-      "\t  energy 0.64\n",
-      "\t  Energy 0.24\n",
-      "\t  matter 0.10\n",
-      "\n",
-      "The Local Group of galaxies is pulled toward The Great [MASK] .\n",
-      "original: Attractor.\n",
-      "\t  Wall 0.96\n",
-      "\t  East 0.01\n",
-      "\t  Planet 0.00\n",
-      "\n",
-      "The Moon is the only [MASK] of the Earth.\n",
-      "original: satellite\n",
-      "\t  satellite 0.38\n",
-      "\t  moon 0.31\n",
-      "\t  constituent 0.07\n",
-      "\n",
-      "Galaxies are categorized according to their visual morphology as [MASK] , spiral, or irregular.\n",
-      "original: elliptical,\n",
-      "\t  elliptical 0.92\n",
-      "\t  spheroidal 0.02\n",
-      "\t  irregular 0.01\n",
-      "\n",
-      "Stars are made mostly of [MASK] .\n",
-      "original: hydrogen.\n",
-      "\t  hydrogen 0.20\n",
-      "\t  helium 0.14\n",
-      "\t  carbon 0.12\n",
-      "\n",
-      "Comet tails are created as comets approach the [MASK] .\n",
-      "original: Sun.\n",
-      "\t  Sun 0.45\n",
-      "\t  sun 0.23\n",
-      "\t  Earth 0.19\n",
-      "\n",
-      "Pluto is a dwarf [MASK] in the Kuiper Belt.\n",
-      "original: planet\n",
-      "\t  planet 0.96\n",
-      "\t  satellite 0.02\n",
-      "\t  nova 0.00\n",
-      "\n",
-      "The Milky Way has a [MASK] black hole, Sagittarius A*, at its center.\n",
-      "original: supermassive\n",
-      "\t  supermassive 0.80\n",
-      "\t  massive 0.17\n",
-      "\t  stellar 0.00\n",
-      "\n",
-      "Andromeda is the nearest large [MASK] to the Milky Way and is roughly its equal in mass.\n",
-      "original: galaxy\n",
-      "\t  galaxy 0.68\n",
-      "\t  spiral 0.12\n",
-      "\t  satellite 0.09\n",
-      "\n",
-      "The [MASK] medium is the gas and dust between stars.\n",
-      "original: interstellar\n",
-      "\t  interstellar 0.87\n",
-      "\t  interplanetary 0.05\n",
-      "\t  intracluster 0.03\n",
-      "\n",
-      "The cosmic microwave background (CMB, CMBR), in Big Bang cosmology, is electromagnetic radiation which is a remnant from an early stage of the [MASK] .\n",
-      "original: universe.\n",
-      "\t  universe 0.45\n",
-      "\t  Universe 0.26\n",
-      "\t  expansion 0.09\n",
-      "\n",
-      "The Large and Small Magellanic Clouds are irregular [MASK] galaxies and are two satellite galaxies of the Milky Way.\n",
-      "original: dwarf\n",
-      "\t  dwarf 0.68\n",
-      "\t  satellite 0.13\n",
-      "\t  Magellanic 0.08\n",
-      "\n"
-     ]
-    }
-   ],
-   "source": [
-    "for w, s, rs in zip(masked_words, masked_sentences, results):\n",
-    "    print(s)\n",
-    "    print('original: {}'.format(w))\n",
-    "    for r in rs:\n",
-    "        print('\\t  {} {:0.2f}'.format(r['token_str'], r['score']))\n",
-    "    print()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.8.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

Tutorials/0_Embeddings.ipynb

@@ -40,7 +40,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2022-10-20 16:07:24.705905: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
+      "2022-10-19 10:05:02.842926: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0\n"
      ]
     }
    ],
@@ -107,7 +107,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
+      "Some weights of the model checkpoint at adsabs/astroBERT were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight']\n",
       "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
       "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
      ]

config.json

@@ -1,7 +1,7 @@
 {
-  "_name_or_path": "../astroBERT-Tasks/Finetuning_1_NER/trained-models/NER_astroBERT_all_labeled_data_run01/checkpoint-169600/",
+  "_name_or_path": "adsabs/astroBERT",
   "architectures": [
-    "BertForTokenClassification"
+    "BertForPreTraining"
   ],
   "attention_probs_dropout_prob": 0.1,
   "classifier_dropout": null,
@@ -9,138 +9,8 @@
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
-  "id2label": {
-    "0": "LABEL_0",
-    "1": "LABEL_1",
-    "2": "LABEL_2",
-    "3": "LABEL_3",
-    "4": "LABEL_4",
-    "5": "LABEL_5",
-    "6": "LABEL_6",
-    "7": "LABEL_7",
-    "8": "LABEL_8",
-    "9": "LABEL_9",
-    "10": "LABEL_10",
-    "11": "LABEL_11",
-    "12": "LABEL_12",
-    "13": "LABEL_13",
-    "14": "LABEL_14",
-    "15": "LABEL_15",
-    "16": "LABEL_16",
-    "17": "LABEL_17",
-    "18": "LABEL_18",
-    "19": "LABEL_19",
-    "20": "LABEL_20",
-    "21": "LABEL_21",
-    "22": "LABEL_22",
-    "23": "LABEL_23",
-    "24": "LABEL_24",
-    "25": "LABEL_25",
-    "26": "LABEL_26",
-    "27": "LABEL_27",
-    "28": "LABEL_28",
-    "29": "LABEL_29",
-    "30": "LABEL_30",
-    "31": "LABEL_31",
-    "32": "LABEL_32",
-    "33": "LABEL_33",
-    "34": "LABEL_34",
-    "35": "LABEL_35",
-    "36": "LABEL_36",
-    "37": "LABEL_37",
-    "38": "LABEL_38",
-    "39": "LABEL_39",
-    "40": "LABEL_40",
-    "41": "LABEL_41",
-    "42": "LABEL_42",
-    "43": "LABEL_43",
-    "44": "LABEL_44",
-    "45": "LABEL_45",
-    "46": "LABEL_46",
-    "47": "LABEL_47",
-    "48": "LABEL_48",
-    "49": "LABEL_49",
-    "50": "LABEL_50",
-    "51": "LABEL_51",
-    "52": "LABEL_52",
-    "53": "LABEL_53",
-    "54": "LABEL_54",
-    "55": "LABEL_55",
-    "56": "LABEL_56",
-    "57": "LABEL_57",
-    "58": "LABEL_58",
-    "59": "LABEL_59",
-    "60": "LABEL_60",
-    "61": "LABEL_61",
-    "62": "LABEL_62"
-  },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
-  "label2id": {
-    "LABEL_0": 0,
-    "LABEL_1": 1,
-    "LABEL_10": 10,
-    "LABEL_11": 11,
-    "LABEL_12": 12,
-    "LABEL_13": 13,
-    "LABEL_14": 14,
-    "LABEL_15": 15,
-    "LABEL_16": 16,
-    "LABEL_17": 17,
-    "LABEL_18": 18,
-    "LABEL_19": 19,
-    "LABEL_2": 2,
-    "LABEL_20": 20,
-    "LABEL_21": 21,
-    "LABEL_22": 22,
-    "LABEL_23": 23,
-    "LABEL_24": 24,
-    "LABEL_25": 25,
-    "LABEL_26": 26,
-    "LABEL_27": 27,
-    "LABEL_28": 28,
-    "LABEL_29": 29,
-    "LABEL_3": 3,
-    "LABEL_30": 30,
-    "LABEL_31": 31,
-    "LABEL_32": 32,
-    "LABEL_33": 33,
-    "LABEL_34": 34,
-    "LABEL_35": 35,
-    "LABEL_36": 36,
-    "LABEL_37": 37,
-    "LABEL_38": 38,
-    "LABEL_39": 39,
-    "LABEL_4": 4,
-    "LABEL_40": 40,
-    "LABEL_41": 41,
-    "LABEL_42": 42,
-    "LABEL_43": 43,
-    "LABEL_44": 44,
-    "LABEL_45": 45,
-    "LABEL_46": 46,
-    "LABEL_47": 47,
-    "LABEL_48": 48,
-    "LABEL_49": 49,
-    "LABEL_5": 5,
-    "LABEL_50": 50,
-    "LABEL_51": 51,
-    "LABEL_52": 52,
-    "LABEL_53": 53,
-    "LABEL_54": 54,
-    "LABEL_55": 55,
-    "LABEL_56": 56,
-    "LABEL_57": 57,
-    "LABEL_58": 58,
-    "LABEL_59": 59,
-    "LABEL_6": 6,
-    "LABEL_60": 60,
-    "LABEL_61": 61,
-    "LABEL_62": 62,
-    "LABEL_7": 7,
-    "LABEL_8": 8,
-    "LABEL_9": 9
-  },
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
   "model_type": "bert",
@@ -148,7 +18,6 @@
   "num_hidden_layers": 12,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
-  "return_dict": false,
   "torch_dtype": "float32",
   "transformers_version": "4.17.0",
   "type_vocab_size": 2,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:069bc02d7c193d06fc9d23ce25f59129ff2c6724d424d60d600343e52f9e12f7
-size 434238321
+oid sha256:3268b70d3529eb896bf43c0c8fa933f6282b82fa196e2659ab116d49fbbca6ac
+size 438904355