add tiny random mdoel

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,4 @@
+---
+{}
+---
+This is a tiny random {mname_tiny} model to be used for basic testing

config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bc33117b12fec0c72c3a433734042a0487c9010417fca5110624fd91aa16ded
+size 920

generation_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdc3cc912fbcccffc929bad5ad222dfb58ea5438290e7b5c9ba277727699ab42
+size 144

make-tiny-xlm-roberta.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python
+# coding: utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script creates a tiny random model
+#
+# It will be used then as "hf-internal-testing/tiny-xlm-roberta"
+
+# ***To build from scratch***
+#
+# 1. clone sentencepiece into a parent dir
+# git clone https://github.com/google/sentencepiece
+#
+# 2. create a new repo at https://huggingface.co/new
+# make sure to choose 'hf-internal-testing' as the Owner
+#
+# 3. clone
+# git clone https://huggingface.co/hf-internal-testing/tiny-xlm-roberta
+# cd tiny-xlm-roberta
+#
+# 4. start with some pre-existing script from one of the https://huggingface.co/hf-internal-testing/ tiny model repos, e.g.
+# wget https://huggingface.co/hf-internal-testing/tiny-albert/raw/main/make-tiny-albert.py
+# chmod a+x ./make-tiny-albert.py
+# mv ./make-tiny-albert.py ./make-tiny-xlm-roberta.py
+#
+# 5. automatically rename things from the old names to new ones
+# perl -pi -e 's|Albert|XLMRoberta|g' make-tiny-xlm-roberta.py
+# perl -pi -e 's|albert|xlm-roberta|g' make-tiny-xlm-roberta.py
+#
+# 6. edit and re-run this script while fixing it up
+# ./make-tiny-xlm-roberta.py
+#
+# 7. add/commit/push
+# git add *
+# git commit -m "new tiny model"
+# git push
+
+# ***To update***
+#
+# 1. clone the existing repo
+# git clone https://huggingface.co/hf-internal-testing/tiny-xlm-roberta
+# cd tiny-xlm-roberta
+#
+# 2. edit and re-run this script after doing whatever changes are needed
+# ./make-tiny-xlm-roberta.py
+#
+# 3. commit/push
+# git commit -m "new tiny model"
+# git push
+
+import sys
+import os
+
+# workaround for fast tokenizer protobuf issue, and it's much faster too!
+os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
+from transformers import XLMRobertaTokenizerFast, XLMRobertaConfig, XLMRobertaForCausalLM
+
+mname_orig = "xlm-roberta-base"
+mname_tiny = "tiny-xlm-roberta"
+
+### Tokenizer
+
+# Shrink the orig vocab to keep things small
+vocab_keep_items = 5000
+tmp_dir = f"/tmp/{mname_tiny}"
+vocab_orig_path = f"{tmp_dir}/sentencepiece.bpe.model"
+vocab_short_path = f"{tmp_dir}/spiece-short.model"
+if 1: # set to 0 to skip this after running once to speed things up during tune up
+    # HACK: need the sentencepiece source to get sentencepiece_model_pb2, as it doesn't get installed
+    sys.path.append("../sentencepiece/python/src/sentencepiece")
+    import sentencepiece_model_pb2 as model
+    tokenizer_orig = XLMRobertaTokenizerFast.from_pretrained(mname_orig)
+    tokenizer_orig.save_pretrained(tmp_dir)
+    with open(vocab_orig_path, 'rb') as f: data = f.read()
+    # adapted from https://blog.ceshine.net/post/trim-down-sentencepiece-vocabulary/
+    m = model.ModelProto()
+    m.ParseFromString(data)
+    print(f"Shrinking vocab from original {len(m.pieces)} dict items")
+    for i in range(len(m.pieces) - vocab_keep_items): _ = m.pieces.pop()
+    print(f"new dict {len(m.pieces)}")
+    with open(vocab_short_path, 'wb') as f: f.write(m.SerializeToString())
+    m = None
+
+tokenizer_fast_tiny = XLMRobertaTokenizerFast(vocab_file=vocab_short_path)
+
+### Config
+
+config_tiny = XLMRobertaConfig.from_pretrained(mname_orig)
+print(config_tiny)
+# remember to update this to the actual config as each model is different and then shrink the numbers
+config_tiny.update(dict(
+    vocab_size=vocab_keep_items+12,
+    d_ff=256,
+    d_kv=8,
+    d_model=64,
+    hidden_size=256,
+    intermediate_size=256,
+    max_position_embeddings=64,
+    num_attention_heads=2,
+    num_decoder_layers=2,
+    num_heads=2,
+    num_hidden_layers=2,
+    num_layers=2,
+    relative_attention_num_buckets=32,
+))
+print("New config", config_tiny)
+
+### Model
+
+model_tiny = XLMRobertaForCausalLM(config_tiny)
+print(f"{mname_tiny}: num of params {model_tiny.num_parameters()}")
+model_tiny.resize_token_embeddings(len(tokenizer_fast_tiny))
+
+# Test
+inputs = tokenizer_fast_tiny("hello", return_tensors="pt")
+outputs = model_tiny(**inputs)
+print("Test with fast tokenizer:", len(outputs.logits[0]))
+
+# Save
+model_tiny.half() # makes it smaller
+model_tiny.save_pretrained(".")
+tokenizer_fast_tiny.save_pretrained(".")
+
+readme = "README.md"
+if not os.path.exists(readme):
+    with open(readme, "w") as f:
+        f.write(f"This is a {mname_tiny} random model to be used for basic testing.\n")
+
+print(f"Generated {mname_tiny}")

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cd9aefdcce328560a21d7a9bf0c65c4ea2a945a72162efee05faa521c0cfc70
+size 9085000

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb0bccafb4bee811f2138956ea9e94596e1bfdfc868b5364d7b678fac4b2d559
+size 4334436

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef67a60b933d0da430d4b839301301ada0179b0c71102b0eef4567386faa1588
+size 309222

special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58dc0a3d15e3bad4861bcf2dee770ea327344d185182d4c5d2bf253285e53a83
+size 238

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91442440b17efb24b2c9aa4a79431be606ba0a850b0cba1ff408e30707ece947
+size 466347

tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f998ef74b999a23775eb90c18e1d8775319a998f8c7c5fa58296f272456b1c1c
+size 327