update model

.gitattributes

@@ -0,0 +1,2 @@
+events.out.tfevents.1631471189.blg4302.int.ets1.calculquebec.ca.240020.0 filter=lfs diff=lfs merge=lfs -text
+pytorch_model.bin filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1 @@
+{"$pi$": 30522, "$vert$": 30523, "$alpha$": 30524, "$minus$": 30525, "$p$": 30526, "$frac$": 30527, "$q$": 30528, "$lt$": 30529, "$1$": 30530, "$($": 30531, "$2$": 30532, "$supscript$": 30533, "$)$": 30534, "$varepsilon$": 30535, "$x$": 30536, "$subscript$": 30537, "${$": 30538, "$n$": 30539, "$}$": 30540, "$v$": 30541, "$a$": 30542, "$r$": 30543, "$rightarrow$": 30544, "$infty$": 30545, "$omega$": 30546, "$equal$": 30547, "$[$": 30548, "$comma$": 30549, "$]$": 30550, "$\\begin{cases}$": 30551, "$column$": 30552, "$0$": 30553, "$le$": 30554, "$row$": 30555, "$ast$": 30556, "$\\end{cases}$": 30557, "$qquad$": 30558, "$in$": 30559, "$e$": 30560, "$to$": 30561, "$l$": 30562, "$phi$": 30563, "$colon$": 30564, "$z$": 30565, "$i$": 30566, "$5$": 30567, "$plus$": 30568, "$cong$": 30569, "$b$": 30570, "$mod$": 30571, "$ker$": 30572, "$mapsto$": 30573, "$equiv$": 30574, "$subset$": 30575, "$f$": 30576, "$y$": 30577, "$sin$": 30578, "$times$": 30579, "$cos$": 30580, "$t$": 30581, "$float$": 30582, "$fact$": 30583, "$3$": 30584, "$dots$": 30585, "$k$": 30586, "$sum$": 30587, "$gt$": 30588, "$\\left($": 30589, "$\\right)$": 30590, "$m$": 30591, "$ldots$": 30592, "$9$": 30593, "$leftarrow$": 30594, "$prime$": 30595, "$h$": 30596, "$approx$": 30597, "$root$": 30598, "$pm$": 30599, "$c$": 30600, "$d$": 30601, "$beta$": 30602, "$ge$": 30603, "$g$": 30604, "$epsilon$": 30605, "$j$": 30606, "$u$": 30607, "$aleph$": 30608, "$neq$": 30609, "$cdots$": 30610, "$lambda$": 30611, "$mu$": 30612, "$nu$": 30613, "$geq$": 30614, "$\\{$": 30615, "$\\}$": 30616, "$s$": 30617, "$subseteq$": 30618, "$max$": 30619, "$kappa$": 30620, "$oplus$": 30621, "$dim$": 30622, "$prod$": 30623, "$8$": 30624, "$somenum$": 30625, "$circ$": 30626, "$4$": 30627, "$7$": 30628, "$forall$": 30629, "$ne$": 30630, "$o$": 30631, "$w$": 30632, "$arg$": 30633, "$exp$": 30634, "$6$": 30635, "$\\begin{pmatrix}$": 30636, "$\\end{pmatrix}$": 30637, "$theta$": 30638, "$tan$": 30639, "$semicolon$": 30640, "$quad$": 30641, "$newline$": 30642, "$ni$": 30643, "$cup$": 30644, "$varpi$": 30645, "$vee$": 30646, "$langle$": 30647, "$rangle$": 30648, "$delta$": 30649, "$gg$": 30650, "$log$": 30651, "$\\lceil$": 30652, "$\\rceil$": 30653, "$\\begin{array}$": 30654, "$\\end{array}$": 30655, "$pr$": 30656, "$bigcap$": 30657, "$sim$": 30658, "$lim$": 30659, "$\\left [$": 30660, "$\\right ]$": 30661, "$\\left ($": 30662, "$\\right )$": 30663, "$longmapsto$": 30664, "$rvect$": 30665, "$rho$": 30666, "$int$": 30667, "$nabla$": 30668, "$sigma$": 30669, "$cap$": 30670, "$iff$": 30671, "$\\over$": 30672, "$wedge$": 30673, "$bigoplus$": 30674, "$otimes$": 30675, "$partial$": 30676, "$\\begin{bmatrix}$": 30677, "$\\end{bmatrix}$": 30678, "$\\left\\langle$": 30679, "$\\right\\rangle$": 30680, "$\\left\\$": 30681, "$\\right\\$": 30682, "$\\left|$": 30683, "$\\right|$": 30684, "$chi$": 30685, "$\\left.$": 30686, "$psi$": 30687, "$gamma$": 30688, "$cot$": 30689, "$\\left[$": 30690, "$\\right]$": 30691, "$zeta$": 30692, "$xrightarrow$": 30693, "$ln$": 30694, "$setminus$": 30695, "$unlhd$": 30696, "$xi$": 30697, "$ll$": 30698, "$implies$": 30699, "$uparrow$": 30700, "$\\underbrace$": 30701, "$cr$": 30702, "$longleftrightarrow$": 30703, "$tau$": 30704, "$mid$": 30705, "$varphi$": 30706, "$geqslant$": 30707, "$angle$": 30708, "$longrightarrow$": 30709, "$exists$": 30710, "$inf$": 30711, "$sup$": 30712, "$\\begin{matrix}$": 30713, "$\\end{matrix}$": 30714, "$top$": 30715, "$bot$": 30716, "$simeq$": 30717, "$det$": 30718, "$\\right/$": 30719, "$\\overset$": 30720, "$emptyset$": 30721, "$\\stackrel$": 30722, "$\\left\\{$": 30723, "$\\lfloor$": 30724, "$\\rfloor$": 30725, "$\\right.$": 30726, "$binom$": 30727, "$ell$": 30728, "$sec$": 30729, "$arccos$": 30730, "$vdots$": 30731, "$ddots$": 30732, "$\\right\\}$": 30733, "$leqslant$": 30734, "$eta$": 30735, "$\\begin{smallmatrix}$": 30736, "$\\end{smallmatrix}$": 30737, "$percent$": 30738, "$oint$": 30739, "$min$": 30740, "$hbar$": 30741, "$ddot$": 30742, "$varnothing$": 30743, "$cosh$": 30744, "$downarrow$": 30745, "$gcd$": 30746, "$rightarrowtail$": 30747, "$supset$": 30748, "$\\underset$": 30749, "$bigcup$": 30750, "$preceq$": 30751, "$\\array{$": 30752, "$triangleq$": 30753, "$iota$": 30754, "$leftrightarrow$": 30755, "$arctan$": 30756, "$arcsin$": 30757, "$sinh$": 30758, "$triangle$": 30759, "$coprod$": 30760, "$neg$": 30761, "$land$": 30762, "$lor$": 30763, "$measuredangle$": 30764, "$wp$": 30765, "$backslash$": 30766, "$vartheta$": 30767, "$odot$": 30768, "$perp$": 30769, "$tanh$": 30770, "$trianglelefteq$": 30771, "$\\left\\lfloor$": 30772, "$\\right\\rfloor$": 30773, "$supseteq$": 30774, "$sign$": 30775, "$dotsc$": 30776, "$nmid$": 30777, "$smallsetminus$": 30778, "$and$": 30779, "$\\left\\lceil$": 30780, "$\\right\\rceil$": 30781, "$deg$": 30782, "$impliedby$": 30783, "$\\left<$": 30784, "$\\right>$": 30785, "$searrow$": 30786, "$limsup$": 30787, "$succeq$": 30788, "$rightharpoonup$": 30789, "$bigtriangleup$": 30790, "$sqcup$": 30791, "$subsetneq$": 30792, "$\\left \\{$": 30793, "$\\right \\}$": 30794, "$\\left |$": 30795, "$\\right |$": 30796, "$nle$": 30797, "$lnot$": 30798, "$iint$": 30799, "$hom$": 30800, "$leadsto$": 30801, "$nexists$": 30802, "$re$": 30803, "$\\begin{vmatrix}$": 30804, "$\\end{vmatrix}$": 30805, "$parallel$": 30806, "$dotsb$": 30807, "$bigwedge$": 30808, "$succ$": 30809, "$\\buildrel$": 30810, "$liminf$": 30811, "$csc$": 30812, "$wr$": 30813, "$hookrightarrow$": 30814, "$\\lbrace$": 30815, "$\\rbrace$": 30816, "$dotsm$": 30817, "$rtimes$": 30818, "$ltimes$": 30819, "$\\of$": 30820, "$updownarrow$": 30821, "$nearrow$": 30822, "$\\left \\lfloor$": 30823, "$\\right \\rfloor$": 30824, "$approxeq$": 30825, "$dashv$": 30826, "$bigcirc$": 30827, "$triangledown$": 30828, "$lcm$": 30829, "$prec$": 30830, "$propto$": 30831, "$triangleleft$": 30832, "$ncong$": 30833, "$coth$": 30834, "$longleftarrow$": 30835, "$upsilon$": 30836, "$thicksim$": 30837, "$\\left$": 30838, "$\\right$": 30839, "$bigtriangledown$": 30840, "$varliminf$": 30841, "$varlimsup$": 30842, "$atop$": 30843, "$\\overbrace$": 30844, "$\\left \\langle$": 30845, "$\\right \\rangle$": 30846, "$gets$": 30847, "$vartriangleleft$": 30848, "$iiint$": 30849, "$varinjlim$": 30850, "$varprojlim$": 30851, "$bigotimes$": 30852, "$varrho$": 30853, "$lesssim$": 30854, "$\\left \\$": 30855, "$\\right \\$": 30856, "$bigsqcup$": 30857, "$supsetneq$": 30858, "$curvearrowright$": 30859, "$dotso$": 30860, "$preccurlyeq$": 30861, "$imath$": 30862, "$omicron$": 30863, "$nrightarrow$": 30864, "$lneq$": 30865, "$upharpoonright$": 30866, "$nsubseteq$": 30867, "$enspace$": 30868, "$geqq$": 30869, "$rightrightarrows$": 30870, "$uplus$": 30871, "$owns$": 30872, "$rightsquigarrow$": 30873, "$vartriangle$": 30874, "$trianglerighteq$": 30875, "$amalg$": 30876, "$frown$": 30877, "$jmath$": 30878, "$\\left <$": 30879, "$\\right >$": 30880, "$beth$": 30881, "$sphericalangle$": 30882, "$gtrapprox$": 30883, "$lessapprox$": 30884, "$bigvee$": 30885, "$nsim$": 30886, "$swarrow$": 30887, "$asymp$": 30888, "$bigodot$": 30889, "$dotsi$": 30890, "$\\left  |$": 30891, "$intop$": 30892, "$sqsupset$": 30893, "$\\left \\lceil$": 30894, "$\\right \\rceil$": 30895, "$empty$": 30896, "$rightleftharpoons$": 30897, "$\\left .$": 30898, "$smallint$": 30899, "$gtrless$": 30900, "$ngtr$": 30901, "$\\lbrack$": 30902, "$arrowvert$": 30903, "$xleftarrow$": 30904, "$\\left  \\{$": 30905, "$\\right  \\}$": 30906, "$vardelta$": 30907, "$looparrowright$": 30908, "$nge$": 30909, "$gneq$": 30910, "$\\left\\downarrow$": 30911, "$subsetneqq$": 30912, "$idotsint$": 30913, "$nless$": 30914, "$nprec$": 30915, "$nwarrow$": 30916, "$circlearrowright$": 30917, "$ominus$": 30918, "$\\left/$": 30919, "$gtrsim$": 30920, "$succcurlyeq$": 30921, "$sqsubseteq$": 30922, "$backsim$": 30923, "$dddot$": 30924, "$precsim$": 30925, "$divideontimes$": 30926, "$triangleright$": 30927, "$sqcap$": 30928, "$\\right   )$": 30929, "$lessdot$": 30930, "$nsupseteq$": 30931, "$nleqslant$": 30932, "$\\left /$": 30933, "$varsubsetneq$": 30934, "$\\rbrack$": 30935, "$rightharpoondown$": 30936, "$\\right .$": 30937, "$succsim$": 30938, "$leftleftarrows$": 30939, "$varsupsetneq$": 30940, "$iddots$": 30941, "$vargamma$": 30942, "$sqsubset$": 30943, "$downharpoonright$": 30944, "$sqsupseteq$": 30945, "$varsigma$": 30946, "$\\right     .$": 30947, "$veebar$": 30948, "$ddddot$": 30949, "$circlearrowleft$": 30950, "$leftrightarrows$": 30951, "$\\right    )$": 30952, "$thickapprox$": 30953, "$npreceq$": 30954, "$biguplus$": 30955, "$supsetneqq$": 30956, "$rrightarrow$": 30957, "$smallfrown$": 30958, "$multimap$": 30959, "$subseteqq$": 30960, "$hookleftarrow$": 30961, "$rightleftarrows$": 30962, "$nparallel$": 30963, "$nsucc$": 30964, "$ggg$": 30965, "$vartriangleright$": 30966, "$varpropto$": 30967, "$\\left  ($": 30968, "$\\right    ]$": 30969, "$\\left  [$": 30970, "$\\left \\lgroup$": 30971, "$\\right \\rgroup$": 30972, "$\\left  \\$": 30973, "$leftrightsquigarrow$": 30974, "$\\right  )$": 30975, "$lsh$": 30976, "$\\lgroup$": 30977, "$\\rgroup$": 30978, "$mho$": 30979, "$upharpoonleft$": 30980, "$eth$": 30981, "$nleftarrow$": 30982, "$smallsmile$": 30983, "$nleftrightarrow$": 30984, "$iiiint$": 30985, "$gneqq$": 30986, "$lneqq$": 30987, "$downdownarrows$": 30988, "$varkappa$": 30989, "$backsimeq$": 30990, "$barwedge$": 30991, "$unrhd$": 30992, "$bowtie$": 30993, "$backepsilon$": 30994, "$bracevert$": 30995, "$\\right  \\rfloor$": 30996, "$\\left  \\lfloor$": 30997, "$precneqq$": 30998, "$\\left\\lgroup$": 30999, "$\\right\\rgroup$": 31000, "$\\left  \\langle$": 31001, "$\\lmoustache$": 31002, "$\\right   \\}$": 31003, "$\\right  |$": 31004, "$curlywedge$": 31005, "$\\right  ]$": 31006, "$gtreqqless$": 31007, "$gtreqless$": 31008, "$ngeqslant$": 31009, "$between$": 31010, "$dotplus$": 31011, "$leftrightharpoons$": 31012, "$\\right  \\rceil$": 31013, "$leftharpoondown$": 31014, "$projlim$": 31015, "$\\right                   )$": 31016, "$varsubsetneqq$": 31017, "$lvect$": 31018, "$\\left\\backslash$": 31019, "$supseteqq$": 31020, "$join$": 31021, "$injlim$": 31022, "$eqsim$": 31023, "$curvearrowleft$": 31024, "$succneqq$": 31025, "$\\left\\uparrow$": 31026, "$leftarrowtail$": 31027, "$upuparrows$": 31028, "$gvertneqq$": 31029, "$lvertneqq$": 31030, "$nsucceq$": 31031, "$gtrdot$": 31032, "$eqslantless$": 31033, "$\\right  \\$": 31034, "$circeq$": 31035, "$\\left               ($": 31036, "$curlyvee$": 31037, "$\\right    \\}$": 31038, "$\\right   \\rangle$": 31039, "$downharpoonleft$": 31040, "$\\left    ($": 31041, "$\\right  \\rangle$": 31042, "$\\left    \\{$": 31043, "$\\left\\updownarrow$": 31044, "$\\right                                          )$": 31045, "$\\right  >$": 31046, "$leftharpoonup$": 31047, "$curlyeqprec$": 31048, "$\\left   [$": 31049, "$eqslantgtr$": 31050, "$varxi$": 31051, "$\\right   \\rfloor$": 31052, "$\\right     \\}$": 31053, "$varpsi$": 31054, "$\\left  .$": 31055, "$\\left    |$": 31056, "$\\right /$": 31057, "$\\left   ($": 31058, "$\\left   \\$": 31059, "$rsh$": 31060}

config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "data.arjmPWtGwzKrkmR/bert-base-uncased",
+  "architectures": [
+    "BertForPreTraining"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.9.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 31061
+}

events.out.tfevents.1631471189.blg4302.int.ets1.calculquebec.ca.240020.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2caf095b1b4f07fc75087fdecc23f9c0e9edf3201ed001624f4fc06c9a8a83e7
+size 53784629

job-25031358-head.out

@@ -0,0 +1,500 @@
++ TRAINER=pretrain
++ SETUP=for-newvocab
+++ cd pya0
+++ pwd
+++ git rev-parse HEAD
++ CODE_VER='/home/w32zhong/projects/rrg-jimmylin/w32zhong/pya0
+8f207c0036a9f81f91e26f7ecedcfa84025ae680'
++ COMMAND='/var/spool/slurmd/job25031358/slurm_script pretrain for-newvocab'
++ EPOCHS=40
++ TEST_CYCLE=100
++ case $TRAINER-${SETUP} in
++ DEV_BSIZE=8
++ SAVE_FOLD=10
++ DATA_VER=arjmPWtGwzKrkmR
++ START_POINT=bert-base-uncased
++ TOK_CKPOINT=bert-tokenizer
++ SHARDS_LIST=shards-for-newvocab.txt
++ TEST_FILE=test.txt
++ EXTRA_DAT=mse-aops-2021-vocab.pkl
++ EXTRA_ARG=
++ DATA_DIR=data.arjmPWtGwzKrkmR
++ set -e
++ '[' '!' -e data.arjmPWtGwzKrkmR ']'
++ set +e
+++ cat /var/spool/slurmd/job25031358/slurm_script
+++ grep -Po '(?<=SBATCH --nodes=)[0-9]+'
++ N_NODE=4
+++ cat /var/spool/slurmd/job25031358/slurm_script
+++ grep -Po '(?<=SBATCH --gres=gpu:)[0-9]+'
++ N_GPUS=2
++ export NCCL_BLOCKING_WAIT=1
++ NCCL_BLOCKING_WAIT=1
++ export SLURM_ACCOUNT=def-jimmylin
++ SLURM_ACCOUNT=def-jimmylin
++ export SBATCH_ACCOUNT=def-jimmylin
++ SBATCH_ACCOUNT=def-jimmylin
++ export SALLOC_ACCOUNT=def-jimmylin
++ SALLOC_ACCOUNT=def-jimmylin
++ which srun
+/opt/software/slurm/bin/srun
+++ hostname
++ srun --unbuffered python ./pya0/utils/transformer.py pretrain data.arjmPWtGwzKrkmR/bert-base-uncased data.arjmPWtGwzKrkmR/bert-tokenizer data.arjmPWtGwzKrkmR/mse-aops-2021-vocab.pkl --test_file data.arjmPWtGwzKrkmR/test.txt --test_cycle 100 --shards_list data.arjmPWtGwzKrkmR/shards-for-newvocab.txt --cluster tcp://blg4302.int.ets1.calculquebec.ca:8912 --batch_size 64 --save_fold 10 --epochs 40
+Loading model data.arjmPWtGwzKrkmR/bert-base-uncased...
+{
+  "_name_or_path": "data.arjmPWtGwzKrkmR/bert-base-uncased",
+  "add_cross_attention": false,
+  "architectures": [
+    "BertForPreTraining"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bad_words_ids": null,
+  "bos_token_id": null,
+  "chunk_size_feed_forward": 0,
+  "decoder_start_token_id": null,
+  "diversity_penalty": 0.0,
+  "do_sample": false,
+  "early_stopping": false,
+  "encoder_no_repeat_ngram_size": 0,
+  "eos_token_id": null,
+  "finetuning_task": null,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "max_position_embeddings": 512,
+  "min_length": 0,
+  "model_type": "bert",
+  "no_repeat_ngram_size": 0,
+  "num_attention_heads": 12,
+  "num_beam_groups": 1,
+  "num_beams": 1,
+  "num_hidden_layers": 12,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_scores": false,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "prefix": null,
+  "problem_type": null,
+  "pruned_heads": {},
+  "remove_invalid_values": false,
+  "repetition_penalty": 1.0,
+  "return_dict": true,
+  "return_dict_in_generate": false,
+  "sep_token_id": null,
+  "task_specific_params": null,
+  "temperature": 1.0,
+  "tie_encoder_decoder": false,
+  "tie_word_embeddings": true,
+  "tokenizer_class": null,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torch_dtype": null,
+  "torchscript": false,
+  "transformers_version": "4.9.2",
+  "type_vocab_size": 2,
+  "use_bfloat16": false,
+  "use_cache": true,
+  "vocab_size": 30522
+}
+
+Before loading new vocabulary: 30522
+After loading new vocabulary: 31061
+Resize model embedding and save new tokenizer ...
+Invoke training ...
+[caller] pretrain
+[node#3 rank#6] Training on device cuda:0
+[node#3 rank#6] 2 x Tesla V100-SXM2-16GB: 0%
+[node#3 rank#7] Training on device cuda:1
+[node#3 rank#7] 2 x Tesla V100-SXM2-16GB: 5%
+[node#3 rank#6] Initialized process group ...
+[node#3 rank#7] Initialized process group ...
+Loading model data.arjmPWtGwzKrkmR/bert-base-uncased...
+Loading model data.arjmPWtGwzKrkmR/bert-base-uncased...
+Loading model data.arjmPWtGwzKrkmR/bert-base-uncased...
+{
+  "_name_or_path": "data.arjmPWtGwzKrkmR/bert-base-uncased",
+  "add_cross_attention": false,
+  "architectures": [
+    "BertForPreTraining"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bad_words_ids": null,
+  "bos_token_id": null,
+  "chunk_size_feed_forward": 0,
+  "decoder_start_token_id": null,
+  "diversity_penalty": 0.0,
+  "do_sample": false,
+  "early_stopping": false,
+  "encoder_no_repeat_ngram_size": 0,
+  "eos_token_id": null,
+  "finetuning_task": null,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "max_position_embeddings": 512,
+  "min_length": 0,
+  "model_type": "bert",
+  "no_repeat_ngram_size": 0,
+  "num_attention_heads": 12,
+  "num_beam_groups": 1,
+  "num_beams": 1,
+  "num_hidden_layers": 12,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_scores": false,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "prefix": null,
+  "problem_type": null,
+  "pruned_heads": {},
+  "remove_invalid_values": false,
+  "repetition_penalty": 1.0,
+  "return_dict": true,
+  "return_dict_in_generate": false,
+  "sep_token_id": null,
+  "task_specific_params": null,
+  "temperature": 1.0,
+  "tie_encoder_decoder": false,
+  "tie_word_embeddings": true,
+  "tokenizer_class": null,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torch_dtype": null,
+  "torchscript": false,
+  "transformers_version": "4.9.2",
+  "type_vocab_size": 2,
+  "use_bfloat16": false,
+  "use_cache": true,
+  "vocab_size": 30522
+}
+
+{
+  "_name_or_path": "data.arjmPWtGwzKrkmR/bert-base-uncased",
+  "add_cross_attention": false,
+  "architectures": [
+    "BertForPreTraining"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bad_words_ids": null,
+  "bos_token_id": null,
+  "chunk_size_feed_forward": 0,
+  "decoder_start_token_id": null,
+  "diversity_penalty": 0.0,
+  "do_sample": false,
+  "early_stopping": false,
+  "encoder_no_repeat_ngram_size": 0,
+  "eos_token_id": null,
+  "finetuning_task": null,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "max_position_embeddings": 512,
+  "min_length": 0,
+  "model_type": "bert",
+  "no_repeat_ngram_size": 0,
+  "num_attention_heads": 12,
+  "num_beam_groups": 1,
+  "num_beams": 1,
+  "num_hidden_layers": 12,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_scores": false,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "prefix": null,
+  "problem_type": null,
+  "pruned_heads": {},
+  "remove_invalid_values": false,
+  "repetition_penalty": 1.0,
+  "return_dict": true,
+  "return_dict_in_generate": false,
+  "sep_token_id": null,
+  "task_specific_params": null,
+  "temperature": 1.0,
+  "tie_encoder_decoder": false,
+  "tie_word_embeddings": true,
+  "tokenizer_class": null,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torch_dtype": null,
+  "torchscript": false,
+  "transformers_version": "4.9.2",
+  "type_vocab_size": 2,
+  "use_bfloat16": false,
+  "use_cache": true,
+  "vocab_size": 30522
+}
+
+{
+  "_name_or_path": "data.arjmPWtGwzKrkmR/bert-base-uncased",
+  "add_cross_attention": false,
+  "architectures": [
+    "BertForPreTraining"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bad_words_ids": null,
+  "bos_token_id": null,
+  "chunk_size_feed_forward": 0,
+  "decoder_start_token_id": null,
+  "diversity_penalty": 0.0,
+  "do_sample": false,
+  "early_stopping": false,
+  "encoder_no_repeat_ngram_size": 0,
+  "eos_token_id": null,
+  "finetuning_task": null,
+  "forced_bos_token_id": null,
+  "forced_eos_token_id": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "is_decoder": false,
+  "is_encoder_decoder": false,
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "layer_norm_eps": 1e-12,
+  "length_penalty": 1.0,
+  "max_length": 20,
+  "max_position_embeddings": 512,
+  "min_length": 0,
+  "model_type": "bert",
+  "no_repeat_ngram_size": 0,
+  "num_attention_heads": 12,
+  "num_beam_groups": 1,
+  "num_beams": 1,
+  "num_hidden_layers": 12,
+  "num_return_sequences": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "output_scores": false,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "prefix": null,
+  "problem_type": null,
+  "pruned_heads": {},
+  "remove_invalid_values": false,
+  "repetition_penalty": 1.0,
+  "return_dict": true,
+  "return_dict_in_generate": false,
+  "sep_token_id": null,
+  "task_specific_params": null,
+  "temperature": 1.0,
+  "tie_encoder_decoder": false,
+  "tie_word_embeddings": true,
+  "tokenizer_class": null,
+  "top_k": 50,
+  "top_p": 1.0,
+  "torch_dtype": null,
+  "torchscript": false,
+  "transformers_version": "4.9.2",
+  "type_vocab_size": 2,
+  "use_bfloat16": false,
+  "use_cache": true,
+  "vocab_size": 30522
+}
+
+Before loading new vocabulary: 30522
+Before loading new vocabulary: 30522
+After loading new vocabulary: 31061
+Resize model embedding and save new tokenizer ...
+Before loading new vocabulary: 30522
+After loading new vocabulary: 31061
+Resize model embedding and save new tokenizer ...
+After loading new vocabulary: 31061
+Resize model embedding and save new tokenizer ...
+Invoke training ...
+Invoke training ...
+Invoke training ...
+[caller] pretrain
+[caller] pretrain
+[caller] pretrain
+[node#0 rank#0] Training on device cuda:0
+[node#2 rank#4] Training on device cuda:0
+[node#0 rank#0] 2 x Tesla V100-SXM2-16GB: 0%
+[node#2 rank#4] 2 x Tesla V100-SXM2-16GB: 0%
+[node#1 rank#2] Training on device cuda:0
+[node#1 rank#2] 2 x Tesla V100-SXM2-16GB: 0%
+[node#0 rank#1] Training on device cuda:1
+[node#2 rank#5] Training on device cuda:1
+[node#0 rank#1] 2 x Tesla V100-SXM2-16GB: 2%
+[node#2 rank#5] 2 x Tesla V100-SXM2-16GB: 2%
+[node#1 rank#3] Training on device cuda:1
+[node#1 rank#3] 2 x Tesla V100-SXM2-16GB: 2%
+[node#2 rank#4] Initialized process group ...
+[node#0 rank#0] Initialized process group ...
+[node#1 rank#2] Initialized process group ...
+[node#0 rank#1] Initialized process group ...
+[node#2 rank#5] Initialized process group ...
+[node#1 rank#3] Initialized process group ...
+[node#0 rank#0] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 0 using best-guess GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[node#2 rank#4] Enter Torch DDP.
+[node#0 rank#1] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 1 using best-guess GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[W ProcessGroupNCCL.cpp:1569] Rank 4 using best-guess GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[node#1 rank#3] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 3 using best-guess GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[node#3 rank#7] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 7 using best-guess GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[node#2 rank#5] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 5 using best-guess GPU 1 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[node#3 rank#6] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 6 using best-guess GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+[node#1 rank#2] Enter Torch DDP.
+[W ProcessGroupNCCL.cpp:1569] Rank 2 using best-guess GPU 0 to perform barrier as devices used by this process are currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect.Specify device_ids in barrier() to force use of a particular device.
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+[node#3 rank#7] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#3 rank#7] Start training at: (0, 0, -1)
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+Loading test data: data.arjmPWtGwzKrkmR/test.txt (bsize=8)
+[node#3 rank#6] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#3 rank#6] Start training at: (0, 0, -1)
+[node#3 rank#7] Loading shard data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730 ...
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+AdamW (
+Parameter Group 0
+    betas: (0.9, 0.999)
+    correct_bias: True
+    eps: 1e-06
+    lr: 1e-06
+    weight_decay: 0.01
+)
+[node#1 rank#2] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#1 rank#2] Start training at: (0, 0, -1)
+Loading test data: data.arjmPWtGwzKrkmR/test.txt (bsize=8)
+[node#3 rank#6] Loading shard data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730 ...
+[node#1 rank#3] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#1 rank#3] Start training at: (0, 0, -1)
+[node#2 rank#4] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#2 rank#4] Start training at: (0, 0, -1)
+[node#2 rank#5] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#2 rank#5] Start training at: (0, 0, -1)
+Loading test data: data.arjmPWtGwzKrkmR/test.txt (bsize=8)
+Loading test data: data.arjmPWtGwzKrkmR/test.txt (bsize=8)
+[node#2 rank#4] Loading shard data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730 ...
+[node#2 rank#5] Loading shard data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730 ...
+Loading test data: data.arjmPWtGwzKrkmR/test.txt (bsize=8)
+Loading test data: data.arjmPWtGwzKrkmR/test.txt (bsize=8)
+[node#1 rank#3] Loading shard data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730 ...
+[node#1 rank#2] Loading shard data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730 ...
+[node#0 rank#1] Shards: ['data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.6632730', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7074912', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7517094', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.7959276', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8401458', 'data.arjmPWtGwzKrkmR/mse-aops-2021-data.pkl.pairs.8843640']
+[node#0 rank#1] Start training at: (0, 0, -1)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e609127390ca34a2bdc307d1f66fc68b056d1550973adbee02979df8e091162e
+size 442169891

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-uncased", "tokenizer_class": "BertTokenizer"}