flexitok/bpe_vie_Latn_4000_v2
Updated
fineweb2-hq-tokenizers-v2
-
-
flexitok/bpe_dan_Latn_4000_v2
Updated -
flexitok/bpe_swe_Latn_4000_v2
Updated -
flexitok/bpe_fas_Arab_4000_v2
Updated -
flexitok/bpe_ell_Grek_4000_v2
Updated -
flexitok/bpe_hun_Latn_4000_v2
Updated -
flexitok/bpe_tur_Latn_4000_v2
Updated -
flexitok/bpe_ces_Latn_4000_v2
Updated -
flexitok/bpe_ind_Latn_4000_v2
Updated -
flexitok/bpe_arb_Arab_4000_v2
Updated -
flexitok/bpe_pol_Latn_4000_v2
Updated -
flexitok/bpe_nld_Latn_4000_v2
Updated -
flexitok/bpe_por_Latn_4000_v2
Updated -
flexitok/bpe_ita_Latn_4000_v2
Updated -
flexitok/bpe_fra_Latn_4000_v2
Updated -
flexitok/bpe_tur_Latn_1000_v2
Updated -
flexitok/bpe_spa_Latn_4000_v2
Updated -
flexitok/bpe_hun_Latn_32000_v2
Updated -
flexitok/bpe_tur_Latn_16000_v2
Updated -
flexitok/bpe_tur_Latn_8000_v2
Updated -
flexitok/bpe_tur_Latn_32000_v2
Updated -
flexitok/bpe_ind_Latn_8000_v2
Updated -
flexitok/bpe_deu_Latn_4000_v2
Updated -
flexitok/bpe_ita_Latn_16000_v2
Updated -
flexitok/bpe_fw_edu_4000_v2
Updated -
flexitok/bpe_rus_Cyrl_4000_v2
Updated -
flexitok/bpe_dan_Latn_8000_v2
Updated -
flexitok/bpe_vie_Latn_8000_v2
Updated -
flexitok/bpe_dan_Latn_32000_v2
Updated -
flexitok/bpe_swe_Latn_8000_v2
Updated -
flexitok/bpe_fas_Arab_16000_v2
Updated -
flexitok/bpe_fas_Arab_8000_v2
Updated -
flexitok/bpe_swe_Latn_32000_v2
Updated -
flexitok/bpe_ell_Grek_8000_v2
Updated -
flexitok/bpe_hun_Latn_8000_v2
Updated -
flexitok/bpe_arb_Arab_16000_v2
Updated -
flexitok/bpe_ces_Latn_8000_v2
Updated -
flexitok/bpe_ces_Latn_32000_v2
Updated -
flexitok/bpe_dan_Latn_16000_v2
Updated -
flexitok/bpe_ell_Grek_32000_v2
Updated -
flexitok/bpe_pol_Latn_8000_v2
Updated -
flexitok/bpe_vie_Latn_16000_v2
Updated -
flexitok/bpe_hun_Latn_16000_v2
Updated -
flexitok/bpe_nld_Latn_16000_v2
Updated -
flexitok/bpe_arb_Arab_32000_v2
Updated -
flexitok/bpe_ind_Latn_32000_v2
Updated • 1 -
flexitok/bpe_vie_Latn_32000_v2
Updated -
flexitok/bpe_ind_Latn_16000_v2
Updated -
flexitok/bpe_por_Latn_16000_v2
Updated -
flexitok/bpe_fas_Arab_32000_v2
Updated -
flexitok/bpe_pol_Latn_32000_v2
Updated -
flexitok/bpe_arb_Arab_8000_v2
Updated -
flexitok/bpe_nld_Latn_32000_v2
Updated -
flexitok/bpe_pol_Latn_16000_v2
Updated -
flexitok/bpe_por_Latn_8000_v2
Updated -
flexitok/bpe_swe_Latn_16000_v2
Updated -
flexitok/bpe_ces_Latn_16000_v2
Updated -
flexitok/bpe_ell_Grek_16000_v2
Updated -
flexitok/bpe_ita_Latn_8000_v2
Updated -
flexitok/bpe_nld_Latn_8000_v2
Updated -
flexitok/bpe_ita_Latn_32000_v2
Updated -
flexitok/bpe_por_Latn_32000_v2
Updated -
flexitok/bpe_fra_Latn_8000_v2
Updated -
flexitok/bpe_fra_Latn_32000_v2
Updated -
flexitok/bpe_fra_Latn_16000_v2
Updated -
flexitok/bpe_spa_Latn_8000_v2
Updated -
flexitok/bpe_spa_Latn_32000_v2
Updated -
flexitok/bpe_spa_Latn_16000_v2
Updated -
flexitok/bpe_deu_Latn_8000_v2
Updated -
flexitok/bpe_deu_Latn_16000_v2
Updated -
flexitok/bpe_deu_Latn_32000_v2
Updated -
flexitok/bpe_rus_Cyrl_8000_v2
Updated -
flexitok/bpe_rus_Cyrl_16000_v2
Updated -
flexitok/bpe_rus_Cyrl_32000_v2
Updated -
flexitok/bpe_fw_edu_16000_v2
Updated -
flexitok/bpe_fw_edu_8000_v2
Updated -
flexitok/bpe_fw_edu_32000_v2
Updated -
flexitok/bpe_ltr_vie_Latn_1000_v2
Updated -
flexitok/bpe_ltr_swe_Latn_8000_v2
Updated -
flexitok/bpe_ltr_dan_Latn_16000_v2
Updated -
flexitok/bpe_ltr_dan_Latn_8000_v2
Updated -
flexitok/bpe_ltr_dan_Latn_32000_v2
Updated -
flexitok/bpe_ltr_ell_Grek_4000_v2
Updated -
flexitok/bpe_ltr_hun_Latn_4000_v2
Updated -
flexitok/bpe_ltr_tur_Latn_1000_v2
Updated -
flexitok/bpe_ltr_tur_Latn_4000_v2
Updated -
flexitok/bpe_ltr_tur_Latn_8000_v2
Updated -
flexitok/bpe_ltr_hun_Latn_16000_v2
Updated -
flexitok/bpe_ltr_tur_Latn_32000_v2
Updated -
flexitok/bpe_ltr_dan_Latn_1000_v2
Updated -
flexitok/bpe_ltr_dan_Latn_4000_v2
Updated -
flexitok/bpe_ltr_vie_Latn_4000_v2
Updated -
flexitok/bpe_ltr_swe_Latn_1000_v2
Updated -
flexitok/bpe_ltr_swe_Latn_4000_v2
Updated -
flexitok/bpe_ltr_hun_Latn_1000_v2
Updated -
flexitok/bpe_ltr_ell_Grek_1000_v2
Updated -
flexitok/bpe_ltr_ell_Grek_8000_v2
Updated -
flexitok/bpe_ltr_ces_Latn_8000_v2
Updated -
flexitok/bpe_ltr_ces_Latn_4000_v2
Updated -
flexitok/bpe_ltr_ces_Latn_1000_v2
Updated -
flexitok/bpe_ltr_ind_Latn_4000_v2
Updated -
flexitok/bpe_ltr_ind_Latn_1000_v2
Updated -
flexitok/bpe_ltr_pol_Latn_4000_v2
Updated -
flexitok/bpe_ltr_nld_Latn_1000_v2
Updated -
flexitok/bpe_ltr_pol_Latn_1000_v2
Updated -
flexitok/bpe_ltr_nld_Latn_4000_v2
Updated -
flexitok/bpe_ltr_hun_Latn_8000_v2
Updated -
flexitok/bpe_ltr_vie_Latn_8000_v2
Updated -
flexitok/bpe_ltr_ind_Latn_8000_v2
Updated -
flexitok/bpe_ltr_ell_Grek_16000_v2
Updated -
flexitok/bpe_ltr_ces_Latn_16000_v2
Updated -
flexitok/bpe_ltr_por_Latn_1000_v2
Updated -
flexitok/bpe_ltr_por_Latn_4000_v2
Updated -
flexitok/bpe_ltr_nld_Latn_8000_v2
Updated -
flexitok/bpe_ltr_pol_Latn_8000_v2
Updated -
flexitok/bpe_ltr_swe_Latn_16000_v2
Updated -
flexitok/bpe_ltr_ind_Latn_16000_v2
Updated -
flexitok/bpe_ltr_ita_Latn_4000_v2
Updated -
flexitok/bpe_ltr_ita_Latn_1000_v2
Updated -
flexitok/bpe_ltr_vie_Latn_16000_v2
Updated -
flexitok/bpe_ltr_tur_Latn_16000_v2
Updated -
flexitok/bpe_ltr_pol_Latn_16000_v2
Updated -
flexitok/bpe_ltr_nld_Latn_16000_v2
Updated -
flexitok/bpe_ltr_por_Latn_8000_v2
Updated -
flexitok/bpe_ltr_ita_Latn_8000_v2
Updated -
flexitok/bpe_ltr_ces_Latn_32000_v2
Updated -
flexitok/bpe_ltr_ell_Grek_32000_v2
Updated -
flexitok/bpe_ltr_hun_Latn_32000_v2
Updated -
flexitok/bpe_ltr_vie_Latn_32000_v2
Updated -
flexitok/bpe_ltr_swe_Latn_32000_v2
Updated -
flexitok/bpe_ltr_por_Latn_16000_v2
Updated -
flexitok/bpe_ltr_ind_Latn_32000_v2
Updated -
flexitok/bpe_ltr_ita_Latn_16000_v2
Updated -
flexitok/bpe_ltr_pol_Latn_32000_v2
Updated -
flexitok/bpe_ltr_nld_Latn_32000_v2
Updated -
flexitok/bpe_ltr_fra_Latn_4000_v2
Updated -
flexitok/bpe_ltr_fra_Latn_1000_v2
Updated -
flexitok/bpe_ltr_por_Latn_32000_v2
Updated -
flexitok/bpe_ltr_spa_Latn_1000_v2
Updated -
flexitok/bpe_ltr_ita_Latn_32000_v2
Updated -
flexitok/bpe_ltr_spa_Latn_4000_v2
Updated -
flexitok/bpe_ltr_fra_Latn_8000_v2
Updated -
flexitok/bpe_ltr_spa_Latn_8000_v2
Updated -
flexitok/bpe_ltr_fra_Latn_16000_v2
Updated -
flexitok/bpe_ltr_spa_Latn_16000_v2
Updated -
flexitok/bpe_ltr_deu_Latn_1000_v2
Updated -
flexitok/bpe_ltr_deu_Latn_4000_v2
Updated -
flexitok/bpe_ltr_deu_Latn_8000_v2
Updated -
flexitok/bpe_ltr_fra_Latn_32000_v2
Updated -
flexitok/bpe_ltr_spa_Latn_32000_v2
Updated -
flexitok/bpe_ltr_deu_Latn_16000_v2
Updated -
flexitok/bpe_ltr_deu_Latn_32000_v2
Updated -
flexitok/bpe_ltr_rus_Cyrl_1000_v2
Updated -
flexitok/bpe_ltr_rus_Cyrl_4000_v2
Updated -
flexitok/bpe_ltr_rus_Cyrl_8000_v2
Updated -
flexitok/bpe_ltr_fw_edu_1000_v2
Updated -
flexitok/bpe_ltr_rus_Cyrl_16000_v2
Updated -
flexitok/bpe_ltr_fw_edu_4000_v2
Updated -
flexitok/bpe_ltr_fw_edu_8000_v2
Updated -
flexitok/bpe_ltr_rus_Cyrl_32000_v2
Updated -
flexitok/bpe_ltr_fw_edu_16000_v2
Updated -
flexitok/bpe_ltr_fw_edu_32000_v2
Updated -
flexitok/bpe_ltr_fas_Arab_1000_v2
Updated -
flexitok/bpe_ltr_fas_Arab_4000_v2
Updated -
flexitok/bpe_ltr_arb_Arab_1000_v2
Updated -
flexitok/bpe_ltr_jpn_Jpan_1000_v2
Updated -
flexitok/bpe_ltr_arb_Arab_4000_v2
Updated -
flexitok/bpe_ltr_fas_Arab_8000_v2
Updated -
flexitok/bpe_ltr_arb_Arab_8000_v2
Updated -
flexitok/bpe_ltr_fas_Arab_16000_v2
Updated -
flexitok/bpe_ltr_arb_Arab_16000_v2
Updated -
flexitok/bpe_ltr_arb_Arab_32000_v2
Updated -
flexitok/bpe_ltr_fas_Arab_32000_v2
Updated -
flexitok/bpe_ltr_cmn_Hani_1000_v2
Updated -
flexitok/bpe_ltr_jpn_Jpan_8000_v2
Updated -
flexitok/bpe_ltr_jpn_Jpan_4000_v2
Updated -
flexitok/bpe_ltr_jpn_Jpan_16000_v2
Updated -
flexitok/bpe_ltr_jpn_Jpan_32000_v2
Updated -
flexitok/bpe_ltr_cmn_Hani_8000_v2
Updated -
flexitok/bpe_ltr_cmn_Hani_4000_v2
Updated -
flexitok/bpe_ltr_cmn_Hani_16000_v2
Updated -
flexitok/bpe_ltr_cmn_Hani_32000_v2
Updated