gsaltintas commited on
Commit
f209ee9
·
verified ·
1 Parent(s): e7fd830

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. README.md +1 -1
  2. merges.txt +0 -0
  3. tokenizer.json +2 -2
  4. tokenizer_config.json +2 -2
  5. vocab.json +0 -0
README.md CHANGED
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
- | `1234500119 mod 67` | `12345, 0, 0, 119, , mod, , 67` | `40377, 67819, 67819, 70599, 6, 4, 6, 48909` |
 
46
  ## Sample Encoding
47
  | Text | Tokens | Token IDs |
48
  |------|--------|-----------|
49
+ | `123500119 mod 67` | `1235, 00, 119, , mod, , 67` | `2294, 87, 134, 6, 4, 6, 53` |
merges.txt CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:d9c6a4df9b046d6b9240cce74f0d5e05fbe7c59548f366bcfde7dfd29ca3a4db
3
- size 18078802
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6dae90ea60ba79a55e839445adb30e231c6364c4af0a880a23b705dea6b1c6ea
3
+ size 6668460
tokenizer_config.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:6e987318a04452b25918150fdcb99fbaedee52f798bde7e811724f10e4218f4f
3
- size 17078169
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e84647e941ebc01bde9fcdafa000745ca6f272a7c5ad3595129f0dece939a4e5
3
+ size 1558
vocab.json CHANGED
The diff for this file is too large to render. See raw diff