Upload folder using huggingface_hub
Browse files- README.md +1 -1
- merges.txt +0 -0
- tokenizer.json +2 -2
- tokenizer_config.json +2 -2
- vocab.json +0 -0
README.md
CHANGED
|
@@ -46,4 +46,4 @@ tokens = tokenizer.encode("Hello, world!")
|
|
| 46 |
## Sample Encoding
|
| 47 |
| Text | Tokens | Token IDs |
|
| 48 |
|------|--------|-----------|
|
| 49 |
-
| `
|
|
|
|
| 46 |
## Sample Encoding
|
| 47 |
| Text | Tokens | Token IDs |
|
| 48 |
|------|--------|-----------|
|
| 49 |
+
| `123500119 mod 67` | `1235, 00, 119, , mod, , 67` | `2294, 87, 134, 6, 4, 6, 53` |
|
merges.txt
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
tokenizer.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:6dae90ea60ba79a55e839445adb30e231c6364c4af0a880a23b705dea6b1c6ea
|
| 3 |
+
size 6668460
|
tokenizer_config.json
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e84647e941ebc01bde9fcdafa000745ca6f272a7c5ad3595129f0dece939a4e5
|
| 3 |
+
size 1558
|
vocab.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|