| { |
| "version": "0.3.8", |
| "rust_available": true, |
| "cpu_count": 128, |
| "samples": 10000, |
| "total_bytes": 4353919, |
| "thread_counts": [ |
| 1, |
| 2, |
| 4, |
| 8 |
| ], |
| "runs": 3, |
| "results": { |
| "SARFTokenizer (HF)": { |
| "name": "SARFTokenizer (HF)", |
| "type": "sarf", |
| "vocab_size": 64641, |
| "benchmarks": [ |
| { |
| "threads": 1, |
| "time_sec": 1.2484327759593725, |
| "texts_per_sec": 8010.04282534587, |
| "bytes_per_sec": 3487507.7648087065, |
| "mb_per_sec": 3.325946583565432 |
| }, |
| { |
| "threads": 2, |
| "time_sec": 0.24223483664294085, |
| "texts_per_sec": 41282.25377731365, |
| "bytes_per_sec": 17973958.908386767, |
| "mb_per_sec": 17.141302975069777, |
| "speedup": 5.1538118681070975 |
| }, |
| { |
| "threads": 4, |
| "time_sec": 0.25055884880324203, |
| "texts_per_sec": 39910.78362533811, |
| "bytes_per_sec": 17376831.91312485, |
| "mb_per_sec": 16.57183829605565, |
| "speedup": 4.982593039209473 |
| }, |
| { |
| "threads": 8, |
| "time_sec": 0.23175926196078458, |
| "texts_per_sec": 43148.22163047825, |
| "bytes_per_sec": 18786386.197315022, |
| "mb_per_sec": 17.916094014468214, |
| "speedup": 5.38676541078482 |
| } |
| ], |
| "memory": { |
| "current_mb": 6.103515625e-05, |
| "peak_mb": 0.2074756622314453, |
| "texts_processed": 10000 |
| } |
| }, |
| "SARFTokenizer (Local)": { |
| "name": "SARFTokenizer (Local)", |
| "type": "sarf", |
| "vocab_size": 64641, |
| "benchmarks": [ |
| { |
| "threads": 1, |
| "time_sec": 1.2962954323738813, |
| "texts_per_sec": 7714.2908555090635, |
| "bytes_per_sec": 3358739.7527327165, |
| "mb_per_sec": 3.2031438376738706 |
| }, |
| { |
| "threads": 2, |
| "time_sec": 0.24298642265299955, |
| "texts_per_sec": 41154.562838602105, |
| "bytes_per_sec": 17918363.307968363, |
| "mb_per_sec": 17.08828287884556, |
| "speedup": 5.3348471829023785 |
| }, |
| { |
| "threads": 4, |
| "time_sec": 0.23486537175873914, |
| "texts_per_sec": 42577.58359658189, |
| "bytes_per_sec": 18537935.019524623, |
| "mb_per_sec": 17.679152507328627, |
| "speedup": 5.51931271377662 |
| }, |
| { |
| "threads": 8, |
| "time_sec": 0.24369121653338274, |
| "texts_per_sec": 41035.53727645379, |
| "bytes_per_sec": 17866540.542316042, |
| "mb_per_sec": 17.038860838237802, |
| "speedup": 5.319417953647519 |
| } |
| ], |
| "memory": { |
| "current_mb": 6.103515625e-05, |
| "peak_mb": 0.2074756622314453, |
| "texts_processed": 10000 |
| } |
| }, |
| "tiktoken (o200k)": { |
| "name": "tiktoken (o200k)", |
| "type": "tiktoken", |
| "vocab_size": 200019, |
| "benchmarks": [ |
| { |
| "threads": 1, |
| "time_sec": 0.46731498713294667, |
| "texts_per_sec": 21398.842911826185, |
| "bytes_per_sec": 9316882.873181535, |
| "mb_per_sec": 8.885271905118499 |
| }, |
| { |
| "threads": 2, |
| "time_sec": 0.52791434712708, |
| "texts_per_sec": 18942.466811936807, |
| "bytes_per_sec": 8247396.61593611, |
| "mb_per_sec": 7.865330329834089, |
| "speedup": 0.8852098634486518 |
| }, |
| { |
| "threads": 4, |
| "time_sec": 0.48348489341636497, |
| "texts_per_sec": 20683.169497476425, |
| "bytes_per_sec": 9005284.465528306, |
| "mb_per_sec": 8.58810850670653, |
| "speedup": 0.966555508758175 |
| }, |
| { |
| "threads": 8, |
| "time_sec": 0.8265109478185574, |
| "texts_per_sec": 12099.053287065815, |
| "bytes_per_sec": 5267829.798856831, |
| "mb_per_sec": 5.023793982369262, |
| "speedup": 0.5654068931165998 |
| } |
| ], |
| "memory": { |
| "current_mb": 0.0, |
| "peak_mb": 0.21294403076171875, |
| "texts_processed": 10000 |
| } |
| }, |
| "tiktoken (cl100k)": { |
| "name": "tiktoken (cl100k)", |
| "type": "tiktoken", |
| "vocab_size": 100277, |
| "benchmarks": [ |
| { |
| "threads": 1, |
| "time_sec": 0.3374793001761039, |
| "texts_per_sec": 29631.447009584845, |
| "bytes_per_sec": 12901292.013252463, |
| "mb_per_sec": 12.303630841495956 |
| }, |
| { |
| "threads": 2, |
| "time_sec": 0.5219240722556909, |
| "texts_per_sec": 19159.875030828225, |
| "bytes_per_sec": 8342054.393434861, |
| "mb_per_sec": 7.955603021082745, |
| "speedup": 0.6466061216865516 |
| }, |
| { |
| "threads": 4, |
| "time_sec": 0.514385993902882, |
| "texts_per_sec": 19440.653747442506, |
| "bytes_per_sec": 8464303.172341112, |
| "mb_per_sec": 8.072188541737663, |
| "speedup": 0.6560818221652849 |
| }, |
| { |
| "threads": 8, |
| "time_sec": 0.8330062093834082, |
| "texts_per_sec": 12004.71243473924, |
| "bytes_per_sec": 5226754.555914744, |
| "mb_per_sec": 4.984621578135246, |
| "speedup": 0.4051341951291171 |
| } |
| ], |
| "memory": { |
| "current_mb": 0.0, |
| "peak_mb": 0.4065704345703125, |
| "texts_processed": 10000 |
| } |
| } |
| } |
| } |
