Add quantization validation reports
Browse files
reports/validation/q4_k_m_vs_bf16.json
ADDED
|
@@ -0,0 +1,800 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bf16": "fresh\\out\\gguf\\HRM-Text-1B-BF16.gguf",
|
| 3 |
+
"quant": "fresh\\out\\gguf\\HRM-Text-1B-Q4_K_M.gguf",
|
| 4 |
+
"quant_name": "Q4_K_M",
|
| 5 |
+
"created_at_unix": 1779350220.949973,
|
| 6 |
+
"elapsed_seconds": 88.01678323745728,
|
| 7 |
+
"metadata": {
|
| 8 |
+
"bf16_general_architecture": "hrm_text",
|
| 9 |
+
"quant_general_architecture": "hrm_text"
|
| 10 |
+
},
|
| 11 |
+
"checks": {
|
| 12 |
+
"prompt_tokens_match_all_prompts": true,
|
| 13 |
+
"same_next_top1_at_least_3_of_4": true,
|
| 14 |
+
"top1_match_count": 3,
|
| 15 |
+
"top10_overlap_at_least_7_all_prompts": true,
|
| 16 |
+
"quant_general_architecture_is_hrm_text": true,
|
| 17 |
+
"no_new_repetition_loop": false,
|
| 18 |
+
"passed": false
|
| 19 |
+
},
|
| 20 |
+
"prompts": [
|
| 21 |
+
{
|
| 22 |
+
"prompt": "The quick brown fox",
|
| 23 |
+
"bf16_input_ids": [
|
| 24 |
+
341,
|
| 25 |
+
3568,
|
| 26 |
+
9054,
|
| 27 |
+
20102
|
| 28 |
+
],
|
| 29 |
+
"quant_input_ids": [
|
| 30 |
+
341,
|
| 31 |
+
3568,
|
| 32 |
+
9054,
|
| 33 |
+
20102
|
| 34 |
+
],
|
| 35 |
+
"tokens_match": true,
|
| 36 |
+
"bf16_next_top20": [
|
| 37 |
+
{
|
| 38 |
+
"token_id": 20102,
|
| 39 |
+
"logit": 14.297157287597656
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"token_id": 40036,
|
| 43 |
+
"logit": 7.994879722595215
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"token_id": 11979,
|
| 47 |
+
"logit": 6.699657440185547
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"token_id": 3568,
|
| 51 |
+
"logit": 6.626206398010254
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"token_id": 4349,
|
| 55 |
+
"logit": 6.35108757019043
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"token_id": 42,
|
| 59 |
+
"logit": 6.271071434020996
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"token_id": 3152,
|
| 63 |
+
"logit": 5.424627304077148
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"token_id": 2301,
|
| 67 |
+
"logit": 5.3609843254089355
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"token_id": 44,
|
| 71 |
+
"logit": 5.178180694580078
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"token_id": 3558,
|
| 75 |
+
"logit": 4.875790596008301
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"token_id": 5002,
|
| 79 |
+
"logit": 4.839702606201172
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"token_id": 3412,
|
| 83 |
+
"logit": 4.824095249176025
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"token_id": 761,
|
| 87 |
+
"logit": 4.718862056732178
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"token_id": 19444,
|
| 91 |
+
"logit": 4.698058128356934
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"token_id": 446,
|
| 95 |
+
"logit": 4.689098358154297
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"token_id": 16001,
|
| 99 |
+
"logit": 4.590203762054443
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"token_id": 3297,
|
| 103 |
+
"logit": 4.408023357391357
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"token_id": 322,
|
| 107 |
+
"logit": 4.394444942474365
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"token_id": 8636,
|
| 111 |
+
"logit": 4.318354606628418
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"token_id": 12672,
|
| 115 |
+
"logit": 4.2834038734436035
|
| 116 |
+
}
|
| 117 |
+
],
|
| 118 |
+
"quant_next_top20": [
|
| 119 |
+
{
|
| 120 |
+
"token_id": 20102,
|
| 121 |
+
"logit": 14.285414695739746
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"token_id": 40036,
|
| 125 |
+
"logit": 8.034632682800293
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"token_id": 44,
|
| 129 |
+
"logit": 6.610910892486572
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"token_id": 42,
|
| 133 |
+
"logit": 6.567035675048828
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"token_id": 11979,
|
| 137 |
+
"logit": 6.461189270019531
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"token_id": 3568,
|
| 141 |
+
"logit": 6.058647632598877
|
| 142 |
+
},
|
| 143 |
+
{
|
| 144 |
+
"token_id": 4349,
|
| 145 |
+
"logit": 6.0295610427856445
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"token_id": 2301,
|
| 149 |
+
"logit": 5.308095932006836
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"token_id": 3152,
|
| 153 |
+
"logit": 5.21979284286499
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"token_id": 3297,
|
| 157 |
+
"logit": 4.831073760986328
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"token_id": 5002,
|
| 161 |
+
"logit": 4.794489860534668
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"token_id": 19444,
|
| 165 |
+
"logit": 4.792886734008789
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"token_id": 16001,
|
| 169 |
+
"logit": 4.7607221603393555
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"token_id": 12672,
|
| 173 |
+
"logit": 4.729791641235352
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"token_id": 3558,
|
| 177 |
+
"logit": 4.723749160766602
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"token_id": 3412,
|
| 181 |
+
"logit": 4.651169776916504
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"token_id": 8636,
|
| 185 |
+
"logit": 4.551083087921143
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"token_id": 322,
|
| 189 |
+
"logit": 4.406239986419678
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"token_id": 761,
|
| 193 |
+
"logit": 4.104559421539307
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
"token_id": 446,
|
| 197 |
+
"logit": 4.097617149353027
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"top1_match": true,
|
| 201 |
+
"top10_overlap": 9,
|
| 202 |
+
"full_vocab_mean_abs_logit_error": 0.11568036675453186,
|
| 203 |
+
"top20_union_mean_abs_logit_error": 0.30603691935539246,
|
| 204 |
+
"bf16_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 205 |
+
"quant_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 206 |
+
"bf16_repetition_loop": true,
|
| 207 |
+
"quant_repetition_loop": true,
|
| 208 |
+
"new_repetition_loop": false
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"prompt": "In a distant future, humanity",
|
| 212 |
+
"bf16_input_ids": [
|
| 213 |
+
931,
|
| 214 |
+
236,
|
| 215 |
+
95,
|
| 216 |
+
9494,
|
| 217 |
+
2025,
|
| 218 |
+
42,
|
| 219 |
+
13279
|
| 220 |
+
],
|
| 221 |
+
"quant_input_ids": [
|
| 222 |
+
931,
|
| 223 |
+
236,
|
| 224 |
+
95,
|
| 225 |
+
9494,
|
| 226 |
+
2025,
|
| 227 |
+
42,
|
| 228 |
+
13279
|
| 229 |
+
],
|
| 230 |
+
"tokens_match": true,
|
| 231 |
+
"bf16_next_top20": [
|
| 232 |
+
{
|
| 233 |
+
"token_id": 385,
|
| 234 |
+
"logit": 8.094616889953613
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"token_id": 42,
|
| 238 |
+
"logit": 7.381193161010742
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"token_id": 322,
|
| 242 |
+
"logit": 7.22892951965332
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"token_id": 5106,
|
| 246 |
+
"logit": 6.968372344970703
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"token_id": 5835,
|
| 250 |
+
"logit": 6.5758562088012695
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"token_id": 769,
|
| 254 |
+
"logit": 6.294882297515869
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"token_id": 599,
|
| 258 |
+
"logit": 6.259007453918457
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"token_id": 1980,
|
| 262 |
+
"logit": 6.067761421203613
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"token_id": 295,
|
| 266 |
+
"logit": 6.029950141906738
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"token_id": 2156,
|
| 270 |
+
"logit": 5.428796768188477
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"token_id": 379,
|
| 274 |
+
"logit": 5.263803005218506
|
| 275 |
+
},
|
| 276 |
+
{
|
| 277 |
+
"token_id": 5226,
|
| 278 |
+
"logit": 5.066612243652344
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"token_id": 446,
|
| 282 |
+
"logit": 4.910100936889648
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"token_id": 10755,
|
| 286 |
+
"logit": 4.904460430145264
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"token_id": 236,
|
| 290 |
+
"logit": 4.739671230316162
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"token_id": 2026,
|
| 294 |
+
"logit": 4.661045074462891
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"token_id": 307,
|
| 298 |
+
"logit": 4.627893447875977
|
| 299 |
+
},
|
| 300 |
+
{
|
| 301 |
+
"token_id": 1287,
|
| 302 |
+
"logit": 4.507448196411133
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"token_id": 3891,
|
| 306 |
+
"logit": 4.162245273590088
|
| 307 |
+
},
|
| 308 |
+
{
|
| 309 |
+
"token_id": 5671,
|
| 310 |
+
"logit": 4.059610366821289
|
| 311 |
+
}
|
| 312 |
+
],
|
| 313 |
+
"quant_next_top20": [
|
| 314 |
+
{
|
| 315 |
+
"token_id": 42,
|
| 316 |
+
"logit": 8.654781341552734
|
| 317 |
+
},
|
| 318 |
+
{
|
| 319 |
+
"token_id": 385,
|
| 320 |
+
"logit": 7.868644714355469
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"token_id": 322,
|
| 324 |
+
"logit": 7.357276916503906
|
| 325 |
+
},
|
| 326 |
+
{
|
| 327 |
+
"token_id": 769,
|
| 328 |
+
"logit": 6.930787563323975
|
| 329 |
+
},
|
| 330 |
+
{
|
| 331 |
+
"token_id": 5835,
|
| 332 |
+
"logit": 6.524726390838623
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"token_id": 5106,
|
| 336 |
+
"logit": 6.356590270996094
|
| 337 |
+
},
|
| 338 |
+
{
|
| 339 |
+
"token_id": 295,
|
| 340 |
+
"logit": 5.941123962402344
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"token_id": 1980,
|
| 344 |
+
"logit": 5.909348487854004
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"token_id": 599,
|
| 348 |
+
"logit": 5.899454116821289
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"token_id": 2156,
|
| 352 |
+
"logit": 5.540234565734863
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"token_id": 379,
|
| 356 |
+
"logit": 5.285621643066406
|
| 357 |
+
},
|
| 358 |
+
{
|
| 359 |
+
"token_id": 5226,
|
| 360 |
+
"logit": 5.137611389160156
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"token_id": 307,
|
| 364 |
+
"logit": 5.102499961853027
|
| 365 |
+
},
|
| 366 |
+
{
|
| 367 |
+
"token_id": 1287,
|
| 368 |
+
"logit": 4.631147384643555
|
| 369 |
+
},
|
| 370 |
+
{
|
| 371 |
+
"token_id": 236,
|
| 372 |
+
"logit": 4.547019958496094
|
| 373 |
+
},
|
| 374 |
+
{
|
| 375 |
+
"token_id": 446,
|
| 376 |
+
"logit": 4.481627464294434
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"token_id": 2026,
|
| 380 |
+
"logit": 4.348045825958252
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"token_id": 5111,
|
| 384 |
+
"logit": 4.30531644821167
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"token_id": 10755,
|
| 388 |
+
"logit": 4.068432807922363
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"token_id": 11702,
|
| 392 |
+
"logit": 4.025401592254639
|
| 393 |
+
}
|
| 394 |
+
],
|
| 395 |
+
"top1_match": false,
|
| 396 |
+
"top10_overlap": 10,
|
| 397 |
+
"full_vocab_mean_abs_logit_error": 0.18694043159484863,
|
| 398 |
+
"top20_union_mean_abs_logit_error": 0.34929159283638,
|
| 399 |
+
"bf16_generated_suffix": "'s distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant\n\n",
|
| 400 |
+
"quant_generated_suffix": ", humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity, humanity\n\n",
|
| 401 |
+
"bf16_repetition_loop": false,
|
| 402 |
+
"quant_repetition_loop": true,
|
| 403 |
+
"new_repetition_loop": true
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"prompt": "Question: What is 2+2?\nAnswer:",
|
| 407 |
+
"bf16_input_ids": [
|
| 408 |
+
1069,
|
| 409 |
+
56,
|
| 410 |
+
866,
|
| 411 |
+
322,
|
| 412 |
+
236,
|
| 413 |
+
48,
|
| 414 |
+
41,
|
| 415 |
+
48,
|
| 416 |
+
1047,
|
| 417 |
+
1180,
|
| 418 |
+
56
|
| 419 |
+
],
|
| 420 |
+
"quant_input_ids": [
|
| 421 |
+
1069,
|
| 422 |
+
56,
|
| 423 |
+
866,
|
| 424 |
+
322,
|
| 425 |
+
236,
|
| 426 |
+
48,
|
| 427 |
+
41,
|
| 428 |
+
48,
|
| 429 |
+
1047,
|
| 430 |
+
1180,
|
| 431 |
+
56
|
| 432 |
+
],
|
| 433 |
+
"tokens_match": true,
|
| 434 |
+
"bf16_next_top20": [
|
| 435 |
+
{
|
| 436 |
+
"token_id": 236,
|
| 437 |
+
"logit": 9.97547435760498
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"token_id": 391,
|
| 441 |
+
"logit": 6.012524604797363
|
| 442 |
+
},
|
| 443 |
+
{
|
| 444 |
+
"token_id": 315,
|
| 445 |
+
"logit": 5.968222618103027
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"token_id": 866,
|
| 449 |
+
"logit": 5.672043323516846
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"token_id": 395,
|
| 453 |
+
"logit": 5.606063365936279
|
| 454 |
+
},
|
| 455 |
+
{
|
| 456 |
+
"token_id": 5444,
|
| 457 |
+
"logit": 5.2098188400268555
|
| 458 |
+
},
|
| 459 |
+
{
|
| 460 |
+
"token_id": 4746,
|
| 461 |
+
"logit": 4.777042388916016
|
| 462 |
+
},
|
| 463 |
+
{
|
| 464 |
+
"token_id": 1827,
|
| 465 |
+
"logit": 4.54682731628418
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"token_id": 369,
|
| 469 |
+
"logit": 3.9637503623962402
|
| 470 |
+
},
|
| 471 |
+
{
|
| 472 |
+
"token_id": 970,
|
| 473 |
+
"logit": 3.765744686126709
|
| 474 |
+
},
|
| 475 |
+
{
|
| 476 |
+
"token_id": 725,
|
| 477 |
+
"logit": 3.7245254516601562
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"token_id": 421,
|
| 481 |
+
"logit": 3.630314350128174
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"token_id": 731,
|
| 485 |
+
"logit": 3.2161593437194824
|
| 486 |
+
},
|
| 487 |
+
{
|
| 488 |
+
"token_id": 401,
|
| 489 |
+
"logit": 3.1881794929504395
|
| 490 |
+
},
|
| 491 |
+
{
|
| 492 |
+
"token_id": 2286,
|
| 493 |
+
"logit": 3.168586254119873
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"token_id": 1422,
|
| 497 |
+
"logit": 3.151968240737915
|
| 498 |
+
},
|
| 499 |
+
{
|
| 500 |
+
"token_id": 10528,
|
| 501 |
+
"logit": 3.148939847946167
|
| 502 |
+
},
|
| 503 |
+
{
|
| 504 |
+
"token_id": 1544,
|
| 505 |
+
"logit": 3.1278254985809326
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"token_id": 973,
|
| 509 |
+
"logit": 3.1256284713745117
|
| 510 |
+
},
|
| 511 |
+
{
|
| 512 |
+
"token_id": 4286,
|
| 513 |
+
"logit": 3.0892391204833984
|
| 514 |
+
}
|
| 515 |
+
],
|
| 516 |
+
"quant_next_top20": [
|
| 517 |
+
{
|
| 518 |
+
"token_id": 236,
|
| 519 |
+
"logit": 9.75040054321289
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"token_id": 315,
|
| 523 |
+
"logit": 5.8260698318481445
|
| 524 |
+
},
|
| 525 |
+
{
|
| 526 |
+
"token_id": 391,
|
| 527 |
+
"logit": 5.598740577697754
|
| 528 |
+
},
|
| 529 |
+
{
|
| 530 |
+
"token_id": 395,
|
| 531 |
+
"logit": 5.483684539794922
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"token_id": 5444,
|
| 535 |
+
"logit": 4.982985496520996
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"token_id": 866,
|
| 539 |
+
"logit": 4.9333295822143555
|
| 540 |
+
},
|
| 541 |
+
{
|
| 542 |
+
"token_id": 4746,
|
| 543 |
+
"logit": 4.889163970947266
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"token_id": 1827,
|
| 547 |
+
"logit": 4.411078453063965
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"token_id": 2286,
|
| 551 |
+
"logit": 3.8549227714538574
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"token_id": 421,
|
| 555 |
+
"logit": 3.721518039703369
|
| 556 |
+
},
|
| 557 |
+
{
|
| 558 |
+
"token_id": 10528,
|
| 559 |
+
"logit": 3.7057790756225586
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"token_id": 369,
|
| 563 |
+
"logit": 3.527693033218384
|
| 564 |
+
},
|
| 565 |
+
{
|
| 566 |
+
"token_id": 541,
|
| 567 |
+
"logit": 3.308436393737793
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"token_id": 1661,
|
| 571 |
+
"logit": 3.251553535461426
|
| 572 |
+
},
|
| 573 |
+
{
|
| 574 |
+
"token_id": 970,
|
| 575 |
+
"logit": 3.241942882537842
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"token_id": 401,
|
| 579 |
+
"logit": 3.202249050140381
|
| 580 |
+
},
|
| 581 |
+
{
|
| 582 |
+
"token_id": 1422,
|
| 583 |
+
"logit": 3.120091676712036
|
| 584 |
+
},
|
| 585 |
+
{
|
| 586 |
+
"token_id": 725,
|
| 587 |
+
"logit": 2.985598087310791
|
| 588 |
+
},
|
| 589 |
+
{
|
| 590 |
+
"token_id": 1544,
|
| 591 |
+
"logit": 2.884519338607788
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"token_id": 973,
|
| 595 |
+
"logit": 2.8762009143829346
|
| 596 |
+
}
|
| 597 |
+
],
|
| 598 |
+
"top1_match": true,
|
| 599 |
+
"top10_overlap": 8,
|
| 600 |
+
"full_vocab_mean_abs_logit_error": 0.22683359682559967,
|
| 601 |
+
"top20_union_mean_abs_logit_error": 0.34118449687957764,
|
| 602 |
+
"bf16_generated_suffix": " 6 [end of text]\n\n\n",
|
| 603 |
+
"quant_generated_suffix": " 6 [end of text]\n\n\n",
|
| 604 |
+
"bf16_repetition_loop": false,
|
| 605 |
+
"quant_repetition_loop": false,
|
| 606 |
+
"new_repetition_loop": false
|
| 607 |
+
},
|
| 608 |
+
{
|
| 609 |
+
"prompt": "def fibonacci(n):",
|
| 610 |
+
"bf16_input_ids": [
|
| 611 |
+
26763,
|
| 612 |
+
8430,
|
| 613 |
+
21728,
|
| 614 |
+
3145,
|
| 615 |
+
10018
|
| 616 |
+
],
|
| 617 |
+
"quant_input_ids": [
|
| 618 |
+
26763,
|
| 619 |
+
8430,
|
| 620 |
+
21728,
|
| 621 |
+
3145,
|
| 622 |
+
10018
|
| 623 |
+
],
|
| 624 |
+
"tokens_match": true,
|
| 625 |
+
"bf16_next_top20": [
|
| 626 |
+
{
|
| 627 |
+
"token_id": 56,
|
| 628 |
+
"logit": 10.327644348144531
|
| 629 |
+
},
|
| 630 |
+
{
|
| 631 |
+
"token_id": 2904,
|
| 632 |
+
"logit": 7.035848140716553
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"token_id": 11,
|
| 636 |
+
"logit": 6.860950946807861
|
| 637 |
+
},
|
| 638 |
+
{
|
| 639 |
+
"token_id": 10018,
|
| 640 |
+
"logit": 6.495933532714844
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"token_id": 108,
|
| 644 |
+
"logit": 6.341927528381348
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"token_id": 318,
|
| 648 |
+
"logit": 6.146999359130859
|
| 649 |
+
},
|
| 650 |
+
{
|
| 651 |
+
"token_id": 236,
|
| 652 |
+
"logit": 6.0702805519104
|
| 653 |
+
},
|
| 654 |
+
{
|
| 655 |
+
"token_id": 38,
|
| 656 |
+
"logit": 5.865842819213867
|
| 657 |
+
},
|
| 658 |
+
{
|
| 659 |
+
"token_id": 68,
|
| 660 |
+
"logit": 5.02256965637207
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"token_id": 39,
|
| 664 |
+
"logit": 4.66829252243042
|
| 665 |
+
},
|
| 666 |
+
{
|
| 667 |
+
"token_id": 76,
|
| 668 |
+
"logit": 4.6676859855651855
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"token_id": 59,
|
| 672 |
+
"logit": 4.662688255310059
|
| 673 |
+
},
|
| 674 |
+
{
|
| 675 |
+
"token_id": 57,
|
| 676 |
+
"logit": 4.66217041015625
|
| 677 |
+
},
|
| 678 |
+
{
|
| 679 |
+
"token_id": 1941,
|
| 680 |
+
"logit": 4.598584175109863
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"token_id": 401,
|
| 684 |
+
"logit": 4.574923515319824
|
| 685 |
+
},
|
| 686 |
+
{
|
| 687 |
+
"token_id": 395,
|
| 688 |
+
"logit": 4.532867431640625
|
| 689 |
+
},
|
| 690 |
+
{
|
| 691 |
+
"token_id": 8430,
|
| 692 |
+
"logit": 4.5034050941467285
|
| 693 |
+
},
|
| 694 |
+
{
|
| 695 |
+
"token_id": 95,
|
| 696 |
+
"logit": 4.318176746368408
|
| 697 |
+
},
|
| 698 |
+
{
|
| 699 |
+
"token_id": 364,
|
| 700 |
+
"logit": 4.2872209548950195
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"token_id": 48,
|
| 704 |
+
"logit": 4.243288516998291
|
| 705 |
+
}
|
| 706 |
+
],
|
| 707 |
+
"quant_next_top20": [
|
| 708 |
+
{
|
| 709 |
+
"token_id": 56,
|
| 710 |
+
"logit": 10.145026206970215
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"token_id": 11,
|
| 714 |
+
"logit": 7.260679244995117
|
| 715 |
+
},
|
| 716 |
+
{
|
| 717 |
+
"token_id": 2904,
|
| 718 |
+
"logit": 7.198648452758789
|
| 719 |
+
},
|
| 720 |
+
{
|
| 721 |
+
"token_id": 108,
|
| 722 |
+
"logit": 6.379225254058838
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"token_id": 318,
|
| 726 |
+
"logit": 6.159180641174316
|
| 727 |
+
},
|
| 728 |
+
{
|
| 729 |
+
"token_id": 38,
|
| 730 |
+
"logit": 5.719315052032471
|
| 731 |
+
},
|
| 732 |
+
{
|
| 733 |
+
"token_id": 10018,
|
| 734 |
+
"logit": 5.555755615234375
|
| 735 |
+
},
|
| 736 |
+
{
|
| 737 |
+
"token_id": 236,
|
| 738 |
+
"logit": 5.3692545890808105
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"token_id": 76,
|
| 742 |
+
"logit": 5.005756855010986
|
| 743 |
+
},
|
| 744 |
+
{
|
| 745 |
+
"token_id": 59,
|
| 746 |
+
"logit": 4.653585433959961
|
| 747 |
+
},
|
| 748 |
+
{
|
| 749 |
+
"token_id": 68,
|
| 750 |
+
"logit": 4.610100746154785
|
| 751 |
+
},
|
| 752 |
+
{
|
| 753 |
+
"token_id": 1941,
|
| 754 |
+
"logit": 4.513223171234131
|
| 755 |
+
},
|
| 756 |
+
{
|
| 757 |
+
"token_id": 57,
|
| 758 |
+
"logit": 4.4056077003479
|
| 759 |
+
},
|
| 760 |
+
{
|
| 761 |
+
"token_id": 395,
|
| 762 |
+
"logit": 4.349212646484375
|
| 763 |
+
},
|
| 764 |
+
{
|
| 765 |
+
"token_id": 364,
|
| 766 |
+
"logit": 4.287387847900391
|
| 767 |
+
},
|
| 768 |
+
{
|
| 769 |
+
"token_id": 8430,
|
| 770 |
+
"logit": 4.239393711090088
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"token_id": 540,
|
| 774 |
+
"logit": 4.205807685852051
|
| 775 |
+
},
|
| 776 |
+
{
|
| 777 |
+
"token_id": 401,
|
| 778 |
+
"logit": 4.138382911682129
|
| 779 |
+
},
|
| 780 |
+
{
|
| 781 |
+
"token_id": 391,
|
| 782 |
+
"logit": 4.093988418579102
|
| 783 |
+
},
|
| 784 |
+
{
|
| 785 |
+
"token_id": 3145,
|
| 786 |
+
"logit": 4.082869529724121
|
| 787 |
+
}
|
| 788 |
+
],
|
| 789 |
+
"top1_match": true,
|
| 790 |
+
"top10_overlap": 8,
|
| 791 |
+
"full_vocab_mean_abs_logit_error": 0.15026512742042542,
|
| 792 |
+
"top20_union_mean_abs_logit_error": 0.3331286907196045,
|
| 793 |
+
"bf16_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 794 |
+
"quant_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 795 |
+
"bf16_repetition_loop": false,
|
| 796 |
+
"quant_repetition_loop": false,
|
| 797 |
+
"new_repetition_loop": false
|
| 798 |
+
}
|
| 799 |
+
]
|
| 800 |
+
}
|
reports/validation/q5_k_m_vs_bf16.json
ADDED
|
@@ -0,0 +1,800 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bf16": "fresh\\out\\gguf\\HRM-Text-1B-BF16.gguf",
|
| 3 |
+
"quant": "fresh\\out\\gguf\\HRM-Text-1B-Q5_K_M.gguf",
|
| 4 |
+
"quant_name": "Q5_K_M",
|
| 5 |
+
"created_at_unix": 1779350132.6506834,
|
| 6 |
+
"elapsed_seconds": 84.19470143318176,
|
| 7 |
+
"metadata": {
|
| 8 |
+
"bf16_general_architecture": "hrm_text",
|
| 9 |
+
"quant_general_architecture": "hrm_text"
|
| 10 |
+
},
|
| 11 |
+
"checks": {
|
| 12 |
+
"prompt_tokens_match_all_prompts": true,
|
| 13 |
+
"same_next_top1_at_least_3_of_4": true,
|
| 14 |
+
"top1_match_count": 4,
|
| 15 |
+
"top10_overlap_at_least_7_all_prompts": true,
|
| 16 |
+
"quant_general_architecture_is_hrm_text": true,
|
| 17 |
+
"no_new_repetition_loop": true,
|
| 18 |
+
"passed": true
|
| 19 |
+
},
|
| 20 |
+
"prompts": [
|
| 21 |
+
{
|
| 22 |
+
"prompt": "The quick brown fox",
|
| 23 |
+
"bf16_input_ids": [
|
| 24 |
+
341,
|
| 25 |
+
3568,
|
| 26 |
+
9054,
|
| 27 |
+
20102
|
| 28 |
+
],
|
| 29 |
+
"quant_input_ids": [
|
| 30 |
+
341,
|
| 31 |
+
3568,
|
| 32 |
+
9054,
|
| 33 |
+
20102
|
| 34 |
+
],
|
| 35 |
+
"tokens_match": true,
|
| 36 |
+
"bf16_next_top20": [
|
| 37 |
+
{
|
| 38 |
+
"token_id": 20102,
|
| 39 |
+
"logit": 14.297157287597656
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"token_id": 40036,
|
| 43 |
+
"logit": 7.994879722595215
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"token_id": 11979,
|
| 47 |
+
"logit": 6.699657440185547
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"token_id": 3568,
|
| 51 |
+
"logit": 6.626206398010254
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"token_id": 4349,
|
| 55 |
+
"logit": 6.35108757019043
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"token_id": 42,
|
| 59 |
+
"logit": 6.271071434020996
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"token_id": 3152,
|
| 63 |
+
"logit": 5.424627304077148
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"token_id": 2301,
|
| 67 |
+
"logit": 5.3609843254089355
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"token_id": 44,
|
| 71 |
+
"logit": 5.178180694580078
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"token_id": 3558,
|
| 75 |
+
"logit": 4.875790596008301
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"token_id": 5002,
|
| 79 |
+
"logit": 4.839702606201172
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"token_id": 3412,
|
| 83 |
+
"logit": 4.824095249176025
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"token_id": 761,
|
| 87 |
+
"logit": 4.718862056732178
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"token_id": 19444,
|
| 91 |
+
"logit": 4.698058128356934
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"token_id": 446,
|
| 95 |
+
"logit": 4.689098358154297
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"token_id": 16001,
|
| 99 |
+
"logit": 4.590203762054443
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"token_id": 3297,
|
| 103 |
+
"logit": 4.408023357391357
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"token_id": 322,
|
| 107 |
+
"logit": 4.394444942474365
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"token_id": 8636,
|
| 111 |
+
"logit": 4.318354606628418
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"token_id": 12672,
|
| 115 |
+
"logit": 4.2834038734436035
|
| 116 |
+
}
|
| 117 |
+
],
|
| 118 |
+
"quant_next_top20": [
|
| 119 |
+
{
|
| 120 |
+
"token_id": 20102,
|
| 121 |
+
"logit": 13.345020294189453
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"token_id": 40036,
|
| 125 |
+
"logit": 7.160383701324463
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"token_id": 3568,
|
| 129 |
+
"logit": 6.903276443481445
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"token_id": 42,
|
| 133 |
+
"logit": 6.725387096405029
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"token_id": 4349,
|
| 137 |
+
"logit": 6.552280426025391
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"token_id": 11979,
|
| 141 |
+
"logit": 6.0859174728393555
|
| 142 |
+
},
|
| 143 |
+
{
|
| 144 |
+
"token_id": 44,
|
| 145 |
+
"logit": 5.871312141418457
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"token_id": 3152,
|
| 149 |
+
"logit": 5.2989726066589355
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"token_id": 2301,
|
| 153 |
+
"logit": 4.973495006561279
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"token_id": 5002,
|
| 157 |
+
"logit": 4.772021293640137
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"token_id": 322,
|
| 161 |
+
"logit": 4.630204677581787
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"token_id": 446,
|
| 165 |
+
"logit": 4.590030193328857
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"token_id": 3558,
|
| 169 |
+
"logit": 4.585133075714111
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"token_id": 3412,
|
| 173 |
+
"logit": 4.512120246887207
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"token_id": 761,
|
| 177 |
+
"logit": 4.390371322631836
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"token_id": 22263,
|
| 181 |
+
"logit": 4.160024642944336
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"token_id": 3297,
|
| 185 |
+
"logit": 4.139768123626709
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"token_id": 385,
|
| 189 |
+
"logit": 4.068934440612793
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"token_id": 19444,
|
| 193 |
+
"logit": 4.001613140106201
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
"token_id": 16001,
|
| 197 |
+
"logit": 3.997398853302002
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"top1_match": true,
|
| 201 |
+
"top10_overlap": 9,
|
| 202 |
+
"full_vocab_mean_abs_logit_error": 0.35568374395370483,
|
| 203 |
+
"top20_union_mean_abs_logit_error": 0.3906019330024719,
|
| 204 |
+
"bf16_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 205 |
+
"quant_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 206 |
+
"bf16_repetition_loop": true,
|
| 207 |
+
"quant_repetition_loop": true,
|
| 208 |
+
"new_repetition_loop": false
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"prompt": "In a distant future, humanity",
|
| 212 |
+
"bf16_input_ids": [
|
| 213 |
+
931,
|
| 214 |
+
236,
|
| 215 |
+
95,
|
| 216 |
+
9494,
|
| 217 |
+
2025,
|
| 218 |
+
42,
|
| 219 |
+
13279
|
| 220 |
+
],
|
| 221 |
+
"quant_input_ids": [
|
| 222 |
+
931,
|
| 223 |
+
236,
|
| 224 |
+
95,
|
| 225 |
+
9494,
|
| 226 |
+
2025,
|
| 227 |
+
42,
|
| 228 |
+
13279
|
| 229 |
+
],
|
| 230 |
+
"tokens_match": true,
|
| 231 |
+
"bf16_next_top20": [
|
| 232 |
+
{
|
| 233 |
+
"token_id": 385,
|
| 234 |
+
"logit": 8.094616889953613
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"token_id": 42,
|
| 238 |
+
"logit": 7.381193161010742
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"token_id": 322,
|
| 242 |
+
"logit": 7.22892951965332
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"token_id": 5106,
|
| 246 |
+
"logit": 6.968372344970703
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"token_id": 5835,
|
| 250 |
+
"logit": 6.5758562088012695
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"token_id": 769,
|
| 254 |
+
"logit": 6.294882297515869
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"token_id": 599,
|
| 258 |
+
"logit": 6.259007453918457
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"token_id": 1980,
|
| 262 |
+
"logit": 6.067761421203613
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"token_id": 295,
|
| 266 |
+
"logit": 6.029950141906738
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"token_id": 2156,
|
| 270 |
+
"logit": 5.428796768188477
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"token_id": 379,
|
| 274 |
+
"logit": 5.263803005218506
|
| 275 |
+
},
|
| 276 |
+
{
|
| 277 |
+
"token_id": 5226,
|
| 278 |
+
"logit": 5.066612243652344
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"token_id": 446,
|
| 282 |
+
"logit": 4.910100936889648
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"token_id": 10755,
|
| 286 |
+
"logit": 4.904460430145264
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"token_id": 236,
|
| 290 |
+
"logit": 4.739671230316162
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"token_id": 2026,
|
| 294 |
+
"logit": 4.661045074462891
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"token_id": 307,
|
| 298 |
+
"logit": 4.627893447875977
|
| 299 |
+
},
|
| 300 |
+
{
|
| 301 |
+
"token_id": 1287,
|
| 302 |
+
"logit": 4.507448196411133
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"token_id": 3891,
|
| 306 |
+
"logit": 4.162245273590088
|
| 307 |
+
},
|
| 308 |
+
{
|
| 309 |
+
"token_id": 5671,
|
| 310 |
+
"logit": 4.059610366821289
|
| 311 |
+
}
|
| 312 |
+
],
|
| 313 |
+
"quant_next_top20": [
|
| 314 |
+
{
|
| 315 |
+
"token_id": 385,
|
| 316 |
+
"logit": 8.010936737060547
|
| 317 |
+
},
|
| 318 |
+
{
|
| 319 |
+
"token_id": 322,
|
| 320 |
+
"logit": 7.2971720695495605
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"token_id": 5106,
|
| 324 |
+
"logit": 7.011316299438477
|
| 325 |
+
},
|
| 326 |
+
{
|
| 327 |
+
"token_id": 42,
|
| 328 |
+
"logit": 6.86990213394165
|
| 329 |
+
},
|
| 330 |
+
{
|
| 331 |
+
"token_id": 5835,
|
| 332 |
+
"logit": 6.764315605163574
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"token_id": 769,
|
| 336 |
+
"logit": 6.508327484130859
|
| 337 |
+
},
|
| 338 |
+
{
|
| 339 |
+
"token_id": 295,
|
| 340 |
+
"logit": 6.344803333282471
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"token_id": 599,
|
| 344 |
+
"logit": 6.149338245391846
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"token_id": 1980,
|
| 348 |
+
"logit": 6.00590705871582
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"token_id": 2156,
|
| 352 |
+
"logit": 5.4494452476501465
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"token_id": 5226,
|
| 356 |
+
"logit": 5.214221000671387
|
| 357 |
+
},
|
| 358 |
+
{
|
| 359 |
+
"token_id": 379,
|
| 360 |
+
"logit": 5.1674041748046875
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"token_id": 446,
|
| 364 |
+
"logit": 4.9904022216796875
|
| 365 |
+
},
|
| 366 |
+
{
|
| 367 |
+
"token_id": 10755,
|
| 368 |
+
"logit": 4.970067024230957
|
| 369 |
+
},
|
| 370 |
+
{
|
| 371 |
+
"token_id": 1287,
|
| 372 |
+
"logit": 4.824834823608398
|
| 373 |
+
},
|
| 374 |
+
{
|
| 375 |
+
"token_id": 2026,
|
| 376 |
+
"logit": 4.765458583831787
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"token_id": 307,
|
| 380 |
+
"logit": 4.68974494934082
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"token_id": 236,
|
| 384 |
+
"logit": 4.656124114990234
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"token_id": 3891,
|
| 388 |
+
"logit": 4.48333740234375
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"token_id": 5671,
|
| 392 |
+
"logit": 4.341298580169678
|
| 393 |
+
}
|
| 394 |
+
],
|
| 395 |
+
"top1_match": true,
|
| 396 |
+
"top10_overlap": 10,
|
| 397 |
+
"full_vocab_mean_abs_logit_error": 0.1026391088962555,
|
| 398 |
+
"top20_union_mean_abs_logit_error": 0.15874910354614258,
|
| 399 |
+
"bf16_generated_suffix": "'s distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant\n\n",
|
| 400 |
+
"quant_generated_suffix": "'s future is uncertain, and the world's trajectory remains unpredictable. [end of text]\n\n\n",
|
| 401 |
+
"bf16_repetition_loop": false,
|
| 402 |
+
"quant_repetition_loop": false,
|
| 403 |
+
"new_repetition_loop": false
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"prompt": "Question: What is 2+2?\nAnswer:",
|
| 407 |
+
"bf16_input_ids": [
|
| 408 |
+
1069,
|
| 409 |
+
56,
|
| 410 |
+
866,
|
| 411 |
+
322,
|
| 412 |
+
236,
|
| 413 |
+
48,
|
| 414 |
+
41,
|
| 415 |
+
48,
|
| 416 |
+
1047,
|
| 417 |
+
1180,
|
| 418 |
+
56
|
| 419 |
+
],
|
| 420 |
+
"quant_input_ids": [
|
| 421 |
+
1069,
|
| 422 |
+
56,
|
| 423 |
+
866,
|
| 424 |
+
322,
|
| 425 |
+
236,
|
| 426 |
+
48,
|
| 427 |
+
41,
|
| 428 |
+
48,
|
| 429 |
+
1047,
|
| 430 |
+
1180,
|
| 431 |
+
56
|
| 432 |
+
],
|
| 433 |
+
"tokens_match": true,
|
| 434 |
+
"bf16_next_top20": [
|
| 435 |
+
{
|
| 436 |
+
"token_id": 236,
|
| 437 |
+
"logit": 9.97547435760498
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"token_id": 391,
|
| 441 |
+
"logit": 6.012524604797363
|
| 442 |
+
},
|
| 443 |
+
{
|
| 444 |
+
"token_id": 315,
|
| 445 |
+
"logit": 5.968222618103027
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"token_id": 866,
|
| 449 |
+
"logit": 5.672043323516846
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"token_id": 395,
|
| 453 |
+
"logit": 5.606063365936279
|
| 454 |
+
},
|
| 455 |
+
{
|
| 456 |
+
"token_id": 5444,
|
| 457 |
+
"logit": 5.2098188400268555
|
| 458 |
+
},
|
| 459 |
+
{
|
| 460 |
+
"token_id": 4746,
|
| 461 |
+
"logit": 4.777042388916016
|
| 462 |
+
},
|
| 463 |
+
{
|
| 464 |
+
"token_id": 1827,
|
| 465 |
+
"logit": 4.54682731628418
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"token_id": 369,
|
| 469 |
+
"logit": 3.9637503623962402
|
| 470 |
+
},
|
| 471 |
+
{
|
| 472 |
+
"token_id": 970,
|
| 473 |
+
"logit": 3.765744686126709
|
| 474 |
+
},
|
| 475 |
+
{
|
| 476 |
+
"token_id": 725,
|
| 477 |
+
"logit": 3.7245254516601562
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"token_id": 421,
|
| 481 |
+
"logit": 3.630314350128174
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"token_id": 731,
|
| 485 |
+
"logit": 3.2161593437194824
|
| 486 |
+
},
|
| 487 |
+
{
|
| 488 |
+
"token_id": 401,
|
| 489 |
+
"logit": 3.1881794929504395
|
| 490 |
+
},
|
| 491 |
+
{
|
| 492 |
+
"token_id": 2286,
|
| 493 |
+
"logit": 3.168586254119873
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"token_id": 1422,
|
| 497 |
+
"logit": 3.151968240737915
|
| 498 |
+
},
|
| 499 |
+
{
|
| 500 |
+
"token_id": 10528,
|
| 501 |
+
"logit": 3.148939847946167
|
| 502 |
+
},
|
| 503 |
+
{
|
| 504 |
+
"token_id": 1544,
|
| 505 |
+
"logit": 3.1278254985809326
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"token_id": 973,
|
| 509 |
+
"logit": 3.1256284713745117
|
| 510 |
+
},
|
| 511 |
+
{
|
| 512 |
+
"token_id": 4286,
|
| 513 |
+
"logit": 3.0892391204833984
|
| 514 |
+
}
|
| 515 |
+
],
|
| 516 |
+
"quant_next_top20": [
|
| 517 |
+
{
|
| 518 |
+
"token_id": 236,
|
| 519 |
+
"logit": 10.075109481811523
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"token_id": 866,
|
| 523 |
+
"logit": 6.403327941894531
|
| 524 |
+
},
|
| 525 |
+
{
|
| 526 |
+
"token_id": 391,
|
| 527 |
+
"logit": 6.247835636138916
|
| 528 |
+
},
|
| 529 |
+
{
|
| 530 |
+
"token_id": 315,
|
| 531 |
+
"logit": 5.9470744132995605
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"token_id": 395,
|
| 535 |
+
"logit": 5.720576286315918
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"token_id": 5444,
|
| 539 |
+
"logit": 5.346616744995117
|
| 540 |
+
},
|
| 541 |
+
{
|
| 542 |
+
"token_id": 4746,
|
| 543 |
+
"logit": 4.786258697509766
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"token_id": 1827,
|
| 547 |
+
"logit": 4.581784725189209
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"token_id": 725,
|
| 551 |
+
"logit": 4.345588684082031
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"token_id": 970,
|
| 555 |
+
"logit": 4.137524604797363
|
| 556 |
+
},
|
| 557 |
+
{
|
| 558 |
+
"token_id": 369,
|
| 559 |
+
"logit": 3.8343424797058105
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"token_id": 421,
|
| 563 |
+
"logit": 3.592531204223633
|
| 564 |
+
},
|
| 565 |
+
{
|
| 566 |
+
"token_id": 2286,
|
| 567 |
+
"logit": 3.5441107749938965
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"token_id": 401,
|
| 571 |
+
"logit": 3.355380058288574
|
| 572 |
+
},
|
| 573 |
+
{
|
| 574 |
+
"token_id": 214,
|
| 575 |
+
"logit": 3.288057804107666
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"token_id": 1544,
|
| 579 |
+
"logit": 3.241110324859619
|
| 580 |
+
},
|
| 581 |
+
{
|
| 582 |
+
"token_id": 731,
|
| 583 |
+
"logit": 3.23519229888916
|
| 584 |
+
},
|
| 585 |
+
{
|
| 586 |
+
"token_id": 973,
|
| 587 |
+
"logit": 3.1882829666137695
|
| 588 |
+
},
|
| 589 |
+
{
|
| 590 |
+
"token_id": 4286,
|
| 591 |
+
"logit": 3.172485113143921
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"token_id": 10528,
|
| 595 |
+
"logit": 3.0761098861694336
|
| 596 |
+
}
|
| 597 |
+
],
|
| 598 |
+
"top1_match": true,
|
| 599 |
+
"top10_overlap": 9,
|
| 600 |
+
"full_vocab_mean_abs_logit_error": 0.1289779245853424,
|
| 601 |
+
"top20_union_mean_abs_logit_error": 0.17950297892093658,
|
| 602 |
+
"bf16_generated_suffix": " 6 [end of text]\n\n\n",
|
| 603 |
+
"quant_generated_suffix": " 6 [end of text]\n\n\n",
|
| 604 |
+
"bf16_repetition_loop": false,
|
| 605 |
+
"quant_repetition_loop": false,
|
| 606 |
+
"new_repetition_loop": false
|
| 607 |
+
},
|
| 608 |
+
{
|
| 609 |
+
"prompt": "def fibonacci(n):",
|
| 610 |
+
"bf16_input_ids": [
|
| 611 |
+
26763,
|
| 612 |
+
8430,
|
| 613 |
+
21728,
|
| 614 |
+
3145,
|
| 615 |
+
10018
|
| 616 |
+
],
|
| 617 |
+
"quant_input_ids": [
|
| 618 |
+
26763,
|
| 619 |
+
8430,
|
| 620 |
+
21728,
|
| 621 |
+
3145,
|
| 622 |
+
10018
|
| 623 |
+
],
|
| 624 |
+
"tokens_match": true,
|
| 625 |
+
"bf16_next_top20": [
|
| 626 |
+
{
|
| 627 |
+
"token_id": 56,
|
| 628 |
+
"logit": 10.327644348144531
|
| 629 |
+
},
|
| 630 |
+
{
|
| 631 |
+
"token_id": 2904,
|
| 632 |
+
"logit": 7.035848140716553
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"token_id": 11,
|
| 636 |
+
"logit": 6.860950946807861
|
| 637 |
+
},
|
| 638 |
+
{
|
| 639 |
+
"token_id": 10018,
|
| 640 |
+
"logit": 6.495933532714844
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"token_id": 108,
|
| 644 |
+
"logit": 6.341927528381348
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"token_id": 318,
|
| 648 |
+
"logit": 6.146999359130859
|
| 649 |
+
},
|
| 650 |
+
{
|
| 651 |
+
"token_id": 236,
|
| 652 |
+
"logit": 6.0702805519104
|
| 653 |
+
},
|
| 654 |
+
{
|
| 655 |
+
"token_id": 38,
|
| 656 |
+
"logit": 5.865842819213867
|
| 657 |
+
},
|
| 658 |
+
{
|
| 659 |
+
"token_id": 68,
|
| 660 |
+
"logit": 5.02256965637207
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"token_id": 39,
|
| 664 |
+
"logit": 4.66829252243042
|
| 665 |
+
},
|
| 666 |
+
{
|
| 667 |
+
"token_id": 76,
|
| 668 |
+
"logit": 4.6676859855651855
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"token_id": 59,
|
| 672 |
+
"logit": 4.662688255310059
|
| 673 |
+
},
|
| 674 |
+
{
|
| 675 |
+
"token_id": 57,
|
| 676 |
+
"logit": 4.66217041015625
|
| 677 |
+
},
|
| 678 |
+
{
|
| 679 |
+
"token_id": 1941,
|
| 680 |
+
"logit": 4.598584175109863
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"token_id": 401,
|
| 684 |
+
"logit": 4.574923515319824
|
| 685 |
+
},
|
| 686 |
+
{
|
| 687 |
+
"token_id": 395,
|
| 688 |
+
"logit": 4.532867431640625
|
| 689 |
+
},
|
| 690 |
+
{
|
| 691 |
+
"token_id": 8430,
|
| 692 |
+
"logit": 4.5034050941467285
|
| 693 |
+
},
|
| 694 |
+
{
|
| 695 |
+
"token_id": 95,
|
| 696 |
+
"logit": 4.318176746368408
|
| 697 |
+
},
|
| 698 |
+
{
|
| 699 |
+
"token_id": 364,
|
| 700 |
+
"logit": 4.2872209548950195
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"token_id": 48,
|
| 704 |
+
"logit": 4.243288516998291
|
| 705 |
+
}
|
| 706 |
+
],
|
| 707 |
+
"quant_next_top20": [
|
| 708 |
+
{
|
| 709 |
+
"token_id": 56,
|
| 710 |
+
"logit": 10.113146781921387
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"token_id": 10018,
|
| 714 |
+
"logit": 7.379585266113281
|
| 715 |
+
},
|
| 716 |
+
{
|
| 717 |
+
"token_id": 11,
|
| 718 |
+
"logit": 7.0373640060424805
|
| 719 |
+
},
|
| 720 |
+
{
|
| 721 |
+
"token_id": 2904,
|
| 722 |
+
"logit": 6.770742893218994
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"token_id": 236,
|
| 726 |
+
"logit": 6.06136417388916
|
| 727 |
+
},
|
| 728 |
+
{
|
| 729 |
+
"token_id": 38,
|
| 730 |
+
"logit": 5.741509914398193
|
| 731 |
+
},
|
| 732 |
+
{
|
| 733 |
+
"token_id": 108,
|
| 734 |
+
"logit": 5.6297454833984375
|
| 735 |
+
},
|
| 736 |
+
{
|
| 737 |
+
"token_id": 318,
|
| 738 |
+
"logit": 5.541906833648682
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"token_id": 39,
|
| 742 |
+
"logit": 5.26960563659668
|
| 743 |
+
},
|
| 744 |
+
{
|
| 745 |
+
"token_id": 68,
|
| 746 |
+
"logit": 4.772350311279297
|
| 747 |
+
},
|
| 748 |
+
{
|
| 749 |
+
"token_id": 57,
|
| 750 |
+
"logit": 4.503805160522461
|
| 751 |
+
},
|
| 752 |
+
{
|
| 753 |
+
"token_id": 401,
|
| 754 |
+
"logit": 4.451050758361816
|
| 755 |
+
},
|
| 756 |
+
{
|
| 757 |
+
"token_id": 2559,
|
| 758 |
+
"logit": 4.421682834625244
|
| 759 |
+
},
|
| 760 |
+
{
|
| 761 |
+
"token_id": 48,
|
| 762 |
+
"logit": 4.40238618850708
|
| 763 |
+
},
|
| 764 |
+
{
|
| 765 |
+
"token_id": 395,
|
| 766 |
+
"logit": 4.320408344268799
|
| 767 |
+
},
|
| 768 |
+
{
|
| 769 |
+
"token_id": 59,
|
| 770 |
+
"logit": 4.310557842254639
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"token_id": 2316,
|
| 774 |
+
"logit": 4.291357517242432
|
| 775 |
+
},
|
| 776 |
+
{
|
| 777 |
+
"token_id": 1941,
|
| 778 |
+
"logit": 4.208161354064941
|
| 779 |
+
},
|
| 780 |
+
{
|
| 781 |
+
"token_id": 76,
|
| 782 |
+
"logit": 4.146447658538818
|
| 783 |
+
},
|
| 784 |
+
{
|
| 785 |
+
"token_id": 41,
|
| 786 |
+
"logit": 4.066195487976074
|
| 787 |
+
}
|
| 788 |
+
],
|
| 789 |
+
"top1_match": true,
|
| 790 |
+
"top10_overlap": 10,
|
| 791 |
+
"full_vocab_mean_abs_logit_error": 0.10505230724811554,
|
| 792 |
+
"top20_union_mean_abs_logit_error": 0.34302181005477905,
|
| 793 |
+
"bf16_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 794 |
+
"quant_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 795 |
+
"bf16_repetition_loop": false,
|
| 796 |
+
"quant_repetition_loop": false,
|
| 797 |
+
"new_repetition_loop": false
|
| 798 |
+
}
|
| 799 |
+
]
|
| 800 |
+
}
|
reports/validation/q6_k_vs_bf16.json
ADDED
|
@@ -0,0 +1,800 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bf16": "fresh\\out\\gguf\\HRM-Text-1B-BF16.gguf",
|
| 3 |
+
"quant": "fresh\\out\\gguf\\HRM-Text-1B-Q6_K.gguf",
|
| 4 |
+
"quant_name": "Q6_K",
|
| 5 |
+
"created_at_unix": 1779350048.1454997,
|
| 6 |
+
"elapsed_seconds": 92.39457774162292,
|
| 7 |
+
"metadata": {
|
| 8 |
+
"bf16_general_architecture": "hrm_text",
|
| 9 |
+
"quant_general_architecture": "hrm_text"
|
| 10 |
+
},
|
| 11 |
+
"checks": {
|
| 12 |
+
"prompt_tokens_match_all_prompts": true,
|
| 13 |
+
"same_next_top1_at_least_3_of_4": true,
|
| 14 |
+
"top1_match_count": 4,
|
| 15 |
+
"top10_overlap_at_least_7_all_prompts": true,
|
| 16 |
+
"quant_general_architecture_is_hrm_text": true,
|
| 17 |
+
"no_new_repetition_loop": true,
|
| 18 |
+
"passed": true
|
| 19 |
+
},
|
| 20 |
+
"prompts": [
|
| 21 |
+
{
|
| 22 |
+
"prompt": "The quick brown fox",
|
| 23 |
+
"bf16_input_ids": [
|
| 24 |
+
341,
|
| 25 |
+
3568,
|
| 26 |
+
9054,
|
| 27 |
+
20102
|
| 28 |
+
],
|
| 29 |
+
"quant_input_ids": [
|
| 30 |
+
341,
|
| 31 |
+
3568,
|
| 32 |
+
9054,
|
| 33 |
+
20102
|
| 34 |
+
],
|
| 35 |
+
"tokens_match": true,
|
| 36 |
+
"bf16_next_top20": [
|
| 37 |
+
{
|
| 38 |
+
"token_id": 20102,
|
| 39 |
+
"logit": 14.297157287597656
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"token_id": 40036,
|
| 43 |
+
"logit": 7.994879722595215
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"token_id": 11979,
|
| 47 |
+
"logit": 6.699657440185547
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"token_id": 3568,
|
| 51 |
+
"logit": 6.626206398010254
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"token_id": 4349,
|
| 55 |
+
"logit": 6.35108757019043
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"token_id": 42,
|
| 59 |
+
"logit": 6.271071434020996
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"token_id": 3152,
|
| 63 |
+
"logit": 5.424627304077148
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"token_id": 2301,
|
| 67 |
+
"logit": 5.3609843254089355
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"token_id": 44,
|
| 71 |
+
"logit": 5.178180694580078
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"token_id": 3558,
|
| 75 |
+
"logit": 4.875790596008301
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"token_id": 5002,
|
| 79 |
+
"logit": 4.839702606201172
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"token_id": 3412,
|
| 83 |
+
"logit": 4.824095249176025
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"token_id": 761,
|
| 87 |
+
"logit": 4.718862056732178
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"token_id": 19444,
|
| 91 |
+
"logit": 4.698058128356934
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"token_id": 446,
|
| 95 |
+
"logit": 4.689098358154297
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"token_id": 16001,
|
| 99 |
+
"logit": 4.590203762054443
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"token_id": 3297,
|
| 103 |
+
"logit": 4.408023357391357
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"token_id": 322,
|
| 107 |
+
"logit": 4.394444942474365
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"token_id": 8636,
|
| 111 |
+
"logit": 4.318354606628418
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"token_id": 12672,
|
| 115 |
+
"logit": 4.2834038734436035
|
| 116 |
+
}
|
| 117 |
+
],
|
| 118 |
+
"quant_next_top20": [
|
| 119 |
+
{
|
| 120 |
+
"token_id": 20102,
|
| 121 |
+
"logit": 14.177587509155273
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"token_id": 40036,
|
| 125 |
+
"logit": 7.92909574508667
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"token_id": 11979,
|
| 129 |
+
"logit": 6.7863616943359375
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"token_id": 3568,
|
| 133 |
+
"logit": 6.753141403198242
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"token_id": 42,
|
| 137 |
+
"logit": 6.335923194885254
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"token_id": 4349,
|
| 141 |
+
"logit": 6.27409029006958
|
| 142 |
+
},
|
| 143 |
+
{
|
| 144 |
+
"token_id": 3152,
|
| 145 |
+
"logit": 5.374554634094238
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"token_id": 44,
|
| 149 |
+
"logit": 5.344453811645508
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"token_id": 2301,
|
| 153 |
+
"logit": 5.301711082458496
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"token_id": 3558,
|
| 157 |
+
"logit": 4.961206436157227
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"token_id": 5002,
|
| 161 |
+
"logit": 4.878804683685303
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"token_id": 3412,
|
| 165 |
+
"logit": 4.8427839279174805
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"token_id": 446,
|
| 169 |
+
"logit": 4.783636093139648
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"token_id": 761,
|
| 173 |
+
"logit": 4.661725044250488
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"token_id": 19444,
|
| 177 |
+
"logit": 4.640953063964844
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"token_id": 16001,
|
| 181 |
+
"logit": 4.5871052742004395
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"token_id": 3297,
|
| 185 |
+
"logit": 4.52216100692749
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"token_id": 322,
|
| 189 |
+
"logit": 4.450130462646484
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"token_id": 22263,
|
| 193 |
+
"logit": 4.20174503326416
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
"token_id": 12672,
|
| 197 |
+
"logit": 4.159463405609131
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"top1_match": true,
|
| 201 |
+
"top10_overlap": 10,
|
| 202 |
+
"full_vocab_mean_abs_logit_error": 0.0928385853767395,
|
| 203 |
+
"top20_union_mean_abs_logit_error": 0.08213842660188675,
|
| 204 |
+
"bf16_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 205 |
+
"quant_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 206 |
+
"bf16_repetition_loop": true,
|
| 207 |
+
"quant_repetition_loop": true,
|
| 208 |
+
"new_repetition_loop": false
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"prompt": "In a distant future, humanity",
|
| 212 |
+
"bf16_input_ids": [
|
| 213 |
+
931,
|
| 214 |
+
236,
|
| 215 |
+
95,
|
| 216 |
+
9494,
|
| 217 |
+
2025,
|
| 218 |
+
42,
|
| 219 |
+
13279
|
| 220 |
+
],
|
| 221 |
+
"quant_input_ids": [
|
| 222 |
+
931,
|
| 223 |
+
236,
|
| 224 |
+
95,
|
| 225 |
+
9494,
|
| 226 |
+
2025,
|
| 227 |
+
42,
|
| 228 |
+
13279
|
| 229 |
+
],
|
| 230 |
+
"tokens_match": true,
|
| 231 |
+
"bf16_next_top20": [
|
| 232 |
+
{
|
| 233 |
+
"token_id": 385,
|
| 234 |
+
"logit": 8.094616889953613
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"token_id": 42,
|
| 238 |
+
"logit": 7.381193161010742
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"token_id": 322,
|
| 242 |
+
"logit": 7.22892951965332
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"token_id": 5106,
|
| 246 |
+
"logit": 6.968372344970703
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"token_id": 5835,
|
| 250 |
+
"logit": 6.5758562088012695
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"token_id": 769,
|
| 254 |
+
"logit": 6.294882297515869
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"token_id": 599,
|
| 258 |
+
"logit": 6.259007453918457
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"token_id": 1980,
|
| 262 |
+
"logit": 6.067761421203613
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"token_id": 295,
|
| 266 |
+
"logit": 6.029950141906738
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"token_id": 2156,
|
| 270 |
+
"logit": 5.428796768188477
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"token_id": 379,
|
| 274 |
+
"logit": 5.263803005218506
|
| 275 |
+
},
|
| 276 |
+
{
|
| 277 |
+
"token_id": 5226,
|
| 278 |
+
"logit": 5.066612243652344
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"token_id": 446,
|
| 282 |
+
"logit": 4.910100936889648
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"token_id": 10755,
|
| 286 |
+
"logit": 4.904460430145264
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"token_id": 236,
|
| 290 |
+
"logit": 4.739671230316162
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"token_id": 2026,
|
| 294 |
+
"logit": 4.661045074462891
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"token_id": 307,
|
| 298 |
+
"logit": 4.627893447875977
|
| 299 |
+
},
|
| 300 |
+
{
|
| 301 |
+
"token_id": 1287,
|
| 302 |
+
"logit": 4.507448196411133
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"token_id": 3891,
|
| 306 |
+
"logit": 4.162245273590088
|
| 307 |
+
},
|
| 308 |
+
{
|
| 309 |
+
"token_id": 5671,
|
| 310 |
+
"logit": 4.059610366821289
|
| 311 |
+
}
|
| 312 |
+
],
|
| 313 |
+
"quant_next_top20": [
|
| 314 |
+
{
|
| 315 |
+
"token_id": 385,
|
| 316 |
+
"logit": 8.302244186401367
|
| 317 |
+
},
|
| 318 |
+
{
|
| 319 |
+
"token_id": 42,
|
| 320 |
+
"logit": 7.396821975708008
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"token_id": 322,
|
| 324 |
+
"logit": 7.26978874206543
|
| 325 |
+
},
|
| 326 |
+
{
|
| 327 |
+
"token_id": 5106,
|
| 328 |
+
"logit": 6.974541664123535
|
| 329 |
+
},
|
| 330 |
+
{
|
| 331 |
+
"token_id": 5835,
|
| 332 |
+
"logit": 6.505857467651367
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"token_id": 769,
|
| 336 |
+
"logit": 6.336202144622803
|
| 337 |
+
},
|
| 338 |
+
{
|
| 339 |
+
"token_id": 599,
|
| 340 |
+
"logit": 6.306015491485596
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"token_id": 1980,
|
| 344 |
+
"logit": 6.225838661193848
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"token_id": 295,
|
| 348 |
+
"logit": 6.034883499145508
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"token_id": 2156,
|
| 352 |
+
"logit": 5.360622882843018
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"token_id": 379,
|
| 356 |
+
"logit": 5.355396747589111
|
| 357 |
+
},
|
| 358 |
+
{
|
| 359 |
+
"token_id": 446,
|
| 360 |
+
"logit": 4.922514915466309
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"token_id": 5226,
|
| 364 |
+
"logit": 4.9152607917785645
|
| 365 |
+
},
|
| 366 |
+
{
|
| 367 |
+
"token_id": 307,
|
| 368 |
+
"logit": 4.718687057495117
|
| 369 |
+
},
|
| 370 |
+
{
|
| 371 |
+
"token_id": 10755,
|
| 372 |
+
"logit": 4.703006267547607
|
| 373 |
+
},
|
| 374 |
+
{
|
| 375 |
+
"token_id": 2026,
|
| 376 |
+
"logit": 4.668638706207275
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"token_id": 1287,
|
| 380 |
+
"logit": 4.494301795959473
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"token_id": 236,
|
| 384 |
+
"logit": 4.2818989753723145
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"token_id": 3891,
|
| 388 |
+
"logit": 4.090405464172363
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"token_id": 5671,
|
| 392 |
+
"logit": 4.061459541320801
|
| 393 |
+
}
|
| 394 |
+
],
|
| 395 |
+
"top1_match": true,
|
| 396 |
+
"top10_overlap": 10,
|
| 397 |
+
"full_vocab_mean_abs_logit_error": 0.07374933362007141,
|
| 398 |
+
"top20_union_mean_abs_logit_error": 0.08798019587993622,
|
| 399 |
+
"bf16_generated_suffix": "'s distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant\n\n",
|
| 400 |
+
"quant_generated_suffix": "'s distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant\n\n",
|
| 401 |
+
"bf16_repetition_loop": false,
|
| 402 |
+
"quant_repetition_loop": false,
|
| 403 |
+
"new_repetition_loop": false
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"prompt": "Question: What is 2+2?\nAnswer:",
|
| 407 |
+
"bf16_input_ids": [
|
| 408 |
+
1069,
|
| 409 |
+
56,
|
| 410 |
+
866,
|
| 411 |
+
322,
|
| 412 |
+
236,
|
| 413 |
+
48,
|
| 414 |
+
41,
|
| 415 |
+
48,
|
| 416 |
+
1047,
|
| 417 |
+
1180,
|
| 418 |
+
56
|
| 419 |
+
],
|
| 420 |
+
"quant_input_ids": [
|
| 421 |
+
1069,
|
| 422 |
+
56,
|
| 423 |
+
866,
|
| 424 |
+
322,
|
| 425 |
+
236,
|
| 426 |
+
48,
|
| 427 |
+
41,
|
| 428 |
+
48,
|
| 429 |
+
1047,
|
| 430 |
+
1180,
|
| 431 |
+
56
|
| 432 |
+
],
|
| 433 |
+
"tokens_match": true,
|
| 434 |
+
"bf16_next_top20": [
|
| 435 |
+
{
|
| 436 |
+
"token_id": 236,
|
| 437 |
+
"logit": 9.97547435760498
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"token_id": 391,
|
| 441 |
+
"logit": 6.012524604797363
|
| 442 |
+
},
|
| 443 |
+
{
|
| 444 |
+
"token_id": 315,
|
| 445 |
+
"logit": 5.968222618103027
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"token_id": 866,
|
| 449 |
+
"logit": 5.672043323516846
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"token_id": 395,
|
| 453 |
+
"logit": 5.606063365936279
|
| 454 |
+
},
|
| 455 |
+
{
|
| 456 |
+
"token_id": 5444,
|
| 457 |
+
"logit": 5.2098188400268555
|
| 458 |
+
},
|
| 459 |
+
{
|
| 460 |
+
"token_id": 4746,
|
| 461 |
+
"logit": 4.777042388916016
|
| 462 |
+
},
|
| 463 |
+
{
|
| 464 |
+
"token_id": 1827,
|
| 465 |
+
"logit": 4.54682731628418
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"token_id": 369,
|
| 469 |
+
"logit": 3.9637503623962402
|
| 470 |
+
},
|
| 471 |
+
{
|
| 472 |
+
"token_id": 970,
|
| 473 |
+
"logit": 3.765744686126709
|
| 474 |
+
},
|
| 475 |
+
{
|
| 476 |
+
"token_id": 725,
|
| 477 |
+
"logit": 3.7245254516601562
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"token_id": 421,
|
| 481 |
+
"logit": 3.630314350128174
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"token_id": 731,
|
| 485 |
+
"logit": 3.2161593437194824
|
| 486 |
+
},
|
| 487 |
+
{
|
| 488 |
+
"token_id": 401,
|
| 489 |
+
"logit": 3.1881794929504395
|
| 490 |
+
},
|
| 491 |
+
{
|
| 492 |
+
"token_id": 2286,
|
| 493 |
+
"logit": 3.168586254119873
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"token_id": 1422,
|
| 497 |
+
"logit": 3.151968240737915
|
| 498 |
+
},
|
| 499 |
+
{
|
| 500 |
+
"token_id": 10528,
|
| 501 |
+
"logit": 3.148939847946167
|
| 502 |
+
},
|
| 503 |
+
{
|
| 504 |
+
"token_id": 1544,
|
| 505 |
+
"logit": 3.1278254985809326
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"token_id": 973,
|
| 509 |
+
"logit": 3.1256284713745117
|
| 510 |
+
},
|
| 511 |
+
{
|
| 512 |
+
"token_id": 4286,
|
| 513 |
+
"logit": 3.0892391204833984
|
| 514 |
+
}
|
| 515 |
+
],
|
| 516 |
+
"quant_next_top20": [
|
| 517 |
+
{
|
| 518 |
+
"token_id": 236,
|
| 519 |
+
"logit": 9.89577865600586
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"token_id": 391,
|
| 523 |
+
"logit": 6.210731506347656
|
| 524 |
+
},
|
| 525 |
+
{
|
| 526 |
+
"token_id": 315,
|
| 527 |
+
"logit": 5.948070526123047
|
| 528 |
+
},
|
| 529 |
+
{
|
| 530 |
+
"token_id": 866,
|
| 531 |
+
"logit": 5.716897964477539
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"token_id": 395,
|
| 535 |
+
"logit": 5.679378986358643
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"token_id": 5444,
|
| 539 |
+
"logit": 5.323817253112793
|
| 540 |
+
},
|
| 541 |
+
{
|
| 542 |
+
"token_id": 1827,
|
| 543 |
+
"logit": 4.6884050369262695
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"token_id": 4746,
|
| 547 |
+
"logit": 4.628040790557861
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"token_id": 369,
|
| 551 |
+
"logit": 4.035506725311279
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"token_id": 725,
|
| 555 |
+
"logit": 3.7037086486816406
|
| 556 |
+
},
|
| 557 |
+
{
|
| 558 |
+
"token_id": 970,
|
| 559 |
+
"logit": 3.6759777069091797
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"token_id": 421,
|
| 563 |
+
"logit": 3.559971332550049
|
| 564 |
+
},
|
| 565 |
+
{
|
| 566 |
+
"token_id": 1544,
|
| 567 |
+
"logit": 3.344578981399536
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"token_id": 731,
|
| 571 |
+
"logit": 3.3388290405273438
|
| 572 |
+
},
|
| 573 |
+
{
|
| 574 |
+
"token_id": 973,
|
| 575 |
+
"logit": 3.1739864349365234
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"token_id": 214,
|
| 579 |
+
"logit": 3.162808418273926
|
| 580 |
+
},
|
| 581 |
+
{
|
| 582 |
+
"token_id": 1422,
|
| 583 |
+
"logit": 3.1502134799957275
|
| 584 |
+
},
|
| 585 |
+
{
|
| 586 |
+
"token_id": 401,
|
| 587 |
+
"logit": 3.148956298828125
|
| 588 |
+
},
|
| 589 |
+
{
|
| 590 |
+
"token_id": 2286,
|
| 591 |
+
"logit": 3.1337287425994873
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"token_id": 4286,
|
| 595 |
+
"logit": 3.003777027130127
|
| 596 |
+
}
|
| 597 |
+
],
|
| 598 |
+
"top1_match": true,
|
| 599 |
+
"top10_overlap": 9,
|
| 600 |
+
"full_vocab_mean_abs_logit_error": 0.0646180585026741,
|
| 601 |
+
"top20_union_mean_abs_logit_error": 0.09386428445577621,
|
| 602 |
+
"bf16_generated_suffix": " 6 [end of text]\n\n\n",
|
| 603 |
+
"quant_generated_suffix": " 6 [end of text]\n\n\n",
|
| 604 |
+
"bf16_repetition_loop": false,
|
| 605 |
+
"quant_repetition_loop": false,
|
| 606 |
+
"new_repetition_loop": false
|
| 607 |
+
},
|
| 608 |
+
{
|
| 609 |
+
"prompt": "def fibonacci(n):",
|
| 610 |
+
"bf16_input_ids": [
|
| 611 |
+
26763,
|
| 612 |
+
8430,
|
| 613 |
+
21728,
|
| 614 |
+
3145,
|
| 615 |
+
10018
|
| 616 |
+
],
|
| 617 |
+
"quant_input_ids": [
|
| 618 |
+
26763,
|
| 619 |
+
8430,
|
| 620 |
+
21728,
|
| 621 |
+
3145,
|
| 622 |
+
10018
|
| 623 |
+
],
|
| 624 |
+
"tokens_match": true,
|
| 625 |
+
"bf16_next_top20": [
|
| 626 |
+
{
|
| 627 |
+
"token_id": 56,
|
| 628 |
+
"logit": 10.327644348144531
|
| 629 |
+
},
|
| 630 |
+
{
|
| 631 |
+
"token_id": 2904,
|
| 632 |
+
"logit": 7.035848140716553
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"token_id": 11,
|
| 636 |
+
"logit": 6.860950946807861
|
| 637 |
+
},
|
| 638 |
+
{
|
| 639 |
+
"token_id": 10018,
|
| 640 |
+
"logit": 6.495933532714844
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"token_id": 108,
|
| 644 |
+
"logit": 6.341927528381348
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"token_id": 318,
|
| 648 |
+
"logit": 6.146999359130859
|
| 649 |
+
},
|
| 650 |
+
{
|
| 651 |
+
"token_id": 236,
|
| 652 |
+
"logit": 6.0702805519104
|
| 653 |
+
},
|
| 654 |
+
{
|
| 655 |
+
"token_id": 38,
|
| 656 |
+
"logit": 5.865842819213867
|
| 657 |
+
},
|
| 658 |
+
{
|
| 659 |
+
"token_id": 68,
|
| 660 |
+
"logit": 5.02256965637207
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"token_id": 39,
|
| 664 |
+
"logit": 4.66829252243042
|
| 665 |
+
},
|
| 666 |
+
{
|
| 667 |
+
"token_id": 76,
|
| 668 |
+
"logit": 4.6676859855651855
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"token_id": 59,
|
| 672 |
+
"logit": 4.662688255310059
|
| 673 |
+
},
|
| 674 |
+
{
|
| 675 |
+
"token_id": 57,
|
| 676 |
+
"logit": 4.66217041015625
|
| 677 |
+
},
|
| 678 |
+
{
|
| 679 |
+
"token_id": 1941,
|
| 680 |
+
"logit": 4.598584175109863
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"token_id": 401,
|
| 684 |
+
"logit": 4.574923515319824
|
| 685 |
+
},
|
| 686 |
+
{
|
| 687 |
+
"token_id": 395,
|
| 688 |
+
"logit": 4.532867431640625
|
| 689 |
+
},
|
| 690 |
+
{
|
| 691 |
+
"token_id": 8430,
|
| 692 |
+
"logit": 4.5034050941467285
|
| 693 |
+
},
|
| 694 |
+
{
|
| 695 |
+
"token_id": 95,
|
| 696 |
+
"logit": 4.318176746368408
|
| 697 |
+
},
|
| 698 |
+
{
|
| 699 |
+
"token_id": 364,
|
| 700 |
+
"logit": 4.2872209548950195
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"token_id": 48,
|
| 704 |
+
"logit": 4.243288516998291
|
| 705 |
+
}
|
| 706 |
+
],
|
| 707 |
+
"quant_next_top20": [
|
| 708 |
+
{
|
| 709 |
+
"token_id": 56,
|
| 710 |
+
"logit": 10.115074157714844
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"token_id": 11,
|
| 714 |
+
"logit": 7.1996660232543945
|
| 715 |
+
},
|
| 716 |
+
{
|
| 717 |
+
"token_id": 2904,
|
| 718 |
+
"logit": 7.055414199829102
|
| 719 |
+
},
|
| 720 |
+
{
|
| 721 |
+
"token_id": 10018,
|
| 722 |
+
"logit": 6.49235200881958
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"token_id": 236,
|
| 726 |
+
"logit": 6.069934844970703
|
| 727 |
+
},
|
| 728 |
+
{
|
| 729 |
+
"token_id": 108,
|
| 730 |
+
"logit": 5.70982027053833
|
| 731 |
+
},
|
| 732 |
+
{
|
| 733 |
+
"token_id": 38,
|
| 734 |
+
"logit": 5.509694576263428
|
| 735 |
+
},
|
| 736 |
+
{
|
| 737 |
+
"token_id": 318,
|
| 738 |
+
"logit": 5.377516746520996
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"token_id": 401,
|
| 742 |
+
"logit": 4.638413429260254
|
| 743 |
+
},
|
| 744 |
+
{
|
| 745 |
+
"token_id": 68,
|
| 746 |
+
"logit": 4.632800579071045
|
| 747 |
+
},
|
| 748 |
+
{
|
| 749 |
+
"token_id": 48,
|
| 750 |
+
"logit": 4.553812026977539
|
| 751 |
+
},
|
| 752 |
+
{
|
| 753 |
+
"token_id": 57,
|
| 754 |
+
"logit": 4.551402568817139
|
| 755 |
+
},
|
| 756 |
+
{
|
| 757 |
+
"token_id": 395,
|
| 758 |
+
"logit": 4.443292140960693
|
| 759 |
+
},
|
| 760 |
+
{
|
| 761 |
+
"token_id": 1941,
|
| 762 |
+
"logit": 4.424618721008301
|
| 763 |
+
},
|
| 764 |
+
{
|
| 765 |
+
"token_id": 76,
|
| 766 |
+
"logit": 4.370553016662598
|
| 767 |
+
},
|
| 768 |
+
{
|
| 769 |
+
"token_id": 39,
|
| 770 |
+
"logit": 4.348079204559326
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"token_id": 59,
|
| 774 |
+
"logit": 4.284107208251953
|
| 775 |
+
},
|
| 776 |
+
{
|
| 777 |
+
"token_id": 95,
|
| 778 |
+
"logit": 4.113048076629639
|
| 779 |
+
},
|
| 780 |
+
{
|
| 781 |
+
"token_id": 391,
|
| 782 |
+
"logit": 4.051790237426758
|
| 783 |
+
},
|
| 784 |
+
{
|
| 785 |
+
"token_id": 378,
|
| 786 |
+
"logit": 3.9965059757232666
|
| 787 |
+
}
|
| 788 |
+
],
|
| 789 |
+
"top1_match": true,
|
| 790 |
+
"top10_overlap": 9,
|
| 791 |
+
"full_vocab_mean_abs_logit_error": 0.13137772679328918,
|
| 792 |
+
"top20_union_mean_abs_logit_error": 0.2805418372154236,
|
| 793 |
+
"bf16_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 794 |
+
"quant_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 795 |
+
"bf16_repetition_loop": false,
|
| 796 |
+
"quant_repetition_loop": false,
|
| 797 |
+
"new_repetition_loop": false
|
| 798 |
+
}
|
| 799 |
+
]
|
| 800 |
+
}
|
reports/validation/q8_0_vs_bf16.json
ADDED
|
@@ -0,0 +1,800 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"bf16": "fresh\\out\\gguf\\HRM-Text-1B-BF16.gguf",
|
| 3 |
+
"quant": "fresh\\out\\gguf\\HRM-Text-1B-Q8_0.gguf",
|
| 4 |
+
"quant_name": "Q8_0",
|
| 5 |
+
"created_at_unix": 1779349955.394709,
|
| 6 |
+
"elapsed_seconds": 94.64691495895386,
|
| 7 |
+
"metadata": {
|
| 8 |
+
"bf16_general_architecture": "hrm_text",
|
| 9 |
+
"quant_general_architecture": "hrm_text"
|
| 10 |
+
},
|
| 11 |
+
"checks": {
|
| 12 |
+
"prompt_tokens_match_all_prompts": true,
|
| 13 |
+
"same_next_top1_at_least_3_of_4": true,
|
| 14 |
+
"top1_match_count": 4,
|
| 15 |
+
"top10_overlap_at_least_7_all_prompts": true,
|
| 16 |
+
"quant_general_architecture_is_hrm_text": true,
|
| 17 |
+
"no_new_repetition_loop": true,
|
| 18 |
+
"passed": true
|
| 19 |
+
},
|
| 20 |
+
"prompts": [
|
| 21 |
+
{
|
| 22 |
+
"prompt": "The quick brown fox",
|
| 23 |
+
"bf16_input_ids": [
|
| 24 |
+
341,
|
| 25 |
+
3568,
|
| 26 |
+
9054,
|
| 27 |
+
20102
|
| 28 |
+
],
|
| 29 |
+
"quant_input_ids": [
|
| 30 |
+
341,
|
| 31 |
+
3568,
|
| 32 |
+
9054,
|
| 33 |
+
20102
|
| 34 |
+
],
|
| 35 |
+
"tokens_match": true,
|
| 36 |
+
"bf16_next_top20": [
|
| 37 |
+
{
|
| 38 |
+
"token_id": 20102,
|
| 39 |
+
"logit": 14.297157287597656
|
| 40 |
+
},
|
| 41 |
+
{
|
| 42 |
+
"token_id": 40036,
|
| 43 |
+
"logit": 7.994879722595215
|
| 44 |
+
},
|
| 45 |
+
{
|
| 46 |
+
"token_id": 11979,
|
| 47 |
+
"logit": 6.699657440185547
|
| 48 |
+
},
|
| 49 |
+
{
|
| 50 |
+
"token_id": 3568,
|
| 51 |
+
"logit": 6.626206398010254
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"token_id": 4349,
|
| 55 |
+
"logit": 6.35108757019043
|
| 56 |
+
},
|
| 57 |
+
{
|
| 58 |
+
"token_id": 42,
|
| 59 |
+
"logit": 6.271071434020996
|
| 60 |
+
},
|
| 61 |
+
{
|
| 62 |
+
"token_id": 3152,
|
| 63 |
+
"logit": 5.424627304077148
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"token_id": 2301,
|
| 67 |
+
"logit": 5.3609843254089355
|
| 68 |
+
},
|
| 69 |
+
{
|
| 70 |
+
"token_id": 44,
|
| 71 |
+
"logit": 5.178180694580078
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"token_id": 3558,
|
| 75 |
+
"logit": 4.875790596008301
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"token_id": 5002,
|
| 79 |
+
"logit": 4.839702606201172
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"token_id": 3412,
|
| 83 |
+
"logit": 4.824095249176025
|
| 84 |
+
},
|
| 85 |
+
{
|
| 86 |
+
"token_id": 761,
|
| 87 |
+
"logit": 4.718862056732178
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"token_id": 19444,
|
| 91 |
+
"logit": 4.698058128356934
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"token_id": 446,
|
| 95 |
+
"logit": 4.689098358154297
|
| 96 |
+
},
|
| 97 |
+
{
|
| 98 |
+
"token_id": 16001,
|
| 99 |
+
"logit": 4.590203762054443
|
| 100 |
+
},
|
| 101 |
+
{
|
| 102 |
+
"token_id": 3297,
|
| 103 |
+
"logit": 4.408023357391357
|
| 104 |
+
},
|
| 105 |
+
{
|
| 106 |
+
"token_id": 322,
|
| 107 |
+
"logit": 4.394444942474365
|
| 108 |
+
},
|
| 109 |
+
{
|
| 110 |
+
"token_id": 8636,
|
| 111 |
+
"logit": 4.318354606628418
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"token_id": 12672,
|
| 115 |
+
"logit": 4.2834038734436035
|
| 116 |
+
}
|
| 117 |
+
],
|
| 118 |
+
"quant_next_top20": [
|
| 119 |
+
{
|
| 120 |
+
"token_id": 20102,
|
| 121 |
+
"logit": 14.367386817932129
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"token_id": 40036,
|
| 125 |
+
"logit": 8.008993148803711
|
| 126 |
+
},
|
| 127 |
+
{
|
| 128 |
+
"token_id": 3568,
|
| 129 |
+
"logit": 6.713944435119629
|
| 130 |
+
},
|
| 131 |
+
{
|
| 132 |
+
"token_id": 11979,
|
| 133 |
+
"logit": 6.684269905090332
|
| 134 |
+
},
|
| 135 |
+
{
|
| 136 |
+
"token_id": 4349,
|
| 137 |
+
"logit": 6.3594069480896
|
| 138 |
+
},
|
| 139 |
+
{
|
| 140 |
+
"token_id": 42,
|
| 141 |
+
"logit": 6.222784519195557
|
| 142 |
+
},
|
| 143 |
+
{
|
| 144 |
+
"token_id": 3152,
|
| 145 |
+
"logit": 5.488882064819336
|
| 146 |
+
},
|
| 147 |
+
{
|
| 148 |
+
"token_id": 2301,
|
| 149 |
+
"logit": 5.374516487121582
|
| 150 |
+
},
|
| 151 |
+
{
|
| 152 |
+
"token_id": 44,
|
| 153 |
+
"logit": 5.038797378540039
|
| 154 |
+
},
|
| 155 |
+
{
|
| 156 |
+
"token_id": 3558,
|
| 157 |
+
"logit": 4.9473724365234375
|
| 158 |
+
},
|
| 159 |
+
{
|
| 160 |
+
"token_id": 3412,
|
| 161 |
+
"logit": 4.896827697753906
|
| 162 |
+
},
|
| 163 |
+
{
|
| 164 |
+
"token_id": 5002,
|
| 165 |
+
"logit": 4.861919403076172
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"token_id": 761,
|
| 169 |
+
"logit": 4.789654731750488
|
| 170 |
+
},
|
| 171 |
+
{
|
| 172 |
+
"token_id": 19444,
|
| 173 |
+
"logit": 4.6751813888549805
|
| 174 |
+
},
|
| 175 |
+
{
|
| 176 |
+
"token_id": 446,
|
| 177 |
+
"logit": 4.631736755371094
|
| 178 |
+
},
|
| 179 |
+
{
|
| 180 |
+
"token_id": 16001,
|
| 181 |
+
"logit": 4.598655700683594
|
| 182 |
+
},
|
| 183 |
+
{
|
| 184 |
+
"token_id": 3297,
|
| 185 |
+
"logit": 4.449357986450195
|
| 186 |
+
},
|
| 187 |
+
{
|
| 188 |
+
"token_id": 8636,
|
| 189 |
+
"logit": 4.359951496124268
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"token_id": 322,
|
| 193 |
+
"logit": 4.300343036651611
|
| 194 |
+
},
|
| 195 |
+
{
|
| 196 |
+
"token_id": 12672,
|
| 197 |
+
"logit": 4.295340538024902
|
| 198 |
+
}
|
| 199 |
+
],
|
| 200 |
+
"top1_match": true,
|
| 201 |
+
"top10_overlap": 10,
|
| 202 |
+
"full_vocab_mean_abs_logit_error": 0.03179311007261276,
|
| 203 |
+
"top20_union_mean_abs_logit_error": 0.04881145805120468,
|
| 204 |
+
"bf16_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 205 |
+
"quant_generated_suffix": " fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox fox\n\n",
|
| 206 |
+
"bf16_repetition_loop": true,
|
| 207 |
+
"quant_repetition_loop": true,
|
| 208 |
+
"new_repetition_loop": false
|
| 209 |
+
},
|
| 210 |
+
{
|
| 211 |
+
"prompt": "In a distant future, humanity",
|
| 212 |
+
"bf16_input_ids": [
|
| 213 |
+
931,
|
| 214 |
+
236,
|
| 215 |
+
95,
|
| 216 |
+
9494,
|
| 217 |
+
2025,
|
| 218 |
+
42,
|
| 219 |
+
13279
|
| 220 |
+
],
|
| 221 |
+
"quant_input_ids": [
|
| 222 |
+
931,
|
| 223 |
+
236,
|
| 224 |
+
95,
|
| 225 |
+
9494,
|
| 226 |
+
2025,
|
| 227 |
+
42,
|
| 228 |
+
13279
|
| 229 |
+
],
|
| 230 |
+
"tokens_match": true,
|
| 231 |
+
"bf16_next_top20": [
|
| 232 |
+
{
|
| 233 |
+
"token_id": 385,
|
| 234 |
+
"logit": 8.094616889953613
|
| 235 |
+
},
|
| 236 |
+
{
|
| 237 |
+
"token_id": 42,
|
| 238 |
+
"logit": 7.381193161010742
|
| 239 |
+
},
|
| 240 |
+
{
|
| 241 |
+
"token_id": 322,
|
| 242 |
+
"logit": 7.22892951965332
|
| 243 |
+
},
|
| 244 |
+
{
|
| 245 |
+
"token_id": 5106,
|
| 246 |
+
"logit": 6.968372344970703
|
| 247 |
+
},
|
| 248 |
+
{
|
| 249 |
+
"token_id": 5835,
|
| 250 |
+
"logit": 6.5758562088012695
|
| 251 |
+
},
|
| 252 |
+
{
|
| 253 |
+
"token_id": 769,
|
| 254 |
+
"logit": 6.294882297515869
|
| 255 |
+
},
|
| 256 |
+
{
|
| 257 |
+
"token_id": 599,
|
| 258 |
+
"logit": 6.259007453918457
|
| 259 |
+
},
|
| 260 |
+
{
|
| 261 |
+
"token_id": 1980,
|
| 262 |
+
"logit": 6.067761421203613
|
| 263 |
+
},
|
| 264 |
+
{
|
| 265 |
+
"token_id": 295,
|
| 266 |
+
"logit": 6.029950141906738
|
| 267 |
+
},
|
| 268 |
+
{
|
| 269 |
+
"token_id": 2156,
|
| 270 |
+
"logit": 5.428796768188477
|
| 271 |
+
},
|
| 272 |
+
{
|
| 273 |
+
"token_id": 379,
|
| 274 |
+
"logit": 5.263803005218506
|
| 275 |
+
},
|
| 276 |
+
{
|
| 277 |
+
"token_id": 5226,
|
| 278 |
+
"logit": 5.066612243652344
|
| 279 |
+
},
|
| 280 |
+
{
|
| 281 |
+
"token_id": 446,
|
| 282 |
+
"logit": 4.910100936889648
|
| 283 |
+
},
|
| 284 |
+
{
|
| 285 |
+
"token_id": 10755,
|
| 286 |
+
"logit": 4.904460430145264
|
| 287 |
+
},
|
| 288 |
+
{
|
| 289 |
+
"token_id": 236,
|
| 290 |
+
"logit": 4.739671230316162
|
| 291 |
+
},
|
| 292 |
+
{
|
| 293 |
+
"token_id": 2026,
|
| 294 |
+
"logit": 4.661045074462891
|
| 295 |
+
},
|
| 296 |
+
{
|
| 297 |
+
"token_id": 307,
|
| 298 |
+
"logit": 4.627893447875977
|
| 299 |
+
},
|
| 300 |
+
{
|
| 301 |
+
"token_id": 1287,
|
| 302 |
+
"logit": 4.507448196411133
|
| 303 |
+
},
|
| 304 |
+
{
|
| 305 |
+
"token_id": 3891,
|
| 306 |
+
"logit": 4.162245273590088
|
| 307 |
+
},
|
| 308 |
+
{
|
| 309 |
+
"token_id": 5671,
|
| 310 |
+
"logit": 4.059610366821289
|
| 311 |
+
}
|
| 312 |
+
],
|
| 313 |
+
"quant_next_top20": [
|
| 314 |
+
{
|
| 315 |
+
"token_id": 385,
|
| 316 |
+
"logit": 8.11342716217041
|
| 317 |
+
},
|
| 318 |
+
{
|
| 319 |
+
"token_id": 42,
|
| 320 |
+
"logit": 7.516848087310791
|
| 321 |
+
},
|
| 322 |
+
{
|
| 323 |
+
"token_id": 322,
|
| 324 |
+
"logit": 7.227662563323975
|
| 325 |
+
},
|
| 326 |
+
{
|
| 327 |
+
"token_id": 5106,
|
| 328 |
+
"logit": 6.920186996459961
|
| 329 |
+
},
|
| 330 |
+
{
|
| 331 |
+
"token_id": 5835,
|
| 332 |
+
"logit": 6.690338134765625
|
| 333 |
+
},
|
| 334 |
+
{
|
| 335 |
+
"token_id": 769,
|
| 336 |
+
"logit": 6.335032939910889
|
| 337 |
+
},
|
| 338 |
+
{
|
| 339 |
+
"token_id": 599,
|
| 340 |
+
"logit": 6.263925552368164
|
| 341 |
+
},
|
| 342 |
+
{
|
| 343 |
+
"token_id": 295,
|
| 344 |
+
"logit": 6.026804447174072
|
| 345 |
+
},
|
| 346 |
+
{
|
| 347 |
+
"token_id": 1980,
|
| 348 |
+
"logit": 6.014459609985352
|
| 349 |
+
},
|
| 350 |
+
{
|
| 351 |
+
"token_id": 2156,
|
| 352 |
+
"logit": 5.370973587036133
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"token_id": 379,
|
| 356 |
+
"logit": 5.30171537399292
|
| 357 |
+
},
|
| 358 |
+
{
|
| 359 |
+
"token_id": 5226,
|
| 360 |
+
"logit": 5.06598424911499
|
| 361 |
+
},
|
| 362 |
+
{
|
| 363 |
+
"token_id": 10755,
|
| 364 |
+
"logit": 4.898964881896973
|
| 365 |
+
},
|
| 366 |
+
{
|
| 367 |
+
"token_id": 446,
|
| 368 |
+
"logit": 4.843714714050293
|
| 369 |
+
},
|
| 370 |
+
{
|
| 371 |
+
"token_id": 236,
|
| 372 |
+
"logit": 4.761955738067627
|
| 373 |
+
},
|
| 374 |
+
{
|
| 375 |
+
"token_id": 2026,
|
| 376 |
+
"logit": 4.637201309204102
|
| 377 |
+
},
|
| 378 |
+
{
|
| 379 |
+
"token_id": 307,
|
| 380 |
+
"logit": 4.629420280456543
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"token_id": 1287,
|
| 384 |
+
"logit": 4.561810493469238
|
| 385 |
+
},
|
| 386 |
+
{
|
| 387 |
+
"token_id": 5671,
|
| 388 |
+
"logit": 4.1997294425964355
|
| 389 |
+
},
|
| 390 |
+
{
|
| 391 |
+
"token_id": 3891,
|
| 392 |
+
"logit": 4.0533857345581055
|
| 393 |
+
}
|
| 394 |
+
],
|
| 395 |
+
"top1_match": true,
|
| 396 |
+
"top10_overlap": 10,
|
| 397 |
+
"full_vocab_mean_abs_logit_error": 0.028626615181565285,
|
| 398 |
+
"top20_union_mean_abs_logit_error": 0.04695785045623779,
|
| 399 |
+
"bf16_generated_suffix": "'s distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant\n\n",
|
| 400 |
+
"quant_generated_suffix": "'s distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant future, humanity's distant\n\n",
|
| 401 |
+
"bf16_repetition_loop": false,
|
| 402 |
+
"quant_repetition_loop": false,
|
| 403 |
+
"new_repetition_loop": false
|
| 404 |
+
},
|
| 405 |
+
{
|
| 406 |
+
"prompt": "Question: What is 2+2?\nAnswer:",
|
| 407 |
+
"bf16_input_ids": [
|
| 408 |
+
1069,
|
| 409 |
+
56,
|
| 410 |
+
866,
|
| 411 |
+
322,
|
| 412 |
+
236,
|
| 413 |
+
48,
|
| 414 |
+
41,
|
| 415 |
+
48,
|
| 416 |
+
1047,
|
| 417 |
+
1180,
|
| 418 |
+
56
|
| 419 |
+
],
|
| 420 |
+
"quant_input_ids": [
|
| 421 |
+
1069,
|
| 422 |
+
56,
|
| 423 |
+
866,
|
| 424 |
+
322,
|
| 425 |
+
236,
|
| 426 |
+
48,
|
| 427 |
+
41,
|
| 428 |
+
48,
|
| 429 |
+
1047,
|
| 430 |
+
1180,
|
| 431 |
+
56
|
| 432 |
+
],
|
| 433 |
+
"tokens_match": true,
|
| 434 |
+
"bf16_next_top20": [
|
| 435 |
+
{
|
| 436 |
+
"token_id": 236,
|
| 437 |
+
"logit": 9.97547435760498
|
| 438 |
+
},
|
| 439 |
+
{
|
| 440 |
+
"token_id": 391,
|
| 441 |
+
"logit": 6.012524604797363
|
| 442 |
+
},
|
| 443 |
+
{
|
| 444 |
+
"token_id": 315,
|
| 445 |
+
"logit": 5.968222618103027
|
| 446 |
+
},
|
| 447 |
+
{
|
| 448 |
+
"token_id": 866,
|
| 449 |
+
"logit": 5.672043323516846
|
| 450 |
+
},
|
| 451 |
+
{
|
| 452 |
+
"token_id": 395,
|
| 453 |
+
"logit": 5.606063365936279
|
| 454 |
+
},
|
| 455 |
+
{
|
| 456 |
+
"token_id": 5444,
|
| 457 |
+
"logit": 5.2098188400268555
|
| 458 |
+
},
|
| 459 |
+
{
|
| 460 |
+
"token_id": 4746,
|
| 461 |
+
"logit": 4.777042388916016
|
| 462 |
+
},
|
| 463 |
+
{
|
| 464 |
+
"token_id": 1827,
|
| 465 |
+
"logit": 4.54682731628418
|
| 466 |
+
},
|
| 467 |
+
{
|
| 468 |
+
"token_id": 369,
|
| 469 |
+
"logit": 3.9637503623962402
|
| 470 |
+
},
|
| 471 |
+
{
|
| 472 |
+
"token_id": 970,
|
| 473 |
+
"logit": 3.765744686126709
|
| 474 |
+
},
|
| 475 |
+
{
|
| 476 |
+
"token_id": 725,
|
| 477 |
+
"logit": 3.7245254516601562
|
| 478 |
+
},
|
| 479 |
+
{
|
| 480 |
+
"token_id": 421,
|
| 481 |
+
"logit": 3.630314350128174
|
| 482 |
+
},
|
| 483 |
+
{
|
| 484 |
+
"token_id": 731,
|
| 485 |
+
"logit": 3.2161593437194824
|
| 486 |
+
},
|
| 487 |
+
{
|
| 488 |
+
"token_id": 401,
|
| 489 |
+
"logit": 3.1881794929504395
|
| 490 |
+
},
|
| 491 |
+
{
|
| 492 |
+
"token_id": 2286,
|
| 493 |
+
"logit": 3.168586254119873
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"token_id": 1422,
|
| 497 |
+
"logit": 3.151968240737915
|
| 498 |
+
},
|
| 499 |
+
{
|
| 500 |
+
"token_id": 10528,
|
| 501 |
+
"logit": 3.148939847946167
|
| 502 |
+
},
|
| 503 |
+
{
|
| 504 |
+
"token_id": 1544,
|
| 505 |
+
"logit": 3.1278254985809326
|
| 506 |
+
},
|
| 507 |
+
{
|
| 508 |
+
"token_id": 973,
|
| 509 |
+
"logit": 3.1256284713745117
|
| 510 |
+
},
|
| 511 |
+
{
|
| 512 |
+
"token_id": 4286,
|
| 513 |
+
"logit": 3.0892391204833984
|
| 514 |
+
}
|
| 515 |
+
],
|
| 516 |
+
"quant_next_top20": [
|
| 517 |
+
{
|
| 518 |
+
"token_id": 236,
|
| 519 |
+
"logit": 9.934441566467285
|
| 520 |
+
},
|
| 521 |
+
{
|
| 522 |
+
"token_id": 391,
|
| 523 |
+
"logit": 6.028039932250977
|
| 524 |
+
},
|
| 525 |
+
{
|
| 526 |
+
"token_id": 315,
|
| 527 |
+
"logit": 5.897328853607178
|
| 528 |
+
},
|
| 529 |
+
{
|
| 530 |
+
"token_id": 866,
|
| 531 |
+
"logit": 5.70652961730957
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"token_id": 395,
|
| 535 |
+
"logit": 5.564680099487305
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"token_id": 5444,
|
| 539 |
+
"logit": 5.2897419929504395
|
| 540 |
+
},
|
| 541 |
+
{
|
| 542 |
+
"token_id": 4746,
|
| 543 |
+
"logit": 4.698920726776123
|
| 544 |
+
},
|
| 545 |
+
{
|
| 546 |
+
"token_id": 1827,
|
| 547 |
+
"logit": 4.585605621337891
|
| 548 |
+
},
|
| 549 |
+
{
|
| 550 |
+
"token_id": 369,
|
| 551 |
+
"logit": 4.039481163024902
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"token_id": 725,
|
| 555 |
+
"logit": 3.783412456512451
|
| 556 |
+
},
|
| 557 |
+
{
|
| 558 |
+
"token_id": 970,
|
| 559 |
+
"logit": 3.707461357116699
|
| 560 |
+
},
|
| 561 |
+
{
|
| 562 |
+
"token_id": 421,
|
| 563 |
+
"logit": 3.656597137451172
|
| 564 |
+
},
|
| 565 |
+
{
|
| 566 |
+
"token_id": 731,
|
| 567 |
+
"logit": 3.231766700744629
|
| 568 |
+
},
|
| 569 |
+
{
|
| 570 |
+
"token_id": 10528,
|
| 571 |
+
"logit": 3.195836067199707
|
| 572 |
+
},
|
| 573 |
+
{
|
| 574 |
+
"token_id": 2286,
|
| 575 |
+
"logit": 3.1658873558044434
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"token_id": 973,
|
| 579 |
+
"logit": 3.1603126525878906
|
| 580 |
+
},
|
| 581 |
+
{
|
| 582 |
+
"token_id": 401,
|
| 583 |
+
"logit": 3.149336814880371
|
| 584 |
+
},
|
| 585 |
+
{
|
| 586 |
+
"token_id": 4286,
|
| 587 |
+
"logit": 3.12637996673584
|
| 588 |
+
},
|
| 589 |
+
{
|
| 590 |
+
"token_id": 1422,
|
| 591 |
+
"logit": 3.116433620452881
|
| 592 |
+
},
|
| 593 |
+
{
|
| 594 |
+
"token_id": 1544,
|
| 595 |
+
"logit": 3.1147987842559814
|
| 596 |
+
}
|
| 597 |
+
],
|
| 598 |
+
"top1_match": true,
|
| 599 |
+
"top10_overlap": 9,
|
| 600 |
+
"full_vocab_mean_abs_logit_error": 0.02864035964012146,
|
| 601 |
+
"top20_union_mean_abs_logit_error": 0.04218750074505806,
|
| 602 |
+
"bf16_generated_suffix": " 6 [end of text]\n\n\n",
|
| 603 |
+
"quant_generated_suffix": " 6 [end of text]\n\n\n",
|
| 604 |
+
"bf16_repetition_loop": false,
|
| 605 |
+
"quant_repetition_loop": false,
|
| 606 |
+
"new_repetition_loop": false
|
| 607 |
+
},
|
| 608 |
+
{
|
| 609 |
+
"prompt": "def fibonacci(n):",
|
| 610 |
+
"bf16_input_ids": [
|
| 611 |
+
26763,
|
| 612 |
+
8430,
|
| 613 |
+
21728,
|
| 614 |
+
3145,
|
| 615 |
+
10018
|
| 616 |
+
],
|
| 617 |
+
"quant_input_ids": [
|
| 618 |
+
26763,
|
| 619 |
+
8430,
|
| 620 |
+
21728,
|
| 621 |
+
3145,
|
| 622 |
+
10018
|
| 623 |
+
],
|
| 624 |
+
"tokens_match": true,
|
| 625 |
+
"bf16_next_top20": [
|
| 626 |
+
{
|
| 627 |
+
"token_id": 56,
|
| 628 |
+
"logit": 10.327644348144531
|
| 629 |
+
},
|
| 630 |
+
{
|
| 631 |
+
"token_id": 2904,
|
| 632 |
+
"logit": 7.035848140716553
|
| 633 |
+
},
|
| 634 |
+
{
|
| 635 |
+
"token_id": 11,
|
| 636 |
+
"logit": 6.860950946807861
|
| 637 |
+
},
|
| 638 |
+
{
|
| 639 |
+
"token_id": 10018,
|
| 640 |
+
"logit": 6.495933532714844
|
| 641 |
+
},
|
| 642 |
+
{
|
| 643 |
+
"token_id": 108,
|
| 644 |
+
"logit": 6.341927528381348
|
| 645 |
+
},
|
| 646 |
+
{
|
| 647 |
+
"token_id": 318,
|
| 648 |
+
"logit": 6.146999359130859
|
| 649 |
+
},
|
| 650 |
+
{
|
| 651 |
+
"token_id": 236,
|
| 652 |
+
"logit": 6.0702805519104
|
| 653 |
+
},
|
| 654 |
+
{
|
| 655 |
+
"token_id": 38,
|
| 656 |
+
"logit": 5.865842819213867
|
| 657 |
+
},
|
| 658 |
+
{
|
| 659 |
+
"token_id": 68,
|
| 660 |
+
"logit": 5.02256965637207
|
| 661 |
+
},
|
| 662 |
+
{
|
| 663 |
+
"token_id": 39,
|
| 664 |
+
"logit": 4.66829252243042
|
| 665 |
+
},
|
| 666 |
+
{
|
| 667 |
+
"token_id": 76,
|
| 668 |
+
"logit": 4.6676859855651855
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"token_id": 59,
|
| 672 |
+
"logit": 4.662688255310059
|
| 673 |
+
},
|
| 674 |
+
{
|
| 675 |
+
"token_id": 57,
|
| 676 |
+
"logit": 4.66217041015625
|
| 677 |
+
},
|
| 678 |
+
{
|
| 679 |
+
"token_id": 1941,
|
| 680 |
+
"logit": 4.598584175109863
|
| 681 |
+
},
|
| 682 |
+
{
|
| 683 |
+
"token_id": 401,
|
| 684 |
+
"logit": 4.574923515319824
|
| 685 |
+
},
|
| 686 |
+
{
|
| 687 |
+
"token_id": 395,
|
| 688 |
+
"logit": 4.532867431640625
|
| 689 |
+
},
|
| 690 |
+
{
|
| 691 |
+
"token_id": 8430,
|
| 692 |
+
"logit": 4.5034050941467285
|
| 693 |
+
},
|
| 694 |
+
{
|
| 695 |
+
"token_id": 95,
|
| 696 |
+
"logit": 4.318176746368408
|
| 697 |
+
},
|
| 698 |
+
{
|
| 699 |
+
"token_id": 364,
|
| 700 |
+
"logit": 4.2872209548950195
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"token_id": 48,
|
| 704 |
+
"logit": 4.243288516998291
|
| 705 |
+
}
|
| 706 |
+
],
|
| 707 |
+
"quant_next_top20": [
|
| 708 |
+
{
|
| 709 |
+
"token_id": 56,
|
| 710 |
+
"logit": 10.283997535705566
|
| 711 |
+
},
|
| 712 |
+
{
|
| 713 |
+
"token_id": 2904,
|
| 714 |
+
"logit": 6.931118965148926
|
| 715 |
+
},
|
| 716 |
+
{
|
| 717 |
+
"token_id": 10018,
|
| 718 |
+
"logit": 6.879117012023926
|
| 719 |
+
},
|
| 720 |
+
{
|
| 721 |
+
"token_id": 11,
|
| 722 |
+
"logit": 6.837632179260254
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"token_id": 108,
|
| 726 |
+
"logit": 6.313412666320801
|
| 727 |
+
},
|
| 728 |
+
{
|
| 729 |
+
"token_id": 318,
|
| 730 |
+
"logit": 6.080738067626953
|
| 731 |
+
},
|
| 732 |
+
{
|
| 733 |
+
"token_id": 236,
|
| 734 |
+
"logit": 6.061807632446289
|
| 735 |
+
},
|
| 736 |
+
{
|
| 737 |
+
"token_id": 38,
|
| 738 |
+
"logit": 5.88294792175293
|
| 739 |
+
},
|
| 740 |
+
{
|
| 741 |
+
"token_id": 39,
|
| 742 |
+
"logit": 5.063222885131836
|
| 743 |
+
},
|
| 744 |
+
{
|
| 745 |
+
"token_id": 68,
|
| 746 |
+
"logit": 4.971517086029053
|
| 747 |
+
},
|
| 748 |
+
{
|
| 749 |
+
"token_id": 76,
|
| 750 |
+
"logit": 4.6997175216674805
|
| 751 |
+
},
|
| 752 |
+
{
|
| 753 |
+
"token_id": 59,
|
| 754 |
+
"logit": 4.695155620574951
|
| 755 |
+
},
|
| 756 |
+
{
|
| 757 |
+
"token_id": 57,
|
| 758 |
+
"logit": 4.625309467315674
|
| 759 |
+
},
|
| 760 |
+
{
|
| 761 |
+
"token_id": 401,
|
| 762 |
+
"logit": 4.536755561828613
|
| 763 |
+
},
|
| 764 |
+
{
|
| 765 |
+
"token_id": 1941,
|
| 766 |
+
"logit": 4.511960983276367
|
| 767 |
+
},
|
| 768 |
+
{
|
| 769 |
+
"token_id": 395,
|
| 770 |
+
"logit": 4.4221577644348145
|
| 771 |
+
},
|
| 772 |
+
{
|
| 773 |
+
"token_id": 8430,
|
| 774 |
+
"logit": 4.333778381347656
|
| 775 |
+
},
|
| 776 |
+
{
|
| 777 |
+
"token_id": 364,
|
| 778 |
+
"logit": 4.322488307952881
|
| 779 |
+
},
|
| 780 |
+
{
|
| 781 |
+
"token_id": 2559,
|
| 782 |
+
"logit": 4.306702136993408
|
| 783 |
+
},
|
| 784 |
+
{
|
| 785 |
+
"token_id": 95,
|
| 786 |
+
"logit": 4.283227443695068
|
| 787 |
+
}
|
| 788 |
+
],
|
| 789 |
+
"top1_match": true,
|
| 790 |
+
"top10_overlap": 10,
|
| 791 |
+
"full_vocab_mean_abs_logit_error": 0.026543496176600456,
|
| 792 |
+
"top20_union_mean_abs_logit_error": 0.09682635962963104,
|
| 793 |
+
"bf16_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 794 |
+
"quant_generated_suffix": "::::::::::::::::::::::::::::::::\n\n",
|
| 795 |
+
"bf16_repetition_loop": false,
|
| 796 |
+
"quant_repetition_loop": false,
|
| 797 |
+
"new_repetition_loop": false
|
| 798 |
+
}
|
| 799 |
+
]
|
| 800 |
+
}
|
reports/validation/quantization_report.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# HRM-Text-1B GGUF Quantization Report
|
| 2 |
+
|
| 3 |
+
Date: 2026-05-21
|
| 4 |
+
|
| 5 |
+
## Source
|
| 6 |
+
|
| 7 |
+
- Source GGUF: `fresh\out\gguf\HRM-Text-1B-BF16.gguf`
|
| 8 |
+
- Source GGUF SHA256: `2DD5E2EF55E40C46DB0D0CB4CF1427A4E72DA34FEE36F0D2B73D081D0E1C2010`
|
| 9 |
+
- Quantization tool: `fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-quantize.exe`
|
| 10 |
+
- llama.cpp base commit: `6a257d44633d4a752183ed778b88d2924d0a6b9d`
|
| 11 |
+
- Importance matrix: not used
|
| 12 |
+
|
| 13 |
+
## Generated Quantizations
|
| 14 |
+
|
| 15 |
+
| Variant | File | Size (bytes) | SHA256 | Validation | Upload |
|
| 16 |
+
| --- | --- | ---: | --- | --- | --- |
|
| 17 |
+
| BF16 | `HRM-Text-1B-BF16.gguf` | `2367995648` | `2DD5E2EF55E40C46DB0D0CB4CF1427A4E72DA34FEE36F0D2B73D081D0E1C2010` | Source of truth | Yes |
|
| 18 |
+
| Q8_0 | `HRM-Text-1B-Q8_0.gguf` | `1259126560` | `C0729C267C3421E1F6DE0488AC5448E98EA30E56514DAF210596B70AC3F9786D` | Pass | Yes |
|
| 19 |
+
| Q6_K | `HRM-Text-1B-Q6_K.gguf` | `972668704` | `24D93CA4EF4A02CFE415E3EA56A78AD65198A165A4157B928004B58DBDA2D93C` | Pass | Yes |
|
| 20 |
+
| Q5_K_M | `HRM-Text-1B-Q5_K_M.gguf` | `851509024` | `F6CE71A076EC897174C555D810ED6E379767D52F9396D485B42E42BF8DB1D0B7` | Pass | Yes |
|
| 21 |
+
| Q4_K_M | `HRM-Text-1B-Q4_K_M.gguf` | `737476384` | `A1AF847D2D3AEB69730D8D46D63AFCA84091BB18285856A45DA8E4A1419224DA` | Fail | No |
|
| 22 |
+
|
| 23 |
+
## Validation Summary
|
| 24 |
+
|
| 25 |
+
Each quantized GGUF was compared against the BF16 GGUF with the patched HRM-enabled runtime on four fixed prompts.
|
| 26 |
+
|
| 27 |
+
| Variant | Token IDs | Top-1 matches | Min top-10 overlap | `hrm_text` metadata | New loop check | Result |
|
| 28 |
+
| --- | --- | ---: | ---: | --- | --- | --- |
|
| 29 |
+
| Q8_0 | Pass | `4/4` | `9/10` | Pass | Pass | Pass |
|
| 30 |
+
| Q6_K | Pass | `4/4` | `9/10` | Pass | Pass | Pass |
|
| 31 |
+
| Q5_K_M | Pass | `4/4` | `9/10` | Pass | Pass | Pass |
|
| 32 |
+
| Q4_K_M | Pass | `3/4` | `8/10` | Pass | Fail | Fail |
|
| 33 |
+
|
| 34 |
+
`Q4_K_M` is not uploaded because it introduced a new single-token repetition loop for `In a distant future, humanity`: the quantized output repeated `humanity,` while BF16 repeated a longer source-model phrase. This matches the safety rule for this release: do not upload a quantization that introduces new repetition pathology.
|
| 35 |
+
|
| 36 |
+
## Commands
|
| 37 |
+
|
| 38 |
+
```powershell
|
| 39 |
+
fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-quantize.exe fresh\out\gguf\HRM-Text-1B-BF16.gguf fresh\out\gguf\HRM-Text-1B-Q8_0.gguf Q8_0 4
|
| 40 |
+
fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-quantize.exe fresh\out\gguf\HRM-Text-1B-BF16.gguf fresh\out\gguf\HRM-Text-1B-Q6_K.gguf Q6_K 4
|
| 41 |
+
fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-quantize.exe fresh\out\gguf\HRM-Text-1B-BF16.gguf fresh\out\gguf\HRM-Text-1B-Q5_K_M.gguf Q5_K_M 4
|
| 42 |
+
fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-quantize.exe fresh\out\gguf\HRM-Text-1B-BF16.gguf fresh\out\gguf\HRM-Text-1B-Q4_K_M.gguf Q4_K_M 4
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
```powershell
|
| 46 |
+
python fresh\tools\validate_quant_runtime.py --bf16 fresh\out\gguf\HRM-Text-1B-BF16.gguf --quant fresh\out\gguf\HRM-Text-1B-Q8_0.gguf --quant-name Q8_0 --llama-results fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-results.exe --llama-completion fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-completion.exe --out fresh\reports\validation\q8_0_vs_bf16.json --work-dir fresh\reports\validation\runtime_tmp\q8_0 --n-generate 32
|
| 47 |
+
python fresh\tools\validate_quant_runtime.py --bf16 fresh\out\gguf\HRM-Text-1B-BF16.gguf --quant fresh\out\gguf\HRM-Text-1B-Q6_K.gguf --quant-name Q6_K --llama-results fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-results.exe --llama-completion fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-completion.exe --out fresh\reports\validation\q6_k_vs_bf16.json --work-dir fresh\reports\validation\runtime_tmp\q6_k --n-generate 32
|
| 48 |
+
python fresh\tools\validate_quant_runtime.py --bf16 fresh\out\gguf\HRM-Text-1B-BF16.gguf --quant fresh\out\gguf\HRM-Text-1B-Q5_K_M.gguf --quant-name Q5_K_M --llama-results fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-results.exe --llama-completion fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-completion.exe --out fresh\reports\validation\q5_k_m_vs_bf16.json --work-dir fresh\reports\validation\runtime_tmp\q5_k_m --n-generate 32
|
| 49 |
+
python fresh\tools\validate_quant_runtime.py --bf16 fresh\out\gguf\HRM-Text-1B-BF16.gguf --quant fresh\out\gguf\HRM-Text-1B-Q4_K_M.gguf --quant-name Q4_K_M --llama-results fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-results.exe --llama-completion fresh\third_party\llama.cpp\build-hrm-nmake\bin\llama-completion.exe --out fresh\reports\validation\q4_k_m_vs_bf16.json --work-dir fresh\reports\validation\runtime_tmp\q4_k_m --n-generate 32
|
| 50 |
+
```
|