Extended reasoning benchmark - 2025-12-13

benchmark_results/chronos_extended_results.json

@@ -0,0 +1,298 @@
+{
+  "model": "Chronos-1.5B (GGUF)",
+  "base_model": "VibeThinker-1.5B",
+  "principle": "Spectrum-to-Signal",
+  "gguf_file": "chronos-o1-1.5b-f16.gguf",
+  "evaluation_date": "2025-12-13T18:23:52.731239",
+  "backend": "llama-cpp-python",
+  "context_window": 16384,
+  "max_generation": 15000,
+  "results": {
+    "aime_2024": {
+      "score": 60.0,
+      "total": 30,
+      "correct": 18,
+      "baseline": 50.4,
+      "delta": 9.6,
+      "avg_response_length": 35209,
+      "reasoning_shown_count": 28,
+      "results": [
+        {
+          "question_id": 0,
+          "question": "Every morning Aya goes for a $9$-kilometer-long walk and stops at a coffee shop afterwards. When she...",
+          "correct": true,
+          "model_answer": "204",
+          "correct_answer": "204",
+          "response_length": 20814,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 1,
+          "question": "Let $ABC$ be a triangle inscribed in circle $\\omega$. Let the tangents to $\\omega$ at $B$ and $C$ in...",
+          "correct": true,
+          "model_answer": "113",
+          "correct_answer": "113",
+          "response_length": 38590,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 2,
+          "question": "Each vertex of a regular octagon is independently colored either red or blue with equal probability....",
+          "correct": false,
+          "model_answer": "any subset missing one element will have difference set equal to the entire group. Because even though that element is missing, other pairs can generate all differences.",
+          "correct_answer": "371",
+          "response_length": 51650,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 3,
+          "question": "Define $f(x)=|| x|-\\tfrac{1}{2}|$ and $g(x)=|| x|-\\tfrac{1}{4}|$. Find the number of intersections o...",
+          "correct": false,
+          "model_answer": "2",
+          "correct_answer": "385",
+          "response_length": 39251,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 4,
+          "question": "Let $p$ be the least prime number for which there exists a positive integer $n$ such that $n^{4}+1$ ...",
+          "correct": true,
+          "model_answer": "110",
+          "correct_answer": "110",
+          "response_length": 32837,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 5,
+          "question": "Let $ABCD$ be a tetrahedron such that $AB=CD= \\sqrt{41}$, $AC=BD= \\sqrt{80}$, and $BC=AD= \\sqrt{89}$...",
+          "correct": false,
+          "model_answer": "2",
+          "correct_answer": "104",
+          "response_length": 36863,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 6,
+          "question": "Let $\\mathcal{B}$ be the set of rectangular boxes with surface area $54$ and volume $23$. Let $r$ be...",
+          "correct": true,
+          "model_answer": "721",
+          "correct_answer": "721",
+          "response_length": 30271,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 7,
+          "question": "There exist real numbers $x$ and $y$, both greater than 1, such that $\\log_x\\left(y^x\\right)=\\log_y\\...",
+          "correct": true,
+          "model_answer": "25",
+          "correct_answer": "025",
+          "response_length": 28317,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 8,
+          "question": "Alice and Bob play the following game. A stack of $n$ tokens lies before them. The players take turn...",
+          "correct": true,
+          "model_answer": "809",
+          "correct_answer": "809",
+          "response_length": 36569,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 9,
+          "question": "Jen enters a lottery by picking $4$ distinct numbers from $S=\\{1,2,3,\\cdots,9,10\\}.$ $4$ numbers are...",
+          "correct": true,
+          "model_answer": "116",
+          "correct_answer": "116",
+          "response_length": 39577,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 10,
+          "question": "Rectangles $ABCD$ and $EFGH$ are drawn such that $D,E,C,F$ are collinear. Also, $A,D,H,G$ all lie on...",
+          "correct": true,
+          "model_answer": "104",
+          "correct_answer": "104",
+          "response_length": 34939,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 11,
+          "question": "Consider the paths of length $16$ that follow the lines from the lower left corner to the upper righ...",
+          "correct": true,
+          "model_answer": "294",
+          "correct_answer": "294",
+          "response_length": 21751,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 12,
+          "question": "Find the largest possible real part of \\[(75+117i)z+\\frac{96+144i}{z}\\]where $z$ is a complex number...",
+          "correct": true,
+          "model_answer": "540",
+          "correct_answer": "540",
+          "response_length": 22756,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 13,
+          "question": "Eight circles of radius $34$ are sequentially tangent, and two of the circles are tangent to $AB$ an...",
+          "correct": false,
+          "model_answer": "2024",
+          "correct_answer": "197",
+          "response_length": 62139,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 14,
+          "question": "Let $A$, $B$, $C$, and $D$ be point on the hyperbola $\\frac{x^2}{20}- \\frac{y^2}{24} = 1$ such that ...",
+          "correct": false,
+          "model_answer": "",
+          "correct_answer": "480",
+          "response_length": 26,
+          "reasoning_shown": false
+        },
+        {
+          "question_id": 15,
+          "question": "Among the 900 residents of Aimeville, there are 195 who own a diamond ring, 367 who own a set of gol...",
+          "correct": true,
+          "model_answer": "73",
+          "correct_answer": "073",
+          "response_length": 43730,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 16,
+          "question": "Let $\\triangle ABC$ have circumcenter $O$ and incenter $I$ with $\\overline{IA}\\perp\\overline{OI}$, c...",
+          "correct": true,
+          "model_answer": "468",
+          "correct_answer": "468",
+          "response_length": 41499,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 17,
+          "question": "Find the number of triples of nonnegative integers \\((a,b,c)\\) satisfying \\(a + b + c = 300\\) and\n\\b...",
+          "correct": true,
+          "model_answer": "601",
+          "correct_answer": "601",
+          "response_length": 40149,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 18,
+          "question": "Let \\(O=(0,0)\\), \\(A=\\left(\\tfrac{1}{2},0\\right)\\), and \\(B=\\left(0,\\tfrac{\\sqrt{3}}{2}\\right)\\) be ...",
+          "correct": false,
+          "model_answer": "1",
+          "correct_answer": "023",
+          "response_length": 45824,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 19,
+          "question": "Let $\\omega\\neq 1$ be a 13th root of unity. Find the remainder when\n\\[\\prod_{k=0}^{12}(2-2\\omega^k+\\...",
+          "correct": true,
+          "model_answer": "321",
+          "correct_answer": "321",
+          "response_length": 31675,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 20,
+          "question": "Let \\(b\\ge 2\\) be an integer. Call a positive integer \\(n\\) \\(b\\text-\\textit{eautiful}\\) if it has e...",
+          "correct": false,
+          "model_answer": "15",
+          "correct_answer": "211",
+          "response_length": 40970,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 21,
+          "question": "Find the number of rectangles that can be formed inside a fixed regular dodecagon ($12$-gon) where e...",
+          "correct": false,
+          "model_answer": "6",
+          "correct_answer": "315",
+          "response_length": 42574,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 22,
+          "question": "A list of positive integers has the following properties:\n$\\bullet$ The sum of the items in the list...",
+          "correct": true,
+          "model_answer": "236",
+          "correct_answer": "236",
+          "response_length": 52111,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 23,
+          "question": "Find the number of ways to place a digit in each cell of a 2x3 grid so that the sum of the two numbe...",
+          "correct": true,
+          "model_answer": "45",
+          "correct_answer": "045",
+          "response_length": 34868,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 24,
+          "question": "Let $x,y$ and $z$ be positive real numbers that satisfy the following system of equations:\n\\[\\log_2\\...",
+          "correct": true,
+          "model_answer": "33",
+          "correct_answer": "033",
+          "response_length": 12557,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 25,
+          "question": "Let ABCDEF be a convex equilateral hexagon in which all pairs of opposite sides are parallel. The tr...",
+          "correct": false,
+          "model_answer": "",
+          "correct_answer": "080",
+          "response_length": 26,
+          "reasoning_shown": false
+        },
+        {
+          "question_id": 26,
+          "question": "Alice chooses a set $A$ of positive integers. Then Bob lists all finite nonempty sets $B$ of positiv...",
+          "correct": true,
+          "model_answer": "55",
+          "correct_answer": "055",
+          "response_length": 26137,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 27,
+          "question": "Let $N$ be the greatest four-digit positive integer with the property that whenever one of its digit...",
+          "correct": false,
+          "model_answer": "5694",
+          "correct_answer": "699",
+          "response_length": 39108,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 28,
+          "question": "Torus $T$ is the surface produced by revolving a circle with radius $3$ around an axis in the plane ...",
+          "correct": false,
+          "model_answer": "0",
+          "correct_answer": "127",
+          "response_length": 47624,
+          "reasoning_shown": true
+        },
+        {
+          "question_id": 29,
+          "question": "There is a collection of $25$ indistinguishable white chips and $25$ indistinguishable black chips. ...",
+          "correct": false,
+          "model_answer": "the columns' colors must be set such that for each row i with color R_i = C, the columns intersecting that row can only have color C if they include any cell from the row.",
+          "correct_answer": "902",
+          "response_length": 61055,
+          "reasoning_shown": true
+        }
+      ]
+    },
+    "aime_2025": {
+      "score": 0,
+      "total": 0,
+      "error": "Config name is missing.\nPlease pick one among the available configs: ['AIME2025-I', 'AIME2025-II']\nExample of usage:\n\t`load_dataset('opencompass/AIME2025', 'AIME2025-I')`"
+    }
+  }
+}