Upload 50 files

.gitattributes

@@ -40,3 +40,6 @@ UM_Handbook/assets/tensortalk_demo_chat.jpg filter=lfs diff=lfs merge=lfs -text
 UM_Handbook/outputs/qwen3_um_handbook_baseline2_rag_a100/lora_adapter/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 UM_Handbook/outputs/qwen3_um_handbook_baseline2_rag_a100/trainer_runs/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 UM_Handbook/outputs/qwen3_um_handbook_baseline2_rag_a100/trainer_runs/checkpoint-900/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/tokenizer.json filter=lfs diff=lfs merge=lfs -text

UM_Handbook/outputs/baseline2_rag_harness_agent/dataset_split_summary.json

@@ -0,0 +1,14 @@
+{
+  "dataset_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/Dataset/SFT_Dataset/SFT_QA_Training_Ready.jsonl",
+  "metadata_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/Dataset/SFT_Dataset/SFT_QA_Metadata.jsonl",
+  "rag_kb_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/Dataset/RAG/UM_RAG_Knowledge_Base.jsonl",
+  "rag_augmented_dataset_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/outputs/baseline2_rag_harness_agent/rag_augmented_dataset/rag_augmented_sft_dataset.jsonl",
+  "total_examples": 1000,
+  "train_examples": 800,
+  "validation_examples": 100,
+  "test_examples": 100,
+  "seed": 42,
+  "train_ratio": 0.8,
+  "validation_ratio": 0.1,
+  "test_ratio": 0.1
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/generation_eval/generation_details.jsonl

@@ -0,0 +1,20 @@
+{"qa_id": "qa_000031", "index_id": "UMI-0031", "query": "What are the programme goals, objectives, or learning outcomes for Master in Data Science?", "gold_answer": "The programme objectives describe data scientists who can extract meaningful insights, deliver effective data science solutions, sustain lifelong learning with an entrepreneurial mindset, and lead or contribute responsibly in public or private organisations. The learning outcomes highlight mastery of important data science concepts, recommending innovative data science and scientific computing solutions, constructing efficient data science tools, communicating with stakeholders, publishing and presenting solutions, using digital and analytic skills to acquire and interpret knowledge, leadership, lifelong learning, entrepreneurship, and professional ethics.", "plain_answer": "<think>\n\n</think>\n\nThe programme learning outcomes for Master in Data Science include constructing mathematical models for data science problem solving, applying programming and data science techniques to collect, clean, visualise, and analyse data, combining domain knowledge with data analytic techniques to recommend or design data-driven solutions, using communication and teamwork skills to deliver data science solutions through appropriate technical and soft skills, and demonstrating ethical practice in data science.", "rag_answer": "<think>\n\n</think>\n\nThe programme objectives describe data scientists who can extract meaningful insights, deliver effective data science solutions, sustain lifelong learning with an entrepreneurial mindset, and lead or contribute responsibly in public or private organisations. The learning outcomes highlight mastery of important data science concepts, recommending innovative data science and scientific computing solutions, constructing efficient data science tools, communicating with stakeholders, publishing and presenting solutions, using digital and analytic skills to acquire and interpret knowledge, leadership, lifelong learning, entrepreneurship, and professional ethics.", "plain_exact_match": 0.0, "plain_token_f1": 0.4, "rag_exact_match": 0.0, "rag_token_f1": 0.9877, "retrieved_kb_ids": ["RAGKB-0427", "RAGKB-0428", "RAGKB-0429"], "retrieved_sections": ["Master in Data Science", "Master in Data Science", "Master in Data Science"], "retrieved_scopes": ["postgraduate", "postgraduate", "postgraduate"]}
+{"qa_id": "qa_000883", "index_id": "UMI-0883", "query": "Ngl, what distinctive specialization focus is highlighted for Bachelor of Computer Science (Information Systems)?", "gold_answer": "The distinctive specialization focus is Information Systems, with deeper emphasis on Information Systems techniques for analysing information, designing solutions, and developing computer-based systems.", "plain_answer": "<think>\n\n</think>\n\nThe Bachelor of Computer Science (Information Systems) focus is highlighted as a specialization in Information Systems.", "rag_answer": "<think>\n\n</think>\n\nThe distinctive specialization focus is Information Systems, with deeper emphasis on Information Systems techniques for analysing information, designing solutions, and developing computer-based systems.", "plain_exact_match": 0.0, "plain_token_f1": 0.4286, "rag_exact_match": 0.0, "rag_token_f1": 0.96, "retrieved_kb_ids": ["RAGKB-0336", "RAGKB-0338", "RAGKB-0341"], "retrieved_sections": ["Undergraduate Programme Goals and Learning Outcomes", "Undergraduate Programme Goals and Learning Outcomes", "Undergraduate Programme Goals and Learning Outcomes"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000387", "index_id": "UMI-0387", "query": "What policy links, regulations, or university rule documents are referenced for undergraduate students?", "gold_answer": "The undergraduate rules section refers students to the University of Malaya Bachelor's Degree rules and regulations, together with related university policy and administrative references.", "plain_answer": "<think>\n\n</think>\n\nThe undergraduate policy references include the Universiti Malaya (Undergraduate) Rules 2019, as well as related university regulations and guidelines linked in the policy section.", "rag_answer": "<think>\n\n</think>\n\nThe undergraduate rules section refers students to the University of Malaya Bachelor's Degree rules and regulations, together with related university policy and administrative references.", "plain_exact_match": 0.0, "plain_token_f1": 0.4706, "rag_exact_match": 0.0, "rag_token_f1": 0.9615, "retrieved_kb_ids": ["RAGKB-0320", "RAGKB-0142", "RAGKB-0049"], "retrieved_sections": ["Undergraduate Rules and Regulations", "Undergraduate Faculty Identity", "Postgraduate General Information"], "retrieved_scopes": ["undergraduate", "undergraduate", "postgraduate"]}
+{"qa_id": "qa_000254", "index_id": "UMI-0254", "query": "What major milestones appear in the Graduate on Time schedule for Master of Computer Science (By Research)?", "gold_answer": "The Graduate on Time schedule for Master of Computer Science (By Research) lays out the expected semester-by-semester milestones for timely completion, including candidature progression, research-related stages, and expected outputs where applicable.", "plain_answer": "<think>\n\n</think>\n\nThe retrieved evidence shows that the Graduate on Time schedule for Master of Computer Science (By Research) includes milestones such as Progress Report I at the end of Semester I, Progress Report II at the end of Semester II, Proposal Presentation and Proposal Review Board for the end of Semester II or the start of Semester III, Examination of Completion for the end of Semester III or the start of Semester IV, and submission of the dissertation.", "rag_answer": "<think>\n\n</think>\n\nThe Graduate on Time schedule for Master of Computer Science (By Research) shows a staged completion path that begins with research confirmation, includes milestones such as proposal development and defence, dissertation editing, feedback incorporation, dissertation submission, progress report submission, and final completion after external examiner assessment.", "plain_exact_match": 0.0, "plain_token_f1": 0.3363, "rag_exact_match": 0.0, "rag_token_f1": 0.3902, "retrieved_kb_ids": ["RAGKB-0509", "RAGKB-0510", "RAGKB-0374"], "retrieved_sections": ["Master of Computer Science (By Research)", "Master of Computer Science (By Research)", "Master of Computer Science (Applied Computing)"], "retrieved_scopes": ["postgraduate", "postgraduate", "postgraduate"]}
+{"qa_id": "qa_000129", "index_id": "UMI-0129", "query": "When was the Faculty of Computer Science and Information Technology officially formed?", "gold_answer": "The Universiti Malaya Council agreed to the formation of FCSIT on 22 September 1994.", "plain_answer": "<think>\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: History of FCSIT\nSubsection: Formation and Early History\nPages: 10, 11\nKnowledge group: KG-0012\nChunk id: SC-00003\nText:\nformed of 53 students from the 1978/1979 Session. The first undergraduate programme offered by the Department of Computer Centre was the Bachelor of Science in Computer Studies. In 1980, the University Senate agreed to the establishment of the Computer Centre as an", "rag_answer": "<think>\n\n</think>\n\nThe Universiti Malaya Council agreed to the formation of FCSIT on 22 September 1994.", "plain_exact_match": 0.0, "plain_token_f1": 0.1591, "rag_exact_match": 0.0, "rag_token_f1": 0.9333, "retrieved_kb_ids": ["RAGKB-0003", "RAGKB-0004", "RAGKB-0002"], "retrieved_sections": ["History of the Faculty", "History of the Faculty", "History of the Faculty"], "retrieved_scopes": ["general", "general", "general"]}
+{"qa_id": "qa_000308", "index_id": "UMI-0308", "query": "What distinctive specialization focus is highlighted for Bachelor of Computer Science (Artificial Intelligence)?", "gold_answer": "The specialization focus highlighted for Bachelor of Computer Science (Artificial Intelligence) is artificial intelligence, as reflected by the programme title and its learning-outcome emphasis.", "plain_answer": "<think>\n\n</think>\n\nThe Bachelor of Computer Science (Artificial Intelligence) programme emphasizes artificial intelligence as its specialization focus, as reflected by the programme title and focus areas.", "rag_answer": "<think>\n\n</think>\n\nThe specialization focus highlighted for Bachelor of Computer Science (Artificial Intelligence) is artificial intelligence, as reflected by the programme title and its learning-outcome emphasis.", "plain_exact_match": 0.0, "plain_token_f1": 0.7451, "rag_exact_match": 0.0, "rag_token_f1": 0.9615, "retrieved_kb_ids": ["RAGKB-0328", "RAGKB-0330", "RAGKB-0333"], "retrieved_sections": ["Undergraduate Programme Goals and Learning Outcomes", "Undergraduate Programme Goals and Learning Outcomes", "Undergraduate Programme Goals and Learning Outcomes"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000641", "index_id": "UMI-0641", "query": "Could you list the main lecture periods for bachelor degree students in Semester II of session 2025/2026?", "gold_answer": "For bachelor degree students in Semester II, the main lecture periods are 09.03.2026 to 26.04.2026 and 04.05.2026 to 21.06.2026.", "plain_answer": "<think>\n\n</think>\n\nThe main lecture periods for Bachelor's degree students in Semester II 2025/2026 are Week 1 to Week 6 as the first lecture period, and Week 7 to Week 12 as the second lecture period.", "rag_answer": "<think>\n\n</think>\n\nThe handbook shows the academic calendar for Bachelor Degree Level in session 2025/2026. It breaks down the calendar into orientation or semester preparation, teaching periods divided by scheduled breaks, special revision and examination periods, and the final examination block for each semester.", "plain_exact_match": 0.0, "plain_token_f1": 0.4923, "rag_exact_match": 0.0, "rag_token_f1": 0.25, "retrieved_kb_ids": ["RAGKB-0008", "RAGKB-0009", "RAGKB-0006"], "retrieved_sections": ["Academic Calendar 2025/2026", "Academic Calendar 2025/2026", "Academic Calendar 2025/2026"], "retrieved_scopes": ["undergraduate", "undergraduate", "postgraduate"]}
+{"qa_id": "qa_000799", "index_id": "UMI-0799", "query": "Quick one: what graduate or research capabilities are highlighted in the learning objectives and outcomes for Master of Computer Science (By Research)?", "gold_answer": "The learning objectives and outcomes highlight the ability to apply and integrate knowledge of current computer science research issues, evaluate and analyse computing solutions, develop and assess research outcomes, use appropriate research methods to acquire and extend knowledge, communicate and publish technical work effectively, work independently and in groups, and demonstrate ethical and professionally responsible research conduct.", "plain_answer": "<think>\n\n</think>\n\nThe learning objectives and outcomes for Master of Computer Science (By Research) focus on conducting advanced computer science research, producing original research work capable of publication, and contributing to new knowledge in the field.", "rag_answer": "<think>\n\n</think>\n\nThe learning objectives and outcomes highlight the ability to apply and integrate knowledge of current computer science research issues, evaluate and analyse computing solutions, develop and assess research outcomes, use appropriate research methods to acquire and extend knowledge, communicate and publish technical work effectively, work independently and in groups, and demonstrate ethical and professionally responsible research conduct.", "plain_exact_match": 0.0, "plain_token_f1": 0.3656, "rag_exact_match": 0.0, "rag_token_f1": 0.9828, "retrieved_kb_ids": ["RAGKB-0504", "RAGKB-0505", "RAGKB-0506"], "retrieved_sections": ["Master of Computer Science (By Research)", "Master of Computer Science (By Research)", "Master of Computer Science (By Research)"], "retrieved_scopes": ["postgraduate", "postgraduate", "postgraduate"]}
+{"qa_id": "qa_000986", "index_id": "UMI-0986", "query": "Could you tell me what dress expectations are illustrated for official events?", "gold_answer": "For official events, the poster illustrates formal attire, including suit-style clothing and traditional formal wear, to convey a neat and official appearance appropriate for formal university occasions.", "plain_answer": "<think>\n\n[Retrieved Evidence 1]\nSource document: Complete Handbook\nScope: general\nSection: Official Events\nSubsection: Dress Code for Official Events\nPages: 12\nKnowledge group: KG-0009\nChunk id: SC-00035\nText:\nDRESS CODE FOR OFFICIAL EVENTS Whether big or small, formal functions and events that involve the Faculty, University or external guests usually require a presentable formal of dress by participants. This does not mean that everyone has to wear the same suit or the same colour of shirt or the same brand of footwear. What is important is that", "rag_answer": "<think>\n\n</think>\n\nThe poster illustrates formal or traditional formal attire for official university occasions.", "plain_exact_match": 0.0, "plain_token_f1": 0.2393, "rag_exact_match": 0.0, "rag_token_f1": 0.5238, "retrieved_kb_ids": ["RAGKB-0319", "RAGKB-0138", "RAGKB-0071"], "retrieved_sections": ["Student Dress Code", "Research Guidance", "Research Guidance"], "retrieved_scopes": ["general", "postgraduate", "postgraduate"]}
+{"qa_id": "qa_000566", "index_id": "UMI-0566", "query": "Could you tell me what schedule must students follow during industrial training?", "gold_answer": "A week-by-week schedule for students during Industrial Training, including placement approval, logbook submission, faculty supervision checkpoints, and final report submission.", "plain_answer": "<think>\n\n[Retrieved Evidence 1]\nSource document: Complete Handbook\nScope: undergraduate\nSection: Industrial Training\nSubsection: Industrial Training Guidelines\nPages: 270, 271\nKnowledge group: KG-0087\nChunk id: SC-00273\nText:\n5.0 Schedule of Industrial Training Week 1 to 5: 6.0 Week 6 to 10: 7.0 Week 11 to 15: 8.0 Week 16 to 20: 9.0 Week 21 to", "rag_answer": "<think>\n\n</think>\n\nStudents must follow the industrial training schedule set out in the handbook, including the approved placement period, weekly logging requirements, and other administrative timing requirements that apply to the internship component.", "plain_exact_match": 0.0, "plain_token_f1": 0.119, "rag_exact_match": 0.0, "rag_token_f1": 0.2545, "retrieved_kb_ids": ["RAGKB-0285", "RAGKB-0287", "RAGKB-0289"], "retrieved_sections": ["Industrial Training", "Industrial Training", "Industrial Training"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000994", "index_id": "UMI-0994", "query": "Quick check: what consequences or disciplinary actions are implied for examination dishonesty?", "gold_answer": "The consequences are disciplinary in nature. The handbook points students to official Universiti Malaya academic dishonesty, examination misconduct, and student discipline rules, which means dishonest conduct may trigger action under the university's disciplinary framework.", "plain_answer": "<think>\n\n</think>\n\nThe consequences include disciplinary action under the University's regulations for dishonesty in examinations.", "rag_answer": "<think>\n\n</think>\n\nThe consequences are disciplinary in nature. The handbook points students to official Universiti Malaya academic dishonesty, examination misconduct, and student discipline rules, which means dishonest conduct may trigger action under the university's disciplinary framework.", "plain_exact_match": 0.0, "plain_token_f1": 0.3922, "rag_exact_match": 0.0, "rag_token_f1": 0.9722, "retrieved_kb_ids": ["RAGKB-0320", "RAGKB-0321", "RAGKB-0292"], "retrieved_sections": ["Undergraduate Rules and Regulations", "Examination Grading Scheme", "Industrial Training"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000105", "index_id": "UMI-0105", "query": "What is the process flow for Academic Project II?", "gold_answer": "The handbook presents Academic Project as a staged process covering topic or title approval, supervision, progress monitoring, submission through the ilmiah system, viva or evaluation sessions, and final report completion across Academic Project I and Academic Project II.", "plain_answer": "<think>\n\n[Retrieved Evidence 1]\nSource document: Complete Handbook\nScope: postgraduate\nSection: Research Guidance\nSubsection: Process Flow of Academic Project I and Academic Project II\nPages: 12\nKnowledge group: KG-0085\nChunk id: SC-00059\nText:\nFigure 1 shows the process flow of Academic Project I and Academic Project II for the selected topics. The major activities under the supervision component include the supervisor’s meetings with the student, verification of the student’s progress, and the supervisor’s recommendations to the Academic Project Committee if the student is not meeting the", "rag_answer": "<think>\n\n</think>\n\nThe handbook presents Academic Project as a staged process covering topic or title approval, supervision, progress monitoring, submission through the ilmiah system, viva or evaluation sessions, and final report completion across Academic Project I and Academic Project II.", "plain_exact_match": 0.0, "plain_token_f1": 0.2481, "rag_exact_match": 0.0, "rag_token_f1": 0.9744, "retrieved_kb_ids": ["RAGKB-0301", "RAGKB-0296", "RAGKB-0297"], "retrieved_sections": ["Academic Project", "Academic Project", "Academic Project"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000028", "index_id": "UMI-0028", "query": "What is the course plan for Master of Software Engineering (Software Technology)?", "gold_answer": "The course plan organises the programme across multiple semesters for each intake. The core set includes Research Methodology, Architecting Software Systems, Framework-Based Software Design and Development, Software Verification and Validation, Software Project Management, and Dissertation. Students then select two electives such as Big Data Processing, Requirements Engineering, User Experience Design Studio, Advanced Internet of Things, Advanced Machine Learning, or Security Risk Analysis and Evaluation. The handbook notes that Research Methodology must be taken in the first semester, and dissertation registration requires at least 6 credits of passed coursework together with having taken or being enrolled in Research Methodology.", "plain_answer": "<think>\n\n[Retrieved Evidence 1]\nSource document: Course Plan for Master of Software Engineering (Software Technology)\nScope: postgraduate\nSection: Course Plan\nSubsection: Course Plan\nPages: 11\nKnowledge group: KG-0065\nChunk id: SC-00010\nText:\nLampiran B2 COURSE PLAN MASTERS OF SOFTWARE ENGINEERING SOFTWARE TECHNOLOGY ‘Semester | ‘Semester Il ‘Semester I (Marko?) COURSE CODE fean xx x xf 3001 | Research Methodology vv vv vv 3", "rag_answer": "<think>\n\n</think>\n\nThe course plan organises the programme across multiple semesters for each intake. The core set includes Research Methodology, Architecting Software Systems, Framework-Based Software Design and Development, Software Verification and Validation, Software Project Management, and Dissertation. Students then select two electives such as Big Data Processing, Requirements Engineering, User Experience Design Studio, Advanced Internet of Things, Advanced Machine Learning, or Security Risk Analysis and Evaluation. The handbook notes that Research Methodology must be taken in the first semester, and dissertation registration requires at least 6 credits of passed coursework together with having taken or being enrolled in Research Methodology.", "plain_exact_match": 0.0, "plain_token_f1": 0.1605, "rag_exact_match": 0.0, "rag_token_f1": 0.99, "retrieved_kb_ids": ["RAGKB-0404", "RAGKB-0402", "RAGKB-0403"], "retrieved_sections": ["Master of Software Engineering (Software Technology)", "Master of Software Engineering (Software Technology)", "Master of Software Engineering (Software Technology)"], "retrieved_scopes": ["postgraduate", "postgraduate", "postgraduate"]}
+{"qa_id": "qa_000972", "index_id": "UMI-0972", "query": "Could you tell me what conditions or achievements make a project eligible for the Excellence Academic Project Award (APAC)?", "gold_answer": "The Excellence Academic Project Award (APAC), introduced in 2012, is an annual event that motivates students to produce excellent projects, recognizes strong systems, identifies good projects for reference and exhibition, and promotes them for competitions, exhibitions, and intellectual property purposes.", "plain_answer": "<think>\n\n</think>\n\nThe Excellence Academic Project Award (APAC) is given to projects that are evaluated as outstanding by the faculty's academic project assessment panel and subsequently approved for the award.", "rag_answer": "<think>\n\n</think>\n\nThe Excellence Academic Project Award (APAC), introduced in 2012, is an annual event that motivates students to produce excellent projects, recognizes strong systems, identifies good projects for reference and exhibition, and promotes them for competitions, exhibitions, and intellectual property purposes.", "plain_exact_match": 0.0, "plain_token_f1": 0.338, "rag_exact_match": 0.0, "rag_token_f1": 0.9756, "retrieved_kb_ids": ["RAGKB-0308", "RAGKB-0297", "RAGKB-0296"], "retrieved_sections": ["Academic Project", "Academic Project", "Academic Project"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000170", "index_id": "UMI-0170", "query": "What facilities and research focus are described for the Robotedge AI Robotic Lab?", "gold_answer": "The Robotedge AI Robotic Lab, previously known as the Natural Language Processing Lab, is equipped for AI robotics research and development, especially in environmental, home services, and search-and-rescue areas.", "plain_answer": "<think>\n\n</think>\n\nThe Robotedge AI Robotic Lab focuses on AI-powered robotics research and experimentation, with a research theme that combines artificial intelligence with robotic systems.", "rag_answer": "<think>\n\n</think>\n\nThe Robotedge AI Robotic Lab, previously known as the Natural Language Processing Lab, is equipped for AI robotics research and development, especially in environmental, home services, and search-and-rescue areas.", "plain_exact_match": 0.0, "plain_token_f1": 0.3158, "rag_exact_match": 0.0, "rag_token_f1": 0.9688, "retrieved_kb_ids": ["RAGKB-0017", "RAGKB-0019", "RAGKB-0016"], "retrieved_sections": ["Teaching and Learning Facilities", "Teaching and Learning Facilities", "Teaching and Learning Facilities"], "retrieved_scopes": ["general", "general", "general"]}
+{"qa_id": "qa_000914", "index_id": "UMI-0914", "query": "Could you explain how should students interpret or use the programme core courses list when planning their studies?", "gold_answer": "Students should use the programme core courses list as compulsory study-planning guidance: they are expected to pass all Faculty and Programme Core courses and to check the stated course prerequisites, with Industrial Training treated separately in the prerequisite note.", "plain_answer": "<think>\n\n</think>\n\nStudents should use the programme core courses list as their required study path; together with the handbook's course description and curriculum table, it shows what every student in the programme must study.", "rag_answer": "<think>\n\n</think>\n\nStudents should use the programme core courses list as compulsory study-planning guidance: they are expected to pass all Faculty and Programme Core courses and to check the stated course prerequisites, with Industrial Training treated separately in the prerequisite note.", "plain_exact_match": 0.0, "plain_token_f1": 0.4533, "rag_exact_match": 0.0, "rag_token_f1": 0.9756, "retrieved_kb_ids": ["RAGKB-0185", "RAGKB-0186", "RAGKB-0184"], "retrieved_sections": ["Shared Undergraduate Curriculum", "Shared Undergraduate Curriculum", "Shared Undergraduate Curriculum"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000651", "index_id": "UMI-0651", "query": "Could you tell me what facilities and access arrangements are described for the Postgraduate Lab (ML)?", "gold_answer": "The Postgraduate Lab has 33 computer units connected to Windows Active Directory servers and the Internet, uses Windows 10, and is open to all FSKTM postgraduate students.", "plain_answer": "<think>\n\n</think>\n\nThe Postgraduate Lab (ML) at FCSIT, Universiti Malaya is listed under lab facility category and is intended for postgraduate use. The document states that students must be wear name tags and cordial when using the facility, and that data ownership follows university and lab regulations.", "rag_answer": "<think>\n\n</think>\n\nThe Postgraduate Lab has 33 computer units connected to Windows Active Directory servers and the Internet, uses Windows 10, and is open to all FSKTM postgraduate students.", "plain_exact_match": 0.0, "plain_token_f1": 0.2432, "rag_exact_match": 0.0, "rag_token_f1": 0.9643, "retrieved_kb_ids": ["RAGKB-0010", "RAGKB-0012", "RAGKB-0011"], "retrieved_sections": ["Teaching and Learning Facilities", "Teaching and Learning Facilities", "Teaching and Learning Facilities"], "retrieved_scopes": ["general", "general", "general"]}
+{"qa_id": "qa_000292", "index_id": "UMI-0292", "query": "What rules apply to saving data, using personal devices, or protecting laboratory resources?", "gold_answer": "The laboratory regulations prohibit misuse of laboratory facilities, disruptive or improper behaviour, and conduct that may damage equipment, compromise safety, or interfere with other users.", "plain_answer": "<think>\n\n</think>\n\nThe handbook emphasizes proper saving, guarding, and protecting laboratory resources at all times.", "rag_answer": "<think>\n\n</think>\n\nThe laboratory regulations prohibit misuse of laboratory facilities, disruptive or improper behaviour, and conduct that may damage equipment, compromise safety, or interfere with other users.", "plain_exact_match": 0.0, "plain_token_f1": 0.15, "rag_exact_match": 0.0, "rag_token_f1": 0.9615, "retrieved_kb_ids": ["RAGKB-0139", "RAGKB-0140", "RAGKB-0141"], "retrieved_sections": ["Laboratory Regulations and Support", "Laboratory Regulations and Support", "Laboratory Regulations and Support"], "retrieved_scopes": ["general", "general", "general"]}
+{"qa_id": "qa_000525", "index_id": "UMI-0525", "query": "Could you tell me what curriculum structure or course planning information is shown for Bachelor of Computer Science (Computer System and Network)?", "gold_answer": "The curriculum structure semester by semester, showing how university courses, faculty core courses, programme core courses, specialization electives, industrial training, and the academic project fit into the degree plan.", "plain_answer": "<think>\n\n</think>\n\nThe Bachelor of Computer Science (Computer System and Network) curriculum is illustrated as a 8-semester structure. It contains approximately 53 credits of programme core courses, 39 credits of specialization electives, and 14 credits of other specialization courses, with the remaining space made up by university courses, practical components, and the academic project.", "rag_answer": "<think>\n\n</think>\n\nThe curriculum structure semester by semester, showing how university courses, faculty core courses, programme core courses, specialization electives, industrial training, and the academic project fit into the degree plan.", "plain_exact_match": 0.0, "plain_token_f1": 0.4286, "rag_exact_match": 0.0, "rag_token_f1": 0.9667, "retrieved_kb_ids": ["RAGKB-0324", "RAGKB-0322", "RAGKB-0325"], "retrieved_sections": ["Undergraduate Programme Goals and Learning Outcomes", "Undergraduate Programme Goals and Learning Outcomes", "Undergraduate Programme Goals and Learning Outcomes"], "retrieved_scopes": ["undergraduate", "undergraduate", "undergraduate"]}
+{"qa_id": "qa_000673", "index_id": "UMI-0673", "query": "Hey, what facilities and access arrangements are described for the Knowledge Engineering Lab?", "gold_answer": "The Knowledge Engineering Lab is an open-space concept lab at Wisma R&D that is accessible to all postgraduate students, regardless of field, with permission for access.", "plain_answer": "<think>\n\n</think>\n\nThe Knowledge Engineering Lab provides computer lab facilities and makes equipment access arrangements through the lab management system for its students and authorized users.", "rag_answer": "<think>\n\n</think>\n\nThe Knowledge Engineering Lab is an open-space concept lab at Wisma R&D that is accessible to all postgraduate students, regardless of field, with permission for access.", "plain_exact_match": 0.0, "plain_token_f1": 0.2963, "rag_exact_match": 0.0, "rag_token_f1": 0.9655, "retrieved_kb_ids": ["RAGKB-0016", "RAGKB-0017", "RAGKB-0015"], "retrieved_sections": ["Teaching and Learning Facilities", "Teaching and Learning Facilities", "Teaching and Learning Facilities"], "retrieved_scopes": ["general", "general", "general"]}

UM_Handbook/outputs/baseline2_rag_harness_agent/generation_eval/generation_metrics.json

@@ -0,0 +1,10 @@
+{
+  "generation_eval_size": 20,
+  "top_k": 3,
+  "plain_exact_match": 0.0,
+  "plain_token_f1": 0.3391,
+  "rag_exact_match": 0.0,
+  "rag_token_f1": 0.846,
+  "rag_minus_plain_exact_match": 0.0,
+  "rag_minus_plain_token_f1": 0.5069
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/README.md

@@ -0,0 +1,209 @@
+---
+base_model: /scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54282e9eff87e798234f8fa408e6caf2409416eed4516465894c29c08265a18e
+size 87360584

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/lora_adapter/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20789f4b467b5ff6e691e6d696bc04daebaaa023439a4918e4927d5d9b849aee
+size 5304

UM_Handbook/outputs/baseline2_rag_harness_agent/rag_augmented_dataset/rag_augmented_sft_dataset_preview.json

@@ -0,0 +1,212 @@
+[
+  {
+    "qa_id": "qa_000001",
+    "index_id": "UMI-0001",
+    "question": "What are the faculty objectives?",
+    "answer": "The faculty objectives are to sustain excellence in undergraduate and postgraduate teaching, learning, and research; contribute to national development through quality research and publications; provide innovative academic programmes that respond to societal needs; and produce quality graduates with advanced knowledge and skills in computer science and information technology.",
+    "scope_label": "general",
+    "source_doc": "General Handbook",
+    "section": "Faculty Objectives",
+    "subsection": "Faculty Objectives",
+    "retrieved_kb_ids": [
+      "RAGKB-0001",
+      "RAGKB-0515",
+      "RAGKB-0504"
+    ],
+    "retrieved_sections": [
+      "Faculty Objectives",
+      "Doctor of Philosophy",
+      "Master of Computer Science (By Research)"
+    ],
+    "retrieved_scopes": [
+      "general",
+      "postgraduate",
+      "postgraduate"
+    ],
+    "prompt": [
+      {
+        "role": "system",
+        "content": "You are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly."
+      },
+      {
+        "role": "user",
+        "content": "Use the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: Faculty Objectives\nSubsection: Faculty Objectives\nPages: 9\nKnowledge group: KG-0009\nChunk id: SC-00001\nText:\nVISION A global faculty impacting the world MISSION Propelling computing technology and producing world class leaders OBJECTIVES To sustain an outstanding faculty dedicated to excellence in undergraduate and postgraduate teaching, learning and research. To contribute towards the development of the nation through the production of quality research and publications. To provide innovative academic programs that can respond to the changing needs of the society. To produce quality graduates who are equipped with advanced knowledge and skills of computer science and information technology.\n\n[Retrieved Evidence 2]\nSource document: Complete Handbook\nScope: postgraduate\nSection: Doctor of Philosophy\nSubsection: Programme Education Objectives\nPages: 121\nKnowledge group: KG-0021\nChunk id: SC-00515\nText:\nPROGRAMME EDUCATION OBJETIVES oe PEO 1: Foster innovation of new ideas, methods and techniques in relevant research fields PEO 2: Lead research and establish a career as a skilled researcher and/or practitioner PEO 3. Disseminate research output and provide expert advice through various mechanisms in an ethical and professional manner\n\n[Retrieved Evidence 3]\nSource document: Complete Handbook\nScope: postgraduate\nSection: Master of Computer Science (By Research)\nSubsection: Learning Objectives and Outcomes\nPages: 114\nKnowledge group: KG-0039\nChunk id: SC-00504\nText:\nPROGRAMME LEARNING OBJECTIVES & OUTCOMES SS PROGRAMME EDUCATIONAL OBJECTIVES (PEO) 4. To produce researchers who can contribute to the development and knowledge of computer science 2. To produce professionals in computer science research 3. To equip graduates with technical and soft skills PROGRAMME LEARNING OUTCOMES (PLO) Taxonomic No. Programme Learning Outcomes PLOs Gubecte 1. [Apply and integrate knowledge on the latest research issues |p) Gy 3S in computer science and produce state-of-the-art research. 2. [Evaluate and analyse computing solutions in terms of re Pp usability, efficiency and effectiveness 3. [Produce computing solutions and use appropriate tools to lanalyse the performanc ...[truncated]\n\nQuestion: What are the faculty objectives?"
+      }
+    ],
+    "completion": [
+      {
+        "role": "assistant",
+        "content": "The faculty objectives are to sustain excellence in undergraduate and postgraduate teaching, learning, and research; contribute to national development through quality research and publications; provide innovative academic programmes that respond to societal needs; and produce quality graduates with advanced knowledge and skills in computer science and information technology."
+      }
+    ],
+    "text": "<|im_start|>system\nYou are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly.<|im_end|>\n<|im_start|>user\nUse the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: Faculty Objectives\nSubsection: Faculty Objectives\nPages: 9\nKnowledge group: KG-0009\nChunk id: SC-00001\nText:\nVISION A global faculty impacting the world MISSION Propelling computing technology and producing world class leaders OBJECTIVES To sustain an outstanding faculty dedicated to excellence in undergraduate and postgraduate teaching, learning and research. To contribute towards the development of the nation through the production of quality research and publications. To provide innovative academic programs that can respond to the changing needs of the society. To produce quality graduates who are equipped with advanced knowledge and skills of computer science and information technology.\n\n[Retrieved Evidence 2]\nSource document: Complete Handbook\nScope: postgraduate\nSection: Doctor of Philosophy\nSubsection: Programme Education Objectives\nPages: 121\nKnowledge group: KG-0021\nChunk id: SC-00515\nText:\nPROGRAMME EDUCATION OBJETIVES oe PEO 1: Foster innovation of new ideas, methods and techniques in relevant research fields PEO 2: Lead research and establish a career as a skilled researcher and/or practitioner PEO 3. Disseminate research output and provide expert advice through various mechanisms in an ethical and professional manner\n\n[Retrieved Evidence 3]\nSource document: Complete Handbook\nScope: postgraduate\nSection: Master of Computer Science (By Research)\nSubsection: Learning Objectives and Outcomes\nPages: 114\nKnowledge group: KG-0039\nChunk id: SC-00504\nText:\nPROGRAMME LEARNING OBJECTIVES & OUTCOMES SS PROGRAMME EDUCATIONAL OBJECTIVES (PEO) 4. To produce researchers who can contribute to the development and knowledge of computer science 2. To produce professionals in computer science research 3. To equip graduates with technical and soft skills PROGRAMME LEARNING OUTCOMES (PLO) Taxonomic No. Programme Learning Outcomes PLOs Gubecte 1. [Apply and integrate knowledge on the latest research issues |p) Gy 3S in computer science and produce state-of-the-art research. 2. [Evaluate and analyse computing solutions in terms of re Pp usability, efficiency and effectiveness 3. [Produce computing solutions and use appropriate tools to lanalyse the performanc ...[truncated]\n\nQuestion: What are the faculty objectives?<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nThe faculty objectives are to sustain excellence in undergraduate and postgraduate teaching, learning, and research; contribute to national development through quality research and publications; provide innovative academic programmes that respond to societal needs; and produce quality graduates with advanced knowledge and skills in computer science and information technology.<|im_end|>\n"
+  },
+  {
+    "qa_id": "qa_000002",
+    "index_id": "UMI-0002",
+    "question": "What is the history overview of the Faculty of Computer Science and Information Technology?",
+    "answer": "Universiti Malaya's computing facilities began with the Computer Centre in the 1960s, later expanded into teaching and research, introduced the Diploma in Computer Science in 1974 and research degrees in 1985, then moved toward a full faculty structure in 1993. FCSIT was formally approved in 1994, its main building was opened in 1996, the Bachelor of Information Technology began in 1996/97, and four departments were established in 1997.",
+    "scope_label": "general",
+    "source_doc": "General Handbook",
+    "section": "History of the Faculty",
+    "subsection": "History Overview",
+    "retrieved_kb_ids": [
+      "RAGKB-0004",
+      "RAGKB-0003",
+      "RAGKB-0002"
+    ],
+    "retrieved_sections": [
+      "History of the Faculty",
+      "History of the Faculty",
+      "History of the Faculty"
+    ],
+    "retrieved_scopes": [
+      "general",
+      "general",
+      "general"
+    ],
+    "prompt": [
+      {
+        "role": "system",
+        "content": "You are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly."
+      },
+      {
+        "role": "user",
+        "content": "Use the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 11\nKnowledge group: KG-0011\nChunk id: SC-00004\nText:\nThe Bachelor of Information Technology programme started in the 1996/1997 Session, with an initial intake of 50 students. To accommodate an increase student population, an additional building was built which was officially opened by Datuk Fong Chan Onn, Deputy Minister of Education on September 21st, 1998. Since its establishment, the Faculty of Computer Science and Information Technology have been led by a number of distinguished persons. The following have served as Directors/Dean: HISTORY OF THE FACULTY 1967 – 1973 1973 – 1975 1975 – 1978 1978 – 1982 1982 – 1990 1990 – 1992 1992 – 2000 2000 – 2002 2002 – 2004 2004 – 2005 2005 – 2006 2006 – 2007 2007 – 2009 2009 – 2010 2010 – 2011 2011 – 2 ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00003\nText:\nprogramme. The first undergraduate enrolment for the 1990/1991 Session was 50 students. In April 1st, 1993, the University Senate agreed to the formation of the Computer Centre Study Board. The Board proposed the establishment of a faculty to be called the Faculty of Computer Science and Information Technology (FCSIT). The existing Computer Centre was to be annulled and replaced by a Computer Services Division which was placed under the Chancellery. On September 22nd, 1994, the University of Malaya Council agreed to the formation of the Faculty of Computer Science and Information Technology (FCSIT), and the Computer Services Division. A sum of 4.2 million was obtained from the Ministry of Ed ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00002\nText:\nThe provision of computer facilities and services at the Universiti Malaya (UM) began soon after the Computer Centre was officially formed in 1965. This made the university one of the pioneers in computer usage in Malaysia. In December 1969, the Computer Centre took on an additional role of teaching and research of computer science and information technology. The Computer Centre Board was formed, comprising the Vice-Chancellor (as Chairman), the Director of Computer Centre (as Secretary), and a representative from each Faculty, Institute, Centre and the University Senate. In 1974, the Diploma in Computer Science programme was introduced. From its inception in the 1974/1975 Session to the 199 ...[truncated]\n\nQuestion: What is the history overview of the Faculty of Computer Science and Information Technology?"
+      }
+    ],
+    "completion": [
+      {
+        "role": "assistant",
+        "content": "Universiti Malaya's computing facilities began with the Computer Centre in the 1960s, later expanded into teaching and research, introduced the Diploma in Computer Science in 1974 and research degrees in 1985, then moved toward a full faculty structure in 1993. FCSIT was formally approved in 1994, its main building was opened in 1996, the Bachelor of Information Technology began in 1996/97, and four departments were established in 1997."
+      }
+    ],
+    "text": "<|im_start|>system\nYou are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly.<|im_end|>\n<|im_start|>user\nUse the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 11\nKnowledge group: KG-0011\nChunk id: SC-00004\nText:\nThe Bachelor of Information Technology programme started in the 1996/1997 Session, with an initial intake of 50 students. To accommodate an increase student population, an additional building was built which was officially opened by Datuk Fong Chan Onn, Deputy Minister of Education on September 21st, 1998. Since its establishment, the Faculty of Computer Science and Information Technology have been led by a number of distinguished persons. The following have served as Directors/Dean: HISTORY OF THE FACULTY 1967 – 1973 1973 – 1975 1975 – 1978 1978 – 1982 1982 – 1990 1990 – 1992 1992 – 2000 2000 – 2002 2002 – 2004 2004 – 2005 2005 – 2006 2006 – 2007 2007 – 2009 2009 – 2010 2010 – 2011 2011 – 2 ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00003\nText:\nprogramme. The first undergraduate enrolment for the 1990/1991 Session was 50 students. In April 1st, 1993, the University Senate agreed to the formation of the Computer Centre Study Board. The Board proposed the establishment of a faculty to be called the Faculty of Computer Science and Information Technology (FCSIT). The existing Computer Centre was to be annulled and replaced by a Computer Services Division which was placed under the Chancellery. On September 22nd, 1994, the University of Malaya Council agreed to the formation of the Faculty of Computer Science and Information Technology (FCSIT), and the Computer Services Division. A sum of 4.2 million was obtained from the Ministry of Ed ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00002\nText:\nThe provision of computer facilities and services at the Universiti Malaya (UM) began soon after the Computer Centre was officially formed in 1965. This made the university one of the pioneers in computer usage in Malaysia. In December 1969, the Computer Centre took on an additional role of teaching and research of computer science and information technology. The Computer Centre Board was formed, comprising the Vice-Chancellor (as Chairman), the Director of Computer Centre (as Secretary), and a representative from each Faculty, Institute, Centre and the University Senate. In 1974, the Diploma in Computer Science programme was introduced. From its inception in the 1974/1975 Session to the 199 ...[truncated]\n\nQuestion: What is the history overview of the Faculty of Computer Science and Information Technology?<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nUniversiti Malaya's computing facilities began with the Computer Centre in the 1960s, later expanded into teaching and research, introduced the Diploma in Computer Science in 1974 and research degrees in 1985, then moved toward a full faculty structure in 1993. FCSIT was formally approved in 1994, its main building was opened in 1996, the Bachelor of Information Technology began in 1996/97, and four departments were established in 1997.<|im_end|>\n"
+  },
+  {
+    "qa_id": "qa_000003",
+    "index_id": "UMI-0003",
+    "question": "What key milestones are mentioned in the faculty history?",
+    "answer": "Key milestones include the formation of the Computer Centre, the start of teaching and research in 1969, the introduction of the Diploma in Computer Science in 1974, approval and administration of the Master of Computer Science and PhD research programmes in 1985, the 1993 proposal to establish FCSIT, formal approval in 1994, the opening of the faculty building in 1996, the start of the Bachelor of Information Technology in 1996/97, and the establishment of four departments in 1997.",
+    "scope_label": "general",
+    "source_doc": "General Handbook",
+    "section": "History of the Faculty",
+    "subsection": "History Overview",
+    "retrieved_kb_ids": [
+      "RAGKB-0002",
+      "RAGKB-0003",
+      "RAGKB-0005"
+    ],
+    "retrieved_sections": [
+      "History of the Faculty",
+      "History of the Faculty",
+      "History of the Faculty"
+    ],
+    "retrieved_scopes": [
+      "general",
+      "general",
+      "general"
+    ],
+    "prompt": [
+      {
+        "role": "system",
+        "content": "You are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly."
+      },
+      {
+        "role": "user",
+        "content": "Use the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00002\nText:\nThe provision of computer facilities and services at the Universiti Malaya (UM) began soon after the Computer Centre was officially formed in 1965. This made the university one of the pioneers in computer usage in Malaysia. In December 1969, the Computer Centre took on an additional role of teaching and research of computer science and information technology. The Computer Centre Board was formed, comprising the Vice-Chancellor (as Chairman), the Director of Computer Centre (as Secretary), and a representative from each Faculty, Institute, Centre and the University Senate. In 1974, the Diploma in Computer Science programme was introduced. From its inception in the 1974/1975 Session to the 199 ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00003\nText:\nprogramme. The first undergraduate enrolment for the 1990/1991 Session was 50 students. In April 1st, 1993, the University Senate agreed to the formation of the Computer Centre Study Board. The Board proposed the establishment of a faculty to be called the Faculty of Computer Science and Information Technology (FCSIT). The existing Computer Centre was to be annulled and replaced by a Computer Services Division which was placed under the Chancellery. On September 22nd, 1994, the University of Malaya Council agreed to the formation of the Faculty of Computer Science and Information Technology (FCSIT), and the Computer Services Division. A sum of 4.2 million was obtained from the Ministry of Ed ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 11\nKnowledge group: KG-0011\nChunk id: SC-00005\nText:\nProfessor Dato' Dr. Ir. Mashkuri Hj. Yaacob Professor Dr. Mohd Sapiyan Baba Professor Dr. David Ngo Chek Ling Professor Dr. Wan Ahmad Tajuddin Wan Abdullah Professor Dr. Siti Salwah Salim Professor Dr. Abdullah Gani Professor Dr. Abrizah Abdullah Professor Datin Dr. Sameem Abdul Kareem Professor Dr. Loo Chu Kiong Professor Ir. Dr. Chan Chee Seng Associate Professor Dr. Norisma Idris\n\nQuestion: What key milestones are mentioned in the faculty history?"
+      }
+    ],
+    "completion": [
+      {
+        "role": "assistant",
+        "content": "Key milestones include the formation of the Computer Centre, the start of teaching and research in 1969, the introduction of the Diploma in Computer Science in 1974, approval and administration of the Master of Computer Science and PhD research programmes in 1985, the 1993 proposal to establish FCSIT, formal approval in 1994, the opening of the faculty building in 1996, the start of the Bachelor of Information Technology in 1996/97, and the establishment of four departments in 1997."
+      }
+    ],
+    "text": "<|im_start|>system\nYou are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly.<|im_end|>\n<|im_start|>user\nUse the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00002\nText:\nThe provision of computer facilities and services at the Universiti Malaya (UM) began soon after the Computer Centre was officially formed in 1965. This made the university one of the pioneers in computer usage in Malaysia. In December 1969, the Computer Centre took on an additional role of teaching and research of computer science and information technology. The Computer Centre Board was formed, comprising the Vice-Chancellor (as Chairman), the Director of Computer Centre (as Secretary), and a representative from each Faculty, Institute, Centre and the University Senate. In 1974, the Diploma in Computer Science programme was introduced. From its inception in the 1974/1975 Session to the 199 ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 10\nKnowledge group: KG-0011\nChunk id: SC-00003\nText:\nprogramme. The first undergraduate enrolment for the 1990/1991 Session was 50 students. In April 1st, 1993, the University Senate agreed to the formation of the Computer Centre Study Board. The Board proposed the establishment of a faculty to be called the Faculty of Computer Science and Information Technology (FCSIT). The existing Computer Centre was to be annulled and replaced by a Computer Services Division which was placed under the Chancellery. On September 22nd, 1994, the University of Malaya Council agreed to the formation of the Faculty of Computer Science and Information Technology (FCSIT), and the Computer Services Division. A sum of 4.2 million was obtained from the Ministry of Ed ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: general\nSection: History of the Faculty\nSubsection: History Overview\nPages: 11\nKnowledge group: KG-0011\nChunk id: SC-00005\nText:\nProfessor Dato' Dr. Ir. Mashkuri Hj. Yaacob Professor Dr. Mohd Sapiyan Baba Professor Dr. David Ngo Chek Ling Professor Dr. Wan Ahmad Tajuddin Wan Abdullah Professor Dr. Siti Salwah Salim Professor Dr. Abdullah Gani Professor Dr. Abrizah Abdullah Professor Datin Dr. Sameem Abdul Kareem Professor Dr. Loo Chu Kiong Professor Ir. Dr. Chan Chee Seng Associate Professor Dr. Norisma Idris\n\nQuestion: What key milestones are mentioned in the faculty history?<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nKey milestones include the formation of the Computer Centre, the start of teaching and research in 1969, the introduction of the Diploma in Computer Science in 1974, approval and administration of the Master of Computer Science and PhD research programmes in 1985, the 1993 proposal to establish FCSIT, formal approval in 1994, the opening of the faculty building in 1996, the start of the Bachelor of Information Technology in 1996/97, and the establishment of four departments in 1997.<|im_end|>\n"
+  },
+  {
+    "qa_id": "qa_000004",
+    "index_id": "UMI-0004",
+    "question": "What is the academic calendar for master and doctorate level in session 2025/2026?",
+    "answer": "The 2025/2026 master and doctorate academic calendar with Semester I, Semester II, and a special semester, covering orientation, lecture periods, mid-semester breaks, revision weeks, final examinations, and semester breaks. Semester I: Orientation (WOW) 05.10.2025-12.10.2025; lectures 13.10.2025-23.11.2025 and 01.12.2025-25.01.2026; mid-semester break 24.11.2025-30.11.2025; revision week 26.01.2026-01.02.2026; final examination 02.02.2026-22.02.2026; semester break 23.02.2026-08.03.2026. Semester II: lectures 09.03.2026-26.04.2026 and 04.05.2026-21.06.2026; break 27.04.2026-03.05.2026; revision week 22.06.2026-28.06.2026; final examination 29.06.2026-19.07.2026; semester break 20.07.2026-16.08.2026. Special semester: lectures 27.07.2026-13.09.2026; final examination 14.09.2026-20.09.2026; break 21.09.2026-28.09.2026. Public holidays listed include Deepavali, Christmas, New Year, Thaipusam, Federal Territory Day, Chinese New Year, Nuzul Al-Quran, Eidul Fitri, Labour Day, Eidul Adha, Wesak Day, His Majesty the King's Birthday, Awal Muharram, Prophet Muhammad's Birthday, National Day, and Malaysia Day. The notes state that the module registration and examination schedule can be referred to in the UM system guide, the calendar is subject to change, and UM PG iFest 2026 is scheduled for 09-11 June 2026.",
+    "scope_label": "postgraduate",
+    "source_doc": "General Handbook",
+    "section": "Academic Calendar 2025/2026",
+    "subsection": "Master and Doctorate Level Academic Calendar",
+    "retrieved_kb_ids": [
+      "RAGKB-0006",
+      "RAGKB-0007",
+      "RAGKB-0009"
+    ],
+    "retrieved_sections": [
+      "Academic Calendar 2025/2026",
+      "Academic Calendar 2025/2026",
+      "Academic Calendar 2025/2026"
+    ],
+    "retrieved_scopes": [
+      "postgraduate",
+      "postgraduate",
+      "undergraduate"
+    ],
+    "prompt": [
+      {
+        "role": "system",
+        "content": "You are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly."
+      },
+      {
+        "role": "user",
+        "content": "Use the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: postgraduate\nSection: Academic Calendar 2025/2026\nSubsection: Master and Doctorate Level Academic Calendar\nPages: 4\nKnowledge group: KG-0064\nChunk id: SC-00006\nText:\nLampiran B2 ACADEMIC CALENDAR 2025/2026 ACADEMIC SESSION (MASTER AND DOCTORATE LEVEL) AMENDMENT ‘SEMESTER | ‘Orientation (Week of Welcome)-WOW 1 week 05.10.2025 = «= 12.10.2025 Lectures, 6 weeks\" 19.10.2028 - 23.11.2025 Mid Semester | Break 1 week 24.11.2025 = © 30.11.2025 Lectures. 8 weeks\" 01.12.2025 = © 25.01.2026 Revision Week 1 weeks\" 26.01.2028 © - 01.02.2026 ‘Semester | Final Examination 3 weeks\" 02.02.2026 - © 2202.2026 ‘Semester | Break 2 week 23.02.2028 - 08.03.2026 22 weeks ‘SEMESTER I Lectures: 7 weeks\" 09.03.2026 - 26.04.2026 ‘Mid Semester II Break 1 week 27.04.2028 © - 03.05.2026 Lectures, 7 weeks* 04052026 - 21.06.2026 Revision Week 1 week* 22.06.2026 - 28.06.2026 ‘Semester II ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: postgraduate\nSection: Academic Calendar 2025/2026\nSubsection: Master and Doctorate Level Academic Calendar\nPages: 4\nKnowledge group: KG-0064\nChunk id: SC-00007\nText:\n(1) The Module Registration and Examination Schedule can be referred to at tips /umsitsquide um edu my. All, information is subject to change. (0) The Academic Calendar has taken into account public and festive holidays and is subject to change Deepavali 20 October 2025 (Monday) CChvistnas Day 2 December 2025 (Thursday) New Year 04 Janvary 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertitory Day 01 February 2026 (Sunday) ‘Chinese New Year 17 & 18 Fetruary 2026 (Tuesday & Wednesday) Nuzul Al-Quran, 07 March 2026 (Saturday) Edu Fn 20.21 March 2026 (Friday & Saturday) Labour Day 1 May 2026 (Friday) idl Adhal 27 May 2026 (Wednesday) Wesak Day 34 May 2026 (Sunday) His Majesty th ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: undergraduate\nSection: Academic Calendar 2025/2026\nSubsection: Bachelor Degree Level Academic Calendar\nPages: 12\nKnowledge group: KG-0087\nChunk id: SC-00009\nText:\n(1) The Module Registration and Examination Schedule can be referred to at https://umsitsquide um edumy, All information is subject to change. (() The Academic Calendar has taken into account public and festive holidays and is subject to change’ Deepavali 20 October 2028 (Monday) ‘Christmas Day 25 December 2025 (Thursday) New Year 01 January 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertory Day (01 February 2026 (Sunday) Chinese New Year 17 & 18 February 2026 (Tuesday & Wednesday) ‘Nuzul Al-Quran 07 March 2026 (Saturday) Eidul Ft 20 & 21 March 2026 (Friday & Saturday) Labour Day, 01 May 2026 (Friday) Eidul Adna (27 May 2026 (Wednesday) Wesak Day 31 May 2026 (Sunday) His Ma ...[truncated]\n\nQuestion: What is the academic calendar for master and doctorate level in session 2025/2026?"
+      }
+    ],
+    "completion": [
+      {
+        "role": "assistant",
+        "content": "The 2025/2026 master and doctorate academic calendar with Semester I, Semester II, and a special semester, covering orientation, lecture periods, mid-semester breaks, revision weeks, final examinations, and semester breaks. Semester I: Orientation (WOW) 05.10.2025-12.10.2025; lectures 13.10.2025-23.11.2025 and 01.12.2025-25.01.2026; mid-semester break 24.11.2025-30.11.2025; revision week 26.01.2026-01.02.2026; final examination 02.02.2026-22.02.2026; semester break 23.02.2026-08.03.2026. Semester II: lectures 09.03.2026-26.04.2026 and 04.05.2026-21.06.2026; break 27.04.2026-03.05.2026; revision week 22.06.2026-28.06.2026; final examination 29.06.2026-19.07.2026; semester break 20.07.2026-16.08.2026. Special semester: lectures 27.07.2026-13.09.2026; final examination 14.09.2026-20.09.2026; break 21.09.2026-28.09.2026. Public holidays listed include Deepavali, Christmas, New Year, Thaipusam, Federal Territory Day, Chinese New Year, Nuzul Al-Quran, Eidul Fitri, Labour Day, Eidul Adha, Wesak Day, His Majesty the King's Birthday, Awal Muharram, Prophet Muhammad's Birthday, National Day, and Malaysia Day. The notes state that the module registration and examination schedule can be referred to in the UM system guide, the calendar is subject to change, and UM PG iFest 2026 is scheduled for 09-11 June 2026."
+      }
+    ],
+    "text": "<|im_start|>system\nYou are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly.<|im_end|>\n<|im_start|>user\nUse the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: postgraduate\nSection: Academic Calendar 2025/2026\nSubsection: Master and Doctorate Level Academic Calendar\nPages: 4\nKnowledge group: KG-0064\nChunk id: SC-00006\nText:\nLampiran B2 ACADEMIC CALENDAR 2025/2026 ACADEMIC SESSION (MASTER AND DOCTORATE LEVEL) AMENDMENT ‘SEMESTER | ‘Orientation (Week of Welcome)-WOW 1 week 05.10.2025 = «= 12.10.2025 Lectures, 6 weeks\" 19.10.2028 - 23.11.2025 Mid Semester | Break 1 week 24.11.2025 = © 30.11.2025 Lectures. 8 weeks\" 01.12.2025 = © 25.01.2026 Revision Week 1 weeks\" 26.01.2028 © - 01.02.2026 ‘Semester | Final Examination 3 weeks\" 02.02.2026 - © 2202.2026 ‘Semester | Break 2 week 23.02.2028 - 08.03.2026 22 weeks ‘SEMESTER I Lectures: 7 weeks\" 09.03.2026 - 26.04.2026 ‘Mid Semester II Break 1 week 27.04.2028 © - 03.05.2026 Lectures, 7 weeks* 04052026 - 21.06.2026 Revision Week 1 week* 22.06.2026 - 28.06.2026 ‘Semester II ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: postgraduate\nSection: Academic Calendar 2025/2026\nSubsection: Master and Doctorate Level Academic Calendar\nPages: 4\nKnowledge group: KG-0064\nChunk id: SC-00007\nText:\n(1) The Module Registration and Examination Schedule can be referred to at tips /umsitsquide um edu my. All, information is subject to change. (0) The Academic Calendar has taken into account public and festive holidays and is subject to change Deepavali 20 October 2025 (Monday) CChvistnas Day 2 December 2025 (Thursday) New Year 04 Janvary 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertitory Day 01 February 2026 (Sunday) ‘Chinese New Year 17 & 18 Fetruary 2026 (Tuesday & Wednesday) Nuzul Al-Quran, 07 March 2026 (Saturday) Edu Fn 20.21 March 2026 (Friday & Saturday) Labour Day 1 May 2026 (Friday) idl Adhal 27 May 2026 (Wednesday) Wesak Day 34 May 2026 (Sunday) His Majesty th ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: undergraduate\nSection: Academic Calendar 2025/2026\nSubsection: Bachelor Degree Level Academic Calendar\nPages: 12\nKnowledge group: KG-0087\nChunk id: SC-00009\nText:\n(1) The Module Registration and Examination Schedule can be referred to at https://umsitsquide um edumy, All information is subject to change. (() The Academic Calendar has taken into account public and festive holidays and is subject to change’ Deepavali 20 October 2028 (Monday) ‘Christmas Day 25 December 2025 (Thursday) New Year 01 January 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertory Day (01 February 2026 (Sunday) Chinese New Year 17 & 18 February 2026 (Tuesday & Wednesday) ‘Nuzul Al-Quran 07 March 2026 (Saturday) Eidul Ft 20 & 21 March 2026 (Friday & Saturday) Labour Day, 01 May 2026 (Friday) Eidul Adna (27 May 2026 (Wednesday) Wesak Day 31 May 2026 (Sunday) His Ma ...[truncated]\n\nQuestion: What is the academic calendar for master and doctorate level in session 2025/2026?<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nThe 2025/2026 master and doctorate academic calendar with Semester I, Semester II, and a special semester, covering orientation, lecture periods, mid-semester breaks, revision weeks, final examinations, and semester breaks. Semester I: Orientation (WOW) 05.10.2025-12.10.2025; lectures 13.10.2025-23.11.2025 and 01.12.2025-25.01.2026; mid-semester break 24.11.2025-30.11.2025; revision week 26.01.2026-01.02.2026; final examination 02.02.2026-22.02.2026; semester break 23.02.2026-08.03.2026. Semester II: lectures 09.03.2026-26.04.2026 and 04.05.2026-21.06.2026; break 27.04.2026-03.05.2026; revision week 22.06.2026-28.06.2026; final examination 29.06.2026-19.07.2026; semester break 20.07.2026-16.08.2026. Special semester: lectures 27.07.2026-13.09.2026; final examination 14.09.2026-20.09.2026; break 21.09.2026-28.09.2026. Public holidays listed include Deepavali, Christmas, New Year, Thaipusam, Federal Territory Day, Chinese New Year, Nuzul Al-Quran, Eidul Fitri, Labour Day, Eidul Adha, Wesak Day, His Majesty the King's Birthday, Awal Muharram, Prophet Muhammad's Birthday, National Day, and Malaysia Day. The notes state that the module registration and examination schedule can be referred to in the UM system guide, the calendar is subject to change, and UM PG iFest 2026 is scheduled for 09-11 June 2026.<|im_end|>\n"
+  },
+  {
+    "qa_id": "qa_000005",
+    "index_id": "UMI-0005",
+    "question": "What is the academic calendar for bachelor degree level in session 2025/2026?",
+    "answer": "The 2025/2026 bachelor degree academic calendar with Semester I, Semester II, and a special semester, covering orientation, lecture periods, mid-semester breaks, revision weeks, final examinations, and semester breaks. Semester I: Orientation (WOW) 05.10.2025-12.10.2025; lectures 13.10.2025-23.11.2025 and 01.12.2025-25.01.2026; mid-semester break 24.11.2025-30.11.2025; revision week 26.01.2026-01.02.2026; final examination 02.02.2026-22.02.2026; semester break 23.02.2026-08.03.2026. Semester II: lectures 09.03.2026-26.04.2026 and 04.05.2026-21.06.2026; break 27.04.2026-03.05.2026; revision week 22.06.2026-28.06.2026; final examination 29.06.2026-19.07.2026; semester break 20.07.2026-16.08.2026. Special semester: lectures 27.07.2026-13.09.2026; final examination 14.09.2026-20.09.2026; break 21.09.2026-28.09.2026. Public holidays listed include Deepavali, Christmas, New Year, Thaipusam, Federal Territory Day, Chinese New Year, Nuzul Al-Quran, Eidul Fitri, Labour Day, Eidul Adha, Wesak Day, His Majesty the King's Birthday, Awal Muharram, Prophet Muhammad's Birthday, National Day, and Malaysia Day. The notes state that the module registration and examination schedule can be referred to in the UM system guide, the calendar is subject to change, and UM UG iFest 2026 is scheduled for 09-11 June 2026.",
+    "scope_label": "undergraduate",
+    "source_doc": "General Handbook",
+    "section": "Academic Calendar 2025/2026",
+    "subsection": "Bachelor Degree Level Academic Calendar",
+    "retrieved_kb_ids": [
+      "RAGKB-0009",
+      "RAGKB-0008",
+      "RAGKB-0007"
+    ],
+    "retrieved_sections": [
+      "Academic Calendar 2025/2026",
+      "Academic Calendar 2025/2026",
+      "Academic Calendar 2025/2026"
+    ],
+    "retrieved_scopes": [
+      "undergraduate",
+      "undergraduate",
+      "postgraduate"
+    ],
+    "prompt": [
+      {
+        "role": "system",
+        "content": "You are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly."
+      },
+      {
+        "role": "user",
+        "content": "Use the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: undergraduate\nSection: Academic Calendar 2025/2026\nSubsection: Bachelor Degree Level Academic Calendar\nPages: 12\nKnowledge group: KG-0087\nChunk id: SC-00009\nText:\n(1) The Module Registration and Examination Schedule can be referred to at https://umsitsquide um edumy, All information is subject to change. (() The Academic Calendar has taken into account public and festive holidays and is subject to change’ Deepavali 20 October 2028 (Monday) ‘Christmas Day 25 December 2025 (Thursday) New Year 01 January 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertory Day (01 February 2026 (Sunday) Chinese New Year 17 & 18 February 2026 (Tuesday & Wednesday) ‘Nuzul Al-Quran 07 March 2026 (Saturday) Eidul Ft 20 & 21 March 2026 (Friday & Saturday) Labour Day, 01 May 2026 (Friday) Eidul Adna (27 May 2026 (Wednesday) Wesak Day 31 May 2026 (Sunday) His Ma ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: undergraduate\nSection: Academic Calendar 2025/2026\nSubsection: Bachelor Degree Level Academic Calendar\nPages: 12\nKnowledge group: KG-0087\nChunk id: SC-00008\nText:\nEy ACADEMIC CALENDAR et SESSION 2025/2026 A eee ACADEMIC CALENDAR 2025/2026 ACADEMIC SESSION (BACHELOR DEGREE LEVEL) AMENDMENT SEMESTER | Orientation (Week of Welcome)-WOW 1 week 05.10.2025 - 12.10.2025 Lectures 6 weeks* 13.10.2025 - = 23.11.2025 Mid Semester | Break 1 week 24.11.2025 - 30.11.2025 Lectures, 8 weeks* 01.12.2025 - — 26.01.2026 Revision Week 1 weeks* 26.01.2026 - 01.02.2026 ‘Semester | Final Examination 3 weeks* 02.02.2026 - 22.02.2026 ‘Semester | Break 2 week 23.02.2026 - 08.03.2026 22 weeks ‘SEMESTER I Lectures 7 weeks* 09.03.2026 - 26.04.2026 Mid Semester II Break 1 week 27.04.2026 - 03.05.2026 Lectures. 7 weeks* 04.05.2026 - 21.06.2026 Revision Week 1 week\" 22.06.2026 - 28. ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: postgraduate\nSection: Academic Calendar 2025/2026\nSubsection: Master and Doctorate Level Academic Calendar\nPages: 4\nKnowledge group: KG-0064\nChunk id: SC-00007\nText:\n(1) The Module Registration and Examination Schedule can be referred to at tips /umsitsquide um edu my. All, information is subject to change. (0) The Academic Calendar has taken into account public and festive holidays and is subject to change Deepavali 20 October 2025 (Monday) CChvistnas Day 2 December 2025 (Thursday) New Year 04 Janvary 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertitory Day 01 February 2026 (Sunday) ‘Chinese New Year 17 & 18 Fetruary 2026 (Tuesday & Wednesday) Nuzul Al-Quran, 07 March 2026 (Saturday) Edu Fn 20.21 March 2026 (Friday & Saturday) Labour Day 1 May 2026 (Friday) idl Adhal 27 May 2026 (Wednesday) Wesak Day 34 May 2026 (Sunday) His Majesty th ...[truncated]\n\nQuestion: What is the academic calendar for bachelor degree level in session 2025/2026?"
+      }
+    ],
+    "completion": [
+      {
+        "role": "assistant",
+        "content": "The 2025/2026 bachelor degree academic calendar with Semester I, Semester II, and a special semester, covering orientation, lecture periods, mid-semester breaks, revision weeks, final examinations, and semester breaks. Semester I: Orientation (WOW) 05.10.2025-12.10.2025; lectures 13.10.2025-23.11.2025 and 01.12.2025-25.01.2026; mid-semester break 24.11.2025-30.11.2025; revision week 26.01.2026-01.02.2026; final examination 02.02.2026-22.02.2026; semester break 23.02.2026-08.03.2026. Semester II: lectures 09.03.2026-26.04.2026 and 04.05.2026-21.06.2026; break 27.04.2026-03.05.2026; revision week 22.06.2026-28.06.2026; final examination 29.06.2026-19.07.2026; semester break 20.07.2026-16.08.2026. Special semester: lectures 27.07.2026-13.09.2026; final examination 14.09.2026-20.09.2026; break 21.09.2026-28.09.2026. Public holidays listed include Deepavali, Christmas, New Year, Thaipusam, Federal Territory Day, Chinese New Year, Nuzul Al-Quran, Eidul Fitri, Labour Day, Eidul Adha, Wesak Day, His Majesty the King's Birthday, Awal Muharram, Prophet Muhammad's Birthday, National Day, and Malaysia Day. The notes state that the module registration and examination schedule can be referred to in the UM system guide, the calendar is subject to change, and UM UG iFest 2026 is scheduled for 09-11 June 2026."
+      }
+    ],
+    "text": "<|im_start|>system\nYou are an academic assistant for the Faculty of Computer Science and Information Technology, Universiti Malaya. Answer only with handbook-grounded information. Use the retrieved handbook evidence when it is provided. Keep the answer direct, specific, and faithful to the handbook. If the evidence does not support a claim, say so clearly and briefly.<|im_end|>\n<|im_start|>user\nUse the retrieved handbook evidence to answer the question. Base the answer on the evidence and keep it faithful to the handbook.\n\n[Retrieved Evidence 1]\nSource document: General Handbook\nScope: undergraduate\nSection: Academic Calendar 2025/2026\nSubsection: Bachelor Degree Level Academic Calendar\nPages: 12\nKnowledge group: KG-0087\nChunk id: SC-00009\nText:\n(1) The Module Registration and Examination Schedule can be referred to at https://umsitsquide um edumy, All information is subject to change. (() The Academic Calendar has taken into account public and festive holidays and is subject to change’ Deepavali 20 October 2028 (Monday) ‘Christmas Day 25 December 2025 (Thursday) New Year 01 January 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertory Day (01 February 2026 (Sunday) Chinese New Year 17 & 18 February 2026 (Tuesday & Wednesday) ‘Nuzul Al-Quran 07 March 2026 (Saturday) Eidul Ft 20 & 21 March 2026 (Friday & Saturday) Labour Day, 01 May 2026 (Friday) Eidul Adna (27 May 2026 (Wednesday) Wesak Day 31 May 2026 (Sunday) His Ma ...[truncated]\n\n[Retrieved Evidence 2]\nSource document: General Handbook\nScope: undergraduate\nSection: Academic Calendar 2025/2026\nSubsection: Bachelor Degree Level Academic Calendar\nPages: 12\nKnowledge group: KG-0087\nChunk id: SC-00008\nText:\nEy ACADEMIC CALENDAR et SESSION 2025/2026 A eee ACADEMIC CALENDAR 2025/2026 ACADEMIC SESSION (BACHELOR DEGREE LEVEL) AMENDMENT SEMESTER | Orientation (Week of Welcome)-WOW 1 week 05.10.2025 - 12.10.2025 Lectures 6 weeks* 13.10.2025 - = 23.11.2025 Mid Semester | Break 1 week 24.11.2025 - 30.11.2025 Lectures, 8 weeks* 01.12.2025 - — 26.01.2026 Revision Week 1 weeks* 26.01.2026 - 01.02.2026 ‘Semester | Final Examination 3 weeks* 02.02.2026 - 22.02.2026 ‘Semester | Break 2 week 23.02.2026 - 08.03.2026 22 weeks ‘SEMESTER I Lectures 7 weeks* 09.03.2026 - 26.04.2026 Mid Semester II Break 1 week 27.04.2026 - 03.05.2026 Lectures. 7 weeks* 04.05.2026 - 21.06.2026 Revision Week 1 week\" 22.06.2026 - 28. ...[truncated]\n\n[Retrieved Evidence 3]\nSource document: General Handbook\nScope: postgraduate\nSection: Academic Calendar 2025/2026\nSubsection: Master and Doctorate Level Academic Calendar\nPages: 4\nKnowledge group: KG-0064\nChunk id: SC-00007\nText:\n(1) The Module Registration and Examination Schedule can be referred to at tips /umsitsquide um edu my. All, information is subject to change. (0) The Academic Calendar has taken into account public and festive holidays and is subject to change Deepavali 20 October 2025 (Monday) CChvistnas Day 2 December 2025 (Thursday) New Year 04 Janvary 2026 (Thursday) ‘Thaipusam 01 February 2026 (Sunday) Federal Tertitory Day 01 February 2026 (Sunday) ‘Chinese New Year 17 & 18 Fetruary 2026 (Tuesday & Wednesday) Nuzul Al-Quran, 07 March 2026 (Saturday) Edu Fn 20.21 March 2026 (Friday & Saturday) Labour Day 1 May 2026 (Friday) idl Adhal 27 May 2026 (Wednesday) Wesak Day 34 May 2026 (Sunday) His Majesty th ...[truncated]\n\nQuestion: What is the academic calendar for bachelor degree level in session 2025/2026?<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nThe 2025/2026 bachelor degree academic calendar with Semester I, Semester II, and a special semester, covering orientation, lecture periods, mid-semester breaks, revision weeks, final examinations, and semester breaks. Semester I: Orientation (WOW) 05.10.2025-12.10.2025; lectures 13.10.2025-23.11.2025 and 01.12.2025-25.01.2026; mid-semester break 24.11.2025-30.11.2025; revision week 26.01.2026-01.02.2026; final examination 02.02.2026-22.02.2026; semester break 23.02.2026-08.03.2026. Semester II: lectures 09.03.2026-26.04.2026 and 04.05.2026-21.06.2026; break 27.04.2026-03.05.2026; revision week 22.06.2026-28.06.2026; final examination 29.06.2026-19.07.2026; semester break 20.07.2026-16.08.2026. Special semester: lectures 27.07.2026-13.09.2026; final examination 14.09.2026-20.09.2026; break 21.09.2026-28.09.2026. Public holidays listed include Deepavali, Christmas, New Year, Thaipusam, Federal Territory Day, Chinese New Year, Nuzul Al-Quran, Eidul Fitri, Labour Day, Eidul Adha, Wesak Day, His Majesty the King's Birthday, Awal Muharram, Prophet Muhammad's Birthday, National Day, and Malaysia Day. The notes state that the module registration and examination schedule can be referred to in the UM system guide, the calendar is subject to change, and UM UG iFest 2026 is scheduled for 09-11 June 2026.<|im_end|>\n"
+  }
+]

UM_Handbook/outputs/baseline2_rag_harness_agent/rag_augmented_dataset/rag_text_length_summary.json

@@ -0,0 +1,11 @@
+{
+  "count": 1000,
+  "min_tokens": 446,
+  "p50_tokens": 800,
+  "p90_tokens": 1036,
+  "p95_tokens": 1234,
+  "p99_tokens": 1488,
+  "max_tokens": 1987,
+  "configured_max_seq_length": 1024,
+  "examples_over_max_seq_length": 114
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/retrieval_eval/retrieval_metrics.json

@@ -0,0 +1,10 @@
+{
+  "retrieval_eval_size": 1000,
+  "top_k": 3,
+  "hit_at_1_primary": 0.821,
+  "hit_at_k_primary": 0.954,
+  "hit_at_k_same_group": 0.991,
+  "scope_match_at_1": 0.996,
+  "retriever_type": "dense_embedding + faiss + metadata_rerank",
+  "embedding_model_name": "BAAI/bge-base-en-v1.5"
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/README.md

@@ -0,0 +1,57 @@
+---
+library_name: transformers
+model_name: trainer_runs
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for trainer_runs
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.1.0
+- Transformers: 5.3.0
+- Pytorch: 2.5.1+cu121
+- Datasets: 2.19.2
+- Tokenizers: 0.22.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/README.md

@@ -0,0 +1,209 @@
+---
+base_model: /scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54282e9eff87e798234f8fa408e6caf2409416eed4516465894c29c08265a18e
+size 87360584

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1aa3f44290e2213c380548285e633d4ec72f400599c12bf70522b5b505b4627
+size 175019258

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf117f629d64665e096b2be07b01ac7ba6e77931c35740af816bf218c7b9127
+size 14244

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85590a667b733e66a7f08005b8e865d1bbbeba867a23d9a471017de48d623149
+size 1064

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/trainer_state.json

@@ -0,0 +1,604 @@
+{
+  "best_global_step": 1100,
+  "best_metric": 0.11906072497367859,
+  "best_model_checkpoint": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100",
+  "epoch": 11.0,
+  "eval_steps": 500,
+  "global_step": 1100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4356346559524535,
+      "epoch": 0.25,
+      "grad_norm": 0.24204915761947632,
+      "learning_rate": 7.999919440291627e-05,
+      "loss": 2.3678880310058594,
+      "mean_token_accuracy": 0.5813769394159317,
+      "num_tokens": 160725.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.8811942958831787,
+      "epoch": 0.5,
+      "grad_norm": 0.17289401590824127,
+      "learning_rate": 7.995766313163572e-05,
+      "loss": 1.7852125549316407,
+      "mean_token_accuracy": 0.6483715295791626,
+      "num_tokens": 327066.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.4657313966751098,
+      "epoch": 0.75,
+      "grad_norm": 0.16354545950889587,
+      "learning_rate": 7.985326923450766e-05,
+      "loss": 1.4699134826660156,
+      "mean_token_accuracy": 0.7016089594364167,
+      "num_tokens": 490539.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.3245354783535004,
+      "epoch": 1.0,
+      "grad_norm": 0.279362291097641,
+      "learning_rate": 7.968617694722514e-05,
+      "loss": 1.3126080322265625,
+      "mean_token_accuracy": 0.7263329613208771,
+      "num_tokens": 656891.0,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.1590555493648236,
+      "eval_loss": 1.1591017246246338,
+      "eval_mean_token_accuracy": 0.7455048836194552,
+      "eval_num_tokens": 656891.0,
+      "eval_runtime": 14.0274,
+      "eval_samples_per_second": 7.129,
+      "eval_steps_per_second": 0.927,
+      "step": 100
+    },
+    {
+      "entropy": 0.9851375865936279,
+      "epoch": 1.25,
+      "grad_norm": 0.405873566865921,
+      "learning_rate": 7.945664914451888e-05,
+      "loss": 0.9941243743896484,
+      "mean_token_accuracy": 0.7774216139316559,
+      "num_tokens": 820900.0,
+      "step": 125
+    },
+    {
+      "entropy": 0.9000456941127777,
+      "epoch": 1.5,
+      "grad_norm": 0.4880390465259552,
+      "learning_rate": 7.916504692659478e-05,
+      "loss": 0.8994422149658203,
+      "mean_token_accuracy": 0.8044012486934662,
+      "num_tokens": 983751.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.7057543742656708,
+      "epoch": 1.75,
+      "grad_norm": 0.6458895206451416,
+      "learning_rate": 7.881182905103986e-05,
+      "loss": 0.69064697265625,
+      "mean_token_accuracy": 0.8439933550357819,
+      "num_tokens": 1148624.0,
+      "step": 175
+    },
+    {
+      "entropy": 0.5627843302488327,
+      "epoch": 2.0,
+      "grad_norm": 0.5327032804489136,
+      "learning_rate": 7.839755121109077e-05,
+      "loss": 0.5385809707641601,
+      "mean_token_accuracy": 0.8772480082511902,
+      "num_tokens": 1313782.0,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.6312807431587806,
+      "eval_loss": 0.5925119519233704,
+      "eval_mean_token_accuracy": 0.861605126124162,
+      "eval_num_tokens": 1313782.0,
+      "eval_runtime": 14.0529,
+      "eval_samples_per_second": 7.116,
+      "eval_steps_per_second": 0.925,
+      "step": 200
+    },
+    {
+      "entropy": 0.44288761138916016,
+      "epoch": 2.25,
+      "grad_norm": 0.5565536022186279,
+      "learning_rate": 7.792286516139999e-05,
+      "loss": 0.4078195571899414,
+      "mean_token_accuracy": 0.9039785468578339,
+      "num_tokens": 1480714.0,
+      "step": 225
+    },
+    {
+      "entropy": 0.44057594060897826,
+      "epoch": 2.5,
+      "grad_norm": 1.3234641551971436,
+      "learning_rate": 7.738851769267544e-05,
+      "loss": 0.4113604736328125,
+      "mean_token_accuracy": 0.9079041624069214,
+      "num_tokens": 1645148.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.34622341573238374,
+      "epoch": 2.75,
+      "grad_norm": 0.4904310405254364,
+      "learning_rate": 7.679534945680635e-05,
+      "loss": 0.316553897857666,
+      "mean_token_accuracy": 0.9257376372814179,
+      "num_tokens": 1805051.0,
+      "step": 275
+    },
+    {
+      "entropy": 0.37137761026620864,
+      "epoch": 3.0,
+      "grad_norm": 0.8455039858818054,
+      "learning_rate": 7.614429364432367e-05,
+      "loss": 0.3286180114746094,
+      "mean_token_accuracy": 0.9244595277309418,
+      "num_tokens": 1970673.0,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.42906437584987056,
+      "eval_loss": 0.37689468264579773,
+      "eval_mean_token_accuracy": 0.9097504982581506,
+      "eval_num_tokens": 1970673.0,
+      "eval_runtime": 14.1516,
+      "eval_samples_per_second": 7.066,
+      "eval_steps_per_second": 0.919,
+      "step": 300
+    },
+    {
+      "entropy": 0.25241469651460646,
+      "epoch": 3.25,
+      "grad_norm": 0.4860665500164032,
+      "learning_rate": 7.543637451627623e-05,
+      "loss": 0.20183673858642578,
+      "mean_token_accuracy": 0.9500378572940826,
+      "num_tokens": 2133206.0,
+      "step": 325
+    },
+    {
+      "entropy": 0.2856204105913639,
+      "epoch": 3.5,
+      "grad_norm": 0.7240539193153381,
+      "learning_rate": 7.467270579283153e-05,
+      "loss": 0.2465129852294922,
+      "mean_token_accuracy": 0.944399631023407,
+      "num_tokens": 2297449.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.23731404632329942,
+      "epoch": 3.75,
+      "grad_norm": 0.9069561958312988,
+      "learning_rate": 7.385448890113697e-05,
+      "loss": 0.19913478851318359,
+      "mean_token_accuracy": 0.9529761910438538,
+      "num_tokens": 2462390.0,
+      "step": 375
+    },
+    {
+      "entropy": 0.23737725749611854,
+      "epoch": 4.0,
+      "grad_norm": 0.8365960717201233,
+      "learning_rate": 7.29830110851977e-05,
+      "loss": 0.20468717575073242,
+      "mean_token_accuracy": 0.9518069052696227,
+      "num_tokens": 2627564.0,
+      "step": 400
+    },
+    {
+      "epoch": 4.0,
+      "eval_entropy": 0.3099417881323741,
+      "eval_loss": 0.28181254863739014,
+      "eval_mean_token_accuracy": 0.9319974917631882,
+      "eval_num_tokens": 2627564.0,
+      "eval_runtime": 14.0762,
+      "eval_samples_per_second": 7.104,
+      "eval_steps_per_second": 0.924,
+      "step": 400
+    },
+    {
+      "entropy": 0.17392504781484605,
+      "epoch": 4.25,
+      "grad_norm": 0.573874831199646,
+      "learning_rate": 7.205964338074462e-05,
+      "loss": 0.13600427627563477,
+      "mean_token_accuracy": 0.966830027103424,
+      "num_tokens": 2789214.0,
+      "step": 425
+    },
+    {
+      "entropy": 0.1851605027168989,
+      "epoch": 4.5,
+      "grad_norm": 0.3293238878250122,
+      "learning_rate": 7.108583845827884e-05,
+      "loss": 0.14525031089782714,
+      "mean_token_accuracy": 0.9660202085971832,
+      "num_tokens": 2955470.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.18264326125383376,
+      "epoch": 4.75,
+      "grad_norm": 0.60421222448349,
+      "learning_rate": 7.00631283376857e-05,
+      "loss": 0.15052377700805664,
+      "mean_token_accuracy": 0.9645662713050842,
+      "num_tokens": 3119188.0,
+      "step": 475
+    },
+    {
+      "entropy": 0.16633620224893092,
+      "epoch": 5.0,
+      "grad_norm": 0.6254115700721741,
+      "learning_rate": 6.899312197801396e-05,
+      "loss": 0.1348426628112793,
+      "mean_token_accuracy": 0.9678875958919525,
+      "num_tokens": 3284455.0,
+      "step": 500
+    },
+    {
+      "epoch": 5.0,
+      "eval_entropy": 0.24236516998364374,
+      "eval_loss": 0.22977329790592194,
+      "eval_mean_token_accuracy": 0.9444432900502131,
+      "eval_num_tokens": 3284455.0,
+      "eval_runtime": 13.8939,
+      "eval_samples_per_second": 7.197,
+      "eval_steps_per_second": 0.936,
+      "step": 500
+    },
+    {
+      "entropy": 0.13423476360738276,
+      "epoch": 5.25,
+      "grad_norm": 0.36637410521507263,
+      "learning_rate": 6.787750274621175e-05,
+      "loss": 0.10897525787353515,
+      "mean_token_accuracy": 0.9732790982723236,
+      "num_tokens": 3449400.0,
+      "step": 525
+    },
+    {
+      "entropy": 0.1397440279275179,
+      "epoch": 5.5,
+      "grad_norm": 0.5530615448951721,
+      "learning_rate": 6.671802576880203e-05,
+      "loss": 0.0994186782836914,
+      "mean_token_accuracy": 0.9755746126174927,
+      "num_tokens": 3613152.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.12368398927152156,
+      "epoch": 5.75,
+      "grad_norm": 0.9692996144294739,
+      "learning_rate": 6.551651517066341e-05,
+      "loss": 0.09958896636962891,
+      "mean_token_accuracy": 0.9760513007640839,
+      "num_tokens": 3776158.0,
+      "step": 575
+    },
+    {
+      "entropy": 0.12079268284142017,
+      "epoch": 6.0,
+      "grad_norm": 0.4396827220916748,
+      "learning_rate": 6.427486120526078e-05,
+      "loss": 0.09088186264038085,
+      "mean_token_accuracy": 0.9771714675426483,
+      "num_tokens": 3941346.0,
+      "step": 600
+    },
+    {
+      "epoch": 6.0,
+      "eval_entropy": 0.1859686219921479,
+      "eval_loss": 0.19470009207725525,
+      "eval_mean_token_accuracy": 0.9534304233697745,
+      "eval_num_tokens": 3941346.0,
+      "eval_runtime": 13.9237,
+      "eval_samples_per_second": 7.182,
+      "eval_steps_per_second": 0.934,
+      "step": 600
+    },
+    {
+      "entropy": 0.12473276376724243,
+      "epoch": 6.25,
+      "grad_norm": 0.34290623664855957,
+      "learning_rate": 6.29950172808403e-05,
+      "loss": 0.09884612083435058,
+      "mean_token_accuracy": 0.9764992237091065,
+      "num_tokens": 4106933.0,
+      "step": 625
+    },
+    {
+      "entropy": 0.085831069201231,
+      "epoch": 6.5,
+      "grad_norm": 0.48308831453323364,
+      "learning_rate": 6.167899688726744e-05,
+      "loss": 0.056854352951049805,
+      "mean_token_accuracy": 0.9848374032974243,
+      "num_tokens": 4268402.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.10950819738209247,
+      "epoch": 6.75,
+      "grad_norm": 0.856299102306366,
+      "learning_rate": 6.032887042834277e-05,
+      "loss": 0.08215932846069336,
+      "mean_token_accuracy": 0.9799254047870636,
+      "num_tokens": 4434736.0,
+      "step": 675
+    },
+    {
+      "entropy": 0.07904369287192821,
+      "epoch": 7.0,
+      "grad_norm": 0.36722248792648315,
+      "learning_rate": 5.89467619645789e-05,
+      "loss": 0.05222135543823242,
+      "mean_token_accuracy": 0.9856019783020019,
+      "num_tokens": 4598237.0,
+      "step": 700
+    },
+    {
+      "epoch": 7.0,
+      "eval_entropy": 0.1536997648385855,
+      "eval_loss": 0.17453822493553162,
+      "eval_mean_token_accuracy": 0.958331937973316,
+      "eval_num_tokens": 4598237.0,
+      "eval_runtime": 14.081,
+      "eval_samples_per_second": 7.102,
+      "eval_steps_per_second": 0.923,
+      "step": 700
+    },
+    {
+      "entropy": 0.09128913629800081,
+      "epoch": 7.25,
+      "grad_norm": 0.3062293827533722,
+      "learning_rate": 5.75348458715631e-05,
+      "loss": 0.0616093111038208,
+      "mean_token_accuracy": 0.9844523584842682,
+      "num_tokens": 4762560.0,
+      "step": 725
+    },
+    {
+      "entropy": 0.08702207624912262,
+      "epoch": 7.5,
+      "grad_norm": 0.4253726601600647,
+      "learning_rate": 5.609534341916273e-05,
+      "loss": 0.05795002460479736,
+      "mean_token_accuracy": 0.9847881937026978,
+      "num_tokens": 4927776.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.05697074756026268,
+      "epoch": 7.75,
+      "grad_norm": 0.2904660403728485,
+      "learning_rate": 5.463051927695499e-05,
+      "loss": 0.03703362941741943,
+      "mean_token_accuracy": 0.9894271802902221,
+      "num_tokens": 5093008.0,
+      "step": 775
+    },
+    {
+      "entropy": 0.07158231139183044,
+      "epoch": 8.0,
+      "grad_norm": 0.18282686173915863,
+      "learning_rate": 5.3142677951379145e-05,
+      "loss": 0.044614124298095706,
+      "mean_token_accuracy": 0.9877620100975036,
+      "num_tokens": 5255128.0,
+      "step": 800
+    },
+    {
+      "epoch": 8.0,
+      "eval_entropy": 0.12773674267988938,
+      "eval_loss": 0.15855245292186737,
+      "eval_mean_token_accuracy": 0.9648683758882376,
+      "eval_num_tokens": 5255128.0,
+      "eval_runtime": 14.1317,
+      "eval_samples_per_second": 7.076,
+      "eval_steps_per_second": 0.92,
+      "step": 800
+    },
+    {
+      "entropy": 0.05208727709949017,
+      "epoch": 8.25,
+      "grad_norm": 0.341634601354599,
+      "learning_rate": 5.163416016021597e-05,
+      "loss": 0.029668822288513183,
+      "mean_token_accuracy": 0.9911812472343445,
+      "num_tokens": 5418711.0,
+      "step": 825
+    },
+    {
+      "entropy": 0.06870386745780706,
+      "epoch": 8.5,
+      "grad_norm": 0.4494340121746063,
+      "learning_rate": 5.010733915009846e-05,
+      "loss": 0.041636109352111816,
+      "mean_token_accuracy": 0.9882563936710358,
+      "num_tokens": 5585074.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.061128609701991084,
+      "epoch": 8.75,
+      "grad_norm": 0.27014654874801636,
+      "learning_rate": 4.856461696284722e-05,
+      "loss": 0.0354688835144043,
+      "mean_token_accuracy": 0.9902391600608825,
+      "num_tokens": 5748942.0,
+      "step": 875
+    },
+    {
+      "entropy": 0.06322231356054545,
+      "epoch": 9.0,
+      "grad_norm": 0.44081079959869385,
+      "learning_rate": 4.7008420656504246e-05,
+      "loss": 0.04019084930419922,
+      "mean_token_accuracy": 0.9888646650314331,
+      "num_tokens": 5912019.0,
+      "step": 900
+    },
+    {
+      "epoch": 9.0,
+      "eval_entropy": 0.10374558430451614,
+      "eval_loss": 0.13253994286060333,
+      "eval_mean_token_accuracy": 0.9727041262846726,
+      "eval_num_tokens": 5912019.0,
+      "eval_runtime": 13.9522,
+      "eval_samples_per_second": 7.167,
+      "eval_steps_per_second": 0.932,
+      "step": 900
+    },
+    {
+      "entropy": 0.05023659970611334,
+      "epoch": 9.25,
+      "grad_norm": 0.17937935888767242,
+      "learning_rate": 4.5441198487010574e-05,
+      "loss": 0.027511301040649413,
+      "mean_token_accuracy": 0.9921806335449219,
+      "num_tokens": 6076215.0,
+      "step": 925
+    },
+    {
+      "entropy": 0.04388071570545435,
+      "epoch": 9.5,
+      "grad_norm": 0.19000087678432465,
+      "learning_rate": 4.386541605653456e-05,
+      "loss": 0.023981449604034425,
+      "mean_token_accuracy": 0.9924513137340546,
+      "num_tokens": 6239231.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.047076338082551954,
+      "epoch": 9.75,
+      "grad_norm": 0.32662832736968994,
+      "learning_rate": 4.228355243451073e-05,
+      "loss": 0.029095897674560545,
+      "mean_token_accuracy": 0.9913454830646515,
+      "num_tokens": 6404599.0,
+      "step": 975
+    },
+    {
+      "entropy": 0.043895740248262885,
+      "epoch": 10.0,
+      "grad_norm": 0.2123415768146515,
+      "learning_rate": 4.0698096257491346e-05,
+      "loss": 0.025430002212524415,
+      "mean_token_accuracy": 0.9922421896457672,
+      "num_tokens": 6568910.0,
+      "step": 1000
+    },
+    {
+      "epoch": 10.0,
+      "eval_entropy": 0.09539230053241436,
+      "eval_loss": 0.13063254952430725,
+      "eval_mean_token_accuracy": 0.972310735629155,
+      "eval_num_tokens": 6568910.0,
+      "eval_runtime": 13.8054,
+      "eval_samples_per_second": 7.244,
+      "eval_steps_per_second": 0.942,
+      "step": 1000
+    },
+    {
+      "entropy": 0.04173858344554901,
+      "epoch": 10.25,
+      "grad_norm": 0.2956293225288391,
+      "learning_rate": 3.911154181394682e-05,
+      "loss": 0.024647462368011474,
+      "mean_token_accuracy": 0.9924235260486602,
+      "num_tokens": 6732529.0,
+      "step": 1025
+    },
+    {
+      "entropy": 0.035978049002587796,
+      "epoch": 10.5,
+      "grad_norm": 0.19044138491153717,
+      "learning_rate": 3.7526385120174296e-05,
+      "loss": 0.019353010654449464,
+      "mean_token_accuracy": 0.9938314747810364,
+      "num_tokens": 6897778.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.03361038889735937,
+      "epoch": 10.75,
+      "grad_norm": 0.1833367496728897,
+      "learning_rate": 3.594511999348787e-05,
+      "loss": 0.018770104646682738,
+      "mean_token_accuracy": 0.9937199604511261,
+      "num_tokens": 7061058.0,
+      "step": 1075
+    },
+    {
+      "entropy": 0.0329610108025372,
+      "epoch": 11.0,
+      "grad_norm": 0.1862572729587555,
+      "learning_rate": 3.437023412886855e-05,
+      "loss": 0.01859575867652893,
+      "mean_token_accuracy": 0.9939316511154175,
+      "num_tokens": 7225801.0,
+      "step": 1100
+    },
+    {
+      "epoch": 11.0,
+      "eval_entropy": 0.07002563731601605,
+      "eval_loss": 0.11906072497367859,
+      "eval_mean_token_accuracy": 0.9767012825379005,
+      "eval_num_tokens": 7225801.0,
+      "eval_runtime": 13.7737,
+      "eval_samples_per_second": 7.26,
+      "eval_steps_per_second": 0.944,
+      "step": 1100
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.795035557618483e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20789f4b467b5ff6e691e6d696bc04daebaaa023439a4918e4927d5d9b849aee
+size 5304

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/README.md

@@ -0,0 +1,209 @@
+---
+base_model: /scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B
+- lora
+- sft
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/models/Qwen3-8B",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "q_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "o_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d68d0d9a654d1eb05183823ecd5454f8992f9e496fa6e5ecde9ffb8804eb2036
+size 87360584

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/chat_template.jinja

@@ -0,0 +1,89 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {%- if loop.last or (not loop.last and reasoning_content) %}
+                {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
+            {%- else %}
+                {{- '<|im_start|>' + message.role + '\n' + content }}
+            {%- endif %}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2f20969db1d5d958e322a2117c0eb3586841fe48295a3ae9117feada1454001
+size 175019258

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77b91d1b8dc7aac8a06ffacb1744b237d3ef30130e28567e0f0f6be4f5ee1e4d
+size 14244

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee1e0f2b5d2ce8cedcbbda81df0279f5aa839f1a9da3b1113c83ef7fa3c6d03f
+size 1064

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
+size 11422650

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/trainer_state.json

@@ -0,0 +1,706 @@
+{
+  "best_global_step": 1100,
+  "best_metric": 0.11906072497367859,
+  "best_model_checkpoint": "/scr/user/kevin2002/TensorCat/NLP/UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1100",
+  "epoch": 13.0,
+  "eval_steps": 500,
+  "global_step": 1300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "entropy": 1.4356346559524535,
+      "epoch": 0.25,
+      "grad_norm": 0.24204915761947632,
+      "learning_rate": 7.999919440291627e-05,
+      "loss": 2.3678880310058594,
+      "mean_token_accuracy": 0.5813769394159317,
+      "num_tokens": 160725.0,
+      "step": 25
+    },
+    {
+      "entropy": 1.8811942958831787,
+      "epoch": 0.5,
+      "grad_norm": 0.17289401590824127,
+      "learning_rate": 7.995766313163572e-05,
+      "loss": 1.7852125549316407,
+      "mean_token_accuracy": 0.6483715295791626,
+      "num_tokens": 327066.0,
+      "step": 50
+    },
+    {
+      "entropy": 1.4657313966751098,
+      "epoch": 0.75,
+      "grad_norm": 0.16354545950889587,
+      "learning_rate": 7.985326923450766e-05,
+      "loss": 1.4699134826660156,
+      "mean_token_accuracy": 0.7016089594364167,
+      "num_tokens": 490539.0,
+      "step": 75
+    },
+    {
+      "entropy": 1.3245354783535004,
+      "epoch": 1.0,
+      "grad_norm": 0.279362291097641,
+      "learning_rate": 7.968617694722514e-05,
+      "loss": 1.3126080322265625,
+      "mean_token_accuracy": 0.7263329613208771,
+      "num_tokens": 656891.0,
+      "step": 100
+    },
+    {
+      "epoch": 1.0,
+      "eval_entropy": 1.1590555493648236,
+      "eval_loss": 1.1591017246246338,
+      "eval_mean_token_accuracy": 0.7455048836194552,
+      "eval_num_tokens": 656891.0,
+      "eval_runtime": 14.0274,
+      "eval_samples_per_second": 7.129,
+      "eval_steps_per_second": 0.927,
+      "step": 100
+    },
+    {
+      "entropy": 0.9851375865936279,
+      "epoch": 1.25,
+      "grad_norm": 0.405873566865921,
+      "learning_rate": 7.945664914451888e-05,
+      "loss": 0.9941243743896484,
+      "mean_token_accuracy": 0.7774216139316559,
+      "num_tokens": 820900.0,
+      "step": 125
+    },
+    {
+      "entropy": 0.9000456941127777,
+      "epoch": 1.5,
+      "grad_norm": 0.4880390465259552,
+      "learning_rate": 7.916504692659478e-05,
+      "loss": 0.8994422149658203,
+      "mean_token_accuracy": 0.8044012486934662,
+      "num_tokens": 983751.0,
+      "step": 150
+    },
+    {
+      "entropy": 0.7057543742656708,
+      "epoch": 1.75,
+      "grad_norm": 0.6458895206451416,
+      "learning_rate": 7.881182905103986e-05,
+      "loss": 0.69064697265625,
+      "mean_token_accuracy": 0.8439933550357819,
+      "num_tokens": 1148624.0,
+      "step": 175
+    },
+    {
+      "entropy": 0.5627843302488327,
+      "epoch": 2.0,
+      "grad_norm": 0.5327032804489136,
+      "learning_rate": 7.839755121109077e-05,
+      "loss": 0.5385809707641601,
+      "mean_token_accuracy": 0.8772480082511902,
+      "num_tokens": 1313782.0,
+      "step": 200
+    },
+    {
+      "epoch": 2.0,
+      "eval_entropy": 0.6312807431587806,
+      "eval_loss": 0.5925119519233704,
+      "eval_mean_token_accuracy": 0.861605126124162,
+      "eval_num_tokens": 1313782.0,
+      "eval_runtime": 14.0529,
+      "eval_samples_per_second": 7.116,
+      "eval_steps_per_second": 0.925,
+      "step": 200
+    },
+    {
+      "entropy": 0.44288761138916016,
+      "epoch": 2.25,
+      "grad_norm": 0.5565536022186279,
+      "learning_rate": 7.792286516139999e-05,
+      "loss": 0.4078195571899414,
+      "mean_token_accuracy": 0.9039785468578339,
+      "num_tokens": 1480714.0,
+      "step": 225
+    },
+    {
+      "entropy": 0.44057594060897826,
+      "epoch": 2.5,
+      "grad_norm": 1.3234641551971436,
+      "learning_rate": 7.738851769267544e-05,
+      "loss": 0.4113604736328125,
+      "mean_token_accuracy": 0.9079041624069214,
+      "num_tokens": 1645148.0,
+      "step": 250
+    },
+    {
+      "entropy": 0.34622341573238374,
+      "epoch": 2.75,
+      "grad_norm": 0.4904310405254364,
+      "learning_rate": 7.679534945680635e-05,
+      "loss": 0.316553897857666,
+      "mean_token_accuracy": 0.9257376372814179,
+      "num_tokens": 1805051.0,
+      "step": 275
+    },
+    {
+      "entropy": 0.37137761026620864,
+      "epoch": 3.0,
+      "grad_norm": 0.8455039858818054,
+      "learning_rate": 7.614429364432367e-05,
+      "loss": 0.3286180114746094,
+      "mean_token_accuracy": 0.9244595277309418,
+      "num_tokens": 1970673.0,
+      "step": 300
+    },
+    {
+      "epoch": 3.0,
+      "eval_entropy": 0.42906437584987056,
+      "eval_loss": 0.37689468264579773,
+      "eval_mean_token_accuracy": 0.9097504982581506,
+      "eval_num_tokens": 1970673.0,
+      "eval_runtime": 14.1516,
+      "eval_samples_per_second": 7.066,
+      "eval_steps_per_second": 0.919,
+      "step": 300
+    },
+    {
+      "entropy": 0.25241469651460646,
+      "epoch": 3.25,
+      "grad_norm": 0.4860665500164032,
+      "learning_rate": 7.543637451627623e-05,
+      "loss": 0.20183673858642578,
+      "mean_token_accuracy": 0.9500378572940826,
+      "num_tokens": 2133206.0,
+      "step": 325
+    },
+    {
+      "entropy": 0.2856204105913639,
+      "epoch": 3.5,
+      "grad_norm": 0.7240539193153381,
+      "learning_rate": 7.467270579283153e-05,
+      "loss": 0.2465129852294922,
+      "mean_token_accuracy": 0.944399631023407,
+      "num_tokens": 2297449.0,
+      "step": 350
+    },
+    {
+      "entropy": 0.23731404632329942,
+      "epoch": 3.75,
+      "grad_norm": 0.9069561958312988,
+      "learning_rate": 7.385448890113697e-05,
+      "loss": 0.19913478851318359,
+      "mean_token_accuracy": 0.9529761910438538,
+      "num_tokens": 2462390.0,
+      "step": 375
+    },
+    {
+      "entropy": 0.23737725749611854,
+      "epoch": 4.0,
+      "grad_norm": 0.8365960717201233,
+      "learning_rate": 7.29830110851977e-05,
+      "loss": 0.20468717575073242,
+      "mean_token_accuracy": 0.9518069052696227,
+      "num_tokens": 2627564.0,
+      "step": 400
+    },
+    {
+      "epoch": 4.0,
+      "eval_entropy": 0.3099417881323741,
+      "eval_loss": 0.28181254863739014,
+      "eval_mean_token_accuracy": 0.9319974917631882,
+      "eval_num_tokens": 2627564.0,
+      "eval_runtime": 14.0762,
+      "eval_samples_per_second": 7.104,
+      "eval_steps_per_second": 0.924,
+      "step": 400
+    },
+    {
+      "entropy": 0.17392504781484605,
+      "epoch": 4.25,
+      "grad_norm": 0.573874831199646,
+      "learning_rate": 7.205964338074462e-05,
+      "loss": 0.13600427627563477,
+      "mean_token_accuracy": 0.966830027103424,
+      "num_tokens": 2789214.0,
+      "step": 425
+    },
+    {
+      "entropy": 0.1851605027168989,
+      "epoch": 4.5,
+      "grad_norm": 0.3293238878250122,
+      "learning_rate": 7.108583845827884e-05,
+      "loss": 0.14525031089782714,
+      "mean_token_accuracy": 0.9660202085971832,
+      "num_tokens": 2955470.0,
+      "step": 450
+    },
+    {
+      "entropy": 0.18264326125383376,
+      "epoch": 4.75,
+      "grad_norm": 0.60421222448349,
+      "learning_rate": 7.00631283376857e-05,
+      "loss": 0.15052377700805664,
+      "mean_token_accuracy": 0.9645662713050842,
+      "num_tokens": 3119188.0,
+      "step": 475
+    },
+    {
+      "entropy": 0.16633620224893092,
+      "epoch": 5.0,
+      "grad_norm": 0.6254115700721741,
+      "learning_rate": 6.899312197801396e-05,
+      "loss": 0.1348426628112793,
+      "mean_token_accuracy": 0.9678875958919525,
+      "num_tokens": 3284455.0,
+      "step": 500
+    },
+    {
+      "epoch": 5.0,
+      "eval_entropy": 0.24236516998364374,
+      "eval_loss": 0.22977329790592194,
+      "eval_mean_token_accuracy": 0.9444432900502131,
+      "eval_num_tokens": 3284455.0,
+      "eval_runtime": 13.8939,
+      "eval_samples_per_second": 7.197,
+      "eval_steps_per_second": 0.936,
+      "step": 500
+    },
+    {
+      "entropy": 0.13423476360738276,
+      "epoch": 5.25,
+      "grad_norm": 0.36637410521507263,
+      "learning_rate": 6.787750274621175e-05,
+      "loss": 0.10897525787353515,
+      "mean_token_accuracy": 0.9732790982723236,
+      "num_tokens": 3449400.0,
+      "step": 525
+    },
+    {
+      "entropy": 0.1397440279275179,
+      "epoch": 5.5,
+      "grad_norm": 0.5530615448951721,
+      "learning_rate": 6.671802576880203e-05,
+      "loss": 0.0994186782836914,
+      "mean_token_accuracy": 0.9755746126174927,
+      "num_tokens": 3613152.0,
+      "step": 550
+    },
+    {
+      "entropy": 0.12368398927152156,
+      "epoch": 5.75,
+      "grad_norm": 0.9692996144294739,
+      "learning_rate": 6.551651517066341e-05,
+      "loss": 0.09958896636962891,
+      "mean_token_accuracy": 0.9760513007640839,
+      "num_tokens": 3776158.0,
+      "step": 575
+    },
+    {
+      "entropy": 0.12079268284142017,
+      "epoch": 6.0,
+      "grad_norm": 0.4396827220916748,
+      "learning_rate": 6.427486120526078e-05,
+      "loss": 0.09088186264038085,
+      "mean_token_accuracy": 0.9771714675426483,
+      "num_tokens": 3941346.0,
+      "step": 600
+    },
+    {
+      "epoch": 6.0,
+      "eval_entropy": 0.1859686219921479,
+      "eval_loss": 0.19470009207725525,
+      "eval_mean_token_accuracy": 0.9534304233697745,
+      "eval_num_tokens": 3941346.0,
+      "eval_runtime": 13.9237,
+      "eval_samples_per_second": 7.182,
+      "eval_steps_per_second": 0.934,
+      "step": 600
+    },
+    {
+      "entropy": 0.12473276376724243,
+      "epoch": 6.25,
+      "grad_norm": 0.34290623664855957,
+      "learning_rate": 6.29950172808403e-05,
+      "loss": 0.09884612083435058,
+      "mean_token_accuracy": 0.9764992237091065,
+      "num_tokens": 4106933.0,
+      "step": 625
+    },
+    {
+      "entropy": 0.085831069201231,
+      "epoch": 6.5,
+      "grad_norm": 0.48308831453323364,
+      "learning_rate": 6.167899688726744e-05,
+      "loss": 0.056854352951049805,
+      "mean_token_accuracy": 0.9848374032974243,
+      "num_tokens": 4268402.0,
+      "step": 650
+    },
+    {
+      "entropy": 0.10950819738209247,
+      "epoch": 6.75,
+      "grad_norm": 0.856299102306366,
+      "learning_rate": 6.032887042834277e-05,
+      "loss": 0.08215932846069336,
+      "mean_token_accuracy": 0.9799254047870636,
+      "num_tokens": 4434736.0,
+      "step": 675
+    },
+    {
+      "entropy": 0.07904369287192821,
+      "epoch": 7.0,
+      "grad_norm": 0.36722248792648315,
+      "learning_rate": 5.89467619645789e-05,
+      "loss": 0.05222135543823242,
+      "mean_token_accuracy": 0.9856019783020019,
+      "num_tokens": 4598237.0,
+      "step": 700
+    },
+    {
+      "epoch": 7.0,
+      "eval_entropy": 0.1536997648385855,
+      "eval_loss": 0.17453822493553162,
+      "eval_mean_token_accuracy": 0.958331937973316,
+      "eval_num_tokens": 4598237.0,
+      "eval_runtime": 14.081,
+      "eval_samples_per_second": 7.102,
+      "eval_steps_per_second": 0.923,
+      "step": 700
+    },
+    {
+      "entropy": 0.09128913629800081,
+      "epoch": 7.25,
+      "grad_norm": 0.3062293827533722,
+      "learning_rate": 5.75348458715631e-05,
+      "loss": 0.0616093111038208,
+      "mean_token_accuracy": 0.9844523584842682,
+      "num_tokens": 4762560.0,
+      "step": 725
+    },
+    {
+      "entropy": 0.08702207624912262,
+      "epoch": 7.5,
+      "grad_norm": 0.4253726601600647,
+      "learning_rate": 5.609534341916273e-05,
+      "loss": 0.05795002460479736,
+      "mean_token_accuracy": 0.9847881937026978,
+      "num_tokens": 4927776.0,
+      "step": 750
+    },
+    {
+      "entropy": 0.05697074756026268,
+      "epoch": 7.75,
+      "grad_norm": 0.2904660403728485,
+      "learning_rate": 5.463051927695499e-05,
+      "loss": 0.03703362941741943,
+      "mean_token_accuracy": 0.9894271802902221,
+      "num_tokens": 5093008.0,
+      "step": 775
+    },
+    {
+      "entropy": 0.07158231139183044,
+      "epoch": 8.0,
+      "grad_norm": 0.18282686173915863,
+      "learning_rate": 5.3142677951379145e-05,
+      "loss": 0.044614124298095706,
+      "mean_token_accuracy": 0.9877620100975036,
+      "num_tokens": 5255128.0,
+      "step": 800
+    },
+    {
+      "epoch": 8.0,
+      "eval_entropy": 0.12773674267988938,
+      "eval_loss": 0.15855245292186737,
+      "eval_mean_token_accuracy": 0.9648683758882376,
+      "eval_num_tokens": 5255128.0,
+      "eval_runtime": 14.1317,
+      "eval_samples_per_second": 7.076,
+      "eval_steps_per_second": 0.92,
+      "step": 800
+    },
+    {
+      "entropy": 0.05208727709949017,
+      "epoch": 8.25,
+      "grad_norm": 0.341634601354599,
+      "learning_rate": 5.163416016021597e-05,
+      "loss": 0.029668822288513183,
+      "mean_token_accuracy": 0.9911812472343445,
+      "num_tokens": 5418711.0,
+      "step": 825
+    },
+    {
+      "entropy": 0.06870386745780706,
+      "epoch": 8.5,
+      "grad_norm": 0.4494340121746063,
+      "learning_rate": 5.010733915009846e-05,
+      "loss": 0.041636109352111816,
+      "mean_token_accuracy": 0.9882563936710358,
+      "num_tokens": 5585074.0,
+      "step": 850
+    },
+    {
+      "entropy": 0.061128609701991084,
+      "epoch": 8.75,
+      "grad_norm": 0.27014654874801636,
+      "learning_rate": 4.856461696284722e-05,
+      "loss": 0.0354688835144043,
+      "mean_token_accuracy": 0.9902391600608825,
+      "num_tokens": 5748942.0,
+      "step": 875
+    },
+    {
+      "entropy": 0.06322231356054545,
+      "epoch": 9.0,
+      "grad_norm": 0.44081079959869385,
+      "learning_rate": 4.7008420656504246e-05,
+      "loss": 0.04019084930419922,
+      "mean_token_accuracy": 0.9888646650314331,
+      "num_tokens": 5912019.0,
+      "step": 900
+    },
+    {
+      "epoch": 9.0,
+      "eval_entropy": 0.10374558430451614,
+      "eval_loss": 0.13253994286060333,
+      "eval_mean_token_accuracy": 0.9727041262846726,
+      "eval_num_tokens": 5912019.0,
+      "eval_runtime": 13.9522,
+      "eval_samples_per_second": 7.167,
+      "eval_steps_per_second": 0.932,
+      "step": 900
+    },
+    {
+      "entropy": 0.05023659970611334,
+      "epoch": 9.25,
+      "grad_norm": 0.17937935888767242,
+      "learning_rate": 4.5441198487010574e-05,
+      "loss": 0.027511301040649413,
+      "mean_token_accuracy": 0.9921806335449219,
+      "num_tokens": 6076215.0,
+      "step": 925
+    },
+    {
+      "entropy": 0.04388071570545435,
+      "epoch": 9.5,
+      "grad_norm": 0.19000087678432465,
+      "learning_rate": 4.386541605653456e-05,
+      "loss": 0.023981449604034425,
+      "mean_token_accuracy": 0.9924513137340546,
+      "num_tokens": 6239231.0,
+      "step": 950
+    },
+    {
+      "entropy": 0.047076338082551954,
+      "epoch": 9.75,
+      "grad_norm": 0.32662832736968994,
+      "learning_rate": 4.228355243451073e-05,
+      "loss": 0.029095897674560545,
+      "mean_token_accuracy": 0.9913454830646515,
+      "num_tokens": 6404599.0,
+      "step": 975
+    },
+    {
+      "entropy": 0.043895740248262885,
+      "epoch": 10.0,
+      "grad_norm": 0.2123415768146515,
+      "learning_rate": 4.0698096257491346e-05,
+      "loss": 0.025430002212524415,
+      "mean_token_accuracy": 0.9922421896457672,
+      "num_tokens": 6568910.0,
+      "step": 1000
+    },
+    {
+      "epoch": 10.0,
+      "eval_entropy": 0.09539230053241436,
+      "eval_loss": 0.13063254952430725,
+      "eval_mean_token_accuracy": 0.972310735629155,
+      "eval_num_tokens": 6568910.0,
+      "eval_runtime": 13.8054,
+      "eval_samples_per_second": 7.244,
+      "eval_steps_per_second": 0.942,
+      "step": 1000
+    },
+    {
+      "entropy": 0.04173858344554901,
+      "epoch": 10.25,
+      "grad_norm": 0.2956293225288391,
+      "learning_rate": 3.911154181394682e-05,
+      "loss": 0.024647462368011474,
+      "mean_token_accuracy": 0.9924235260486602,
+      "num_tokens": 6732529.0,
+      "step": 1025
+    },
+    {
+      "entropy": 0.035978049002587796,
+      "epoch": 10.5,
+      "grad_norm": 0.19044138491153717,
+      "learning_rate": 3.7526385120174296e-05,
+      "loss": 0.019353010654449464,
+      "mean_token_accuracy": 0.9938314747810364,
+      "num_tokens": 6897778.0,
+      "step": 1050
+    },
+    {
+      "entropy": 0.03361038889735937,
+      "epoch": 10.75,
+      "grad_norm": 0.1833367496728897,
+      "learning_rate": 3.594511999348787e-05,
+      "loss": 0.018770104646682738,
+      "mean_token_accuracy": 0.9937199604511261,
+      "num_tokens": 7061058.0,
+      "step": 1075
+    },
+    {
+      "entropy": 0.0329610108025372,
+      "epoch": 11.0,
+      "grad_norm": 0.1862572729587555,
+      "learning_rate": 3.437023412886855e-05,
+      "loss": 0.01859575867652893,
+      "mean_token_accuracy": 0.9939316511154175,
+      "num_tokens": 7225801.0,
+      "step": 1100
+    },
+    {
+      "epoch": 11.0,
+      "eval_entropy": 0.07002563731601605,
+      "eval_loss": 0.11906072497367859,
+      "eval_mean_token_accuracy": 0.9767012825379005,
+      "eval_num_tokens": 7225801.0,
+      "eval_runtime": 13.7737,
+      "eval_samples_per_second": 7.26,
+      "eval_steps_per_second": 0.944,
+      "step": 1100
+    },
+    {
+      "entropy": 0.029755542390048503,
+      "epoch": 11.25,
+      "grad_norm": 0.13579222559928894,
+      "learning_rate": 3.280420518524585e-05,
+      "loss": 0.016698834896087648,
+      "mean_token_accuracy": 0.9944922685623169,
+      "num_tokens": 7392621.0,
+      "step": 1125
+    },
+    {
+      "entropy": 0.02796286340802908,
+      "epoch": 11.5,
+      "grad_norm": 0.14825628697872162,
+      "learning_rate": 3.124949688756846e-05,
+      "loss": 0.015717124938964842,
+      "mean_token_accuracy": 0.9946336007118225,
+      "num_tokens": 7556769.0,
+      "step": 1150
+    },
+    {
+      "entropy": 0.0260389681532979,
+      "epoch": 11.75,
+      "grad_norm": 0.1284666806459427,
+      "learning_rate": 2.9708555150796346e-05,
+      "loss": 0.016165239810943602,
+      "mean_token_accuracy": 0.9943143832683563,
+      "num_tokens": 7719224.0,
+      "step": 1175
+    },
+    {
+      "entropy": 0.03029863853007555,
+      "epoch": 12.0,
+      "grad_norm": 0.2951240837574005,
+      "learning_rate": 2.8183804231912004e-05,
+      "loss": 0.01682651400566101,
+      "mean_token_accuracy": 0.9942524111270905,
+      "num_tokens": 7882692.0,
+      "step": 1200
+    },
+    {
+      "epoch": 12.0,
+      "eval_entropy": 0.06255707159065284,
+      "eval_loss": 0.11945243179798126,
+      "eval_mean_token_accuracy": 0.9772726480777447,
+      "eval_num_tokens": 7882692.0,
+      "eval_runtime": 13.7991,
+      "eval_samples_per_second": 7.247,
+      "eval_steps_per_second": 0.942,
+      "step": 1200
+    },
+    {
+      "entropy": 0.02317443212494254,
+      "epoch": 12.25,
+      "grad_norm": 0.13226298987865448,
+      "learning_rate": 2.6677642916004772e-05,
+      "loss": 0.012957496643066406,
+      "mean_token_accuracy": 0.9950127184391022,
+      "num_tokens": 8049057.0,
+      "step": 1225
+    },
+    {
+      "entropy": 0.0250219264626503,
+      "epoch": 12.5,
+      "grad_norm": 0.11246883124113083,
+      "learning_rate": 2.519244074242825e-05,
+      "loss": 0.01474016547203064,
+      "mean_token_accuracy": 0.9946307218074799,
+      "num_tokens": 8212177.0,
+      "step": 1250
+    },
+    {
+      "entropy": 0.024466073755174875,
+      "epoch": 12.75,
+      "grad_norm": 0.12266653031110764,
+      "learning_rate": 2.3730534276968e-05,
+      "loss": 0.014008394479751586,
+      "mean_token_accuracy": 0.9946387422084808,
+      "num_tokens": 8376959.0,
+      "step": 1275
+    },
+    {
+      "entropy": 0.02350047092884779,
+      "epoch": 13.0,
+      "grad_norm": 0.10886064171791077,
+      "learning_rate": 2.2294223435884402e-05,
+      "loss": 0.014072380065917968,
+      "mean_token_accuracy": 0.9947505962848663,
+      "num_tokens": 8539583.0,
+      "step": 1300
+    },
+    {
+      "epoch": 13.0,
+      "eval_entropy": 0.05241998714896349,
+      "eval_loss": 0.12038982659578323,
+      "eval_mean_token_accuracy": 0.9782212514143723,
+      "eval_num_tokens": 8539583.0,
+      "eval_runtime": 13.7732,
+      "eval_samples_per_second": 7.26,
+      "eval_steps_per_second": 0.944,
+      "step": 1300
+    }
+  ],
+  "logging_steps": 25,
+  "max_steps": 2000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 2,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 2
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.4872917665132544e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/checkpoint-1300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20789f4b467b5ff6e691e6d696bc04daebaaa023439a4918e4927d5d9b849aee
+size 5304

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/dataset_dict.json

@@ -0,0 +1 @@
+{"splits": ["train", "validation", "test"]}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/test/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f025159053af787b7124324d5700120792c277fbd717701fed35627723dd5c86
+size 787088

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/test/dataset_info.json

@@ -0,0 +1,89 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "qa_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "index_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "question": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "answer": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "scope_label": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "source_doc": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "section": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "subsection": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "retrieved_kb_ids": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "retrieved_sections": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "retrieved_scopes": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "prompt": [
+      {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      }
+    ],
+    "completion": [
+      {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      }
+    ],
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/test/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "936c078af06a1d62",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/train/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a886ad01c88df3805ff1754bc09af8c94c3cf72fe5908edaff92b5b70943c8c7
+size 6287752

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/train/dataset_info.json

@@ -0,0 +1,89 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "qa_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "index_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "question": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "answer": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "scope_label": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "source_doc": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "section": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "subsection": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "retrieved_kb_ids": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "retrieved_sections": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "retrieved_scopes": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "prompt": [
+      {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      }
+    ],
+    "completion": [
+      {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      }
+    ],
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/train/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "a4eeb3b4abeec226",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/validation/data-00000-of-00001.arrow

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e91442ff072666e35d8227c37e03637290a2bfbbff9b277b8fe61cff2b1862
+size 782200

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/validation/dataset_info.json

@@ -0,0 +1,89 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "qa_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "index_id": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "question": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "answer": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "scope_label": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "source_doc": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "section": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "subsection": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "retrieved_kb_ids": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "retrieved_sections": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "retrieved_scopes": {
+      "feature": {
+        "dtype": "string",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "prompt": [
+      {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      }
+    ],
+    "completion": [
+      {
+        "content": {
+          "dtype": "string",
+          "_type": "Value"
+        },
+        "role": {
+          "dtype": "string",
+          "_type": "Value"
+        }
+      }
+    ],
+    "text": {
+      "dtype": "string",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/hf_dataset_dict/validation/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "6cb74bcfac21e0ef",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

UM_Handbook/outputs/baseline2_rag_harness_agent/trainer_runs/train_metrics.json

@@ -0,0 +1,7 @@
+{
+  "train_runtime": 3233.2916,
+  "train_samples_per_second": 4.949,
+  "train_steps_per_second": 0.619,
+  "total_flos": 4.4872917665132544e+17,
+  "train_loss": 0.2747548096684309
+}