diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,1194 @@ +2024-07-04 20:03:05,004 INFO StreamThr :136 [internal.py:wandb_internal():85] W&B internal server running at pid: 136, started at: 2024-07-04 20:03:05.004020 +2024-07-04 20:03:05,006 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status +2024-07-04 20:03:05,320 INFO WriterThread:136 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240704_200304-v5ofm505/run-v5ofm505.wandb +2024-07-04 20:03:05,321 DEBUG SenderThread:136 [sender.py:send():378] send: header +2024-07-04 20:03:05,324 DEBUG SenderThread:136 [sender.py:send():378] send: run +2024-07-04 20:03:05,534 INFO SenderThread:136 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files +2024-07-04 20:03:05,534 INFO SenderThread:136 [sender.py:_start_run_threads():1123] run started: v5ofm505 with start time 1720123385.005429 +2024-07-04 20:03:05,549 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: check_version +2024-07-04 20:03:05,549 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: check_version +2024-07-04 20:03:05,639 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: run_start +2024-07-04 20:03:05,653 DEBUG HandlerThread:136 [system_info.py:__init__():26] System info init +2024-07-04 20:03:05,653 DEBUG HandlerThread:136 [system_info.py:__init__():41] System info init done +2024-07-04 20:03:05,653 INFO HandlerThread:136 [system_monitor.py:start():194] Starting system monitor +2024-07-04 20:03:05,653 INFO SystemMonitor:136 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-04 20:03:05,654 INFO HandlerThread:136 [system_monitor.py:probe():214] Collecting system info +2024-07-04 20:03:05,654 INFO SystemMonitor:136 [interfaces.py:start():188] Started cpu monitoring +2024-07-04 20:03:05,655 INFO SystemMonitor:136 [interfaces.py:start():188] Started disk monitoring +2024-07-04 20:03:05,655 INFO SystemMonitor:136 [interfaces.py:start():188] Started gpu monitoring +2024-07-04 20:03:05,657 INFO SystemMonitor:136 [interfaces.py:start():188] Started memory monitoring +2024-07-04 20:03:05,658 INFO SystemMonitor:136 [interfaces.py:start():188] Started network monitoring +2024-07-04 20:03:05,663 DEBUG HandlerThread:136 [system_info.py:probe():150] Probing system +2024-07-04 20:03:05,666 DEBUG HandlerThread:136 [gitlib.py:_init_repo():56] git repository is invalid +2024-07-04 20:03:05,666 DEBUG HandlerThread:136 [system_info.py:probe():198] Probing system done +2024-07-04 20:03:05,666 DEBUG HandlerThread:136 [system_monitor.py:probe():223] {'os': 'Linux-5.15.154+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-07-04T20:03:05.663552', 'startedAt': '2024-07-04T20:03:04.997852', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'bcbb291c3718', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.134, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5689.499000549316}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357574462890625}} +2024-07-04 20:03:05,666 INFO HandlerThread:136 [system_monitor.py:probe():224] Finished collecting system info +2024-07-04 20:03:05,666 INFO HandlerThread:136 [system_monitor.py:probe():227] Publishing system info +2024-07-04 20:03:05,667 DEBUG HandlerThread:136 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-07-04 20:03:06,536 INFO Thread-12 :136 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/conda-environment.yaml +2024-07-04 20:03:20,681 ERROR HandlerThread:136 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +Traceback (most recent call last): + File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda + subprocess.call( + File "/opt/conda/lib/python3.10/subprocess.py", line 347, in call + return p.wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1209, in wait + return self._wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-07-04 20:03:20,684 DEBUG HandlerThread:136 [system_info.py:_save_conda():222] Saving conda packages done +2024-07-04 20:03:20,684 INFO HandlerThread:136 [system_monitor.py:probe():229] Finished publishing system info +2024-07-04 20:03:20,693 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:20,693 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: keepalive +2024-07-04 20:03:20,694 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:20,694 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: keepalive +2024-07-04 20:03:20,694 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:20,694 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: keepalive +2024-07-04 20:03:20,694 DEBUG SenderThread:136 [sender.py:send():378] send: files +2024-07-04 20:03:20,695 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now +2024-07-04 20:03:21,011 INFO wandb-upload_0:136 [upload_job.py:push():130] Uploaded file /tmp/tmp8n8odwqhwandb/kxconlu9-wandb-metadata.json +2024-07-04 20:03:21,539 INFO Thread-12 :136 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-metadata.json +2024-07-04 20:03:21,971 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: python_packages +2024-07-04 20:03:21,972 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: python_packages +2024-07-04 20:03:21,974 DEBUG SenderThread:136 [sender.py:send():378] send: telemetry +2024-07-04 20:03:21,975 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:03:21,976 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:03:22,117 DEBUG SenderThread:136 [sender.py:send():378] send: config +2024-07-04 20:03:22,118 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:03:22,119 DEBUG SenderThread:136 [sender.py:send():378] send: telemetry +2024-07-04 20:03:22,119 DEBUG SenderThread:136 [sender.py:send():378] send: telemetry +2024-07-04 20:03:22,119 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:03:22,119 WARNING SenderThread:136 [sender.py:send_metric():1340] Seen metric with glob (shouldn't happen) +2024-07-04 20:03:22,119 DEBUG SenderThread:136 [sender.py:send():378] send: telemetry +2024-07-04 20:03:22,540 INFO Thread-12 :136 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/requirements.txt +2024-07-04 20:03:26,120 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:31,120 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:36,126 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:36,545 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/config.yaml +2024-07-04 20:03:36,972 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:03:36,973 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:03:42,091 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:47,092 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:51,972 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:03:51,973 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:03:53,042 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:03:58,043 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:03,044 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:05,658 DEBUG SystemMonitor:136 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-04 20:04:05,659 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:04:06,972 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:04:06,973 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:04:08,101 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:13,102 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:18,103 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:21,972 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:04:21,973 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:04:24,057 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:29,058 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:34,059 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:35,660 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:04:36,974 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:04:36,975 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:04:39,127 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:44,128 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:49,129 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:04:51,973 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:04:51,973 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:04:55,076 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:00,077 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:05,078 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:05,661 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:05:07,010 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:05:07,011 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:05:10,100 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:15,101 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:20,102 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:22,046 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:05:22,047 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:05:25,127 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:30,128 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:35,129 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:35,662 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:05:37,025 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:05:37,026 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:05:40,159 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:45,161 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:50,162 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:05:52,178 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:05:52,178 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:05:55,246 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:00,247 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:05,249 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:05,662 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:06:07,177 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:06:07,178 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:06:11,242 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:16,243 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:21,244 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:22,177 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:06:22,178 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:06:26,267 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:31,268 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:35,663 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:06:36,664 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:37,244 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:06:37,245 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:06:42,377 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:47,378 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:52,243 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:06:52,244 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:06:53,356 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:06:58,357 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:03,359 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:05,664 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:07:07,243 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:07:07,244 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:07:09,315 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:14,316 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:19,317 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:22,288 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:07:22,288 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:07:24,348 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:29,349 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:34,350 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:35,665 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:07:37,275 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:07:37,275 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:07:39,409 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:44,410 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:49,411 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:07:52,275 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:07:52,275 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:07:55,370 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:00,371 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:05,372 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:05,666 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:08:07,275 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:08:07,276 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:08:10,436 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:15,437 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:20,438 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:22,275 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:08:22,276 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:08:26,367 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:31,368 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:35,667 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:08:36,668 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:37,319 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:08:37,320 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:08:42,438 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:47,439 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:52,298 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:08:52,298 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:08:53,357 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:08:58,358 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:03,359 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:05,668 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:09:07,298 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:09:07,298 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:09:08,380 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:13,381 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:18,383 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:22,336 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:09:22,337 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:09:23,495 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:28,496 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:33,497 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:35,669 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:09:37,315 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:09:37,315 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:09:39,440 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:44,441 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:49,442 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:09:52,339 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:09:52,340 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:09:55,438 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:00,439 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:05,440 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:05,669 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:10:07,518 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:10:07,519 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:10:10,692 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:15,693 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:20,694 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:22,700 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:10:22,700 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:10:25,760 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:26,645 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:10:26,647 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:10:26,648 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:10:26,648 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:10:26,648 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:10:26,648 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:10:26,648 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:10:26,648 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:10:26,700 INFO Thread-12 :136 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:10:31,650 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:35,670 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:10:36,671 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:37,700 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:10:37,700 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:10:41,839 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:46,840 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:51,846 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:10:52,700 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:10:52,701 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:10:52,710 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/config.yaml +2024-07-04 20:10:57,806 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:02,807 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:05,671 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:11:07,700 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:11:07,701 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:11:07,845 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:12,846 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:17,847 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:22,701 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:11:22,702 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:11:22,860 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:27,861 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:32,862 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:35,672 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:11:37,701 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:11:37,701 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:11:38,797 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:43,797 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:48,798 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:52,701 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:11:52,701 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:11:53,833 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:11:58,834 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:03,834 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:05,673 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:12:07,701 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:12:07,701 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:12:08,850 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:13,851 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:18,852 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:22,877 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:12:22,878 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:12:23,991 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:28,992 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:33,993 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:35,674 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:12:37,876 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:12:37,877 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:12:39,035 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:44,036 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:49,037 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:12:52,876 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:12:52,877 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:12:55,029 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:00,030 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:05,031 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:05,674 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:13:08,085 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:13:08,086 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:13:10,235 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:15,236 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:20,237 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:23,084 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:13:23,085 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:13:26,154 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:31,155 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:35,675 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:13:36,676 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:38,085 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:13:38,085 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:13:42,179 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:47,180 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:52,182 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:13:53,137 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:13:53,137 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:13:57,242 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:02,243 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:05,676 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:14:07,677 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:08,194 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:14:08,194 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:14:13,328 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:18,329 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:23,173 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:14:23,174 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:14:24,309 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:29,310 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:34,311 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:35,677 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:14:38,173 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:14:38,174 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:14:40,243 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:45,244 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:50,245 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:14:53,222 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:14:53,223 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:14:55,332 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:00,333 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:05,334 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:05,678 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:15:08,201 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:15:08,201 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:15:11,333 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:16,334 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:21,335 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:23,201 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:15:23,201 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:15:27,318 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:32,319 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:35,679 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:15:37,680 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:38,201 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:15:38,201 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:15:43,317 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:48,318 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:53,201 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:15:53,202 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:15:54,285 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:15:59,286 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:04,287 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:05,680 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:16:08,210 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:16:08,210 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:16:09,369 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:14,370 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:19,371 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:23,209 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:16:23,210 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:16:25,282 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:30,283 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:35,284 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:35,681 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:16:38,360 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:16:38,361 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:16:40,448 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:45,449 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:50,450 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:16:53,361 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:16:53,361 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:16:55,481 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:00,482 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:05,483 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:05,681 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:17:08,361 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:17:08,361 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:17:11,443 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:16,444 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:21,445 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:23,361 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:17:23,362 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:17:27,434 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:30,624 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:17:30,626 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:17:30,626 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:17:30,626 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:17:30,860 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:17:32,627 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:35,682 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:17:37,683 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:38,398 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:17:38,398 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:17:43,492 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:48,493 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:53,445 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:17:53,445 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:17:53,549 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:17:58,550 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:03,551 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:05,683 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:18:08,422 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:18:08,423 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:18:09,550 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:14,551 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:19,553 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:23,423 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:18:23,424 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:18:24,569 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:29,569 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:34,571 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:35,684 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:18:38,466 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:18:38,467 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:18:40,540 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:45,541 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:50,542 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:18:53,445 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:18:53,445 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:18:56,506 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:01,508 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:05,685 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:19:06,686 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:08,461 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:19:08,462 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:19:12,550 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:17,550 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:22,551 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:23,460 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:19:23,461 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:19:27,597 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:32,598 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:35,686 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:19:37,687 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:38,460 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:19:38,461 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:19:43,566 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:48,567 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:53,468 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:19:53,469 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:19:54,540 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:19:59,541 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:04,542 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:05,687 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:20:08,467 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:20:08,467 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:20:10,540 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:15,541 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:20,541 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:23,480 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:20:23,481 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:20:25,590 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:30,591 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:35,592 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:35,688 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:20:38,479 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:20:38,480 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:20:40,635 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:45,636 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:50,637 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:20:53,479 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:20:53,480 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:20:56,631 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:01,632 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:05,689 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:21:06,690 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:08,479 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:21:08,480 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:21:12,620 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:17,621 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:22,621 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:23,480 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:21:23,480 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:21:28,581 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:33,582 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:35,690 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:21:38,640 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:21:38,641 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:21:38,643 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:43,723 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:48,724 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:53,656 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:21:53,657 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:21:53,756 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:21:58,757 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:03,757 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:05,691 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:22:08,653 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:22:08,654 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:22:08,760 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:13,762 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:18,763 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:23,695 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:22:23,695 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:22:23,810 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:28,811 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:33,812 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:35,695 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:22:38,731 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:22:38,732 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:22:38,884 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:43,885 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:48,886 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:53,708 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:22:53,708 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:22:54,858 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:22:59,859 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:04,860 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:05,692 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:23:08,844 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:23:08,844 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:23:09,981 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:14,982 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:19,983 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:23,897 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:23:23,898 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:23:25,043 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:30,044 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:35,045 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:35,695 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:23:39,100 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:23:39,101 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:23:40,165 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:45,166 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:50,167 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:23:54,139 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:23:54,140 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:23:55,277 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:00,279 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:05,280 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:05,694 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:24:09,119 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:24:09,120 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:24:11,193 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:16,194 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:21,195 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:24,120 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:24:24,120 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:24:26,259 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:31,260 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:35,695 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:24:35,700 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:24:35,702 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:24:35,702 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:24:35,703 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:24:36,020 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:24:36,703 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:39,180 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:24:39,181 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:24:42,313 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:47,314 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:52,315 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:24:54,119 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:24:54,120 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:24:58,195 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:03,196 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:05,695 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:25:08,696 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:09,166 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:25:09,166 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:25:14,249 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:19,250 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:24,163 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:25:24,164 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:25:24,312 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:29,314 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:34,315 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:35,696 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:25:39,306 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:25:39,306 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:25:39,461 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:44,461 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:49,462 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:25:54,305 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:25:54,306 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:25:55,391 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:00,392 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:05,393 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:05,697 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:26:09,305 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:26:09,306 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:26:10,422 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:15,423 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:20,424 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:24,337 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:26:24,337 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:26:25,499 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:30,500 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:35,501 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:35,698 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:26:39,316 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:26:39,317 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:26:41,398 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:46,399 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:51,400 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:26:54,316 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:26:54,317 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:26:56,453 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:01,454 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:05,699 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:27:06,700 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:09,518 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:27:09,518 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:27:12,639 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:17,640 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:22,641 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:24,517 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:27:24,518 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:27:28,636 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:33,637 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:35,699 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:27:38,701 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:39,521 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:27:39,521 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:27:44,654 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:49,655 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:54,573 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:27:54,573 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:27:54,702 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:27:59,703 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:04,704 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:05,700 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:28:09,571 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:28:09,571 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:28:09,728 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:14,729 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:19,730 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:24,572 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:28:24,573 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:28:25,687 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:30,688 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:35,692 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:35,701 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:28:39,573 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:28:39,573 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:28:40,705 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:45,706 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:50,707 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:28:54,573 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:28:54,573 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:28:55,712 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:00,713 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:05,702 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:29:06,703 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:09,610 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:29:09,610 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:29:11,727 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:16,729 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:21,729 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:24,597 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:29:24,598 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:29:27,685 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:32,686 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:35,703 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:29:37,704 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:39,597 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:29:39,598 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:29:43,696 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:48,697 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:53,699 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:29:54,597 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:29:54,598 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:29:58,770 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:03,771 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:05,704 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:30:09,588 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:09,598 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:30:09,598 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:30:14,730 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:19,731 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:24,810 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:30:24,811 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:30:24,935 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:29,937 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:34,938 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:35,705 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:30:40,019 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:30:40,020 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:30:40,098 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:45,100 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:50,100 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:30:55,019 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:30:55,020 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:30:55,118 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:00,120 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:05,121 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:05,706 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:31:10,020 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:31:10,020 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:31:10,151 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:15,152 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:20,153 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:25,199 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:31:25,200 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:31:25,334 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:30,335 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:35,337 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:35,706 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:31:40,199 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:31:40,200 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:31:41,292 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:42,067 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:31:42,068 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:31:42,068 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:31:42,069 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:31:42,182 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:31:47,070 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:52,071 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:31:55,199 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:31:55,200 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:31:57,269 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:02,270 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:05,707 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:32:07,708 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:10,364 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:32:10,365 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:32:13,467 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:18,468 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:23,469 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:25,364 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:32:25,365 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:32:28,485 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:33,486 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:35,708 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:32:38,708 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:40,383 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:32:40,384 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:32:44,505 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:49,506 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:54,507 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:32:55,567 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:32:55,567 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:32:59,717 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:04,718 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:05,708 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:33:09,769 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:10,567 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:33:10,567 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:33:15,638 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:20,639 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:25,567 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:33:25,568 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:33:25,644 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:30,645 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:35,647 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:35,709 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:33:40,709 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:33:40,709 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:33:40,849 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:45,850 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:50,851 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:33:55,752 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:33:55,753 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:33:56,828 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:01,829 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:05,710 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:34:07,711 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:10,732 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:34:10,733 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:34:12,850 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:17,851 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:22,852 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:25,932 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:34:25,932 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:34:28,032 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:33,033 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:35,711 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:34:38,712 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:40,964 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:34:40,965 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:34:44,073 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:49,074 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:54,075 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:34:55,944 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:34:55,944 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:34:59,999 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:05,000 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:05,712 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:35:10,944 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:35:10,944 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:35:11,027 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:16,028 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:21,029 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:25,978 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:35:25,978 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:35:26,122 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:31,124 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:35,712 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:35:36,713 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:40,956 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:35:40,957 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:35:42,094 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:47,096 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:52,097 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:35:55,979 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:35:55,979 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:35:57,137 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:02,138 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:05,713 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:36:07,714 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:11,089 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:36:11,089 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:36:13,204 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:18,205 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:23,206 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:26,089 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:36:26,089 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:36:28,240 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:33,240 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:35,714 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:36:38,715 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:41,089 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:36:41,089 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:36:44,198 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:49,199 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:54,200 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:36:56,089 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:36:56,090 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:36:59,244 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:04,245 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:05,715 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:37:09,716 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:11,089 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:37:11,090 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:37:15,199 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:20,199 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:25,200 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:26,253 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:37:26,254 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:37:30,372 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:35,373 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:35,716 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:37:40,404 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:41,253 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:37:41,254 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:37:46,383 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:51,384 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:37:56,253 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:37:56,254 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:37:57,324 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:02,324 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:05,716 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:38:07,717 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:11,254 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:38:11,254 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:38:13,355 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:18,356 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:23,357 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:26,254 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:38:26,254 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:38:29,350 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:34,351 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:35,717 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:38:39,718 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:41,302 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:38:41,303 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:38:45,441 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:47,016 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:38:47,017 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:38:47,018 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:38:47,018 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:38:47,329 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:38:51,019 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:38:56,410 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:38:56,410 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:38:56,542 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:01,543 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:05,718 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:39:06,719 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:11,410 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:39:11,410 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:39:12,560 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:17,561 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:22,562 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:26,410 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:39:26,410 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:39:27,567 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:32,568 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:35,719 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:39:37,720 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:41,484 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:39:41,484 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:39:43,563 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:48,564 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:53,565 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:39:56,600 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:39:56,600 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:39:58,676 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:03,677 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:05,720 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:40:08,721 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:11,600 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:40:11,600 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:40:14,701 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:19,702 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:24,702 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:26,600 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:40:26,600 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:40:30,698 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:35,698 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:35,720 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:40:40,721 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:41,713 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:40:41,713 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:40:45,781 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:50,782 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:55,783 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:40:56,713 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:40:56,714 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:41:00,838 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:05,721 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:41:06,722 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:11,713 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:41:11,714 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:41:11,802 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:16,803 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:21,804 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:26,728 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:41:26,728 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:41:26,879 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:31,880 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:35,722 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:41:37,723 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:41,756 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:41:41,756 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:41:42,877 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:47,878 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:52,878 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:41:56,735 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:41:56,736 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:41:58,867 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:03,869 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:05,723 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:42:09,724 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:11,736 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:42:11,736 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:42:14,875 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:19,876 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:24,876 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:26,736 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:42:26,736 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:42:30,858 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:35,724 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:42:36,724 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:41,784 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:42:41,785 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:42:41,895 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:46,897 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:51,898 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:42:56,764 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:42:56,765 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:42:57,840 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:02,841 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:05,725 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:43:08,726 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:11,800 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:43:11,800 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:43:13,961 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:18,961 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:23,962 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:26,780 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:43:26,780 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:43:29,887 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:34,888 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:35,725 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:43:40,726 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:41,780 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:43:41,780 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:43:41,896 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:43:41,898 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:43:41,898 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:43:41,898 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:43:41,898 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:43:41,898 DEBUG SenderThread:136 [sender.py:send():378] send: metric +2024-07-04 20:43:41,899 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:43:41,899 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:43:41,899 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:43:42,432 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:43:42,432 INFO Thread-12 :136 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/output.log +2024-07-04 20:43:44,433 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/output.log +2024-07-04 20:43:45,900 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:50,902 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:43:56,780 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:43:56,781 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:43:56,950 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:01,951 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:05,726 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:44:07,727 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:11,780 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:44:11,781 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:44:12,905 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:13,443 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/config.yaml +2024-07-04 20:44:18,034 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:23,035 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:26,823 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:44:26,824 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:44:28,961 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:33,962 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:35,728 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:44:39,729 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:41,823 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:44:41,824 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:44:44,916 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:49,916 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:54,918 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:44:56,824 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:44:56,824 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:45:00,899 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:05,727 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:45:06,728 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:11,922 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:45:11,922 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:45:12,009 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:17,010 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:22,011 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:27,105 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:45:27,105 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:45:27,253 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:32,254 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:35,728 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:45:37,729 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:42,231 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:45:42,231 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:45:43,322 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:48,323 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:53,324 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:45:57,232 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:45:57,232 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:45:58,389 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:03,390 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:05,729 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:46:08,730 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:12,231 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:46:12,232 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:46:14,313 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:19,315 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:24,316 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:27,231 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:46:27,232 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:46:29,345 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:34,346 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:35,735 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:46:39,736 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:42,378 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:46:42,379 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:46:45,529 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:50,530 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:55,531 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:46:57,515 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:46:57,515 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:47:00,647 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:05,648 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:05,731 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:47:10,732 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:12,515 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:47:12,515 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:47:16,625 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:21,626 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:26,627 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:27,515 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:47:27,515 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:47:32,599 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:35,733 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:47:37,734 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:42,664 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:47:42,664 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:47:42,773 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:47,774 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:52,775 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:47:57,725 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:47:57,726 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:47:57,883 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:02,884 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:05,733 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:48:08,734 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:12,664 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:48:12,664 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:48:13,793 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:18,794 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:23,795 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:27,664 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:48:27,664 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:48:28,806 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:32,303 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: partial_history +2024-07-04 20:48:32,304 DEBUG SenderThread:136 [sender.py:send():378] send: history +2024-07-04 20:48:32,305 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: summary_record +2024-07-04 20:48:32,305 INFO SenderThread:136 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end +2024-07-04 20:48:32,539 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/wandb-summary.json +2024-07-04 20:48:34,306 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:35,733 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:48:39,734 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:42,721 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:48:42,722 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:48:44,785 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:49,786 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:54,787 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:48:57,721 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:48:57,722 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:48:59,811 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:04,812 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:05,734 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:49:10,735 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:12,722 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:49:12,722 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:49:15,852 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:20,853 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:25,854 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:27,722 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:49:27,722 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:49:31,804 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:35,735 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:49:37,736 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:42,722 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:49:42,722 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:49:42,935 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:47,936 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:52,937 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:49:57,777 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:49:57,777 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:49:58,899 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:03,900 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:05,735 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:50:09,736 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:12,777 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:50:12,777 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:50:14,879 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:19,880 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:24,881 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:27,777 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:50:27,777 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:50:30,864 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:35,736 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:50:36,737 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:41,738 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:42,954 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:50:42,955 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:50:47,105 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:52,106 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:57,107 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:50:58,021 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:50:58,021 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:51:02,126 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:05,737 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:51:07,738 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:12,995 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:51:12,995 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:51:13,147 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:18,148 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:23,148 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:28,162 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:51:28,163 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:51:28,295 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:33,296 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:35,738 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:51:38,739 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:43,162 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:51:43,163 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:51:44,271 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:49,272 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:54,273 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:51:58,163 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:51:58,163 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:51:59,286 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:04,287 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:05,739 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:52:09,739 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:13,217 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:52:13,218 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:52:15,370 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:20,371 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:25,373 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:28,196 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:52:28,197 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status +2024-07-04 20:52:31,338 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:34,984 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: pause +2024-07-04 20:52:34,984 INFO HandlerThread:136 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-04 20:52:34,984 INFO HandlerThread:136 [system_monitor.py:finish():203] Stopping system monitor +2024-07-04 20:52:34,985 INFO HandlerThread:136 [interfaces.py:finish():200] Joined cpu monitor +2024-07-04 20:52:34,985 DEBUG SystemMonitor:136 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-04 20:52:34,985 DEBUG SystemMonitor:136 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-04 20:52:34,986 INFO HandlerThread:136 [interfaces.py:finish():200] Joined disk monitor +2024-07-04 20:52:34,992 INFO HandlerThread:136 [interfaces.py:finish():200] Joined gpu monitor +2024-07-04 20:52:34,992 INFO HandlerThread:136 [interfaces.py:finish():200] Joined memory monitor +2024-07-04 20:52:34,992 INFO HandlerThread:136 [interfaces.py:finish():200] Joined network monitor +2024-07-04 20:52:34,993 DEBUG SenderThread:136 [sender.py:send():378] send: stats +2024-07-04 20:52:35,214 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: resume +2024-07-04 20:52:35,214 INFO HandlerThread:136 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-04 20:52:35,214 INFO HandlerThread:136 [system_monitor.py:start():194] Starting system monitor +2024-07-04 20:52:35,215 INFO SystemMonitor:136 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-04 20:52:35,215 INFO SystemMonitor:136 [interfaces.py:start():188] Started cpu monitoring +2024-07-04 20:52:35,216 INFO SystemMonitor:136 [interfaces.py:start():188] Started disk monitoring +2024-07-04 20:52:35,216 INFO SystemMonitor:136 [interfaces.py:start():188] Started gpu monitoring +2024-07-04 20:52:35,218 INFO SystemMonitor:136 [interfaces.py:start():188] Started memory monitoring +2024-07-04 20:52:35,220 INFO SystemMonitor:136 [interfaces.py:start():188] Started network monitoring +2024-07-04 20:52:36,554 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:36,629 INFO Thread-12 :136 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240704_200304-v5ofm505/files/output.log +2024-07-04 20:52:41,555 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: status_report +2024-07-04 20:52:43,196 DEBUG HandlerThread:136 [handler.py:handle_request():158] handle_request: stop_status +2024-07-04 20:52:43,197 DEBUG SenderThread:136 [sender.py:send_request():405] send_request: stop_status