diff --git "a/wandb/run-20231119_155754-znnni89w/logs/debug-internal.log" "b/wandb/run-20231119_155754-znnni89w/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20231119_155754-znnni89w/logs/debug-internal.log" @@ -0,0 +1,3065 @@ +2023-11-19 15:57:54,807 INFO StreamThr :652 [internal.py:wandb_internal():86] W&B internal server running at pid: 652, started at: 2023-11-19 15:57:54.807238 +2023-11-19 15:57:54,808 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status +2023-11-19 15:57:54,814 INFO WriterThread:652 [datastore.py:open_for_write():85] open: /home/ubuntu/wandb/run-20231119_155754-znnni89w/run-znnni89w.wandb +2023-11-19 15:57:54,814 DEBUG SenderThread:652 [sender.py:send():380] send: header +2023-11-19 15:57:54,820 DEBUG SenderThread:652 [sender.py:send():380] send: run +2023-11-19 15:57:56,061 INFO SenderThread:652 [dir_watcher.py:__init__():211] watching files in: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files +2023-11-19 15:57:56,062 INFO SenderThread:652 [sender.py:_start_run_threads():1124] run started: znnni89w with start time 1700409474.807221 +2023-11-19 15:57:56,070 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: check_version +2023-11-19 15:57:56,071 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: check_version +2023-11-19 15:57:56,183 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: run_start +2023-11-19 15:57:56,209 DEBUG HandlerThread:652 [system_info.py:__init__():32] System info init +2023-11-19 15:57:56,209 DEBUG HandlerThread:652 [system_info.py:__init__():47] System info init done +2023-11-19 15:57:56,209 INFO HandlerThread:652 [system_monitor.py:start():194] Starting system monitor +2023-11-19 15:57:56,209 INFO SystemMonitor:652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-19 15:57:56,209 INFO HandlerThread:652 [system_monitor.py:probe():214] Collecting system info +2023-11-19 15:57:56,210 INFO SystemMonitor:652 [interfaces.py:start():190] Started cpu monitoring +2023-11-19 15:57:56,210 INFO SystemMonitor:652 [interfaces.py:start():190] Started disk monitoring +2023-11-19 15:57:56,211 INFO SystemMonitor:652 [interfaces.py:start():190] Started gpu monitoring +2023-11-19 15:57:56,212 INFO SystemMonitor:652 [interfaces.py:start():190] Started memory monitoring +2023-11-19 15:57:56,213 INFO SystemMonitor:652 [interfaces.py:start():190] Started network monitoring +2023-11-19 15:57:56,236 DEBUG HandlerThread:652 [system_info.py:probe():196] Probing system +2023-11-19 15:57:56,238 DEBUG HandlerThread:652 [gitlib.py:_init_repo():56] git repository is invalid +2023-11-19 15:57:56,238 DEBUG HandlerThread:652 [system_info.py:probe():244] Probing system done +2023-11-19 15:57:56,238 DEBUG HandlerThread:652 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1021-aws-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2023-11-19T15:57:56.237031', 'startedAt': '2023-11-19T15:57:54.802501', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'mistral-finetune-own-data.ipynb', 'codePathLocal': 'mistral-finetune-own-data.ipynb', 'root': '/home/ubuntu', 'host': 'verb-workspace', 'username': 'ubuntu', 'executable': '/home/ubuntu/.pyenv/versions/3.10.13/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2921.4885, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 3285.96, 'min': 0.0, 'max': 0.0}, {'current': 2799.998, 'min': 0.0, 'max': 0.0}, {'current': 2799.998, 'min': 0.0, 'max': 0.0}, {'current': 2799.998, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 116.25236511230469, 'used': 56.017494201660156}}, 'gpu': 'NVIDIA A10G', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A10G', 'memory_total': 24146608128}], 'memory': {'total': 15.44485092163086}} +2023-11-19 15:57:56,238 INFO HandlerThread:652 [system_monitor.py:probe():224] Finished collecting system info +2023-11-19 15:57:56,238 INFO HandlerThread:652 [system_monitor.py:probe():227] Publishing system info +2023-11-19 15:57:56,238 DEBUG HandlerThread:652 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-11-19 15:57:56,238 DEBUG HandlerThread:652 [system_info.py:_save_pip():68] Saving pip packages done +2023-11-19 15:57:56,239 INFO HandlerThread:652 [system_monitor.py:probe():229] Finished publishing system info +2023-11-19 15:57:56,241 DEBUG SenderThread:652 [sender.py:send():380] send: files +2023-11-19 15:57:56,241 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-metadata.json with policy now +2023-11-19 15:57:56,254 DEBUG SenderThread:652 [sender.py:send():380] send: telemetry +2023-11-19 15:57:56,261 DEBUG SenderThread:652 [sender.py:send():380] send: config +2023-11-19 15:57:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:57:56,305 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:57:56,305 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 15:57:56,305 DEBUG SenderThread:652 [sender.py:send():380] send: telemetry +2023-11-19 15:57:56,305 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 15:57:56,305 WARNING SenderThread:652 [sender.py:send_metric():1331] Seen metric with glob (shouldn't happen) +2023-11-19 15:57:56,305 DEBUG SenderThread:652 [sender.py:send():380] send: telemetry +2023-11-19 15:57:56,306 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:57:56,756 INFO wandb-upload_0:652 [upload_job.py:push():131] Uploaded file /tmp/tmp0mbmfolhwandb/0sr1vx9s-wandb-metadata.json +2023-11-19 15:57:57,063 INFO Thread-12 :652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-metadata.json +2023-11-19 15:57:57,064 INFO Thread-12 :652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/requirements.txt +2023-11-19 15:57:57,064 INFO Thread-12 :652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 15:57:57,310 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: pause +2023-11-19 15:57:57,310 INFO HandlerThread:652 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-11-19 15:57:57,310 INFO HandlerThread:652 [system_monitor.py:finish():203] Stopping system monitor +2023-11-19 15:57:57,311 DEBUG SystemMonitor:652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-19 15:57:57,311 INFO HandlerThread:652 [interfaces.py:finish():202] Joined cpu monitor +2023-11-19 15:57:57,311 DEBUG SystemMonitor:652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-19 15:57:57,311 INFO HandlerThread:652 [interfaces.py:finish():202] Joined disk monitor +2023-11-19 15:57:57,312 DEBUG SystemMonitor:652 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-19 15:57:57,318 INFO HandlerThread:652 [interfaces.py:finish():202] Joined gpu monitor +2023-11-19 15:57:57,319 INFO HandlerThread:652 [interfaces.py:finish():202] Joined memory monitor +2023-11-19 15:57:57,319 INFO HandlerThread:652 [interfaces.py:finish():202] Joined network monitor +2023-11-19 15:57:57,319 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 15:57:59,063 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 15:58:00,320 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:05,320 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:10,321 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:11,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:11,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:11,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:15,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:20,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:25,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:26,068 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml +2023-11-19 15:58:26,253 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:26,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:31,376 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:36,377 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:41,253 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:41,253 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:42,329 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:47,330 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:52,330 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:58:56,253 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:58:56,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:58:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:58:57,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:02,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:07,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:11,253 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:11,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:13,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:18,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:23,405 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:26,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:26,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:29,320 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:34,321 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:39,321 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:41,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:41,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:44,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:49,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:54,139 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: resume +2023-11-19 15:59:54,139 INFO HandlerThread:652 [handler.py:handle_request_resume():696] starting system metrics thread +2023-11-19 15:59:54,139 INFO HandlerThread:652 [system_monitor.py:start():194] Starting system monitor +2023-11-19 15:59:54,139 INFO SystemMonitor:652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-19 15:59:54,140 INFO SystemMonitor:652 [interfaces.py:start():190] Started cpu monitoring +2023-11-19 15:59:54,140 INFO SystemMonitor:652 [interfaces.py:start():190] Started disk monitoring +2023-11-19 15:59:54,141 INFO SystemMonitor:652 [interfaces.py:start():190] Started gpu monitoring +2023-11-19 15:59:54,142 INFO SystemMonitor:652 [interfaces.py:start():190] Started memory monitoring +2023-11-19 15:59:54,142 INFO SystemMonitor:652 [interfaces.py:start():190] Started network monitoring +2023-11-19 15:59:54,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 15:59:54,436 DEBUG SenderThread:652 [sender.py:send():380] send: config +2023-11-19 15:59:54,436 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 15:59:54,437 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 15:59:54,437 WARNING SenderThread:652 [sender.py:send_metric():1331] Seen metric with glob (shouldn't happen) +2023-11-19 15:59:55,083 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 15:59:56,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 15:59:56,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 15:59:56,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 15:59:59,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:00,084 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml +2023-11-19 16:00:04,491 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:09,492 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:11,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:11,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:15,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:20,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:25,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:26,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:26,254 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:30,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:35,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:40,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:41,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:41,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:46,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:50,048 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: pause +2023-11-19 16:00:50,048 INFO HandlerThread:652 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-11-19 16:00:50,048 INFO HandlerThread:652 [system_monitor.py:finish():203] Stopping system monitor +2023-11-19 16:00:50,049 DEBUG SystemMonitor:652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-19 16:00:50,049 DEBUG SystemMonitor:652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-19 16:00:50,049 DEBUG SystemMonitor:652 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-19 16:00:50,050 INFO HandlerThread:652 [interfaces.py:finish():202] Joined cpu monitor +2023-11-19 16:00:50,050 INFO HandlerThread:652 [interfaces.py:finish():202] Joined disk monitor +2023-11-19 16:00:50,055 INFO HandlerThread:652 [interfaces.py:finish():202] Joined gpu monitor +2023-11-19 16:00:50,055 INFO HandlerThread:652 [interfaces.py:finish():202] Joined memory monitor +2023-11-19 16:00:50,055 INFO HandlerThread:652 [interfaces.py:finish():202] Joined network monitor +2023-11-19 16:00:50,056 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:00:52,056 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:00:56,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:00:56,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:00:56,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:00:57,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:02,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:07,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:11,254 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:11,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:13,368 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:15,665 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: resume +2023-11-19 16:01:15,665 INFO HandlerThread:652 [handler.py:handle_request_resume():696] starting system metrics thread +2023-11-19 16:01:15,665 INFO HandlerThread:652 [system_monitor.py:start():194] Starting system monitor +2023-11-19 16:01:15,665 INFO SystemMonitor:652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-19 16:01:15,665 INFO SystemMonitor:652 [interfaces.py:start():190] Started cpu monitoring +2023-11-19 16:01:15,666 INFO SystemMonitor:652 [interfaces.py:start():190] Started disk monitoring +2023-11-19 16:01:15,667 INFO SystemMonitor:652 [interfaces.py:start():190] Started gpu monitoring +2023-11-19 16:01:15,667 INFO SystemMonitor:652 [interfaces.py:start():190] Started memory monitoring +2023-11-19 16:01:15,668 INFO SystemMonitor:652 [interfaces.py:start():190] Started network monitoring +2023-11-19 16:01:15,987 DEBUG SenderThread:652 [sender.py:send():380] send: config +2023-11-19 16:01:15,988 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:01:15,988 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:01:15,988 WARNING SenderThread:652 [sender.py:send_metric():1331] Seen metric with glob (shouldn't happen) +2023-11-19 16:01:18,989 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:23,989 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:26,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:26,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:26,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:29,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:34,405 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:35,106 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml +2023-11-19 16:01:39,509 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:41,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:41,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:01:41,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:45,374 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:50,374 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:55,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:01:56,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:01:56,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:01:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:01,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:06,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:11,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:11,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:11,395 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:15,668 DEBUG SystemMonitor:652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-19 16:02:15,669 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:02:16,669 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:21,670 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:26,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:26,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:27,383 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:32,384 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:35,367 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:02:35,369 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:02:35,369 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:02:35,369 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:02:35,370 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:02:35,370 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:02:35,371 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:02:36,119 INFO Thread-12 :652 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:02:38,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:39,119 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml +2023-11-19 16:02:41,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:41,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:02:41,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:44,399 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:45,669 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:02:49,670 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:54,671 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:02:56,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:02:56,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:02:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:00,309 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:05,310 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:10,311 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:11,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:11,255 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:15,319 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:15,670 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:03:20,670 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:25,671 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:26,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:26,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:31,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:36,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:41,255 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:41,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:42,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:45,670 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:03:47,671 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:52,671 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:03:56,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:03:56,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:03:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:03:58,333 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:03,334 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:08,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:11,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:11,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:13,406 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:15,671 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:04:18,672 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:23,672 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:26,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:26,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:29,364 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:34,365 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:39,366 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:41,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:41,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:04:44,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:45,671 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:04:49,672 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:54,673 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:04:56,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:04:56,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:04:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:00,312 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:05,313 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:10,313 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:11,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:11,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:15,397 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:15,672 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:05:20,673 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:25,673 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:26,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:26,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:31,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:36,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:41,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:41,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:41,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:45,672 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:05:46,673 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:51,673 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:05:56,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:05:56,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:05:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:05:57,409 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:02,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:07,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:11,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:11,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:13,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:15,673 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:06:17,366 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:06:17,367 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:06:17,367 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:06:17,367 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:06:17,367 DEBUG SenderThread:652 [sender.py:send():380] send: metric +2023-11-19 16:06:17,367 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:06:17,367 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:06:17,368 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:06:18,167 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:06:19,108 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:19,167 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:06:20,167 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml +2023-11-19 16:06:24,237 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:26,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:26,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:26,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:29,374 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:34,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:39,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:41,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:41,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:06:45,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:45,673 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:06:50,674 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:55,675 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:06:56,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:06:56,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:06:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:01,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:06,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:11,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:11,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:12,314 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:15,674 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:07:17,674 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:22,675 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:26,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:26,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:28,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:33,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:37,326 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:07:37,327 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:07:37,327 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:07:37,328 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:07:38,186 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:07:39,328 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:41,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:41,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:07:41,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:44,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:45,674 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:07:49,675 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:54,676 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:07:56,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:07:56,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:07:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:00,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:05,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:10,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:11,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:11,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:15,675 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:08:15,676 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:20,676 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:25,676 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:26,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:26,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:31,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:36,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:41,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:41,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:42,333 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:45,675 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:08:47,676 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:52,676 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:08:56,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:08:56,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:08:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:08:58,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:03,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:08,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:11,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:11,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:14,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:15,676 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:09:19,677 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:24,677 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:26,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:26,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:30,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:35,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:40,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:41,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:41,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:09:45,676 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:09:45,677 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:50,677 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:55,678 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:09:56,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:09:56,258 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:09:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:01,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:06,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:11,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:11,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:11,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:15,679 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:10:16,680 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:21,680 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:26,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:26,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:27,315 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:32,316 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:37,316 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:41,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:41,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:42,388 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:45,677 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:10:47,678 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:52,678 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:10:56,258 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:10:56,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:10:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:10:58,306 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:03,306 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:08,307 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:11,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:11,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:13,325 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:15,678 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:11:18,679 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:19,062 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:11:19,063 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:11:19,063 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:11:19,063 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:11:19,234 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:11:22,235 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:11:23,711 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:26,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:26,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:26,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:29,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:34,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:39,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:41,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:41,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:11:45,312 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:45,678 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:11:50,679 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:55,680 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:11:56,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:11:56,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:11:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:01,415 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:06,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:11,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:11,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:12,392 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:15,679 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:12:17,680 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:22,680 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:26,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:26,259 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:28,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:33,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:38,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:38,950 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:12:38,950 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:12:38,951 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:12:38,951 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:12:39,254 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:12:41,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:41,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:12:41,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:44,313 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:45,679 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:12:49,680 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:54,681 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:12:56,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:12:56,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:12:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:00,319 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:05,320 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:10,320 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:11,259 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:11,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:15,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:15,680 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:13:20,681 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:25,681 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:26,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:26,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:31,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:36,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:41,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:41,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:42,317 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:45,680 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:13:47,681 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:52,682 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:13:56,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:13:56,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:13:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:13:58,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:03,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:08,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:11,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:11,260 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:14,330 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:15,681 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:14:19,682 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:24,682 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:26,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:26,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:30,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:35,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:40,383 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:41,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:41,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:14:45,681 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:14:45,682 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:50,682 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:55,683 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:14:56,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:14:56,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:14:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:01,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:06,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:11,260 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:11,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:12,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:15,682 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:15:17,683 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:22,683 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:26,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:26,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:28,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:33,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:38,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:41,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:41,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:15:44,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:45,683 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:15:49,684 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:54,684 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:15:56,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:15:56,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:15:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:00,398 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:05,399 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:10,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:11,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:11,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:15,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:15,683 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:16:20,684 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:20,713 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:16:20,713 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:16:20,714 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:16:20,714 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:16:21,303 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:16:24,303 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:16:26,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:26,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:26,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:26,387 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:31,388 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:36,388 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:41,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:41,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:42,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:45,684 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:16:47,684 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:52,685 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:16:56,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:16:56,261 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:16:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:16:58,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:03,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:08,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:11,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:11,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:13,477 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:15,684 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:17:18,685 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:23,686 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:26,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:26,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:29,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:34,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:39,383 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:40,621 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:17:40,622 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:17:40,623 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:17:40,623 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:17:41,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:41,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:17:41,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:41,321 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:17:45,325 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:45,685 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:17:50,686 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:55,686 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:17:56,261 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:17:56,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:17:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:01,440 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:06,440 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:11,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:11,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:12,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:15,685 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:18:17,686 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:22,686 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:26,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:26,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:28,361 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:33,362 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:38,363 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:41,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:41,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:43,443 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:45,686 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:18:48,686 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:53,687 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:18:56,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:18:56,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:18:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:18:59,369 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:04,369 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:09,370 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:11,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:11,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:15,327 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:15,686 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:19:20,687 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:25,688 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:26,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:26,262 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:31,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:36,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:41,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:41,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:41,433 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:45,687 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:19:46,687 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:51,688 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:19:56,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:19:56,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:19:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:19:57,363 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:02,364 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:07,364 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:11,262 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:11,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:12,389 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:15,687 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:20:17,688 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:22,689 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:26,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:26,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:28,383 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:33,384 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:38,384 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:41,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:41,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:20:44,316 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:45,688 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:20:49,689 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:54,689 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:20:56,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:20:56,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:20:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:00,314 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:05,314 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:10,315 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:11,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:11,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:15,327 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:15,688 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:21:20,689 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:22,379 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:21:22,380 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:21:22,380 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:21:22,380 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:21:23,374 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:21:24,374 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:21:26,024 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:26,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:26,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:26,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:31,334 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:36,334 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:41,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:41,263 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:41,390 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:45,689 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:21:46,690 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:51,690 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:21:56,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:21:56,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:21:56,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:21:57,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:02,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:07,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:11,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:11,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:12,391 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:15,690 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:22:17,690 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:22,691 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:26,263 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:26,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:28,390 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:33,391 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:38,391 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:41,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:41,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:22:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:42,249 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:22:42,250 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:22:42,250 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:22:42,250 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:22:42,392 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:22:44,251 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:45,690 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:22:49,691 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:54,692 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:22:56,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:22:56,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:22:56,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:00,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:05,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:10,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:11,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:11,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:11,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:15,691 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:23:15,692 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:20,692 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:25,692 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:26,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:26,264 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:26,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:31,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:36,375 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:41,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:41,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:41,304 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:42,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:45,692 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:23:47,692 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:52,693 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:23:56,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:23:56,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:23:56,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:23:58,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:03,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:08,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:11,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:11,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:11,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:13,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:15,692 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:24:18,693 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:23,693 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:26,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:26,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:26,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:29,395 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:34,396 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:39,397 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:41,264 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:41,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:41,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:24:45,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:45,692 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:24:50,693 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:55,694 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:24:56,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:24:56,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:24:56,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:01,372 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:06,373 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:11,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:11,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:11,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:12,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:15,693 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:25:17,693 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:22,694 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:26,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:26,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:26,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:28,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:33,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:38,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:41,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:41,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:41,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:43,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:45,693 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:25:48,694 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:53,695 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:25:56,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:25:56,265 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:25:56,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:25:59,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:04,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:09,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:11,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:11,266 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:11,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:15,369 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:15,694 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:26:20,695 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:23,989 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:26:23,990 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:26:23,990 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:26:23,990 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:26:24,440 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:26:26,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:26,266 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:26,266 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:26,334 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:26,441 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:26:31,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:36,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:41,265 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:41,266 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:41,308 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:42,324 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:45,695 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:26:47,695 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:52,696 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:26:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:26:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:26:56,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:26:58,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:03,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:08,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:11,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:11,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:11,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:14,330 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:15,695 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:27:19,696 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:24,696 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:26,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:26,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:26,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:30,389 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:35,390 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:40,390 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:41,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:41,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:41,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:27:43,874 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:27:43,875 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:27:43,875 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:27:43,875 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:27:44,459 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:27:45,695 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:27:45,697 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:50,696 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:55,697 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:27:56,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:27:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:27:56,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:01,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:06,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:11,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:11,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:11,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:11,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:15,696 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:28:16,697 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:21,697 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:26,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:26,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:26,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:27,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:32,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:37,433 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:41,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:41,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:41,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:43,390 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:45,697 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:28:48,697 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:53,698 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:28:56,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:28:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:28:56,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:28:59,392 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:04,392 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:09,393 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:11,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:11,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:11,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:15,366 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:15,697 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:29:20,698 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:25,699 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:26,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:26,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:26,278 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:31,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:36,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:41,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:41,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:41,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:42,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:45,698 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:29:47,699 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:52,699 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:29:56,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:29:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:29:56,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:29:58,393 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:03,394 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:08,394 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:11,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:11,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:11,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:14,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:15,698 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:30:19,699 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:24,700 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:26,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:26,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:26,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:30,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:35,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:40,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:41,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:41,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:41,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:30:45,388 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:45,699 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:30:50,700 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:55,700 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:30:56,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:30:56,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:30:56,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:01,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:06,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:11,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:11,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:11,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:12,380 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:15,700 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:31:17,700 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:22,701 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:25,609 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:31:25,610 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:31:25,610 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:31:25,610 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:31:26,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:26,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:26,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:26,508 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:31:28,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:29,508 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:31:33,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:38,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:41,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:41,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:41,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:31:44,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:45,702 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:31:49,703 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:54,704 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:31:56,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:31:56,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:31:56,279 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:00,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:05,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:10,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:11,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:11,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:11,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:15,700 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:32:15,701 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:20,701 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:25,702 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:26,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:26,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:26,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:31,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:36,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:41,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:41,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:41,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:45,513 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:32:45,514 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:32:45,514 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:32:45,516 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:32:45,526 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:32:45,701 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:32:46,701 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:51,702 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:32:56,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:32:56,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:32:56,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:32:57,461 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:02,461 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:07,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:11,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:11,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:11,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:13,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:15,702 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:33:18,703 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:23,704 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:26,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:26,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:26,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:29,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:34,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:39,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:41,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:41,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:33:45,399 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:45,702 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:33:50,703 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:55,704 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:33:56,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:33:56,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:33:56,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:01,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:06,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:11,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:11,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:11,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:11,398 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:15,703 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:34:16,704 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:21,704 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:26,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:26,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:26,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:27,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:32,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:37,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:41,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:41,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:42,477 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:45,703 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:34:47,704 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:52,705 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:34:56,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:34:56,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:34:56,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:34:58,327 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:03,327 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:08,328 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:11,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:11,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:11,280 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:13,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:15,704 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:35:18,705 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:23,705 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:26,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:26,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:26,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:29,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:34,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:39,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:41,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:41,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:35:45,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:45,704 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:35:50,705 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:55,706 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:35:56,274 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:35:56,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:35:56,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:01,362 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:06,363 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:11,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:11,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:11,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:12,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:15,705 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:36:17,705 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:22,706 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:26,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:26,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:26,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:27,266 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:36:27,266 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:36:27,266 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:36:27,267 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:36:27,573 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:36:27,927 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:29,573 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:36:32,927 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:37,928 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:41,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:41,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:41,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:43,405 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:45,705 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:36:48,706 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:53,707 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:36:56,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:36:56,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:36:56,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:36:59,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:04,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:09,426 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:11,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:11,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:11,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:14,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:15,706 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:37:19,706 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:24,707 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:26,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:26,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:26,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:30,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:35,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:40,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:41,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:41,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:41,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:37:45,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:45,706 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:37:47,161 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:37:47,162 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:37:47,162 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:37:47,163 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:37:47,593 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:37:51,163 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:56,164 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:37:56,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:37:56,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:37:56,281 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:01,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:06,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:11,275 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:11,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:11,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:11,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:15,707 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:38:16,707 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:21,708 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:26,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:26,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:26,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:27,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:32,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:37,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:41,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:41,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:41,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:42,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:45,707 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:38:47,708 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:52,708 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:38:56,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:38:56,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:38:56,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:38:58,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:03,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:08,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:11,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:11,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:11,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:14,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:15,708 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:39:19,708 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:24,709 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:26,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:26,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:26,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:30,380 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:35,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:40,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:41,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:41,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:41,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:39:45,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:45,708 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:39:50,709 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:55,710 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:39:56,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:39:56,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:39:56,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:01,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:06,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:11,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:11,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:11,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:12,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:15,709 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:40:17,710 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:22,710 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:26,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:26,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:26,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:28,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:33,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:38,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:41,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:41,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:41,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:40:44,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:45,710 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:40:49,710 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:54,711 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:40:56,276 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:40:56,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:40:56,282 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:00,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:05,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:10,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:11,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:11,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:11,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:15,433 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:15,710 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:41:20,711 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:25,711 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:26,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:26,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:26,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:28,904 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:41:28,905 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:41:28,905 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:41:28,906 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:41:29,643 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:41:31,550 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:31,643 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:41:36,551 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:41,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:41,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:41,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:42,367 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:45,711 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:41:47,711 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:52,712 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:41:56,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:41:56,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:41:56,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:41:58,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:03,360 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:08,360 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:11,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:11,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:11,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:13,434 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:15,711 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:42:18,712 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:23,713 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:26,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:26,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:26,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:29,444 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:34,444 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:39,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:41,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:41,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:41,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:42:45,374 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:45,712 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:42:48,770 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:42:48,771 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:42:48,771 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:42:48,771 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:42:49,661 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:42:50,772 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:55,772 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:42:56,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:42:56,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:42:56,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:01,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:06,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:11,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:11,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:11,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:12,366 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:15,712 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:43:17,713 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:22,713 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:26,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:26,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:26,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:28,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:33,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:38,382 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:41,277 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:41,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:41,283 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:43,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:45,713 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:43:48,714 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:53,714 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:43:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:43:56,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:43:56,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:43:59,391 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:04,392 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:09,392 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:11,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:11,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:11,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:15,381 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:15,713 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:44:20,714 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:25,715 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:26,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:26,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:26,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:31,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:36,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:41,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:41,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:41,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:42,326 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:45,714 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:44:47,714 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:52,715 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:44:56,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:44:56,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:44:56,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:44:58,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:03,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:08,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:11,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:11,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:11,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:14,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:15,714 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:45:19,715 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:24,715 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:26,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:26,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:26,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:30,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:35,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:40,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:41,278 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:41,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:41,284 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:45:45,396 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:45,714 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:45:50,715 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:55,716 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:45:56,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:45:56,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:45:56,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:01,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:06,430 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:11,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:11,334 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:11,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:11,440 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:15,715 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:46:16,715 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:21,716 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:26,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:26,334 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:26,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:27,477 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:30,512 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:46:30,513 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:46:30,513 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:46:30,514 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:46:30,707 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:46:33,243 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:33,708 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:46:38,243 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:41,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:41,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:41,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:43,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:45,715 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:46:48,716 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:53,717 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:46:56,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:46:56,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:46:56,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:46:59,397 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:04,398 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:09,399 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:11,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:11,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:11,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:14,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:15,716 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:47:19,717 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:24,717 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:26,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:26,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:26,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:30,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:35,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:40,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:41,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:41,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:41,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:45,716 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:47:45,717 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:50,466 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:47:50,467 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:47:50,467 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:47:50,468 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:47:50,726 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:47:51,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:47:56,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:47:56,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:47:56,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:47:57,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:02,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:07,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:11,279 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:11,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:11,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:12,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:15,717 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:48:17,717 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:22,718 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:26,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:26,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:26,335 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:28,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:33,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:38,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:41,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:41,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:43,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:45,717 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:48:48,718 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:53,719 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:48:56,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:48:56,335 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:48:56,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:48:59,478 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:04,480 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:09,480 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:11,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:11,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:11,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:15,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:15,718 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:49:20,719 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:25,719 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:26,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:26,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:26,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:31,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:36,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:41,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:41,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:41,480 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:45,718 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:49:46,719 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:51,719 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:49:56,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:49:56,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:49:56,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:49:57,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:02,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:07,412 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:11,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:11,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:11,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:12,415 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:15,719 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:50:17,720 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:22,720 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:26,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:26,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:26,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:28,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:33,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:38,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:41,280 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:41,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:41,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:43,435 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:45,719 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:50:48,720 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:53,721 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:50:56,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:50:56,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:50:56,336 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:50:59,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:04,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:09,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:11,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:11,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:11,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:15,398 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:15,720 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:51:20,721 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:25,721 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:26,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:26,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:26,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:31,395 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:32,211 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:51:32,212 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:51:32,212 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:51:32,212 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:51:32,776 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:51:34,777 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:51:36,856 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:41,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:41,336 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:41,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:42,399 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:45,720 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:51:47,721 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:52,721 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:51:56,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:51:56,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:51:56,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:51:58,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:03,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:08,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:11,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:11,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:11,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:13,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:15,721 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:52:18,722 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:23,722 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:26,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:26,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:26,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:29,395 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:34,395 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:39,396 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:41,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:41,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:41,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:52:44,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:45,721 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:52:49,722 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:52,087 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:52:52,088 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:52:52,088 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:52:52,089 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:52:52,795 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:52:55,089 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:52:56,281 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:52:56,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:52:56,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:00,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:05,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:10,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:11,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:11,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:11,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:15,724 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:53:15,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:20,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:25,726 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:26,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:26,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:26,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:31,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:36,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:41,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:41,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:41,337 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:42,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:45,723 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:53:47,723 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:52,724 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:53:56,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:53:56,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:53:56,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:53:58,457 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:03,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:08,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:11,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:11,337 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:11,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:14,451 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:15,724 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:54:19,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:24,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:26,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:26,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:26,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:30,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:35,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:40,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:41,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:41,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:41,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:54:45,724 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:54:45,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:50,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:55,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:54:56,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:54:56,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:54:56,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:01,392 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:06,393 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:11,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:11,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:11,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:11,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:15,724 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:55:16,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:21,726 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:26,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:26,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:26,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:27,440 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:32,440 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:37,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:41,282 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:41,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:41,338 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:43,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:45,725 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:55:48,725 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:53,726 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:55:56,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:55:56,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:55:56,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:55:59,430 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:04,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:09,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:11,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:11,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:11,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:14,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:15,725 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:56:19,726 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:24,727 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:26,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:26,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:26,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:30,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:33,821 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:56:33,822 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:56:33,822 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:56:33,822 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:56:33,844 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:56:35,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:36,845 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 16:56:40,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:41,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:41,338 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:41,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:56:45,726 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:56:45,726 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:50,727 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:55,728 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:56:56,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:56:56,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:56:56,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:01,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:06,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:11,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:11,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:11,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:12,385 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:15,726 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:57:17,727 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:22,728 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:26,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:26,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:26,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:28,430 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:33,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:38,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:41,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:41,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:41,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:57:43,466 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:45,727 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:57:48,728 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:53,683 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 16:57:53,684 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 16:57:53,684 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 16:57:53,685 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 16:57:53,863 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 16:57:54,685 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:57:56,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:57:56,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:57:56,339 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:00,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:05,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:10,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:11,283 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:11,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:11,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:15,494 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:15,727 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:58:20,728 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:25,729 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:26,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:26,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:26,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:31,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:36,454 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:41,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:41,339 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:41,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:42,394 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:45,728 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:58:47,729 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:52,729 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:58:56,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:58:56,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:58:56,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:58:58,488 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:03,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:08,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:11,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:11,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:11,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:14,450 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:15,729 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:59:19,729 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:24,730 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:26,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:26,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:26,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:30,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:35,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:40,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:41,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:41,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:41,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 16:59:45,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:45,729 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 16:59:50,730 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:55,730 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 16:59:56,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 16:59:56,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 16:59:56,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:01,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:06,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:11,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:11,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:11,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:12,426 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:15,730 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:00:17,730 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:22,731 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:26,284 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:26,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:26,340 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:28,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:33,447 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:38,447 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:41,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:41,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:41,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:43,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:45,730 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:00:48,731 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:53,731 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:00:56,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:00:56,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:00:56,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:00:59,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:04,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:09,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:11,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:11,340 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:11,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:14,492 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:15,730 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:01:19,731 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:24,732 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:26,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:26,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:26,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:30,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:35,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:01:35,421 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 17:01:35,422 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:01:35,422 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:01:35,912 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 17:01:36,069 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:36,912 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 17:01:41,069 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:41,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:41,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:41,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:45,731 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:01:46,732 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:51,732 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:01:56,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:01:56,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:01:56,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:01:57,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:02,496 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:07,497 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:11,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:11,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:11,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:13,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:15,732 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:02:18,732 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:23,733 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:26,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:26,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:26,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:29,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:34,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:39,415 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:41,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:41,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:41,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:02:44,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:45,732 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:02:49,733 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:54,734 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:02:55,303 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: partial_history +2023-11-19 17:02:55,303 DEBUG SenderThread:652 [sender.py:send():380] send: history +2023-11-19 17:02:55,304 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: summary_record +2023-11-19 17:02:55,304 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:02:55,931 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 17:02:56,285 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:02:56,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:02:56,341 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:00,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:05,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:10,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:11,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:11,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:11,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:15,435 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:15,733 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:03:20,734 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:25,734 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:26,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:26,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:26,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:31,475 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:36,475 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:41,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:41,341 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:41,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:42,412 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:45,733 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:03:47,734 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:52,734 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:03:53,113 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: pause +2023-11-19 17:03:53,113 INFO HandlerThread:652 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-11-19 17:03:53,113 INFO HandlerThread:652 [system_monitor.py:finish():203] Stopping system monitor +2023-11-19 17:03:53,113 DEBUG SystemMonitor:652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-19 17:03:53,113 DEBUG SystemMonitor:652 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-19 17:03:53,114 INFO HandlerThread:652 [interfaces.py:finish():202] Joined cpu monitor +2023-11-19 17:03:53,114 INFO HandlerThread:652 [interfaces.py:finish():202] Joined disk monitor +2023-11-19 17:03:53,119 INFO HandlerThread:652 [interfaces.py:finish():202] Joined gpu monitor +2023-11-19 17:03:53,119 INFO HandlerThread:652 [interfaces.py:finish():202] Joined memory monitor +2023-11-19 17:03:53,120 INFO HandlerThread:652 [interfaces.py:finish():202] Joined network monitor +2023-11-19 17:03:53,120 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:03:56,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:03:56,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:03:56,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:03:58,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:03,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:08,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:11,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:11,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:11,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:13,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:18,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:23,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:26,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:26,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:26,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:28,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:33,488 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:38,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:41,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:41,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:41,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:43,503 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:48,503 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:53,504 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:04:56,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:04:56,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:04:56,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:04:59,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:04,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:09,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:11,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:11,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:11,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:15,461 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:20,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:25,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:26,286 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:26,342 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:26,342 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:31,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:36,454 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:41,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:41,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:41,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:41,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:46,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:51,488 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:05:56,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:05:56,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:05:56,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:05:57,449 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:02,449 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:07,450 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:11,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:11,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:11,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:12,479 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:17,479 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:22,480 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:26,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:26,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:26,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:27,493 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:32,493 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:37,494 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:41,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:41,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:41,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:43,455 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:48,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:53,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:06:56,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:06:56,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:06:56,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:06:59,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:04,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:09,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:11,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:11,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:11,343 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:15,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:20,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:25,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:26,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:26,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:26,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:30,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:35,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:40,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:41,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:41,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:41,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:46,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:51,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:07:56,287 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:07:56,343 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:07:56,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:07:56,482 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:01,483 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:06,483 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:11,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:11,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:11,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:12,430 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:17,430 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:22,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:26,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:26,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:26,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:27,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:32,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:37,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:41,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:41,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:41,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:43,435 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:48,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:53,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:08:56,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:08:56,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:08:56,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:08:58,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:03,460 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:08,460 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:11,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:11,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:11,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:13,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:18,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:23,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:26,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:26,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:26,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:29,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:34,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:39,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:41,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:41,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:41,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:09:44,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:49,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:54,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:09:56,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:09:56,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:09:56,344 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:00,443 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:05,444 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:10,444 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:11,288 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:11,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:11,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:15,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:20,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:25,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:26,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:26,344 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:26,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:31,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:36,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:41,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:41,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:41,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:41,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:46,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:51,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:10:56,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:10:56,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:10:56,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:10:57,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:02,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:07,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:11,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:11,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:11,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:13,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:18,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:23,404 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:26,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:26,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:26,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:29,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:34,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:39,403 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:41,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:41,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:41,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:44,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:49,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:54,440 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:11:56,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:11:56,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:11:56,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:11:59,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:04,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:09,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:11,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:11,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:11,345 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:14,467 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:19,467 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:24,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:26,289 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:26,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:26,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:29,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:34,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:39,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:41,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:41,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:41,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:12:45,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:50,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:55,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:12:56,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:12:56,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:12:56,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:01,492 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:06,493 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:11,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:11,345 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:11,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:12,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:17,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:22,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:26,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:26,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:26,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:27,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:32,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:37,470 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:41,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:41,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:41,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:42,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:47,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:52,488 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:13:56,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:13:56,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:13:56,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:13:58,482 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:03,483 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:08,484 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:11,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:11,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:11,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:14,476 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:19,476 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:24,477 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:26,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:26,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:26,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:29,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:34,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:39,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:41,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:41,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:41,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:14:45,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:50,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:55,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:14:56,290 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:14:56,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:14:56,346 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:01,396 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:06,396 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:11,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:11,346 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:11,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:11,497 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:16,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:21,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:26,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:26,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:26,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:27,396 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:32,397 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:37,398 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:41,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:41,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:41,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:42,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:47,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:52,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:15:56,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:15:56,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:15:56,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:15:58,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:03,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:08,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:11,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:11,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:11,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:13,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:18,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:23,470 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:26,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:26,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:26,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:28,494 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:33,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:38,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:41,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:41,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:41,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:44,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:49,491 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:54,491 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:16:56,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:16:56,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:16:56,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:16:59,530 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:04,530 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:09,531 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:11,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:11,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:11,347 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:15,465 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:20,465 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:25,466 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:26,291 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:26,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:26,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:31,450 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:36,451 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:41,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:41,347 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:41,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:41,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:46,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:51,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:17:56,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:17:56,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:17:56,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:17:57,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:02,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:07,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:11,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:11,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:11,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:13,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:18,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:23,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:26,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:26,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:26,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:28,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:33,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:38,465 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:41,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:41,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:41,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:43,497 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:48,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:53,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:18:56,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:18:56,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:18:56,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:18:59,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:04,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:09,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:11,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:11,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:11,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:15,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:20,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:25,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:26,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:26,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:26,348 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:30,494 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:35,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:40,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:41,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:41,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:41,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:19:45,501 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:50,501 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:55,502 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:19:56,292 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:19:56,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:19:56,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:01,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:06,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:11,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:11,348 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:11,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:11,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:16,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:21,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:26,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:26,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:26,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:27,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:32,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:37,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:41,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:41,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:41,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:42,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:47,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:52,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:20:56,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:20:56,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:20:56,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:20:58,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:03,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:08,412 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:11,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:11,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:11,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:13,433 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:18,433 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:23,434 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:26,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:26,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:26,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:29,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:34,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:39,414 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:41,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:41,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:41,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:21:44,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:49,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:54,443 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:21:56,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:21:56,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:21:56,349 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:00,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:05,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:10,402 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:11,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:11,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:11,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:15,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:20,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:25,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:26,293 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:26,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:26,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:30,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:35,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:40,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:41,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:41,349 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:41,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:22:45,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:50,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:55,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:22:56,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:22:56,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:22:56,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:01,486 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:06,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:11,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:11,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:11,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:12,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:17,458 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:22,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:26,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:26,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:26,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:28,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:33,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:38,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:41,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:41,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:41,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:43,470 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:48,471 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:53,471 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:23:56,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:23:56,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:23:56,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:23:59,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:04,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:09,465 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:11,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:11,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:11,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:15,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:20,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:25,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:26,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:26,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:26,350 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:30,478 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:35,478 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:40,479 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:41,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:41,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:41,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:46,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:51,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:24:56,294 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:24:56,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:24:56,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:24:57,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:02,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:07,443 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:11,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:11,350 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:11,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:13,415 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:18,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:23,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:26,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:26,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:26,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:28,450 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:33,451 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:38,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:41,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:41,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:41,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:43,482 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:48,483 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:53,483 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:25:56,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:25:56,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:25:56,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:25:58,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:03,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:08,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:11,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:11,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:11,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:14,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:19,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:24,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:26,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:26,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:26,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:30,444 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:35,444 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:40,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:41,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:41,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:41,351 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:26:45,454 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:50,455 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:55,455 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:26:56,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:26:56,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:26:56,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:00,471 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:05,471 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:10,472 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:11,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:11,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:11,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:16,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:21,432 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:26,295 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:26,351 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:26,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:26,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:31,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:36,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:41,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:41,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:42,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:47,443 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:52,443 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:27:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:27:56,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:27:56,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:27:57,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:02,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:07,447 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:11,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:11,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:13,406 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:18,406 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:23,406 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:26,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:26,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:28,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:33,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:38,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:41,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:41,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:43,477 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:48,478 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:53,478 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:28:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:28:56,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:28:56,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:28:59,455 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:04,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:09,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:11,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:11,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:11,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:15,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:20,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:25,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:26,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:26,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:26,352 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:31,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:36,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:41,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:41,352 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:41,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:41,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:46,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:51,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:29:56,296 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:29:56,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:29:56,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:29:56,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:01,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:06,501 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:11,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:11,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:11,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:12,426 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:17,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:22,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:26,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:26,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:26,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:28,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:33,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:38,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:41,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:41,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:41,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:43,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:48,465 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:53,465 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:30:56,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:30:56,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:30:56,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:30:59,445 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:04,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:09,446 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:11,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:11,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:11,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:15,426 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:20,426 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:25,427 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:26,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:26,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:26,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:30,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:35,457 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:40,457 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:41,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:41,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:41,353 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:31:45,491 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:50,491 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:55,492 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:31:56,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:31:56,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:31:56,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:01,468 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:06,469 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:11,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:11,353 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:11,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:12,460 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:17,461 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:22,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:26,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:26,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:26,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:27,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:32,496 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:37,496 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:41,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:41,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:41,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:43,428 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:48,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:53,429 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:32:56,297 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:32:56,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:32:56,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:32:58,472 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:03,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:08,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:11,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:11,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:11,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:14,447 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:19,447 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:24,448 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:26,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:26,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:26,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:29,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:34,499 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:39,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:41,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:41,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:41,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:33:45,493 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:50,493 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:55,494 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:33:56,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:33:56,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:33:56,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:00,518 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:05,519 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:10,519 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:11,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:11,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:11,354 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:16,496 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:21,496 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:26,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:26,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:26,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:27,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:32,400 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:37,401 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:41,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:41,354 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:41,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:42,461 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:47,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:52,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:34:56,298 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:34:56,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:34:56,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:34:58,455 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:03,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:08,456 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:11,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:11,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:11,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:13,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:18,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:23,460 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:26,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:26,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:26,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:29,459 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:34,460 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:39,461 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:41,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:41,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:41,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:44,506 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:49,507 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:54,507 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:35:56,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:35:56,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:35:56,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:35:59,508 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:04,509 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:09,509 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:11,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:11,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:11,355 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:15,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:20,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:25,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:26,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:26,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:26,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:30,501 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:35,502 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:40,502 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:41,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:41,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:41,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:46,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:51,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:36:56,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:36:56,355 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:36:56,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:36:56,484 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:01,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:06,485 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:11,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:11,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:11,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:12,423 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:17,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:22,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:26,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:26,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:26,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:27,508 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:32,509 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:37,509 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:41,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:41,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:41,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:43,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:48,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:53,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:37:56,299 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:37:56,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:37:56,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:37:59,420 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:04,421 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:09,422 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:11,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:11,356 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:14,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:19,490 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:24,491 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:26,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:26,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:30,487 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:35,488 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:40,489 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:41,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:41,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:46,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:51,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:38:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:38:56,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:38:56,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:38:57,416 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:02,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:07,417 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:11,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:11,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:11,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:13,412 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:18,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:23,413 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:26,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:26,356 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:26,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:29,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:34,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:39,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:41,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:41,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:41,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:44,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:49,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:54,425 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:39:56,300 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:39:56,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:39:56,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:39:59,435 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:04,436 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:09,437 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:11,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:11,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:11,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:15,430 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:20,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:25,431 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:26,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:26,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:26,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:30,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:35,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:40,464 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:41,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:41,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:41,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:40:45,497 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:50,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:55,498 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:40:56,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:40:56,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:40:56,357 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:01,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:06,424 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:11,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:11,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:11,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:12,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:17,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:22,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:26,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:26,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:26,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:27,466 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:32,466 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:37,467 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:41,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:41,357 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:41,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:42,512 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:47,513 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:52,514 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:41:56,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:41:56,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:41:56,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:41:58,411 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:03,412 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:08,412 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:11,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:11,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:11,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:13,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:18,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:23,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:26,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:26,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:26,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:28,502 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:33,503 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:38,504 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:41,301 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:41,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:41,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:42:44,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:49,462 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:54,463 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:42:56,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:42:56,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:42:56,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:00,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:05,452 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:10,453 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:11,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:11,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:11,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:15,509 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:20,510 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:25,510 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:26,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:26,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:26,358 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:31,495 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:36,496 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:41,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:41,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:41,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:42,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:47,438 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:52,439 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:43:56,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:43:56,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:43:56,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:43:58,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:03,407 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:08,408 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:11,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:11,358 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:11,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:13,418 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:18,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:23,419 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:26,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:26,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:26,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:28,481 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:33,481 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:38,482 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:41,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:41,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:41,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:43,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:48,500 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:53,501 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:44:56,302 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:44:56,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:44:56,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:44:59,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:04,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:09,474 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:11,303 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:11,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:11,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:15,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:20,441 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:25,442 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:26,303 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:26,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:26,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:28,694 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: resume +2023-11-19 17:45:28,694 INFO HandlerThread:652 [handler.py:handle_request_resume():696] starting system metrics thread +2023-11-19 17:45:28,694 INFO HandlerThread:652 [system_monitor.py:start():194] Starting system monitor +2023-11-19 17:45:28,694 INFO SystemMonitor:652 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-11-19 17:45:28,694 INFO SystemMonitor:652 [interfaces.py:start():190] Started cpu monitoring +2023-11-19 17:45:28,695 INFO SystemMonitor:652 [interfaces.py:start():190] Started disk monitoring +2023-11-19 17:45:28,696 INFO SystemMonitor:652 [interfaces.py:start():190] Started gpu monitoring +2023-11-19 17:45:28,697 INFO SystemMonitor:652 [interfaces.py:start():190] Started memory monitoring +2023-11-19 17:45:28,698 INFO SystemMonitor:652 [interfaces.py:start():190] Started network monitoring +2023-11-19 17:45:29,460 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 17:45:30,700 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:30,948 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: pause +2023-11-19 17:45:30,949 INFO HandlerThread:652 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-11-19 17:45:30,949 INFO HandlerThread:652 [system_monitor.py:finish():203] Stopping system monitor +2023-11-19 17:45:30,949 DEBUG SystemMonitor:652 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-11-19 17:45:30,949 DEBUG SystemMonitor:652 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-11-19 17:45:30,949 DEBUG SystemMonitor:652 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-11-19 17:45:30,950 INFO HandlerThread:652 [interfaces.py:finish():202] Joined cpu monitor +2023-11-19 17:45:30,950 INFO HandlerThread:652 [interfaces.py:finish():202] Joined disk monitor +2023-11-19 17:45:30,955 INFO HandlerThread:652 [interfaces.py:finish():202] Joined gpu monitor +2023-11-19 17:45:30,955 INFO HandlerThread:652 [interfaces.py:finish():202] Joined memory monitor +2023-11-19 17:45:30,955 INFO HandlerThread:652 [interfaces.py:finish():202] Joined network monitor +2023-11-19 17:45:30,955 DEBUG SenderThread:652 [sender.py:send():380] send: stats +2023-11-19 17:45:35,956 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:40,957 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:41,303 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:41,359 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: stop_status +2023-11-19 17:45:41,359 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: stop_status +2023-11-19 17:45:46,409 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:51,410 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:52,269 DEBUG SenderThread:652 [sender.py:send():380] send: exit +2023-11-19 17:45:52,269 INFO SenderThread:652 [sender.py:send_exit():585] handling exit code: 0 +2023-11-19 17:45:52,269 INFO SenderThread:652 [sender.py:send_exit():587] handling runtime: 3816 +2023-11-19 17:45:52,269 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:45:52,270 INFO SenderThread:652 [sender.py:send_exit():593] send defer +2023-11-19 17:45:52,270 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,270 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 0 +2023-11-19 17:45:52,270 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,270 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 0 +2023-11-19 17:45:52,270 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 1 +2023-11-19 17:45:52,270 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,270 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 1 +2023-11-19 17:45:52,270 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,270 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 1 +2023-11-19 17:45:52,271 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 2 +2023-11-19 17:45:52,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,271 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 2 +2023-11-19 17:45:52,271 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,271 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 2 +2023-11-19 17:45:52,271 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 3 +2023-11-19 17:45:52,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,271 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 3 +2023-11-19 17:45:52,271 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,271 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 3 +2023-11-19 17:45:52,271 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 4 +2023-11-19 17:45:52,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,271 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 4 +2023-11-19 17:45:52,272 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,272 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 4 +2023-11-19 17:45:52,272 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 5 +2023-11-19 17:45:52,272 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,272 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 5 +2023-11-19 17:45:52,272 DEBUG SenderThread:652 [sender.py:send():380] send: summary +2023-11-19 17:45:52,272 INFO SenderThread:652 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-19 17:45:52,273 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,273 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 5 +2023-11-19 17:45:52,273 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 6 +2023-11-19 17:45:52,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,273 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 6 +2023-11-19 17:45:52,273 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,273 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 6 +2023-11-19 17:45:52,273 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 7 +2023-11-19 17:45:52,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:45:52,273 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:52,273 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 7 +2023-11-19 17:45:52,274 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:52,274 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 7 +2023-11-19 17:45:52,465 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 17:45:53,255 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 8 +2023-11-19 17:45:53,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:53,256 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 8 +2023-11-19 17:45:53,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:53,256 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 8 +2023-11-19 17:45:53,256 INFO SenderThread:652 [job_builder.py:build():298] Attempting to build job artifact +2023-11-19 17:45:53,257 INFO SenderThread:652 [job_builder.py:_get_source_type():439] no source found +2023-11-19 17:45:53,257 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 9 +2023-11-19 17:45:53,257 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:53,257 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 9 +2023-11-19 17:45:53,257 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:53,258 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 9 +2023-11-19 17:45:53,258 INFO SenderThread:652 [dir_watcher.py:finish():358] shutting down directory watcher +2023-11-19 17:45:53,269 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-19 17:45:53,465 INFO Thread-12 :652 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 17:45:53,465 INFO SenderThread:652 [dir_watcher.py:finish():388] scan: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files +2023-11-19 17:45:53,465 INFO SenderThread:652 [dir_watcher.py:finish():402] scan save: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-metadata.json wandb-metadata.json +2023-11-19 17:45:53,465 INFO SenderThread:652 [dir_watcher.py:finish():402] scan save: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/requirements.txt requirements.txt +2023-11-19 17:45:53,465 INFO SenderThread:652 [dir_watcher.py:finish():402] scan save: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml config.yaml +2023-11-19 17:45:53,466 INFO SenderThread:652 [dir_watcher.py:finish():402] scan save: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log output.log +2023-11-19 17:45:53,471 INFO SenderThread:652 [dir_watcher.py:finish():402] scan save: /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json wandb-summary.json +2023-11-19 17:45:53,473 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 10 +2023-11-19 17:45:53,473 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: poll_exit +2023-11-19 17:45:53,473 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:53,475 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 10 +2023-11-19 17:45:53,475 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:53,475 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 10 +2023-11-19 17:45:53,476 INFO SenderThread:652 [file_pusher.py:finish():175] shutting down file pusher +2023-11-19 17:45:54,270 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-19 17:45:54,271 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: poll_exit +2023-11-19 17:45:54,592 INFO wandb-upload_0:652 [upload_job.py:push():131] Uploaded file /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/requirements.txt +2023-11-19 17:45:54,892 INFO wandb-upload_2:652 [upload_job.py:push():131] Uploaded file /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/output.log +2023-11-19 17:45:54,893 INFO wandb-upload_3:652 [upload_job.py:push():131] Uploaded file /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/wandb-summary.json +2023-11-19 17:45:54,893 INFO wandb-upload_1:652 [upload_job.py:push():131] Uploaded file /home/ubuntu/wandb/run-20231119_155754-znnni89w/files/config.yaml +2023-11-19 17:45:55,094 INFO Thread-11 (_thread_body):652 [sender.py:transition_state():613] send defer: 11 +2023-11-19 17:45:55,094 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:55,094 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 11 +2023-11-19 17:45:55,095 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:55,095 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 11 +2023-11-19 17:45:55,095 INFO SenderThread:652 [file_pusher.py:join():181] waiting for file pusher +2023-11-19 17:45:55,095 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 12 +2023-11-19 17:45:55,095 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:55,095 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 12 +2023-11-19 17:45:55,095 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:55,095 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 12 +2023-11-19 17:45:55,095 INFO SenderThread:652 [file_stream.py:finish():595] file stream finish called +2023-11-19 17:45:55,271 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-19 17:45:56,211 INFO SenderThread:652 [file_stream.py:finish():599] file stream finish is done +2023-11-19 17:45:56,212 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 13 +2023-11-19 17:45:56,212 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: poll_exit +2023-11-19 17:45:56,212 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:56,212 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 13 +2023-11-19 17:45:56,212 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:56,212 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 13 +2023-11-19 17:45:56,212 INFO SenderThread:652 [sender.py:transition_state():613] send defer: 14 +2023-11-19 17:45:56,213 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: defer +2023-11-19 17:45:56,213 DEBUG SenderThread:652 [sender.py:send():380] send: final +2023-11-19 17:45:56,213 INFO HandlerThread:652 [handler.py:handle_request_defer():172] handle defer: 14 +2023-11-19 17:45:56,213 DEBUG SenderThread:652 [sender.py:send():380] send: footer +2023-11-19 17:45:56,213 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: defer +2023-11-19 17:45:56,213 INFO SenderThread:652 [sender.py:send_request_defer():609] handle sender defer: 14 +2023-11-19 17:45:56,214 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-19 17:45:56,214 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: poll_exit +2023-11-19 17:45:56,214 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: poll_exit +2023-11-19 17:45:56,214 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: poll_exit +2023-11-19 17:45:56,215 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: server_info +2023-11-19 17:45:56,215 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: server_info +2023-11-19 17:45:56,217 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: get_summary +2023-11-19 17:45:56,217 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: sampled_history +2023-11-19 17:45:56,218 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: internal_messages +2023-11-19 17:45:56,218 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: job_info +2023-11-19 17:45:56,256 DEBUG SenderThread:652 [sender.py:send_request():407] send_request: job_info +2023-11-19 17:45:56,256 INFO MainThread:652 [wandb_run.py:_footer_history_summary_info():3760] rendering history +2023-11-19 17:45:56,256 INFO MainThread:652 [wandb_run.py:_footer_history_summary_info():3792] rendering summary +2023-11-19 17:46:02,256 DEBUG HandlerThread:652 [handler.py:handle_request():146] handle_request: status_report +2023-11-19 17:46:02,437 WARNING StreamThr :652 [internal.py:is_dead():414] Internal process exiting, parent pid 248 disappeared +2023-11-19 17:46:02,438 ERROR StreamThr :652 [internal.py:wandb_internal():152] Internal process shutdown. +2023-11-19 17:46:03,257 INFO SenderThread:652 [sender.py:finish():1536] shutting down sender +2023-11-19 17:46:03,257 INFO SenderThread:652 [file_pusher.py:finish():175] shutting down file pusher +2023-11-19 17:46:03,257 INFO SenderThread:652 [file_pusher.py:join():181] waiting for file pusher +2023-11-19 17:46:03,257 INFO HandlerThread:652 [handler.py:finish():866] shutting down handler +2023-11-19 17:46:03,257 INFO WriterThread:652 [datastore.py:close():294] close: /home/ubuntu/wandb/run-20231119_155754-znnni89w/run-znnni89w.wandb