Training in progress, step 500

config.json

@@ -9,14 +9,14 @@
   "eos_token_id": 0,
   "hidden_act": "gelu",
   "hidden_dropout": 0.0,
-  "hidden_size": 1024,
+  "hidden_size": 128,
   "initializer_range": 0.02,
-  "intermediate_size": 4096,
+  "intermediate_size": 512,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 2048,
   "model_type": "gpt_neox",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
   "partial_rotary_factor": 0.25,
   "rope_scaling": null,
   "rope_theta": 10000,

dap_multiplexer.2809d093debe.root.log.INFO.20250506-220315.133

@@ -0,0 +1,22 @@
+Log file created at: 2025/05/06 22:03:15
+Running on machine: 2809d093debe
+Binary: Built on Dec 31 1969 16:00:00 -0800 (0)
+Binary: Built at redacted@redacted:.
+Binary: Built with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+Previous log: <none>
+Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
+I0506 22:03:15.564100       1 log_spam.go:43] Process id 133
+I0506 22:03:15.564506       1 log_spam.go:47] Current working directory /
+I0506 22:03:15.564513       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
+I0506 22:03:15.564537       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
+ at redacted@redacted:.
+ as //research/colab/dap_multiplexer:dap_multiplexer
+ with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+ from changelist 0 in a unknown client based on redacted
+Build tool: unknown
+Build target: //research/colab/dap_multiplexer:dap_multiplexer
+Build id: unknown
+Built with PGO profile: unknown
+I0506 22:03:15.564543       1 log_spam.go:51] Command line arguments:
+I0506 22:03:15.564546       1 log_spam.go:53]  argv[0]: '/usr/local/bin/dap_multiplexer'
+I0506 22:03:15.564548       1 log_spam.go:53]  argv[1]: '--domain_socket_path=/tmp/debugger_2nmn5rw6k1'

dap_multiplexer.INFO

@@ -1,14 +1,14 @@
-Log file created at: 2025/05/06 18:08:35
-Running on machine: b07af655c6a1
+Log file created at: 2025/05/06 22:03:15
+Running on machine: 2809d093debe
 Binary: Built on Dec 31 1969 16:00:00 -0800 (0)
 Binary: Built at redacted@redacted:.
 Binary: Built with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
 Previous log: <none>
 Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
-I0506 18:08:35.903164       1 log_spam.go:43] Process id 137
-I0506 18:08:35.903840       1 log_spam.go:47] Current working directory /
-I0506 18:08:35.903848       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
-I0506 18:08:35.903913       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
+I0506 22:03:15.564100       1 log_spam.go:43] Process id 133
+I0506 22:03:15.564506       1 log_spam.go:47] Current working directory /
+I0506 22:03:15.564513       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
+I0506 22:03:15.564537       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
  at redacted@redacted:.
  as //research/colab/dap_multiplexer:dap_multiplexer
  with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
@@ -17,6 +17,6 @@ Build tool: unknown
 Build target: //research/colab/dap_multiplexer:dap_multiplexer
 Build id: unknown
 Built with PGO profile: unknown
-I0506 18:08:35.903922       1 log_spam.go:51] Command line arguments:
-I0506 18:08:35.903927       1 log_spam.go:53]  argv[0]: '/usr/local/bin/dap_multiplexer'
-I0506 18:08:35.903931       1 log_spam.go:53]  argv[1]: '--domain_socket_path=/tmp/debugger_1ihn10rabp'
+I0506 22:03:15.564543       1 log_spam.go:51] Command line arguments:
+I0506 22:03:15.564546       1 log_spam.go:53]  argv[0]: '/usr/local/bin/dap_multiplexer'
+I0506 22:03:15.564548       1 log_spam.go:53]  argv[1]: '--domain_socket_path=/tmp/debugger_2nmn5rw6k1'

language_service.2809d093debe.root.log.INFO.20250506-220346.370

@@ -0,0 +1,48 @@
+Log file created at: 2025/05/06 22:03:46
+Running on machine: 2809d093debe
+Binary: Built on Dec 31 1969 16:00:00 -0800 (0)
+Binary: Built at redacted@redacted:.
+Binary: Built with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+Previous log: <none>
+Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
+I0506 22:03:46.934232       1 log_spam.go:43] Process id 370
+I0506 22:03:46.934676       1 log_spam.go:47] Current working directory /datalab/web
+I0506 22:03:46.934687       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
+I0506 22:03:46.934756       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
+ at redacted@redacted:.
+ as //research/colab/datalab/language_service:language_service
+ with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+ from changelist 0 in a unknown client based on redacted
+Build tool: unknown
+Build target: //research/colab/datalab/language_service:language_service
+Build id: unknown
+Built with PGO profile: unknown
+I0506 22:03:46.934768       1 log_spam.go:51] Command line arguments:
+I0506 22:03:46.934778       1 log_spam.go:53]  argv[0]: '/usr/colab/bin/language_service'
+I0506 22:03:46.934782       1 log_spam.go:53]  argv[1]: '--lsp_search_dirs=/datalab/web/pyright/typeshed-fallback/stdlib,/usr/local/lib/python3.10/dist-packages'
+I0506 22:03:46.934784       1 log_spam.go:53]  argv[2]: '--language_services_request_root_url=http://172.28.0.1:8013/'
+I0506 22:03:46.934787       1 log_spam.go:53]  argv[3]: '--language_services_request_timeout=30s'
+I0506 22:03:46.934789       1 log_spam.go:53]  argv[4]: '--'
+I0506 22:03:46.934791       1 log_spam.go:53]  argv[5]: 'node'
+I0506 22:03:46.934793       1 log_spam.go:53]  argv[6]: '/datalab/web/pyright/pyright-langserver.js'
+I0506 22:03:46.934795       1 log_spam.go:53]  argv[7]: '--stdio'
+I0506 22:03:46.934797       1 log_spam.go:53]  argv[8]: '--cancellationReceive=file:ab4e7a9cbedef186eb18f9686827b88b5b51e9fd97'
+I0506 22:03:59.772284      29 ls.go:1241] cancel was requested for: 16, needs to be canceled: false
+I0506 22:04:42.864484      29 ls.go:1241] cancel was requested for: 45, needs to be canceled: false
+I0506 22:04:45.317416      29 ls.go:1241] cancel was requested for: 48, needs to be canceled: false
+I0506 22:04:51.849654      29 ls.go:1241] cancel was requested for: 57, needs to be canceled: false
+I0506 22:04:52.100740      29 ls.go:1241] cancel was requested for: 60, needs to be canceled: false
+I0506 22:04:52.865330      29 ls.go:1241] cancel was requested for: 67, needs to be canceled: false
+I0506 22:04:53.314938      29 ls.go:1241] cancel was requested for: 70, needs to be canceled: false
+I0506 22:04:54.869203      29 ls.go:1241] cancel was requested for: 78, needs to be canceled: false
+I0506 22:04:57.912455      29 ls.go:1241] cancel was requested for: 80, needs to be canceled: false
+I0506 22:05:00.408619      29 ls.go:1241] cancel was requested for: 87, needs to be canceled: false
+I0506 22:05:00.417580      29 ls.go:1241] cancel was requested for: 88, needs to be canceled: false
+I0506 22:05:10.262280      29 ls.go:1241] cancel was requested for: 96, needs to be canceled: false
+I0506 22:05:10.522558      29 ls.go:1241] cancel was requested for: 95, needs to be canceled: false
+I0506 22:05:11.170890      29 ls.go:1241] cancel was requested for: 97, needs to be canceled: false
+I0506 22:05:12.690277      29 ls.go:1241] cancel was requested for: 101, needs to be canceled: false
+I0506 22:05:12.696166      29 ls.go:1241] cancel was requested for: 102, needs to be canceled: false
+I0506 22:05:12.696371      29 ls.go:1241] cancel was requested for: 103, needs to be canceled: false
+I0506 22:05:13.468052      29 ls.go:1241] cancel was requested for: 110, needs to be canceled: false
+I0506 22:05:13.474022      29 ls.go:1241] cancel was requested for: 111, needs to be canceled: false

language_service.2809d093debe.root.log.INFO.20250506-220534.897

@@ -0,0 +1,7 @@
+Log file created at: 2025/05/06 22:05:34
+Running on machine: 2809d093debe
+Binary: Built on Dec 31 1969 16:00:00 -0800 (0)
+Binary: Built at redacted@redacted:.
+Binary: Built with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+Previous log: <none>
+Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg

language_service.2809d093debe.root.log.INFO.20250506-220535.922

@@ -0,0 +1,41 @@
+Log file created at: 2025/05/06 22:05:35
+Running on machine: 2809d093debe
+Binary: Built on Dec 31 1969 16:00:00 -0800 (0)
+Binary: Built at redacted@redacted:.
+Binary: Built with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+Previous log: <none>
+Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
+I0506 22:05:35.933100       1 log_spam.go:43] Process id 922
+I0506 22:05:35.933378       1 log_spam.go:47] Current working directory /datalab/web
+I0506 22:05:35.933386       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
+I0506 22:05:35.933447       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
+ at redacted@redacted:.
+ as //research/colab/datalab/language_service:language_service
+ with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
+ from changelist 0 in a unknown client based on redacted
+Build tool: unknown
+Build target: //research/colab/datalab/language_service:language_service
+Build id: unknown
+Built with PGO profile: unknown
+I0506 22:05:35.933452       1 log_spam.go:51] Command line arguments:
+I0506 22:05:35.933454       1 log_spam.go:53]  argv[0]: '/usr/colab/bin/language_service'
+I0506 22:05:35.933457       1 log_spam.go:53]  argv[1]: '--lsp_search_dirs=/datalab/web/pyright/typeshed-fallback/stdlib,/usr/local/lib/python3.10/dist-packages'
+I0506 22:05:35.933459       1 log_spam.go:53]  argv[2]: '--language_services_request_root_url=http://172.28.0.1:8013/'
+I0506 22:05:35.933460       1 log_spam.go:53]  argv[3]: '--language_services_request_timeout=30s'
+I0506 22:05:35.933462       1 log_spam.go:53]  argv[4]: '--'
+I0506 22:05:35.933463       1 log_spam.go:53]  argv[5]: 'node'
+I0506 22:05:35.933465       1 log_spam.go:53]  argv[6]: '/datalab/web/pyright/pyright-langserver.js'
+I0506 22:05:35.933466       1 log_spam.go:53]  argv[7]: '--stdio'
+I0506 22:05:35.933471       1 log_spam.go:53]  argv[8]: '--cancellationReceive=file:7e3261f6b4c29b135e4abbae7651fb35f873820f17'
+I0506 22:06:06.691284      16 ls.go:1241] cancel was requested for: 20, needs to be canceled: false
+I0506 22:06:13.663339      16 ls.go:1241] cancel was requested for: 28, needs to be canceled: false
+I0506 22:06:18.962970      16 ls.go:1241] cancel was requested for: 35, needs to be canceled: false
+I0506 22:06:20.104719      16 ls.go:1241] cancel was requested for: 36, needs to be canceled: false
+I0506 22:06:20.807271      16 ls.go:1241] cancel was requested for: 39, needs to be canceled: false
+I0506 22:06:25.022875      16 ls.go:1241] cancel was requested for: 52, needs to be canceled: false
+I0506 22:06:36.708695      16 ls.go:1241] cancel was requested for: 55, needs to be canceled: false
+I0506 22:06:49.697244      16 ls.go:1241] cancel was requested for: 59, needs to be canceled: false
+I0506 22:06:56.794684      16 ls.go:1241] cancel was requested for: 61, needs to be canceled: false
+I0506 22:07:05.137377      16 ls.go:1241] cancel was requested for: 70, needs to be canceled: false
+I0506 22:07:05.143186      16 ls.go:1241] cancel was requested for: 71, needs to be canceled: false
+I0506 22:21:37.746597      16 ls.go:1241] cancel was requested for: 76, needs to be canceled: false

language_service.INFO

@@ -1,14 +1,14 @@
-Log file created at: 2025/05/06 18:19:41
-Running on machine: b07af655c6a1
+Log file created at: 2025/05/06 22:05:35
+Running on machine: 2809d093debe
 Binary: Built on Dec 31 1969 16:00:00 -0800 (0)
 Binary: Built at redacted@redacted:.
 Binary: Built with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
 Previous log: <none>
 Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
-I0506 18:19:41.717097       1 log_spam.go:43] Process id 3031
-I0506 18:19:41.717582       1 log_spam.go:47] Current working directory /datalab/web
-I0506 18:19:41.717595       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
-I0506 18:19:41.717645       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
+I0506 22:05:35.933100       1 log_spam.go:43] Process id 922
+I0506 22:05:35.933378       1 log_spam.go:47] Current working directory /datalab/web
+I0506 22:05:35.933386       1 log_spam.go:49] Current timezone is UTC (currently UTC +00:00)
+I0506 22:05:35.933447       1 log_spam.go:50] Built on Dec 31 1969 16:00:00 -0800 (0)
  at redacted@redacted:.
  as //research/colab/datalab/language_service:language_service
  with gc go1.25-20250420-RC00 cl/749505989 +c893e1cf82 X:fieldtrack,boringcrypto for linux/amd64
@@ -17,42 +17,25 @@ Build tool: unknown
 Build target: //research/colab/datalab/language_service:language_service
 Build id: unknown
 Built with PGO profile: unknown
-I0506 18:19:41.717652       1 log_spam.go:51] Command line arguments:
-I0506 18:19:41.717656       1 log_spam.go:53]  argv[0]: '/usr/colab/bin/language_service'
-I0506 18:19:41.717661       1 log_spam.go:53]  argv[1]: '--lsp_search_dirs=/datalab/web/pyright/typeshed-fallback/stdlib,/usr/local/lib/python3.10/dist-packages'
-I0506 18:19:41.717665       1 log_spam.go:53]  argv[2]: '--language_services_request_root_url=http://172.28.0.1:8013/'
-I0506 18:19:41.717668       1 log_spam.go:53]  argv[3]: '--language_services_request_timeout=30s'
-I0506 18:19:41.717671       1 log_spam.go:53]  argv[4]: '--'
-I0506 18:19:41.717682       1 log_spam.go:53]  argv[5]: 'node'
-I0506 18:19:41.717685       1 log_spam.go:53]  argv[6]: '/datalab/web/pyright/pyright-langserver.js'
-I0506 18:19:41.717688       1 log_spam.go:53]  argv[7]: '--stdio'
-I0506 18:19:41.717692       1 log_spam.go:53]  argv[8]: '--cancellationReceive=file:15252db1c886861deb82a831bf172cb79d84703f3f'
-I0506 18:19:47.753115      52 ls.go:1241] cancel was requested for: 10, needs to be canceled: false
-I0506 18:21:52.634700      52 ls.go:1241] cancel was requested for: 15, needs to be canceled: false
-I0506 18:21:54.200669      52 ls.go:1241] cancel was requested for: 16, needs to be canceled: false
-I0506 18:21:54.854416      52 ls.go:1241] cancel was requested for: 17, needs to be canceled: false
-I0506 18:22:12.179738      52 ls.go:1241] cancel was requested for: 23, needs to be canceled: false
-I0506 18:22:12.664717      52 ls.go:1241] cancel was requested for: 25, needs to be canceled: false
-I0506 18:22:40.269723      52 ls.go:1241] cancel was requested for: 32, needs to be canceled: false
-I0506 18:23:27.618397      52 ls.go:1241] cancel was requested for: 54, needs to be canceled: false
-I0506 18:23:31.491393      52 ls.go:1241] cancel was requested for: 60, needs to be canceled: false
-I0506 18:23:40.130561      52 ls.go:1241] cancel was requested for: 67, needs to be canceled: false
-I0506 18:23:42.525602      52 ls.go:1241] cancel was requested for: 68, needs to be canceled: false
-I0506 18:24:04.685209      52 ls.go:1241] cancel was requested for: 86, needs to be canceled: false
-I0506 18:24:08.577608      52 ls.go:1241] cancel was requested for: 88, needs to be canceled: false
-I0506 18:24:09.471818      52 ls.go:1241] cancel was requested for: 89, needs to be canceled: false
-I0506 18:24:09.783382      52 ls.go:1241] cancel was requested for: 90, needs to be canceled: false
-I0506 18:24:13.972705      52 ls.go:1241] cancel was requested for: 94, needs to be canceled: false
-I0506 18:24:17.720220      52 ls.go:1241] cancel was requested for: 96, needs to be canceled: false
-I0506 18:24:20.054453      52 ls.go:1241] cancel was requested for: 97, needs to be canceled: false
-I0506 18:24:29.919637      52 ls.go:1241] cancel was requested for: 102, needs to be canceled: false
-I0506 18:24:30.274732      52 ls.go:1241] cancel was requested for: 103, needs to be canceled: false
-I0506 18:24:33.118910      52 ls.go:1241] cancel was requested for: 110, needs to be canceled: false
-I0506 18:24:33.313052      52 ls.go:1241] cancel was requested for: 111, needs to be canceled: false
-I0506 18:24:37.702974      52 ls.go:1241] cancel was requested for: 115, needs to be canceled: false
-I0506 18:24:46.033624      52 ls.go:1241] cancel was requested for: 120, needs to be canceled: false
-I0506 18:24:46.628875      52 ls.go:1241] cancel was requested for: 121, needs to be canceled: false
-I0506 18:24:46.953272      52 ls.go:1241] cancel was requested for: 122, needs to be canceled: false
-I0506 18:24:47.636919      52 ls.go:1241] cancel was requested for: 123, needs to be canceled: false
-I0506 18:27:52.393660      52 ls.go:1241] cancel was requested for: 139, needs to be canceled: false
-I0506 18:27:59.676321      52 ls.go:1241] cancel was requested for: 143, needs to be canceled: false
+I0506 22:05:35.933452       1 log_spam.go:51] Command line arguments:
+I0506 22:05:35.933454       1 log_spam.go:53]  argv[0]: '/usr/colab/bin/language_service'
+I0506 22:05:35.933457       1 log_spam.go:53]  argv[1]: '--lsp_search_dirs=/datalab/web/pyright/typeshed-fallback/stdlib,/usr/local/lib/python3.10/dist-packages'
+I0506 22:05:35.933459       1 log_spam.go:53]  argv[2]: '--language_services_request_root_url=http://172.28.0.1:8013/'
+I0506 22:05:35.933460       1 log_spam.go:53]  argv[3]: '--language_services_request_timeout=30s'
+I0506 22:05:35.933462       1 log_spam.go:53]  argv[4]: '--'
+I0506 22:05:35.933463       1 log_spam.go:53]  argv[5]: 'node'
+I0506 22:05:35.933465       1 log_spam.go:53]  argv[6]: '/datalab/web/pyright/pyright-langserver.js'
+I0506 22:05:35.933466       1 log_spam.go:53]  argv[7]: '--stdio'
+I0506 22:05:35.933471       1 log_spam.go:53]  argv[8]: '--cancellationReceive=file:7e3261f6b4c29b135e4abbae7651fb35f873820f17'
+I0506 22:06:06.691284      16 ls.go:1241] cancel was requested for: 20, needs to be canceled: false
+I0506 22:06:13.663339      16 ls.go:1241] cancel was requested for: 28, needs to be canceled: false
+I0506 22:06:18.962970      16 ls.go:1241] cancel was requested for: 35, needs to be canceled: false
+I0506 22:06:20.104719      16 ls.go:1241] cancel was requested for: 36, needs to be canceled: false
+I0506 22:06:20.807271      16 ls.go:1241] cancel was requested for: 39, needs to be canceled: false
+I0506 22:06:25.022875      16 ls.go:1241] cancel was requested for: 52, needs to be canceled: false
+I0506 22:06:36.708695      16 ls.go:1241] cancel was requested for: 55, needs to be canceled: false
+I0506 22:06:49.697244      16 ls.go:1241] cancel was requested for: 59, needs to be canceled: false
+I0506 22:06:56.794684      16 ls.go:1241] cancel was requested for: 61, needs to be canceled: false
+I0506 22:07:05.137377      16 ls.go:1241] cancel was requested for: 70, needs to be canceled: false
+I0506 22:07:05.143186      16 ls.go:1241] cancel was requested for: 71, needs to be canceled: false
+I0506 22:21:37.746597      16 ls.go:1241] cancel was requested for: 76, needs to be canceled: false

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19c7b7b41cea0b5ea361266fc167993b90350977170156a59763e6d01030aaf4
-size 1621375948
+oid sha256:f57e87bd3b6d14a31cf9aca59ea9b1c086e4ec311042e0989b44529d4b3683cd
+size 56279344

runs/May06_22-29-20_2809d093debe/events.out.tfevents.1746570561.2809d093debe.301.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ffdcc9d89c12a4db5d2877e2d648d747b8e2b374c18b7810518cd5af3f6584e
+size 5831

tmp6yvqll6r/_remote_module_non_scriptable.py

@@ -0,0 +1,81 @@
+from typing import *
+
+import torch
+import torch.distributed.rpc as rpc
+from torch import Tensor
+from torch._jit_internal import Future
+from torch.distributed.rpc import RRef
+from typing import Tuple  # pyre-ignore: unused import
+
+
+module_interface_cls = None
+
+
+def forward_async(self, *args, **kwargs):
+    args = (self.module_rref, self.device, self.is_device_map_set, *args)
+    kwargs = {**kwargs}
+    return rpc.rpc_async(
+        self.module_rref.owner(),
+        _remote_forward,
+        args,
+        kwargs,
+    )
+
+
+def forward(self, *args, **kwargs):
+    args = (self.module_rref, self.device, self.is_device_map_set, *args)
+    kwargs = {**kwargs}
+    ret_fut = rpc.rpc_async(
+        self.module_rref.owner(),
+        _remote_forward,
+        args,
+        kwargs,
+    )
+    return ret_fut.wait()
+
+
+_generated_methods = [
+    forward_async,
+    forward,
+]
+
+
+
+
+def _remote_forward(
+    module_rref: RRef[module_interface_cls], device: str, is_device_map_set: bool, *args, **kwargs):
+    module = module_rref.local_value()
+    device = torch.device(device)
+
+    if device.type != "cuda":
+        return module.forward(*args, **kwargs)
+
+    # If the module is on a cuda device,
+    # move any CPU tensor in args or kwargs to the same cuda device.
+    # Since torch script does not support generator expression,
+    # have to use concatenation instead of
+    # ``tuple(i.to(device) if isinstance(i, Tensor) else i for i in *args)``.
+    args = (*args,)
+    out_args: Tuple[()] = ()
+    for arg in args:
+        arg = (arg.to(device),) if isinstance(arg, Tensor) else (arg,)
+        out_args = out_args + arg
+
+    kwargs = {**kwargs}
+    for k, v in kwargs.items():
+        if isinstance(v, Tensor):
+            kwargs[k] = kwargs[k].to(device)
+
+    if is_device_map_set:
+        return module.forward(*out_args, **kwargs)
+
+    # If the device map is empty, then only CPU tensors are allowed to send over wire,
+    # so have to move any GPU tensor to CPU in the output.
+    # Since torch script does not support generator expression,
+    # have to use concatenation instead of
+    # ``tuple(i.cpu() if isinstance(i, Tensor) else i for i in module.forward(*out_args, **kwargs))``.
+    ret: Tuple[()] = ()
+    for i in module.forward(*out_args, **kwargs):
+        i = (i.cpu(),) if isinstance(i, Tensor) else (i,)
+        ret = ret + i
+    return ret

tokenizer_config.json

@@ -205,7 +205,7 @@
     }
   },
   "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": false,
+  "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
   "extra_special_tokens": {},
   "model_max_length": 1000000000000000019884624838656,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40e360dd6496ec917b8d06dbf50496f10b40e7a0c2bdc1cef254b784d703ca12
+oid sha256:4652eb213ea1c556e5dd8e62fe40aab85e2fc204836b6881aac3b5634cdb04fd
 size 5624