Ashira Pitchayapakayakul commited on
Commit
eaaf1cf
Β·
1 Parent(s): c8d5d6b

fix: bash heredoc + python triple-quote interpolation crashes

Browse files

surrogate-dev-loop.sh + surrogate-daemon.sh used '''\$VAR''' which broke when
$VAR contained quotes (e.g. error msgs like [err] 'choices'). Replaced with
env-var passing + sys.argv. This was the root cause of the cron 'unterminated
string literal' errors blocking the dev-loop and daemon chains.

Files changed (2) hide show
  1. bin/surrogate-daemon.sh +36 -24
  2. bin/surrogate-dev-loop.sh +33 -38
bin/surrogate-daemon.sh CHANGED
@@ -33,13 +33,20 @@ case "$CMD" in
33
  shift
34
  TASK="$*"
35
  [[ -z "$TASK" ]] && { echo "need task"; exit 2; }
36
- python3 -c "
37
- import json, uuid
38
  from datetime import datetime
39
- task = {'id': uuid.uuid4().hex[:12], 'ts': datetime.utcnow().isoformat(), 'task': '''$TASK''', 'status': 'pending', 'priority': 'P0-user'}
40
- open('$QUEUE','a').write(json.dumps(task, ensure_ascii=False) + '\n')
41
- print(f\"enqueued: {task['id']} {task['task'][:60]}\")
42
- "
 
 
 
 
 
 
 
43
  exit 0
44
  ;;
45
  plan)
@@ -152,7 +159,7 @@ print(f\"enqueued: {task['id']} {task['task'][:60]}\")
152
  _worker)
153
  # ── Pop one task from queue (P0-user first, then plan, then self-gen) ──────
154
  _pop_queue() {
155
- python3 <<PYEOF
156
  import json, os, sys, fcntl
157
  from pathlib import Path
158
  q = Path(os.path.expanduser('$QUEUE'))
@@ -181,7 +188,7 @@ PYEOF
181
 
182
  # ── Pop next task from active plan (no sleep needed β€” plan drives work) ──
183
  _pop_plan() {
184
- python3 <<'PYEOF'
185
  import sys, json, os, re, uuid
186
  from pathlib import Path
187
  from datetime import datetime
@@ -216,7 +223,7 @@ PYEOF
216
 
217
  # ── Self-generate task from pool (fallback when no plan + queue empty) ──
218
  _self_gen() {
219
- AUTO_TASK=$(python3 <<'PYEOF'
220
  import json, os, random
221
  from pathlib import Path
222
  ep = Path(os.path.expanduser('~/.claude/state/surrogate-memory/episodes.jsonl'))
@@ -271,7 +278,7 @@ for t in random.sample(pool, len(pool)):
271
  print(chosen or pool[0])
272
  PYEOF
273
  )
274
- echo "{\"id\":\"auto-$(python3 -c 'import uuid; print(uuid.uuid4().hex[:8])')\",\"task\":\"$AUTO_TASK\",\"self_generated\":true,\"source\":\"self-gen\"}"
275
  }
276
 
277
  # ── Task resolution: queue β†’ plan β†’ self-gen (no 60s sleep) ─────────────
@@ -290,9 +297,9 @@ PYEOF
290
  fi
291
 
292
  # Extract task
293
- TASK=$(echo "$TASK_JSON" | python3 -c "import json,sys; print(json.loads(sys.stdin.read())['task'])")
294
- TID=$(echo "$TASK_JSON" | python3 -c "import json,sys; print(json.loads(sys.stdin.read())['id'])")
295
- SOURCE=$(echo "$TASK_JSON" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('source','queue'))")
296
 
297
  echo "[$(date +%H:%M:%S)] worker picked $TID [$SOURCE]: ${TASK:0:80}" >> "$LOG"
298
  START=$(date +%s)
@@ -302,30 +309,35 @@ PYEOF
302
  END=$(date +%s)
303
  DUR=$((END - START))
304
 
305
- # If task came from plan, mark as done ([ ] β†’ [x])
306
  if [[ "$SOURCE" == "plan" ]]; then
307
- python3 <<PYEOF >> "$LOG" 2>&1
308
- import re
309
  from pathlib import Path
310
  plan_file = Path.home() / '.surrogate' / 'active-plan.md'
311
  if plan_file.exists():
312
- task = '''$TASK'''
313
  text = plan_file.read_text()
314
- # Mark [~] in-progress β†’ [x] done
315
  new_text = text.replace(f'- [~] {task}', f'- [x] {task}', 1)
316
  plan_file.write_text(new_text)
317
- # Count remaining
318
  remaining = len(re.findall(r'^- \[ \]', new_text, re.MULTILINE))
319
  print(f"[plan] marked done: {task[:60]} | remaining: {remaining}")
320
  PYEOF
321
  fi
322
 
323
  # Mark done in audit log
324
- python3 <<PYEOF >> "$LOG" 2>&1
325
- import json
326
- done = {'id': '$TID', 'source': '$SOURCE', 'task': '''$TASK''', 'duration_sec': $DUR, 'output_tail': '''$(echo "$OUTPUT" | tail -20 | python3 -c "import sys; print(sys.stdin.read().replace(chr(39),chr(34))[:2000])")'''}
327
- open('$DONE','a').write(json.dumps(done, ensure_ascii=False) + '\n')
328
- print(f"[{'$TID'}] done in {$DUR}s")
 
 
 
 
 
 
 
329
  PYEOF
330
  exit 0
331
  ;;
 
33
  shift
34
  TASK="$*"
35
  [[ -z "$TASK" ]] && { echo "need task"; exit 2; }
36
+ ENQUEUE_TASK="$TASK" /usr/bin/python3 - "$QUEUE" <<'PYEOF'
37
+ import json, uuid, os, sys
38
  from datetime import datetime
39
+ queue_path = sys.argv[1]
40
+ task = {
41
+ 'id': uuid.uuid4().hex[:12],
42
+ 'ts': datetime.utcnow().isoformat(),
43
+ 'task': os.environ.get('ENQUEUE_TASK', ''),
44
+ 'status': 'pending',
45
+ 'priority': 'P0-user',
46
+ }
47
+ open(queue_path, 'a').write(json.dumps(task, ensure_ascii=False) + '\n')
48
+ print(f"enqueued: {task['id']} {task['task'][:60]}")
49
+ PYEOF
50
  exit 0
51
  ;;
52
  plan)
 
159
  _worker)
160
  # ── Pop one task from queue (P0-user first, then plan, then self-gen) ──────
161
  _pop_queue() {
162
+ /usr/bin/python3 <<PYEOF
163
  import json, os, sys, fcntl
164
  from pathlib import Path
165
  q = Path(os.path.expanduser('$QUEUE'))
 
188
 
189
  # ── Pop next task from active plan (no sleep needed β€” plan drives work) ──
190
  _pop_plan() {
191
+ /usr/bin/python3 <<'PYEOF'
192
  import sys, json, os, re, uuid
193
  from pathlib import Path
194
  from datetime import datetime
 
223
 
224
  # ── Self-generate task from pool (fallback when no plan + queue empty) ──
225
  _self_gen() {
226
+ AUTO_TASK=$(/usr/bin/python3 <<'PYEOF'
227
  import json, os, random
228
  from pathlib import Path
229
  ep = Path(os.path.expanduser('~/.claude/state/surrogate-memory/episodes.jsonl'))
 
278
  print(chosen or pool[0])
279
  PYEOF
280
  )
281
+ echo "{\"id\":\"auto-$(/usr/bin/python3 -c 'import uuid; print(uuid.uuid4().hex[:8])')\",\"task\":\"$AUTO_TASK\",\"self_generated\":true,\"source\":\"self-gen\"}"
282
  }
283
 
284
  # ── Task resolution: queue β†’ plan β†’ self-gen (no 60s sleep) ─────────────
 
297
  fi
298
 
299
  # Extract task
300
+ TASK=$(echo "$TASK_JSON" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read())['task'])")
301
+ TID=$(echo "$TASK_JSON" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read())['id'])")
302
+ SOURCE=$(echo "$TASK_JSON" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('source','queue'))")
303
 
304
  echo "[$(date +%H:%M:%S)] worker picked $TID [$SOURCE]: ${TASK:0:80}" >> "$LOG"
305
  START=$(date +%s)
 
309
  END=$(date +%s)
310
  DUR=$((END - START))
311
 
312
+ # If task came from plan, mark as done ([ ] β†’ [x]) β€” env vars = safe quoting
313
  if [[ "$SOURCE" == "plan" ]]; then
314
+ DAEMON_TASK="$TASK" /usr/bin/python3 - >> "$LOG" 2>&1 <<'PYEOF'
315
+ import re, os
316
  from pathlib import Path
317
  plan_file = Path.home() / '.surrogate' / 'active-plan.md'
318
  if plan_file.exists():
319
+ task = os.environ.get('DAEMON_TASK', '')
320
  text = plan_file.read_text()
 
321
  new_text = text.replace(f'- [~] {task}', f'- [x] {task}', 1)
322
  plan_file.write_text(new_text)
 
323
  remaining = len(re.findall(r'^- \[ \]', new_text, re.MULTILINE))
324
  print(f"[plan] marked done: {task[:60]} | remaining: {remaining}")
325
  PYEOF
326
  fi
327
 
328
  # Mark done in audit log
329
+ DAEMON_TASK="$TASK" DAEMON_OUTPUT="$(echo "$OUTPUT" | tail -20)" \
330
+ /usr/bin/python3 - "$TID" "$SOURCE" "$DUR" "$DONE" >> "$LOG" 2>&1 <<'PYEOF'
331
+ import json, os, sys
332
+ tid, source, dur, done_path = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4]
333
+ done = {
334
+ 'id': tid, 'source': source,
335
+ 'task': os.environ.get('DAEMON_TASK', ''),
336
+ 'duration_sec': dur,
337
+ 'output_tail': os.environ.get('DAEMON_OUTPUT', '')[:2000],
338
+ }
339
+ open(done_path, 'a').write(json.dumps(done, ensure_ascii=False) + '\n')
340
+ print(f"[{tid}] done in {dur}s")
341
  PYEOF
342
  exit 0
343
  ;;
bin/surrogate-dev-loop.sh CHANGED
@@ -33,7 +33,7 @@ SEARCH_ROOTS=(
33
 
34
  # ── Task generators (pick one per cycle, weighted random) ────────────────────
35
  pick_task() {
36
- python3 <<'PYEOF'
37
  import os, random, re, subprocess, json
38
  from pathlib import Path
39
 
@@ -47,7 +47,7 @@ ROOTS = [p for p in ROOTS if p.exists()]
47
 
48
  def find_todo():
49
  """Find a TODO/FIXME/XXX/HACK comment in user code (uses ripgrep β€” fast)."""
50
- cmd = ['rg', '--no-heading', '-n', '-m', '3',
51
  '--type', 'py', '--type', 'sh', '--type', 'ts', '--type', 'go',
52
  '-g', '!node_modules', '-g', '!.venv', '-g', '!__pycache__',
53
  '-g', '!.git', '-g', '!dist', '-g', '!build',
@@ -178,7 +178,7 @@ load_reflexion_lessons() {
178
  local kind="$1"
179
  local file="$HOME/.hermes/workspace/reflexion/lessons-${kind}.jsonl"
180
  [[ ! -f "$file" ]] && { echo ""; return; }
181
- python3 <<PYEOF
182
  import json
183
  from pathlib import Path
184
  p = Path("$file")
@@ -209,17 +209,15 @@ save_reflexion_lesson() {
209
  local kind="$1" task="$2" response="$3" duration="$4"
210
  local file="$HOME/.hermes/workspace/reflexion/lessons-${kind}.jsonl"
211
  mkdir -p "$(dirname "$file")"
212
- python3 <<PYEOF
213
- import json, re, sys
214
- from pathlib import Path
 
215
  from datetime import datetime
 
 
 
216
 
217
- resp = '''$response'''
218
- task = '''$task'''[:200]
219
- dur = $duration
220
-
221
- # Heuristic: extract a "lesson" line from the response.
222
- # Look for explicit "lesson:", "key insight:", "note:", or use first concrete-sounding sentence.
223
  lesson = None
224
  for pat in [
225
  r'(?:lesson|key insight|key takeaway|note):\s*([^\n]{20,200})',
@@ -227,23 +225,17 @@ for pat in [
227
  ]:
228
  m = re.search(pat, resp, re.IGNORECASE)
229
  if m: lesson = m.group(1).strip(); break
230
-
231
  if not lesson:
232
- # Fallback: first declarative sentence in response (after prelude)
233
  sentences = [s.strip() for s in re.split(r'[\.\n]+', resp) if 30 < len(s.strip()) < 200]
234
- if sentences:
235
- lesson = sentences[0]
236
-
237
  if lesson:
238
  record = {
239
  'ts': datetime.utcnow().isoformat(),
240
- 'kind': '$kind',
241
- 'task': task,
242
- 'lesson': lesson[:300],
243
  'duration_sec': dur,
244
- 'score': 1.0 if dur < 60 else 0.5, # fast cycle = better quality usually
245
  }
246
- with open("$file", 'a') as f:
247
  f.write(json.dumps(record, ensure_ascii=False) + '\n')
248
  PYEOF
249
  }
@@ -259,11 +251,11 @@ run_cycle() {
259
  fi
260
 
261
  local kind path line task_text context
262
- kind=$(echo "$task_json" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('kind',''))")
263
- path=$(echo "$task_json" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('path',''))")
264
- line=$(echo "$task_json" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('line',0))")
265
- task_text=$(echo "$task_json" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('task',''))")
266
- context=$(echo "$task_json" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('context',''))")
267
 
268
  local id="$(date +%s)-${kind}"
269
  local out="$OUT_DIR/${id}.md"
@@ -285,7 +277,7 @@ $context
285
 
286
  # Call Surrogate-1 via Ollama (keep_alive=5m so model stays warm between cycles)
287
  local body
288
- body=$(PROMPT_VAR="$prompt" python3 <<'PYEOF'
289
  import json, os
290
  print(json.dumps({
291
  "model": "surrogate-1",
@@ -298,13 +290,13 @@ print(json.dumps({
298
  PYEOF
299
  )
300
  local resp
301
- resp=$(curl -sS --max-time 120 \
302
  http://localhost:11434/v1/chat/completions \
303
  -H 'Content-Type: application/json' \
304
  -d "$body" 2>/dev/null)
305
 
306
  local answer
307
- answer=$(echo "$resp" | python3 -c "
308
  import json, sys
309
  try:
310
  d = json.load(sys.stdin)
@@ -341,18 +333,21 @@ EOF
341
  # Reflexion: extract & save lesson from this cycle
342
  save_reflexion_lesson "$kind" "$task_text" "$answer" "$dur"
343
 
344
- # Append to training-data candidate (will be reviewed before promoting to JSONL)
345
- python3 <<PYEOF
346
- import json
 
347
  from pathlib import Path
 
 
348
  candidate = Path.home() / 'axentx/surrogate/data/training-jsonl/local-dev-pending.jsonl'
349
  candidate.parent.mkdir(parents=True, exist_ok=True)
350
  record = {
351
- 'ts': '$(date -u +%Y-%m-%dT%H:%M:%SZ)',
352
- 'kind': '$kind',
353
- 'task': '''$task_text''',
354
- 'response': '''$answer'''[:5000],
355
- 'duration_sec': $dur,
356
  'source': 'surrogate-dev-loop',
357
  }
358
  with open(candidate, 'a') as f:
 
33
 
34
  # ── Task generators (pick one per cycle, weighted random) ────────────────────
35
  pick_task() {
36
+ /usr/bin/python3 <<'PYEOF'
37
  import os, random, re, subprocess, json
38
  from pathlib import Path
39
 
 
47
 
48
  def find_todo():
49
  """Find a TODO/FIXME/XXX/HACK comment in user code (uses ripgrep β€” fast)."""
50
+ cmd = ['/opt/homebrew/bin/rg', '--no-heading', '-n', '-m', '3',
51
  '--type', 'py', '--type', 'sh', '--type', 'ts', '--type', 'go',
52
  '-g', '!node_modules', '-g', '!.venv', '-g', '!__pycache__',
53
  '-g', '!.git', '-g', '!dist', '-g', '!build',
 
178
  local kind="$1"
179
  local file="$HOME/.hermes/workspace/reflexion/lessons-${kind}.jsonl"
180
  [[ ! -f "$file" ]] && { echo ""; return; }
181
+ /usr/bin/python3 <<PYEOF
182
  import json
183
  from pathlib import Path
184
  p = Path("$file")
 
209
  local kind="$1" task="$2" response="$3" duration="$4"
210
  local file="$HOME/.hermes/workspace/reflexion/lessons-${kind}.jsonl"
211
  mkdir -p "$(dirname "$file")"
212
+ # Pass payload via env vars + sys.argv (safe β€” no shell quoting issues with embedded quotes)
213
+ REFLEX_RESP="$response" REFLEX_TASK="$task" \
214
+ /usr/bin/python3 - "$kind" "$duration" "$file" <<'PYEOF'
215
+ import json, re, os, sys
216
  from datetime import datetime
217
+ kind, dur, out_file = sys.argv[1], int(sys.argv[2]), sys.argv[3]
218
+ resp = os.environ.get('REFLEX_RESP', '')
219
+ task = os.environ.get('REFLEX_TASK', '')[:200]
220
 
 
 
 
 
 
 
221
  lesson = None
222
  for pat in [
223
  r'(?:lesson|key insight|key takeaway|note):\s*([^\n]{20,200})',
 
225
  ]:
226
  m = re.search(pat, resp, re.IGNORECASE)
227
  if m: lesson = m.group(1).strip(); break
 
228
  if not lesson:
 
229
  sentences = [s.strip() for s in re.split(r'[\.\n]+', resp) if 30 < len(s.strip()) < 200]
230
+ if sentences: lesson = sentences[0]
 
 
231
  if lesson:
232
  record = {
233
  'ts': datetime.utcnow().isoformat(),
234
+ 'kind': kind, 'task': task, 'lesson': lesson[:300],
 
 
235
  'duration_sec': dur,
236
+ 'score': 1.0 if dur < 60 else 0.5,
237
  }
238
+ with open(out_file, 'a') as f:
239
  f.write(json.dumps(record, ensure_ascii=False) + '\n')
240
  PYEOF
241
  }
 
251
  fi
252
 
253
  local kind path line task_text context
254
+ kind=$(echo "$task_json" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('kind',''))")
255
+ path=$(echo "$task_json" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('path',''))")
256
+ line=$(echo "$task_json" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('line',0))")
257
+ task_text=$(echo "$task_json" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('task',''))")
258
+ context=$(echo "$task_json" | /usr/bin/python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('context',''))")
259
 
260
  local id="$(date +%s)-${kind}"
261
  local out="$OUT_DIR/${id}.md"
 
277
 
278
  # Call Surrogate-1 via Ollama (keep_alive=5m so model stays warm between cycles)
279
  local body
280
+ body=$(PROMPT_VAR="$prompt" /usr/bin/python3 <<'PYEOF'
281
  import json, os
282
  print(json.dumps({
283
  "model": "surrogate-1",
 
290
  PYEOF
291
  )
292
  local resp
293
+ resp=$(/usr/bin/curl -sS --max-time 120 \
294
  http://localhost:11434/v1/chat/completions \
295
  -H 'Content-Type: application/json' \
296
  -d "$body" 2>/dev/null)
297
 
298
  local answer
299
+ answer=$(echo "$resp" | /usr/bin/python3 -c "
300
  import json, sys
301
  try:
302
  d = json.load(sys.stdin)
 
333
  # Reflexion: extract & save lesson from this cycle
334
  save_reflexion_lesson "$kind" "$task_text" "$answer" "$dur"
335
 
336
+ # Append to training-data candidate (env vars + argv = safe quoting)
337
+ DEV_TASK="$task_text" DEV_ANSWER="$answer" \
338
+ /usr/bin/python3 - "$kind" "$dur" <<'PYEOF'
339
+ import json, os, sys
340
  from pathlib import Path
341
+ from datetime import datetime
342
+ kind, dur = sys.argv[1], int(sys.argv[2])
343
  candidate = Path.home() / 'axentx/surrogate/data/training-jsonl/local-dev-pending.jsonl'
344
  candidate.parent.mkdir(parents=True, exist_ok=True)
345
  record = {
346
+ 'ts': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ'),
347
+ 'kind': kind,
348
+ 'task': os.environ.get('DEV_TASK', '')[:8000],
349
+ 'response': os.environ.get('DEV_ANSWER', '')[:5000],
350
+ 'duration_sec': dur,
351
  'source': 'surrogate-dev-loop',
352
  }
353
  with open(candidate, 'a') as f: