srishtichugh commited on
Commit
8f2f756
Β·
1 Parent(s): 2c0acc5

pass model explicitly to all pipeline stages

Browse files
Files changed (1) hide show
  1. main.py +10 -5
main.py CHANGED
@@ -59,14 +59,16 @@ async def generate(
59
  else:
60
  source_args = ["--url", url.strip()]
61
 
62
- provider = os.getenv("PROVIDER", "gemini")
63
- model = os.getenv("MODEL", "")
64
- extra_env = {"MODEL": model} if model else {}
 
 
65
 
66
  blog_json = data_dir / "blog_data.json"
67
 
68
  def run_pipeline():
69
- # Stage 0 – parse blog
70
  _run("0_blog_process.py",
71
  source_args + ["--output_json_path", str(blog_json)],
72
  extra_env)
@@ -85,9 +87,10 @@ async def generate(
85
  "--output_dir", str(data_dir),
86
  "--provider", provider,
87
  "--content_type", "blog",
 
88
  ], extra_env)
89
 
90
- # Stage 1.1 – extract config
91
  _run("1_1_extract_config.py", [
92
  "--output_dir", str(data_dir),
93
  ], extra_env)
@@ -101,6 +104,7 @@ async def generate(
101
  "--pdf_json_path", str(blog_json_path),
102
  "--output_dir", str(data_dir),
103
  "--provider", provider,
 
104
  ], extra_env)
105
 
106
  # Stage 3 – code generation
@@ -109,6 +113,7 @@ async def generate(
109
  "--output_dir", str(data_dir),
110
  "--output_repo_dir", str(output_dir),
111
  "--provider", provider,
 
112
  ], extra_env)
113
 
114
  await asyncio.get_event_loop().run_in_executor(None, run_pipeline)
 
59
  else:
60
  source_args = ["--url", url.strip()]
61
 
62
+ provider = os.getenv("PROVIDER", "gemma")
63
+ # Default model for NVIDIA/Llama β€” overridable via MODEL env var
64
+ default_model = "meta/llama-3.3-70b-instruct"
65
+ model = os.getenv("MODEL", default_model)
66
+ extra_env = {"MODEL": model}
67
 
68
  blog_json = data_dir / "blog_data.json"
69
 
70
  def run_pipeline():
71
+ # Stage 0 – parse blog (no LLM, no --model needed)
72
  _run("0_blog_process.py",
73
  source_args + ["--output_json_path", str(blog_json)],
74
  extra_env)
 
87
  "--output_dir", str(data_dir),
88
  "--provider", provider,
89
  "--content_type", "blog",
90
+ "--model", model,
91
  ], extra_env)
92
 
93
+ # Stage 1.1 – extract config (no LLM, no --model needed)
94
  _run("1_1_extract_config.py", [
95
  "--output_dir", str(data_dir),
96
  ], extra_env)
 
104
  "--pdf_json_path", str(blog_json_path),
105
  "--output_dir", str(data_dir),
106
  "--provider", provider,
107
+ "--model", model,
108
  ], extra_env)
109
 
110
  # Stage 3 – code generation
 
113
  "--output_dir", str(data_dir),
114
  "--output_repo_dir", str(output_dir),
115
  "--provider", provider,
116
+ "--model", model,
117
  ], extra_env)
118
 
119
  await asyncio.get_event_loop().run_in_executor(None, run_pipeline)