Spaces:
Sleeping
Sleeping
Commit Β·
8f2f756
1
Parent(s): 2c0acc5
pass model explicitly to all pipeline stages
Browse files
main.py
CHANGED
|
@@ -59,14 +59,16 @@ async def generate(
|
|
| 59 |
else:
|
| 60 |
source_args = ["--url", url.strip()]
|
| 61 |
|
| 62 |
-
provider
|
| 63 |
-
model
|
| 64 |
-
|
|
|
|
|
|
|
| 65 |
|
| 66 |
blog_json = data_dir / "blog_data.json"
|
| 67 |
|
| 68 |
def run_pipeline():
|
| 69 |
-
# Stage 0 β parse blog
|
| 70 |
_run("0_blog_process.py",
|
| 71 |
source_args + ["--output_json_path", str(blog_json)],
|
| 72 |
extra_env)
|
|
@@ -85,9 +87,10 @@ async def generate(
|
|
| 85 |
"--output_dir", str(data_dir),
|
| 86 |
"--provider", provider,
|
| 87 |
"--content_type", "blog",
|
|
|
|
| 88 |
], extra_env)
|
| 89 |
|
| 90 |
-
# Stage 1.1 β extract config
|
| 91 |
_run("1_1_extract_config.py", [
|
| 92 |
"--output_dir", str(data_dir),
|
| 93 |
], extra_env)
|
|
@@ -101,6 +104,7 @@ async def generate(
|
|
| 101 |
"--pdf_json_path", str(blog_json_path),
|
| 102 |
"--output_dir", str(data_dir),
|
| 103 |
"--provider", provider,
|
|
|
|
| 104 |
], extra_env)
|
| 105 |
|
| 106 |
# Stage 3 β code generation
|
|
@@ -109,6 +113,7 @@ async def generate(
|
|
| 109 |
"--output_dir", str(data_dir),
|
| 110 |
"--output_repo_dir", str(output_dir),
|
| 111 |
"--provider", provider,
|
|
|
|
| 112 |
], extra_env)
|
| 113 |
|
| 114 |
await asyncio.get_event_loop().run_in_executor(None, run_pipeline)
|
|
|
|
| 59 |
else:
|
| 60 |
source_args = ["--url", url.strip()]
|
| 61 |
|
| 62 |
+
provider = os.getenv("PROVIDER", "gemma")
|
| 63 |
+
# Default model for NVIDIA/Llama β overridable via MODEL env var
|
| 64 |
+
default_model = "meta/llama-3.3-70b-instruct"
|
| 65 |
+
model = os.getenv("MODEL", default_model)
|
| 66 |
+
extra_env = {"MODEL": model}
|
| 67 |
|
| 68 |
blog_json = data_dir / "blog_data.json"
|
| 69 |
|
| 70 |
def run_pipeline():
|
| 71 |
+
# Stage 0 β parse blog (no LLM, no --model needed)
|
| 72 |
_run("0_blog_process.py",
|
| 73 |
source_args + ["--output_json_path", str(blog_json)],
|
| 74 |
extra_env)
|
|
|
|
| 87 |
"--output_dir", str(data_dir),
|
| 88 |
"--provider", provider,
|
| 89 |
"--content_type", "blog",
|
| 90 |
+
"--model", model,
|
| 91 |
], extra_env)
|
| 92 |
|
| 93 |
+
# Stage 1.1 β extract config (no LLM, no --model needed)
|
| 94 |
_run("1_1_extract_config.py", [
|
| 95 |
"--output_dir", str(data_dir),
|
| 96 |
], extra_env)
|
|
|
|
| 104 |
"--pdf_json_path", str(blog_json_path),
|
| 105 |
"--output_dir", str(data_dir),
|
| 106 |
"--provider", provider,
|
| 107 |
+
"--model", model,
|
| 108 |
], extra_env)
|
| 109 |
|
| 110 |
# Stage 3 β code generation
|
|
|
|
| 113 |
"--output_dir", str(data_dir),
|
| 114 |
"--output_repo_dir", str(output_dir),
|
| 115 |
"--provider", provider,
|
| 116 |
+
"--model", model,
|
| 117 |
], extra_env)
|
| 118 |
|
| 119 |
await asyncio.get_event_loop().run_in_executor(None, run_pipeline)
|