refactor: enhance Cloudflare outbound proxy with improved request patching, robust header handling, and secure environment configuration.
Browse files- Dockerfile +2 -2
- cloudflare-proxy-setup.py +23 -7
- cloudflare-proxy.js +250 -73
- n8n-sync.py +60 -11
- start.sh +2 -2
Dockerfile
CHANGED
|
@@ -45,8 +45,8 @@ USER node
|
|
| 45 |
|
| 46 |
EXPOSE 7861
|
| 47 |
|
| 48 |
-
HEALTHCHECK --interval=30s --timeout=5s --start-period=
|
| 49 |
-
CMD curl -
|
| 50 |
|
| 51 |
ENTRYPOINT ["/usr/bin/tini", "--"]
|
| 52 |
CMD ["/home/node/app/start.sh"]
|
|
|
|
| 45 |
|
| 46 |
EXPOSE 7861
|
| 47 |
|
| 48 |
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=90s \
|
| 49 |
+
CMD curl -fsS http://localhost:7861/health || exit 1
|
| 50 |
|
| 51 |
ENTRYPOINT ["/usr/bin/tini", "--"]
|
| 52 |
CMD ["/home/node/app/start.sh"]
|
cloudflare-proxy-setup.py
CHANGED
|
@@ -154,6 +154,12 @@ def write_env(proxy_url: str, proxy_secret: str) -> None:
|
|
| 154 |
+ "\n",
|
| 155 |
encoding="utf-8",
|
| 156 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
|
| 158 |
|
| 159 |
def main() -> int:
|
|
@@ -165,9 +171,19 @@ def main() -> int:
|
|
| 165 |
)
|
| 166 |
|
| 167 |
if existing_url:
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
return 0
|
| 172 |
|
| 173 |
if not workers_token:
|
|
@@ -217,22 +233,22 @@ def main() -> int:
|
|
| 217 |
|
| 218 |
proxy_url = f"https://{worker_name}.{subdomain}.workers.dev"
|
| 219 |
write_env(proxy_url, proxy_secret)
|
| 220 |
-
print(f"
|
| 221 |
return 0
|
| 222 |
except urllib.error.HTTPError as error:
|
| 223 |
detail = error.read().decode("utf-8", errors="replace")
|
| 224 |
if error.code == 403 and '"code":9109' in detail:
|
| 225 |
print(
|
| 226 |
-
"
|
| 227 |
"Use a Cloudflare API Token in CLOUDFLARE_WORKERS_TOKEN "
|
| 228 |
"(not a Global API Key, tunnel token, or worker secret). "
|
| 229 |
"For auto-setup, it should have account-level 'Workers Scripts: Edit'. "
|
| 230 |
"The setup can auto-discover your account; CLOUDFLARE_ACCOUNT_ID is not required."
|
| 231 |
)
|
| 232 |
-
print(f"
|
| 233 |
return 1
|
| 234 |
except Exception as error:
|
| 235 |
-
print(f"
|
| 236 |
return 1
|
| 237 |
|
| 238 |
|
|
|
|
| 154 |
+ "\n",
|
| 155 |
encoding="utf-8",
|
| 156 |
)
|
| 157 |
+
# Belt-and-suspenders: even with umask 0077 on the parent shell, force
|
| 158 |
+
# 0600 since the file holds the worker shared secret.
|
| 159 |
+
try:
|
| 160 |
+
ENV_FILE.chmod(0o600)
|
| 161 |
+
except OSError:
|
| 162 |
+
pass
|
| 163 |
|
| 164 |
|
| 165 |
def main() -> int:
|
|
|
|
| 171 |
)
|
| 172 |
|
| 173 |
if existing_url:
|
| 174 |
+
# Always write the env file so downstream `. $CF_PROXY_ENV_FILE` in
|
| 175 |
+
# start.sh has CLOUDFLARE_PROXY_URL set even when no secret was
|
| 176 |
+
# supplied. Empty secret means we send no x-proxy-key header — that
|
| 177 |
+
# only works if the deployed worker also has no secret baked in.
|
| 178 |
+
write_env(existing_url, existing_secret)
|
| 179 |
+
if not existing_secret:
|
| 180 |
+
print(
|
| 181 |
+
"Warning: CLOUDFLARE_PROXY_URL is set but CLOUDFLARE_PROXY_SECRET "
|
| 182 |
+
"is empty. Requests will succeed only if the deployed worker "
|
| 183 |
+
"was built without PROXY_SHARED_SECRET; otherwise you'll see "
|
| 184 |
+
"401 Unauthorized."
|
| 185 |
+
)
|
| 186 |
+
print(f"Using configured Cloudflare proxy: {existing_url}")
|
| 187 |
return 0
|
| 188 |
|
| 189 |
if not workers_token:
|
|
|
|
| 233 |
|
| 234 |
proxy_url = f"https://{worker_name}.{subdomain}.workers.dev"
|
| 235 |
write_env(proxy_url, proxy_secret)
|
| 236 |
+
print(f"Cloudflare proxy ready: {proxy_url}")
|
| 237 |
return 0
|
| 238 |
except urllib.error.HTTPError as error:
|
| 239 |
detail = error.read().decode("utf-8", errors="replace")
|
| 240 |
if error.code == 403 and '"code":9109' in detail:
|
| 241 |
print(
|
| 242 |
+
"Cloudflare proxy setup failed: invalid Workers token. "
|
| 243 |
"Use a Cloudflare API Token in CLOUDFLARE_WORKERS_TOKEN "
|
| 244 |
"(not a Global API Key, tunnel token, or worker secret). "
|
| 245 |
"For auto-setup, it should have account-level 'Workers Scripts: Edit'. "
|
| 246 |
"The setup can auto-discover your account; CLOUDFLARE_ACCOUNT_ID is not required."
|
| 247 |
)
|
| 248 |
+
print(f"Cloudflare proxy setup failed: HTTP {error.code} {detail}")
|
| 249 |
return 1
|
| 250 |
except Exception as error:
|
| 251 |
+
print(f"Cloudflare proxy setup failed: {error}")
|
| 252 |
return 1
|
| 253 |
|
| 254 |
|
cloudflare-proxy.js
CHANGED
|
@@ -1,14 +1,17 @@
|
|
| 1 |
/**
|
| 2 |
* Cloudflare Proxy: Transparent Fix for Blocked Domains
|
| 3 |
*
|
| 4 |
-
* Patches https.request/http.request/fetch to redirect traffic
|
| 5 |
-
* hosts through a Cloudflare Worker proxy.
|
| 6 |
*/
|
| 7 |
"use strict";
|
| 8 |
|
| 9 |
const https = require("https");
|
| 10 |
const http = require("http");
|
| 11 |
|
|
|
|
|
|
|
|
|
|
| 12 |
let PROXY_URL = process.env.CLOUDFLARE_PROXY_URL;
|
| 13 |
if (
|
| 14 |
PROXY_URL &&
|
|
@@ -41,61 +44,64 @@ if (PROXY_URL) {
|
|
| 41 |
const isInternal =
|
| 42 |
normalized === "localhost" ||
|
| 43 |
normalized === "127.0.0.1" ||
|
|
|
|
|
|
|
|
|
|
| 44 |
normalized.endsWith(".hf.space") ||
|
| 45 |
normalized.endsWith(".huggingface.co") ||
|
| 46 |
normalized === "huggingface.co";
|
| 47 |
|
| 48 |
-
|
| 49 |
-
return !isInternal;
|
| 50 |
-
}
|
| 51 |
-
|
| 52 |
-
return BLOCKED_DOMAINS.some(
|
| 53 |
(domain) =>
|
| 54 |
normalized === domain || normalized.endsWith(`.${domain}`),
|
| 55 |
);
|
|
|
|
|
|
|
| 56 |
};
|
| 57 |
|
| 58 |
const patch = (original, originalModuleName) => {
|
| 59 |
-
return function patchedRequest(
|
| 60 |
-
let
|
| 61 |
-
let
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
} else {
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
(options.host ? String(options.host).split(":")[0] : "");
|
| 76 |
-
path = options.path || "/";
|
| 77 |
-
headers = options.headers || {};
|
| 78 |
}
|
| 79 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
const shouldProxy = shouldProxyHost(hostname);
|
| 81 |
-
const alreadyProxied =
|
| 82 |
-
options && typeof options === "object" && options._proxied;
|
| 83 |
const hasTargetHeader =
|
| 84 |
-
headers
|
| 85 |
-
(headers["x-target-host"] || headers["X-Target-Host"]);
|
| 86 |
|
| 87 |
if (shouldProxy && !alreadyProxied && !hasTargetHeader) {
|
| 88 |
if (DEBUG) {
|
| 89 |
-
|
| 90 |
`[cloudflare-proxy] Redirecting ${originalModuleName}://${hostname}${path} -> ${proxy.hostname}`,
|
| 91 |
);
|
| 92 |
}
|
| 93 |
|
| 94 |
-
const newOptions =
|
| 95 |
-
typeof options === "string" || options instanceof URL
|
| 96 |
-
? { protocol: "https:", path }
|
| 97 |
-
: { ...options };
|
| 98 |
-
|
| 99 |
newOptions._proxied = true;
|
| 100 |
newOptions.protocol = "https:";
|
| 101 |
newOptions.hostname = proxy.hostname;
|
|
@@ -105,7 +111,7 @@ if (PROXY_URL) {
|
|
| 105 |
delete newOptions.agent;
|
| 106 |
|
| 107 |
newOptions.headers = {
|
| 108 |
-
...(
|
| 109 |
host: proxy.host,
|
| 110 |
"x-target-host": hostname,
|
| 111 |
};
|
|
@@ -117,7 +123,7 @@ if (PROXY_URL) {
|
|
| 117 |
return originalHttpsRequest.call(https, newOptions, callback);
|
| 118 |
}
|
| 119 |
|
| 120 |
-
return original.call(this,
|
| 121 |
};
|
| 122 |
};
|
| 123 |
|
|
@@ -127,72 +133,243 @@ if (PROXY_URL) {
|
|
| 127 |
if (originalFetch) {
|
| 128 |
globalThis.fetch = async function patchedFetch(input, init) {
|
| 129 |
const request = input instanceof Request ? input : null;
|
| 130 |
-
const
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
|
|
|
|
|
|
| 136 |
|
| 137 |
const hostname = url.hostname;
|
| 138 |
const shouldProxy = shouldProxyHost(hostname);
|
| 139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
const alreadyProxied =
|
| 141 |
-
|
| 142 |
|
| 143 |
if (!shouldProxy || alreadyProxied) {
|
| 144 |
return originalFetch(input, init);
|
| 145 |
}
|
| 146 |
|
| 147 |
if (DEBUG) {
|
| 148 |
-
|
| 149 |
`[cloudflare-proxy] Redirecting fetch://${hostname}${url.pathname}${url.search} -> ${proxy.hostname}`,
|
| 150 |
);
|
| 151 |
}
|
| 152 |
|
| 153 |
-
|
| 154 |
if (PROXY_SHARED_SECRET) {
|
| 155 |
-
|
| 156 |
}
|
| 157 |
|
| 158 |
const proxiedUrl = new URL(url.pathname + url.search, proxy);
|
| 159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
if (request) {
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
);
|
| 170 |
}
|
| 171 |
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
};
|
| 177 |
}
|
| 178 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
if (DEBUG) {
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
);
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
`[cloudflare-proxy] Transparent proxy active for: ${BLOCKED_DOMAINS.join(", ")}`,
|
| 187 |
);
|
|
|
|
|
|
|
| 188 |
}
|
| 189 |
-
console.log(`[cloudflare-proxy] Target proxy: ${proxy.hostname}`);
|
| 190 |
}
|
| 191 |
} catch (error) {
|
| 192 |
-
|
| 193 |
-
console.error(
|
| 194 |
-
`[cloudflare-proxy] Failed to initialize: ${error.message}`,
|
| 195 |
-
);
|
| 196 |
-
}
|
| 197 |
}
|
| 198 |
}
|
|
|
|
| 1 |
/**
|
| 2 |
* Cloudflare Proxy: Transparent Fix for Blocked Domains
|
| 3 |
*
|
| 4 |
+
* Patches https.request/http.request/fetch and undici to redirect traffic
|
| 5 |
+
* for blocked hosts through a Cloudflare Worker proxy.
|
| 6 |
*/
|
| 7 |
"use strict";
|
| 8 |
|
| 9 |
const https = require("https");
|
| 10 |
const http = require("http");
|
| 11 |
|
| 12 |
+
// Use stderr for logs to avoid breaking child processes that communicate via stdout JSON
|
| 13 |
+
const log = (...args) => console.error(...args);
|
| 14 |
+
|
| 15 |
let PROXY_URL = process.env.CLOUDFLARE_PROXY_URL;
|
| 16 |
if (
|
| 17 |
PROXY_URL &&
|
|
|
|
| 44 |
const isInternal =
|
| 45 |
normalized === "localhost" ||
|
| 46 |
normalized === "127.0.0.1" ||
|
| 47 |
+
normalized === "::1" ||
|
| 48 |
+
normalized === "0.0.0.0" ||
|
| 49 |
+
normalized === proxy.hostname ||
|
| 50 |
normalized.endsWith(".hf.space") ||
|
| 51 |
normalized.endsWith(".huggingface.co") ||
|
| 52 |
normalized === "huggingface.co";
|
| 53 |
|
| 54 |
+
const should = PROXY_ALL ? !isInternal : BLOCKED_DOMAINS.some(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
(domain) =>
|
| 56 |
normalized === domain || normalized.endsWith(`.${domain}`),
|
| 57 |
);
|
| 58 |
+
|
| 59 |
+
return should;
|
| 60 |
};
|
| 61 |
|
| 62 |
const patch = (original, originalModuleName) => {
|
| 63 |
+
return function patchedRequest(arg1, arg2, arg3) {
|
| 64 |
+
let options = {};
|
| 65 |
+
let callback;
|
| 66 |
+
|
| 67 |
+
if (typeof arg1 === "string" || arg1 instanceof URL) {
|
| 68 |
+
const url = typeof arg1 === "string" ? new URL(arg1) : arg1;
|
| 69 |
+
options = {
|
| 70 |
+
protocol: url.protocol,
|
| 71 |
+
hostname: url.hostname,
|
| 72 |
+
port: url.port,
|
| 73 |
+
path: url.pathname + url.search,
|
| 74 |
+
};
|
| 75 |
+
if (typeof arg2 === "object" && arg2 !== null) {
|
| 76 |
+
options = { ...options, ...arg2 };
|
| 77 |
+
callback = arg3;
|
| 78 |
+
} else {
|
| 79 |
+
callback = arg2;
|
| 80 |
+
}
|
| 81 |
} else {
|
| 82 |
+
options = { ...arg1 };
|
| 83 |
+
callback = arg2;
|
|
|
|
|
|
|
|
|
|
| 84 |
}
|
| 85 |
|
| 86 |
+
const hostname =
|
| 87 |
+
options.hostname ||
|
| 88 |
+
(options.host ? String(options.host).split(":")[0] : "");
|
| 89 |
+
const path = options.path || "/";
|
| 90 |
+
const headers = options.headers || {};
|
| 91 |
+
|
| 92 |
const shouldProxy = shouldProxyHost(hostname);
|
| 93 |
+
const alreadyProxied = options._proxied;
|
|
|
|
| 94 |
const hasTargetHeader =
|
| 95 |
+
headers["x-target-host"] || headers["X-Target-Host"];
|
|
|
|
| 96 |
|
| 97 |
if (shouldProxy && !alreadyProxied && !hasTargetHeader) {
|
| 98 |
if (DEBUG) {
|
| 99 |
+
log(
|
| 100 |
`[cloudflare-proxy] Redirecting ${originalModuleName}://${hostname}${path} -> ${proxy.hostname}`,
|
| 101 |
);
|
| 102 |
}
|
| 103 |
|
| 104 |
+
const newOptions = { ...options };
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
newOptions._proxied = true;
|
| 106 |
newOptions.protocol = "https:";
|
| 107 |
newOptions.hostname = proxy.hostname;
|
|
|
|
| 111 |
delete newOptions.agent;
|
| 112 |
|
| 113 |
newOptions.headers = {
|
| 114 |
+
...(options.headers || {}),
|
| 115 |
host: proxy.host,
|
| 116 |
"x-target-host": hostname,
|
| 117 |
};
|
|
|
|
| 123 |
return originalHttpsRequest.call(https, newOptions, callback);
|
| 124 |
}
|
| 125 |
|
| 126 |
+
return original.call(this, arg1, arg2, arg3);
|
| 127 |
};
|
| 128 |
};
|
| 129 |
|
|
|
|
| 133 |
if (originalFetch) {
|
| 134 |
globalThis.fetch = async function patchedFetch(input, init) {
|
| 135 |
const request = input instanceof Request ? input : null;
|
| 136 |
+
const urlStr = request ? request.url : String(input);
|
| 137 |
+
|
| 138 |
+
let url;
|
| 139 |
+
try {
|
| 140 |
+
url = new URL(urlStr);
|
| 141 |
+
} catch (e) {
|
| 142 |
+
return originalFetch(input, init);
|
| 143 |
+
}
|
| 144 |
|
| 145 |
const hostname = url.hostname;
|
| 146 |
const shouldProxy = shouldProxyHost(hostname);
|
| 147 |
+
|
| 148 |
+
let mergedHeaders;
|
| 149 |
+
if (request) {
|
| 150 |
+
mergedHeaders = new Headers(request.headers);
|
| 151 |
+
} else {
|
| 152 |
+
mergedHeaders = new Headers(init?.headers || {});
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
const alreadyProxied =
|
| 156 |
+
mergedHeaders.has("x-target-host") || mergedHeaders.has("X-Target-Host");
|
| 157 |
|
| 158 |
if (!shouldProxy || alreadyProxied) {
|
| 159 |
return originalFetch(input, init);
|
| 160 |
}
|
| 161 |
|
| 162 |
if (DEBUG) {
|
| 163 |
+
log(
|
| 164 |
`[cloudflare-proxy] Redirecting fetch://${hostname}${url.pathname}${url.search} -> ${proxy.hostname}`,
|
| 165 |
);
|
| 166 |
}
|
| 167 |
|
| 168 |
+
mergedHeaders.set("x-target-host", hostname);
|
| 169 |
if (PROXY_SHARED_SECRET) {
|
| 170 |
+
mergedHeaders.set("x-proxy-key", PROXY_SHARED_SECRET);
|
| 171 |
}
|
| 172 |
|
| 173 |
const proxiedUrl = new URL(url.pathname + url.search, proxy);
|
| 174 |
|
| 175 |
+
const logProxyError = (promise, debugInfo) => {
|
| 176 |
+
promise
|
| 177 |
+
.then(r => {
|
| 178 |
+
if (DEBUG && !r.ok) {
|
| 179 |
+
log(`[cloudflare-proxy] Proxy HTTP ${r.status} for ${hostname}: ${r.statusText}`);
|
| 180 |
+
}
|
| 181 |
+
})
|
| 182 |
+
.catch(err => {
|
| 183 |
+
const cause = err?.cause;
|
| 184 |
+
const causeStr = cause
|
| 185 |
+
? ` | cause: ${cause?.code || cause?.message || String(cause)}`
|
| 186 |
+
: "";
|
| 187 |
+
log(`[cloudflare-proxy] Proxy FAILED ${hostname}: ${err?.message}${causeStr}`);
|
| 188 |
+
if (DEBUG && debugInfo) {
|
| 189 |
+
log(`[cloudflare-proxy] Debug: ${debugInfo}`);
|
| 190 |
+
}
|
| 191 |
+
});
|
| 192 |
+
return promise;
|
| 193 |
+
};
|
| 194 |
+
|
| 195 |
if (request) {
|
| 196 |
+
const fetchOpts = {
|
| 197 |
+
method: request.method,
|
| 198 |
+
headers: mergedHeaders,
|
| 199 |
+
redirect: request.redirect,
|
| 200 |
+
};
|
| 201 |
+
if (request.body) {
|
| 202 |
+
fetchOpts.body = request.body;
|
| 203 |
+
fetchOpts.duplex = "half";
|
| 204 |
+
}
|
| 205 |
+
return logProxyError(
|
| 206 |
+
originalFetch(String(proxiedUrl), fetchOpts),
|
| 207 |
+
`request-mode method=${request.method} hasBody=${!!request.body}`,
|
| 208 |
);
|
| 209 |
}
|
| 210 |
|
| 211 |
+
// Build a fresh init: do NOT spread `init` because it may carry a
|
| 212 |
+
// `dispatcher`/`client` pinned to the original target's connection
|
| 213 |
+
// pool, which causes undici to throw UND_ERR_INVALID_ARG when we
|
| 214 |
+
// change the origin. Forward only well-known fetch options.
|
| 215 |
+
const newInit = {
|
| 216 |
+
method: init?.method || "GET",
|
| 217 |
+
headers: mergedHeaders,
|
| 218 |
+
};
|
| 219 |
+
if (init?.body != null) {
|
| 220 |
+
newInit.body = init.body;
|
| 221 |
+
if (init.body instanceof ReadableStream) {
|
| 222 |
+
newInit.duplex = init.duplex || "half";
|
| 223 |
+
}
|
| 224 |
+
}
|
| 225 |
+
if (init?.signal) newInit.signal = init.signal;
|
| 226 |
+
if (init?.redirect) newInit.redirect = init.redirect;
|
| 227 |
+
if (init?.credentials) newInit.credentials = init.credentials;
|
| 228 |
+
if (init?.cache) newInit.cache = init.cache;
|
| 229 |
+
if (init?.mode) newInit.mode = init.mode;
|
| 230 |
+
if (init?.referrer) newInit.referrer = init.referrer;
|
| 231 |
+
if (init?.referrerPolicy) newInit.referrerPolicy = init.referrerPolicy;
|
| 232 |
+
if (init?.integrity) newInit.integrity = init.integrity;
|
| 233 |
+
if (init?.keepalive != null) newInit.keepalive = init.keepalive;
|
| 234 |
+
|
| 235 |
+
const bodyType = init?.body == null
|
| 236 |
+
? "none"
|
| 237 |
+
: init.body instanceof ReadableStream
|
| 238 |
+
? "ReadableStream"
|
| 239 |
+
: (init.body?.constructor?.name || typeof init.body);
|
| 240 |
+
|
| 241 |
+
return logProxyError(
|
| 242 |
+
originalFetch(String(proxiedUrl), newInit),
|
| 243 |
+
`init-mode method=${newInit.method} body=${bodyType} initKeys=${Object.keys(init || {}).join(",")}`,
|
| 244 |
+
);
|
| 245 |
};
|
| 246 |
}
|
| 247 |
|
| 248 |
+
// undici patching
|
| 249 |
+
const patchUndiciInstance = (exports) => {
|
| 250 |
+
if (!exports) return;
|
| 251 |
+
|
| 252 |
+
const patchDispatch = (proto, name) => {
|
| 253 |
+
if (proto && proto.dispatch && !proto.dispatch._patched) {
|
| 254 |
+
const origDispatch = proto.dispatch;
|
| 255 |
+
proto.dispatch = function(options, handler) {
|
| 256 |
+
let origin = options.origin || this.origin;
|
| 257 |
+
if (origin && typeof origin !== 'string') {
|
| 258 |
+
try { origin = origin.origin || origin.toString(); } catch (e) { origin = ""; }
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
let hostname = "";
|
| 262 |
+
try {
|
| 263 |
+
hostname = new URL(String(origin)).hostname;
|
| 264 |
+
} catch(e) {
|
| 265 |
+
hostname = String(origin || "").split(':')[0];
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
if (hostname && shouldProxyHost(hostname)) {
|
| 269 |
+
if (DEBUG) log(`[cloudflare-proxy] Redirecting undici ${name}.dispatch: ${hostname}${options.path || ""} -> ${proxy.hostname}`);
|
| 270 |
+
|
| 271 |
+
const targetHeader = "x-target-host";
|
| 272 |
+
const secretHeader = "x-proxy-key";
|
| 273 |
+
|
| 274 |
+
if (Array.isArray(options.headers)) {
|
| 275 |
+
let foundTarget = false;
|
| 276 |
+
for (let i = 0; i < options.headers.length; i += 2) {
|
| 277 |
+
if (String(options.headers[i]).toLowerCase() === targetHeader) {
|
| 278 |
+
foundTarget = true;
|
| 279 |
+
break;
|
| 280 |
+
}
|
| 281 |
+
}
|
| 282 |
+
if (!foundTarget) {
|
| 283 |
+
options.headers.push(targetHeader, hostname);
|
| 284 |
+
if (PROXY_SHARED_SECRET) options.headers.push(secretHeader, PROXY_SHARED_SECRET);
|
| 285 |
+
}
|
| 286 |
+
} else {
|
| 287 |
+
options.headers = options.headers || {};
|
| 288 |
+
if (options.headers instanceof Map || (typeof options.headers.set === 'function')) {
|
| 289 |
+
options.headers.set(targetHeader, hostname);
|
| 290 |
+
if (PROXY_SHARED_SECRET) options.headers.set(secretHeader, PROXY_SHARED_SECRET);
|
| 291 |
+
} else {
|
| 292 |
+
options.headers[targetHeader] = hostname;
|
| 293 |
+
if (PROXY_SHARED_SECRET) options.headers[secretHeader] = PROXY_SHARED_SECRET;
|
| 294 |
+
}
|
| 295 |
+
}
|
| 296 |
+
options.origin = `https://${proxy.hostname}`;
|
| 297 |
+
}
|
| 298 |
+
return origDispatch.call(this, options, handler);
|
| 299 |
+
};
|
| 300 |
+
proto.dispatch._patched = true;
|
| 301 |
+
}
|
| 302 |
+
};
|
| 303 |
+
|
| 304 |
+
for (const key in exports) {
|
| 305 |
+
if (exports[key] && exports[key].prototype && typeof exports[key].prototype.dispatch === 'function') {
|
| 306 |
+
patchDispatch(exports[key].prototype, key);
|
| 307 |
+
}
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
if (exports.getGlobalDispatcher) {
|
| 311 |
+
try {
|
| 312 |
+
const globalDispatcher = exports.getGlobalDispatcher();
|
| 313 |
+
if (globalDispatcher && globalDispatcher.dispatch && !globalDispatcher.dispatch._patched) {
|
| 314 |
+
patchDispatch(globalDispatcher, "GlobalDispatcherInstance");
|
| 315 |
+
}
|
| 316 |
+
} catch (e) {}
|
| 317 |
+
}
|
| 318 |
+
|
| 319 |
+
// Also patch Agent and other potentially unexported classes if they have dispatch
|
| 320 |
+
if (exports.Agent && exports.Agent.prototype) patchDispatch(exports.Agent.prototype, "Agent");
|
| 321 |
+
if (exports.Pool && exports.Pool.prototype) patchDispatch(exports.Pool.prototype, "Pool");
|
| 322 |
+
if (exports.Client && exports.Client.prototype) patchDispatch(exports.Client.prototype, "Client");
|
| 323 |
+
|
| 324 |
+
if (exports.fetch && !exports.fetch._patched) {
|
| 325 |
+
const origFetch = exports.fetch;
|
| 326 |
+
exports.fetch = async function (input, init) {
|
| 327 |
+
// If we are calling undici.fetch, it should use our globalThis.fetch which is patched
|
| 328 |
+
return globalThis.fetch(input, init);
|
| 329 |
+
};
|
| 330 |
+
exports.fetch._patched = true;
|
| 331 |
+
}
|
| 332 |
+
};
|
| 333 |
+
|
| 334 |
+
// Try to require undici immediately
|
| 335 |
+
try {
|
| 336 |
+
const undici = require("undici");
|
| 337 |
+
patchUndiciInstance(undici);
|
| 338 |
+
} catch (e) {}
|
| 339 |
+
|
| 340 |
+
// Hook require() to patch any undici instance the moment it loads.
|
| 341 |
+
// Match either the bare "undici" id or paths whose final package
|
| 342 |
+
// segment IS undici (e.g. "/foo/node_modules/undici/index.js"). The
|
| 343 |
+
// earlier substring check `id.includes("/undici/")` would also match
|
| 344 |
+
// unrelated packages like "super-undici-x".
|
| 345 |
+
const Module = require("module");
|
| 346 |
+
const originalRequire = Module.prototype.require;
|
| 347 |
+
const UNDICI_PATH_RE = /(?:^|\/)node_modules\/undici(?:\/|$)/;
|
| 348 |
+
Module.prototype.require = function (id) {
|
| 349 |
+
const exports = originalRequire.apply(this, arguments);
|
| 350 |
+
if (id === "undici" || UNDICI_PATH_RE.test(id)) {
|
| 351 |
+
try { patchUndiciInstance(exports); } catch (e) {}
|
| 352 |
+
}
|
| 353 |
+
return exports;
|
| 354 |
+
};
|
| 355 |
+
|
| 356 |
+
// Startup banner: print once across all Node spawns. Use a file marker
|
| 357 |
+
// because every Node process (health-server, gateway, sync subprocess)
|
| 358 |
+
// is spawned fresh from bash with NODE_OPTIONS=--require, so an env-var
|
| 359 |
+
// marker won't propagate. /tmp is per-container so it resets on rebuild.
|
| 360 |
if (DEBUG) {
|
| 361 |
+
try {
|
| 362 |
+
require("fs").writeFileSync("/tmp/.cf-proxy-banner-shown", "1", {
|
| 363 |
+
flag: "wx",
|
| 364 |
+
});
|
| 365 |
+
log(
|
| 366 |
+
`[cloudflare-proxy] active (${PROXY_ALL ? "wildcard" : "list"}) -> ${proxy.hostname}`,
|
|
|
|
| 367 |
);
|
| 368 |
+
} catch (_) {
|
| 369 |
+
// marker exists — banner already shown by another process
|
| 370 |
}
|
|
|
|
| 371 |
}
|
| 372 |
} catch (error) {
|
| 373 |
+
log(`[cloudflare-proxy] Failed to initialize: ${error.message}`);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 374 |
}
|
| 375 |
}
|
n8n-sync.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
|
| 3 |
import hashlib
|
| 4 |
import json
|
|
|
|
| 5 |
import os
|
| 6 |
import shutil
|
| 7 |
import signal
|
|
@@ -13,13 +14,23 @@ import threading
|
|
| 13 |
from pathlib import Path
|
| 14 |
|
| 15 |
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
from huggingface_hub import HfApi, snapshot_download, upload_folder
|
| 18 |
-
from huggingface_hub.errors import RepositoryNotFoundError
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
N8N_HOME = Path(os.environ.get("N8N_USER_FOLDER", "/home/node/.n8n"))
|
| 21 |
STATUS_FILE = Path("/tmp/hugging8n-sync-status.json")
|
| 22 |
INTERVAL = int(os.environ.get("SYNC_INTERVAL", "180"))
|
|
|
|
| 23 |
HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
|
| 24 |
HF_USERNAME = (
|
| 25 |
os.environ.get("HF_USERNAME", "").strip()
|
|
@@ -28,6 +39,7 @@ HF_USERNAME = (
|
|
| 28 |
BACKUP_DATASET_NAME = os.environ.get("BACKUP_DATASET_NAME", "hugging8n-backup").strip()
|
| 29 |
HF_API = HfApi(token=HF_TOKEN) if HF_TOKEN else None
|
| 30 |
STOP_EVENT = threading.Event()
|
|
|
|
| 31 |
|
| 32 |
|
| 33 |
def write_status(status: str, message: str) -> None:
|
|
@@ -64,14 +76,29 @@ def metadata_marker(root: Path) -> tuple[int, int, int]:
|
|
| 64 |
return (file_count, total_size, newest_mtime)
|
| 65 |
|
| 66 |
|
| 67 |
-
def
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
|
| 72 |
|
| 73 |
def ensure_repo_exists() -> str:
|
| 74 |
-
repo_id =
|
| 75 |
try:
|
| 76 |
HF_API.repo_info(repo_id=repo_id, repo_type="dataset")
|
| 77 |
except RepositoryNotFoundError:
|
|
@@ -138,7 +165,7 @@ def restore() -> bool:
|
|
| 138 |
write_status("disabled", "HF_TOKEN is not configured.")
|
| 139 |
return False
|
| 140 |
|
| 141 |
-
repo_id =
|
| 142 |
write_status("restoring", f"Restoring state from {repo_id}")
|
| 143 |
|
| 144 |
try:
|
|
@@ -178,6 +205,13 @@ def restore() -> bool:
|
|
| 178 |
except RepositoryNotFoundError:
|
| 179 |
write_status("fresh", f"Backup dataset {repo_id} does not exist yet.")
|
| 180 |
return True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 181 |
except Exception as exc:
|
| 182 |
write_status("error", f"Restore failed: {exc}")
|
| 183 |
print(f"Restore failed: {exc}", file=sys.stderr)
|
|
@@ -229,9 +263,19 @@ def loop() -> int:
|
|
| 229 |
signal.signal(signal.SIGTERM, handle_signal)
|
| 230 |
signal.signal(signal.SIGINT, handle_signal)
|
| 231 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
last_fingerprint = fingerprint_dir(N8N_HOME)
|
| 233 |
last_marker = metadata_marker(N8N_HOME)
|
| 234 |
-
|
|
|
|
|
|
|
| 235 |
|
| 236 |
while not STOP_EVENT.is_set():
|
| 237 |
try:
|
|
@@ -239,7 +283,7 @@ def loop() -> int:
|
|
| 239 |
except Exception as exc:
|
| 240 |
write_status("error", f"Sync failed: {exc}")
|
| 241 |
print(f"Sync failed: {exc}", file=sys.stderr)
|
| 242 |
-
|
| 243 |
if STOP_EVENT.wait(INTERVAL):
|
| 244 |
break
|
| 245 |
|
|
@@ -255,8 +299,13 @@ def main() -> int:
|
|
| 255 |
if command == "restore":
|
| 256 |
return 0 if restore() else 1
|
| 257 |
if command == "sync-once":
|
| 258 |
-
|
| 259 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
if command == "loop":
|
| 261 |
return loop()
|
| 262 |
|
|
|
|
| 2 |
|
| 3 |
import hashlib
|
| 4 |
import json
|
| 5 |
+
import logging
|
| 6 |
import os
|
| 7 |
import shutil
|
| 8 |
import signal
|
|
|
|
| 14 |
from pathlib import Path
|
| 15 |
|
| 16 |
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
|
| 17 |
+
# huggingface_hub reads HF_HUB_VERBOSITY at import time and overrides any
|
| 18 |
+
# logging.getLogger().setLevel() we apply afterwards. Set it before import
|
| 19 |
+
# to silence the "No files have been modified..." spam from
|
| 20 |
+
# upload_large_folder workers (logger.warning level).
|
| 21 |
+
os.environ.setdefault("HF_HUB_VERBOSITY", "error")
|
| 22 |
|
| 23 |
from huggingface_hub import HfApi, snapshot_download, upload_folder
|
| 24 |
+
from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError
|
| 25 |
+
|
| 26 |
+
# Belt-and-suspenders: also raise the level after import in case the env var
|
| 27 |
+
# wasn't honored (older hub versions, or message logged via a sub-logger).
|
| 28 |
+
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)
|
| 29 |
|
| 30 |
N8N_HOME = Path(os.environ.get("N8N_USER_FOLDER", "/home/node/.n8n"))
|
| 31 |
STATUS_FILE = Path("/tmp/hugging8n-sync-status.json")
|
| 32 |
INTERVAL = int(os.environ.get("SYNC_INTERVAL", "180"))
|
| 33 |
+
INITIAL_DELAY = int(os.environ.get("SYNC_START_DELAY", "10"))
|
| 34 |
HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
|
| 35 |
HF_USERNAME = (
|
| 36 |
os.environ.get("HF_USERNAME", "").strip()
|
|
|
|
| 39 |
BACKUP_DATASET_NAME = os.environ.get("BACKUP_DATASET_NAME", "hugging8n-backup").strip()
|
| 40 |
HF_API = HfApi(token=HF_TOKEN) if HF_TOKEN else None
|
| 41 |
STOP_EVENT = threading.Event()
|
| 42 |
+
_REPO_ID_CACHE: str | None = None
|
| 43 |
|
| 44 |
|
| 45 |
def write_status(status: str, message: str) -> None:
|
|
|
|
| 76 |
return (file_count, total_size, newest_mtime)
|
| 77 |
|
| 78 |
|
| 79 |
+
def resolve_backup_namespace() -> str:
|
| 80 |
+
global _REPO_ID_CACHE
|
| 81 |
+
if _REPO_ID_CACHE:
|
| 82 |
+
return _REPO_ID_CACHE
|
| 83 |
+
|
| 84 |
+
namespace = HF_USERNAME
|
| 85 |
+
if not namespace and HF_API is not None:
|
| 86 |
+
whoami = HF_API.whoami()
|
| 87 |
+
namespace = whoami.get("name") or whoami.get("user") or ""
|
| 88 |
+
|
| 89 |
+
namespace = str(namespace).strip()
|
| 90 |
+
if not namespace:
|
| 91 |
+
raise RuntimeError(
|
| 92 |
+
"Could not determine the Hugging Face username for backups. "
|
| 93 |
+
"Set HF_USERNAME or use a token tied to your account."
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
_REPO_ID_CACHE = f"{namespace}/{BACKUP_DATASET_NAME}"
|
| 97 |
+
return _REPO_ID_CACHE
|
| 98 |
|
| 99 |
|
| 100 |
def ensure_repo_exists() -> str:
|
| 101 |
+
repo_id = resolve_backup_namespace()
|
| 102 |
try:
|
| 103 |
HF_API.repo_info(repo_id=repo_id, repo_type="dataset")
|
| 104 |
except RepositoryNotFoundError:
|
|
|
|
| 165 |
write_status("disabled", "HF_TOKEN is not configured.")
|
| 166 |
return False
|
| 167 |
|
| 168 |
+
repo_id = resolve_backup_namespace()
|
| 169 |
write_status("restoring", f"Restoring state from {repo_id}")
|
| 170 |
|
| 171 |
try:
|
|
|
|
| 205 |
except RepositoryNotFoundError:
|
| 206 |
write_status("fresh", f"Backup dataset {repo_id} does not exist yet.")
|
| 207 |
return True
|
| 208 |
+
except HfHubHTTPError as exc:
|
| 209 |
+
if exc.response is not None and exc.response.status_code == 404:
|
| 210 |
+
write_status("fresh", f"Backup dataset {repo_id} does not exist yet.")
|
| 211 |
+
return True
|
| 212 |
+
write_status("error", f"Restore failed: {exc}")
|
| 213 |
+
print(f"Restore failed: {exc}", file=sys.stderr)
|
| 214 |
+
return False
|
| 215 |
except Exception as exc:
|
| 216 |
write_status("error", f"Restore failed: {exc}")
|
| 217 |
print(f"Restore failed: {exc}", file=sys.stderr)
|
|
|
|
| 263 |
signal.signal(signal.SIGTERM, handle_signal)
|
| 264 |
signal.signal(signal.SIGINT, handle_signal)
|
| 265 |
|
| 266 |
+
try:
|
| 267 |
+
repo_id = resolve_backup_namespace()
|
| 268 |
+
write_status("configured", f"Backup loop active for {repo_id} with {INTERVAL}s interval.")
|
| 269 |
+
except Exception as exc:
|
| 270 |
+
write_status("error", str(exc))
|
| 271 |
+
print(f"Sync error: {exc}")
|
| 272 |
+
return 1
|
| 273 |
+
|
| 274 |
last_fingerprint = fingerprint_dir(N8N_HOME)
|
| 275 |
last_marker = metadata_marker(N8N_HOME)
|
| 276 |
+
|
| 277 |
+
time.sleep(INITIAL_DELAY)
|
| 278 |
+
print(f"State sync started: every {INTERVAL}s -> {repo_id}")
|
| 279 |
|
| 280 |
while not STOP_EVENT.is_set():
|
| 281 |
try:
|
|
|
|
| 283 |
except Exception as exc:
|
| 284 |
write_status("error", f"Sync failed: {exc}")
|
| 285 |
print(f"Sync failed: {exc}", file=sys.stderr)
|
| 286 |
+
|
| 287 |
if STOP_EVENT.wait(INTERVAL):
|
| 288 |
break
|
| 289 |
|
|
|
|
| 299 |
if command == "restore":
|
| 300 |
return 0 if restore() else 1
|
| 301 |
if command == "sync-once":
|
| 302 |
+
try:
|
| 303 |
+
sync_once()
|
| 304 |
+
return 0
|
| 305 |
+
except Exception as exc:
|
| 306 |
+
write_status("error", f"Shutdown sync failed: {exc}")
|
| 307 |
+
print(f"State sync: shutdown sync failed: {exc}")
|
| 308 |
+
return 1
|
| 309 |
if command == "loop":
|
| 310 |
return loop()
|
| 311 |
|
start.sh
CHANGED
|
@@ -79,11 +79,11 @@ CF_PROXY_ENV_FILE="/tmp/hugging8n-cloudflare-proxy.env"
|
|
| 79 |
if [ -n "${CLOUDFLARE_WORKERS_TOKEN:-}" ] || [ -n "${CLOUDFLARE_PROXY_URL:-}" ]; then
|
| 80 |
export CLOUDFLARE_PROXY_DOMAINS="${CLOUDFLARE_PROXY_DOMAINS:-*}"
|
| 81 |
export CLOUDFLARE_PROXY_DEBUG="${CLOUDFLARE_PROXY_DEBUG:-false}"
|
| 82 |
-
echo "
|
| 83 |
python3 "$APP_DIR/cloudflare-proxy-setup.py" || true
|
| 84 |
if [ -f "$CF_PROXY_ENV_FILE" ]; then
|
| 85 |
. "$CF_PROXY_ENV_FILE"
|
| 86 |
-
echo "
|
| 87 |
fi
|
| 88 |
fi
|
| 89 |
|
|
|
|
| 79 |
if [ -n "${CLOUDFLARE_WORKERS_TOKEN:-}" ] || [ -n "${CLOUDFLARE_PROXY_URL:-}" ]; then
|
| 80 |
export CLOUDFLARE_PROXY_DOMAINS="${CLOUDFLARE_PROXY_DOMAINS:-*}"
|
| 81 |
export CLOUDFLARE_PROXY_DEBUG="${CLOUDFLARE_PROXY_DEBUG:-false}"
|
| 82 |
+
echo "Preparing Cloudflare outbound proxy..."
|
| 83 |
python3 "$APP_DIR/cloudflare-proxy-setup.py" || true
|
| 84 |
if [ -f "$CF_PROXY_ENV_FILE" ]; then
|
| 85 |
. "$CF_PROXY_ENV_FILE"
|
| 86 |
+
echo " Proxy environment loaded: ${CLOUDFLARE_PROXY_URL:-none}"
|
| 87 |
fi
|
| 88 |
fi
|
| 89 |
|