fix: allow POST with url

backend/functions/src/cloud-functions/crawler.ts

@@ -571,12 +571,14 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
             res: Response,
         },
         auth: JinaEmbeddingsAuthDTO,
-        crawlerOptions: CrawlerOptionsHeaderOnly,
+        crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly,
+        crawlerOptionsParamsAllowed: CrawlerOptions,
     ) {
         const uid = await auth.solveUID();
         let chargeAmount = 0;
         const noSlashURL = ctx.req.url.slice(1);
-        if (!noSlashURL) {
+        const crawlerOptions = ctx.req.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed;
+        if (!noSlashURL && !crawlerOptions.url) {
             const latestUser = uid ? await auth.assertUser() : undefined;
             if (!ctx.req.accepts('text/plain') && (ctx.req.accepts('text/json') || ctx.req.accepts('application/json'))) {
                 return this.getIndex(latestUser);
@@ -636,7 +638,17 @@ ${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`;
         let urlToCrawl;
         const normalizeUrl = (await pNormalizeUrl).default;
         try {
-            urlToCrawl = new URL(normalizeUrl(noSlashURL.trim(), { stripWWW: false, removeTrailingSlash: false, removeSingleSlash: false, sortQueryParameters: false }));
+            urlToCrawl = new URL(
+                normalizeUrl(
+                    (crawlerOptions.url || noSlashURL).trim(),
+                    {
+                        stripWWW: false,
+                        removeTrailingSlash: false,
+                        removeSingleSlash: false,
+                        sortQueryParameters: false,
+                    }
+                )
+            );
         } catch (err) {
             throw new ParamValidationError({
                 message: `${err}`,

backend/functions/src/dto/scrapping-options.ts

@@ -116,6 +116,9 @@ import { parseString as parseSetCookieString } from 'set-cookie-parser';
 })
 export class CrawlerOptions extends AutoCastable {
 
+    @Prop()
+    url?: string;
+
     @Prop({
         default: 'default',
     })

thinapps-shared

@@ -1 +1 @@
-Subproject commit 128201c8a6b3b5f1145c310fb144e669b1feba30
+Subproject commit a26b633136a2651f01dcf02b47efc35b2401d807