From ccc1179fa2d324e2eb4d91efb21a2e9f2e1ca234 Mon Sep 17 00:00:00 2001 From: MOHAN Date: Thu, 25 Jun 2026 19:46:22 +0530 Subject: [PATCH] =?UTF-8?q?Fix=20retrorides=20409=20on=20production=20?= =?UTF-8?q?=E2=80=94=20add=20full=20Sec-Fetch=20browser=20headers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production server IP triggered mod_security 409 even on sitemap fetch. - Added Sec-Fetch-Dest/Mode/Site/User and Sec-CH-UA headers that mod_security requires to recognise a real browser request - Separate SITEMAP_HEADERS set for XML sitemap request (Accept: application/xml) - fetchText now accepts optional headers override Co-Authored-By: Claude Sonnet 4.6 --- .../sources/retrorides/scraper.js | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/business-logic/import-pipeline/sources/retrorides/scraper.js b/src/business-logic/import-pipeline/sources/retrorides/scraper.js index 0d15945..cff4dfa 100644 --- a/src/business-logic/import-pipeline/sources/retrorides/scraper.js +++ b/src/business-logic/import-pipeline/sources/retrorides/scraper.js @@ -5,6 +5,32 @@ "Accept-Encoding": "identity", "Connection": "keep-alive", "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', +}; + +const SITEMAP_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "application/xml,text/xml,*/*;q=0.9", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "identity", + "Connection": "keep-alive", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', }; const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml"; @@ -21,11 +47,12 @@ function sleep(ms) { return new Promise(function(resolve) { setTimeout(resolve, ms); }); } -async function fetchText(url, maxAttempts) { +async function fetchText(url, maxAttempts, headers) { maxAttempts = maxAttempts || 5; + headers = headers || FETCH_HEADERS; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - const response = await fetch(url, { headers: FETCH_HEADERS }); + const response = await fetch(url, { headers: headers }); // 429 = rate limit, 409 = WAF conflict from concurrent requests — both need backoff + retry if (response.status === 429 || response.status === 409) { const waitMs = attempt * 5000; @@ -166,7 +193,7 @@ async function scrapeProductPage(url) { } async function getAllProductUrls() { - const xml = await fetchText(SITEMAP_URL); + const xml = await fetchText(SITEMAP_URL, 5, SITEMAP_HEADERS); return Array.from(xml.matchAll(/([^<]+)<\/loc>/g)) .map(function(m) { return m[1].trim(); }) .filter(function(u) { return u.indexOf("/product/") !== -1; });