diff --git a/src/business-logic/import-pipeline/sources/retrorides/scraper.js b/src/business-logic/import-pipeline/sources/retrorides/scraper.js index 0d15945..cff4dfa 100644 --- a/src/business-logic/import-pipeline/sources/retrorides/scraper.js +++ b/src/business-logic/import-pipeline/sources/retrorides/scraper.js @@ -5,6 +5,32 @@ "Accept-Encoding": "identity", "Connection": "keep-alive", "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', +}; + +const SITEMAP_HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Accept": "application/xml,text/xml,*/*;q=0.9", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "identity", + "Connection": "keep-alive", + "Cache-Control": "no-cache", + "Pragma": "no-cache", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"', + "Sec-CH-UA-Mobile": "?0", + "Sec-CH-UA-Platform": '"Windows"', }; const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml"; @@ -21,11 +47,12 @@ function sleep(ms) { return new Promise(function(resolve) { setTimeout(resolve, ms); }); } -async function fetchText(url, maxAttempts) { +async function fetchText(url, maxAttempts, headers) { maxAttempts = maxAttempts || 5; + headers = headers || FETCH_HEADERS; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - const response = await fetch(url, { headers: FETCH_HEADERS }); + const response = await fetch(url, { headers: headers }); // 429 = rate limit, 409 = WAF conflict from concurrent requests — both need backoff + retry if (response.status === 429 || response.status === 409) { const waitMs = attempt * 5000; @@ -166,7 +193,7 @@ async function scrapeProductPage(url) { } async function getAllProductUrls() { - const xml = await fetchText(SITEMAP_URL); + const xml = await fetchText(SITEMAP_URL, 5, SITEMAP_HEADERS); return Array.from(xml.matchAll(/([^<]+)<\/loc>/g)) .map(function(m) { return m[1].trim(); }) .filter(function(u) { return u.indexOf("/product/") !== -1; });