admin/Race-Nation-Shopify-Backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix retrorides 409 on production — add full Sec-Fetch browser headers

Browse Source 
Production server IP triggered mod_security 409 even on sitemap fetch.
- Added Sec-Fetch-Dest/Mode/Site/User and Sec-CH-UA headers that
  mod_security requires to recognise a real browser request
- Separate SITEMAP_HEADERS set for XML sitemap request (Accept: application/xml)
- fetchText now accepts optional headers override

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MOHAN 2026-06-25 19:46:22 +05:30
parent 6938c113ae
commit ccc1179fa2
1 changed files with 30 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/business-logic/import-pipeline/sources/retrorides/scraper.js
View File

@ -5,6 +5,32 @@
"Accept-Encoding": "identity", "Accept-Encoding": "identity",
"Connection": "keep-alive", "Connection": "keep-alive",
"Cache-Control": "no-cache", "Cache-Control": "no-cache",
"Pragma": "no-cache",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
};
const SITEMAP_HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Accept": "application/xml,text/xml,*/*;q=0.9",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "identity",
"Connection": "keep-alive",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Sec-CH-UA": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"Sec-CH-UA-Mobile": "?0",
"Sec-CH-UA-Platform": '"Windows"',
}; };
const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml"; const SITEMAP_URL = "https://retrorides.co.in/wp-sitemap-posts-product-1.xml";
@ -21,11 +47,12 @@ function sleep(ms) {
return new Promise(function(resolve) { setTimeout(resolve, ms); }); return new Promise(function(resolve) { setTimeout(resolve, ms); });
} }
async function fetchText(url, maxAttempts) { async function fetchText(url, maxAttempts, headers) {
maxAttempts = maxAttempts || 5; maxAttempts = maxAttempts || 5;
headers = headers || FETCH_HEADERS;
for (let attempt = 1; attempt <= maxAttempts; attempt++) { for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try { try {
const response = await fetch(url, { headers: FETCH_HEADERS }); const response = await fetch(url, { headers: headers });
// 429 = rate limit, 409 = WAF conflict from concurrent requests — both need backoff + retry // 429 = rate limit, 409 = WAF conflict from concurrent requests — both need backoff + retry
if (response.status === 429 || response.status === 409) { if (response.status === 429 || response.status === 409) {
const waitMs = attempt * 5000; const waitMs = attempt * 5000;
@ -166,7 +193,7 @@ async function scrapeProductPage(url) {
} }
async function getAllProductUrls() { async function getAllProductUrls() {
const xml = await fetchText(SITEMAP_URL); const xml = await fetchText(SITEMAP_URL, 5, SITEMAP_HEADERS);
return Array.from(xml.matchAll(/<loc>([^<]+)<\/loc>/g)) return Array.from(xml.matchAll(/<loc>([^<]+)<\/loc>/g))
.map(function(m) { return m[1].trim(); }) .map(function(m) { return m[1].trim(); })
.filter(function(u) { return u.indexOf("/product/") !== -1; }); .filter(function(u) { return u.indexOf("/product/") !== -1; });