feat: standardize uber error handling and transient retry policy

docs/developer-portal/09-errors-guide-audit.md

@@ -0,0 +1,35 @@
+# 09 Errors Guide Audit
+
+Source checked: Uber Eats "Errors" section shared by you.
+
+## Implemented Now
+
+- Centralized Uber error normalization with:
+  - HTTP status
+  - platform/error code
+  - transient classification
+  - raw upstream details
+- Safe retry strategy implemented in proxy/auth/reporting calls:
+  - retries only transient failures
+  - exponential backoff with jitter
+  - max 4 attempts
+- Global API error response now returns:
+  - `code`
+  - `transient`
+  - `message`
+  - `requestId`
+
+## Retry Classification Used
+
+- Retryable:
+  - `408`, `429`, `502`, `503`, `504`
+  - network timeout/connectivity/reset failures
+- Non-retryable by default:
+  - validation/auth/resource conflict 4xx responses
+  - `500` (unless network failure classification applies)
+
+## Pending
+
+- Per-endpoint custom retry budgets for high-impact operations
+- Circuit-breaker layer for prolonged upstream instability
+

docs/developer-portal/09-errors-retries.md

@@ -7,3 +7,15 @@ Standards:
 - Enforce idempotent writes where possible
 - Capture request/response logs for debugging
 
+Implemented behavior:
+
+- 4xx errors are treated as client/fatal (no retry by default).
+- Transient retries use exponential backoff + jitter for:
+  - `408`, `429`, `502`, `503`, `504`
+  - network timeout/reset/connectivity failures
+- Max attempts: `4` (initial + 3 retries).
+- Error response contract includes:
+  - `code`
+  - `transient`
+  - `message`
+  - `requestId`

src/middleware/errorHandler.js

@@ -2,11 +2,14 @@ module.exports = function errorHandler(err, req, res, next) {
   const status = err.status || 500;
   const payload = {
     success: false,
+    code: err.code || `http_${status}`,
+    transient: Boolean(err.transient),
     message: err.message || "Internal server error",
     requestId: req.requestId
   };
 
   if (process.env.NODE_ENV !== "production") {
+    payload.details = err.details;
     payload.stack = err.stack;
   }
 
@@ -17,4 +20,3 @@ module.exports = function errorHandler(err, req, res, next) {
 
   res.status(status).json(payload);
 };
-

src/modules/auth/auth.service.js

@@ -1,6 +1,8 @@
 const axios = require("axios");
 const env = require("../../config/env");
 const { appTokenRepository, tokenRequestLogRepository } = require("../../db/adapter");
+const { normalizeUberError, isRetryableUberError } = require("../common/errors/uberError");
+const { withExponentialBackoffRetry } = require("../common/http/retry");
 
 const AUTH_GRANT_TYPES = {
   CLIENT_CREDENTIALS: "client_credentials",
@@ -42,10 +44,18 @@ async function exchangeCodeForToken(code) {
     code
   });
 
-  const { data } = await uberAuthClient.post("/oauth/v2/token", payload.toString(), {
+  const { data } = await withExponentialBackoffRetry({
+    fn: async () =>
+      uberAuthClient.post("/oauth/v2/token", payload.toString(), {
         headers: {
           "Content-Type": "application/x-www-form-urlencoded"
         }
+      }),
+    maxAttempts: 4,
+    baseDelayMs: 250,
+    shouldRetry: (error) => isRetryableUberError(error)
+  }).catch((error) => {
+    throw normalizeUberError(error);
   });
 
   return data;
@@ -59,10 +69,18 @@ async function refreshToken(refreshToken) {
     refresh_token: refreshToken
   });
 
-  const { data } = await uberAuthClient.post("/oauth/v2/token", payload.toString(), {
+  const { data } = await withExponentialBackoffRetry({
+    fn: async () =>
+      uberAuthClient.post("/oauth/v2/token", payload.toString(), {
         headers: {
           "Content-Type": "application/x-www-form-urlencoded"
         }
+      }),
+    maxAttempts: 4,
+    baseDelayMs: 250,
+    shouldRetry: (error) => isRetryableUberError(error)
+  }).catch((error) => {
+    throw normalizeUberError(error);
   });
 
   return data;
@@ -76,10 +94,18 @@ async function getClientCredentialsToken({ scope = "eats.order" } = {}) {
     scope
   });
 
-  const { data } = await uberAuthClient.post("/oauth/v2/token", payload.toString(), {
+  const { data } = await withExponentialBackoffRetry({
+    fn: async () =>
+      uberAuthClient.post("/oauth/v2/token", payload.toString(), {
         headers: {
           "Content-Type": "application/x-www-form-urlencoded"
         }
+      }),
+    maxAttempts: 4,
+    baseDelayMs: 250,
+    shouldRetry: (error) => isRetryableUberError(error)
+  }).catch((error) => {
+    throw normalizeUberError(error);
   });
 
   return data;

src/modules/common/errors/uberError.js

@@ -0,0 +1,57 @@
+const TRANSIENT_HTTP_STATUSES = new Set([408, 429, 502, 503, 504]);
+
+function isNetworkOrTimeout(error) {
+  const code = String(error?.code || "").toUpperCase();
+  return (
+    code.includes("ETIMEDOUT") ||
+    code.includes("ECONNRESET") ||
+    code.includes("ENOTFOUND") ||
+    code.includes("EAI_AGAIN") ||
+    code.includes("ECONNABORTED")
+  );
+}
+
+function extractUberCode(responseData, fallbackStatus) {
+  if (!responseData || typeof responseData !== "object") {
+    return fallbackStatus ? `http_${fallbackStatus}` : "unknown_error";
+  }
+  return responseData.code || responseData.error || responseData.error_code || "unknown_error";
+}
+
+function extractUberMessage(responseData, fallbackMessage) {
+  if (!responseData || typeof responseData !== "object") {
+    return fallbackMessage || "Unknown error";
+  }
+  return responseData.message || responseData.error_description || fallbackMessage || "Unknown error";
+}
+
+function normalizeUberError(error) {
+  const status = error?.response?.status || 500;
+  const responseData = error?.response?.data;
+  const code = extractUberCode(responseData, status);
+  const message = extractUberMessage(responseData, error?.message);
+  const transient = TRANSIENT_HTTP_STATUSES.has(status) || isNetworkOrTimeout(error);
+
+  const normalized = new Error(`Uber API request failed: ${status} ${code}`);
+  normalized.status = status;
+  normalized.code = code;
+  normalized.transient = transient;
+  normalized.details = responseData || { message };
+  return normalized;
+}
+
+function isRetryableUberError(error) {
+  if (error?.transient === true) {
+    return true;
+  }
+  if (error?.response?.status && TRANSIENT_HTTP_STATUSES.has(error.response.status)) {
+    return true;
+  }
+  return isNetworkOrTimeout(error);
+}
+
+module.exports = {
+  normalizeUberError,
+  isRetryableUberError
+};
+

src/modules/common/http/retry.js

@@ -0,0 +1,35 @@
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function withExponentialBackoffRetry({
+  fn,
+  maxAttempts = 4,
+  baseDelayMs = 300,
+  shouldRetry
+}) {
+  let attempt = 0;
+  let lastError = null;
+
+  while (attempt < maxAttempts) {
+    try {
+      return await fn(attempt);
+    } catch (error) {
+      lastError = error;
+      attempt += 1;
+      if (attempt >= maxAttempts || !shouldRetry(error, attempt)) {
+        throw error;
+      }
+      const jitter = Math.floor(Math.random() * 200);
+      const delay = baseDelayMs * 2 ** attempt + jitter;
+      await sleep(delay);
+    }
+  }
+
+  throw lastError;
+}
+
+module.exports = {
+  withExponentialBackoffRetry
+};
+

src/modules/proxy/proxy.service.js

@@ -3,6 +3,8 @@ const env = require("../../config/env");
 const uberEndpoints = require("../../config/uberEndpoints");
 const { uberConnectionRepository, apiLogRepository } = require("../../db/adapter");
 const { getCachedClientCredentialsToken, AUTH_SCOPES } = require("../auth/auth.service");
+const { normalizeUberError, isRetryableUberError } = require("../common/errors/uberError");
+const { withExponentialBackoffRetry } = require("../common/http/retry");
 
 const uberApiClient = axios.create({
   baseURL: env.UBER_API_BASE_URL,
@@ -57,7 +59,9 @@ async function callUberApi({ merchantId, method, uberPath, query, body, wrapperR
   const resolvedAuth = await resolveAuthToken({ authMode, merchantId, scopes });
 
   try {
-    const response = await uberApiClient.request({
+    const response = await withExponentialBackoffRetry({
+      fn: async () =>
+        uberApiClient.request({
           method,
           url: uberPath,
           params: query,
@@ -66,6 +70,10 @@ async function callUberApi({ merchantId, method, uberPath, query, body, wrapperR
             Authorization: buildAuthHeader(resolvedAuth.tokenType, resolvedAuth.accessToken),
             "Content-Type": "application/json"
           }
+        }),
+      maxAttempts: 4,
+      baseDelayMs: 300,
+      shouldRetry: (error) => isRetryableUberError(error)
     });
 
     apiLogRepository.insert({
@@ -80,23 +88,24 @@ async function callUberApi({ merchantId, method, uberPath, query, body, wrapperR
 
     return response.data;
   } catch (error) {
-    const status = error.response?.status || 500;
+    const normalized = normalizeUberError(error);
-    const responseBody = error.response?.data || { message: error.message };
 
     apiLogRepository.insert({
       merchantId,
       method,
       wrapperRoute,
       uberPath,
-      responseStatus: status,
+      responseStatus: normalized.status,
       requestBody: body,
-      responseBody
+      responseBody: {
+        code: normalized.code,
+        message: normalized.message,
+        transient: normalized.transient,
+        details: normalized.details
+      }
     });
 
-    const wrapped = new Error(`Uber API request failed: ${status}`);
+    throw normalized;
-    wrapped.status = status;
-    wrapped.details = responseBody;
-    throw wrapped;
   }
 }
 

src/modules/reporting/reporting.service.js

@@ -2,51 +2,18 @@ const axios = require("axios");
 const env = require("../../config/env");
 const { apiLogRepository } = require("../../db/adapter");
 const { getCachedClientCredentialsToken, AUTH_SCOPES } = require("../auth/auth.service");
+const { normalizeUberError, isRetryableUberError } = require("../common/errors/uberError");
+const { withExponentialBackoffRetry } = require("../common/http/retry");
 
 const reportingClient = axios.create({
   baseURL: env.UBER_API_BASE_URL,
   timeout: 45000
 });
 
-const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
-
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
 function buildAuthorizationHeader(tokenType, accessToken) {
   return `${tokenType || "Bearer"} ${accessToken}`;
 }
 
-function isRetryableError(error) {
-  if (!error.response) {
-    return true;
-  }
-  return RETRYABLE_STATUSES.has(error.response.status);
-}
-
-async function withRetry(fn, maxAttempts = 4, baseDelayMs = 300) {
-  let attempt = 0;
-  let lastError = null;
-
-  while (attempt < maxAttempts) {
-    try {
-      return await fn();
-    } catch (error) {
-      lastError = error;
-      attempt += 1;
-      if (attempt >= maxAttempts || !isRetryableError(error)) {
-        throw error;
-      }
-      const jitter = Math.floor(Math.random() * 200);
-      const delay = baseDelayMs * 2 ** attempt + jitter;
-      await sleep(delay);
-    }
-  }
-
-  throw lastError;
-}
-
 function parseCsvLine(line) {
   const output = [];
   let current = "";
@@ -128,8 +95,8 @@ async function fetchReport({
   const requestMethod = String(method || "GET").toUpperCase();
 
   try {
-    const response = await withRetry(
+    const response = await withExponentialBackoffRetry({
-      async () =>
+      fn: async () =>
         reportingClient.request({
           method: requestMethod,
           url: upstreamPath,
@@ -141,9 +108,10 @@ async function fetchReport({
             "Content-Type": "application/json"
           }
         }),
-      4,
+      maxAttempts: 4,
-      400
+      baseDelayMs: 400,
-    );
+      shouldRetry: (error) => isRetryableUberError(error)
+    });
 
     const textData =
       typeof response.data === "string" ? response.data : JSON.stringify(response.data || {});
@@ -165,22 +133,24 @@ async function fetchReport({
 
     return payload;
   } catch (error) {
-    const status = error.response?.status || 500;
+    const normalized = normalizeUberError(error);
-    const responseBody = error.response?.data || { message: error.message };
 
     apiLogRepository.insert({
       method: requestMethod,
       wrapperRoute,
       uberPath: upstreamPath,
-      responseStatus: status,
+      responseStatus: normalized.status,
       requestBody: body,
-      responseBody: typeof responseBody === "string" ? { raw: responseBody } : responseBody
+      responseBody: {
+        code: normalized.code,
+        message: normalized.message,
+        transient: normalized.transient,
+        details:
+          typeof normalized.details === "string" ? { raw: normalized.details } : normalized.details
+      }
     });
 
-    const wrapped = new Error(`Reporting fetch failed: ${status}`);
+    throw normalized;
-    wrapped.status = status;
-    wrapped.details = responseBody;
-    throw wrapped;
   }
 }
 
@@ -188,4 +158,3 @@ module.exports = {
   fetchReport,
   parseCsvByHeader
 };
-