Update downloader

safaribooks/cookies.js

@@ -44,3 +44,7 @@ export async function collectSessionCookies() {
     cookies: jar
   };
 }
+
+export function updateRequestContext() {
+  // Chrome/Edge do not require extra request header handling.
+}

safaribooks/downloader.js

@@ -14,6 +14,7 @@ import {
   isLikelyImageAsset
 } from "./utils.js";
 import { SimpleZip } from "./zip.js";
+import { collectSessionCookies, updateRequestContext } from "./cookies.js";
 
 const SAFARI_BASE_URL = "https://learning.oreilly.com";
 const API_V2_TEMPLATE = `${SAFARI_BASE_URL}/api/v2/epubs/urn:orm:book:`;
@@ -32,6 +33,10 @@ const KINDLE_CSS =
   "#sbo-rt-content table,#sbo-rt-content pre{overflow-x:unset!important;overflow:unset!important;" +
   "overflow-y:unset!important;white-space:pre-wrap!important;}";
 
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, Math.max(ms, 0)));
+}
+
 function formatTwoDigits(value) {
   return value.toString().padStart(2, "0");
 }
@@ -94,6 +99,12 @@ export class SafariBooksDownloader {
     this.coverPath = "";
     this.themeMode = "white";
     this.options = { theme: "none", kindle: false };
+    this.referrerUrl = SAFARI_BASE_URL;
+    this.baseRequestSpacingMs = 200;
+    this.nextRequestTime = Date.now();
+    this.rateLimitPenaltyMs = 0;
+    this.sessionRefreshCooldownMs = 30000;
+    this.lastSessionRefresh = 0;
 
     this.cssSources = [];
     this.cssFilenameMap = new Map();
@@ -105,6 +116,167 @@ export class SafariBooksDownloader {
     this.imagesFromLinks = new Set();
   }
 
+  getReferrerUrl() {
+    return this.referrerUrl || SAFARI_BASE_URL;
+  }
+
+  updateReferrer(url) {
+    if (typeof url === "string" && url.startsWith("http")) {
+      this.referrerUrl = url;
+    }
+  }
+
+  async ensureSessionRefreshed({ force = false } = {}) {
+    if (typeof collectSessionCookies !== "function") {
+      return;
+    }
+    const now = Date.now();
+    if (!force && now - this.lastSessionRefresh < this.sessionRefreshCooldownMs) {
+      return;
+    }
+    try {
+      await collectSessionCookies();
+      this.lastSessionRefresh = Date.now();
+    } catch (error) {
+      this.log(`Warning: unable to refresh session cookies (${error.message}).`);
+    }
+  }
+
+  async applyRateLimit() {
+    const waitTime = this.nextRequestTime - Date.now();
+    if (waitTime > 0) {
+      await sleep(waitTime);
+    }
+  }
+
+  registerRequestOutcome(status) {
+    const penalizedStatuses = new Set([0, 401, 403, 429, 500, 502, 503, 504]);
+    if (penalizedStatuses.has(status)) {
+      if (status === 429 || status === 503) {
+        this.rateLimitPenaltyMs = Math.min(Math.max(this.rateLimitPenaltyMs * 2, 500), 8000);
+      } else {
+        this.rateLimitPenaltyMs = Math.min(this.rateLimitPenaltyMs + 250, 6000);
+      }
+    } else {
+      this.rateLimitPenaltyMs = Math.max(Math.floor(this.rateLimitPenaltyMs * 0.5), 0);
+    }
+    const delay = this.baseRequestSpacingMs + this.rateLimitPenaltyMs;
+    this.nextRequestTime = Date.now() + delay;
+  }
+
+  shouldRetryResponse(response, attempt, maxAttempts, options = {}) {
+    if (attempt >= maxAttempts - 1) {
+      return false;
+    }
+    const status = response?.status ?? 0;
+    if (status === 0) {
+      return true;
+    }
+    if ([429, 500, 502, 503, 504].includes(status)) {
+      return true;
+    }
+    if ((status === 401 || status === 403) && options.retryOnForbidden !== false) {
+      return true;
+    }
+    return false;
+  }
+
+  computeRetryDelay(response, attempt, options = {}) {
+    const status = response?.status ?? 0;
+    const base = options.baseRetryDelay ?? 750;
+    const cap = options.maxRetryDelay ?? 8000;
+    const growth = Math.min(cap, base * Math.pow(2, attempt));
+    const jitter = Math.random() * 250;
+    if (status === 429 || status === 503) {
+      return Math.min(cap, growth + 1000 + jitter);
+    }
+    return Math.min(cap, growth + jitter);
+  }
+
+  buildFetchInit(init = {}, { requireDocument = false } = {}) {
+    const { headers, ...rest } = init || {};
+    const mergedHeaders = new Headers(headers || {});
+    if (!mergedHeaders.has("X-Requested-With")) {
+      mergedHeaders.set("X-Requested-With", "XMLHttpRequest");
+    }
+    if (requireDocument && !mergedHeaders.has("Accept")) {
+      mergedHeaders.set(
+        "Accept",
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+      );
+    }
+    if (!mergedHeaders.has("Accept-Language")) {
+      mergedHeaders.set("Accept-Language", "en-US,en;q=0.9");
+    }
+
+    const initWithDefaults = {
+      credentials: "include",
+      mode: "cors",
+      ...rest,
+      headers: mergedHeaders
+    };
+
+    if (!initWithDefaults.referrer) {
+      initWithDefaults.referrer = this.getReferrerUrl();
+    }
+    if (!initWithDefaults.referrerPolicy) {
+      initWithDefaults.referrerPolicy = "strict-origin-when-cross-origin";
+    }
+    return initWithDefaults;
+  }
+
+  async fetchWithSession(url, init = {}, options = {}) {
+    const maxAttempts = Math.max(1, options.maxAttempts ?? 4);
+    let lastError = null;
+    let lastResponse = null;
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      await this.applyRateLimit();
+
+      try {
+        const response = await fetch(url, this.buildFetchInit(init, options));
+        lastResponse = response;
+        this.registerRequestOutcome(response.status);
+
+        if (response.ok) {
+          return response;
+        }
+
+        const shouldRetry = this.shouldRetryResponse(response, attempt, maxAttempts, options);
+        if (!shouldRetry) {
+          return response;
+        }
+
+        if ((response.status === 401 || response.status === 403) && options.retryOnForbidden !== false) {
+          await this.ensureSessionRefreshed({ force: true });
+        }
+
+        const retryDelay = this.computeRetryDelay(response, attempt, options);
+        const targetTime = Date.now() + retryDelay;
+        if (targetTime > this.nextRequestTime) {
+          this.nextRequestTime = targetTime;
+        }
+      } catch (error) {
+        lastError = error;
+        this.registerRequestOutcome(0);
+        if (attempt >= maxAttempts - 1) {
+          throw error;
+        }
+
+        const retryDelay = this.computeRetryDelay(null, attempt, options);
+        const targetTime = Date.now() + retryDelay;
+        if (targetTime > this.nextRequestTime) {
+          this.nextRequestTime = targetTime;
+        }
+      }
+    }
+
+    if (lastResponse) {
+      return lastResponse;
+    }
+    throw lastError ?? new Error(`Request to ${url} failed without a response.`);
+  }
+
   async download(bookId, options = {}) {
     this.reset();
     this.bookId = bookId;
@@ -113,6 +285,8 @@ export class SafariBooksDownloader {
       kindle: Boolean(options.kindle)
     };
     this.themeMode = this.options.theme === "none" ? "white" : this.options.theme;
+    this.updateReferrer(`${SAFARI_BASE_URL}/library/view/${this.bookId}/`);
+    updateRequestContext({ referer: this.getReferrerUrl() });
 
     this.log(`Verifying session by loading profile page...`);
     await this.checkLogin();
@@ -120,6 +294,8 @@ export class SafariBooksDownloader {
     this.log(`Retrieving book metadata for ${bookId}...`);
     this.bookInfo = await this.fetchBookInfo(bookId);
     this.baseUrl = this.bookInfo.web_url || this.bookInfo.url;
+    this.updateReferrer(this.bookInfo.web_url || this.bookInfo.url);
+    updateRequestContext({ referer: this.getReferrerUrl() });
     this.cleanTitle = makeFolderFriendlyName(this.bookInfo.title, this.bookId);
     const authorNames = (this.bookInfo.authors || []).map((author) => author.name).filter(Boolean);
     this.primaryAuthors = authorNames.join(", ") || "Unknown Author";
@@ -147,7 +323,7 @@ export class SafariBooksDownloader {
   }
 
   async checkLogin() {
-    const response = await fetch(PROFILE_URL, { credentials: "include" });
+    const response = await this.fetchWithSession(PROFILE_URL, {}, { requireDocument: true });
     if (!response.ok) {
       throw new Error(`Authentication failed (status ${response.status}) when accessing ${PROFILE_URL}`);
     }
@@ -191,7 +367,11 @@ export class SafariBooksDownloader {
   }
 
   async fetchJson(url) {
-    const response = await fetch(url, { credentials: "include" });
+    const response = await this.fetchWithSession(url, {
+      headers: {
+        Accept: "application/json, text/plain, */*"
+      }
+    });
     if (!response.ok) {
       throw new Error(`Request to ${url} failed with status ${response.status}`);
     }
@@ -361,7 +541,15 @@ export class SafariBooksDownloader {
   }
 
   async loadChapterDocument(url) {
-    const response = await fetch(url, { credentials: "include" });
+    const response = await this.fetchWithSession(
+      url,
+      {
+        headers: {
+          Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        }
+      },
+      { requireDocument: true }
+    );
     if (!response.ok) {
       throw new Error(`Failed to download chapter at ${url} (status ${response.status})`);
     }
@@ -481,7 +669,11 @@ export class SafariBooksDownloader {
       if (this.cssContentMap.has(source)) {
         continue;
       }
-      const response = await fetch(source, { credentials: "include" });
+      const response = await this.fetchWithSession(source, {
+        headers: {
+          Accept: "text/css,*/*;q=0.1"
+        }
+      });
       if (!response.ok) {
         this.log(`Warning: unable to retrieve CSS from ${source} (status ${response.status})`);
         continue;
@@ -527,7 +719,11 @@ export class SafariBooksDownloader {
       if (this.cssImageDownloads.has(localPath)) {
         continue;
       }
-      const response = await fetch(url, { credentials: "include" });
+      const response = await this.fetchWithSession(url, {
+        headers: {
+          Accept: "font/woff2,application/font-woff,application/octet-stream,*/*"
+        }
+      });
       if (!response.ok) {
         this.log(`Warning: unable to retrieve font ${url} (status ${response.status})`);
         continue;
@@ -557,7 +753,11 @@ export class SafariBooksDownloader {
       }
       tried.add(candidate);
       try {
-        const response = await fetch(candidate, { credentials: "include" });
+        const response = await this.fetchWithSession(candidate, {
+          headers: {
+            Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
+          }
+        });
         if (!response.ok) {
           attempts.push(`${candidate} (status ${response.status})`);
           continue;
@@ -623,7 +823,11 @@ export class SafariBooksDownloader {
       if (entry?.data) {
         continue;
       }
-      const response = await fetch(entry, { credentials: "include" });
+      const response = await this.fetchWithSession(entry, {
+        headers: {
+          Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
+        }
+      });
       if (!response.ok) {
         this.log(`Warning: unable to fetch CSS image ${entry} (status ${response.status})`);
         continue;