Firefox update

README.md

@@ -1,6 +1,6 @@
-# Reilly SafariBooks Downloader (Chrome & Edge)
+# Reilly SafariBooks Downloader (Chrome, Edge & Firefox)
 
-This extension ports the `safaribooks.py` script into a browser workflow so you can download EPUBs directly from [learning.oreilly.com](https://learning.oreilly.com).
+This extension ports the `safaribooks.py` script into a browser workflow so you can download EPUBs directly from [learning.oreilly.com](https://learning.oreilly.com) in Chrome, Edge, or Firefox.
 
 ## Installation (Chrome or Edge)
 
@@ -10,8 +10,27 @@ This extension ports the `safaribooks.py` script into a browser workflow so you
    - **Edge**: `edge://extensions/`
 3. Enable **Developer mode** (toggle in the top-right corner).
 4. Click **Load unpacked** and select this project folder.
-5. The “Reilly SafariBooks Downloader” icon should appear in the toolbar.
+5. The "Reilly SafariBooks Downloader" icon should appear in the toolbar.
 
+
+## Installation (Firefox)
+
+The packaged `.xpi` is unsigned, so the Firefox release channel blocks it. Pick the approach that matches your setup:
+
+- **Temporary load (any Firefox build)**
+  1. Open `about:debugging#/runtime/this-firefox`.
+  2. Click **Load Temporary Add-on...** and select `firefox-extension.xpi`.
+  3. Keep the tab open while testing; Firefox unloads temporary add-ons after a restart.
+
+- **Persistent install for testing (Firefox Developer Edition or Nightly)**
+  1. Switch to Developer Edition or Nightly—only these builds allow signing to be disabled.
+  2. In `about:config`, set `xpinstall.signatures.required` to `false`.
+  3. Open `about:addons`, click the gear icon, and choose **Install Add-on From File...**.
+  4. Pick `firefox-extension.xpi`, confirm the prompts, then flip the preference back to `true` when you return to normal browsing.
+
+- **Permanent install on Firefox Release**
+  1. Sign the add-on through Mozilla’s Add-on Developer Hub (https://addons.mozilla.org/developers/) or `web-ext sign`.
+  2. Use the signed `.xpi` that Mozilla returns; only signed builds will install on stable Firefox.
 ## Usage
 
 1. Log into [learning.oreilly.com](https://learning.oreilly.com) in the same browser profile.
@@ -27,3 +46,4 @@ This extension ports the `safaribooks.py` script into a browser workflow so you
 - The extension uses your authenticated cookies; ensure you are signed in before downloading.
 - If an image fetch fails, the downloader logs each attempt and tries alternate URLs.
 - Placeholder icons live in the `images/` directory—swap them out with custom artwork if you prefer.
+

firefox/manifest.json

@@ -43,6 +43,8 @@
     "cookies",
     "tabs",
     "storage",
+    "webRequest",
+    "webRequestBlocking",
     "https://learning.oreilly.com/*",
     "https://www.oreilly.com/*",
     "https://api.oreilly.com/*"

firefox/safaribooks/cookies.js

@@ -3,6 +3,16 @@ const rawChrome = typeof chrome !== "undefined" ? chrome : null;
 
 const PROFILE_URL = "https://learning.oreilly.com/profile/";
 const COOKIE_DOMAINS = ["learning.oreilly.com", ".oreilly.com", "www.oreilly.com"];
+const COOKIE_URL_FILTERS = [
+  "https://learning.oreilly.com/*",
+  "https://www.oreilly.com/*",
+  "https://api.oreilly.com/*"
+];
+
+let cookieHeader = "";
+let refererHeader = "";
+let originHeader = "";
+let webRequestHookReady = false;
 
 function callCookiesGetAll(filter) {
   const result = runtime.cookies.getAll(filter);
@@ -23,17 +33,101 @@ function callCookiesGetAll(filter) {
 }
 
 async function gatherCookies() {
-  const collected = {};
+  const collected = new Map();
+  let stores = [];
+  try {
+    if (typeof runtime.cookies.getAllCookieStores === "function") {
+      stores = await runtime.cookies.getAllCookieStores();
+    }
+  } catch (_) {
+    stores = [];
+  }
+
+  const storeIds = stores.length ? stores.map((store) => store.id) : [undefined];
+
   for (const domain of COOKIE_DOMAINS) {
-    const cookies = await callCookiesGetAll({ domain });
+    for (const storeId of storeIds) {
-    cookies.forEach((cookie) => {
+      const filter = storeId ? { domain, storeId } : { domain };
-      collected[cookie.name] = cookie.value;
+      const cookies = await callCookiesGetAll(filter);
-    });
+      cookies.forEach((cookie) => {
+        collected.set(cookie.name, cookie.value);
+      });
+    }
   }
   return collected;
 }
 
+function buildCookieHeader(jar) {
+  if (!jar || jar.size === 0) {
+    return "";
+  }
+  return Array.from(jar.entries())
+    .map(([name, value]) => `${name}=${value}`)
+    .join("; ");
+}
+
+function ensureWebRequestHook() {
+  if (webRequestHookReady || !runtime.webRequest?.onBeforeSendHeaders) {
+    return;
+  }
+
+  runtime.webRequest.onBeforeSendHeaders.addListener(
+    (details) => {
+      const isExtensionRequest =
+        details.tabId === -1 ||
+        (typeof details.initiator === "string" && details.initiator.startsWith("moz-extension://"));
+      if (!isExtensionRequest) {
+        return {};
+      }
+
+      const headers = details.requestHeaders ? [...details.requestHeaders] : [];
+      const lower = (name) => name.toLowerCase();
+
+      if (cookieHeader) {
+        const existingCookie = headers.find((header) => lower(header.name) === "cookie");
+        if (existingCookie) {
+          existingCookie.value = cookieHeader;
+        } else {
+          headers.push({ name: "Cookie", value: cookieHeader });
+        }
+      }
+
+      if (refererHeader) {
+        const existingReferer = headers.find((header) => lower(header.name) === "referer");
+        if (existingReferer) {
+          existingReferer.value = refererHeader;
+        } else {
+          headers.push({ name: "Referer", value: refererHeader });
+        }
+      }
+
+      if (originHeader) {
+        const existingOrigin = headers.find((header) => lower(header.name) === "origin");
+        if (existingOrigin) {
+          existingOrigin.value = originHeader;
+        } else {
+          headers.push({ name: "Origin", value: originHeader });
+        }
+      }
+      return { requestHeaders: headers };
+    },
+    { urls: COOKIE_URL_FILTERS, types: ["xmlhttprequest", "other"] },
+    ["blocking", "requestHeaders"]
+  );
+
+  webRequestHookReady = true;
+}
+
 export async function collectSessionCookies() {
+  const jar = await gatherCookies();
+  const count = jar.size;
+  if (!count) {
+    throw new Error("No O'Reilly cookies detected. Make sure you visited learning.oreilly.com in this profile.");
+  }
+
+  cookieHeader = buildCookieHeader(jar);
+  ensureWebRequestHook();
+
   let response;
   try {
     response = await fetch(PROFILE_URL, { credentials: "include" });
@@ -49,16 +143,22 @@ export async function collectSessionCookies() {
     throw new Error(`Profile check failed (status ${response.status}).`);
   }
 
-  const jar = await gatherCookies();
-  const count = Object.keys(jar).length;
-  if (!count) {
-    throw new Error("No O'Reilly cookies detected. Make sure you visited learning.oreilly.com in this profile.");
-  }
-
   return {
     status: response.status,
     count,
-    cookies: jar
+    cookies: Object.fromEntries(jar.entries())
   };
 }
 
+export function updateRequestContext({ referer } = {}) {
+  if (!referer) {
+    return;
+  }
+  try {
+    refererHeader = referer;
+    const url = new URL(referer);
+    originHeader = `${url.protocol}//${url.host}`;
+  } catch (_) {
+    // ignore invalid referrer values
+  }
+}

firefox/safaribooks/downloader.js

@@ -14,6 +14,7 @@ import {
   isLikelyImageAsset
 } from "./utils.js";
 import { SimpleZip } from "./zip.js";
+import { collectSessionCookies, updateRequestContext } from "./cookies.js";
 
 const SAFARI_BASE_URL = "https://learning.oreilly.com";
 const API_V2_TEMPLATE = `${SAFARI_BASE_URL}/api/v2/epubs/urn:orm:book:`;
@@ -32,25 +33,6 @@ const KINDLE_CSS =
   "#sbo-rt-content table,#sbo-rt-content pre{overflow-x:unset!important;overflow:unset!important;" +
   "overflow-y:unset!important;white-space:pre-wrap!important;}";
 
-function downloadWithWebExtensionsApi(options) {
-  if (typeof browser !== "undefined" && browser?.downloads?.download) {
-    return browser.downloads.download(options);
-  }
-  if (typeof chrome !== "undefined" && chrome?.downloads?.download) {
-    return new Promise((resolve, reject) => {
-      chrome.downloads.download(options, (downloadId) => {
-        const err = chrome.runtime?.lastError;
-        if (err) {
-          reject(new Error(err.message));
-        } else {
-          resolve(downloadId);
-        }
-      });
-    });
-  }
-  return Promise.reject(new Error("Downloads API is not available in this browser."));
-}
-
 function formatTwoDigits(value) {
   return value.toString().padStart(2, "0");
 }
@@ -113,6 +95,12 @@ export class SafariBooksDownloader {
     this.coverPath = "";
     this.themeMode = "white";
     this.options = { theme: "none", kindle: false };
+    this.referrerUrl = SAFARI_BASE_URL;
+    this.baseRequestSpacingMs = 200;
+    this.nextRequestTime = Date.now();
+    this.rateLimitPenaltyMs = 0;
+    this.sessionRefreshCooldownMs = 30000;
+    this.lastSessionRefresh = 0;
 
     this.cssSources = [];
     this.cssFilenameMap = new Map();
@@ -124,6 +112,167 @@ export class SafariBooksDownloader {
     this.imagesFromLinks = new Set();
   }
 
+  getReferrerUrl() {
+    return this.referrerUrl || SAFARI_BASE_URL;
+  }
+
+  updateReferrer(url) {
+    if (typeof url === "string" && url.startsWith("http")) {
+      this.referrerUrl = url;
+    }
+  }
+
+  async ensureSessionRefreshed({ force = false } = {}) {
+    if (typeof collectSessionCookies !== "function") {
+      return;
+    }
+    const now = Date.now();
+    if (!force && now - this.lastSessionRefresh < this.sessionRefreshCooldownMs) {
+      return;
+    }
+    try {
+      await collectSessionCookies();
+      this.lastSessionRefresh = Date.now();
+    } catch (error) {
+      this.log(`Warning: unable to refresh session cookies (${error.message}).`);
+    }
+  }
+
+  async applyRateLimit() {
+    const waitTime = this.nextRequestTime - Date.now();
+    if (waitTime > 0) {
+      await new Promise((resolve) => setTimeout(resolve, waitTime));
+    }
+  }
+
+  registerRequestOutcome(status) {
+    const penalizedStatuses = new Set([0, 401, 403, 429, 500, 502, 503, 504]);
+    if (penalizedStatuses.has(status)) {
+      if (status === 429 || status === 503) {
+        this.rateLimitPenaltyMs = Math.min(Math.max(this.rateLimitPenaltyMs * 2, 500), 8000);
+      } else {
+        this.rateLimitPenaltyMs = Math.min(this.rateLimitPenaltyMs + 250, 6000);
+      }
+    } else {
+      this.rateLimitPenaltyMs = Math.max(Math.floor(this.rateLimitPenaltyMs * 0.5), 0);
+    }
+    const delay = this.baseRequestSpacingMs + this.rateLimitPenaltyMs;
+    this.nextRequestTime = Date.now() + delay;
+  }
+
+  shouldRetryResponse(response, attempt, maxAttempts, options = {}) {
+    if (attempt >= maxAttempts - 1) {
+      return false;
+    }
+    const status = response?.status ?? 0;
+    if (status === 0) {
+      return true;
+    }
+    if ([429, 500, 502, 503, 504].includes(status)) {
+      return true;
+    }
+    if ((status === 401 || status === 403) && options.retryOnForbidden !== false) {
+      return true;
+    }
+    return false;
+  }
+
+  computeRetryDelay(response, attempt, options = {}) {
+    const status = response?.status ?? 0;
+    const base = options.baseRetryDelay ?? 750;
+    const cap = options.maxRetryDelay ?? 8000;
+    const growth = Math.min(cap, base * Math.pow(2, attempt));
+    const jitter = Math.random() * 250;
+    if (status === 429 || status === 503) {
+      return Math.min(cap, growth + 1000 + jitter);
+    }
+    return Math.min(cap, growth + jitter);
+  }
+
+  buildFetchInit(init = {}, { requireDocument = false } = {}) {
+    const { headers, ...rest } = init || {};
+    const mergedHeaders = new Headers(headers || {});
+    if (!mergedHeaders.has("X-Requested-With")) {
+      mergedHeaders.set("X-Requested-With", "XMLHttpRequest");
+    }
+    if (requireDocument && !mergedHeaders.has("Accept")) {
+      mergedHeaders.set(
+        "Accept",
+        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+      );
+    }
+    if (!mergedHeaders.has("Accept-Language")) {
+      mergedHeaders.set("Accept-Language", "en-US,en;q=0.9");
+    }
+
+    const initWithDefaults = {
+      credentials: "include",
+      mode: "cors",
+      ...rest,
+      headers: mergedHeaders
+    };
+
+    if (!initWithDefaults.referrer) {
+      initWithDefaults.referrer = this.getReferrerUrl();
+    }
+    if (!initWithDefaults.referrerPolicy) {
+      initWithDefaults.referrerPolicy = "strict-origin-when-cross-origin";
+    }
+    return initWithDefaults;
+  }
+
+  async fetchWithSession(url, init = {}, options = {}) {
+    const maxAttempts = Math.max(1, options.maxAttempts ?? 4);
+    let lastError = null;
+    let lastResponse = null;
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      await this.applyRateLimit();
+
+      try {
+        const response = await fetch(url, this.buildFetchInit(init, options));
+        lastResponse = response;
+        this.registerRequestOutcome(response.status);
+
+        if (response.ok) {
+          return response;
+        }
+
+        const shouldRetry = this.shouldRetryResponse(response, attempt, maxAttempts, options);
+        if (!shouldRetry) {
+          return response;
+        }
+
+        if ((response.status === 401 || response.status === 403) && options.retryOnForbidden !== false) {
+          await this.ensureSessionRefreshed({ force: true });
+        }
+
+        const retryDelay = this.computeRetryDelay(response, attempt, options);
+        const targetTime = Date.now() + retryDelay;
+        if (targetTime > this.nextRequestTime) {
+          this.nextRequestTime = targetTime;
+        }
+      } catch (error) {
+        lastError = error;
+        this.registerRequestOutcome(0);
+        if (attempt >= maxAttempts - 1) {
+          throw error;
+        }
+
+        const retryDelay = this.computeRetryDelay(null, attempt, options);
+        const targetTime = Date.now() + retryDelay;
+        if (targetTime > this.nextRequestTime) {
+          this.nextRequestTime = targetTime;
+        }
+      }
+    }
+
+    if (lastResponse) {
+      return lastResponse;
+    }
+    throw lastError ?? new Error(`Request to ${url} failed without a response.`);
+  }
+
   async download(bookId, options = {}) {
     this.reset();
     this.bookId = bookId;
@@ -132,6 +281,8 @@ export class SafariBooksDownloader {
       kindle: Boolean(options.kindle)
     };
     this.themeMode = this.options.theme === "none" ? "white" : this.options.theme;
+    this.updateReferrer(`${SAFARI_BASE_URL}/library/view/${this.bookId}/`);
+    updateRequestContext({ referer: this.getReferrerUrl() });
 
     this.log(`Verifying session by loading profile page...`);
     await this.checkLogin();
@@ -139,6 +290,8 @@ export class SafariBooksDownloader {
     this.log(`Retrieving book metadata for ${bookId}...`);
     this.bookInfo = await this.fetchBookInfo(bookId);
     this.baseUrl = this.bookInfo.web_url || this.bookInfo.url;
+    this.updateReferrer(this.bookInfo.web_url || this.bookInfo.url);
+    updateRequestContext({ referer: this.getReferrerUrl() });
     this.cleanTitle = makeFolderFriendlyName(this.bookInfo.title, this.bookId);
     const authorNames = (this.bookInfo.authors || []).map((author) => author.name).filter(Boolean);
     this.primaryAuthors = authorNames.join(", ") || "Unknown Author";
@@ -161,12 +314,12 @@ export class SafariBooksDownloader {
     const url = URL.createObjectURL(blob);
 
     const filename = `${this.cleanTitle}/${this.outputFileName}.epub`;
-    await downloadWithWebExtensionsApi({ url, filename, saveAs: false });
+    await chrome.downloads.download({ url, filename, saveAs: false });
     this.log(`EPUB saved as ${filename}`);
   }
 
   async checkLogin() {
-    const response = await fetch(PROFILE_URL, { credentials: "include" });
+    const response = await this.fetchWithSession(PROFILE_URL, {}, { requireDocument: true });
     if (!response.ok) {
       throw new Error(`Authentication failed (status ${response.status}) when accessing ${PROFILE_URL}`);
     }
@@ -210,7 +363,11 @@ export class SafariBooksDownloader {
   }
 
   async fetchJson(url) {
-    const response = await fetch(url, { credentials: "include" });
+    const response = await this.fetchWithSession(url, {
+      headers: {
+        Accept: "application/json, text/plain, */*"
+      }
+    });
     if (!response.ok) {
       throw new Error(`Request to ${url} failed with status ${response.status}`);
     }
@@ -380,7 +537,15 @@ export class SafariBooksDownloader {
   }
 
   async loadChapterDocument(url) {
-    const response = await fetch(url, { credentials: "include" });
+    const response = await this.fetchWithSession(
+      url,
+      {
+        headers: {
+          Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+        }
+      },
+      { requireDocument: true }
+    );
     if (!response.ok) {
       throw new Error(`Failed to download chapter at ${url} (status ${response.status})`);
     }
@@ -500,7 +665,11 @@ export class SafariBooksDownloader {
       if (this.cssContentMap.has(source)) {
         continue;
       }
-      const response = await fetch(source, { credentials: "include" });
+      const response = await this.fetchWithSession(source, {
+        headers: {
+          Accept: "text/css,*/*;q=0.1"
+        }
+      });
       if (!response.ok) {
         this.log(`Warning: unable to retrieve CSS from ${source} (status ${response.status})`);
         continue;
@@ -546,7 +715,11 @@ export class SafariBooksDownloader {
       if (this.cssImageDownloads.has(localPath)) {
         continue;
       }
-      const response = await fetch(url, { credentials: "include" });
+      const response = await this.fetchWithSession(url, {
+        headers: {
+          Accept: "font/woff2,application/font-woff,application/octet-stream,*/*"
+        }
+      });
       if (!response.ok) {
         this.log(`Warning: unable to retrieve font ${url} (status ${response.status})`);
         continue;
@@ -576,7 +749,11 @@ export class SafariBooksDownloader {
       }
       tried.add(candidate);
       try {
-        const response = await fetch(candidate, { credentials: "include" });
+        const response = await this.fetchWithSession(candidate, {
+          headers: {
+            Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
+          }
+        });
         if (!response.ok) {
           attempts.push(`${candidate} (status ${response.status})`);
           continue;
@@ -642,7 +819,11 @@ export class SafariBooksDownloader {
       if (entry?.data) {
         continue;
       }
-      const response = await fetch(entry, { credentials: "include" });
+      const response = await this.fetchWithSession(entry, {
+        headers: {
+          Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
+        }
+      });
       if (!response.ok) {
         this.log(`Warning: unable to fetch CSS image ${entry} (status ${response.status})`);
         continue;