diff --git a/README.md b/README.md index 526a5ef..441484f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Reilly SafariBooks Downloader (Chrome & Edge) +# Reilly SafariBooks Downloader (Chrome, Edge & Firefox) -This extension ports the `safaribooks.py` script into a browser workflow so you can download EPUBs directly from [learning.oreilly.com](https://learning.oreilly.com). +This extension ports the `safaribooks.py` script into a browser workflow so you can download EPUBs directly from [learning.oreilly.com](https://learning.oreilly.com) in Chrome, Edge, or Firefox. ## Installation (Chrome or Edge) @@ -10,8 +10,27 @@ This extension ports the `safaribooks.py` script into a browser workflow so you - **Edge**: `edge://extensions/` 3. Enable **Developer mode** (toggle in the top-right corner). 4. Click **Load unpacked** and select this project folder. -5. The “Reilly SafariBooks Downloader” icon should appear in the toolbar. +5. The "Reilly SafariBooks Downloader" icon should appear in the toolbar. + +## Installation (Firefox) + +The packaged `.xpi` is unsigned, so the Firefox release channel blocks it. Pick the approach that matches your setup: + +- **Temporary load (any Firefox build)** + 1. Open `about:debugging#/runtime/this-firefox`. + 2. Click **Load Temporary Add-on...** and select `firefox-extension.xpi`. + 3. Keep the tab open while testing; Firefox unloads temporary add-ons after a restart. + +- **Persistent install for testing (Firefox Developer Edition or Nightly)** + 1. Switch to Developer Edition or Nightly—only these builds allow signing to be disabled. + 2. In `about:config`, set `xpinstall.signatures.required` to `false`. + 3. Open `about:addons`, click the gear icon, and choose **Install Add-on From File...**. + 4. Pick `firefox-extension.xpi`, confirm the prompts, then flip the preference back to `true` when you return to normal browsing. + +- **Permanent install on Firefox Release** + 1. Sign the add-on through Mozilla’s Add-on Developer Hub (https://addons.mozilla.org/developers/) or `web-ext sign`. + 2. Use the signed `.xpi` that Mozilla returns; only signed builds will install on stable Firefox. ## Usage 1. Log into [learning.oreilly.com](https://learning.oreilly.com) in the same browser profile. @@ -27,3 +46,4 @@ This extension ports the `safaribooks.py` script into a browser workflow so you - The extension uses your authenticated cookies; ensure you are signed in before downloading. - If an image fetch fails, the downloader logs each attempt and tries alternate URLs. - Placeholder icons live in the `images/` directory—swap them out with custom artwork if you prefer. + diff --git a/firefox/manifest.json b/firefox/manifest.json index 9940134..bf86a3d 100644 --- a/firefox/manifest.json +++ b/firefox/manifest.json @@ -43,6 +43,8 @@ "cookies", "tabs", "storage", + "webRequest", + "webRequestBlocking", "https://learning.oreilly.com/*", "https://www.oreilly.com/*", "https://api.oreilly.com/*" diff --git a/firefox/safaribooks/cookies.js b/firefox/safaribooks/cookies.js index d5ac3a3..165a18b 100644 --- a/firefox/safaribooks/cookies.js +++ b/firefox/safaribooks/cookies.js @@ -3,6 +3,16 @@ const rawChrome = typeof chrome !== "undefined" ? chrome : null; const PROFILE_URL = "https://learning.oreilly.com/profile/"; const COOKIE_DOMAINS = ["learning.oreilly.com", ".oreilly.com", "www.oreilly.com"]; +const COOKIE_URL_FILTERS = [ + "https://learning.oreilly.com/*", + "https://www.oreilly.com/*", + "https://api.oreilly.com/*" +]; + +let cookieHeader = ""; +let refererHeader = ""; +let originHeader = ""; +let webRequestHookReady = false; function callCookiesGetAll(filter) { const result = runtime.cookies.getAll(filter); @@ -23,17 +33,101 @@ function callCookiesGetAll(filter) { } async function gatherCookies() { - const collected = {}; + const collected = new Map(); + let stores = []; + try { + if (typeof runtime.cookies.getAllCookieStores === "function") { + stores = await runtime.cookies.getAllCookieStores(); + } + } catch (_) { + stores = []; + } + + const storeIds = stores.length ? stores.map((store) => store.id) : [undefined]; + for (const domain of COOKIE_DOMAINS) { - const cookies = await callCookiesGetAll({ domain }); - cookies.forEach((cookie) => { - collected[cookie.name] = cookie.value; - }); + for (const storeId of storeIds) { + const filter = storeId ? { domain, storeId } : { domain }; + const cookies = await callCookiesGetAll(filter); + cookies.forEach((cookie) => { + collected.set(cookie.name, cookie.value); + }); + } } return collected; } +function buildCookieHeader(jar) { + if (!jar || jar.size === 0) { + return ""; + } + return Array.from(jar.entries()) + .map(([name, value]) => `${name}=${value}`) + .join("; "); +} + +function ensureWebRequestHook() { + if (webRequestHookReady || !runtime.webRequest?.onBeforeSendHeaders) { + return; + } + + runtime.webRequest.onBeforeSendHeaders.addListener( + (details) => { + const isExtensionRequest = + details.tabId === -1 || + (typeof details.initiator === "string" && details.initiator.startsWith("moz-extension://")); + if (!isExtensionRequest) { + return {}; + } + + const headers = details.requestHeaders ? [...details.requestHeaders] : []; + const lower = (name) => name.toLowerCase(); + + if (cookieHeader) { + const existingCookie = headers.find((header) => lower(header.name) === "cookie"); + if (existingCookie) { + existingCookie.value = cookieHeader; + } else { + headers.push({ name: "Cookie", value: cookieHeader }); + } + } + + if (refererHeader) { + const existingReferer = headers.find((header) => lower(header.name) === "referer"); + if (existingReferer) { + existingReferer.value = refererHeader; + } else { + headers.push({ name: "Referer", value: refererHeader }); + } + } + + if (originHeader) { + const existingOrigin = headers.find((header) => lower(header.name) === "origin"); + if (existingOrigin) { + existingOrigin.value = originHeader; + } else { + headers.push({ name: "Origin", value: originHeader }); + } + } + return { requestHeaders: headers }; + }, + { urls: COOKIE_URL_FILTERS, types: ["xmlhttprequest", "other"] }, + ["blocking", "requestHeaders"] + ); + + webRequestHookReady = true; +} + export async function collectSessionCookies() { + const jar = await gatherCookies(); + const count = jar.size; + if (!count) { + throw new Error("No O'Reilly cookies detected. Make sure you visited learning.oreilly.com in this profile."); + } + + cookieHeader = buildCookieHeader(jar); + ensureWebRequestHook(); + let response; try { response = await fetch(PROFILE_URL, { credentials: "include" }); @@ -49,16 +143,22 @@ export async function collectSessionCookies() { throw new Error(`Profile check failed (status ${response.status}).`); } - const jar = await gatherCookies(); - const count = Object.keys(jar).length; - if (!count) { - throw new Error("No O'Reilly cookies detected. Make sure you visited learning.oreilly.com in this profile."); - } - return { status: response.status, count, - cookies: jar + cookies: Object.fromEntries(jar.entries()) }; } +export function updateRequestContext({ referer } = {}) { + if (!referer) { + return; + } + try { + refererHeader = referer; + const url = new URL(referer); + originHeader = `${url.protocol}//${url.host}`; + } catch (_) { + // ignore invalid referrer values + } +} diff --git a/firefox/safaribooks/downloader.js b/firefox/safaribooks/downloader.js index 5516faa..e22453d 100644 --- a/firefox/safaribooks/downloader.js +++ b/firefox/safaribooks/downloader.js @@ -14,6 +14,7 @@ import { isLikelyImageAsset } from "./utils.js"; import { SimpleZip } from "./zip.js"; +import { collectSessionCookies, updateRequestContext } from "./cookies.js"; const SAFARI_BASE_URL = "https://learning.oreilly.com"; const API_V2_TEMPLATE = `${SAFARI_BASE_URL}/api/v2/epubs/urn:orm:book:`; @@ -32,25 +33,6 @@ const KINDLE_CSS = "#sbo-rt-content table,#sbo-rt-content pre{overflow-x:unset!important;overflow:unset!important;" + "overflow-y:unset!important;white-space:pre-wrap!important;}"; -function downloadWithWebExtensionsApi(options) { - if (typeof browser !== "undefined" && browser?.downloads?.download) { - return browser.downloads.download(options); - } - if (typeof chrome !== "undefined" && chrome?.downloads?.download) { - return new Promise((resolve, reject) => { - chrome.downloads.download(options, (downloadId) => { - const err = chrome.runtime?.lastError; - if (err) { - reject(new Error(err.message)); - } else { - resolve(downloadId); - } - }); - }); - } - return Promise.reject(new Error("Downloads API is not available in this browser.")); -} - function formatTwoDigits(value) { return value.toString().padStart(2, "0"); } @@ -113,6 +95,12 @@ export class SafariBooksDownloader { this.coverPath = ""; this.themeMode = "white"; this.options = { theme: "none", kindle: false }; + this.referrerUrl = SAFARI_BASE_URL; + this.baseRequestSpacingMs = 200; + this.nextRequestTime = Date.now(); + this.rateLimitPenaltyMs = 0; + this.sessionRefreshCooldownMs = 30000; + this.lastSessionRefresh = 0; this.cssSources = []; this.cssFilenameMap = new Map(); @@ -124,6 +112,167 @@ export class SafariBooksDownloader { this.imagesFromLinks = new Set(); } + getReferrerUrl() { + return this.referrerUrl || SAFARI_BASE_URL; + } + + updateReferrer(url) { + if (typeof url === "string" && url.startsWith("http")) { + this.referrerUrl = url; + } + } + + async ensureSessionRefreshed({ force = false } = {}) { + if (typeof collectSessionCookies !== "function") { + return; + } + const now = Date.now(); + if (!force && now - this.lastSessionRefresh < this.sessionRefreshCooldownMs) { + return; + } + try { + await collectSessionCookies(); + this.lastSessionRefresh = Date.now(); + } catch (error) { + this.log(`Warning: unable to refresh session cookies (${error.message}).`); + } + } + + async applyRateLimit() { + const waitTime = this.nextRequestTime - Date.now(); + if (waitTime > 0) { + await new Promise((resolve) => setTimeout(resolve, waitTime)); + } + } + + registerRequestOutcome(status) { + const penalizedStatuses = new Set([0, 401, 403, 429, 500, 502, 503, 504]); + if (penalizedStatuses.has(status)) { + if (status === 429 || status === 503) { + this.rateLimitPenaltyMs = Math.min(Math.max(this.rateLimitPenaltyMs * 2, 500), 8000); + } else { + this.rateLimitPenaltyMs = Math.min(this.rateLimitPenaltyMs + 250, 6000); + } + } else { + this.rateLimitPenaltyMs = Math.max(Math.floor(this.rateLimitPenaltyMs * 0.5), 0); + } + const delay = this.baseRequestSpacingMs + this.rateLimitPenaltyMs; + this.nextRequestTime = Date.now() + delay; + } + + shouldRetryResponse(response, attempt, maxAttempts, options = {}) { + if (attempt >= maxAttempts - 1) { + return false; + } + const status = response?.status ?? 0; + if (status === 0) { + return true; + } + if ([429, 500, 502, 503, 504].includes(status)) { + return true; + } + if ((status === 401 || status === 403) && options.retryOnForbidden !== false) { + return true; + } + return false; + } + + computeRetryDelay(response, attempt, options = {}) { + const status = response?.status ?? 0; + const base = options.baseRetryDelay ?? 750; + const cap = options.maxRetryDelay ?? 8000; + const growth = Math.min(cap, base * Math.pow(2, attempt)); + const jitter = Math.random() * 250; + if (status === 429 || status === 503) { + return Math.min(cap, growth + 1000 + jitter); + } + return Math.min(cap, growth + jitter); + } + + buildFetchInit(init = {}, { requireDocument = false } = {}) { + const { headers, ...rest } = init || {}; + const mergedHeaders = new Headers(headers || {}); + if (!mergedHeaders.has("X-Requested-With")) { + mergedHeaders.set("X-Requested-With", "XMLHttpRequest"); + } + if (requireDocument && !mergedHeaders.has("Accept")) { + mergedHeaders.set( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + ); + } + if (!mergedHeaders.has("Accept-Language")) { + mergedHeaders.set("Accept-Language", "en-US,en;q=0.9"); + } + + const initWithDefaults = { + credentials: "include", + mode: "cors", + ...rest, + headers: mergedHeaders + }; + + if (!initWithDefaults.referrer) { + initWithDefaults.referrer = this.getReferrerUrl(); + } + if (!initWithDefaults.referrerPolicy) { + initWithDefaults.referrerPolicy = "strict-origin-when-cross-origin"; + } + return initWithDefaults; + } + + async fetchWithSession(url, init = {}, options = {}) { + const maxAttempts = Math.max(1, options.maxAttempts ?? 4); + let lastError = null; + let lastResponse = null; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + await this.applyRateLimit(); + + try { + const response = await fetch(url, this.buildFetchInit(init, options)); + lastResponse = response; + this.registerRequestOutcome(response.status); + + if (response.ok) { + return response; + } + + const shouldRetry = this.shouldRetryResponse(response, attempt, maxAttempts, options); + if (!shouldRetry) { + return response; + } + + if ((response.status === 401 || response.status === 403) && options.retryOnForbidden !== false) { + await this.ensureSessionRefreshed({ force: true }); + } + + const retryDelay = this.computeRetryDelay(response, attempt, options); + const targetTime = Date.now() + retryDelay; + if (targetTime > this.nextRequestTime) { + this.nextRequestTime = targetTime; + } + } catch (error) { + lastError = error; + this.registerRequestOutcome(0); + if (attempt >= maxAttempts - 1) { + throw error; + } + + const retryDelay = this.computeRetryDelay(null, attempt, options); + const targetTime = Date.now() + retryDelay; + if (targetTime > this.nextRequestTime) { + this.nextRequestTime = targetTime; + } + } + } + + if (lastResponse) { + return lastResponse; + } + throw lastError ?? new Error(`Request to ${url} failed without a response.`); + } + async download(bookId, options = {}) { this.reset(); this.bookId = bookId; @@ -132,6 +281,8 @@ export class SafariBooksDownloader { kindle: Boolean(options.kindle) }; this.themeMode = this.options.theme === "none" ? "white" : this.options.theme; + this.updateReferrer(`${SAFARI_BASE_URL}/library/view/${this.bookId}/`); + updateRequestContext({ referer: this.getReferrerUrl() }); this.log(`Verifying session by loading profile page...`); await this.checkLogin(); @@ -139,6 +290,8 @@ export class SafariBooksDownloader { this.log(`Retrieving book metadata for ${bookId}...`); this.bookInfo = await this.fetchBookInfo(bookId); this.baseUrl = this.bookInfo.web_url || this.bookInfo.url; + this.updateReferrer(this.bookInfo.web_url || this.bookInfo.url); + updateRequestContext({ referer: this.getReferrerUrl() }); this.cleanTitle = makeFolderFriendlyName(this.bookInfo.title, this.bookId); const authorNames = (this.bookInfo.authors || []).map((author) => author.name).filter(Boolean); this.primaryAuthors = authorNames.join(", ") || "Unknown Author"; @@ -161,12 +314,12 @@ export class SafariBooksDownloader { const url = URL.createObjectURL(blob); const filename = `${this.cleanTitle}/${this.outputFileName}.epub`; - await downloadWithWebExtensionsApi({ url, filename, saveAs: false }); + await chrome.downloads.download({ url, filename, saveAs: false }); this.log(`EPUB saved as ${filename}`); } async checkLogin() { - const response = await fetch(PROFILE_URL, { credentials: "include" }); + const response = await this.fetchWithSession(PROFILE_URL, {}, { requireDocument: true }); if (!response.ok) { throw new Error(`Authentication failed (status ${response.status}) when accessing ${PROFILE_URL}`); } @@ -210,7 +363,11 @@ export class SafariBooksDownloader { } async fetchJson(url) { - const response = await fetch(url, { credentials: "include" }); + const response = await this.fetchWithSession(url, { + headers: { + Accept: "application/json, text/plain, */*" + } + }); if (!response.ok) { throw new Error(`Request to ${url} failed with status ${response.status}`); } @@ -380,7 +537,15 @@ export class SafariBooksDownloader { } async loadChapterDocument(url) { - const response = await fetch(url, { credentials: "include" }); + const response = await this.fetchWithSession( + url, + { + headers: { + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + } + }, + { requireDocument: true } + ); if (!response.ok) { throw new Error(`Failed to download chapter at ${url} (status ${response.status})`); } @@ -500,7 +665,11 @@ export class SafariBooksDownloader { if (this.cssContentMap.has(source)) { continue; } - const response = await fetch(source, { credentials: "include" }); + const response = await this.fetchWithSession(source, { + headers: { + Accept: "text/css,*/*;q=0.1" + } + }); if (!response.ok) { this.log(`Warning: unable to retrieve CSS from ${source} (status ${response.status})`); continue; @@ -546,7 +715,11 @@ export class SafariBooksDownloader { if (this.cssImageDownloads.has(localPath)) { continue; } - const response = await fetch(url, { credentials: "include" }); + const response = await this.fetchWithSession(url, { + headers: { + Accept: "font/woff2,application/font-woff,application/octet-stream,*/*" + } + }); if (!response.ok) { this.log(`Warning: unable to retrieve font ${url} (status ${response.status})`); continue; @@ -576,7 +749,11 @@ export class SafariBooksDownloader { } tried.add(candidate); try { - const response = await fetch(candidate, { credentials: "include" }); + const response = await this.fetchWithSession(candidate, { + headers: { + Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" + } + }); if (!response.ok) { attempts.push(`${candidate} (status ${response.status})`); continue; @@ -642,7 +819,11 @@ export class SafariBooksDownloader { if (entry?.data) { continue; } - const response = await fetch(entry, { credentials: "include" }); + const response = await this.fetchWithSession(entry, { + headers: { + Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" + } + }); if (!response.ok) { this.log(`Warning: unable to fetch CSS image ${entry} (status ${response.status})`); continue;