Firefox update
This commit is contained in:
parent
c5f9dad0f9
commit
8fd5438f6c
4 changed files with 345 additions and 42 deletions
26
README.md
26
README.md
|
|
@ -1,6 +1,6 @@
|
|||
# Reilly SafariBooks Downloader (Chrome & Edge)
|
||||
# Reilly SafariBooks Downloader (Chrome, Edge & Firefox)
|
||||
|
||||
This extension ports the `safaribooks.py` script into a browser workflow so you can download EPUBs directly from [learning.oreilly.com](https://learning.oreilly.com).
|
||||
This extension ports the `safaribooks.py` script into a browser workflow so you can download EPUBs directly from [learning.oreilly.com](https://learning.oreilly.com) in Chrome, Edge, or Firefox.
|
||||
|
||||
## Installation (Chrome or Edge)
|
||||
|
||||
|
|
@ -10,8 +10,27 @@ This extension ports the `safaribooks.py` script into a browser workflow so you
|
|||
- **Edge**: `edge://extensions/`
|
||||
3. Enable **Developer mode** (toggle in the top-right corner).
|
||||
4. Click **Load unpacked** and select this project folder.
|
||||
5. The “Reilly SafariBooks Downloader” icon should appear in the toolbar.
|
||||
5. The "Reilly SafariBooks Downloader" icon should appear in the toolbar.
|
||||
|
||||
|
||||
## Installation (Firefox)
|
||||
|
||||
The packaged `.xpi` is unsigned, so the Firefox release channel blocks it. Pick the approach that matches your setup:
|
||||
|
||||
- **Temporary load (any Firefox build)**
|
||||
1. Open `about:debugging#/runtime/this-firefox`.
|
||||
2. Click **Load Temporary Add-on...** and select `firefox-extension.xpi`.
|
||||
3. Keep the tab open while testing; Firefox unloads temporary add-ons after a restart.
|
||||
|
||||
- **Persistent install for testing (Firefox Developer Edition or Nightly)**
|
||||
1. Switch to Developer Edition or Nightly—only these builds allow signing to be disabled.
|
||||
2. In `about:config`, set `xpinstall.signatures.required` to `false`.
|
||||
3. Open `about:addons`, click the gear icon, and choose **Install Add-on From File...**.
|
||||
4. Pick `firefox-extension.xpi`, confirm the prompts, then flip the preference back to `true` when you return to normal browsing.
|
||||
|
||||
- **Permanent install on Firefox Release**
|
||||
1. Sign the add-on through Mozilla’s Add-on Developer Hub (https://addons.mozilla.org/developers/) or `web-ext sign`.
|
||||
2. Use the signed `.xpi` that Mozilla returns; only signed builds will install on stable Firefox.
|
||||
## Usage
|
||||
|
||||
1. Log into [learning.oreilly.com](https://learning.oreilly.com) in the same browser profile.
|
||||
|
|
@ -27,3 +46,4 @@ This extension ports the `safaribooks.py` script into a browser workflow so you
|
|||
- The extension uses your authenticated cookies; ensure you are signed in before downloading.
|
||||
- If an image fetch fails, the downloader logs each attempt and tries alternate URLs.
|
||||
- Placeholder icons live in the `images/` directory—swap them out with custom artwork if you prefer.
|
||||
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@
|
|||
"cookies",
|
||||
"tabs",
|
||||
"storage",
|
||||
"webRequest",
|
||||
"webRequestBlocking",
|
||||
"https://learning.oreilly.com/*",
|
||||
"https://www.oreilly.com/*",
|
||||
"https://api.oreilly.com/*"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,16 @@ const rawChrome = typeof chrome !== "undefined" ? chrome : null;
|
|||
|
||||
const PROFILE_URL = "https://learning.oreilly.com/profile/";
|
||||
const COOKIE_DOMAINS = ["learning.oreilly.com", ".oreilly.com", "www.oreilly.com"];
|
||||
const COOKIE_URL_FILTERS = [
|
||||
"https://learning.oreilly.com/*",
|
||||
"https://www.oreilly.com/*",
|
||||
"https://api.oreilly.com/*"
|
||||
];
|
||||
|
||||
let cookieHeader = "";
|
||||
let refererHeader = "";
|
||||
let originHeader = "";
|
||||
let webRequestHookReady = false;
|
||||
|
||||
function callCookiesGetAll(filter) {
|
||||
const result = runtime.cookies.getAll(filter);
|
||||
|
|
@ -23,17 +33,101 @@ function callCookiesGetAll(filter) {
|
|||
}
|
||||
|
||||
async function gatherCookies() {
|
||||
const collected = {};
|
||||
const collected = new Map();
|
||||
let stores = [];
|
||||
try {
|
||||
if (typeof runtime.cookies.getAllCookieStores === "function") {
|
||||
stores = await runtime.cookies.getAllCookieStores();
|
||||
}
|
||||
} catch (_) {
|
||||
stores = [];
|
||||
}
|
||||
|
||||
const storeIds = stores.length ? stores.map((store) => store.id) : [undefined];
|
||||
|
||||
for (const domain of COOKIE_DOMAINS) {
|
||||
const cookies = await callCookiesGetAll({ domain });
|
||||
cookies.forEach((cookie) => {
|
||||
collected[cookie.name] = cookie.value;
|
||||
});
|
||||
for (const storeId of storeIds) {
|
||||
const filter = storeId ? { domain, storeId } : { domain };
|
||||
const cookies = await callCookiesGetAll(filter);
|
||||
cookies.forEach((cookie) => {
|
||||
collected.set(cookie.name, cookie.value);
|
||||
});
|
||||
}
|
||||
}
|
||||
return collected;
|
||||
}
|
||||
|
||||
function buildCookieHeader(jar) {
|
||||
if (!jar || jar.size === 0) {
|
||||
return "";
|
||||
}
|
||||
return Array.from(jar.entries())
|
||||
.map(([name, value]) => `${name}=${value}`)
|
||||
.join("; ");
|
||||
}
|
||||
|
||||
function ensureWebRequestHook() {
|
||||
if (webRequestHookReady || !runtime.webRequest?.onBeforeSendHeaders) {
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.webRequest.onBeforeSendHeaders.addListener(
|
||||
(details) => {
|
||||
const isExtensionRequest =
|
||||
details.tabId === -1 ||
|
||||
(typeof details.initiator === "string" && details.initiator.startsWith("moz-extension://"));
|
||||
if (!isExtensionRequest) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const headers = details.requestHeaders ? [...details.requestHeaders] : [];
|
||||
const lower = (name) => name.toLowerCase();
|
||||
|
||||
if (cookieHeader) {
|
||||
const existingCookie = headers.find((header) => lower(header.name) === "cookie");
|
||||
if (existingCookie) {
|
||||
existingCookie.value = cookieHeader;
|
||||
} else {
|
||||
headers.push({ name: "Cookie", value: cookieHeader });
|
||||
}
|
||||
}
|
||||
|
||||
if (refererHeader) {
|
||||
const existingReferer = headers.find((header) => lower(header.name) === "referer");
|
||||
if (existingReferer) {
|
||||
existingReferer.value = refererHeader;
|
||||
} else {
|
||||
headers.push({ name: "Referer", value: refererHeader });
|
||||
}
|
||||
}
|
||||
|
||||
if (originHeader) {
|
||||
const existingOrigin = headers.find((header) => lower(header.name) === "origin");
|
||||
if (existingOrigin) {
|
||||
existingOrigin.value = originHeader;
|
||||
} else {
|
||||
headers.push({ name: "Origin", value: originHeader });
|
||||
}
|
||||
}
|
||||
return { requestHeaders: headers };
|
||||
},
|
||||
{ urls: COOKIE_URL_FILTERS, types: ["xmlhttprequest", "other"] },
|
||||
["blocking", "requestHeaders"]
|
||||
);
|
||||
|
||||
webRequestHookReady = true;
|
||||
}
|
||||
|
||||
export async function collectSessionCookies() {
|
||||
const jar = await gatherCookies();
|
||||
const count = jar.size;
|
||||
if (!count) {
|
||||
throw new Error("No O'Reilly cookies detected. Make sure you visited learning.oreilly.com in this profile.");
|
||||
}
|
||||
|
||||
cookieHeader = buildCookieHeader(jar);
|
||||
ensureWebRequestHook();
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(PROFILE_URL, { credentials: "include" });
|
||||
|
|
@ -49,16 +143,22 @@ export async function collectSessionCookies() {
|
|||
throw new Error(`Profile check failed (status ${response.status}).`);
|
||||
}
|
||||
|
||||
const jar = await gatherCookies();
|
||||
const count = Object.keys(jar).length;
|
||||
if (!count) {
|
||||
throw new Error("No O'Reilly cookies detected. Make sure you visited learning.oreilly.com in this profile.");
|
||||
}
|
||||
|
||||
return {
|
||||
status: response.status,
|
||||
count,
|
||||
cookies: jar
|
||||
cookies: Object.fromEntries(jar.entries())
|
||||
};
|
||||
}
|
||||
|
||||
export function updateRequestContext({ referer } = {}) {
|
||||
if (!referer) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
refererHeader = referer;
|
||||
const url = new URL(referer);
|
||||
originHeader = `${url.protocol}//${url.host}`;
|
||||
} catch (_) {
|
||||
// ignore invalid referrer values
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
isLikelyImageAsset
|
||||
} from "./utils.js";
|
||||
import { SimpleZip } from "./zip.js";
|
||||
import { collectSessionCookies, updateRequestContext } from "./cookies.js";
|
||||
|
||||
const SAFARI_BASE_URL = "https://learning.oreilly.com";
|
||||
const API_V2_TEMPLATE = `${SAFARI_BASE_URL}/api/v2/epubs/urn:orm:book:`;
|
||||
|
|
@ -32,25 +33,6 @@ const KINDLE_CSS =
|
|||
"#sbo-rt-content table,#sbo-rt-content pre{overflow-x:unset!important;overflow:unset!important;" +
|
||||
"overflow-y:unset!important;white-space:pre-wrap!important;}";
|
||||
|
||||
function downloadWithWebExtensionsApi(options) {
|
||||
if (typeof browser !== "undefined" && browser?.downloads?.download) {
|
||||
return browser.downloads.download(options);
|
||||
}
|
||||
if (typeof chrome !== "undefined" && chrome?.downloads?.download) {
|
||||
return new Promise((resolve, reject) => {
|
||||
chrome.downloads.download(options, (downloadId) => {
|
||||
const err = chrome.runtime?.lastError;
|
||||
if (err) {
|
||||
reject(new Error(err.message));
|
||||
} else {
|
||||
resolve(downloadId);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
return Promise.reject(new Error("Downloads API is not available in this browser."));
|
||||
}
|
||||
|
||||
function formatTwoDigits(value) {
|
||||
return value.toString().padStart(2, "0");
|
||||
}
|
||||
|
|
@ -113,6 +95,12 @@ export class SafariBooksDownloader {
|
|||
this.coverPath = "";
|
||||
this.themeMode = "white";
|
||||
this.options = { theme: "none", kindle: false };
|
||||
this.referrerUrl = SAFARI_BASE_URL;
|
||||
this.baseRequestSpacingMs = 200;
|
||||
this.nextRequestTime = Date.now();
|
||||
this.rateLimitPenaltyMs = 0;
|
||||
this.sessionRefreshCooldownMs = 30000;
|
||||
this.lastSessionRefresh = 0;
|
||||
|
||||
this.cssSources = [];
|
||||
this.cssFilenameMap = new Map();
|
||||
|
|
@ -124,6 +112,167 @@ export class SafariBooksDownloader {
|
|||
this.imagesFromLinks = new Set();
|
||||
}
|
||||
|
||||
getReferrerUrl() {
|
||||
return this.referrerUrl || SAFARI_BASE_URL;
|
||||
}
|
||||
|
||||
updateReferrer(url) {
|
||||
if (typeof url === "string" && url.startsWith("http")) {
|
||||
this.referrerUrl = url;
|
||||
}
|
||||
}
|
||||
|
||||
async ensureSessionRefreshed({ force = false } = {}) {
|
||||
if (typeof collectSessionCookies !== "function") {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
if (!force && now - this.lastSessionRefresh < this.sessionRefreshCooldownMs) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await collectSessionCookies();
|
||||
this.lastSessionRefresh = Date.now();
|
||||
} catch (error) {
|
||||
this.log(`Warning: unable to refresh session cookies (${error.message}).`);
|
||||
}
|
||||
}
|
||||
|
||||
async applyRateLimit() {
|
||||
const waitTime = this.nextRequestTime - Date.now();
|
||||
if (waitTime > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
||||
}
|
||||
}
|
||||
|
||||
registerRequestOutcome(status) {
|
||||
const penalizedStatuses = new Set([0, 401, 403, 429, 500, 502, 503, 504]);
|
||||
if (penalizedStatuses.has(status)) {
|
||||
if (status === 429 || status === 503) {
|
||||
this.rateLimitPenaltyMs = Math.min(Math.max(this.rateLimitPenaltyMs * 2, 500), 8000);
|
||||
} else {
|
||||
this.rateLimitPenaltyMs = Math.min(this.rateLimitPenaltyMs + 250, 6000);
|
||||
}
|
||||
} else {
|
||||
this.rateLimitPenaltyMs = Math.max(Math.floor(this.rateLimitPenaltyMs * 0.5), 0);
|
||||
}
|
||||
const delay = this.baseRequestSpacingMs + this.rateLimitPenaltyMs;
|
||||
this.nextRequestTime = Date.now() + delay;
|
||||
}
|
||||
|
||||
shouldRetryResponse(response, attempt, maxAttempts, options = {}) {
|
||||
if (attempt >= maxAttempts - 1) {
|
||||
return false;
|
||||
}
|
||||
const status = response?.status ?? 0;
|
||||
if (status === 0) {
|
||||
return true;
|
||||
}
|
||||
if ([429, 500, 502, 503, 504].includes(status)) {
|
||||
return true;
|
||||
}
|
||||
if ((status === 401 || status === 403) && options.retryOnForbidden !== false) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
computeRetryDelay(response, attempt, options = {}) {
|
||||
const status = response?.status ?? 0;
|
||||
const base = options.baseRetryDelay ?? 750;
|
||||
const cap = options.maxRetryDelay ?? 8000;
|
||||
const growth = Math.min(cap, base * Math.pow(2, attempt));
|
||||
const jitter = Math.random() * 250;
|
||||
if (status === 429 || status === 503) {
|
||||
return Math.min(cap, growth + 1000 + jitter);
|
||||
}
|
||||
return Math.min(cap, growth + jitter);
|
||||
}
|
||||
|
||||
buildFetchInit(init = {}, { requireDocument = false } = {}) {
|
||||
const { headers, ...rest } = init || {};
|
||||
const mergedHeaders = new Headers(headers || {});
|
||||
if (!mergedHeaders.has("X-Requested-With")) {
|
||||
mergedHeaders.set("X-Requested-With", "XMLHttpRequest");
|
||||
}
|
||||
if (requireDocument && !mergedHeaders.has("Accept")) {
|
||||
mergedHeaders.set(
|
||||
"Accept",
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
);
|
||||
}
|
||||
if (!mergedHeaders.has("Accept-Language")) {
|
||||
mergedHeaders.set("Accept-Language", "en-US,en;q=0.9");
|
||||
}
|
||||
|
||||
const initWithDefaults = {
|
||||
credentials: "include",
|
||||
mode: "cors",
|
||||
...rest,
|
||||
headers: mergedHeaders
|
||||
};
|
||||
|
||||
if (!initWithDefaults.referrer) {
|
||||
initWithDefaults.referrer = this.getReferrerUrl();
|
||||
}
|
||||
if (!initWithDefaults.referrerPolicy) {
|
||||
initWithDefaults.referrerPolicy = "strict-origin-when-cross-origin";
|
||||
}
|
||||
return initWithDefaults;
|
||||
}
|
||||
|
||||
async fetchWithSession(url, init = {}, options = {}) {
|
||||
const maxAttempts = Math.max(1, options.maxAttempts ?? 4);
|
||||
let lastError = null;
|
||||
let lastResponse = null;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
await this.applyRateLimit();
|
||||
|
||||
try {
|
||||
const response = await fetch(url, this.buildFetchInit(init, options));
|
||||
lastResponse = response;
|
||||
this.registerRequestOutcome(response.status);
|
||||
|
||||
if (response.ok) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const shouldRetry = this.shouldRetryResponse(response, attempt, maxAttempts, options);
|
||||
if (!shouldRetry) {
|
||||
return response;
|
||||
}
|
||||
|
||||
if ((response.status === 401 || response.status === 403) && options.retryOnForbidden !== false) {
|
||||
await this.ensureSessionRefreshed({ force: true });
|
||||
}
|
||||
|
||||
const retryDelay = this.computeRetryDelay(response, attempt, options);
|
||||
const targetTime = Date.now() + retryDelay;
|
||||
if (targetTime > this.nextRequestTime) {
|
||||
this.nextRequestTime = targetTime;
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
this.registerRequestOutcome(0);
|
||||
if (attempt >= maxAttempts - 1) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const retryDelay = this.computeRetryDelay(null, attempt, options);
|
||||
const targetTime = Date.now() + retryDelay;
|
||||
if (targetTime > this.nextRequestTime) {
|
||||
this.nextRequestTime = targetTime;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lastResponse) {
|
||||
return lastResponse;
|
||||
}
|
||||
throw lastError ?? new Error(`Request to ${url} failed without a response.`);
|
||||
}
|
||||
|
||||
async download(bookId, options = {}) {
|
||||
this.reset();
|
||||
this.bookId = bookId;
|
||||
|
|
@ -132,6 +281,8 @@ export class SafariBooksDownloader {
|
|||
kindle: Boolean(options.kindle)
|
||||
};
|
||||
this.themeMode = this.options.theme === "none" ? "white" : this.options.theme;
|
||||
this.updateReferrer(`${SAFARI_BASE_URL}/library/view/${this.bookId}/`);
|
||||
updateRequestContext({ referer: this.getReferrerUrl() });
|
||||
|
||||
this.log(`Verifying session by loading profile page...`);
|
||||
await this.checkLogin();
|
||||
|
|
@ -139,6 +290,8 @@ export class SafariBooksDownloader {
|
|||
this.log(`Retrieving book metadata for ${bookId}...`);
|
||||
this.bookInfo = await this.fetchBookInfo(bookId);
|
||||
this.baseUrl = this.bookInfo.web_url || this.bookInfo.url;
|
||||
this.updateReferrer(this.bookInfo.web_url || this.bookInfo.url);
|
||||
updateRequestContext({ referer: this.getReferrerUrl() });
|
||||
this.cleanTitle = makeFolderFriendlyName(this.bookInfo.title, this.bookId);
|
||||
const authorNames = (this.bookInfo.authors || []).map((author) => author.name).filter(Boolean);
|
||||
this.primaryAuthors = authorNames.join(", ") || "Unknown Author";
|
||||
|
|
@ -161,12 +314,12 @@ export class SafariBooksDownloader {
|
|||
const url = URL.createObjectURL(blob);
|
||||
|
||||
const filename = `${this.cleanTitle}/${this.outputFileName}.epub`;
|
||||
await downloadWithWebExtensionsApi({ url, filename, saveAs: false });
|
||||
await chrome.downloads.download({ url, filename, saveAs: false });
|
||||
this.log(`EPUB saved as ${filename}`);
|
||||
}
|
||||
|
||||
async checkLogin() {
|
||||
const response = await fetch(PROFILE_URL, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(PROFILE_URL, {}, { requireDocument: true });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Authentication failed (status ${response.status}) when accessing ${PROFILE_URL}`);
|
||||
}
|
||||
|
|
@ -210,7 +363,11 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
|
||||
async fetchJson(url) {
|
||||
const response = await fetch(url, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(url, {
|
||||
headers: {
|
||||
Accept: "application/json, text/plain, */*"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Request to ${url} failed with status ${response.status}`);
|
||||
}
|
||||
|
|
@ -380,7 +537,15 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
|
||||
async loadChapterDocument(url) {
|
||||
const response = await fetch(url, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(
|
||||
url,
|
||||
{
|
||||
headers: {
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
}
|
||||
},
|
||||
{ requireDocument: true }
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download chapter at ${url} (status ${response.status})`);
|
||||
}
|
||||
|
|
@ -500,7 +665,11 @@ export class SafariBooksDownloader {
|
|||
if (this.cssContentMap.has(source)) {
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(source, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(source, {
|
||||
headers: {
|
||||
Accept: "text/css,*/*;q=0.1"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
this.log(`Warning: unable to retrieve CSS from ${source} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
@ -546,7 +715,11 @@ export class SafariBooksDownloader {
|
|||
if (this.cssImageDownloads.has(localPath)) {
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(url, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(url, {
|
||||
headers: {
|
||||
Accept: "font/woff2,application/font-woff,application/octet-stream,*/*"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
this.log(`Warning: unable to retrieve font ${url} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
@ -576,7 +749,11 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
tried.add(candidate);
|
||||
try {
|
||||
const response = await fetch(candidate, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(candidate, {
|
||||
headers: {
|
||||
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
attempts.push(`${candidate} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
@ -642,7 +819,11 @@ export class SafariBooksDownloader {
|
|||
if (entry?.data) {
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(entry, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(entry, {
|
||||
headers: {
|
||||
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
this.log(`Warning: unable to fetch CSS image ${entry} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue