From edd0f9f80928a9691c8485e2ad3e4a047af0f182 Mon Sep 17 00:00:00 2001 From: Joey Yakimowich-Payne Date: Tue, 14 Oct 2025 14:14:46 -0600 Subject: [PATCH] Update downloader --- safaribooks/cookies.js | 4 + safaribooks/downloader.js | 218 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 215 insertions(+), 7 deletions(-) diff --git a/safaribooks/cookies.js b/safaribooks/cookies.js index b07498f..1a0ddfd 100644 --- a/safaribooks/cookies.js +++ b/safaribooks/cookies.js @@ -44,3 +44,7 @@ export async function collectSessionCookies() { cookies: jar }; } + +export function updateRequestContext() { + // Chrome/Edge do not require extra request header handling. +} diff --git a/safaribooks/downloader.js b/safaribooks/downloader.js index 75e4f78..7041a06 100644 --- a/safaribooks/downloader.js +++ b/safaribooks/downloader.js @@ -14,6 +14,7 @@ import { isLikelyImageAsset } from "./utils.js"; import { SimpleZip } from "./zip.js"; +import { collectSessionCookies, updateRequestContext } from "./cookies.js"; const SAFARI_BASE_URL = "https://learning.oreilly.com"; const API_V2_TEMPLATE = `${SAFARI_BASE_URL}/api/v2/epubs/urn:orm:book:`; @@ -32,6 +33,10 @@ const KINDLE_CSS = "#sbo-rt-content table,#sbo-rt-content pre{overflow-x:unset!important;overflow:unset!important;" + "overflow-y:unset!important;white-space:pre-wrap!important;}"; +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(ms, 0))); +} + function formatTwoDigits(value) { return value.toString().padStart(2, "0"); } @@ -94,6 +99,12 @@ export class SafariBooksDownloader { this.coverPath = ""; this.themeMode = "white"; this.options = { theme: "none", kindle: false }; + this.referrerUrl = SAFARI_BASE_URL; + this.baseRequestSpacingMs = 200; + this.nextRequestTime = Date.now(); + this.rateLimitPenaltyMs = 0; + this.sessionRefreshCooldownMs = 30000; + this.lastSessionRefresh = 0; this.cssSources = []; this.cssFilenameMap = new Map(); @@ -105,6 +116,167 @@ export class SafariBooksDownloader { this.imagesFromLinks = new Set(); } + getReferrerUrl() { + return this.referrerUrl || SAFARI_BASE_URL; + } + + updateReferrer(url) { + if (typeof url === "string" && url.startsWith("http")) { + this.referrerUrl = url; + } + } + + async ensureSessionRefreshed({ force = false } = {}) { + if (typeof collectSessionCookies !== "function") { + return; + } + const now = Date.now(); + if (!force && now - this.lastSessionRefresh < this.sessionRefreshCooldownMs) { + return; + } + try { + await collectSessionCookies(); + this.lastSessionRefresh = Date.now(); + } catch (error) { + this.log(`Warning: unable to refresh session cookies (${error.message}).`); + } + } + + async applyRateLimit() { + const waitTime = this.nextRequestTime - Date.now(); + if (waitTime > 0) { + await sleep(waitTime); + } + } + + registerRequestOutcome(status) { + const penalizedStatuses = new Set([0, 401, 403, 429, 500, 502, 503, 504]); + if (penalizedStatuses.has(status)) { + if (status === 429 || status === 503) { + this.rateLimitPenaltyMs = Math.min(Math.max(this.rateLimitPenaltyMs * 2, 500), 8000); + } else { + this.rateLimitPenaltyMs = Math.min(this.rateLimitPenaltyMs + 250, 6000); + } + } else { + this.rateLimitPenaltyMs = Math.max(Math.floor(this.rateLimitPenaltyMs * 0.5), 0); + } + const delay = this.baseRequestSpacingMs + this.rateLimitPenaltyMs; + this.nextRequestTime = Date.now() + delay; + } + + shouldRetryResponse(response, attempt, maxAttempts, options = {}) { + if (attempt >= maxAttempts - 1) { + return false; + } + const status = response?.status ?? 0; + if (status === 0) { + return true; + } + if ([429, 500, 502, 503, 504].includes(status)) { + return true; + } + if ((status === 401 || status === 403) && options.retryOnForbidden !== false) { + return true; + } + return false; + } + + computeRetryDelay(response, attempt, options = {}) { + const status = response?.status ?? 0; + const base = options.baseRetryDelay ?? 750; + const cap = options.maxRetryDelay ?? 8000; + const growth = Math.min(cap, base * Math.pow(2, attempt)); + const jitter = Math.random() * 250; + if (status === 429 || status === 503) { + return Math.min(cap, growth + 1000 + jitter); + } + return Math.min(cap, growth + jitter); + } + + buildFetchInit(init = {}, { requireDocument = false } = {}) { + const { headers, ...rest } = init || {}; + const mergedHeaders = new Headers(headers || {}); + if (!mergedHeaders.has("X-Requested-With")) { + mergedHeaders.set("X-Requested-With", "XMLHttpRequest"); + } + if (requireDocument && !mergedHeaders.has("Accept")) { + mergedHeaders.set( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + ); + } + if (!mergedHeaders.has("Accept-Language")) { + mergedHeaders.set("Accept-Language", "en-US,en;q=0.9"); + } + + const initWithDefaults = { + credentials: "include", + mode: "cors", + ...rest, + headers: mergedHeaders + }; + + if (!initWithDefaults.referrer) { + initWithDefaults.referrer = this.getReferrerUrl(); + } + if (!initWithDefaults.referrerPolicy) { + initWithDefaults.referrerPolicy = "strict-origin-when-cross-origin"; + } + return initWithDefaults; + } + + async fetchWithSession(url, init = {}, options = {}) { + const maxAttempts = Math.max(1, options.maxAttempts ?? 4); + let lastError = null; + let lastResponse = null; + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + await this.applyRateLimit(); + + try { + const response = await fetch(url, this.buildFetchInit(init, options)); + lastResponse = response; + this.registerRequestOutcome(response.status); + + if (response.ok) { + return response; + } + + const shouldRetry = this.shouldRetryResponse(response, attempt, maxAttempts, options); + if (!shouldRetry) { + return response; + } + + if ((response.status === 401 || response.status === 403) && options.retryOnForbidden !== false) { + await this.ensureSessionRefreshed({ force: true }); + } + + const retryDelay = this.computeRetryDelay(response, attempt, options); + const targetTime = Date.now() + retryDelay; + if (targetTime > this.nextRequestTime) { + this.nextRequestTime = targetTime; + } + } catch (error) { + lastError = error; + this.registerRequestOutcome(0); + if (attempt >= maxAttempts - 1) { + throw error; + } + + const retryDelay = this.computeRetryDelay(null, attempt, options); + const targetTime = Date.now() + retryDelay; + if (targetTime > this.nextRequestTime) { + this.nextRequestTime = targetTime; + } + } + } + + if (lastResponse) { + return lastResponse; + } + throw lastError ?? new Error(`Request to ${url} failed without a response.`); + } + async download(bookId, options = {}) { this.reset(); this.bookId = bookId; @@ -113,6 +285,8 @@ export class SafariBooksDownloader { kindle: Boolean(options.kindle) }; this.themeMode = this.options.theme === "none" ? "white" : this.options.theme; + this.updateReferrer(`${SAFARI_BASE_URL}/library/view/${this.bookId}/`); + updateRequestContext({ referer: this.getReferrerUrl() }); this.log(`Verifying session by loading profile page...`); await this.checkLogin(); @@ -120,6 +294,8 @@ export class SafariBooksDownloader { this.log(`Retrieving book metadata for ${bookId}...`); this.bookInfo = await this.fetchBookInfo(bookId); this.baseUrl = this.bookInfo.web_url || this.bookInfo.url; + this.updateReferrer(this.bookInfo.web_url || this.bookInfo.url); + updateRequestContext({ referer: this.getReferrerUrl() }); this.cleanTitle = makeFolderFriendlyName(this.bookInfo.title, this.bookId); const authorNames = (this.bookInfo.authors || []).map((author) => author.name).filter(Boolean); this.primaryAuthors = authorNames.join(", ") || "Unknown Author"; @@ -147,7 +323,7 @@ export class SafariBooksDownloader { } async checkLogin() { - const response = await fetch(PROFILE_URL, { credentials: "include" }); + const response = await this.fetchWithSession(PROFILE_URL, {}, { requireDocument: true }); if (!response.ok) { throw new Error(`Authentication failed (status ${response.status}) when accessing ${PROFILE_URL}`); } @@ -191,7 +367,11 @@ export class SafariBooksDownloader { } async fetchJson(url) { - const response = await fetch(url, { credentials: "include" }); + const response = await this.fetchWithSession(url, { + headers: { + Accept: "application/json, text/plain, */*" + } + }); if (!response.ok) { throw new Error(`Request to ${url} failed with status ${response.status}`); } @@ -361,7 +541,15 @@ export class SafariBooksDownloader { } async loadChapterDocument(url) { - const response = await fetch(url, { credentials: "include" }); + const response = await this.fetchWithSession( + url, + { + headers: { + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + } + }, + { requireDocument: true } + ); if (!response.ok) { throw new Error(`Failed to download chapter at ${url} (status ${response.status})`); } @@ -481,7 +669,11 @@ export class SafariBooksDownloader { if (this.cssContentMap.has(source)) { continue; } - const response = await fetch(source, { credentials: "include" }); + const response = await this.fetchWithSession(source, { + headers: { + Accept: "text/css,*/*;q=0.1" + } + }); if (!response.ok) { this.log(`Warning: unable to retrieve CSS from ${source} (status ${response.status})`); continue; @@ -527,7 +719,11 @@ export class SafariBooksDownloader { if (this.cssImageDownloads.has(localPath)) { continue; } - const response = await fetch(url, { credentials: "include" }); + const response = await this.fetchWithSession(url, { + headers: { + Accept: "font/woff2,application/font-woff,application/octet-stream,*/*" + } + }); if (!response.ok) { this.log(`Warning: unable to retrieve font ${url} (status ${response.status})`); continue; @@ -557,7 +753,11 @@ export class SafariBooksDownloader { } tried.add(candidate); try { - const response = await fetch(candidate, { credentials: "include" }); + const response = await this.fetchWithSession(candidate, { + headers: { + Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" + } + }); if (!response.ok) { attempts.push(`${candidate} (status ${response.status})`); continue; @@ -623,7 +823,11 @@ export class SafariBooksDownloader { if (entry?.data) { continue; } - const response = await fetch(entry, { credentials: "include" }); + const response = await this.fetchWithSession(entry, { + headers: { + Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8" + } + }); if (!response.ok) { this.log(`Warning: unable to fetch CSS image ${entry} (status ${response.status})`); continue;