Update downloader
This commit is contained in:
parent
8fd5438f6c
commit
edd0f9f809
2 changed files with 215 additions and 7 deletions
|
|
@ -44,3 +44,7 @@ export async function collectSessionCookies() {
|
|||
cookies: jar
|
||||
};
|
||||
}
|
||||
|
||||
export function updateRequestContext() {
|
||||
// Chrome/Edge do not require extra request header handling.
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import {
|
|||
isLikelyImageAsset
|
||||
} from "./utils.js";
|
||||
import { SimpleZip } from "./zip.js";
|
||||
import { collectSessionCookies, updateRequestContext } from "./cookies.js";
|
||||
|
||||
const SAFARI_BASE_URL = "https://learning.oreilly.com";
|
||||
const API_V2_TEMPLATE = `${SAFARI_BASE_URL}/api/v2/epubs/urn:orm:book:`;
|
||||
|
|
@ -32,6 +33,10 @@ const KINDLE_CSS =
|
|||
"#sbo-rt-content table,#sbo-rt-content pre{overflow-x:unset!important;overflow:unset!important;" +
|
||||
"overflow-y:unset!important;white-space:pre-wrap!important;}";
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, Math.max(ms, 0)));
|
||||
}
|
||||
|
||||
function formatTwoDigits(value) {
|
||||
return value.toString().padStart(2, "0");
|
||||
}
|
||||
|
|
@ -94,6 +99,12 @@ export class SafariBooksDownloader {
|
|||
this.coverPath = "";
|
||||
this.themeMode = "white";
|
||||
this.options = { theme: "none", kindle: false };
|
||||
this.referrerUrl = SAFARI_BASE_URL;
|
||||
this.baseRequestSpacingMs = 200;
|
||||
this.nextRequestTime = Date.now();
|
||||
this.rateLimitPenaltyMs = 0;
|
||||
this.sessionRefreshCooldownMs = 30000;
|
||||
this.lastSessionRefresh = 0;
|
||||
|
||||
this.cssSources = [];
|
||||
this.cssFilenameMap = new Map();
|
||||
|
|
@ -105,6 +116,167 @@ export class SafariBooksDownloader {
|
|||
this.imagesFromLinks = new Set();
|
||||
}
|
||||
|
||||
getReferrerUrl() {
|
||||
return this.referrerUrl || SAFARI_BASE_URL;
|
||||
}
|
||||
|
||||
updateReferrer(url) {
|
||||
if (typeof url === "string" && url.startsWith("http")) {
|
||||
this.referrerUrl = url;
|
||||
}
|
||||
}
|
||||
|
||||
async ensureSessionRefreshed({ force = false } = {}) {
|
||||
if (typeof collectSessionCookies !== "function") {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
if (!force && now - this.lastSessionRefresh < this.sessionRefreshCooldownMs) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await collectSessionCookies();
|
||||
this.lastSessionRefresh = Date.now();
|
||||
} catch (error) {
|
||||
this.log(`Warning: unable to refresh session cookies (${error.message}).`);
|
||||
}
|
||||
}
|
||||
|
||||
async applyRateLimit() {
|
||||
const waitTime = this.nextRequestTime - Date.now();
|
||||
if (waitTime > 0) {
|
||||
await sleep(waitTime);
|
||||
}
|
||||
}
|
||||
|
||||
registerRequestOutcome(status) {
|
||||
const penalizedStatuses = new Set([0, 401, 403, 429, 500, 502, 503, 504]);
|
||||
if (penalizedStatuses.has(status)) {
|
||||
if (status === 429 || status === 503) {
|
||||
this.rateLimitPenaltyMs = Math.min(Math.max(this.rateLimitPenaltyMs * 2, 500), 8000);
|
||||
} else {
|
||||
this.rateLimitPenaltyMs = Math.min(this.rateLimitPenaltyMs + 250, 6000);
|
||||
}
|
||||
} else {
|
||||
this.rateLimitPenaltyMs = Math.max(Math.floor(this.rateLimitPenaltyMs * 0.5), 0);
|
||||
}
|
||||
const delay = this.baseRequestSpacingMs + this.rateLimitPenaltyMs;
|
||||
this.nextRequestTime = Date.now() + delay;
|
||||
}
|
||||
|
||||
shouldRetryResponse(response, attempt, maxAttempts, options = {}) {
|
||||
if (attempt >= maxAttempts - 1) {
|
||||
return false;
|
||||
}
|
||||
const status = response?.status ?? 0;
|
||||
if (status === 0) {
|
||||
return true;
|
||||
}
|
||||
if ([429, 500, 502, 503, 504].includes(status)) {
|
||||
return true;
|
||||
}
|
||||
if ((status === 401 || status === 403) && options.retryOnForbidden !== false) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
computeRetryDelay(response, attempt, options = {}) {
|
||||
const status = response?.status ?? 0;
|
||||
const base = options.baseRetryDelay ?? 750;
|
||||
const cap = options.maxRetryDelay ?? 8000;
|
||||
const growth = Math.min(cap, base * Math.pow(2, attempt));
|
||||
const jitter = Math.random() * 250;
|
||||
if (status === 429 || status === 503) {
|
||||
return Math.min(cap, growth + 1000 + jitter);
|
||||
}
|
||||
return Math.min(cap, growth + jitter);
|
||||
}
|
||||
|
||||
buildFetchInit(init = {}, { requireDocument = false } = {}) {
|
||||
const { headers, ...rest } = init || {};
|
||||
const mergedHeaders = new Headers(headers || {});
|
||||
if (!mergedHeaders.has("X-Requested-With")) {
|
||||
mergedHeaders.set("X-Requested-With", "XMLHttpRequest");
|
||||
}
|
||||
if (requireDocument && !mergedHeaders.has("Accept")) {
|
||||
mergedHeaders.set(
|
||||
"Accept",
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
);
|
||||
}
|
||||
if (!mergedHeaders.has("Accept-Language")) {
|
||||
mergedHeaders.set("Accept-Language", "en-US,en;q=0.9");
|
||||
}
|
||||
|
||||
const initWithDefaults = {
|
||||
credentials: "include",
|
||||
mode: "cors",
|
||||
...rest,
|
||||
headers: mergedHeaders
|
||||
};
|
||||
|
||||
if (!initWithDefaults.referrer) {
|
||||
initWithDefaults.referrer = this.getReferrerUrl();
|
||||
}
|
||||
if (!initWithDefaults.referrerPolicy) {
|
||||
initWithDefaults.referrerPolicy = "strict-origin-when-cross-origin";
|
||||
}
|
||||
return initWithDefaults;
|
||||
}
|
||||
|
||||
async fetchWithSession(url, init = {}, options = {}) {
|
||||
const maxAttempts = Math.max(1, options.maxAttempts ?? 4);
|
||||
let lastError = null;
|
||||
let lastResponse = null;
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
await this.applyRateLimit();
|
||||
|
||||
try {
|
||||
const response = await fetch(url, this.buildFetchInit(init, options));
|
||||
lastResponse = response;
|
||||
this.registerRequestOutcome(response.status);
|
||||
|
||||
if (response.ok) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const shouldRetry = this.shouldRetryResponse(response, attempt, maxAttempts, options);
|
||||
if (!shouldRetry) {
|
||||
return response;
|
||||
}
|
||||
|
||||
if ((response.status === 401 || response.status === 403) && options.retryOnForbidden !== false) {
|
||||
await this.ensureSessionRefreshed({ force: true });
|
||||
}
|
||||
|
||||
const retryDelay = this.computeRetryDelay(response, attempt, options);
|
||||
const targetTime = Date.now() + retryDelay;
|
||||
if (targetTime > this.nextRequestTime) {
|
||||
this.nextRequestTime = targetTime;
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
this.registerRequestOutcome(0);
|
||||
if (attempt >= maxAttempts - 1) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const retryDelay = this.computeRetryDelay(null, attempt, options);
|
||||
const targetTime = Date.now() + retryDelay;
|
||||
if (targetTime > this.nextRequestTime) {
|
||||
this.nextRequestTime = targetTime;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lastResponse) {
|
||||
return lastResponse;
|
||||
}
|
||||
throw lastError ?? new Error(`Request to ${url} failed without a response.`);
|
||||
}
|
||||
|
||||
async download(bookId, options = {}) {
|
||||
this.reset();
|
||||
this.bookId = bookId;
|
||||
|
|
@ -113,6 +285,8 @@ export class SafariBooksDownloader {
|
|||
kindle: Boolean(options.kindle)
|
||||
};
|
||||
this.themeMode = this.options.theme === "none" ? "white" : this.options.theme;
|
||||
this.updateReferrer(`${SAFARI_BASE_URL}/library/view/${this.bookId}/`);
|
||||
updateRequestContext({ referer: this.getReferrerUrl() });
|
||||
|
||||
this.log(`Verifying session by loading profile page...`);
|
||||
await this.checkLogin();
|
||||
|
|
@ -120,6 +294,8 @@ export class SafariBooksDownloader {
|
|||
this.log(`Retrieving book metadata for ${bookId}...`);
|
||||
this.bookInfo = await this.fetchBookInfo(bookId);
|
||||
this.baseUrl = this.bookInfo.web_url || this.bookInfo.url;
|
||||
this.updateReferrer(this.bookInfo.web_url || this.bookInfo.url);
|
||||
updateRequestContext({ referer: this.getReferrerUrl() });
|
||||
this.cleanTitle = makeFolderFriendlyName(this.bookInfo.title, this.bookId);
|
||||
const authorNames = (this.bookInfo.authors || []).map((author) => author.name).filter(Boolean);
|
||||
this.primaryAuthors = authorNames.join(", ") || "Unknown Author";
|
||||
|
|
@ -147,7 +323,7 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
|
||||
async checkLogin() {
|
||||
const response = await fetch(PROFILE_URL, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(PROFILE_URL, {}, { requireDocument: true });
|
||||
if (!response.ok) {
|
||||
throw new Error(`Authentication failed (status ${response.status}) when accessing ${PROFILE_URL}`);
|
||||
}
|
||||
|
|
@ -191,7 +367,11 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
|
||||
async fetchJson(url) {
|
||||
const response = await fetch(url, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(url, {
|
||||
headers: {
|
||||
Accept: "application/json, text/plain, */*"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Request to ${url} failed with status ${response.status}`);
|
||||
}
|
||||
|
|
@ -361,7 +541,15 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
|
||||
async loadChapterDocument(url) {
|
||||
const response = await fetch(url, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(
|
||||
url,
|
||||
{
|
||||
headers: {
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||
}
|
||||
},
|
||||
{ requireDocument: true }
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to download chapter at ${url} (status ${response.status})`);
|
||||
}
|
||||
|
|
@ -481,7 +669,11 @@ export class SafariBooksDownloader {
|
|||
if (this.cssContentMap.has(source)) {
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(source, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(source, {
|
||||
headers: {
|
||||
Accept: "text/css,*/*;q=0.1"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
this.log(`Warning: unable to retrieve CSS from ${source} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
@ -527,7 +719,11 @@ export class SafariBooksDownloader {
|
|||
if (this.cssImageDownloads.has(localPath)) {
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(url, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(url, {
|
||||
headers: {
|
||||
Accept: "font/woff2,application/font-woff,application/octet-stream,*/*"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
this.log(`Warning: unable to retrieve font ${url} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
@ -557,7 +753,11 @@ export class SafariBooksDownloader {
|
|||
}
|
||||
tried.add(candidate);
|
||||
try {
|
||||
const response = await fetch(candidate, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(candidate, {
|
||||
headers: {
|
||||
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
attempts.push(`${candidate} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
@ -623,7 +823,11 @@ export class SafariBooksDownloader {
|
|||
if (entry?.data) {
|
||||
continue;
|
||||
}
|
||||
const response = await fetch(entry, { credentials: "include" });
|
||||
const response = await this.fetchWithSession(entry, {
|
||||
headers: {
|
||||
Accept: "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8"
|
||||
}
|
||||
});
|
||||
if (!response.ok) {
|
||||
this.log(`Warning: unable to fetch CSS image ${entry} (status ${response.status})`);
|
||||
continue;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue