jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

While downloading, save a cache manifest alongside

Browse source 
This allows us to keep track of the chapter title after the fact.
This commit is contained in:
Domenic Denicola 2015-05-11 20:49:09 -04:00
commit b68b88e17e
4 changed files with 62 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/common.js Normal file
View file

@ -0,0 +1,5 @@
"use strict";
exports.getChapterTitle = function (rawChapterDoc) {
return rawChapterDoc.querySelector("h1.entry-title").textContent;
};

12
lib/convert.js
View file

@ -2,8 +2,9 @@
const path = require("path"); const path = require("path");
const fs = require("mz/fs"); const fs = require("mz/fs");
const throat = require("throat"); const throat = require("throat");
const jsdom = require("./jsdom.js");
const serializeToXml = require("xmlserializer").serializeToString; const serializeToXml = require("xmlserializer").serializeToString;
const jsdom = require("./jsdom.js");
const getChapterTitle = require("./common.js").getChapterTitle;
module.exports = function (cachePath, contentPath) { module.exports = function (cachePath, contentPath) {
return getChapterFilePaths(cachePath) return getChapterFilePaths(cachePath)
@ -50,9 +51,8 @@ function convertChapter(filePath, contentPath) {
} }
function getChapterString(rawChapterDoc) { function getChapterString(rawChapterDoc) {
const headingEl = rawChapterDoc.querySelector("h1.entry-title"); const title = getChapterTitle(rawChapterDoc);
const title = headingEl.textContent; const body = getBodyXml(title, rawChapterDoc.querySelector(".entry-content"));
const body = getBodyXml(headingEl, rawChapterDoc.querySelector(".entry-content"));
return `<?xml version="1.0" encoding="UTF-8" ?> return `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@ -65,7 +65,7 @@ ${body}
</html>`; </html>`;
} }
function getBodyXml(headingEl, contentEl) { function getBodyXml(title, contentEl) {
// Remove initial Next Chapter and Previous Chapter <p> // Remove initial Next Chapter and Previous Chapter <p>
contentEl.removeChild(contentEl.firstElementChild); contentEl.removeChild(contentEl.firstElementChild);
@ -108,7 +108,7 @@ function getBodyXml(headingEl, contentEl) {
// Synthesize a <body> tag to serialize // Synthesize a <body> tag to serialize
const bodyEl = contentEl.ownerDocument.createElement("body"); const bodyEl = contentEl.ownerDocument.createElement("body");
const h1El = contentEl.ownerDocument.createElement("h1"); const h1El = contentEl.ownerDocument.createElement("h1");
h1El.textContent = headingEl.textContent; h1El.textContent = title;
bodyEl.appendChild(h1El); bodyEl.appendChild(h1El);
while (contentEl.firstChild) { while (contentEl.firstChild) {

76
lib/download.js
View file

@ -5,24 +5,20 @@ const mkdirp = require("mkdirp-then");
const request = require("requisition"); const request = require("requisition");
const zfill = require("zfill"); const zfill = require("zfill");
const jsdom = require("./jsdom.js"); const jsdom = require("./jsdom.js");
const getChapterTitle = require("./common.js").getChapterTitle;
const FILENAME_PREFIX = "chapter"; const FILENAME_PREFIX = "chapter";
const URL_SUFFIX = "-url.txt";
module.exports = function (startChapterUrl, cachePath) { module.exports = function (startChapterUrl, cachePath, manifestPath) {
return fs.readdir(cachePath).then( return fs.readFile(manifestPath, { encoding: "utf-8" }).then(
function (filenames) { function (manifestContents) {
const sorted = filenames.filter(function (f) { return f.endsWith(URL_SUFFIX); }).sort(); const manifest = JSON.parse(manifestContents);
const lastChapterUrlFile = sorted[sorted.length - 1];
const lastChapterNumber = getChapterNumber(lastChapterUrlFile);
return fs.readFile(path.resolve(cachePath, lastChapterUrlFile), { encoding: "utf-8" }).then(function (url) { return downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath);
return downloadAllChapters(url, lastChapterNumber, cachePath);
});
}, },
function (e) { function (e) {
if (e.code === "ENOENT") { if (e.code === "ENOENT") {
return downloadAllChapters(startChapterUrl, 1, cachePath); return downloadAllChapters(null, startChapterUrl, cachePath, manifestPath);
} else { } else {
throw e; throw e;
} }
@ -30,21 +26,27 @@ module.exports = function (startChapterUrl, cachePath) {
); );
}; };
function downloadAllChapters(startChapterUrl, startChapterNumber, cachePath) { function downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath) {
let currentChapter = startChapterUrl; let currentChapter = startChapterUrl;
let chapterCounter = startChapterNumber; let chapterIndex = 0;
if (manifest !== null) {
currentChapter = manifest[manifest.length - 1].url;
chapterIndex = manifest.length - 1;
// We're going to re-add it to the manifest later, possibly with an updated title.
manifest.pop();
} else {
manifest = [];
}
return mkdirp(cachePath).then(loop); return mkdirp(cachePath).then(loop);
function loop() { function loop() {
const filename = `${FILENAME_PREFIX}${zfill(chapterCounter, 3)}.html`; const filename = `${FILENAME_PREFIX}${zfill(chapterIndex, 3)}.html`;
console.log(`Downloading ${currentChapter}`); console.log(`Downloading ${currentChapter}`);
// Necessary for https://parahumans.wordpress.com/2011/10/11/interlude-3½-bonus/ return downloadChapter(currentChapter).then(function (response) {
const escapedUrl = encodeURI(currentChapter);
return retry(3, function () { return request(escapedUrl).redirects(10); }).then(function (response) {
console.log("- Response received"); console.log("- Response received");
return response.text(); return response.text();
}) })
@ -54,27 +56,42 @@ function downloadAllChapters(startChapterUrl, startChapterNumber, cachePath) {
console.log("- Response body parsed into DOM"); console.log("- Response body parsed into DOM");
const chapterUrlToSave = currentChapter; const chapterUrlToSave = currentChapter;
const chapterTitle = getChapterTitle(rawChapterDoc);
currentChapter = getNextChapterUrl(rawChapterDoc); currentChapter = getNextChapterUrl(rawChapterDoc);
console.log("- Got next chapter URL");
// TODO: this should probably not be necessary... jsdom bug I guess!? // TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterDoc.defaultView.close(); rawChapterDoc.defaultView.close();
return [ manifest.push({
fs.writeFile(path.resolve(cachePath, filename), contents), url: chapterUrlToSave,
fs.writeFile(path.resolve(cachePath, urlFilename(filename)), chapterUrlToSave) title: chapterTitle,
]; filename: filename
});
fs.writeFile(path.resolve(cachePath, filename), contents);
}) })
.then(function () { .then(function () {
console.log("- Response text saved to cache file"); console.log("- Response text saved to cache file");
// Incrementally update the manifest after every successful download, instead of waiting until the end.
return writeManifest();
})
.then(function () {
console.log("- Manifest updated");
if (currentChapter === null) { if (currentChapter === null) {
return; return;
} }
++chapterCounter; ++chapterIndex;
return loop(); return loop();
}); });
} }
function writeManifest() {
const contents = JSON.stringify(manifest, undefined, 2);
return fs.writeFile(manifestPath, contents);
}
} }
function getNextChapterUrl(rawChapterDoc) { function getNextChapterUrl(rawChapterDoc) {
@ -103,10 +120,11 @@ function retry(times, fn) {
}); });
} }
function urlFilename(filename) { function downloadChapter(url) {
return `${filename}${URL_SUFFIX}`; // Necessary for https://parahumans.wordpress.com/2011/10/11/interlude-3½-bonus/
} const escapedUrl = encodeURI(url);
function getChapterNumber(filename) { return retry(3, function () {
return Number(/0?0?(\d+)\./.exec(filename)[1]); return request(escapedUrl).redirects(10);
});
} }

7
lib/worm-scraper.js
View file

@ -15,12 +15,13 @@ const cachePath = path.resolve("cache");
const outPath = path.resolve("out"); const outPath = path.resolve("out");
const contentPath = path.resolve(outPath, "OEBPS"); const contentPath = path.resolve(outPath, "OEBPS");
const chaptersPath = path.resolve(contentPath, "chapters"); const chaptersPath = path.resolve(contentPath, "chapters");
const manifestPath = path.resolve(cachePath, "manifest.json");
Promise.resolve() Promise.resolve()
.then(function () { .then(function () {
// return download(START_CHAPTER_URL, cachePath); return download(START_CHAPTER_URL, cachePath, manifestPath);
}) })
.then(function () { /* .then(function () {
return rimraf(chaptersPath); return rimraf(chaptersPath);
}) })
.then(function () { .then(function () {
@ -31,7 +32,7 @@ Promise.resolve()
}) })
.then(function () { .then(function () {
return extras(contentPath, chaptersPath); return extras(contentPath, chaptersPath);
}) })*/
.then(function () { .then(function () {
console.log("All done!"); console.log("All done!");
}); });