Update dev dependencies and use new JS features
This commit is contained in:
Domenic Denicola
parent
853b76a881
commit
d5157f2c22
10 changed files with 331 additions and 144 deletions
|
|
@ -2,23 +2,21 @@
|
|||
const path = require("path");
|
||||
const fs = require("mz/fs");
|
||||
const throat = require("throat");
|
||||
const serializeToXml = require("xmlserializer").serializeToString;
|
||||
const serializeToXML = require("xmlserializer").serializeToString;
|
||||
const jsdom = require("./jsdom.js");
|
||||
const substitutions = require("./substitutions.json");
|
||||
|
||||
module.exports = function (cachePath, manifestPath, contentPath) {
|
||||
return fs.readFile(manifestPath, { encoding: "utf-8" }).then(function (manifestContents) {
|
||||
module.exports = (cachePath, manifestPath, contentPath) => {
|
||||
return fs.readFile(manifestPath, { encoding: "utf-8" }).then(manifestContents => {
|
||||
const chapters = JSON.parse(manifestContents);
|
||||
console.log("All chapters downloaded; beginning conversion to EPUB chapters");
|
||||
|
||||
const mapper = throat(10, function (chapter) {
|
||||
const mapper = throat(10, chapter => {
|
||||
return convertChapter(chapter, cachePath, contentPath);
|
||||
});
|
||||
return Promise.all(chapters.map(mapper));
|
||||
})
|
||||
.then(function () {
|
||||
console.log("All chapters converted");
|
||||
});
|
||||
.then(() => console.log("All chapters converted"));
|
||||
};
|
||||
|
||||
function convertChapter(chapter, cachePath, contentPath) {
|
||||
|
|
@ -26,7 +24,7 @@ function convertChapter(chapter, cachePath, contentPath) {
|
|||
const filePath = path.resolve(cachePath, filename);
|
||||
|
||||
console.log(`- Reading ${filename}`);
|
||||
return fs.readFile(filePath, { encoding: "utf-8" }).then(function (contents) {
|
||||
return fs.readFile(filePath, { encoding: "utf-8" }).then(contents => {
|
||||
console.log(`- Read ${filename}`);
|
||||
const rawChapterDoc = jsdom(contents);
|
||||
const output = getChapterString(chapter, rawChapterDoc);
|
||||
|
|
@ -38,13 +36,11 @@ function convertChapter(chapter, cachePath, contentPath) {
|
|||
const destFilePath = path.resolve(contentPath, destFileName);
|
||||
return fs.writeFile(destFilePath, output);
|
||||
})
|
||||
.then(function () {
|
||||
console.log(`- Finished converting ${filename}`);
|
||||
});
|
||||
.then(() => console.log(`- Finished converting ${filename}`));
|
||||
}
|
||||
|
||||
function getChapterString(chapter, rawChapterDoc) {
|
||||
const body = getBodyXml(chapter, rawChapterDoc.querySelector(".entry-content"));
|
||||
const body = getBodyXML(chapter, rawChapterDoc.querySelector(".entry-content"));
|
||||
|
||||
return `<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
||||
|
|
@ -57,7 +53,7 @@ ${body}
|
|||
</html>`;
|
||||
}
|
||||
|
||||
function getBodyXml(chapter, contentEl) {
|
||||
function getBodyXML(chapter, contentEl) {
|
||||
// Remove initial Next Chapter and Previous Chapter <p>
|
||||
contentEl.removeChild(contentEl.firstElementChild);
|
||||
|
||||
|
|
@ -73,7 +69,7 @@ function getBodyXml(chapter, contentEl) {
|
|||
}
|
||||
|
||||
// Remove redundant attributes
|
||||
Array.prototype.forEach.call(contentEl.children, function (child) {
|
||||
Array.prototype.forEach.call(contentEl.children, child => {
|
||||
if (child.getAttribute("dir") === "ltr") {
|
||||
child.removeAttribute("dir");
|
||||
}
|
||||
|
|
@ -90,7 +86,7 @@ function getBodyXml(chapter, contentEl) {
|
|||
// Remove empty <em>s and <i>s
|
||||
// Remove style attributes from them, as they're always messed up.
|
||||
const ems = contentEl.querySelectorAll("em, i");
|
||||
Array.prototype.forEach.call(ems, function (em) {
|
||||
Array.prototype.forEach.call(ems, em => {
|
||||
if (em.textContent.trim() === "") {
|
||||
const replacement = contentEl.ownerDocument.createTextNode(" ");
|
||||
em.parentNode.replaceChild(replacement, em);
|
||||
|
|
@ -101,7 +97,7 @@ function getBodyXml(chapter, contentEl) {
|
|||
|
||||
// In https://parahumans.wordpress.com/2013/01/05/monarch-16-13/ there are some <address>s that should be <p>s O_o
|
||||
const addresses = contentEl.querySelectorAll("address");
|
||||
Array.prototype.forEach.call(addresses, function (address) {
|
||||
Array.prototype.forEach.call(addresses, address => {
|
||||
const p = contentEl.ownerDocument.createElement("p");
|
||||
p.innerHTML = address.innerHTML;
|
||||
address.parentNode.replaceChild(p, address);
|
||||
|
|
@ -110,7 +106,7 @@ function getBodyXml(chapter, contentEl) {
|
|||
// Every <span> except underline ones is pointless at best and frequently messed up. (Weird font size, line spacing,
|
||||
// etc.)
|
||||
const spans = contentEl.querySelectorAll("span");
|
||||
Array.prototype.forEach.call(spans, function (span) {
|
||||
Array.prototype.forEach.call(spans, span => {
|
||||
if (span.getAttribute("style") === "text-decoration:underline;") {
|
||||
return;
|
||||
}
|
||||
|
|
@ -137,7 +133,7 @@ function getBodyXml(chapter, contentEl) {
|
|||
bodyEl.appendChild(contentEl.firstChild);
|
||||
}
|
||||
|
||||
let xml = serializeToXml(bodyEl);
|
||||
let xml = serializeToXML(bodyEl);
|
||||
|
||||
// Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p>
|
||||
xml = xml.replace(/<br\/>\s*<\/em><\/p>/g, "</em></p>");
|
||||
|
|
@ -151,8 +147,8 @@ function getBodyXml(chapter, contentEl) {
|
|||
xml = xml.replace(/<\/em><em>/g, "");
|
||||
xml = xml.replace(/<em>(\s?\s?[^A-Za-z]\s?\s?)<\/em>/g, "$1");
|
||||
xml = xml.replace(/<\/em>(\s?\s?[^A-Za-z]\s?\s?)<em>/g, "$1");
|
||||
xml = xml.replace(/“<em>([^>]+)<\/em>(!|\?|\.)”/g, "“<em>$1$2<\/em>”");
|
||||
xml = xml.replace(/<p><em>([^>]+)<\/em>(!|\?|\.)<\/p>/g, "<p><em>$1$2<\/em></p>");
|
||||
xml = xml.replace(/“<em>([^>]+)<\/em>(!|\?|\.)”/g, "“<em>$1$2</em>”");
|
||||
xml = xml.replace(/<p><em>([^>]+)<\/em>(!|\?|\.)<\/p>/g, "<p><em>$1$2</em></p>");
|
||||
xml = xml.replace(/(!|\?|\.)\s{2}<\/em><\/p>/g, "$1</em></p>");
|
||||
xml = xml.replace(/<em>([a-z]+)\?<\/em>/g, "<em>$1</em>?");
|
||||
|
||||
|
|
@ -173,9 +169,9 @@ function getBodyXml(chapter, contentEl) {
|
|||
// Fixes dashes
|
||||
xml = xml.replace(/ – /g, "—");
|
||||
xml = xml.replace(/“-/g, "“—");
|
||||
xml = xml.replace(/-[,\.]?”/g, "—”");
|
||||
xml = xml.replace(/-[,.]?”/g, "—”");
|
||||
xml = xml.replace(/-(!|\?)”/g, "—$1”");
|
||||
xml = xml.replace(/-[,\.]?<\/em>”/g, "—</em>”");
|
||||
xml = xml.replace(/-[,.]?<\/em>”/g, "—</em>”");
|
||||
xml = xml.replace(/-“/g, "—”");
|
||||
xml = xml.replace(/<p>-/g, "<p>—");
|
||||
xml = xml.replace(/-<\/p>/g, "—</p>");
|
||||
|
|
@ -199,7 +195,7 @@ function getBodyXml(chapter, contentEl) {
|
|||
xml = xml.replace(/the clairvoyant/g, "the Clairvoyant");
|
||||
|
||||
// One-off fixes
|
||||
(substitutions[chapter.url] || []).forEach(function (substitution) {
|
||||
(substitutions[chapter.url] || []).forEach(substitution => {
|
||||
const indexOf = xml.indexOf(substitution.before);
|
||||
if (indexOf === -1) {
|
||||
console.warn(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
|
||||
|
|
@ -225,5 +221,5 @@ function isEmptyOrGarbage(el) {
|
|||
}
|
||||
|
||||
function escapeRegExp(str) {
|
||||
return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
|
||||
return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,19 +8,18 @@ const jsdom = require("./jsdom.js");
|
|||
|
||||
const FILENAME_PREFIX = "chapter";
|
||||
|
||||
module.exports = function (startChapterUrl, cachePath, manifestPath) {
|
||||
module.exports = (startChapterUrl, cachePath, manifestPath) => {
|
||||
return fs.readFile(manifestPath, { encoding: "utf-8" }).then(
|
||||
function (manifestContents) {
|
||||
manifestContents => {
|
||||
const manifest = JSON.parse(manifestContents);
|
||||
|
||||
return downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath);
|
||||
},
|
||||
function (e) {
|
||||
e => {
|
||||
if (e.code === "ENOENT") {
|
||||
return downloadAllChapters(null, startChapterUrl, cachePath, manifestPath);
|
||||
} else {
|
||||
throw e;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
);
|
||||
};
|
||||
|
|
@ -45,11 +44,11 @@ function downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath)
|
|||
|
||||
console.log(`Downloading ${currentChapter}`);
|
||||
|
||||
return downloadChapter(currentChapter).then(function (response) {
|
||||
return downloadChapter(currentChapter).then(response => {
|
||||
console.log("- Response received");
|
||||
return response.text();
|
||||
})
|
||||
.then(function (contents) {
|
||||
.then(contents => {
|
||||
console.log("- Response body received");
|
||||
const rawChapterDoc = jsdom(contents, { url: currentChapter });
|
||||
console.log("- Response body parsed into DOM");
|
||||
|
|
@ -69,16 +68,16 @@ function downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath)
|
|||
|
||||
fs.writeFile(path.resolve(cachePath, filename), contents);
|
||||
})
|
||||
.then(function () {
|
||||
.then(() => {
|
||||
console.log("- Response text saved to cache file");
|
||||
// Incrementally update the manifest after every successful download, instead of waiting until the end.
|
||||
return writeManifest();
|
||||
})
|
||||
.then(function () {
|
||||
.then(() => {
|
||||
console.log("- Manifest updated");
|
||||
|
||||
if (currentChapter === null) {
|
||||
return;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
++chapterIndex;
|
||||
|
|
@ -94,7 +93,7 @@ function downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath)
|
|||
}
|
||||
|
||||
function getNextChapterUrl(rawChapterDoc) {
|
||||
// a[title="Next Chapter"] doesn"t always work. Two different pathologies:
|
||||
// `a[title="Next Chapter"]` doesn"t always work. Two different pathologies:
|
||||
// - https://parahumans.wordpress.com/2011/09/27/shell-4-2/
|
||||
// - https://parahumans.wordpress.com/2012/04/21/sentinel-9-6/
|
||||
// So instead search for the first <a> within the main content area starting with "Next", trimmed.
|
||||
|
|
@ -118,7 +117,7 @@ function retry(times, fn) {
|
|||
return fn();
|
||||
}
|
||||
|
||||
return fn().catch(function () {
|
||||
return fn().catch(() => {
|
||||
return retry(times - 1, fn);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
13
lib/jsdom.js
13
lib/jsdom.js
|
|
@ -1,13 +1,14 @@
|
|||
"use strict";
|
||||
const jsdom = require("jsdom");
|
||||
const xtend = require("xtend");
|
||||
|
||||
// No need to fetch or execute JavaScript
|
||||
module.exports = function (contents, options) {
|
||||
options = xtend(options, { features: {
|
||||
FetchExternalResources: false,
|
||||
ProcessExternalResources: false
|
||||
}});
|
||||
module.exports = (contents, options) => {
|
||||
options = Object.assign({}, options, {
|
||||
features: {
|
||||
FetchExternalResources: false,
|
||||
ProcessExternalResources: false
|
||||
}
|
||||
});
|
||||
|
||||
return jsdom.jsdom(contents, options);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -21,17 +21,17 @@ const COVER_IMG_FILENAME = "cover.png";
|
|||
const COVER_XHTML_FILENAME = "cover.xhtml";
|
||||
const COVER_MIMETYPE = "image/png";
|
||||
|
||||
module.exports = function (scaffoldingPath, bookPath, contentPath, chaptersPath, manifestPath) {
|
||||
module.exports = (scaffoldingPath, bookPath, contentPath, chaptersPath, manifestPath) => {
|
||||
return Promise.all([
|
||||
getChapters(contentPath, chaptersPath, manifestPath).then(function (chapters) {
|
||||
cpr(scaffoldingPath, bookPath, { overwrite: true, confirm: true, filter: noThumbs }),
|
||||
getChapters(contentPath, chaptersPath, manifestPath).then(chapters => {
|
||||
return Promise.all([
|
||||
writeOpf(chapters, contentPath),
|
||||
writeNcx(chapters, contentPath)
|
||||
]);
|
||||
})
|
||||
])
|
||||
.then(function () { });
|
||||
.then(() => undefined);
|
||||
};
|
||||
|
||||
function noThumbs(filePath) {
|
||||
|
|
@ -40,11 +40,11 @@ function noThumbs(filePath) {
|
|||
}
|
||||
|
||||
function writeOpf(chapters, contentPath) {
|
||||
const manifestChapters = chapters.map(function (c) {
|
||||
const manifestChapters = chapters.map(c => {
|
||||
return `<item id="${c.id}" href="${c.href}" media-type="application/xhtml+xml"/>`;
|
||||
}).join("\n");
|
||||
|
||||
const spineChapters = chapters.map(function (c) {
|
||||
const spineChapters = chapters.map(c => {
|
||||
return `<itemref idref="${c.id}"/>`;
|
||||
}).join("\n");
|
||||
|
||||
|
|
@ -82,7 +82,7 @@ ${spineChapters}
|
|||
}
|
||||
|
||||
function writeNcx(chapters, contentPath) {
|
||||
const navPoints = chapters.map(function (c, i) {
|
||||
const navPoints = chapters.map((c, i) => {
|
||||
return `<navPoint class="chapter" id="${c.id}" playOrder="${i + 1}">
|
||||
<navLabel><text>${c.title}</text></navLabel>
|
||||
<content src="${c.href}"/>
|
||||
|
|
@ -118,15 +118,14 @@ ${navPoints}
|
|||
function getChapters(contentPath, chaptersPath, manifestPath) {
|
||||
const hrefPrefix = `${path.relative(contentPath, chaptersPath)}/`;
|
||||
|
||||
return fs.readFile(manifestPath, { encoding: "utf-8" }).then(function (manifestContents) {
|
||||
return fs.readFile(manifestPath, { encoding: "utf-8" }).then(manifestContents => {
|
||||
const manifestChapters = JSON.parse(manifestContents);
|
||||
|
||||
return fs.readdir(chaptersPath).then(function (filenames) {
|
||||
return filenames.filter(function (f) {
|
||||
return path.extname(f) === ".xhtml";
|
||||
})
|
||||
return fs.readdir(chaptersPath).then(filenames => {
|
||||
return filenames
|
||||
.filter(f => path.extname(f) === ".xhtml")
|
||||
.sort()
|
||||
.map(function (f, i) {
|
||||
.map((f, i) => {
|
||||
return {
|
||||
id: path.basename(f),
|
||||
title: manifestChapters[i].title,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
"use strict";
|
||||
/* eslint-disable no-process-exit */
|
||||
const path = require("path");
|
||||
const mkdirp = require("mkdirp-then");
|
||||
const rimraf = require("rimraf-then");
|
||||
|
|
@ -62,41 +63,31 @@ const chaptersPath = path.resolve(contentPath, "chapters");
|
|||
|
||||
const commands = [];
|
||||
|
||||
if (argv._.indexOf("download") !== -1) {
|
||||
commands.push(function () {
|
||||
return download(argv.startUrl, cachePath, manifestPath);
|
||||
if (argv._.includes("download")) {
|
||||
commands.push(() => download(argv.startUrl, cachePath, manifestPath));
|
||||
}
|
||||
|
||||
if (argv._.includes("convert")) {
|
||||
commands.push(() => {
|
||||
return rimraf(chaptersPath)
|
||||
.then(() => mkdirp(chaptersPath))
|
||||
.then(() => convert(cachePath, manifestPath, chaptersPath));
|
||||
});
|
||||
}
|
||||
|
||||
if (argv._.indexOf("convert") !== -1) {
|
||||
commands.push(function () {
|
||||
return rimraf(chaptersPath).then(function () {
|
||||
return mkdirp(chaptersPath);
|
||||
})
|
||||
.then(function () {
|
||||
return convert(cachePath, manifestPath, chaptersPath);
|
||||
});
|
||||
});
|
||||
if (argv._.includes("scaffold")) {
|
||||
commands.push(() => scaffold(scaffoldingPath, bookPath, contentPath, chaptersPath, manifestPath));
|
||||
}
|
||||
|
||||
if (argv._.indexOf("scaffold") !== -1) {
|
||||
commands.push(function () {
|
||||
return scaffold(scaffoldingPath, bookPath, contentPath, chaptersPath, manifestPath);
|
||||
});
|
||||
if (argv._.includes("zip")) {
|
||||
commands.push(() => zip(bookPath, contentPath, path.resolve(argv.out)));
|
||||
}
|
||||
|
||||
if (argv._.indexOf("zip") !== -1) {
|
||||
commands.push(function () {
|
||||
return zip(bookPath, contentPath, path.resolve(argv.out));
|
||||
});
|
||||
}
|
||||
|
||||
commands.reduce(function (previous, command) {
|
||||
commands.reduce((previous, command) => {
|
||||
return previous.then(command);
|
||||
}, Promise.resolve())
|
||||
.then(function () {
|
||||
console.log("All done!");
|
||||
})
|
||||
.catch(function (e) {
|
||||
.then(() => console.log("All done!"))
|
||||
.catch(e => {
|
||||
console.error(e.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -3,14 +3,14 @@ const fs = require("fs");
|
|||
const path = require("path");
|
||||
const archiver = require("archiver");
|
||||
|
||||
module.exports = function (bookPath, contentPath, outPath) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
module.exports = (bookPath, contentPath, outPath) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
console.log(`Zipping up ${bookPath} into an EPUB`);
|
||||
|
||||
const archive = archiver("zip");
|
||||
const destStream = fs.createWriteStream(outPath);
|
||||
|
||||
destStream.on("close", function () {
|
||||
destStream.on("close", () => {
|
||||
console.log(`EPUB written to ${outPath} (${archive.pointer()} bytes)`);
|
||||
resolve();
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue