Improve conversion speed using a worker pool
This commit is contained in:
Domenic Denicola
parent
6848456bc9
commit
ad2c0b0ca5
6 changed files with 501 additions and 488 deletions
473
lib/convert-worker.js
Normal file
473
lib/convert-worker.js
Normal file
|
|
@ -0,0 +1,473 @@
|
|||
"use strict";
|
||||
const workerpool = require("workerpool");
|
||||
const fs = require("fs");
|
||||
const { JSDOM } = require("jsdom");
|
||||
const substitutions = require("./substitutions.json");
|
||||
|
||||
workerpool.worker({ convertChapter });
|
||||
|
||||
function convertChapter(chapter, inputPath, outputPath) {
|
||||
const contents = fs.readFileSync(inputPath, { encoding: "utf-8" });
|
||||
|
||||
const rawChapterJSDOM = new JSDOM(contents);
|
||||
const output = getChapterString(chapter, rawChapterJSDOM.window.document);
|
||||
|
||||
// TODO: this should probably not be necessary... jsdom bug I guess!?
|
||||
rawChapterJSDOM.window.close();
|
||||
|
||||
fs.writeFileSync(outputPath, output);
|
||||
}
|
||||
|
||||
function getChapterString(chapter, rawChapterDoc) {
|
||||
const body = getBodyXML(chapter, rawChapterDoc.querySelector(".entry-content"));
|
||||
|
||||
return `<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
|
||||
<title>${chapter.title}</title>
|
||||
</head>
|
||||
${body}
|
||||
</html>`;
|
||||
}
|
||||
|
||||
function getBodyXML(chapter, contentEl) {
|
||||
// Remove initial Next Chapter and Previous Chapter <p>
|
||||
contentEl.removeChild(contentEl.firstElementChild);
|
||||
|
||||
// Remove everything after the last <p> (e.g. analytics <div>s)
|
||||
const lastP = contentEl.querySelector("p:last-of-type");
|
||||
while (contentEl.lastElementChild !== lastP) {
|
||||
contentEl.removeChild(contentEl.lastElementChild);
|
||||
}
|
||||
|
||||
// Remove empty <p>s or Last Chapter/Next Chapter <p>s
|
||||
while (isEmptyOrGarbage(contentEl.lastElementChild)) {
|
||||
contentEl.removeChild(contentEl.lastElementChild);
|
||||
}
|
||||
|
||||
// Remove redundant attributes and style
|
||||
Array.prototype.forEach.call(contentEl.children, child => {
|
||||
if (child.getAttribute("dir") === "ltr") {
|
||||
child.removeAttribute("dir");
|
||||
}
|
||||
|
||||
// Only ever appears with align="LEFT" (useless) or align="CENTER" overridden by style="text-align: left;" (also
|
||||
// useless)
|
||||
child.removeAttribute("align");
|
||||
|
||||
if (child.getAttribute("style") === "text-align:left;") {
|
||||
child.removeAttribute("style");
|
||||
}
|
||||
if (child.getAttribute("style") === "text-align:left;padding-left:30px;") {
|
||||
child.setAttribute("style", "padding-left:30px;");
|
||||
}
|
||||
});
|
||||
|
||||
// Remove empty <em>s and <i>s
|
||||
// Remove style attributes from them, as they're always messed up.
|
||||
const ems = contentEl.querySelectorAll("em, i");
|
||||
Array.prototype.forEach.call(ems, em => {
|
||||
if (em.textContent.trim() === "") {
|
||||
const replacement = contentEl.ownerDocument.createTextNode(" ");
|
||||
em.parentNode.replaceChild(replacement, em);
|
||||
} else {
|
||||
em.removeAttribute("style");
|
||||
}
|
||||
});
|
||||
|
||||
// In https://parahumans.wordpress.com/2013/01/05/monarch-16-13/ there are some <address>s that should be <p>s O_o
|
||||
const addresses = contentEl.querySelectorAll("address");
|
||||
Array.prototype.forEach.call(addresses, address => {
|
||||
const p = contentEl.ownerDocument.createElement("p");
|
||||
p.innerHTML = address.innerHTML;
|
||||
address.parentNode.replaceChild(p, address);
|
||||
});
|
||||
|
||||
// Every <span> except underline ones is pointless at best and frequently messed up. (Weird font size, line spacing,
|
||||
// etc.)
|
||||
const spans = contentEl.querySelectorAll("span");
|
||||
Array.prototype.forEach.call(spans, span => {
|
||||
if (span.getAttribute("style") === "text-decoration:underline;") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (span.textContent.trim() === "") {
|
||||
span.parentNode.removeChild(span);
|
||||
} else {
|
||||
const docFrag = contentEl.ownerDocument.createDocumentFragment();
|
||||
while (span.firstChild) {
|
||||
docFrag.appendChild(span.firstChild);
|
||||
}
|
||||
span.parentNode.replaceChild(docFrag, span);
|
||||
}
|
||||
});
|
||||
|
||||
// In Ward, CloudFlare email protection obfuscates the email addresses:
|
||||
// https://usamaejaz.com/cloudflare-email-decoding/
|
||||
for (const emailEl of contentEl.querySelectorAll("[data-cfemail]")) {
|
||||
const decoded = decodeCloudFlareEmail(emailEl.dataset.cfemail);
|
||||
emailEl.replaceWith(contentEl.ownerDocument.createTextNode(decoded));
|
||||
}
|
||||
|
||||
// Synthesize a <body> tag to serialize
|
||||
const bodyEl = contentEl.ownerDocument.createElement("body");
|
||||
|
||||
const h1El = contentEl.ownerDocument.createElement("h1");
|
||||
h1El.textContent = chapter.title;
|
||||
bodyEl.appendChild(h1El);
|
||||
|
||||
const comment = contentEl.ownerDocument.createComment(chapter.url);
|
||||
bodyEl.appendChild(comment);
|
||||
|
||||
while (contentEl.firstChild) {
|
||||
bodyEl.appendChild(contentEl.firstChild);
|
||||
}
|
||||
|
||||
const xmlSerializer = new contentEl.ownerDocument.defaultView.XMLSerializer();
|
||||
let xml = xmlSerializer.serializeToString(bodyEl);
|
||||
|
||||
// Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p>
|
||||
xml = xml.replace(/<br \/>\s*<\/em><\/p>/g, "</em></p>");
|
||||
|
||||
// There are way too many nonbreaking spaces where they don't belong.
|
||||
// If they show up three in a row, then let them live. Otherwise, they die.
|
||||
// Also remove any run of them after a period.
|
||||
xml = xml.replace(/([^\xA0])\xA0\xA0?([^\xA0])/g, "$1 $2");
|
||||
xml = xml.replace(/\.\xA0+\s*/, ". ");
|
||||
|
||||
function fixEms() {
|
||||
// Fix recurring broken-up or erroneous <em>s
|
||||
xml = xml.replace(/<\/em>‘s/g, "’s</em>");
|
||||
xml = xml.replace(/<em><\/em>/g, "");
|
||||
xml = xml.replace(/<\/em><em>/g, "");
|
||||
xml = xml.replace(/<em>(\s?\s?[^A-Za-z]\s?\s?)<\/em>/g, "$1");
|
||||
xml = xml.replace(/<\/em>(\s?\s?[^A-Za-z]\s?\s?)<em>/g, "$1");
|
||||
xml = xml.replace(/“<em>([^>]+)<\/em>(!|\?|\.)”/g, "“<em>$1$2</em>”");
|
||||
xml = xml.replace(/<p><em>([^>]+)<\/em>(!|\?|\.)<\/p>/g, "<p><em>$1$2</em></p>");
|
||||
xml = xml.replace(/(!|\?|\.)\s{2}<\/em><\/p>/g, "$1</em></p>");
|
||||
xml = xml.replace(/<em>([a-z]+)(\?|\.)<\/em>/g, "<em>$1</em>$2");
|
||||
xml = xml.replace(/<em>([^>]+?)( +)<\/em>/g, "<em>$1</em>$2");
|
||||
xml = xml.replace(/<em> ([a-zA-Z]+)<\/em>/g, " <em>$1</em>");
|
||||
xml = xml.replace(/<em>‘\s*([^<]+)\s*’<\/em>/g, "‘<em>$1</em>’");
|
||||
xml = xml.replace(/<em>‘\s*([^<]+)\s*<\/em>\s*’/g, "‘<em>$1</em>’");
|
||||
xml = xml.replace(/‘\s*<em>\s*([^<]+)\s*’<\/em>/g, "‘<em>$1</em>’");
|
||||
xml = xml.replace(/<em>“\s*([^<]+)\s*”<\/em>/g, "“<em>$1</em>”");
|
||||
xml = xml.replace(/<em>“\s*([^<]+)\s*<\/em>\s*”/g, "“<em>$1</em>”");
|
||||
xml = xml.replace(/“\s*<em>\s*([^<]+)\s*”<\/em>/g, "“<em>$1</em>”");
|
||||
xml = xml.replace(/([^\n>])<em> ?/g, "$1 <em>");
|
||||
xml = xml.replace(/ ?<\/em>/g, "</em> ");
|
||||
xml = xml.replace(/<p([^>]+)> <em>/g, "<p$1><em>");
|
||||
xml = xml.replace(/<\/em> <\/p>/g, "</em></p>");
|
||||
xml = xml.replace(/<em>([a-z]+),<\/em>/g, "<em>$1</em>,");
|
||||
}
|
||||
|
||||
function fixQuotesAndApostrophes() {
|
||||
// Fix recurring poor quotes and apostrophes
|
||||
xml = xml.replace(/<p>”/g, "<p>“");
|
||||
xml = xml.replace(/“\s*<\/p>/g, "”</p>");
|
||||
xml = xml.replace(/“\s*<\/em><\/p>/g, "</em>”</p>");
|
||||
xml = xml.replace(/‘\s*<\/p>/g, "’</p>");
|
||||
xml = xml.replace(/‘\s*<\/em><\/p>/g, "’</em></p>");
|
||||
xml = xml.replace(/,” <\/em>/g, "</em>,” ");
|
||||
xml = xml.replace(/′/g, "’");
|
||||
xml = xml.replace(/″/g, "”");
|
||||
xml = xml.replace(/([A-Za-z])‘s(\s?)/g, "$1’s$2");
|
||||
xml = xml.replace(/I‘m/g, "I’m");
|
||||
xml = xml.replace(/<p>“\s+/g, "<p>“");
|
||||
xml = xml.replace(/'/g, "’");
|
||||
xml = xml.replace(/’([A-Za-z]+)’/g, "‘$1’");
|
||||
xml = xml.replace(/‘Sup/g, "’Sup");
|
||||
}
|
||||
|
||||
// These interact with each other, so do them a few times.
|
||||
xml = xml.replace(/,” <\/em>/g, "</em>,” ");
|
||||
fixEms();
|
||||
fixQuotesAndApostrophes();
|
||||
fixEms();
|
||||
fixQuotesAndApostrophes();
|
||||
fixEms();
|
||||
|
||||
// Similar problems occur in Ward with <b> and <strong> as do in Worm with <em>s
|
||||
xml = xml.replace(/<b \/>/g, "");
|
||||
xml = xml.replace(/<b>(\s*<br \/>\s*)<\/b>/g, "$1");
|
||||
xml = xml.replace(/<strong>(\s*<br \/>\s*)<\/strong>/g, "$1");
|
||||
xml = xml.replace(/<\/strong>(\s*)<strong>/g, "$1");
|
||||
xml = xml.replace(/<strong>@<\/strong>/g, "@");
|
||||
xml = xml.replace(/<br \/>(\s*)<\/strong>/g, "</strong><br />$1");
|
||||
xml = xml.replace(/(\s*)<\/strong>/g, "</strong>$1");
|
||||
|
||||
// No need for line breaks before paragraph ends
|
||||
// These often occur with the <br>s inside <b>/<strong> fixed above.
|
||||
xml = xml.replace(/<br \/>\s*<\/p>/g, "</p>");
|
||||
|
||||
// Fix possessive of names ending in "s"
|
||||
// Note: if the "s" is unvoiced, as in Marquis, then it doesn't get the second "s".
|
||||
xml = xml.replace(/([^‘])Judas’([^s])/g, "$1Judas’s$2");
|
||||
xml = xml.replace(/([^‘])Brutus’([^s])/g, "$1Brutus’s$2");
|
||||
xml = xml.replace(/([^‘])Jess’([^s])/g, "$1Jess’s$2");
|
||||
xml = xml.replace(/([^‘])Aegis’([^s])/g, "$1Aegis’s$2");
|
||||
xml = xml.replace(/([^‘])Dauntless’([^s])/g, "$1Dauntless’s$2");
|
||||
xml = xml.replace(/([^‘])Circus’([^s])/g, "$1Circus’s$2");
|
||||
xml = xml.replace(/([^‘])Sirius’([^s])/g, "$1Sirius’s$2");
|
||||
xml = xml.replace(/([^‘])Brooks’([^s])/g, "$1Brooks’s$2");
|
||||
xml = xml.replace(/([^‘])Genesis’([^s])/g, "$1Genesis’s$2");
|
||||
xml = xml.replace(/([^‘])Atlas’([^s])/g, "$1Atlas’s$2");
|
||||
xml = xml.replace(/([^‘])Lucas’([^s])/g, "$1Lucas’s$2");
|
||||
xml = xml.replace(/([^‘])Gwerrus’([^s])/g, "$1Gwerrus’s$2");
|
||||
xml = xml.replace(/([^‘])Chris’([^s])/g, "$1Chris’s$2");
|
||||
xml = xml.replace(/([^‘])Eligos’([^s])/g, "$1Eligos’s$2");
|
||||
xml = xml.replace(/([^‘])Animos’([^s])/g, "$1Animos’s$2");
|
||||
xml = xml.replace(/([^‘])Mags’([^s])/g, "$1Mags’s$2");
|
||||
xml = xml.replace(/([^‘])Huntress’([^s])/g, "$1Huntress’s$2");
|
||||
xml = xml.replace(/([^‘])Hephaestus’([^s])/g, "$1Hephaestus’s$2");
|
||||
xml = xml.replace(/([^‘])Lord of Loss’([^s])/g, "$1Lord of Loss’s$2");
|
||||
xml = xml.replace(/([^‘])John Combs’([^s])/g, "$1John Combs’s$2");
|
||||
xml = xml.replace(/([^‘])Mama Mathers’([^s])/g, "$1Mama Mathers’s$2");
|
||||
xml = xml.replace(/([^‘])Monokeros’([^s])/g, "$1Monokeros’s$2");
|
||||
xml = xml.replace(/([^‘])Goddess’([^s])/g, "$1Goddess’s$2");
|
||||
xml = xml.replace(/([^‘])Boundless’([^s])/g, "$1Boundless’s$2");
|
||||
xml = xml.replace(/([^‘])Paris’([^s])/g, "$1Paris’s$2");
|
||||
xml = xml.replace(/([^‘])Tress’([^s])/g, "$1Tress’s$2");
|
||||
xml = xml.replace(/([^‘])Harris’([^s])/g, "$1Harris’s$2");
|
||||
xml = xml.replace(/([^‘])Antares’([^s])/g, "$1Antares’s$2");
|
||||
xml = xml.replace(/([^‘])Nieves’([^s])/g, "$1Nieves’s$2");
|
||||
xml = xml.replace(/([^‘])Backwoods’([^s])/g, "$1Backwoods’s$2");
|
||||
xml = xml.replace(/([^‘])Midas’([^s])/g, "$1Midas’s$2");
|
||||
xml = xml.replace(/([^‘])Mrs. Sims’([^s])/g, "$1Mrs. Sims’s$2");
|
||||
xml = xml.replace(/([^‘])Ms. Stillons’([^s])/g, "$1Ms. Stillons’s$2");
|
||||
xml = xml.replace(/([^‘])Chuckles’([^s])/g, "$1Chuckles’s$2");
|
||||
|
||||
// Fixes dashes
|
||||
xml = xml.replace(/ – /g, "—");
|
||||
xml = xml.replace(/“((?:<em>)?)-/g, "“$1—");
|
||||
xml = xml.replace(/-[,.]?”/g, "—”");
|
||||
xml = xml.replace(/-(!|\?)”/g, "—$1”");
|
||||
xml = xml.replace(/-[,.]?<\/em>”/g, "—</em>”");
|
||||
xml = xml.replace(/-“/g, "—”");
|
||||
xml = xml.replace(/<p>-/g, "<p>—");
|
||||
xml = xml.replace(/-<\/p>/g, "—</p>");
|
||||
xml = xml.replace(/-<\/em><\/p>/g, "—</em></p>");
|
||||
xml = xml.replace(/\s?\s?–\s?\s?/g, "—");
|
||||
xml = xml.replace(/-\s\s?/g, "—");
|
||||
xml = xml.replace(/\s?\s-/g, "—");
|
||||
xml = xml.replace(/\s+—”/g, "—”");
|
||||
xml = xml.replace(/I-I/g, "I—I");
|
||||
xml = xml.replace(/I-uh/g, "I—uh");
|
||||
|
||||
// Joint names should use em dashes
|
||||
xml = xml.replace(/Dallon-Pelham/g, "Dallon–Pelham");
|
||||
xml = xml.replace(/Bet-Gimel/g, "Bet–Gimel");
|
||||
xml = xml.replace(/Tristan-Capricorn/g, "Tristan–Capricorn");
|
||||
xml = xml.replace(/Capricorn-Byron/g, "Capricorn–Byron");
|
||||
xml = xml.replace(/Tristan-Byron/g, "Tristan–Byron");
|
||||
xml = xml.replace(/Earth-Gimel/g, "Earth–Gimel");
|
||||
xml = xml.replace(/Gimel-Europe/g, "Gimel–Europe");
|
||||
xml = xml.replace(/Imp-Damsel/g, "Imp–Damsel");
|
||||
xml = xml.replace(/Damsel-Ashley/g, "Damsel–Ashley");
|
||||
xml = xml.replace(/Antares-Anelace/g, "Antares–Anelace");
|
||||
xml = xml.replace(/Challenger-Gallant/g, "Challenger–Gallant");
|
||||
xml = xml.replace(/Undersider(s?)-(Breakthrough|Ambassador)/g, "Undersider$1–$2");
|
||||
xml = xml.replace(/Norwalk-Fairfield/g, "Norwalk–Fairfield");
|
||||
xml = xml.replace(/East-West/g, "east–west");
|
||||
xml = xml.replace(/(Green|Yellow)-Black/g, "$1–Black");
|
||||
xml = xml.replace(/Creutzfeldt-Jakob/g, "Creutzfeldt–Jakob");
|
||||
xml = xml.replace(/Astaroth-Nidhug/g, "Astaroth–Nidhug");
|
||||
xml = xml.replace(/Capulet-Montague/g, "Capulet–Montague");
|
||||
xml = xml.replace(/Weaver-Clockblocker/g, "Weaver–Clockblocker");
|
||||
xml = xml.replace(/Alexandria-Pretender/g, "Alexandria–Pretender");
|
||||
xml = xml.replace(/Night Hag-Nyx/g, "Night Hag–Nyx");
|
||||
xml = xml.replace(/Crawler-Breed/g, "Crawler–Breed");
|
||||
xml = xml.replace(/Simurgh-Myrddin-plant/g, "Simurgh–Myrddin–plant");
|
||||
xml = xml.replace(/Armsmaster-Defiant/g, "Armsmaster–Defiant");
|
||||
|
||||
// Use <hr> for separators
|
||||
// https://www.parahumans.net/2019/12/21/interlude-18-z-radiation/ has "super-separators" ("⊙ ⊙ ⊙ ⊙ ⊙") which we
|
||||
// leave untouched for now.
|
||||
xml = xml.replace(/<p>■<\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align:center;">■<\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;">⊙<\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;"><strong>⊙<\/strong><\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;"><em><strong>⊙<\/strong><\/em><\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;"><strong>⊙⊙<\/strong><\/p>/g, "<hr />");
|
||||
|
||||
// Fix recurring miscapitalization with questions
|
||||
xml = xml.replace(/\?”\s\s?She asked/g, "?” she asked");
|
||||
xml = xml.replace(/\?”\s\s?He asked/g, "?” he asked");
|
||||
|
||||
// Fix bad periods and spacing/markup surrounding them
|
||||
xml = xml.replace(/\.\.<\/p>/g, ".</p>");
|
||||
xml = xml.replace(/\.\.”<\/p>/g, ".”</p>");
|
||||
xml = xml.replace(/ \. /g, ". ");
|
||||
xml = xml.replace(/ \.<\/p>/g, ".</p>");
|
||||
xml = xml.replace(/\.<em>\.\./g, "<em>…");
|
||||
|
||||
// Fix extra spaces
|
||||
xml = xml.replace(/ ? <\/p>/g, "</p>");
|
||||
xml = xml.replace(/([a-z]) ,/g, "$1,");
|
||||
|
||||
// The author often fails to terminate a sentence, instead using a comma after a dialogue tag. For example,
|
||||
// > “I didn’t get much done,” Greg said, “I got distracted by...
|
||||
// This should instead be
|
||||
// > “I didn’t get much done,” Greg said. “I got distracted by...
|
||||
//
|
||||
// Our heuristic is to try to automatically fix this if the dialogue tag is two words (X said/admitted/sighed/etc.).
|
||||
//
|
||||
// This sometimes overcorrects, as in the following example:
|
||||
// > “Basically,” Alec said, “For your powers to manifest, ...
|
||||
// Here instead we should lowercase the "f". We handle that via one-offs in substitutions.json.
|
||||
//
|
||||
// This applies to ~800 instances, so although we have to correct back in substitutions.json a decent number of
|
||||
// times, it definitely pays for itself. Most of the instances we have to correct back we also need to fix the
|
||||
// capitalization anyway, and that's harder to do automatically, since proper names/"I"/etc. stay capitalized.
|
||||
xml = xml.replace(/,” ([A-Za-z]+ [A-Za-z]+), “([A-Z])/g, ",” $1. “$2");
|
||||
|
||||
// Replace single-word <i>s with <em>s. Other <i>s are probably erroneous too, but these are known-bad.
|
||||
xml = xml.replace(/<i>([A-Za-z]+)<\/i>/g, "<em>$1</em>");
|
||||
|
||||
// This occurs enough times it's better to do here than in one-off fixes. We correct the single instance where
|
||||
// it's incorrect to capitalize in the one-off fixes.
|
||||
// Note that Ward contains much talk of "the clairvoyants", so we don't want to capitalize plurals.
|
||||
xml = xml.replace(/the clairvoyant([^s])/g, "the Clairvoyant$1");
|
||||
|
||||
// ReSound's name is sometimes miscapitalized. The word is never used in a non-name context.
|
||||
xml = xml.replace(/Resound/g, "ReSound");
|
||||
|
||||
// "patrol block" is capitalized three different ways: "patrol block", "Patrol block", and "Patrol Block". I can see
|
||||
// arguments for any of them, so let's go with the most prevalent: "patrol block".
|
||||
xml = xml.replace(/([^ ]) Patrol (?:B|b)lock/g, "$1 patrol block");
|
||||
|
||||
// This is sometimes missing its capitalization.
|
||||
xml = xml.replace(/the birdcage/g, "the Birdcage");
|
||||
|
||||
// This is usually spelled "TV" but sometimes the other ways. Normalize.
|
||||
xml = xml.replace(/(\b)tv(\b)/g, "$1TV$2");
|
||||
xml = xml.replace(/T\.V\./g, "TV");
|
||||
|
||||
// There's no reason why these should be capitalized. (Note that they never appear at the beginning of any sentences.)
|
||||
xml = xml.replace(/Halberd/g, "halberd");
|
||||
xml = xml.replace(/Loft/g, "loft");
|
||||
|
||||
// Especially early in the story, PRT designations are capitalized; they should not be. This fixes the cases where we
|
||||
// can be reasonably sure they don't start a sentence, although more specific instances are done in
|
||||
// substitutions.json, and some need to be back-corrected.
|
||||
//
|
||||
// Note: "Master" is specifically omitted because it fails poorly on Interlude 4. Other instances need to be
|
||||
// corrected via substitutions.json.
|
||||
xml = xml.replace(
|
||||
/([a-zA-Z,] |\/)(Mover|Shaker|Brute|Breaker|Tinker|Blaster|Thinker|Striker|Changer|Trump|Stranger|Shifter|Shaper)/g,
|
||||
(_, prefix, designation) => prefix + designation.toLowerCase()
|
||||
);
|
||||
xml = xml.replace(
|
||||
/(mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)-(\d+)/gi,
|
||||
"$1 $2"
|
||||
);
|
||||
xml = xml.replace(
|
||||
// eslint-disable-next-line max-len
|
||||
/(mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)[ -/](mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)/gi,
|
||||
"$1–$2"
|
||||
);
|
||||
|
||||
// This is consistently missing accents
|
||||
xml = xml.replace(/Yangban/g, "Yàngbǎn");
|
||||
|
||||
// Place names need to always be capitalized
|
||||
xml = xml.replace(/North end/g, "North End");
|
||||
xml = xml.replace(/Stonemast avenue/g, "Stonemast Avenue");
|
||||
xml = xml.replace(/Shale avenue/g, "Shale Avenue");
|
||||
xml = xml.replace(/Lord street/g, "Lord Street");
|
||||
xml = xml.replace(/Slater street/g, "Slater Street");
|
||||
xml = xml.replace(/Hollow point/g, "Hollow Point");
|
||||
xml = xml.replace(/Cedar point/g, "Cedar Point");
|
||||
|
||||
// These are usually not italicized, but sometimes are. Other foreign-language names (like Yàngbǎn) are not
|
||||
// italicized, so we go in the direction of removing the italics.
|
||||
xml = xml.replace(/<em>Garama<\/em>/g, "Garama");
|
||||
xml = xml.replace(/<em>Thanda<\/em>/g, "Thanda");
|
||||
xml = xml.replace(/<em>Sifara([^<]*)<\/em>/g, "Sifara$1");
|
||||
xml = xml.replace(/<em>Moord Nag([^<]*)<\/em>/g, "Moord Nag$1");
|
||||
xml = xml.replace(/<em>Califa de Perro([^<]*)<\/em>/g, "Califa de Perro$1");
|
||||
xml = xml.replace(/<em>Turanta([^<]*)<\/em>/g, "Turanta$1");
|
||||
|
||||
// "okay" is preferred to "ok". This sometimes gets changed back via substitutions.json when people are writing notes
|
||||
// and thus probably the intention was to be less formal. Also it seems per https://en.wikipedia.org/wiki/A-ok the
|
||||
// "A" in "A-okay" should be capitalized.
|
||||
xml = xml.replace(/Ok([,. ])/g, "Okay$1");
|
||||
xml = xml.replace(/([^a-zA-Z])ok([^a])/g, "$1okay$2");
|
||||
xml = xml.replace(/a-okay/g, "A-okay");
|
||||
|
||||
// Signal(l)ing/signal(l)ed are spelled both ways. Both are acceptable in English. Let's standardize on single-L.
|
||||
xml = xml.replace(/(S|s)ignall/g, "$1ignal");
|
||||
|
||||
// Clich(e|é) is spelled both ways. Let's standardize on including the accent.
|
||||
xml = xml.replace(/cliche/g, "cliché");
|
||||
|
||||
// "gray" is the majority spelling, except for "greyhound"
|
||||
xml = xml.replace(/(G|g)rey(?!hound)/g, "$1ray");
|
||||
|
||||
// "Mom" and "Dad" should be capitalized when used as a proper name. These regexps are tuned to catch a good amount of
|
||||
// instances, without over-correcting for non-proper-name-like cases. Many other instances are handled in
|
||||
// substitutions.json.
|
||||
xml = xml.replace(/(?<!mom), dad(?![a-z])/g, ", Dad");
|
||||
xml = xml.replace(/, mom(?![a-z-])/g, ", Mom");
|
||||
|
||||
// These are consistently missing hyphens.
|
||||
xml = xml.replace(/self destruct/g, "self-destruct");
|
||||
xml = xml.replace(/life threatening/g, "life-threatening");
|
||||
xml = xml.replace(/hard headed/g, "hard-headed");
|
||||
xml = xml.replace(/shoulder mounted/g, "shoulder-mounted");
|
||||
xml = xml.replace(/golden skinned/g, "golden-skinned");
|
||||
xml = xml.replace(/creepy crawl/g, "creepy-crawl");
|
||||
xml = xml.replace(/well armed/g, "well-armed");
|
||||
xml = xml.replace(/able bodied/g, "able-bodied");
|
||||
|
||||
// One-off fixes
|
||||
(substitutions[chapter.url] || []).forEach(substitution => {
|
||||
if (substitution.before) {
|
||||
const indexOf = xml.indexOf(substitution.before);
|
||||
if (indexOf === -1) {
|
||||
console.warn(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
|
||||
`updated at the source, in which case, you should edit substitutions.json.`);
|
||||
}
|
||||
if (indexOf !== xml.lastIndexOf(substitution.before)) {
|
||||
console.warn(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
|
||||
`Update substitutions.json for a more precise substitution.`);
|
||||
}
|
||||
|
||||
xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after);
|
||||
} else if (substitution.regExp) {
|
||||
xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement);
|
||||
} else {
|
||||
console.warn(`Invalid substitution specified for ${chapter.url}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a <html>
|
||||
xml = xml.replace(/<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/, "<body>");
|
||||
|
||||
return xml;
|
||||
}
|
||||
|
||||
function isEmptyOrGarbage(el) {
|
||||
const text = el.textContent.trim();
|
||||
return text === "" ||
|
||||
text.startsWith("Last Chapter") ||
|
||||
text.startsWith("Previous Chapter") ||
|
||||
text.startsWith("Next Chapter");
|
||||
}
|
||||
|
||||
function escapeRegExp(str) {
|
||||
return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
|
||||
}
|
||||
|
||||
function decodeCloudFlareEmail(hash) {
|
||||
let email = "";
|
||||
const xorWithThis = parseInt(hash.substring(0, 2), 16);
|
||||
for (let i = 2; i < hash.length; i += 2) {
|
||||
const charCode = parseInt(hash.substring(i, i + 2), 16) ^ xorWithThis;
|
||||
email += String.fromCharCode(charCode);
|
||||
}
|
||||
|
||||
return email;
|
||||
}
|
||||
497
lib/convert.js
497
lib/convert.js
|
|
@ -1,10 +1,8 @@
|
|||
"use strict";
|
||||
const path = require("path");
|
||||
const fs = require("fs").promises;
|
||||
const throat = require("throat");
|
||||
const { JSDOM } = require("jsdom");
|
||||
const workerpool = require("workerpool");
|
||||
const cliProgress = require("cli-progress");
|
||||
const substitutions = require("./substitutions.json");
|
||||
|
||||
module.exports = async (cachePath, manifestPath, contentPath, concurrentJobs) => {
|
||||
const manifestContents = await fs.readFile(manifestPath, { encoding: "utf-8" });
|
||||
|
|
@ -17,483 +15,24 @@ module.exports = async (cachePath, manifestPath, contentPath, concurrentJobs) =>
|
|||
}, cliProgress.Presets.shades_classic);
|
||||
progress.start(chapters.length, 0);
|
||||
|
||||
const mapper = throat(concurrentJobs, async chapter => {
|
||||
await convertChapter(chapter, cachePath, contentPath);
|
||||
const poolOptions = {};
|
||||
if (concurrentJobs !== undefined) {
|
||||
poolOptions.maxWorkers = concurrentJobs;
|
||||
}
|
||||
const pool = workerpool.pool(path.resolve(__dirname, "convert-worker.js"), poolOptions);
|
||||
|
||||
await Promise.all(chapters.map(async chapter => {
|
||||
const inputPath = path.resolve(cachePath, chapter.filename);
|
||||
|
||||
const destFileName = `${path.basename(chapter.filename, ".html")}.xhtml`;
|
||||
const outputPath = path.resolve(contentPath, destFileName);
|
||||
|
||||
await pool.exec("convertChapter", [chapter, inputPath, outputPath]);
|
||||
|
||||
progress.increment();
|
||||
});
|
||||
await Promise.all(chapters.map(mapper));
|
||||
}));
|
||||
|
||||
pool.terminate();
|
||||
|
||||
console.log("All chapters converted");
|
||||
};
|
||||
|
||||
async function convertChapter(chapter, cachePath, contentPath) {
|
||||
const filename = chapter.filename;
|
||||
const filePath = path.resolve(cachePath, filename);
|
||||
|
||||
const contents = await fs.readFile(filePath, { encoding: "utf-8" });
|
||||
|
||||
const rawChapterJSDOM = new JSDOM(contents);
|
||||
const output = getChapterString(chapter, rawChapterJSDOM.window.document);
|
||||
|
||||
// TODO: this should probably not be necessary... jsdom bug I guess!?
|
||||
rawChapterJSDOM.window.close();
|
||||
|
||||
const destFileName = `${path.basename(filename, ".html")}.xhtml`;
|
||||
const destFilePath = path.resolve(contentPath, destFileName);
|
||||
|
||||
await fs.writeFile(destFilePath, output);
|
||||
}
|
||||
|
||||
function getChapterString(chapter, rawChapterDoc) {
|
||||
const body = getBodyXML(chapter, rawChapterDoc.querySelector(".entry-content"));
|
||||
|
||||
return `<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
|
||||
<title>${chapter.title}</title>
|
||||
</head>
|
||||
${body}
|
||||
</html>`;
|
||||
}
|
||||
|
||||
function getBodyXML(chapter, contentEl) {
|
||||
// Remove initial Next Chapter and Previous Chapter <p>
|
||||
contentEl.removeChild(contentEl.firstElementChild);
|
||||
|
||||
// Remove everything after the last <p> (e.g. analytics <div>s)
|
||||
const lastP = contentEl.querySelector("p:last-of-type");
|
||||
while (contentEl.lastElementChild !== lastP) {
|
||||
contentEl.removeChild(contentEl.lastElementChild);
|
||||
}
|
||||
|
||||
// Remove empty <p>s or Last Chapter/Next Chapter <p>s
|
||||
while (isEmptyOrGarbage(contentEl.lastElementChild)) {
|
||||
contentEl.removeChild(contentEl.lastElementChild);
|
||||
}
|
||||
|
||||
// Remove redundant attributes and style
|
||||
Array.prototype.forEach.call(contentEl.children, child => {
|
||||
if (child.getAttribute("dir") === "ltr") {
|
||||
child.removeAttribute("dir");
|
||||
}
|
||||
|
||||
// Only ever appears with align="LEFT" (useless) or align="CENTER" overridden by style="text-align: left;" (also
|
||||
// useless)
|
||||
child.removeAttribute("align");
|
||||
|
||||
if (child.getAttribute("style") === "text-align:left;") {
|
||||
child.removeAttribute("style");
|
||||
}
|
||||
if (child.getAttribute("style") === "text-align:left;padding-left:30px;") {
|
||||
child.setAttribute("style", "padding-left:30px;");
|
||||
}
|
||||
});
|
||||
|
||||
// Remove empty <em>s and <i>s
|
||||
// Remove style attributes from them, as they're always messed up.
|
||||
const ems = contentEl.querySelectorAll("em, i");
|
||||
Array.prototype.forEach.call(ems, em => {
|
||||
if (em.textContent.trim() === "") {
|
||||
const replacement = contentEl.ownerDocument.createTextNode(" ");
|
||||
em.parentNode.replaceChild(replacement, em);
|
||||
} else {
|
||||
em.removeAttribute("style");
|
||||
}
|
||||
});
|
||||
|
||||
// In https://parahumans.wordpress.com/2013/01/05/monarch-16-13/ there are some <address>s that should be <p>s O_o
|
||||
const addresses = contentEl.querySelectorAll("address");
|
||||
Array.prototype.forEach.call(addresses, address => {
|
||||
const p = contentEl.ownerDocument.createElement("p");
|
||||
p.innerHTML = address.innerHTML;
|
||||
address.parentNode.replaceChild(p, address);
|
||||
});
|
||||
|
||||
// Every <span> except underline ones is pointless at best and frequently messed up. (Weird font size, line spacing,
|
||||
// etc.)
|
||||
const spans = contentEl.querySelectorAll("span");
|
||||
Array.prototype.forEach.call(spans, span => {
|
||||
if (span.getAttribute("style") === "text-decoration:underline;") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (span.textContent.trim() === "") {
|
||||
span.parentNode.removeChild(span);
|
||||
} else {
|
||||
const docFrag = contentEl.ownerDocument.createDocumentFragment();
|
||||
while (span.firstChild) {
|
||||
docFrag.appendChild(span.firstChild);
|
||||
}
|
||||
span.parentNode.replaceChild(docFrag, span);
|
||||
}
|
||||
});
|
||||
|
||||
// In Ward, CloudFlare email protection obfuscates the email addresses:
|
||||
// https://usamaejaz.com/cloudflare-email-decoding/
|
||||
for (const emailEl of contentEl.querySelectorAll("[data-cfemail]")) {
|
||||
const decoded = decodeCloudFlareEmail(emailEl.dataset.cfemail);
|
||||
emailEl.replaceWith(contentEl.ownerDocument.createTextNode(decoded));
|
||||
}
|
||||
|
||||
// Synthesize a <body> tag to serialize
|
||||
const bodyEl = contentEl.ownerDocument.createElement("body");
|
||||
|
||||
const h1El = contentEl.ownerDocument.createElement("h1");
|
||||
h1El.textContent = chapter.title;
|
||||
bodyEl.appendChild(h1El);
|
||||
|
||||
const comment = contentEl.ownerDocument.createComment(chapter.url);
|
||||
bodyEl.appendChild(comment);
|
||||
|
||||
while (contentEl.firstChild) {
|
||||
bodyEl.appendChild(contentEl.firstChild);
|
||||
}
|
||||
|
||||
const xmlSerializer = new contentEl.ownerDocument.defaultView.XMLSerializer();
|
||||
let xml = xmlSerializer.serializeToString(bodyEl);
|
||||
|
||||
// Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p>
|
||||
xml = xml.replace(/<br \/>\s*<\/em><\/p>/g, "</em></p>");
|
||||
|
||||
// There are way too many nonbreaking spaces where they don't belong.
|
||||
// If they show up three in a row, then let them live. Otherwise, they die.
|
||||
// Also remove any run of them after a period.
|
||||
xml = xml.replace(/([^\xA0])\xA0\xA0?([^\xA0])/g, "$1 $2");
|
||||
xml = xml.replace(/\.\xA0+\s*/, ". ");
|
||||
|
||||
function fixEms() {
|
||||
// Fix recurring broken-up or erroneous <em>s
|
||||
xml = xml.replace(/<\/em>‘s/g, "’s</em>");
|
||||
xml = xml.replace(/<em><\/em>/g, "");
|
||||
xml = xml.replace(/<\/em><em>/g, "");
|
||||
xml = xml.replace(/<em>(\s?\s?[^A-Za-z]\s?\s?)<\/em>/g, "$1");
|
||||
xml = xml.replace(/<\/em>(\s?\s?[^A-Za-z]\s?\s?)<em>/g, "$1");
|
||||
xml = xml.replace(/“<em>([^>]+)<\/em>(!|\?|\.)”/g, "“<em>$1$2</em>”");
|
||||
xml = xml.replace(/<p><em>([^>]+)<\/em>(!|\?|\.)<\/p>/g, "<p><em>$1$2</em></p>");
|
||||
xml = xml.replace(/(!|\?|\.)\s{2}<\/em><\/p>/g, "$1</em></p>");
|
||||
xml = xml.replace(/<em>([a-z]+)(\?|\.)<\/em>/g, "<em>$1</em>$2");
|
||||
xml = xml.replace(/<em>([^>]+?)( +)<\/em>/g, "<em>$1</em>$2");
|
||||
xml = xml.replace(/<em> ([a-zA-Z]+)<\/em>/g, " <em>$1</em>");
|
||||
xml = xml.replace(/<em>‘\s*([^<]+)\s*’<\/em>/g, "‘<em>$1</em>’");
|
||||
xml = xml.replace(/<em>‘\s*([^<]+)\s*<\/em>\s*’/g, "‘<em>$1</em>’");
|
||||
xml = xml.replace(/‘\s*<em>\s*([^<]+)\s*’<\/em>/g, "‘<em>$1</em>’");
|
||||
xml = xml.replace(/<em>“\s*([^<]+)\s*”<\/em>/g, "“<em>$1</em>”");
|
||||
xml = xml.replace(/<em>“\s*([^<]+)\s*<\/em>\s*”/g, "“<em>$1</em>”");
|
||||
xml = xml.replace(/“\s*<em>\s*([^<]+)\s*”<\/em>/g, "“<em>$1</em>”");
|
||||
xml = xml.replace(/([^\n>])<em> ?/g, "$1 <em>");
|
||||
xml = xml.replace(/ ?<\/em>/g, "</em> ");
|
||||
xml = xml.replace(/<p([^>]+)> <em>/g, "<p$1><em>");
|
||||
xml = xml.replace(/<\/em> <\/p>/g, "</em></p>");
|
||||
xml = xml.replace(/<em>([a-z]+),<\/em>/g, "<em>$1</em>,");
|
||||
}
|
||||
|
||||
function fixQuotesAndApostrophes() {
|
||||
// Fix recurring poor quotes and apostrophes
|
||||
xml = xml.replace(/<p>”/g, "<p>“");
|
||||
xml = xml.replace(/“\s*<\/p>/g, "”</p>");
|
||||
xml = xml.replace(/“\s*<\/em><\/p>/g, "</em>”</p>");
|
||||
xml = xml.replace(/‘\s*<\/p>/g, "’</p>");
|
||||
xml = xml.replace(/‘\s*<\/em><\/p>/g, "’</em></p>");
|
||||
xml = xml.replace(/,” <\/em>/g, "</em>,” ");
|
||||
xml = xml.replace(/′/g, "’");
|
||||
xml = xml.replace(/″/g, "”");
|
||||
xml = xml.replace(/([A-Za-z])‘s(\s?)/g, "$1’s$2");
|
||||
xml = xml.replace(/I‘m/g, "I’m");
|
||||
xml = xml.replace(/<p>“\s+/g, "<p>“");
|
||||
xml = xml.replace(/'/g, "’");
|
||||
xml = xml.replace(/’([A-Za-z]+)’/g, "‘$1’");
|
||||
xml = xml.replace(/‘Sup/g, "’Sup");
|
||||
}
|
||||
|
||||
// These interact with each other, so do them a few times.
|
||||
xml = xml.replace(/,” <\/em>/g, "</em>,” ");
|
||||
fixEms();
|
||||
fixQuotesAndApostrophes();
|
||||
fixEms();
|
||||
fixQuotesAndApostrophes();
|
||||
fixEms();
|
||||
|
||||
// Similar problems occur in Ward with <b> and <strong> as do in Worm with <em>s
|
||||
xml = xml.replace(/<b \/>/g, "");
|
||||
xml = xml.replace(/<b>(\s*<br \/>\s*)<\/b>/g, "$1");
|
||||
xml = xml.replace(/<strong>(\s*<br \/>\s*)<\/strong>/g, "$1");
|
||||
xml = xml.replace(/<\/strong>(\s*)<strong>/g, "$1");
|
||||
xml = xml.replace(/<strong>@<\/strong>/g, "@");
|
||||
xml = xml.replace(/<br \/>(\s*)<\/strong>/g, "</strong><br />$1");
|
||||
xml = xml.replace(/(\s*)<\/strong>/g, "</strong>$1");
|
||||
|
||||
// No need for line breaks before paragraph ends
|
||||
// These often occur with the <br>s inside <b>/<strong> fixed above.
|
||||
xml = xml.replace(/<br \/>\s*<\/p>/g, "</p>");
|
||||
|
||||
// Fix possessive of names ending in "s"
|
||||
// Note: if the "s" is unvoiced, as in Marquis, then it doesn't get the second "s".
|
||||
xml = xml.replace(/([^‘])Judas’([^s])/g, "$1Judas’s$2");
|
||||
xml = xml.replace(/([^‘])Brutus’([^s])/g, "$1Brutus’s$2");
|
||||
xml = xml.replace(/([^‘])Jess’([^s])/g, "$1Jess’s$2");
|
||||
xml = xml.replace(/([^‘])Aegis’([^s])/g, "$1Aegis’s$2");
|
||||
xml = xml.replace(/([^‘])Dauntless’([^s])/g, "$1Dauntless’s$2");
|
||||
xml = xml.replace(/([^‘])Circus’([^s])/g, "$1Circus’s$2");
|
||||
xml = xml.replace(/([^‘])Sirius’([^s])/g, "$1Sirius’s$2");
|
||||
xml = xml.replace(/([^‘])Brooks’([^s])/g, "$1Brooks’s$2");
|
||||
xml = xml.replace(/([^‘])Genesis’([^s])/g, "$1Genesis’s$2");
|
||||
xml = xml.replace(/([^‘])Atlas’([^s])/g, "$1Atlas’s$2");
|
||||
xml = xml.replace(/([^‘])Lucas’([^s])/g, "$1Lucas’s$2");
|
||||
xml = xml.replace(/([^‘])Gwerrus’([^s])/g, "$1Gwerrus’s$2");
|
||||
xml = xml.replace(/([^‘])Chris’([^s])/g, "$1Chris’s$2");
|
||||
xml = xml.replace(/([^‘])Eligos’([^s])/g, "$1Eligos’s$2");
|
||||
xml = xml.replace(/([^‘])Animos’([^s])/g, "$1Animos’s$2");
|
||||
xml = xml.replace(/([^‘])Mags’([^s])/g, "$1Mags’s$2");
|
||||
xml = xml.replace(/([^‘])Huntress’([^s])/g, "$1Huntress’s$2");
|
||||
xml = xml.replace(/([^‘])Hephaestus’([^s])/g, "$1Hephaestus’s$2");
|
||||
xml = xml.replace(/([^‘])Lord of Loss’([^s])/g, "$1Lord of Loss’s$2");
|
||||
xml = xml.replace(/([^‘])John Combs’([^s])/g, "$1John Combs’s$2");
|
||||
xml = xml.replace(/([^‘])Mama Mathers’([^s])/g, "$1Mama Mathers’s$2");
|
||||
xml = xml.replace(/([^‘])Monokeros’([^s])/g, "$1Monokeros’s$2");
|
||||
xml = xml.replace(/([^‘])Goddess’([^s])/g, "$1Goddess’s$2");
|
||||
xml = xml.replace(/([^‘])Boundless’([^s])/g, "$1Boundless’s$2");
|
||||
xml = xml.replace(/([^‘])Paris’([^s])/g, "$1Paris’s$2");
|
||||
xml = xml.replace(/([^‘])Tress’([^s])/g, "$1Tress’s$2");
|
||||
xml = xml.replace(/([^‘])Harris’([^s])/g, "$1Harris’s$2");
|
||||
xml = xml.replace(/([^‘])Antares’([^s])/g, "$1Antares’s$2");
|
||||
xml = xml.replace(/([^‘])Nieves’([^s])/g, "$1Nieves’s$2");
|
||||
xml = xml.replace(/([^‘])Backwoods’([^s])/g, "$1Backwoods’s$2");
|
||||
xml = xml.replace(/([^‘])Midas’([^s])/g, "$1Midas’s$2");
|
||||
xml = xml.replace(/([^‘])Mrs. Sims’([^s])/g, "$1Mrs. Sims’s$2");
|
||||
xml = xml.replace(/([^‘])Ms. Stillons’([^s])/g, "$1Ms. Stillons’s$2");
|
||||
xml = xml.replace(/([^‘])Chuckles’([^s])/g, "$1Chuckles’s$2");
|
||||
|
||||
// Fixes dashes
|
||||
xml = xml.replace(/ – /g, "—");
|
||||
xml = xml.replace(/“((?:<em>)?)-/g, "“$1—");
|
||||
xml = xml.replace(/-[,.]?”/g, "—”");
|
||||
xml = xml.replace(/-(!|\?)”/g, "—$1”");
|
||||
xml = xml.replace(/-[,.]?<\/em>”/g, "—</em>”");
|
||||
xml = xml.replace(/-“/g, "—”");
|
||||
xml = xml.replace(/<p>-/g, "<p>—");
|
||||
xml = xml.replace(/-<\/p>/g, "—</p>");
|
||||
xml = xml.replace(/-<\/em><\/p>/g, "—</em></p>");
|
||||
xml = xml.replace(/\s?\s?–\s?\s?/g, "—");
|
||||
xml = xml.replace(/-\s\s?/g, "—");
|
||||
xml = xml.replace(/\s?\s-/g, "—");
|
||||
xml = xml.replace(/\s+—”/g, "—”");
|
||||
xml = xml.replace(/I-I/g, "I—I");
|
||||
xml = xml.replace(/I-uh/g, "I—uh");
|
||||
|
||||
// Joint names should use em dashes
|
||||
xml = xml.replace(/Dallon-Pelham/g, "Dallon–Pelham");
|
||||
xml = xml.replace(/Bet-Gimel/g, "Bet–Gimel");
|
||||
xml = xml.replace(/Tristan-Capricorn/g, "Tristan–Capricorn");
|
||||
xml = xml.replace(/Capricorn-Byron/g, "Capricorn–Byron");
|
||||
xml = xml.replace(/Tristan-Byron/g, "Tristan–Byron");
|
||||
xml = xml.replace(/Earth-Gimel/g, "Earth–Gimel");
|
||||
xml = xml.replace(/Gimel-Europe/g, "Gimel–Europe");
|
||||
xml = xml.replace(/Imp-Damsel/g, "Imp–Damsel");
|
||||
xml = xml.replace(/Damsel-Ashley/g, "Damsel–Ashley");
|
||||
xml = xml.replace(/Antares-Anelace/g, "Antares–Anelace");
|
||||
xml = xml.replace(/Challenger-Gallant/g, "Challenger–Gallant");
|
||||
xml = xml.replace(/Undersider(s?)-(Breakthrough|Ambassador)/g, "Undersider$1–$2");
|
||||
xml = xml.replace(/Norwalk-Fairfield/g, "Norwalk–Fairfield");
|
||||
xml = xml.replace(/East-West/g, "east–west");
|
||||
xml = xml.replace(/(Green|Yellow)-Black/g, "$1–Black");
|
||||
xml = xml.replace(/Creutzfeldt-Jakob/g, "Creutzfeldt–Jakob");
|
||||
xml = xml.replace(/Astaroth-Nidhug/g, "Astaroth–Nidhug");
|
||||
xml = xml.replace(/Capulet-Montague/g, "Capulet–Montague");
|
||||
xml = xml.replace(/Weaver-Clockblocker/g, "Weaver–Clockblocker");
|
||||
xml = xml.replace(/Alexandria-Pretender/g, "Alexandria–Pretender");
|
||||
xml = xml.replace(/Night Hag-Nyx/g, "Night Hag–Nyx");
|
||||
xml = xml.replace(/Crawler-Breed/g, "Crawler–Breed");
|
||||
xml = xml.replace(/Simurgh-Myrddin-plant/g, "Simurgh–Myrddin–plant");
|
||||
xml = xml.replace(/Armsmaster-Defiant/g, "Armsmaster–Defiant");
|
||||
|
||||
// Use <hr> for separators
|
||||
// https://www.parahumans.net/2019/12/21/interlude-18-z-radiation/ has "super-separators" ("⊙ ⊙ ⊙ ⊙ ⊙") which we
|
||||
// leave untouched for now.
|
||||
xml = xml.replace(/<p>■<\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align:center;">■<\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;">⊙<\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;"><strong>⊙<\/strong><\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;"><em><strong>⊙<\/strong><\/em><\/p>/g, "<hr />");
|
||||
xml = xml.replace(/<p style="text-align: center;"><strong>⊙⊙<\/strong><\/p>/g, "<hr />");
|
||||
|
||||
// Fix recurring miscapitalization with questions
|
||||
xml = xml.replace(/\?”\s\s?She asked/g, "?” she asked");
|
||||
xml = xml.replace(/\?”\s\s?He asked/g, "?” he asked");
|
||||
|
||||
// Fix bad periods and spacing/markup surrounding them
|
||||
xml = xml.replace(/\.\.<\/p>/g, ".</p>");
|
||||
xml = xml.replace(/\.\.”<\/p>/g, ".”</p>");
|
||||
xml = xml.replace(/ \. /g, ". ");
|
||||
xml = xml.replace(/ \.<\/p>/g, ".</p>");
|
||||
xml = xml.replace(/\.<em>\.\./g, "<em>…");
|
||||
|
||||
// Fix extra spaces
|
||||
xml = xml.replace(/ ? <\/p>/g, "</p>");
|
||||
xml = xml.replace(/([a-z]) ,/g, "$1,");
|
||||
|
||||
// The author often fails to terminate a sentence, instead using a comma after a dialogue tag. For example,
|
||||
// > “I didn’t get much done,” Greg said, “I got distracted by...
|
||||
// This should instead be
|
||||
// > “I didn’t get much done,” Greg said. “I got distracted by...
|
||||
//
|
||||
// Our heuristic is to try to automatically fix this if the dialogue tag is two words (X said/admitted/sighed/etc.).
|
||||
//
|
||||
// This sometimes overcorrects, as in the following example:
|
||||
// > “Basically,” Alec said, “For your powers to manifest, ...
|
||||
// Here instead we should lowercase the "f". We handle that via one-offs in substitutions.json.
|
||||
//
|
||||
// This applies to ~800 instances, so although we have to correct back in substitutions.json a decent number of
|
||||
// times, it definitely pays for itself. Most of the instances we have to correct back we also need to fix the
|
||||
// capitalization anyway, and that's harder to do automatically, since proper names/"I"/etc. stay capitalized.
|
||||
xml = xml.replace(/,” ([A-Za-z]+ [A-Za-z]+), “([A-Z])/g, ",” $1. “$2");
|
||||
|
||||
// Replace single-word <i>s with <em>s. Other <i>s are probably erroneous too, but these are known-bad.
|
||||
xml = xml.replace(/<i>([A-Za-z]+)<\/i>/g, "<em>$1</em>");
|
||||
|
||||
// This occurs enough times it's better to do here than in one-off fixes. We correct the single instance where
|
||||
// it's incorrect to capitalize in the one-off fixes.
|
||||
// Note that Ward contains much talk of "the clairvoyants", so we don't want to capitalize plurals.
|
||||
xml = xml.replace(/the clairvoyant([^s])/g, "the Clairvoyant$1");
|
||||
|
||||
// ReSound's name is sometimes miscapitalized. The word is never used in a non-name context.
|
||||
xml = xml.replace(/Resound/g, "ReSound");
|
||||
|
||||
// "patrol block" is capitalized three different ways: "patrol block", "Patrol block", and "Patrol Block". I can see
|
||||
// arguments for any of them, so let's go with the most prevalent: "patrol block".
|
||||
xml = xml.replace(/([^ ]) Patrol (?:B|b)lock/g, "$1 patrol block");
|
||||
|
||||
// This is sometimes missing its capitalization.
|
||||
xml = xml.replace(/the birdcage/g, "the Birdcage");
|
||||
|
||||
// This is usually spelled "TV" but sometimes the other ways. Normalize.
|
||||
xml = xml.replace(/(\b)tv(\b)/g, "$1TV$2");
|
||||
xml = xml.replace(/T\.V\./g, "TV");
|
||||
|
||||
// There's no reason why these should be capitalized. (Note that they never appear at the beginning of any sentences.)
|
||||
xml = xml.replace(/Halberd/g, "halberd");
|
||||
xml = xml.replace(/Loft/g, "loft");
|
||||
|
||||
// Especially early in the story, PRT designations are capitalized; they should not be. This fixes the cases where we
|
||||
// can be reasonably sure they don't start a sentence, although more specific instances are done in
|
||||
// substitutions.json, and some need to be back-corrected.
|
||||
//
|
||||
// Note: "Master" is specifically omitted because it fails poorly on Interlude 4. Other instances need to be
|
||||
// corrected via substitutions.json.
|
||||
xml = xml.replace(
|
||||
/([a-zA-Z,] |\/)(Mover|Shaker|Brute|Breaker|Tinker|Blaster|Thinker|Striker|Changer|Trump|Stranger|Shifter|Shaper)/g,
|
||||
(_, prefix, designation) => prefix + designation.toLowerCase()
|
||||
);
|
||||
xml = xml.replace(
|
||||
/(mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)-(\d+)/gi,
|
||||
"$1 $2"
|
||||
);
|
||||
xml = xml.replace(
|
||||
// eslint-disable-next-line max-len
|
||||
/(mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)[ -/](mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)/gi,
|
||||
"$1–$2"
|
||||
);
|
||||
|
||||
// This is consistently missing accents
|
||||
xml = xml.replace(/Yangban/g, "Yàngbǎn");
|
||||
|
||||
// Place names need to always be capitalized
|
||||
xml = xml.replace(/North end/g, "North End");
|
||||
xml = xml.replace(/Stonemast avenue/g, "Stonemast Avenue");
|
||||
xml = xml.replace(/Shale avenue/g, "Shale Avenue");
|
||||
xml = xml.replace(/Lord street/g, "Lord Street");
|
||||
xml = xml.replace(/Slater street/g, "Slater Street");
|
||||
xml = xml.replace(/Hollow point/g, "Hollow Point");
|
||||
xml = xml.replace(/Cedar point/g, "Cedar Point");
|
||||
|
||||
// These are usually not italicized, but sometimes are. Other foreign-language names (like Yàngbǎn) are not
|
||||
// italicized, so we go in the direction of removing the italics.
|
||||
xml = xml.replace(/<em>Garama<\/em>/g, "Garama");
|
||||
xml = xml.replace(/<em>Thanda<\/em>/g, "Thanda");
|
||||
xml = xml.replace(/<em>Sifara([^<]*)<\/em>/g, "Sifara$1");
|
||||
xml = xml.replace(/<em>Moord Nag([^<]*)<\/em>/g, "Moord Nag$1");
|
||||
xml = xml.replace(/<em>Califa de Perro([^<]*)<\/em>/g, "Califa de Perro$1");
|
||||
xml = xml.replace(/<em>Turanta([^<]*)<\/em>/g, "Turanta$1");
|
||||
|
||||
// "okay" is preferred to "ok". This sometimes gets changed back via substitutions.json when people are writing notes
|
||||
// and thus probably the intention was to be less formal. Also it seems per https://en.wikipedia.org/wiki/A-ok the
|
||||
// "A" in "A-okay" should be capitalized.
|
||||
xml = xml.replace(/Ok([,. ])/g, "Okay$1");
|
||||
xml = xml.replace(/([^a-zA-Z])ok([^a])/g, "$1okay$2");
|
||||
xml = xml.replace(/a-okay/g, "A-okay");
|
||||
|
||||
// Signal(l)ing/signal(l)ed are spelled both ways. Both are acceptable in English. Let's standardize on single-L.
|
||||
xml = xml.replace(/(S|s)ignall/g, "$1ignal");
|
||||
|
||||
// Clich(e|é) is spelled both ways. Let's standardize on including the accent.
|
||||
xml = xml.replace(/cliche/g, "cliché");
|
||||
|
||||
// "gray" is the majority spelling, except for "greyhound"
|
||||
xml = xml.replace(/(G|g)rey(?!hound)/g, "$1ray");
|
||||
|
||||
// "Mom" and "Dad" should be capitalized when used as a proper name. These regexps are tuned to catch a good amount of
|
||||
// instances, without over-correcting for non-proper-name-like cases. Many other instances are handled in
|
||||
// substitutions.json.
|
||||
xml = xml.replace(/(?<!mom), dad(?![a-z])/g, ", Dad");
|
||||
xml = xml.replace(/, mom(?![a-z-])/g, ", Mom");
|
||||
|
||||
// These are consistently missing hyphens.
|
||||
xml = xml.replace(/self destruct/g, "self-destruct");
|
||||
xml = xml.replace(/life threatening/g, "life-threatening");
|
||||
xml = xml.replace(/hard headed/g, "hard-headed");
|
||||
xml = xml.replace(/shoulder mounted/g, "shoulder-mounted");
|
||||
xml = xml.replace(/golden skinned/g, "golden-skinned");
|
||||
xml = xml.replace(/creepy crawl/g, "creepy-crawl");
|
||||
xml = xml.replace(/well armed/g, "well-armed");
|
||||
xml = xml.replace(/able bodied/g, "able-bodied");
|
||||
|
||||
// One-off fixes
|
||||
(substitutions[chapter.url] || []).forEach(substitution => {
|
||||
if (substitution.before) {
|
||||
const indexOf = xml.indexOf(substitution.before);
|
||||
if (indexOf === -1) {
|
||||
console.warn(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
|
||||
`updated at the source, in which case, you should edit substitutions.json.`);
|
||||
}
|
||||
if (indexOf !== xml.lastIndexOf(substitution.before)) {
|
||||
console.warn(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
|
||||
`Update substitutions.json for a more precise substitution.`);
|
||||
}
|
||||
|
||||
xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after);
|
||||
} else if (substitution.regExp) {
|
||||
xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement);
|
||||
} else {
|
||||
console.warn(`Invalid substitution specified for ${chapter.url}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a <html>
|
||||
xml = xml.replace(/<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/, "<body>");
|
||||
|
||||
return xml;
|
||||
}
|
||||
|
||||
function isEmptyOrGarbage(el) {
|
||||
const text = el.textContent.trim();
|
||||
return text === "" ||
|
||||
text.startsWith("Last Chapter") ||
|
||||
text.startsWith("Previous Chapter") ||
|
||||
text.startsWith("Next Chapter");
|
||||
}
|
||||
|
||||
function escapeRegExp(str) {
|
||||
return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
|
||||
}
|
||||
|
||||
function decodeCloudFlareEmail(hash) {
|
||||
let email = "";
|
||||
const xorWithThis = parseInt(hash.substring(0, 2), 16);
|
||||
for (let i = 2; i < hash.length; i += 2) {
|
||||
const charCode = parseInt(hash.substring(i, i + 2), 16) ^ xorWithThis;
|
||||
email += String.fromCharCode(charCode);
|
||||
}
|
||||
|
||||
return email;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,8 +52,9 @@ const argv = yargs
|
|||
})
|
||||
.option("j", {
|
||||
alias: "jobs",
|
||||
default: 10,
|
||||
describe: "Number of concurrent read/write conversion jobs",
|
||||
default: undefined,
|
||||
defaultDescription: "# of CPU cores - 1",
|
||||
describe: "number of concurrent read/write conversion jobs",
|
||||
requiresArg: true,
|
||||
global: true
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue