"use strict"; const workerpool = require("workerpool"); const fs = require("fs"); const { JSDOM } = require("jsdom"); const substitutions = require("./substitutions.json"); workerpool.worker({ convertChapter }); function convertChapter(chapter, book, inputPath, outputPath) { const contents = fs.readFileSync(inputPath, { encoding: "utf-8" }); const rawChapterJSDOM = new JSDOM(contents); const { output, warnings } = getChapterString(chapter, book, rawChapterJSDOM.window.document); // TODO: this should probably not be necessary... jsdom bug I guess!? rawChapterJSDOM.window.close(); fs.writeFileSync(outputPath, output); return warnings; } function getChapterString(chapter, book, rawChapterDoc) { const { xml, warnings } = getBodyXML(chapter, book, rawChapterDoc.querySelector(".entry-content")); const output = ` ${chapter.title} ${xml} `; return { output, warnings }; } function getBodyXML(chapter, book, contentEl) { const warnings = []; // Remove initial Next Chapter and Previous Chapter

contentEl.firstElementChild.remove(); // Remove everything after the last

(e.g. analytics

s) const lastP = contentEl.querySelector("p:last-of-type"); while (contentEl.lastElementChild !== lastP) { contentEl.lastElementChild.remove(); } // Remove empty

s or Last Chapter/Next Chapter

s while (isEmptyOrGarbage(contentEl.lastElementChild)) { contentEl.lastElementChild.remove(); } // Remove redundant attributes and style for (const child of contentEl.children) { if (child.getAttribute("dir") === "ltr") { child.removeAttribute("dir"); } // Only ever appears with align="LEFT" (useless) or align="CENTER" overridden by style="text-align: left;" (also // useless) child.removeAttribute("align"); const style = child.getAttribute("style"); if (style === "text-align:left;" || style === "text-align: left;") { child.removeAttribute("style"); } // Worm uses 30px; Ward mostly uses 40px but sometimes uses 30px/60px. Let's standardize on 30px. if (style === "text-align:left;padding-left:30px;" || style === "text-align: left;padding-left: 40px;" || style === "padding-left: 40px;") { child.setAttribute("style", "padding-left: 30px;"); } } // Remove empty s and s // Remove style attributes from them, as they're always messed up. for (const em of contentEl.querySelectorAll("em, i")) { if (em.textContent.trim() === "") { em.replaceWith(contentEl.ownerDocument.createTextNode(" ")); } else { em.removeAttribute("style"); } } // In https://parahumans.wordpress.com/2013/01/05/monarch-16-13/ there are some

s that should be

s O_o for (const address of contentEl.querySelectorAll("address")) { const p = contentEl.ownerDocument.createElement("p"); p.innerHTML = address.innerHTML; address.replaceWith(p); } // Every except underline ones is pointless at best and frequently messed up. (Weird font size, line spacing, // etc.) for (const span of contentEl.querySelectorAll("span")) { if (span.getAttribute("style") === "text-decoration:underline;") { continue; } if (span.textContent.trim() === "") { span.remove(); } else { const docFrag = contentEl.ownerDocument.createDocumentFragment(); while (span.firstChild) { docFrag.appendChild(span.firstChild); } span.replaceWith(docFrag); } } // In Ward, CloudFlare email protection obfuscates the email addresses: // https://usamaejaz.com/cloudflare-email-decoding/ for (const emailEl of contentEl.querySelectorAll("[data-cfemail]")) { const decoded = decodeCloudFlareEmail(emailEl.dataset.cfemail); emailEl.replaceWith(contentEl.ownerDocument.createTextNode(decoded)); } // Synthesize a tag to serialize const bodyEl = contentEl.ownerDocument.createElement("body"); const h1El = contentEl.ownerDocument.createElement("h1"); h1El.textContent = chapter.title; bodyEl.appendChild(h1El); while (contentEl.firstChild) { bodyEl.appendChild(contentEl.firstChild); } const xmlSerializer = new contentEl.ownerDocument.defaultView.XMLSerializer(); let xml = xmlSerializer.serializeToString(bodyEl); // Fix recurring strange pattern of extra
in

......
\n

xml = xml.replace(/
\s*<\/em><\/p>/g, "

"); // Replace single-word s with s. Other s are probably erroneous too, but these are known-bad. xml = xml.replace(/([^ ]+)<\/i>/g, "$1"); xml = xml.replace(/([^ ]+)( +)<\/i>/g, "$1$2"); // There are way too many nonbreaking spaces where they don't belong. // If they show up three in a row, then let them live. Otherwise, they die. // Also remove any run of them after a period. xml = xml.replace(/([^\xA0])\xA0\xA0?([^\xA0])/g, "$1 $2"); xml = xml.replace(/\.\x20*\xA0[\xA0\x20]*/, ". "); function fixEms() { // Fix recurring broken-up or erroneous s xml = xml.replace(/<\/em>‘s/g, "’s"); xml = xml.replace(/<\/em>/g, ""); xml = xml.replace(/<\/em>/g, ""); xml = xml.replace(/(\s?\s?[^A-Za-z]\s?\s?)<\/em>/g, "$1"); xml = xml.replace(/<\/em>(\s?\s?[^A-Za-z]\s?\s?)/g, "$1"); xml = xml.replace(/“([^>]+)<\/em>(!|\?|\.)”/g, "“$1$2”"); xml = xml.replace(/

([^>]+)<\/em>(!|\?|\.)<\/p>/g, "

$1$2

"); xml = xml.replace(/(!|\?|\.)\s{2}<\/em><\/p>/g, "$1

"); xml = xml.replace(/([a-z]+)(\?|\.)<\/em>/g, "$1$2"); xml = xml.replace(/([^>]+?)( +)<\/em>/g, "$1$2"); xml = xml.replace(/ ([a-zA-Z]+)<\/em>/g, " $1"); xml = xml.replace(/‘\s*([^<]+)\s*’<\/em>/g, "‘$1’"); xml = xml.replace(/‘\s*([^<]+)\s*<\/em>\s*’/g, "‘$1’"); xml = xml.replace(/‘\s*\s*([^<]+)\s*’<\/em>/g, "‘$1’"); xml = xml.replace(/“\s*([^<”]+)\s*”<\/em>/g, "“$1”"); xml = xml.replace(/“\s*([^<”]+)\s*<\/em>\s*”/g, "“$1”"); xml = xml.replace(/“\s*\s*([^<”]+)\s*”<\/em>/g, "“$1”"); xml = xml.replace(/([^\n>]) ?/g, "$1 "); xml = xml.replace(/ ?<\/em>/g, " "); xml = xml.replace(/]+)> /g, ""); xml = xml.replace(/<\/em> <\/p>/g, "

"); xml = xml.replace(/([a-z]+),<\/em>/g, "$1,"); } // These quote/apostrophe/em fixes interact with each other. TODO: try to disentangle so we don't repeat all of // fixEms. xml = xml.replace(/,” <\/em>/g, ",” "); fixEms(); xml = xml.replace(/

”/g, "

“"); xml = xml.replace(/“\s*<\/p>/g, "”

"); xml = xml.replace(/“\s*<\/em><\/p>/g, "

"); xml = xml.replace(/‘\s*<\/p>/g, "’

"); xml = xml.replace(/‘\s*<\/em><\/p>/g, "’

"); xml = xml.replace(/,” <\/em>/g, ",” "); xml = xml.replace(/′/g, "’"); xml = xml.replace(/″/g, "”"); xml = xml.replace(/([A-Za-z])‘s(\s?)/g, "$1’s$2"); xml = xml.replace(/I‘m/g, "I’m"); xml = xml.replace(/

“\s+/g, "

“"); xml = xml.replace(/'/g, "’"); xml = xml.replace(/’([A-Za-z]+)’/g, "‘$1’"); xml = xml.replace(/([a-z])”<\/p>/g, "$1.”

"); fixEms(); xml = xml.replace(/‘([^<]+)<\/em>‘/g, "‘$1’"); xml = xml.replace(/([a-z]+)!<\/em>/g, "$1!"); xml = xml.replace(/(?([\w ’]+)([!.?])”<\/em>/g, "$1$2”"); xml = xml.replace(/([\w ’]+[!.?])”<\/em>/g, "$1”"); xml = xml.replace(/I”(m|ll)/g, "I’$1"); xml = xml.replace(/””<\/p>/g, "”

"); xml = xml.replace(/^([^“]+?) ?”(?![ —<])/gm, "$1 “"); // Similar problems occur in Ward with and as do in Worm with s xml = xml.replace(//g, ""); xml = xml.replace(/(\s*
\s*)<\/b>/g, "$1"); xml = xml.replace(/(\s*
\s*)<\/strong>/g, "$1"); xml = xml.replace(/<\/strong>(\s*)/g, "$1"); xml = xml.replace(/@<\/strong>/g, "@"); xml = xml.replace(/
(\s*)<\/strong>/g, "

$1"); xml = xml.replace(/(\s*)<\/strong>/g, "
$1"); xml = xml.replace(/>(.*)<\/strong>:$1:s inside /// fixed above. xml = xml.replace(/
\s*<\/p>/g, "

"); // Fix missing spaces after commas xml = xml.replace(/([a-zA-Z]+),([a-zA-Z]+)/g, "$1, $2"); // Fix bad periods and spacing/markup surrounding them xml = xml.replace(/\.\.<\/p>/g, ".

"); xml = xml.replace(/\.\.”<\/p>/g, ".”

"); xml = xml.replace(/ \. /g, ". "); xml = xml.replace(/ \.<\/p>/g, ".

"); xml = xml.replace(/\.\.\./g, "…"); xml = xml.replace(/\.\. {2}/g, ". "); // Fix extra spaces xml = xml.replace(/ ? <\/p>/g, "

"); xml = xml.replace(/([a-z]) ,/g, "$1,"); xml = fixTruncatedWords(xml); xml = fixDialogueTags(xml); xml = fixForeignNames(xml); xml = standardizeNames(xml); xml = fixEmDashes(xml); xml = enDashJointNames(xml); xml = fixPossessives(xml); xml = cleanSceneBreaks(xml); xml = fixCapitalization(xml, book); xml = fixMispellings(xml); xml = fixHyphens(xml); xml = standardizeSpellings(xml); xml = fixCaseNumbers(xml); // One-off fixes for (const substitution of substitutions[chapter.url] || []) { if (substitution.before) { const indexOf = xml.indexOf(substitution.before); if (indexOf === -1) { warnings.push(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` + `updated at the source, in which case, you should edit substitutions.json.`); } if (indexOf !== xml.lastIndexOf(substitution.before)) { warnings.push(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` + `Update substitutions.json for a more precise substitution.`); } xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after); } else if (substitution.regExp) { xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement); } else { warnings.push(`Invalid substitution specified for ${chapter.url}`); } } // Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a . // Use this opportunity to insert a comment pointing to the original URL, for reference. xml = xml.replace( //, `\n\n`); return { xml, warnings }; } function fixTruncatedWords(xml) { xml = xml.replace(/‘Sup/g, "’Sup"); xml = xml.replace(/‘cuz/g, "’cuz"); // Short for "Sidepeace" xml = xml.replace(/[‘’][Pp]iece(?![a-z])/g, "’Piece"); // Short for "Disjoint" xml = xml.replace(/[‘’][Jj]oint(?![a-z])/g, "’Joint"); // Short for "Contender" xml = xml.replace(/[‘’][Tt]end(?![a-z])/g, "’Tend"); return xml; } function fixDialogueTags(xml) { // Fix recurring miscapitalization with questions xml = xml.replace(/\?”\s\s?She asked/g, "?” she asked"); xml = xml.replace(/\?”\s\s?He asked/g, "?” he asked"); // The author often fails to terminate a sentence, instead using a comma after a dialogue tag. For example, // > “I didn’t get much done,” Greg said, “I got distracted by... // This should instead be // > “I didn’t get much done,” Greg said. “I got distracted by... // // Our heuristic is to try to automatically fix this if the dialogue tag is two words (X said/admitted/sighed/etc.). // // This sometimes overcorrects, as in the following example: // > “Basically,” Alec said, “For your powers to manifest, ... // Here instead we should lowercase the "f". We handle that via one-offs in substitutions.json. // // This applies to ~800 instances, so although we have to correct back in substitutions.json a decent number of // times, it definitely pays for itself. Most of the instances we have to correct back we also need to fix the // capitalization anyway, and that's harder to do automatically, since proper names/"I"/etc. stay capitalized. xml = xml.replace(/,” ([A-Za-z]+ [A-Za-z]+), “([A-Z])/g, ",” $1. “$2"); return xml; } function fixForeignNames(xml) { // This is consistently missing diacritics xml = xml.replace(/Yangban/g, "Yàngbǎn"); // These are usually not italicized, but sometimes are. Other foreign-language names (like Yàngbǎn) are not // italicized, so we go in the direction of removing the italics. xml = xml.replace(/Garama<\/em>/g, "Garama"); xml = xml.replace(/Thanda<\/em>/g, "Thanda"); xml = xml.replace(/Sifara([^<]*)<\/em>/g, "Sifara$1"); xml = xml.replace(/Moord Nag([^<]*)<\/em>/g, "Moord Nag$1"); xml = xml.replace(/Califa de Perro([^<]*)<\/em>/g, "Califa de Perro$1"); xml = xml.replace(/Turanta([^<]*)<\/em>/g, "Turanta$1"); return xml; } function standardizeNames(xml) { // 197 instances of "Mrs." to 21 of "Ms." xml = xml.replace(/Ms\. Yamada/g, "Mrs. Yamada"); // 25 instances of "Amias" to 3 of "Amais" xml = xml.replace(/Amais/g, "Amias"); return xml; } function fixEmDashes(xml) { xml = xml.replace(/ – /g, "—"); xml = xml.replace(/“((?:)?)-/g, "“$1—"); xml = xml.replace(/-[,.]?”/g, "—”"); xml = xml.replace(/-(!|\?)”/g, "—$1”"); xml = xml.replace(/-[,.]?<\/em>”/g, "—”"); xml = xml.replace(/-“/g, "—”"); xml = xml.replace(/

-/g, "

—"); xml = xml.replace(/-<\/p>/g, "—

"); xml = xml.replace(/-<\/em><\/p>/g, "—

"); xml = xml.replace(/\s?\s?–\s?\s?/g, "—"); xml = xml.replace(/-\s\s?/g, "—"); xml = xml.replace(/\s?\s-/g, "—"); xml = xml.replace(/\s+—”/g, "—”"); xml = xml.replace(/I-I/g, "I—I"); xml = xml.replace(/I-uh/g, "I—uh"); return xml; } function enDashJointNames(xml) { // Joint names should use en dashes xml = xml.replace(/Dallon-Pelham/g, "Dallon–Pelham"); xml = xml.replace(/Bet-Gimel/g, "Bet–Gimel"); xml = xml.replace(/Tristan-Capricorn/g, "Tristan–Capricorn"); xml = xml.replace(/Capricorn-Byron/g, "Capricorn–Byron"); xml = xml.replace(/Tristan-Byron/g, "Tristan–Byron"); xml = xml.replace(/Gimel-Europe/g, "Gimel–Europe"); xml = xml.replace(/G-N/g, "G–N"); xml = xml.replace(/Imp-Damsel/g, "Imp–Damsel"); xml = xml.replace(/Damsel-Ashley/g, "Damsel–Ashley"); xml = xml.replace(/Antares-Anelace/g, "Antares–Anelace"); xml = xml.replace(/Challenger-Gallant/g, "Challenger–Gallant"); xml = xml.replace(/Undersider(s?)-(Breakthrough|Ambassador)/g, "Undersider$1–$2"); xml = xml.replace(/Norwalk-Fairfield/g, "Norwalk–Fairfield"); xml = xml.replace(/East-West/g, "east–west"); xml = xml.replace(/(Green|Yellow)-Black/g, "$1–Black"); xml = xml.replace(/Creutzfeldt-Jakob/g, "Creutzfeldt–Jakob"); xml = xml.replace(/Astaroth-Nidhug/g, "Astaroth–Nidhug"); xml = xml.replace(/Capulet-Montague/g, "Capulet–Montague"); xml = xml.replace(/Weaver-Clockblocker/g, "Weaver–Clockblocker"); xml = xml.replace(/Alexandria-Pretender/g, "Alexandria–Pretender"); xml = xml.replace(/Night Hag-Nyx/g, "Night Hag–Nyx"); xml = xml.replace(/Crawler-Breed/g, "Crawler–Breed"); xml = xml.replace(/Simurgh-Myrddin-plant/g, "Simurgh–Myrddin–plant"); xml = xml.replace(/Armsmaster-Defiant/g, "Armsmaster–Defiant"); return xml; } function fixPossessives(xml) { // Fix possessive of names ending in "s" // Note: if the "s" is unvoiced, as in Marquis, then it doesn't get the second "s". xml = xml.replace( // eslint-disable-next-line max-len /(? would be more semantically appropriate, but loses the author's intent. This is // especially the case in Ward, which uses a variety of different scene breaks. xml = xml.replace(/]*)>■<\/p>/g, `

`); xml = xml.replace(/

⊙<\/strong><\/p>/g, `

`); xml = xml.replace(/

⊙<\/strong><\/em><\/p>/g, `

`); xml = xml.replace(/

⊙⊙<\/strong><\/p>/g, `

`); xml = xml.replace(/

⊙ *⊙ *⊙ *⊙ *⊙<\/strong><\/p>/g, `

⊙ ⊙ ⊙ ⊙ ⊙

`); return xml; } function fixCapitalization(xml, book) { // This occurs enough times it's better to do here than in one-off fixes. We correct the single instance where // it's incorrect to capitalize in the one-off fixes. // Note that Ward contains much talk of "the clairvoyants", so we don't want to capitalize plurals. xml = xml.replace(/([Tt])he clairvoyant([^s])/g, "$1he Clairvoyant$2"); // ReSound's name is sometimes miscapitalized. The word is never used in a non-name context. xml = xml.replace(/Resound/g, "ReSound"); // The Speedrunners team name is missing its capitalization a couple times. xml = xml.replace(/speedrunners/g, "Speedrunners"); // "patrol block" is capitalized three different ways: "patrol block", "Patrol block", and "Patrol Block". "patrol // group" is always lowercased. It seems like "Patrol" is a proper name, and is used as a capitalized modifier in // other contexts (e.g. Patrol leader). So let's standardize on "Patrol ". xml = xml.replace(/patrol (block|group|leader|guard|student|uniform|squad|soldier|officer|crew|girl|bus|training)/ig, (_, $1) => `Patrol ${$1.toLowerCase()}`); // This always works in Ward and has a few false positives in Worm, where it is never needed: if (book === "ward") { xml = xml.replace(/the patrol/g, "the Patrol"); } // This is sometimes missing its capitalization. xml = xml.replace(/the birdcage/g, "the Birdcage"); // There's no reason why these should be capitalized. (Note that they never appear at the beginning of any sentences.) xml = xml.replace(/Halberd/g, "halberd"); xml = xml.replace(/Loft/g, "loft"); // There's no reason why university should be capitalized in most contexts, although sometimes it's used as part of // a compound noun or at the beginning of a sentence. xml = xml.replace(/(? prefix + designation.toLowerCase() ); xml = xml.replace( /(mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)-(\d+)/gi, "$1 $2" ); xml = xml.replace( // eslint-disable-next-line max-len /(mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)[ -/](mover|shaker|brute|breaker|tinker|blaster|thinker|master|striker|changer|trump|stranger|shifter|shaper)/gi, "$1–$2" ); // Place names need to always be capitalized xml = xml.replace(/North end/g, "North End"); xml = xml.replace(/(Stonemast|Shale) avenue/g, "$1 Avenue"); xml = xml.replace(/(Lord|Slater) street/g, "$1 Street"); xml = xml.replace(/(Hollow|Cedar) point/g, "$1 Point"); xml = xml.replace(/(Norwalk|Fenway|Stratford) station/g, "$1 Station"); xml = xml.replace(/the megalopolis/g, "the Megalopolis"); xml = xml.replace(/earths(?![a-z])/g, "Earths"); // "Mom" and "Dad" should be capitalized when used as a proper name. These regexps are tuned to catch a good amount of // instances, without over-correcting for non-proper-name-like cases. Many other instances are handled in // substitutions.json. xml = xml.replace(/(? "Case " + caseNumber[0].toUpperCase() + caseNumber.substring(1)); return xml; } function isEmptyOrGarbage(el) { const text = el.textContent.trim(); return text === "" || text.startsWith("Last Chapter") || text.startsWith("Previous Chapter") || text.startsWith("Next Chapter"); } function escapeRegExp(str) { return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&"); } function decodeCloudFlareEmail(hash) { let email = ""; const xorWithThis = parseInt(hash.substring(0, 2), 16); for (let i = 2; i < hash.length; i += 2) { const charCode = parseInt(hash.substring(i, i + 2), 16) ^ xorWithThis; email += String.fromCharCode(charCode); } return email; }