From 801e28d602ce6986b3b9885dec7f7cdf8f3b9454 Mon Sep 17 00:00:00 2001 From: Domenic Denicola Date: Fri, 15 May 2015 01:01:03 -0400 Subject: [PATCH] A lot more cleanups --- lib/convert.js | 46 +++++++++++- lib/download.js | 2 +- lib/substitutions.json | 154 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 195 insertions(+), 7 deletions(-) diff --git a/lib/convert.js b/lib/convert.js index 8e4451e..9e57706 100644 --- a/lib/convert.js +++ b/lib/convert.js @@ -88,14 +88,43 @@ function getBodyXml(chapter, contentEl) { }); // Remove empty s and s + // Remove style attributes from them, as they're always messed up. const ems = contentEl.querySelectorAll("em, i"); Array.prototype.forEach.call(ems, function (em) { if (em.textContent.trim() === "") { em.parentNode.removeChild(em); + } else { + em.removeAttribute("style"); + } + }); + + // In https://parahumans.wordpress.com/2013/01/05/monarch-16-13/ there are some
s that should be

s O_o + const addresses = contentEl.querySelectorAll("address"); + Array.prototype.forEach.call(addresses, function (address) { + const p = contentEl.ownerDocument.createElement("p"); + p.innerHTML = address.innerHTML; + address.parentNode.replaceChild(p, address); + }); + + // Every except underline ones is pointless at best and frequently messed up. (Weird font size, line spacing, + // etc.) + const spans = contentEl.querySelectorAll("span"); + Array.prototype.forEach.call(spans, function (span) { + if (span.getAttribute("style") === "text-decoration:underline;") { + return; + } + + if (span.textContent.trim() === "") { + span.parentNode.removeChild(span); + } else { + const docFrag = contentEl.ownerDocument.createDocumentFragment(); + Array.prototype.forEach.call(span.childNodes, function (childNode) { + docFrag.appendChild(childNode); + }); + span.parentNode.replaceChild(docFrag, span); } }); - // TODO: remove redundant s inside

s // Synthesize a tag to serialize const bodyEl = contentEl.ownerDocument.createElement("body"); @@ -110,14 +139,25 @@ function getBodyXml(chapter, contentEl) { let xml = serializeToXml(bodyEl); // Fix recurring strange pattern of extra
in

......
\n

- xml = xml.replace(/\s*<\/em><\/p>/g, '

'); + xml = xml.replace(/\s*<\/em><\/p>/g, "

"); + + // Fix recurring broken-up s + xml = xml.replace(/<\/em>‘s/, "’s") + xml = xml.replace(/<\/em>/g, ""); + xml = xml.replace(/<\/em>([^A-Za-z]{1,2})/g, "$1"); + xml = xml.replace(/<\/em>\. /g, ". "); // Fix recurring poor closing quotes xml = xml.replace(/“<\/p>/g, "”

"); // Some fixes for dashes; not comprehensive + xml = xml.replace(/ – /g, "—"); xml = xml.replace(/“-/g, "“—"); xml = xml.replace(/-”/g, "—”"); + xml = xml.replace(/-.”/g, "—”"); + xml = xml.replace(/-“/g, "—”"); + xml = xml.replace(/

-/g, "

—"); + xml = xml.replace(/-<\/p>/g, "—

"); // There are way too many nonbreaking spaces where they don't belong. // If they show up three in a row, then let them live. Otherwise, they die. @@ -139,7 +179,7 @@ function getBodyXml(chapter, contentEl) { }); // Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a - xml = xml.replace(//, ''); + xml = xml.replace(//, ""); return xml; } diff --git a/lib/download.js b/lib/download.js index aaeae7c..46db362 100644 --- a/lib/download.js +++ b/lib/download.js @@ -64,7 +64,7 @@ function downloadAllChapters(manifest, startChapterUrl, cachePath, manifestPath) manifest.push({ url: chapterUrlToSave, title: chapterTitle, - filename: filename + filename }); fs.writeFile(path.resolve(cachePath, filename), contents); diff --git a/lib/substitutions.json b/lib/substitutions.json index 9e0f21a..2889457 100644 --- a/lib/substitutions.json +++ b/lib/substitutions.json @@ -53,16 +53,164 @@ "after": "Lost in thought.”" } ], + "https://parahumans.wordpress.com/2012/12/13/interlude-16-donation-bonus-2/": [ + { + "before": "“", + "after": "”" + } + ], + "https://parahumans.wordpress.com/2012/12/15/monarch-16-7/": [ + { + "before": "Brockton bay", + "after": "Brockton Bay" + } + ], "https://parahumans.wordpress.com/2012/12/29/monarch-16-11/": [ { "before": "attemtps", "after": "attempts" } ], - "https://parahumans.wordpress.com/2013/01/01/monarch-16-12/": [ + "https://parahumans.wordpress.com/2013/01/09/migration-17-2/": [ { - "before": "Did they make her smell like me? They had to have, to keep the dogs from barking distress. But how? Had Calvert had his men raid my stuff? Had he used my dirty laundry?", - "after": "Did they make her smell like me? They had to have, to keep the dogs from barking distress. But how? Had Calvert had his men raid my stuff? Had he used my dirty laundry?" + "before": "fence!", + "after": "fence!" + } + ], + "https://parahumans.wordpress.com/2013/01/10/migration-17-3/": [ + { + "before": "divots intp", + "after": "divots into" + }, + { + "before": "No. the", + "after": "No. The" + }, + { + "before": "help?” He", + "after": "help?” he" + } + ], + "https://parahumans.wordpress.com/2013/01/12/migration-17-5/": [ + { + "before": "been replying", + "after": "been replaying" + }, + { + "before": "sort of stuff that , and", + "after": "sort of stuff that ???, and" + }, + { + "before": "stirred. her eyes", + "after": "stirred. Her eyes" + } + ], + "https://parahumans.wordpress.com/2013/01/13/migration-17-6/": [ + { + "before": "ground. Not exactly", + "after": "ground. “Not exactly" + }, + { + "before": "MWBB ", + "after": "MWBB " + } + ], + "https://parahumans.wordpress.com/2013/01/14/migration-17-7/": [ + { + "before": "focus. This is because of Noelle", + "after": "focus. This is because of Noelle" + } + ], + "https://parahumans.wordpress.com/2013/01/19/queen-18-1/": [ + { + "before": "could beon", + "after": "could be on" + }, + { + "before": "untrustworthy.", + "after": "untrustworthy." + } + ], + "https://parahumans.wordpress.com/2013/01/24/interlude-18x/": [ + { + "before": "french accent", + "after": "French accent" + } + ], + "https://parahumans.wordpress.com/2013/01/31/interlude-18-donation-bonus-2/": [ + { + "before": "Bay.She", + "after": "Bay. She" + } + ], + "https://parahumans.wordpress.com/2013/02/07/interlude-18-donation-bonus-3/": [ + { + "before": "instinctthat", + "after": "instinct that" + } + ], + "https://parahumans.wordpress.com/2013/02/12/queen-18-8/": [ + { + "before": "dragged long", + "after": "dragged along" + } + ], + "https://parahumans.wordpress.com/2013/02/14/interlude-18-donation-bonus-4/": [ + { + "before": "C.U.l China", + "after": "C.U. China" + }, + { + "before": "Faulltine asked", + "after": "Faultline asked" + }, + { + "before": "through Faulltine", + "after": "through Faultline" + } + ], + "https://parahumans.wordpress.com/2013/02/16/interlude-18/": [ + { + "before": "Gruewas", + "after": "Grue was" + }, + { + "before": "askinga", + "after": "asking a" + } + ], + "https://parahumans.wordpress.com/2013/02/19/scourge-19-1/": [ + { + "before": "‘s9′", + "after": "‘s9’" + } + ], + "https://parahumans.wordpress.com/2013/02/23/scourge-19-2/": [ + { + "before": "You’llstop", + "after": "You’ll stop" + } + ], + "https://parahumans.wordpress.com/2013/02/26/scourge-19-3/": [ + { + "before": "areconnected", + "after": "are connected" + } + ], + "https://parahumans.wordpress.com/2013/02/28/interlude-19-donation-bonus-1/": [ + { + "before": "English?” He asked", + "after": "English?” he asked" + }, + { + "before": "and be brought it", + "after": "and he brought it" + } + ], + "https://parahumans.wordpress.com/2013/03/16/interlude-19-y/": [ + { + "before": "Brockton bay", + "after": "Brockton Bay" } ] }