jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tweaks and bug-fixes for the cleanups

Browse source 
Several notable fixes:

- Fixed a bad bug with <span> remover: since moving the child node to a document fragment changes the indices of the childNodes collection, this would leave several nodes in limbo, with the net effect of removing their text from the document.
- Fixed the empty-<em> remover to replace the empty <em> with a space, instead of a removing it entirely; this leads to a lot fewer wordsstuck together, which were starting to accumulate erroneously in substitutions.json.
- Warn instead of error on bad substitutions: this makes it easier to actually find the bad substitution afterward, since then the output still happens.
This commit is contained in:
Domenic Denicola 2015-05-17 15:28:44 -04:00
commit 4019f5d1e6
2 changed files with 163 additions and 75 deletions
Download patch file Download diff file Expand all files Collapse all files

44
lib/convert.js
View file

@ -92,7 +92,8 @@ function getBodyXml(chapter, contentEl) {
const ems = contentEl.querySelectorAll("em, i");
Array.prototype.forEach.call(ems, function (em) {
if (em.textContent.trim() === "") {
em.parentNode.removeChild(em);
const replacement = contentEl.ownerDocument.createTextNode(" ");
em.parentNode.replaceChild(replacement, em);
} else {
em.removeAttribute("style");
}
@ -118,9 +119,9 @@ function getBodyXml(chapter, contentEl) {
span.parentNode.removeChild(span);
} else {
const docFrag = contentEl.ownerDocument.createDocumentFragment();
Array.prototype.forEach.call(span.childNodes, function (childNode) {
docFrag.appendChild(childNode);
});
while (span.firstChild) {
docFrag.appendChild(span.firstChild);
}
span.parentNode.replaceChild(docFrag, span);
}
});
@ -141,14 +142,12 @@ function getBodyXml(chapter, contentEl) {
// Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p>
xml = xml.replace(/<br\/>\s*<\/em><\/p>/g, "</em></p>");
// Fix recurring broken-up <em>s
xml = xml.replace(/<\/em>‘s/, "’s</em>")
xml = xml.replace(/<\/em><em>/g, "");
xml = xml.replace(/<\/em>([^A-Za-z]{1,2})<em>/g, "$1");
xml = xml.replace(/<\/em>\. <em>/g, ". ");
// Fix recurring poor closing quotes
xml = xml.replace(/“<\/p>/g, "”</p>");
xml = xml.replace(/‘<\/p>/g, "’</p>");
// Fix use of ′ instead of closing single quote
xml = xml.replace(/′/g, "’");
// Some fixes for dashes; not comprehensive
xml = xml.replace(/ – /g, "—");
@ -158,21 +157,38 @@ function getBodyXml(chapter, contentEl) {
xml = xml.replace(/-“/g, "—”");
xml = xml.replace(/<p>-/g, "<p>—");
xml = xml.replace(/-<\/p>/g, "—</p>");
xml = xml.replace(/\s?\s?–\s?\s?/g, "—");
// There are way too many nonbreaking spaces where they don't belong.
// If they show up three in a row, then let them live. Otherwise, they die.
xml = xml.replace(/([^\xA0])\xA0\xA0?([^\xA0])/g, "$1 $2");
// Fix recurring broken-up <em>s
xml = xml.replace(/<\/em>‘s/g, "’s</em>")
xml = xml.replace(/<\/em><em>/g, "");
xml = xml.replace(/<\/em>\. <em>/g, ". ");
xml = xml.replace(/“<em>([^>]+)<\/em>(!|\?|\.)”/g, "“<em>$1$2<\/em>”");
// Fix recurring miscapitalization with questions
xml = xml.replace(/\?”\s\s?She asked/g, "?” she asked");
xml = xml.replace(/\?”\s\s?He asked/g, "?” he asked");
// Fix extra periods at the end of paragraphs
xml = xml.replace(/\.\.<\/p>/g, ".</p>");
// Replace single-word <i>s with <em>s. Other <i>s are probably erroneous too, but these are known-bad.
xml = xml.replace(/<i>([A-Za-z]+)<\/i>/g, "<em>$1</em>");
// One-off fixes
(substitutions[chapter.url] || []).forEach(function (substitution) {
const indexOf = xml.indexOf(substitution.before);
if (indexOf === -1) {
throw new Error(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
`updated at the source, in which case, you should edit substitutions.json.`);
console.warn(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
`updated at the source, in which case, you should edit substitutions.json.`);
}
if (indexOf !== xml.lastIndexOf(substitution.before)) {
throw new Error(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
`Update substitutions.json for a more precise substitution.`);
console.warn(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
`Update substitutions.json for a more precise substitution.`);
}
xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after);

194
lib/substitutions.json
View file

@ -1,4 +1,34 @@
{
"https://parahumans.wordpress.com/2011/08/06/interlude-2/": [
{
"before": "<em>principles, </em>Glory",
"after": "<em>principles</em>, Glory"
}
],
"https://parahumans.wordpress.com/2011/12/03/hive-5-9/": [
{
"before": "insult. An exc</em>use to trounce me physically<em>.</em>",
"after": "insult.</em> An excuse to trounce me physically."
}
],
"https://parahumans.wordpress.com/2012/03/13/extermination-8-4/": [
{
"before": "Endbringer<em>. </em>After the laser petered out<em>, h</em>e",
"after": "Endbringer. After the laser petered out, he"
}
],
"https://parahumans.wordpress.com/2012/03/17/extermination-8-5/": [
{
"before": "past tense-, or",
"after": "past tense—or"
}
],
"https://parahumans.wordpress.com/2012/08/11/snare-13-5/": [
{
"before": "<em>similar. </em>",
"after": "<em>similar</em>. "
}
],
"https://parahumans.wordpress.com/2012/09/11/prey-14-3/": [
{
"before": "truck reached<br/>\nthe other Nine",
@ -29,12 +59,6 @@
"after": "this sort of resistance?"
}
],
"https://parahumans.wordpress.com/2012/11/27/monarch-16-2/": [
{
"before": "waseven <i>less</i>",
"after": "was even <em>less</em>"
}
],
"https://parahumans.wordpress.com/2012/12/04/monarch-16-4/": [
{
"before": "well, with Bitch’s civilian",
@ -73,8 +97,8 @@
],
"https://parahumans.wordpress.com/2013/01/09/migration-17-2/": [
{
"before": "fence</em>!",
"after": "fence!</em>"
"before": "left now.</em>’</p>",
"after": "left now.</em></p>"
}
],
"https://parahumans.wordpress.com/2013/01/10/migration-17-3/": [
@ -85,10 +109,6 @@
{
"before": "No. the",
"after": "No. The"
},
{
"before": "help?” He",
"after": "help?” he"
}
],
"https://parahumans.wordpress.com/2013/01/12/migration-17-5/": [
@ -122,10 +142,6 @@
}
],
"https://parahumans.wordpress.com/2013/01/19/queen-18-1/": [
{
"before": "could beon",
"after": "could be on"
},
{
"before": "<em>untrustworthy.</em>",
"after": "<em>untrustworthy</em>."
@ -137,18 +153,6 @@
"after": "French accent"
}
],
"https://parahumans.wordpress.com/2013/01/31/interlude-18-donation-bonus-2/": [
{
"before": "Bay.<em>She",
"after": "Bay. <em>She"
}
],
"https://parahumans.wordpress.com/2013/02/07/interlude-18-donation-bonus-3/": [
{
"before": "instinctthat",
"after": "instinct that"
}
],
"https://parahumans.wordpress.com/2013/02/12/queen-18-8/": [
{
"before": "dragged long",
@ -158,7 +162,7 @@
"https://parahumans.wordpress.com/2013/02/14/interlude-18-donation-bonus-4/": [
{
"before": "C.U.l China",
"after": "C.U. China"
"after": "C.U.I. China"
},
{
"before": "Faulltine asked",
@ -169,48 +173,116 @@
"after": "through Faultline"
}
],
"https://parahumans.wordpress.com/2013/02/16/interlude-18/": [
{
"before": "Gruewas",
"after": "Grue was"
},
{
"before": "askinga",
"after": "asking a"
}
],
"https://parahumans.wordpress.com/2013/02/19/scourge-19-1/": [
{
"before": "‘s9′",
"after": "‘s9’"
}
],
"https://parahumans.wordpress.com/2013/02/23/scourge-19-2/": [
{
"before": "You’llstop",
"after": "You’ll stop"
}
],
"https://parahumans.wordpress.com/2013/02/26/scourge-19-3/": [
{
"before": "areconnected",
"after": "are connected"
}
],
"https://parahumans.wordpress.com/2013/02/28/interlude-19-donation-bonus-1/": [
{
"before": "English?” He asked",
"after": "English?” he asked"
},
{
"before": "and be brought it",
"after": "and he brought it"
}
],
"https://parahumans.wordpress.com/2013/03/09/scourge-19-6/": [
{
"before": "heailng",
"after": "healing"
}
],
"https://parahumans.wordpress.com/2013/03/12/scourge-19-7/": [
{
"before": "they<em> did, but how we</em> dealt",
"after": "they <em>did</em>, but <em>how</em> we dealt"
}
],
"https://parahumans.wordpress.com/2013/03/16/interlude-19-y/": [
{
"before": "Brockton bay",
"after": "Brockton Bay"
}
],
"https://parahumans.wordpress.com/2013/03/30/chrysalis-20-4/": [
{
"before": "guess-, while",
"after": "guess—while"
}
],
"https://parahumans.wordpress.com/2013/04/04/interlude-20-donation-bonus-1/": [
{
"before": "’20th’",
"after": "‘20th’"
}
],
"https://parahumans.wordpress.com/2013/04/06/interlude-20/": [
{
"before": "the costume were studded",
"after": "the costume was studded"
}
],
"https://parahumans.wordpress.com/2013/04/13/imago-21-2/": [
{
"before": "Captain’s hill",
"after": "Captain’s Hill"
}
],
"https://parahumans.wordpress.com/2013/04/25/imago-21-7/": [
{
"before": "Dinah</em>. <em>because this",
"after": "Dinah. Because this"
}
],
"https://parahumans.wordpress.com/2013/04/30/interlude-21/": [
{
"before": "But…Maybe she",
"after": "But… Maybe she"
}
],
"https://parahumans.wordpress.com/2013/05/07/cell-22-2/": [
{
"before": "said, “Is to set",
"after": "said, “is to set"
}
],
"https://parahumans.wordpress.com/2013/05/18/interlude-22/": [
{
"before": "long,” <em>she</em> warned",
"after": "long,” she warned"
}
],
"https://parahumans.wordpress.com/2013/05/21/interlude-22-donation-bonus-1/": [
{
"before": "“is it reassuring",
"after": "“Is it reassuring"
}
],
"https://parahumans.wordpress.com/2013/05/25/drone-23-1/": [
{
"before": "Defiant spoke , “Let’s",
"after": "Defiant spoke. “Let’s"
}
],
"https://parahumans.wordpress.com/2013/05/28/drone-23-2/": [
{
"before": "of the ship. It’s",
"after": "of the ship.’ It’s"
}
],
"https://parahumans.wordpress.com/2013/06/01/drone-23-4/": [
{
"before": "better off?'”",
"after": "better off?’”"
},
{
"before": "said. Someone",
"after": "said. “Someone"
}
],
"https://parahumans.wordpress.com/2013/09/28/venom-29-5/": [
{
"before": "<em>Losing y</em>ou as you get further down<em>.”</em>",
"after": "<em>Losing you as you get further down.</em>”"
}
],
"https://parahumans.wordpress.com/2013/11/05/teneral-e-2/": [
{
"before": "<em>property of Ner</em>o",
"after": "<em>property of Nero</em>"
}
]
}