+
+
+
+
+ ......
\n
([^>]+)<\/em>(!|\?|\.)<\/p>/g, " $1$2
([^>]+)<\/em>(!|\?|\.)<\/p>/ug, " $1$2
”/g, "
“"); - xml = xml.replace(/“\s*<\/p>/g, "”
"); - xml = xml.replace(/“\s*<\/em><\/p>/g, "“\s+/g, "
“"); - xml = xml.replace(/\s+”/g, "”"); - xml = xml.replace(/'/g, "’"); - xml = xml.replace(/’([A-Za-z]+)’/g, "‘$1’"); - xml = xml.replace(/([a-z])”<\/p>/g, "$1.”
"); + xml = xml.replace(/”/ug, "
“"); + xml = xml.replace(/“\s*<\/p>/ug, "”
"); + xml = xml.replace(/“\s*<\/em><\/p>/ug, "”"); + xml = xml.replace(/‘\s*<\/p>/ug, "’"); + xml = xml.replace(/‘\s*<\/em><\/p>/ug, "’"); + xml = xml.replace(/,” <\/em>/ug, ",” "); + xml = xml.replace(/′/ug, "’"); + xml = xml.replace(/″/ug, "”"); + xml = xml.replace(/([A-Za-z])‘s(\s?)/ug, "$1’s$2"); + xml = xml.replace(/I‘m/ug, "I’m"); + xml = xml.replace(/“\s+/ug, "
“"); + xml = xml.replace(/\s+”/ug, "”"); + xml = xml.replace(/'/ug, "’"); + xml = xml.replace(/’([A-Za-z]+)’/ug, "‘$1’"); + xml = xml.replace(/([a-z])”<\/p>/ug, "$1.”
"); fixEms(); - xml = xml.replace(/‘([^<]+)<\/em>‘/g, "‘$1’"); - xml = xml.replace(/([a-z]+)!<\/em>/g, "$1!"); - xml = xml.replace(/(?([\w ’]+)([!.?])”<\/em>/g, "$1$2”"); - xml = xml.replace(/([\w ’]+[!.?])”<\/em>/g, "$1”"); - xml = xml.replace(/I”(m|ll)/g, "I’$1"); - xml = xml.replace(/””<\/p>/g, "”"); - xml = xml.replace(/^([^“]+?) ?”(?![ —<])/gm, "$1 “"); - xml = xml.replace(/(?([A-Za-z]+),<\/em>(?!”| +[A-Za-z]+ thought)/, "$1,"); - xml = xml.replace(/‘([Kk])ay(?!’)/g, "’$1ay"); - xml = xml.replace(/(Why|What|Who|How|Where|When)<\/em>\?/g, "$1?"); - xml = xml.replace(/,<\/em>/g, ","); - xml = xml.replace(/,”<\/p>/g, ".”"); - xml = xml.replace(/(.*),<\/p>/g, "
$1.
"); - xml = xml.replace(/‘(\w+)‘(\w+)’/g, "‘$1’$2’"); - xml = xml.replace(/([a-z]+), ([a-z]+)<\/em>/g, "$1, $2"); + xml = xml.replace(/‘([^<]+)<\/em>‘/ug, "‘$1’"); + xml = xml.replace(/([a-z]+)!<\/em>/ug, "$1!"); + xml = xml.replace(/(?([\w ’]+)([!.?])”<\/em>/ug, "$1$2”"); + xml = xml.replace(/([\w ’]+[!.?])”<\/em>/ug, "$1”"); + xml = xml.replace(/I”(m|ll)/ug, "I’$1"); + xml = xml.replace(/””<\/p>/ug, "”"); + xml = xml.replace(/^([^“]+?) ?”(?![ —<])/ugm, "$1 “"); + xml = xml.replace(/(?([A-Za-z]+),<\/em>(?!”| +[A-Za-z]+ thought)/u, "$1,"); + xml = xml.replace(/‘([Kk])ay(?!’)/ug, "’$1ay"); + xml = xml.replace(/(Why|What|Who|How|Where|When)<\/em>\?/ug, "$1?"); + xml = xml.replace(/,<\/em>/ug, ","); + xml = xml.replace(/,”<\/p>/ug, ".”"); + xml = xml.replace(/(.*),<\/p>/ug, "
$1.
"); + xml = xml.replace(/‘(\w+)‘(\w+)’/ug, "‘$1’$2’"); + xml = xml.replace(/([a-z]+), ([a-z]+)<\/em>/ug, "$1, $2"); // Similar problems occur in Ward with and as do in Worm with s - xml = xml.replace(//g, ""); - xml = xml.replace(/(\s*
\s*/ug, "
"); + + // This is another quote fix but it needs to happen after the line break deletion... so entangled, ugh. + xml = xml.replace(/<\/em>\s*“\s*<\/p>/ug, "
”"); // Fix missing spaces after commas - xml = xml.replace(/([a-zA-Z]+),([a-zA-Z]+)/g, "$1, $2"); + xml = xml.replace(/([a-zA-Z]+),([a-zA-Z]+)/ug, "$1, $2"); // Fix bad periods and spacing/markup surrounding them - xml = xml.replace(/\.\.<\/p>/g, "."); - xml = xml.replace(/\.\.”<\/p>/g, ".”"); - xml = xml.replace(/ \. /g, ". "); - xml = xml.replace(/ \.<\/p>/g, "."); - xml = xml.replace(/\.\.\./g, "…"); - xml = xml.replace(/\.\. {2}/g, ". "); - xml = xml.replace(/\.\./g, "…"); - xml = xml.replace(/(?/ug, "."); + xml = xml.replace(/\.\.”<\/p>/ug, ".”"); + xml = xml.replace(/ \. /ug, ". "); + xml = xml.replace(/ \.<\/p>/ug, "."); + xml = xml.replace(/\.\.\./ug, "…"); + xml = xml.replace(/\.\. {2}/ug, ". "); + xml = xml.replace(/\.\./ug, "…"); + xml = xml.replace(/(?/g, ""); - xml = xml.replace(/([a-z]) ,/g, "$1,"); + xml = xml.replace(/ ? <\/p>/ug, ""); + xml = xml.replace(/([a-z]) ,/ug, "$1,"); + + // Use actual emojis instead of images + xml = xml.replace( + // eslint-disable-next-line max-len + /
/ug,
+ "🤨"
+ );
xml = fixTruncatedWords(xml);
xml = fixDialogueTags(xml);
@@ -270,9 +289,9 @@ function getBodyXML(chapter, book, contentEl) {
`Update substitutions.json for a more precise substitution.`);
}
- xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after);
+ xml = xml.replace(new RegExp(escapeRegExp(substitution.before), "u"), substitution.after);
} else if (substitution.regExp) {
- xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement);
+ xml = xml.replace(new RegExp(substitution.regExp, "ug"), substitution.replacement);
} else {
warnings.push(`Invalid substitution specified for ${chapter.url}`);
}
@@ -281,30 +300,31 @@ function getBodyXML(chapter, book, contentEl) {
// Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a .
// Use this opportunity to insert a comment pointing to the original URL, for reference.
xml = xml.replace(
- //,
- `\n\n`);
+ //u,
+ `\n\n`
+ );
return { xml, warnings };
}
function fixTruncatedWords(xml) {
- xml = xml.replace(/‘Sup/g, "’Sup");
- xml = xml.replace(/‘cuz/g, "’cuz");
+ xml = xml.replace(/‘Sup/ug, "’Sup");
+ xml = xml.replace(/‘cuz/ug, "’cuz");
// Short for "Sidepeace"
- xml = xml.replace(/[‘’][Pp]iece(?![a-z])/g, "’Piece");
+ xml = xml.replace(/[‘’][Pp]iece(?![a-z])/ug, "’Piece");
// Short for "Disjoint"
- xml = xml.replace(/[‘’][Jj]oint(?![a-z])/g, "’Joint");
+ xml = xml.replace(/[‘’][Jj]oint(?![a-z])/ug, "’Joint");
// Short for "Contender"
- xml = xml.replace(/[‘’][Tt]end(?![a-z])/g, "’Tend");
+ xml = xml.replace(/[‘’][Tt]end(?![a-z])/ug, "’Tend");
// Short for "Anelace"
- xml = xml.replace(/[‘’][Ll]ace(?![a-z])/g, "’Lace");
+ xml = xml.replace(/[‘’][Ll]ace(?![a-z])/ug, "’Lace");
// Short for "Birdcage"
- xml = xml.replace(/[‘’][Cc]age(?![a-z])/g, "’Cage");
+ xml = xml.replace(/[‘’][Cc]age(?![a-z])/ug, "’Cage");
// We can't do "’Clear" (short for Crystalclear) here because it appears too much as a normal word preceded by an
// open quote, so we do that in substitutions.json.
@@ -314,8 +334,8 @@ function fixTruncatedWords(xml) {
function fixDialogueTags(xml) {
// Fix recurring miscapitalization with questions
- xml = xml.replace(/\?”\s\s?She asked/g, "?” she asked");
- xml = xml.replace(/\?”\s\s?He asked/g, "?” he asked");
+ xml = xml.replace(/\?”\s\s?She asked/ug, "?” she asked");
+ xml = xml.replace(/\?”\s\s?He asked/ug, "?” he asked");
// The author often fails to terminate a sentence, instead using a comma after a dialogue tag. For example,
// > “I didn’t get much done,” Greg said, “I got distracted by...
@@ -331,105 +351,108 @@ function fixDialogueTags(xml) {
// This applies to ~800 instances, so although we have to correct back in substitutions.json a decent number of
// times, it definitely pays for itself. Most of the instances we have to correct back we also need to fix the
// capitalization anyway, and that's harder to do automatically, since proper names/"I"/etc. stay capitalized.
- xml = xml.replace(/,” ([A-Za-z]+ [A-Za-z]+), “([A-Z])/g, ",” $1. “$2");
+ xml = xml.replace(/,” ([A-Za-z]+ [A-Za-z]+), “([A-Z])/ug, ",” $1. “$2");
return xml;
}
function fixForeignNames(xml) {
// This is consistently missing diacritics
- xml = xml.replace(/Yangban/g, "Yàngbǎn");
+ xml = xml.replace(/Yangban/ug, "Yàngbǎn");
// These are usually not italicized, but sometimes are. Other foreign-language names (like Yàngbǎn) are not
// italicized, so we go in the direction of removing the italics.
- xml = xml.replace(/Garama<\/em>/g, "Garama");
- xml = xml.replace(/Thanda<\/em>/g, "Thanda");
- xml = xml.replace(/Sifara([^<]*)<\/em>/g, "Sifara$1");
- xml = xml.replace(/Moord Nag([^<]*)<\/em>/g, "Moord Nag$1");
- xml = xml.replace(/Califa de Perro([^<]*)<\/em>/g, "Califa de Perro$1");
- xml = xml.replace(/Turanta([^<]*)<\/em>/g, "Turanta$1");
+ xml = xml.replace(/Garama<\/em>/ug, "Garama");
+ xml = xml.replace(/Thanda<\/em>/ug, "Thanda");
+ xml = xml.replace(/Sifara([^<]*)<\/em>/ug, "Sifara$1");
+ xml = xml.replace(/Moord Nag([^<]*)<\/em>/ug, "Moord Nag$1");
+ xml = xml.replace(/Califa de Perro([^<]*)<\/em>/ug, "Califa de Perro$1");
+ xml = xml.replace(/Turanta([^<]*)<\/em>/ug, "Turanta$1");
return xml;
}
function standardizeNames(xml) {
// 197 instances of "Mrs." to 21 of "Ms."
- xml = xml.replace(/Ms\. Yamada/g, "Mrs. Yamada");
+ xml = xml.replace(/Ms\. Yamada/ug, "Mrs. Yamada");
// 25 instances of "Amias" to 3 of "Amais"
- xml = xml.replace(/Amais/g, "Amias");
+ xml = xml.replace(/Amais/ug, "Amias");
// 185 instances of Juliette to 4 of Juliet
- xml = xml.replace(/Juliet(?=\b)/g, "Juliette");
+ xml = xml.replace(/Juliet(?=\b)/ug, "Juliette");
// Earlier chapters have a space; later ones do not. They're separate words, so side with the earlier chapters.
// One location is missing the "k".
- xml = xml.replace(/Crock? o[‘’]Shit/g, "Crock o’ Shit");
+ xml = xml.replace(/Crock? o[‘’]Shit/ug, "Crock o’ Shit");
// 5 instances of "Jotun" to 2 of "Jotunn"
- xml = xml.replace(/Jotunn/g, "Jotun");
+ xml = xml.replace(/Jotunn/ug, "Jotun");
// 13 instances of Elman to 1 of Elmann
- xml = xml.replace(/Elmann/g, "Elman");
+ xml = xml.replace(/Elmann/ug, "Elman");
// Thousands of instances of Tattletale to 4 instances of Tatteltale
- xml = xml.replace(/Tatteltale/g, "Tattletale");
+ xml = xml.replace(/Tatteltale/ug, "Tattletale");
+
+ // 73 instances of Über to 2 of Uber
+ xml = xml.replace(/Uber/ug, "Über");
return xml;
}
function fixEmDashes(xml) {
- xml = xml.replace(/ – /g, "—");
- xml = xml.replace(/“((?:)?)-/g, "“$1—");
- xml = xml.replace(/-[,.]?”/g, "—”");
- xml = xml.replace(/-(!|\?)”/g, "—$1”");
- xml = xml.replace(/-[,.]?<\/([a-z]+)>”/g, "—”");
- xml = xml.replace(/-“/g, "—”");
- xml = xml.replace(/-/g, "
—"); - xml = xml.replace(/-<\/p>/g, "—
"); - xml = xml.replace(/--/ug, "
—"); + xml = xml.replace(/-<\/p>/ug, "—
"); + xml = xml.replace(/-■
`); + xml = xml.replace(/■
`); - xml = xml.replace(/⊙<\/strong><\/p>/g, ` ⊙ ⊙<\/strong><\/em><\/p>/g,
- ` ⊙ ⊙⊙<\/strong><\/p>/g,
- ` ⊙ ⊙<\/strong><\/p>/ug,
+ ` ⊙ ⊙<\/strong><\/em><\/p>/ug,
+ ` ⊙ ⊙⊙<\/strong><\/p>/ug,
+ ` ⊙ ⊙ *⊙ *⊙ *⊙ *⊙<\/strong><\/p>/g,
- ` ⊙ ⊙ ⊙ ⊙ ⊙ ⊙ *⊙ *⊙ *⊙ *⊙<\/strong><\/p>/ug,
+ ` ⊙ ⊙ ⊙ ⊙ ⊙ $1 ",
+ "_comment": "These are clearly paragraphs. Also the leading space is strange."
},
{
- "regExp": "\n ([^<]+)( $1 ([^<\n]+) $1 $2 Swearing: antisocial.",
+ "_comment": "This one has the surrounding the Crazed, kooky, cracked, crazy, Crazed, kooky, cracked, crazy, Crazed, kooky, cracked, crazy, Crazed, kooky, cracked, crazy, She "
},
{
@@ -2132,6 +2140,10 @@
}
],
"https://parahumans.wordpress.com/2012/07/26/interlude-12%C2%BD/": [
+ {
+ "before": "implode, it has created a powerful vacuum in someone’s mouth, that",
+ "after": "implode, and it has created a powerful vacuum in someone’s mouth that"
+ },
{
"before": "doesn’t,” Madcap said. “But",
"after": "doesn’t,” Madcap said, “but"
@@ -2177,6 +2189,11 @@
{
"before": "Grue,” Trickster said. “Get",
"after": "Grue,” Trickster said, “get"
+ },
+ {
+ "before": "Nothing she can’t do outside the bubble",
+ "after": "Nothing she can do outside the bubble",
+ "_comment": "This doesn't make sense logically with 'can’t'."
}
],
"https://parahumans.wordpress.com/2012/08/18/snare-13-7/": [
@@ -2308,6 +2325,11 @@
}
],
"https://parahumans.wordpress.com/2012/10/20/colony-15-2/": [
+ {
+ "before": "turned something",
+ "after": "taken something",
+ "_comment": "'turned' is repeated later in the sentence."
+ },
{
"before": "on,” Tattletale said. “Let’s",
"after": "on,” Tattletale said, “let’s"
@@ -2318,6 +2340,11 @@
}
],
"https://parahumans.wordpress.com/2012/10/23/colony-15-3/": [
+ {
+ "before": "whether or not I agreed or not",
+ "after": "whether I agreed or not",
+ "_comment": "Alternatively, the second 'or not' could be deleted instead of the first."
+ },
{
"before": "Woah,” Regent said. “Relax",
"after": "Woah,” Regent said, “relax"
@@ -2329,6 +2356,11 @@
{
"before": "the street
|
\n\\s*",
+ "replacement": "
|
\n([^>\n]+)
\n",
- "replacement": "\n
\nSwearing: antisocial.",
+ "after": "Word choice, ‘too’: haunted by demons.
, unlike the others."
}
],
"https://parahumans.wordpress.com/2012/03/10/extermination-8-3/": [
@@ -1711,12 +1721,6 @@
"after": "Dinah being kidnapped, and leaving"
}
],
- "https://parahumans.wordpress.com/2012/05/22/infestation-11-2/": [
- {
- "before": "attentio n",
- "after": "attention"
- }
- ],
"https://parahumans.wordpress.com/2012/05/26/infestation-11-3/": [
{
"before": "intimidating: A sea",
@@ -1812,6 +1816,10 @@
"before": "Charlotte,” I frowned. “Look",
"after": "Charlotte,” I frowned, “look"
},
+ {
+ "before": "non-sequitor",
+ "after": "non-sequitur"
+ },
{
"before": "Did they… was he",
"after": "Did they… Was he"
@@ -1819,12 +1827,12 @@
{
"before": "see the Doctor",
"after": "see the doctor",
- "comment": "Unlike the Cauldron Doctor, this is not used as a proper noun"
+ "_comment": "Unlike the Cauldron Doctor, this is not used as a proper noun"
},
{
"before": "the Doctor spoke",
"after": "the doctor spoke",
- "comment": "Unlike the Cauldron Doctor, this is not used as a proper noun"
+ "_comment": "Unlike the Cauldron Doctor, this is not used as a proper noun"
},
{
"before": "asinine confession again, “It’s",
@@ -1941,7 +1949,7 @@
"after": "
\nMental, dotty, whacked, loopy…
\nNutty, screwy, mentally diseased…
\n She ",
+ "before": "
\nNutty, screwy, mentally diseased…
\n
\nShe ",
"after": "
\nNutty, screwy, mentally diseased…
⊙
\n\n", + "before": "⊙
\n\n", "after": "", "_comment": "Our usual heuristics of removing the first paragraph to remove the previous/next chapter links are broken here because of the 'go back and look at the fake out' comment at the top" }, @@ -7488,7 +7637,7 @@ "after": "—don’t shoot it!" } ], - "https://www.parahumans.net/2019/09/15/from-within-16-10/": [ + "https://www.parahumans.net/2019/09/14/from-within-16-10/": [ { "before": "carried on. out of", "after": "carried on, out of" @@ -7787,7 +7936,7 @@ }, { "before": "you proved my sister right? Tired,", - "after": "you proved my sister right? Tired," + "after": "you proved my sister right? Tired," }, { "before": "boundary,” He whispered", @@ -8871,7 +9020,7 @@ "after": "the entities" } ], - "https://www.parahumans.net/2020/03/25/last-20-9/": [ + "https://www.parahumans.net/2020/03/24/last-20-9/": [ { "before": "you could help miss—help Antares", "after": "you could help Miss—help Antares" @@ -9072,7 +9221,7 @@ "after": "cross the ocean" } ], - "https://www.parahumans.net/?p=3365&preview=true": [ + "https://www.parahumans.net/2020/05/02/last-20-end/": [ { "before": "saying dad had custody", "after": "saying Dad had custody" diff --git a/lib/worm-scraper.js b/lib/worm-scraper.js index 4a63eef..5f34617 100644 --- a/lib/worm-scraper.js +++ b/lib/worm-scraper.js @@ -14,7 +14,7 @@ const zip = require("./zip.js"); const OUTPUT_DEFAULT = "(Book name).epub"; -const argv = yargs +const { argv } = yargs .usage(`${packageJson.description}\n\n${packageJson.name} [