jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix warning output

Browse source 
This has been broken since the progress bar addition.
This commit is contained in:
Domenic Denicola 2020-11-07 16:08:25 -05:00
commit b4454f8432
2 changed files with 22 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

26
lib/convert-worker.js
View file

@ -10,29 +10,35 @@ function convertChapter(chapter, book, inputPath, outputPath) {
const contents = fs.readFileSync(inputPath, { encoding: "utf-8" }); const contents = fs.readFileSync(inputPath, { encoding: "utf-8" });
const rawChapterJSDOM = new JSDOM(contents); const rawChapterJSDOM = new JSDOM(contents);
const output = getChapterString(chapter, book, rawChapterJSDOM.window.document); const { output, warnings } = getChapterString(chapter, book, rawChapterJSDOM.window.document);
// TODO: this should probably not be necessary... jsdom bug I guess!? // TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterJSDOM.window.close(); rawChapterJSDOM.window.close();
fs.writeFileSync(outputPath, output); fs.writeFileSync(outputPath, output);
return warnings;
} }
function getChapterString(chapter, book, rawChapterDoc) { function getChapterString(chapter, book, rawChapterDoc) {
const body = getBodyXML(chapter, book, rawChapterDoc.querySelector(".entry-content")); const { xml, warnings } =
getBodyXML(chapter, book, rawChapterDoc.querySelector(".entry-content"));
return `<?xml version="1.0" encoding="UTF-8" ?> const output = `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en"> <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head> <head>
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" /> <meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
<title>${chapter.title}</title> <title>${chapter.title}</title>
</head> </head>
${body} ${xml}
</html>`; </html>`;
return { output, warnings };
} }
function getBodyXML(chapter, book, contentEl) { function getBodyXML(chapter, book, contentEl) {
const warnings = [];
// Remove initial Next Chapter and Previous Chapter <p> // Remove initial Next Chapter and Previous Chapter <p>
contentEl.firstElementChild.remove(); contentEl.firstElementChild.remove();
@ -440,19 +446,19 @@ function getBodyXML(chapter, book, contentEl) {
if (substitution.before) { if (substitution.before) {
const indexOf = xml.indexOf(substitution.before); const indexOf = xml.indexOf(substitution.before);
if (indexOf === -1) { if (indexOf === -1) {
console.warn(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` + warnings.push(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
`updated at the source, in which case, you should edit substitutions.json.`); `updated at the source, in which case, you should edit substitutions.json.`);
} }
if (indexOf !== xml.lastIndexOf(substitution.before)) { if (indexOf !== xml.lastIndexOf(substitution.before)) {
console.warn(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` + warnings.push(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
`Update substitutions.json for a more precise substitution.`); `Update substitutions.json for a more precise substitution.`);
} }
xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after); xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after);
} else if (substitution.regExp) { } else if (substitution.regExp) {
xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement); xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement);
} else { } else {
console.warn(`Invalid substitution specified for ${chapter.url}`); warnings.push(`Invalid substitution specified for ${chapter.url}`);
} }
}); });
@ -462,7 +468,7 @@ function getBodyXML(chapter, book, contentEl) {
/<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/, /<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/,
`<body>\n<!-- ${chapter.url} -->\n`); `<body>\n<!-- ${chapter.url} -->\n`);
return xml; return { xml, warnings };
} }
function isEmptyOrGarbage(el) { function isEmptyOrGarbage(el) {

7
lib/convert.js
View file

@ -21,18 +21,23 @@ module.exports = async (cachePath, manifestPath, contentPath, book, concurrentJo
} }
const pool = workerpool.pool(path.resolve(__dirname, "convert-worker.js"), poolOptions); const pool = workerpool.pool(path.resolve(__dirname, "convert-worker.js"), poolOptions);
const warnings = [];
await Promise.all(chapters.map(async chapter => { await Promise.all(chapters.map(async chapter => {
const inputPath = path.resolve(cachePath, chapter.filename); const inputPath = path.resolve(cachePath, chapter.filename);
const destFileName = `${path.basename(chapter.filename, ".html")}.xhtml`; const destFileName = `${path.basename(chapter.filename, ".html")}.xhtml`;
const outputPath = path.resolve(contentPath, destFileName); const outputPath = path.resolve(contentPath, destFileName);
await pool.exec("convertChapter", [chapter, book, inputPath, outputPath]); warnings.push(...await pool.exec("convertChapter", [chapter, book, inputPath, outputPath]));
progress.increment(); progress.increment();
})); }));
pool.terminate(); pool.terminate();
for (const warning of warnings) {
console.warn(warning);
}
console.log("All chapters converted"); console.log("All chapters converted");
}; };