jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix warning output

Browse source 
This has been broken since the progress bar addition.
This commit is contained in:
Domenic Denicola 2020-11-07 16:08:25 -05:00
commit b4454f8432
2 changed files with 22 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

26
lib/convert-worker.js
View file

@ -10,29 +10,35 @@ function convertChapter(chapter, book, inputPath, outputPath) {
const contents = fs.readFileSync(inputPath, { encoding: "utf-8" });
const rawChapterJSDOM = new JSDOM(contents);
const output = getChapterString(chapter, book, rawChapterJSDOM.window.document);
const { output, warnings } = getChapterString(chapter, book, rawChapterJSDOM.window.document);
// TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterJSDOM.window.close();
fs.writeFileSync(outputPath, output);
return warnings;
}
function getChapterString(chapter, book, rawChapterDoc) {
const body = getBodyXML(chapter, book, rawChapterDoc.querySelector(".entry-content"));
const { xml, warnings } =
getBodyXML(chapter, book, rawChapterDoc.querySelector(".entry-content"));
return `<?xml version="1.0" encoding="UTF-8" ?>
const output = `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
<title>${chapter.title}</title>
</head>
${body}
${xml}
</html>`;
return { output, warnings };
}
function getBodyXML(chapter, book, contentEl) {
const warnings = [];
// Remove initial Next Chapter and Previous Chapter <p>
contentEl.firstElementChild.remove();
@ -440,19 +446,19 @@ function getBodyXML(chapter, book, contentEl) {
if (substitution.before) {
const indexOf = xml.indexOf(substitution.before);
if (indexOf === -1) {
console.warn(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
`updated at the source, in which case, you should edit substitutions.json.`);
warnings.push(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
`updated at the source, in which case, you should edit substitutions.json.`);
}
if (indexOf !== xml.lastIndexOf(substitution.before)) {
console.warn(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
`Update substitutions.json for a more precise substitution.`);
warnings.push(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
`Update substitutions.json for a more precise substitution.`);
}
xml = xml.replace(new RegExp(escapeRegExp(substitution.before)), substitution.after);
} else if (substitution.regExp) {
xml = xml.replace(new RegExp(substitution.regExp, "g"), substitution.replacement);
} else {
console.warn(`Invalid substitution specified for ${chapter.url}`);
warnings.push(`Invalid substitution specified for ${chapter.url}`);
}
});
@ -462,7 +468,7 @@ function getBodyXML(chapter, book, contentEl) {
/<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/,
`<body>\n<!-- ${chapter.url} -->\n`);
return xml;
return { xml, warnings };
}
function isEmptyOrGarbage(el) {

7
lib/convert.js
View file

@ -21,18 +21,23 @@ module.exports = async (cachePath, manifestPath, contentPath, book, concurrentJo
}
const pool = workerpool.pool(path.resolve(__dirname, "convert-worker.js"), poolOptions);
const warnings = [];
await Promise.all(chapters.map(async chapter => {
const inputPath = path.resolve(cachePath, chapter.filename);
const destFileName = `${path.basename(chapter.filename, ".html")}.xhtml`;
const outputPath = path.resolve(contentPath, destFileName);
await pool.exec("convertChapter", [chapter, book, inputPath, outputPath]);
warnings.push(...await pool.exec("convertChapter", [chapter, book, inputPath, outputPath]));
progress.increment();
}));
pool.terminate();
for (const warning of warnings) {
console.warn(warning);
}
console.log("All chapters converted");
};