jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add --jobs option for the convert command

Browse source 
This doesn't seem that helpful as there seems to be a lot of sequential work going on regardless.
This commit is contained in:
Domenic Denicola 2019-08-16 23:37:00 -04:00
commit 4386d48021
2 changed files with 10 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/convert.js
View file

@ -6,12 +6,12 @@ const serializeToXML = require("xmlserializer").serializeToString;
const { JSDOM } = require("jsdom"); const { JSDOM } = require("jsdom");
const substitutions = require("./substitutions.json"); const substitutions = require("./substitutions.json");
module.exports = async (cachePath, manifestPath, contentPath) => { module.exports = async (cachePath, manifestPath, contentPath, concurrentJobs) => {
const manifestContents = await fs.readFile(manifestPath, { encoding: "utf-8" }); const manifestContents = await fs.readFile(manifestPath, { encoding: "utf-8" });
const chapters = JSON.parse(manifestContents); const chapters = JSON.parse(manifestContents);
console.log("All chapters downloaded; beginning conversion to EPUB chapters"); console.log("All chapters downloaded; beginning conversion to EPUB chapters");
const mapper = throat(10, chapter => convertChapter(chapter, cachePath, contentPath)); const mapper = throat(concurrentJobs, chapter => convertChapter(chapter, cachePath, contentPath));
await Promise.all(chapters.map(mapper)); await Promise.all(chapters.map(mapper));
console.log("All chapters converted"); console.log("All chapters converted");

9
lib/worm-scraper.js
View file

@ -48,6 +48,13 @@ const argv = yargs
requiresArg: true, requiresArg: true,
global: true global: true
}) })
.option("j", {
alias: "jobs",
default: 10,
describe: "Number of concurrent read/write jobs to run, for the convert command",
requiresArg: true,
global: true
})
.demandCommand(1) // TODO remove and allow all .demandCommand(1) // TODO remove and allow all
.recommendCommands() .recommendCommands()
.help() .help()
@ -72,7 +79,7 @@ if (argv._.includes("convert")) {
commands.push(() => { commands.push(() => {
return rimraf(chaptersPath) return rimraf(chaptersPath)
.then(() => mkdirp(chaptersPath)) .then(() => mkdirp(chaptersPath))
.then(() => convert(cachePath, manifestPath, chaptersPath)); .then(() => convert(cachePath, manifestPath, chaptersPath, argv.jobs));
}); });
} }