jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Try to parallelize the conversion more

Browse source 
Stalls before it gets to any writing though, it seems.
This commit is contained in:
Domenic Denicola 2015-05-06 23:08:59 +02:00
commit 2236bb2a86
1 changed files with 21 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

35
lib/worm-scraper.js
View file

@ -22,30 +22,37 @@ rimraf(outPath)
.then(function () { .then(function () {
return download(START_CHAPTER_URL, cachePath); return download(START_CHAPTER_URL, cachePath);
}) })
.then(readAllRawChapterDocs) .then(getChapterFilePaths)
.then(function (rawChapterDocs) { .then(function (chapterFilePaths) {
console.log("Extracting content into EPUB chapter files"); console.log("All chapters downloaded; beginning conversion to EPUB chapters");
return Promise.all(rawChapterDocs.map(function (rawChapterDoc, i) { return Promise.all(chapterFilePaths.map(convertChapter));
const output = getChapterString(rawChapterDoc);
const destFilename = path.resolve(contentPath, `chapter${i + 1}.xhtml`);
return fs.writeFile(destFilename, output);
}));
}) })
.then(function () { .then(function () {
console.log("All done!"); console.log("All done!");
}); });
function readAllRawChapterDocs() { function getChapterFilePaths() {
return fs.readdir(cachePath).then(function (filenames) { return fs.readdir(cachePath).then(function (filenames) {
const htmlFiles = filenames.filter(function (f) { return f.endsWith(".html"); }); return filenames.filter(function (f) { return f.endsWith(".html"); })
return Promise.all(htmlFiles.map(readRawChapterDoc)); .map(function (f) { return path.resolve(cachePath, f); });
}); });
} }
function readRawChapterDoc(filename) { function convertChapter(filePath, i) {
const filePath = path.resolve(cachePath, filename); console.log(`- Reading ${filePath}`);
return fs.readFile(filePath, { encoding: "utf-8" }).then(function (contents) { return fs.readFile(filePath, { encoding: "utf-8" }).then(function (contents) {
return jsdom.jsdom(contents); console.log(`- Read ${filePath}`);
const rawChapterDoc = jsdom.jsdom(contents);
const output = getChapterString(rawChapterDoc);
// TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterDoc.defaultView.close();
const destFilename = path.resolve(contentPath, `chapter${i + 1}.xhtml`);
return fs.writeFile(destFilename, output);
})
.then(function () {
console.log(`- Finished converting ${filePath}`);
}); });
} }