jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update conversion to work with cache manifest

Browse source
This commit is contained in:
Domenic Denicola 2015-05-11 21:14:40 -04:00
commit cf6b5c9ab9
4 changed files with 23 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

41
lib/convert.js
View file

@ -4,39 +4,31 @@ const fs = require("mz/fs");
const throat = require("throat");
const serializeToXml = require("xmlserializer").serializeToString;
const jsdom = require("./jsdom.js");
const getChapterTitle = require("./common.js").getChapterTitle;
module.exports = function (cachePath, contentPath) {
return getChapterFilePaths(cachePath)
.then(function (chapterFilePaths) {
console.log("All chapters downloaded; beginning conversion to EPUB chapters");
module.exports = function (cachePath, manifestPath, contentPath) {
return fs.readFile(manifestPath, { encoding: "utf-8" }).then(function (manifestContents) {
const chapters = JSON.parse(manifestContents);
console.log("All chapters downloaded; beginning conversion to EPUB chapters");
const mapper = throat(10, function (filePath) {
return convertChapter(filePath, contentPath);
});
return Promise.all(chapterFilePaths.map(mapper));
})
.then(function () {
console.log("All chapters converted");
const mapper = throat(10, function (chapter) {
return convertChapter(chapter, cachePath, contentPath);
});
return Promise.all(chapters.map(mapper));
})
.then(function () {
console.log("All chapters converted");
});
};
function getChapterFilePaths(cachePath) {
return fs.readdir(cachePath).then(function (filenames) {
return filenames.filter(function (f) { return f.endsWith(".html"); })
.map(function (f) { return path.resolve(cachePath, f); });
});
}
function convertChapter(filePath, contentPath) {
const filename = path.basename(filePath);
function convertChapter(chapter, cachePath, contentPath) {
const filename = chapter.filename;
const filePath = path.resolve(cachePath, filename);
console.log(`- Reading ${filename}`);
return fs.readFile(filePath, { encoding: "utf-8" }).then(function (contents) {
console.log(`- Read ${filename}`);
const rawChapterDoc = jsdom(contents);
const output = getChapterString(rawChapterDoc);
const output = getChapterString(chapter.title, rawChapterDoc);
// TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterDoc.defaultView.close();
@ -50,8 +42,7 @@ function convertChapter(filePath, contentPath) {
});
}
function getChapterString(rawChapterDoc) {
const title = getChapterTitle(rawChapterDoc);
function getChapterString(title, rawChapterDoc) {
const body = getBodyXml(title, rawChapterDoc.querySelector(".entry-content"));
return `<?xml version="1.0" encoding="UTF-8" ?>