jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Throttle conversions; move to a separate file

Browse source
This commit is contained in:
Domenic Denicola 2015-05-07 23:56:53 +02:00
commit 8bb41473d2
3 changed files with 87 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

83
lib/convert.js Normal file
View file

@ -0,0 +1,83 @@
"use strict";
const path = require("path");
const fs = require("mz/fs");
const throat = require("throat");
const jsdom = require("./jsdom.js");
module.exports = function (cachePath, contentPath) {
return getChapterFilePaths(cachePath)
.then(function (chapterFilePaths) {
console.log("All chapters downloaded; beginning conversion to EPUB chapters");
const mapper = throat(10, function (filePath) {
return convertChapter(filePath, contentPath);
});
return Promise.all(chapterFilePaths.map(mapper));
})
.then(function () {
console.log("All chapters converted");
});
};
function getChapterFilePaths(cachePath) {
return fs.readdir(cachePath).then(function (filenames) {
return filenames.filter(function (f) { return f.endsWith(".html"); })
.map(function (f) { return path.resolve(cachePath, f); });
});
}
function convertChapter(filePath, contentPath) {
const filename = path.basename(filePath);
console.log(`- Reading ${filename}`);
return fs.readFile(filePath, { encoding: "utf-8" }).then(function (contents) {
console.log(`- Read ${filename}`);
const rawChapterDoc = jsdom(contents);
const output = getChapterString(rawChapterDoc);
// TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterDoc.defaultView.close();
const destFileName = `${path.basename(filename, ".html")}.xhtml`;
const destFilePath = path.resolve(contentPath, destFileName);
return fs.writeFile(destFilePath, output);
})
.then(function () {
console.log(`- Finished converting ${filename}`);
});
}
function getChapterString(rawChapterDoc) {
const title = rawChapterDoc.querySelector("h1.entry-title").textContent;
const body = cleanContentEl(rawChapterDoc.querySelector(".entry-content")).innerHTML;
return `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
<title>${title}</title>
</head>
<body>
<h1>${title}</h1>
${body}
</body>
</html>`;
}
function cleanContentEl(el) {
// Remove Next Chapter and Previous Chapter <p>s
el.removeChild(el.firstElementChild);
el.removeChild(el.lastElementChild);
// Remove redundant dir="ltr"
Array.prototype.forEach.call(el.children, function (child) {
if (child.getAttribute("dir") === "ltr") {
child.removeAttribute("dir");
}
});
return el;
}

71
lib/worm-scraper.js
View file

@ -1,11 +1,10 @@
"use strict";
const path = require("path");
const fs = require("mz/fs");
const mkdirp = require("mkdirp-then");
const rimraf = require("rimraf-then");
const jsdom = require("./jsdom.js");
const download = require("./download.js");
const convert = require("./convert.js");
require("./track-rejections.js");
@ -22,73 +21,9 @@ rimraf(outPath)
.then(function () {
return download(START_CHAPTER_URL, cachePath);
})
.then(getChapterFilePaths)
.then(function (chapterFilePaths) {
console.log("All chapters downloaded; beginning conversion to EPUB chapters");
return Promise.all(chapterFilePaths.map(convertChapter));
.then(function () {
return convert(cachePath, contentPath);
})
.then(function () {
console.log("All done!");
});
function getChapterFilePaths() {
return fs.readdir(cachePath).then(function (filenames) {
return filenames.filter(function (f) { return f.endsWith(".html"); })
.map(function (f) { return path.resolve(cachePath, f); });
});
}
function convertChapter(filePath) {
const filename = path.basename(filePath);
console.log(`- Reading ${filename}`);
return fs.readFile(filePath, { encoding: "utf-8" }).then(function (contents) {
console.log(`- Read ${filename}`);
const rawChapterDoc = jsdom(contents);
const output = getChapterString(rawChapterDoc);
// TODO: this should probably not be necessary... jsdom bug I guess!?
rawChapterDoc.defaultView.close();
const destFileName = `${path.basename(filename, ".html")}.xhtml`;
const destFilePath = path.resolve(contentPath, destFileName);
return fs.writeFile(destFilePath, output);
})
.then(function () {
console.log(`- Finished converting ${filename}`);
});
}
function getChapterString(rawChapterDoc) {
const title = rawChapterDoc.querySelector("h1.entry-title").textContent;
const body = cleanContentEl(rawChapterDoc.querySelector(".entry-content")).innerHTML;
return `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
<title>${title}</title>
</head>
<body>
<h1>${title}</h1>
${body}
</body>
</html>`;
}
function cleanContentEl(el) {
// Remove Next Chapter and Previous Chapter <p>s
el.removeChild(el.firstElementChild);
el.removeChild(el.lastElementChild);
// Remove redundant dir="ltr"
Array.prototype.forEach.call(el.children, function (child) {
if (child.getAttribute("dir") === "ltr") {
child.removeAttribute("dir");
}
});
return el;
}

1
package.json
View file

@ -23,6 +23,7 @@
"mkdirp-then": "^1.0.1",
"requisition": "^1.5.0",
"rimraf-then": "^1.0.0",
"throat": "^2.0.2",
"xtend": "^4.0.0",
"zfill": "0.0.2"
},