jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Work on assembling the extras

Browse source
This commit is contained in:
Domenic Denicola 2015-05-09 01:15:22 +02:00
commit a26f622fbb
4 changed files with 119 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.editorconfig
View file

@ -7,3 +7,6 @@ trim_trailing_whitespace = true
charset = utf-8
indent_style = space
indent_size = 2
[out/mimetype]
insert_final_newline = false

2
.gitignore vendored
View file

@ -3,3 +3,5 @@
cache/
out/
!out/mimetime
!out/META-INF/

102
lib/extras.js Normal file
View file

@ -0,0 +1,102 @@
"use strict";
const fs = require("mz/fs");
const path = require("path");
const BOOK_TITLE = "Worm";
const BOOK_AUTHOR = "wildbow";
const BOOK_ID = "urn:uuid:e7f3532d-8db6-4888-be80-1976166b7059";
const NCX_FILENAME = "toc.ncx";
module.exports = function (contentPath, chaptersPath) {
return getChapters(contentPath, chaptersPath).then(function (chapters) {
return Promise.all([
writeOpf(chapters, contentPath),
writeNcx(chapters, contentPath)
]);
});
};
function writeOpf(chapters, contentPath) {
const manifestChapters = chapters.map(function (c) {
return `<item id="${c.id}" href="${c.href}" media-type="application/xhtml+xml"/>`;
}).join("\n");
const spineChapters = chapters.map(function (c) {
return `<itemref idref="${c.id}"/>`;
}).join("\n");
const contents = `<?xml version="1.0"?>
<package version="2.0" xmlns="http://www.idpf.org/2007/opf" unique-identifier="BookId">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
<dc:title>${BOOK_TITLE}</dc:title>
<dc:language>en</dc:language>
<dc:identifier id="BookId" opf:scheme="UUID">${BOOK_ID}</dc:identifier>
<dc:creator opf:file-as="${BOOK_AUTHOR}" opf:role="aut">${BOOK_AUTHOR}</dc:creator>
</metadata>
<manifest>
${manifestChapters}
<item id="ncx" href="${NCX_FILENAME}" media-type="application/x-dtbncx+xml"/>
</manifest>
<spine toc="ncx">
${spineChapters}
</spine>
</package>`;
return fs.writeFile(path.resolve(contentPath, "content.opf"), contents);
}
function writeNcx(chapters, contentPath) {
const navPoints = chapters.map(function (c, i) {
return `<navPoint class="chapter" id="${c.id}" playOrder="${i + 1}">
<navLabel><text>${c.title}</text></navLabel>
<content src="${c.href}"/>
</navPoint>`;
}).join("\n");
const contents = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd">
<ncx version="2005-1" xml:lang="en" xmlns="http://www.daisy.org/z3986/2005/ncx/">
<head>
<meta name="dtb:uid" content="${BOOK_ID}"/>
<meta name="dtb:depth" content="1"/>
<meta name="dtb:totalPageCount" content="0"/>
<meta name="dtb:maxPageNumber" content="0"/>
</head>
<docTitle>
<text>${BOOK_TITLE}</text>
</docTitle>
<docAuthor>
<text>${BOOK_AUTHOR}</text>
</docAuthor>
<navMap>
${navPoints}
</navMap>
</ncx>`;
return fs.writeFile(path.resolve(contentPath, NCX_FILENAME), contents);
}
function getChapters(contentPath, chaptersPath) {
const hrefPrefix = `${path.relative(contentPath, chaptersPath)}/`;
return fs.readdir(chaptersPath).then(function (filenames) {
return filenames.filter(function (f) {
return path.extname(f) === ".xhtml";
})
.sort()
.map(function (f) {
return {
id: path.basename(f),
title: path.basename(f), // TODO extract actual title... inconvenient
href: `${hrefPrefix}${f}`
};
});
});
}

16
lib/worm-scraper.js
View file

@ -5,6 +5,7 @@ const rimraf = require("rimraf-then");
const download = require("./download.js");
const convert = require("./convert.js");
const extras = require("./extras.js");
require("./track-rejections.js");
@ -13,16 +14,23 @@ const START_CHAPTER_URL = "https://parahumans.wordpress.com/2011/06/11/1-1/";
const cachePath = path.resolve("cache");
const outPath = path.resolve("out");
const contentPath = path.resolve(outPath, "OEBPS");
const chaptersPath = path.resolve(contentPath, "chapters");
rimraf(outPath)
Promise.resolve()
.then(function () {
return mkdirp(contentPath);
// return download(START_CHAPTER_URL, cachePath);
})
.then(function () {
return download(START_CHAPTER_URL, cachePath);
// return rimraf(chaptersPath);
})
.then(function () {
return convert(cachePath, contentPath);
// return mkdirp(chaptersPath);
})
.then(function () {
// return convert(cachePath, chaptersPath);
})
.then(function () {
return extras(contentPath, chaptersPath);
})
.then(function () {
console.log("All done!");