Restructure to be a bit more efficient

lib/worm-scraper.js

@@ -2,26 +2,25 @@
 const path = require("path");
 const fs = require("mz/fs");
 const mkdirp = require("mkdirp-then");
+const rimraf = require("rimraf-then");
 const request = require("requisition");
 const jsdom = require("jsdom");
 
 require("./track-rejections.js");
 
-const START_CHAPTER = "http://parahumans.wordpress.com/category/stories-arcs-1-10/arc-1-gestation/1-01/";
+const START_CHAPTER = "https://parahumans.wordpress.com/2011/06/11/1-1/";
 
 const cachePath = path.resolve("cache");
 const outPath = path.resolve("out");
 const contentPath = path.resolve(outPath, "OEBPS");
 
-let chapters;
+const rawChapterDocsCache = new Map();
-getChapters()
+
-  .then(function (theChapters) {
+rimraf(outPath)
-    chapters = theChapters;
+  .then(function () {
     return mkdirp(contentPath);
   })
-  .then(function () {
+  .then(getAllRawChapterDocs)
-    return Promise.all(chapters.map(getRawChapterDoc));
-  })
   .then(function (rawChapterDocs) {
     console.log("Extracting content into EPUB chapter files");
     return Promise.all(rawChapterDocs.map(function (rawChapterDoc, i) {
@@ -34,54 +33,71 @@ getChapters()
     console.log("All done!");
   });
 
-function getChapters() {
+function getAllRawChapterDocs() {
-  return fs.readdir(cachePath).catch(function (e) {
+  return fs.readdir(cachePath).then(
-    if (e.code === "ENOENT") {
+    function (filenames) {
-      let currentChapter = START_CHAPTER;
+      return filenames.map(getRawChapterDoc);
-      let chapterCounter = 1;
+    },
-      const filenames = [];
+    function (e) {
-
+      if (e.code === "ENOENT") {
-      return loop().then(function () {
+        return downloadAllChapters();
-        return filenames;
+      } else {
-      });
+        throw e;
-
-      function loop() {
-        const filename = `chapter${chapterCounter}.html`;
-
-        console.log(`Downloading ${currentChapter}`);
-
-        return request(currentChapter).redirects(10).then(function (response) {
-          return response.saveTo(path.resolve(cachePath, filename));
-        })
-        .then(function () {
-          filenames.push(filename);
-
-          // This is inefficient in a number of ways; whatever.
-          return getRawChapterDoc(filename);
-        })
-        .then(function (rawChapterDoc) {
-          currentChapter = getNextChapterUrl(rawChapterDoc);
-
-          if (currentChapter === null) {
-            return;
-          }
-
-          ++chapterCounter;
-          return loop();
-        });
       }
-    } else {
-      throw e;
     }
+  );
+}
+
+function downloadAllChapters() {
+  let currentChapter = START_CHAPTER;
+  let chapterCounter = 1;
+
+  return mkdirp(cachePath).then(loop).then(function () {
+    return toArray(rawChapterDocsCache.values());
   });
+
+  function loop() {
+    const filename = `chapter${chapterCounter}.html`;
+
+    console.log(`Downloading ${currentChapter}`);
+
+    return request(currentChapter).redirects(10).then(function (response) {
+      return response.text();
+    })
+    .then(function (contents) {
+      const rawChapterDoc = setRawChapterDoc(filename, contents);
+      currentChapter = getNextChapterUrl(rawChapterDoc);
+
+      return fs.writeFile(path.resolve(cachePath, filename), jsdom.serializeDocument(rawChapterDoc));
+    })
+    .then(function () {
+      if (currentChapter === null) {
+        return;
+      }
+
+      ++chapterCounter;
+      return loop();
+    });
+  }
 }
 
 function getRawChapterDoc(filename) {
+  const doc = rawChapterDocsCache.get(filename);
+  if (doc !== undefined) {
+    return Promise.resolve(doc);
+  }
+
   return fs.readFile(path.resolve(cachePath, filename), { encoding: "utf-8" }).then(function (contents) {
-    return jsdom.jsdom(contents);
+    return setRawChapterDoc(filename, contents);
   });
 }
 
+function setRawChapterDoc(filename, contents) {
+  const doc = jsdom.jsdom(contents);
+  rawChapterDocsCache.set(filename, doc);
+  return doc;
+}
+
 function getChapterString(rawChapterDoc) {
   const title = rawChapterDoc.querySelector("h1.entry-title").textContent;
   const body = cleanContentEl(rawChapterDoc.querySelector(".entry-content")).innerHTML;
@@ -129,3 +145,11 @@ function getNextChapterUrl(rawChapterDoc) {
 
   return null;
 }
+
+function toArray(iterable) {
+  const array = [];
+  for (const x of iterable) {
+    array.push(x);
+  }
+  return array;
+}

package.json

@@ -21,7 +21,8 @@
   "dependencies": {
     "jsdom": "^5.3.0",
     "mkdirp-then": "^1.0.1",
-    "requisition": "^1.5.0"
+    "requisition": "^1.5.0",
+    "rimraf-then": "^1.0.0"
   },
   "devDependencies": {
     "eslint": "0.20.0",