Restructure to be a bit more efficient

lib/worm-scraper.js

@@ -2,26 +2,25 @@
 const path = require("path");
 const fs = require("mz/fs");
 const mkdirp = require("mkdirp-then");
+const rimraf = require("rimraf-then");
 const request = require("requisition");
 const jsdom = require("jsdom");
 
 require("./track-rejections.js");
 
-const START_CHAPTER = "http://parahumans.wordpress.com/category/stories-arcs-1-10/arc-1-gestation/1-01/";
+const START_CHAPTER = "https://parahumans.wordpress.com/2011/06/11/1-1/";
 
 const cachePath = path.resolve("cache");
 const outPath = path.resolve("out");
 const contentPath = path.resolve(outPath, "OEBPS");
 
-let chapters;
+const rawChapterDocsCache = new Map();
-getChapters()
+
-  .then(function (theChapters) {
+rimraf(outPath)
-    chapters = theChapters;
+  .then(function () {
     return mkdirp(contentPath);
   })
-  .then(function () {
+  .then(getAllRawChapterDocs)
-    return Promise.all(chapters.map(getRawChapterDoc));
-  })
   .then(function (rawChapterDocs) {
     console.log("Extracting content into EPUB chapter files");
     return Promise.all(rawChapterDocs.map(function (rawChapterDoc, i) {
@@ -34,15 +33,27 @@ getChapters()
     console.log("All done!");
   });
 
-function getChapters() {
+function getAllRawChapterDocs() {
-  return fs.readdir(cachePath).catch(function (e) {
+  return fs.readdir(cachePath).then(
+    function (filenames) {
+      return filenames.map(getRawChapterDoc);
+    },
+    function (e) {
       if (e.code === "ENOENT") {
+        return downloadAllChapters();
+      } else {
+        throw e;
+      }
+    }
+  );
+}
+
+function downloadAllChapters() {
   let currentChapter = START_CHAPTER;
   let chapterCounter = 1;
-      const filenames = [];
 
-      return loop().then(function () {
+  return mkdirp(cachePath).then(loop).then(function () {
-        return filenames;
+    return toArray(rawChapterDocsCache.values());
   });
 
   function loop() {
@@ -51,17 +62,15 @@ function getChapters() {
     console.log(`Downloading ${currentChapter}`);
 
     return request(currentChapter).redirects(10).then(function (response) {
-          return response.saveTo(path.resolve(cachePath, filename));
+      return response.text();
     })
-        .then(function () {
+    .then(function (contents) {
-          filenames.push(filename);
+      const rawChapterDoc = setRawChapterDoc(filename, contents);
-
-          // This is inefficient in a number of ways; whatever.
-          return getRawChapterDoc(filename);
-        })
-        .then(function (rawChapterDoc) {
       currentChapter = getNextChapterUrl(rawChapterDoc);
 
+      return fs.writeFile(path.resolve(cachePath, filename), jsdom.serializeDocument(rawChapterDoc));
+    })
+    .then(function () {
       if (currentChapter === null) {
         return;
       }
@@ -70,18 +79,25 @@ function getChapters() {
       return loop();
     });
   }
-    } else {
-      throw e;
-    }
-  });
 }
 
 function getRawChapterDoc(filename) {
+  const doc = rawChapterDocsCache.get(filename);
+  if (doc !== undefined) {
+    return Promise.resolve(doc);
+  }
+
   return fs.readFile(path.resolve(cachePath, filename), { encoding: "utf-8" }).then(function (contents) {
-    return jsdom.jsdom(contents);
+    return setRawChapterDoc(filename, contents);
   });
 }
 
+function setRawChapterDoc(filename, contents) {
+  const doc = jsdom.jsdom(contents);
+  rawChapterDocsCache.set(filename, doc);
+  return doc;
+}
+
 function getChapterString(rawChapterDoc) {
   const title = rawChapterDoc.querySelector("h1.entry-title").textContent;
   const body = cleanContentEl(rawChapterDoc.querySelector(".entry-content")).innerHTML;
@@ -129,3 +145,11 @@ function getNextChapterUrl(rawChapterDoc) {
 
   return null;
 }
+
+function toArray(iterable) {
+  const array = [];
+  for (const x of iterable) {
+    array.push(x);
+  }
+  return array;
+}

package.json

@@ -21,7 +21,8 @@
   "dependencies": {
     "jsdom": "^5.3.0",
     "mkdirp-then": "^1.0.1",
-    "requisition": "^1.5.0"
+    "requisition": "^1.5.0",
+    "rimraf-then": "^1.0.0"
   },
   "devDependencies": {
     "eslint": "0.20.0",