jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Serialize body as XHTML, not HTML

Browse source
This commit is contained in:
Domenic Denicola 2015-05-09 00:40:54 +02:00
commit 1f241b85ac
2 changed files with 32 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

50
lib/convert.js
View file

@ -3,6 +3,7 @@ const path = require("path");
const fs = require("mz/fs"); const fs = require("mz/fs");
const throat = require("throat"); const throat = require("throat");
const jsdom = require("./jsdom.js"); const jsdom = require("./jsdom.js");
const serializeToXml = require("xmlserializer").serializeToString;
module.exports = function (cachePath, contentPath) { module.exports = function (cachePath, contentPath) {
return getChapterFilePaths(cachePath) return getChapterFilePaths(cachePath)
@ -49,8 +50,9 @@ function convertChapter(filePath, contentPath) {
} }
function getChapterString(rawChapterDoc) { function getChapterString(rawChapterDoc) {
const title = rawChapterDoc.querySelector("h1.entry-title").textContent; const headingEl = rawChapterDoc.querySelector("h1.entry-title");
const body = cleanContentEl(rawChapterDoc.querySelector(".entry-content")); const title = headingEl.textContent;
const body = getBodyXml(headingEl, rawChapterDoc.querySelector(".entry-content"));
return `<?xml version="1.0" encoding="UTF-8" ?> return `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@ -59,31 +61,27 @@ function getChapterString(rawChapterDoc) {
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" /> <meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
<title>${title}</title> <title>${title}</title>
</head> </head>
<body>
<h1>${title}</h1>
${body} ${body}
</body>
</html>`; </html>`;
} }
function cleanContentEl(el) { function getBodyXml(headingEl, contentEl) {
// Remove initial Next Chapter and Previous Chapter <p> // Remove initial Next Chapter and Previous Chapter <p>
el.removeChild(el.firstElementChild); contentEl.removeChild(contentEl.firstElementChild);
// Remove everything after the last <p> (e.g. analytics <div>s) // Remove everything after the last <p> (e.g. analytics <div>s)
const lastP = el.querySelector("p:last-of-type"); const lastP = contentEl.querySelector("p:last-of-type");
while (el.lastElementChild !== lastP) { while (contentEl.lastElementChild !== lastP) {
el.removeChild(el.lastElementChild); contentEl.removeChild(contentEl.lastElementChild);
} }
// Remove empty <p>s or Last Chapter/Next Chapter <p>s // Remove empty <p>s or Last Chapter/Next Chapter <p>s
while (isEmptyOrGarbage(el.lastElementChild)) { while (isEmptyOrGarbage(contentEl.lastElementChild)) {
el.removeChild(el.lastElementChild); contentEl.removeChild(contentEl.lastElementChild);
} }
// Remove redundant dir="ltr" and align="LEFT" and style="text-align: left;" // Remove redundant dir="ltr" and align="LEFT" and style="text-align: left;"
Array.prototype.forEach.call(el.children, function (child) { Array.prototype.forEach.call(contentEl.children, function (child) {
if (child.getAttribute("dir") === "ltr") { if (child.getAttribute("dir") === "ltr") {
child.removeAttribute("dir"); child.removeAttribute("dir");
} }
@ -96,23 +94,35 @@ function cleanContentEl(el) {
}); });
// Remove empty <em>s and <i>s // Remove empty <em>s and <i>s
const ems = el.querySelectorAll("em, i"); const ems = contentEl.querySelectorAll("em, i");
Array.prototype.forEach.call(ems, function (em) { Array.prototype.forEach.call(ems, function (em) {
if (em.textContent.trim() === "") { if (em.textContent.trim() === "") {
em.parentNode.removeChild(em); em.parentNode.removeChild(em);
} }
}); });
let html = el.innerHTML; // Synthesize a <body> tag to serialize
const bodyEl = contentEl.ownerDocument.createElement("body");
const h1El = contentEl.ownerDocument.createElement("h1");
h1El.textContent = headingEl.textContent;
bodyEl.appendChild(h1El);
while (contentEl.firstChild) {
bodyEl.appendChild(contentEl.firstChild);
}
let xml = serializeToXml(bodyEl);
// Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p> // Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p>
html = html.replace(/<br>\s*<\/em><\/p>/g, '</em></p>'); xml = xml.replace(/<br\/>\s*<\/em><\/p>/g, '</em></p>');
// One-off fixes // One-off fixes
html = html.replace(/truck reached<br>\nthe other Nine/, 'truck reached the other Nine'); xml = xml.replace(/truck reached<br\/>\nthe other Nine/, 'truck reached the other Nine');
return html; // Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a <html>
xml = xml.replace(/<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/, '<body>');
return xml;
} }
function isEmptyOrGarbage(el) { function isEmptyOrGarbage(el) {

1
package.json
View file

@ -24,6 +24,7 @@
"requisition": "^1.5.0", "requisition": "^1.5.0",
"rimraf-then": "^1.0.0", "rimraf-then": "^1.0.0",
"throat": "^2.0.2", "throat": "^2.0.2",
"xmlserializer": "^0.3.3",
"xtend": "^4.0.0", "xtend": "^4.0.0",
"zfill": "0.0.2" "zfill": "0.0.2"
}, },