jyapayne/worm-scraper
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Serialize body as XHTML, not HTML

Browse source
This commit is contained in:
Domenic Denicola 2015-05-09 00:40:54 +02:00
commit 1f241b85ac
2 changed files with 32 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

52
lib/convert.js
View file

@ -3,6 +3,7 @@ const path = require("path");
const fs = require("mz/fs");
const throat = require("throat");
const jsdom = require("./jsdom.js");
const serializeToXml = require("xmlserializer").serializeToString;
module.exports = function (cachePath, contentPath) {
return getChapterFilePaths(cachePath)
@ -49,8 +50,9 @@ function convertChapter(filePath, contentPath) {
}
function getChapterString(rawChapterDoc) {
const title = rawChapterDoc.querySelector("h1.entry-title").textContent;
const body = cleanContentEl(rawChapterDoc.querySelector(".entry-content"));
const headingEl = rawChapterDoc.querySelector("h1.entry-title");
const title = headingEl.textContent;
const body = getBodyXml(headingEl, rawChapterDoc.querySelector(".entry-content"));
return `<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
@ -59,31 +61,27 @@ function getChapterString(rawChapterDoc) {
<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=utf-8" />
<title>${title}</title>
</head>
<body>
<h1>${title}</h1>
${body}
</body>
${body}
</html>`;
}
function cleanContentEl(el) {
function getBodyXml(headingEl, contentEl) {
// Remove initial Next Chapter and Previous Chapter <p>
el.removeChild(el.firstElementChild);
contentEl.removeChild(contentEl.firstElementChild);
// Remove everything after the last <p> (e.g. analytics <div>s)
const lastP = el.querySelector("p:last-of-type");
while (el.lastElementChild !== lastP) {
el.removeChild(el.lastElementChild);
const lastP = contentEl.querySelector("p:last-of-type");
while (contentEl.lastElementChild !== lastP) {
contentEl.removeChild(contentEl.lastElementChild);
}
// Remove empty <p>s or Last Chapter/Next Chapter <p>s
while (isEmptyOrGarbage(el.lastElementChild)) {
el.removeChild(el.lastElementChild);
while (isEmptyOrGarbage(contentEl.lastElementChild)) {
contentEl.removeChild(contentEl.lastElementChild);
}
// Remove redundant dir="ltr" and align="LEFT" and style="text-align: left;"
Array.prototype.forEach.call(el.children, function (child) {
Array.prototype.forEach.call(contentEl.children, function (child) {
if (child.getAttribute("dir") === "ltr") {
child.removeAttribute("dir");
}
@ -96,23 +94,35 @@ function cleanContentEl(el) {
});
// Remove empty <em>s and <i>s
const ems = el.querySelectorAll("em, i");
const ems = contentEl.querySelectorAll("em, i");
Array.prototype.forEach.call(ems, function (em) {
if (em.textContent.trim() === "") {
em.parentNode.removeChild(em);
}
});
let html = el.innerHTML;
// Synthesize a <body> tag to serialize
const bodyEl = contentEl.ownerDocument.createElement("body");
const h1El = contentEl.ownerDocument.createElement("h1");
h1El.textContent = headingEl.textContent;
bodyEl.appendChild(h1El);
while (contentEl.firstChild) {
bodyEl.appendChild(contentEl.firstChild);
}
let xml = serializeToXml(bodyEl);
// Fix recurring strange pattern of extra <br> in <p>...<em>...<br>\n</em></p>
html = html.replace(/<br>\s*<\/em><\/p>/g, '</em></p>');
xml = xml.replace(/<br\/>\s*<\/em><\/p>/g, '</em></p>');
// One-off fixes
html = html.replace(/truck reached<br>\nthe other Nine/, 'truck reached the other Nine');
xml = xml.replace(/truck reached<br\/>\nthe other Nine/, 'truck reached the other Nine');
return html;
// Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a <html>
xml = xml.replace(/<body xmlns="http:\/\/www.w3.org\/1999\/xhtml">/, '<body>');
return xml;
}
function isEmptyOrGarbage(el) {

1
package.json
View file

@ -24,6 +24,7 @@
"requisition": "^1.5.0",
"rimraf-then": "^1.0.0",
"throat": "^2.0.2",
"xmlserializer": "^0.3.3",
"xtend": "^4.0.0",
"zfill": "0.0.2"
},