jyapayne/ace
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix xml tokenizer tests

Browse source
This commit is contained in:
nightwing 2012-07-07 00:22:02 +04:00
commit 98f3cf04df
2 changed files with 296 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

174
lib/ace/mode/html_highlight_rules_test.js
View file

@ -45,46 +45,146 @@ define(function(require, exports, module) {
var HtmlMode = require("./html").Mode; var HtmlMode = require("./html").Mode;
var assert = require("../test/assertions"); var assert = require("../test/assertions");
var testData = {
"test: tokenize embedded script" : [{
text: "<script a='a'>var</script>'123'",
state: ["start", "start"],
tokens: [{
type: "meta.tag",
value: "<"
}, {
type: "meta.tag.tag-name.script",
value: "script"
}, {
type: "text",
value: " "
}, {
type: "entity.other.attribute-name",
value: "a"
}, {
type: "keyword.operator",
value: "="
}, {
type: "string",
value: "'a'"
}, {
type: "meta.tag",
value: ">"
}, {
type: "storage.type",
value: "var"
}, {
type: "meta.tag",
value: "</"
}, {
type: "meta.tag.tag-name.script",
value: "script"
}, {
type: "meta.tag",
value: ">"
}, {
type: "text",
value: "'123'"
}]
}],
"test: tokenize multiline attribute value with double quotes": [{
text: "<a href=\"abc",
state: [ "start", "tag_qqstring"],
tokens: [{
type: "meta.tag",
value: "<"
}, {
type: "meta.tag.tag-name.anchor",
value: "a"
}, {
type: "text",
value: " "
}, {
type: "entity.other.attribute-name",
value: "href"
}, {
type: "keyword.operator",
value: "="
}, {
type: "string",
value: "\"abc"
}
]
}, {
text: "def\">",
state: [ "tag_qqstring", "start" ],
tokens: [ {
type: "string",
value: "def\""
}, {
type: "meta.tag",
value: ">"
}
]
}],
"test: tokenize multiline attribute value with single quotes": [{
text: "<a href='abc",
state: ["start", "tag_qstring"],
tokens: [{
type: "meta.tag",
value: "<"
}, {
type: "meta.tag.tag-name.anchor",
value: "a"
}, {
type: "text",
value: " "
}, {
type: "entity.other.attribute-name",
value: "href"
}, {
type: "keyword.operator",
value: "="
}, {
type: "string",
value: "'abc"
}
]
}, {
text: "def\"'>",
state: [ "tag_qstring", "start" ],
tokens: [ {
type: "string",
value: "def\"'"
}, {
type: "meta.tag",
value: ">"
}
]
}]
};
function generateTest(exampleData) {
return function testTokenizer() {
for (var i = 0; i < exampleData.length; i++) {
var s = exampleData[i];
var lineTokens = tokenizer.getLineTokens(s.text, s.state[0]);
assert.equal(
JSON.stringify(lineTokens, null, 4),
JSON.stringify({tokens:s.tokens, state: s.state[1]}, null, 4)
);
}
}
}
var tokenizer;
module.exports = { module.exports = {
setUp : function() { setUp : function() {
this.tokenizer = new HtmlMode().getTokenizer(); tokenizer = new HtmlMode().getTokenizer();
}, }
}
"test: tokenize embedded script" : function() {
var line = "<script a='a'>var</script>'123'"; for (var i in testData) {
var tokens = this.tokenizer.getLineTokens(line, "start").tokens; module.exports[i] = generateTest(testData[i])
assert.equal(12, tokens.length);
assert.equal("meta.tag", tokens[0].type);
assert.equal("meta.tag.script", tokens[1].type);
assert.equal("text", tokens[2].type);
assert.equal("entity.other.attribute-name", tokens[3].type);
assert.equal("keyword.operator", tokens[4].type);
assert.equal("string", tokens[5].type);
assert.equal("meta.tag", tokens[6].type);
assert.equal("storage.type", tokens[7].type);
assert.equal("meta.tag", tokens[8].type);
assert.equal("meta.tag.script", tokens[9].type);
assert.equal("meta.tag", tokens[10].type);
assert.equal("text", tokens[11].type);
},
"test: tokenize multiline attribute value with double quotes": function() {
var line1 = this.tokenizer.getLineTokens('<a href="abc', "start");
var t1 = line1.tokens;
var t2 = this.tokenizer.getLineTokens('def">', line1.state).tokens;
assert.equal(t1[t1.length-1].type, "string");
assert.equal(t2[0].type, "string");
},
"test: tokenize multiline attribute value with single quotes": function() {
var line1 = this.tokenizer.getLineTokens("<a href='abc", "start");
var t1 = line1.tokens;
var t2 = this.tokenizer.getLineTokens('def\'>', line1.state).tokens;
assert.equal(t1[t1.length-1].type, "string");
assert.equal(t2[0].type, "string");
} }
};
}); });

202
lib/ace/mode/xml_highlight_rules_test.js
View file

@ -45,57 +45,169 @@ define(function(require, exports, module) {
var XmlMode = require("./xml").Mode; var XmlMode = require("./xml").Mode;
var assert = require("../test/assertions"); var assert = require("../test/assertions");
module.exports = { var testData = {
"test: tokenize1" : [{
text: "<Juhu>//Juhu Kinners</Kinners>",
state: ["start", "start"],
tokens: [
{
type: "meta.tag",
value: "<"
},
{
type: "meta.tag.tag-name",
value: "Juhu"
},
{
type: "meta.tag",
value: ">"
},
{
type: "text",
value: "//Juhu Kinners"
},
{
type: "meta.tag",
value: "</"
},
{
type: "meta.tag.tag-name",
value: "Kinners"
},
{
type: "meta.tag",
value: ">"
}
]
}],
"test: two tags in the same lines should be in separate tokens": [{
text: "<Juhu><Kinners>",
state: [ "start", "start"],
tokens: [
{
type: "meta.tag",
value: "<"
},
{
type: "meta.tag.tag-name",
value: "Juhu"
},
{
type: "meta.tag",
value: ">"
},
{
type: "meta.tag",
value: "<"
},
{
type: "meta.tag.tag-name",
value: "Kinners"
},
{
type: "meta.tag",
value: ">"
}
]
}],
"test: multiline attributes": [{
text: "<copy set=\"{",
state: ["start", "tag_qqstring"],
tokens: [
{
type: "meta.tag",
value: "<"
},
{
type: "meta.tag.tag-name",
value: "copy"
},
{
type: "text",
value: " "
},
{
type: "entity.other.attribute-name",
value: "set"
},
{
type: "keyword.operator",
value: "="
},
{
type: "string",
value: "\"{"
}
]
}, {
text: "}\" undo=\"{",
state: [ "tag_qqstring", "tag_qqstring"],
tokens: [
{
type: "string",
value: "}\""
},
{
type: "text",
value: " "
},
{
type: "entity.other.attribute-name",
value: "undo"
},
{
type: "keyword.operator",
value: "="
},
{
type: "string",
value: "\"{"
}
]
}, {
text: "}\"/>",
state: ["tag_qqstring", "start"],
tokens: [
{
type: "string",
value: "}\""
},
{
type: "meta.tag",
value: "/>"
}
]
}]
};
function generateTest(exampleData) {
return function testTokenizer() {
for (var i = 0; i < exampleData.length; i++) {
var s = exampleData[i];
var lineTokens = tokenizer.getLineTokens(s.text, s.state[0]);
assert.equal(
JSON.stringify(lineTokens, null, 4),
JSON.stringify({tokens:s.tokens, state: s.state[1]}, null, 4)
);
}
}
}
var tokenizer;
module.exports = {
name: "XML Tokenizer", name: "XML Tokenizer",
setUp : function() { setUp : function() {
this.tokenizer = new XmlMode().getTokenizer(); tokenizer = new XmlMode().getTokenizer();
}, }
"test: tokenize1" : function() {
var line = "<Juhu>//Juhu Kinners</Kinners>";
var tokens = this.tokenizer.getLineTokens(line, "start").tokens;
assert.equal(3, tokens.length);
assert.equal("meta.tag", tokens[0].type);
assert.equal("text", tokens[1].type);
assert.equal("meta.tag", tokens[2].type);
},
"test: two tags in the same lines should be in separate tokens" : function() {
var line = "<Juhu><Kinners>";
var tokens = this.tokenizer.getLineTokens(line, "start").tokens;
assert.equal(2, tokens.length);
assert.equal("meta.tag", tokens[0].type);
assert.equal("meta.tag", tokens[1].type);
assert.equal("<Juhu>", tokens[0].value);
assert.equal("<Kinners>", tokens[1].value);
},
"test: multiline attributes": function() {
var multiLine = ['<copy set="{', '}" undo="{', '}"/>'];
var state = "start";
var multiLineTokens = multiLine.map(function(line) {
var tokens = this.tokenizer.getLineTokens(line, state);
state = tokens.state;
return tokens.tokens;
}, this);
assert.equal(multiLineTokens[0].length, 5);
assert.equal(multiLineTokens[1].length, 5);
assert.equal(multiLineTokens[2].length, 2);
assert.equal(multiLineTokens[0][4].type, "string");
assert.equal(multiLineTokens[1][0].type, "string");
assert.equal(multiLineTokens[1][4].type, "string");
assert.equal(multiLineTokens[2][0].type, "string");
} }
};
for (var i in testData) {
module.exports[i] = generateTest(testData[i])
}
}); });
if (typeof module !== "undefined" && module === require.main) { if (typeof module !== "undefined" && module === require.main) {