You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

55 lines
1.3 KiB

var SyntaxError = require('./SyntaxError');
var TAB = 9;
var N = 10;
var F = 12;
var R = 13;
var SPACE = 32;
var Tokenizer = function(str) {
this.str = str;
this.pos = 0;
};
Tokenizer.prototype = {
charCodeAt: function(pos) {
return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
},
charCode: function() {
return this.charCodeAt(this.pos);
},
nextCharCode: function() {
return this.charCodeAt(this.pos + 1);
},
nextNonWsCode: function(pos) {
return this.charCodeAt(this.findWsEnd(pos));
},
findWsEnd: function(pos) {
for (; pos < this.str.length; pos++) {
var code = this.str.charCodeAt(pos);
if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
break;
}
}
return pos;
},
substringToPos: function(end) {
return this.str.substring(this.pos, this.pos = end);
},
eat: function(code) {
if (this.charCode() !== code) {
this.error('Expect `' + String.fromCharCode(code) + '`');
}
this.pos++;
},
peek: function() {
return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
},
error: function(message) {
throw new SyntaxError(message, this.str, this.pos);
}
};
module.exports = Tokenizer;