diff --git a/lib/Tokenizer.js b/lib/Tokenizer.js index 01dc3d2..0bd78b5 100644 --- a/lib/Tokenizer.js +++ b/lib/Tokenizer.js @@ -2,7 +2,6 @@ var EventEmitter = require('events').EventEmitter; var util = require('util'); var assert = require('assert'); var Transform = require('stream').Transform; -var disect = require('disect'); function noop(){} @@ -58,10 +57,11 @@ Tokenizer.prototype._tokenize = function _tokenize(data, nobuffer) { } var self = this; - var maxIndex = disect(0, data.length, function (index) { - var buf = data.substring(0, index + 1); - return self._getMatchingRule(buf) === null; - }); + + for (var maxIndex = data.length; maxIndex > 0; maxIndex--) { + var buf = data.substr(0,maxIndex); + if (self._getMatchingRule(buf)) break; + } if(maxIndex === 0) { // no match found diff --git a/package.json b/package.json index ea984c2..ca7d406 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,5 @@ "nodeunit": "~0.8.1" }, "dependencies": { - "disect": "~1.1.0" } } diff --git a/test/test-tokenizer.js b/test/test-tokenizer.js index 5e3e3c3..548a9a9 100644 --- a/test/test-tokenizer.js +++ b/test/test-tokenizer.js @@ -174,3 +174,28 @@ exports['words in two chunks'] = function(test) { t.write('Hell'); t.end('o World'); }.withDomain(); + +exports["quoted string"] = function(test) { + var t = tokenizer(); + t.addRule(/^\#.*?$/, "comment"); + t.addRule('whitespace'); + t.addRule(/^\d+$/, 'integer'); + t.addRule(/^".*?"$/, "quoted-string"); + t.addRule(/^\;$/, 'end-statement'); + t.addRule(/^\=$/, 'assignment'); + t.addRule(/^(int16|int32|string)$/, 'data-type'); + t.addRule(/^[a-zA-Z0-9_]+$/, 'symbol'); + t.ignore(['comment','whitespace']); + + var idx = 0; + var expect = ['data-type','symbol','assignment','integer','end-statement','data-type','symbol','assignment','quoted-string','end-statement']; + var values = ['int16','ii','=','12345678901234567890',';','string','a1','=','"some long value for the quoted string"', ';']; + t.on('data', function(token) { + test.equal(expect[idx], token.type); + test.equal(values[idx], token.content); + idx += 1; + }); + t.on('end', test.done.bind(test)); + t.write('#this is a commment\n\n\tint16 ii = 12345678901234567890;\nstring a1 = "some long value for the quoted string";'); + t.end(); +}.withDomain();