From 05f18c96192263edaa3c064ee63d4e5459a41d97 Mon Sep 17 00:00:00 2001 From: kettek Date: Tue, 25 Nov 2025 00:01:05 -0800 Subject: [PATCH 1/5] Add support for negative sign numbers --- example_test.go | 4 ++-- parser.go | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/example_test.go b/example_test.go index e95d263..4d3c535 100644 --- a/example_test.go +++ b/example_test.go @@ -122,11 +122,11 @@ func (parser *jsonParser) error(stream *Stream) error { func TestJsonParser(t *testing.T) { parser := newJSONParser() - data, err := parser.Parse([]byte(`{"one": 1, "two": "three", "four": [5, "six", 7.8, {}]}`)) + data, err := parser.Parse([]byte(`{"one": 1, "two": "three", "four": [5, "six", 7.8, -9, -99.9, {}]}`)) require.NoError(t, err) require.Equal(t, map[string]interface{}{ "one": int64(1), "two": "three", - "four": []interface{}{int64(5), "six", 7.8, map[string]interface{}{}}, + "four": []interface{}{int64(5), "six", 7.8, int64(-9), -99.9, map[string]interface{}{}}, }, data) } diff --git a/parser.go b/parser.go index 003608e..02ba810 100644 --- a/parser.go +++ b/parser.go @@ -315,6 +315,14 @@ func (p *parsing) parseNumber() bool { } else { break } + } else if !hasExp && p.curr == '-' { // Allow starting numbers from a negative sign. + if isNumberByte(nextByte) { + if start == -1 { + start = p.pos + } + } else { + break + } } else { break } From 4ceba5f1b6d3d1f2d4333fc5ba3d5a9c273f48d5 Mon Sep 17 00:00:00 2001 From: kettek Date: Tue, 25 Nov 2025 00:03:27 -0800 Subject: [PATCH 2/5] Prevent parsing multiple negative signs in a number --- parser.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parser.go b/parser.go index 02ba810..ffd7195 100644 --- a/parser.go +++ b/parser.go @@ -319,6 +319,8 @@ func (p *parsing) parseNumber() bool { if isNumberByte(nextByte) { if start == -1 { start = p.pos + } else { + break } } else { break From ee2a4ae6a06fa0c877f09a2aa8fb7d83f3474c37 Mon Sep 17 00:00:00 2001 From: kettek Date: Tue, 25 Nov 2025 00:24:02 -0800 Subject: [PATCH 3/5] Change module to use kettek; temporary for now --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3b3570e..bb093e8 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/bzick/tokenizer +module github.com/kettek/tokenizer go 1.13 From 6122091d6a49206469aad5f60a841632981659ba Mon Sep 17 00:00:00 2001 From: kettek Date: Fri, 28 Nov 2025 18:25:42 -0800 Subject: [PATCH 4/5] Increase negative number test cases --- example_test.go | 4 +++- tokenizer_test.go | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/example_test.go b/example_test.go index 4d3c535..c041ace 100644 --- a/example_test.go +++ b/example_test.go @@ -122,11 +122,13 @@ func (parser *jsonParser) error(stream *Stream) error { func TestJsonParser(t *testing.T) { parser := newJSONParser() - data, err := parser.Parse([]byte(`{"one": 1, "two": "three", "four": [5, "six", 7.8, -9, -99.9, {}]}`)) + data, err := parser.Parse([]byte(`{"one": 1, "two": "three", "four": [5, "six", 7.8, -9, -99.9, {}], "five-5": "5-five", "6-six": "six-6"}`)) require.NoError(t, err) require.Equal(t, map[string]interface{}{ "one": int64(1), "two": "three", "four": []interface{}{int64(5), "six", 7.8, int64(-9), -99.9, map[string]interface{}{}}, + "five-5": "5-five", + "6-six": "six-6", }, data) } diff --git a/tokenizer_test.go b/tokenizer_test.go index 55eeadf..942dbc8 100644 --- a/tokenizer_test.go +++ b/tokenizer_test.go @@ -46,6 +46,7 @@ func TestTokenize(t *testing.T) { t.Run("integers", func(t *testing.T) { integers := []item{ {int64(1), Token{key: TokenInteger, value: []byte("1")}}, + {int64(-1), Token{key: TokenInteger, value: []byte("-1")}}, {int64(123456), Token{key: TokenInteger, value: []byte("123456")}}, {int64(123456), Token{key: TokenInteger, value: []byte("123_456")}}, } @@ -63,10 +64,12 @@ func TestTokenize(t *testing.T) { {2.3, Token{key: TokenFloat, value: []byte("2.3")}}, {2.0, Token{key: TokenFloat, value: []byte("2.")}}, {0.2, Token{key: TokenFloat, value: []byte(".2")}}, + {-2.3, Token{key: TokenFloat, value: []byte("-2.3")}}, {2.3e4, Token{key: TokenFloat, value: []byte("2.3e4")}}, {2.3e-4, Token{key: TokenFloat, value: []byte("2.3e-4")}}, {2.3e+4, Token{key: TokenFloat, value: []byte("2.3E+4")}}, {2e4, Token{key: TokenFloat, value: []byte("2e4")}}, + {-1e1, Token{key: TokenFloat, value: []byte("-1e1")}}, } for _, v := range floats { t.Run(string(v.token.value), func(t *testing.T) { From f58843d85ef17571eefcc6a09249aab523ca6f22 Mon Sep 17 00:00:00 2001 From: kettek Date: Mon, 1 Dec 2025 12:52:35 -0800 Subject: [PATCH 5/5] Change module name back to original --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bb093e8..3b3570e 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/kettek/tokenizer +module github.com/bzick/tokenizer go 1.13