From 614aca3a283e23c623088ab1c8d438dbb552b260 Mon Sep 17 00:00:00 2001
From: iyatomi takehiro <iyatomi@gmail.com>
Date: Sat, 15 Dec 2012 03:42:44 +0900
Subject: [PATCH] Squashed commit of the following:

commit f57da257673e628d1af5cc545d3be7e0ea7115fa
Author: iyatomi takehiro <iyatomi@gmail.com>
Date:   Sat Dec 15 03:39:56 2012 +0900

    adjust stream unpack test to project's standard way

commit c389bd113a105f29157312502d2263d91b6a3654
Author: iyatomi takehiro <iyatomi@gmail.com>
Date:   Sat Dec 15 03:04:35 2012 +0900

    add stream unpacker and its test
---
 msgpack.js             | 257 ++++++++++++++++++++++++++++-
 test/codec.htm         |  10 ++
 test/stream_unpack.htm | 355 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 621 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 msgpack.js
 mode change 100644 => 100755 test/codec.htm
 create mode 100755 test/stream_unpack.htm

diff --git a/msgpack.js b/msgpack.js
old mode 100644
new mode 100755
index 46ed0cb..f39ccf2
--- a/msgpack.js
+++ b/msgpack.js
@@ -13,6 +13,7 @@ globalScope.msgpack = {
     unpack:     msgpackunpack,  // msgpack.unpack(data:BinaryString/ByteArray):Mix
                                 //  [1][String to mix]    msgpack.unpack("...") -> {}
                                 //  [2][ByteArray to mix] msgpack.unpack([...]) -> {}
+    stream_unpacker: msgpack_stream_unpacker_new,
     worker:     "msgpack.js",   // msgpack.worker - WebWorkers script filename
     upload:     msgpackupload,  // msgpack.upload(url:String, option:Hash, callback:Function)
     download:   msgpackdownload // msgpack.download(url:String, option:Hash, callback:Function)
@@ -70,6 +71,11 @@ function msgpackunpack(data) { // @param BinaryString/ByteArray:
     return decode(); // mix or undefined
 }
 
+// msgpack.stream_unpacker
+function msgpack_stream_unpacker_new() {
+    return new stream_decoder();
+}
+
 // inner - encoder
 function encode(rv,      // @param ByteArray: result
                 mix,     // @param Mix: source data
@@ -247,7 +253,7 @@ function decode() { // @return Mix:
     var size, i, iz, c, num = 0,
         sign, exp, frac, ary, hash,
         buf = _buf, type = buf[++_idx];
-
+        
     if (type >= 0xe0) {             // Negative FixNum (111x xxxx) (-32 ~ -1)
         return type - 0x100;
     }
@@ -384,6 +390,255 @@ function decode() { // @return Mix:
     return;
 }
 
+// stream decoder
+var stream_decoder = (function () {
+    function _stream_decoder() {
+        var that = this;
+        that._buf        = []; // decode buffer
+        that._idx        = 0;  // decode buffer[index]
+        that._stack      = [];
+        that._type       = null;    //current decoded type
+    }
+    var constant = {
+        ARRAY_VALUE: 1,
+        MAP_VALUE: 2,
+        MAP_KEY: 3,
+    }
+    function stream_decode(that) { // @return Mix:
+        while (that._buf.length > that._idx) {
+            var rv = stream_decode_token(that);
+            if (rv === undefined) { 
+                return; /* not enough buffer length */
+            }
+            else {
+                that._type = null;  //reset type
+                var stack = that._stack;
+                ReduceStack: while (true) {
+                    if (stack.length > 0) {
+                        var top = stack[stack.length - 1];
+                        switch(top.type) {
+                        case constant.ARRAY_VALUE:
+                            top.value.push(rv); 
+                            if (top.value.length == top.reqlen) {
+                                rv = top.value;
+                                stack.pop();
+                                break;
+                            }
+                            break ReduceStack; //try next stream_decode_token
+                        case constant.MAP_KEY:
+                            top.key = rv;
+                            top.type = constant.MAP_VALUE;
+                            break ReduceStack; //try next stream_decode_token
+                        case constant.MAP_VALUE:
+                            top.value[top.key] = rv;
+                            if (Object.keys(top.value).length == top.reqlen) {
+                                rv = top.value;
+                                stack.pop();
+                                break;
+                            }
+                            top.type = constant.MAP_KEY;
+                            break ReduceStack; //try next stream_decode_token
+                        default:
+                            throw "invalid stack type:" + top.type;
+                        }
+                    }
+                    else {
+                        return rv;
+                    }
+                }
+            }
+        }
+        return;/* not enough buffer length */
+    }
+    
+    function push_stack(that, type, reqlen, value) {
+        that._stack.push({
+            type: type,
+            reqlen: reqlen,
+            value: value,
+        });
+        that._type = null;  //when push stack, reset parsed type to parse element of array or hash
+    }
+    function stream_decode_token(that) {
+        var size, i, iz, c, num = 0,
+            sign, exp, frac, ary, hash,
+            buf = that._buf;
+        if (that._type === null) {  //type check is only done when current parsed type not decided
+            if (buf.length <= (that._idx + 1)) { return; }
+            type = buf[++that._idx];
+            if (type >= 0xe0) {             // Negative FixNum (111x xxxx) (-32 ~ -1)
+                return type - 0x100;
+            }
+            if (type < 0xc0) {
+                if (type < 0x80) {          // Positive FixNum (0xxx xxxx) (0 ~ 127)
+                    return type;
+                }
+                if (type < 0x90) {          // FixMap (1000 xxxx)
+                    num  = type - 0x80;
+                    that._type = 0x80;
+                } else if (type < 0xa0) {   // FixArray (1001 xxxx)
+                    num  = type - 0x90;
+                    that._type = 0x90;
+                } else { //if (type < 0xc0)   // FixRaw (101x xxxx)
+                    that._fixrawlen  = type - 0xa0;
+                    that._type = 0xa0;
+                } 
+            }
+            else {
+                that._type = type;
+            }
+        }
+        switch (that._type) {
+        case 0xc0:  return null;
+        case 0xc2:  return false;
+        case 0xc3:  return true;
+        case 0xca:  // float
+                    if (buf.length <= (that._idx + 4)) { return; }
+                    num = buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16) +
+                                                    (buf[++that._idx] <<  8) + buf[++that._idx];
+                    sign =  num & 0x80000000;    //  1bit
+                    exp  = (num >> 23) & 0xff;   //  8bits
+                    frac =  num & 0x7fffff;      // 23bits
+                    if (!num || num === 0x80000000) { // 0.0 or -0.0
+                        return 0;
+                    }
+                    if (exp === 0xff) { // NaN or Infinity
+                        return frac ? NaN : Infinity;
+                    }
+                    return (sign ? -1 : 1) *
+                                (frac | 0x800000) * Math.pow(2, exp - 127 - 23); // 127: bias
+        case 0xcb:  // double
+                    if (buf.length <= (that._idx + 8)) { return; }
+                    num = buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16) +
+                                                    (buf[++that._idx] <<  8) + buf[++that._idx];
+                    sign =  num & 0x80000000;    //  1bit
+                    exp  = (num >> 20) & 0x7ff;  // 11bits
+                    frac =  num & 0xfffff;       // 52bits - 32bits (high word)
+                    if (!num || num === 0x80000000) { // 0.0 or -0.0
+                        that._idx += 4;
+                        return 0;
+                    }
+                    if (exp === 0x7ff) { // NaN or Infinity
+                        that._idx += 4;
+                        return frac ? NaN : Infinity;
+                    }
+                    num = buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16) +
+                                                    (buf[++that._idx] <<  8) + buf[++that._idx];
+                    return (sign ? -1 : 1) *
+                                ((frac | 0x100000) * Math.pow(2, exp - 1023 - 20) // 1023: bias
+                                 + num * Math.pow(2, exp - 1023 - 52));
+        // 0xcf: uint64, 0xce: uint32, 0xcd: uint16
+        case 0xcf:  if (buf.length < (that._idx + 8)) { return; }
+                    num =  buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16) +
+                                                     (buf[++that._idx] <<  8) + buf[++that._idx];
+                    return num * 0x100000000 +
+                           buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16) +
+                                                     (buf[++that._idx] <<  8) + buf[++that._idx];
+        case 0xce:  if (buf.length <= (that._idx + 4)) { return; }
+                    num += buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16);
+        case 0xcd:  if (buf.length <= (that._idx + 2)) { return; }
+                    num += buf[++that._idx] << 8;
+        case 0xcc:  if (buf.length <= (that._idx + 1)) { return; }
+                    return num + buf[++that._idx];
+        // 0xd3: int64, 0xd2: int32, 0xd1: int16, 0xd0: int8
+        case 0xd3:  if (buf.length <= (that._idx + 8)) { return; }
+                    num = buf[++that._idx];
+                    if (num & 0x80) { // sign -> avoid overflow
+                        return ((num         ^ 0xff) * 0x100000000000000 +
+                                (buf[++that._idx] ^ 0xff) *   0x1000000000000 +
+                                (buf[++that._idx] ^ 0xff) *     0x10000000000 +
+                                (buf[++that._idx] ^ 0xff) *       0x100000000 +
+                                (buf[++that._idx] ^ 0xff) *         0x1000000 +
+                                (buf[++that._idx] ^ 0xff) *           0x10000 +
+                                (buf[++that._idx] ^ 0xff) *             0x100 +
+                                (buf[++that._idx] ^ 0xff) + 1) * -1;
+                    }
+                    return num         * 0x100000000000000 +
+                           buf[++that._idx] *   0x1000000000000 +
+                           buf[++that._idx] *     0x10000000000 +
+                           buf[++that._idx] *       0x100000000 +
+                           buf[++that._idx] *         0x1000000 +
+                           buf[++that._idx] *           0x10000 +
+                           buf[++that._idx] *             0x100 +
+                           buf[++that._idx];
+        case 0xd2:  if (buf.length <= (that._idx + 4)) { return; }
+                    num  =  buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16) +
+                           (buf[++that._idx] << 8) + buf[++that._idx];
+                    return num < 0x80000000 ? num : num - 0x100000000; // 0x80000000 * 2
+        case 0xd1:  if (buf.length <= (that._idx + 2)) { return; }
+                    num  = (buf[++that._idx] << 8) + buf[++that._idx];
+                    return num < 0x8000 ? num : num - 0x10000; // 0x8000 * 2
+        case 0xd0:  if (buf.length <= (that._idx + 1)) { return; }
+                    num  =  buf[++that._idx];
+                    return num < 0x80 ? num : num - 0x100; // 0x80 * 2
+        // 0xdb: raw32, 0xda: raw16, 0xa0: raw ( string )
+        case 0xdb:  i = that._idx;
+                    if (buf.length <= (i + 4)) { return; }
+                    num +=  buf[++i] * 0x1000000 + (buf[++i] << 16);
+        case 0xda:  if (i === undefined) { i = that._idx; }
+                    if (buf.length <= (i + 2)) { return; }
+                    num += (buf[++i] << 8)       +  buf[++i];
+        case 0xa0:  // utf8.decode
+                    if (i === undefined) { 
+                        i = that._idx; 
+                        num = that._fixrawlen;
+                    }
+                    if (buf.length <= (i + num)) { return; }
+                    for (ary = [], iz = i + num; i < iz; ) {
+                        c = buf[++i]; // lead byte
+                        ary.push(c < 0x80 ? c : // ASCII(0x00 ~ 0x7f)
+                                 c < 0xe0 ? ((c & 0x1f) <<  6 | (buf[++i] & 0x3f)) :
+                                            ((c & 0x0f) << 12 | (buf[++i] & 0x3f) << 6
+                                                              | (buf[++i] & 0x3f)));
+                    }
+                    that._idx = i;
+                    return ary.length < 10240 ? _toString.apply(null, ary)
+                                              : byteArrayToByteString(ary);
+        // 0xdf: map32, 0xde: map16, 0x80: map
+        case 0xdf:  if (buf.length <= (that._idx + 4)) { return; }
+                    num +=  buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16);
+        case 0xde:  if (buf.length <= (that._idx + 2)) { return; }
+                    num += (buf[++that._idx] << 8)       +  buf[++that._idx];
+        case 0x80:  if (num <= 0) { return {}; } 
+                    push_stack(that, constant.MAP_KEY, num, {});
+                    return stream_decode_token(that);
+        // 0xdd: array32, 0xdc: array16, 0x90: array
+        case 0xdd:  if (buf.length <= (that._idx + 4)) { return; }
+                    num +=  buf[++that._idx] * 0x1000000 + (buf[++that._idx] << 16);
+        case 0xdc:  if (buf.length <= (that._idx + 2)) { return; }
+                    num += (buf[++that._idx] << 8)       +  buf[++that._idx];
+        case 0x90:  if (num <= 0) { return []; } 
+                    push_stack(that, constant.ARRAY_VALUE, num, []);
+                    return stream_decode_token(that);
+        }
+        return;
+    }
+    _stream_decoder.prototype.unpack = function (data) { // @param BinaryString/ByteArray:
+                                   // @return Mix/undefined:
+                                   //       undefined is error return
+        //  [1][String to mix]    msgpack.unpack("...") -> {}
+        //  [2][ByteArray to mix] msgpack.unpack([...]) -> {}
+        var that = this;
+        //console.log("buffer:" + JSON.stringify(data));
+        if (data != null) {
+            var readlen = that._idx + 1;
+            if (that._buf.length > readlen) {
+                // create new bytearray which is [prev remain] + [new data]
+                var tmp = that._buf.subarray(readlen);
+                that._buf = new Uint8Array(that._buf.length - readlen + data.length);
+                that._buf.set(tmp);
+                that._buf.set(typeof data === "string" ? toByteArray(data) : data, tmp.length);
+            }
+            else {
+                that._buf = typeof data === "string" ? toByteArray(data) : data;
+            }
+            that._idx = -1;
+        }
+        return stream_decode(that); // mix or undefined or null (buff short)
+    }
+    return _stream_decoder;
+})();
+
 // inner - byteArray To ByteString
 function byteArrayToByteString(byteArray) { // @param ByteArray
                                             // @return String
diff --git a/test/codec.htm b/test/codec.htm
old mode 100644
new mode 100755
index eb2dbdc..02336b2
--- a/test/codec.htm
+++ b/test/codec.htm
@@ -36,6 +36,16 @@
         return uu.array.dump(ary, "0x", ", 0x");
     };
 
+    var args = window.location.search.substr(1).split('&');
+    for (v in args) {
+        var s = args[v].split('=');
+        if (s[0] == 'stream' && s[1] == '1') {
+            msgpack.unpack = function (data) {
+                return msgpack.stream_unpacker().unpack(data);
+            }
+        }
+    }
+
     uu.test({
         "FixNum, uint and int": "",
             "Number(0) -> [0x00] (Positive FixNum)": function() {
diff --git a/test/stream_unpack.htm b/test/stream_unpack.htm
new file mode 100755
index 0000000..401e55e
--- /dev/null
+++ b/test/stream_unpack.htm
@@ -0,0 +1,355 @@
+<!DOCTYPE html><html lang="ja">
+<head>
+<meta charset="UTF-8" />
+<title>msgpack stream unpack test</title>
+<style>
+.uutest ol { background-color:dodgerblue;color:white }
+.uutest li { padding:5px;border:1px solid #ccc }
+.uutest .li0 { background-color:red   }
+.uutest .li1 { background-color:green }
+.uutest .li2 { background-color:red   }
+.uutest .li3 { background-color:#0a0  }
+.uutestinfo { position:fixed;top:10px;right:10px;font-size:xx-large }
+.uutestinfo a { border: 3px outset gray;padding:3px;background-color:red;color:white }
+.uutestinfo .score { color:black;text-shadow:gray 1px 1px 1px }
+.uutestinfo .ngzone {}
+</style>
+<script src="../msgpack.js" type="text/javascript"></script>
+<script src="10000.js"></script>
+<script src="uupaa.js"></script>
+<script type="text/javascript">
+function equals(a1, a2) {
+    //http://stackoverflow.com/questions/3115982/how-to-check-javascript-array-equals
+    return JSON.stringify(a1) == JSON.stringify(a2);
+}
+/*
+(function sandbox() {
+    var data = {'abc' : [123]};
+    var b = msgpack.pack(data);
+    var u = msgpack.stream_unpacker();
+    var r = u.unpack(b);
+    if (!equals(r, data)) {
+        alert("it should be " + JSON.stringify(data) + ":" + JSON.stringify(r));
+    }
+})(); */
+var solid100 = {
+    "0": null,
+    "1": {
+        "1": {
+            "1": -1
+        }
+    }, "2": "2\u0002\u0002\u0002\u0002",
+    "3": 0.024300000221130002,
+    "4": [null, {
+        "4": true
+    }, [
+        [null, false, {
+            "4": false
+        }], {
+            "4": "4\u0004\u0004\u0004\u0004"
+        }, {
+            "4": {
+                "4": {
+                    "4": -0.032400000294840005
+                }
+            }
+        }]],
+    "5": null,
+    "6": true,
+    "7": [{
+        "7": null
+    }, {
+        "7": true
+    }, false],
+    "8": 8,
+    "9": [0.07290000066339, 9, true],
+    "10": 10,
+    "11": {
+        "11": -11
+    }, "12": ["c\f\f\f\f", 12, false],
+    "13": 13,
+    "14": [true, -14, null],
+    "15": 15,
+    "16": null,
+    "17": "11\u0011\u0011\u0011\u0011",
+    "18": false,
+    "19": [false, {
+        "19": null
+    }, "13\u0013\u0013\u0013\u0013"],
+    "20": [-20, true, true],
+    "21": null,
+    "22": false,
+    "23": -0.18630000169533,
+    "24": false,
+    "25": ["19\u0019\u0019\u0019\u0019", {
+        "25": [false, 0.20250000184275002, 25]
+    }, [
+        [false, false, false], null, {
+            "25": true
+        }]],
+    "26": null,
+    "27": 0.21870000199017,
+    "28": {
+        "28": -0.22680000206388
+    }, "29": {
+        "29": null
+    }, "30": 0.24300000221130002,
+    "31": 0.25110000228501,
+    "32": null,
+    "33": 0.26730000243243,
+    "34": false,
+    "35": -0.28350000257985003,
+    "36": {
+        "36": "24$$$$"
+    }, "37": false,
+    "38": {
+        "38": 0.30780000280098
+    }, "39": null,
+    "40": [null, false, "28(((("],
+    "41": {
+        "41": 0.33210000302211
+    }, "42": true,
+    "43": null,
+    "44": {
+        "44": {
+            "44": 44
+        }
+    }, "45": true,
+    "46": null,
+    "47": {
+        "47": true
+    }, "48": -0.38880000353808003,
+    "49": {
+        "49": {
+            "49": [false, 0.39690000361179, ["311111", true, 0.39690000361179]]
+        }
+    }, "50": [-50, [true, true, 50], false],
+    "51": 0.41310000375921,
+    "52": null,
+    "53": null,
+    "54": {
+        "54": -54
+    }, "55": true,
+    "56": -56,
+    "57": null,
+    "58": {
+        "58": null
+    }, "59": "3b;;;;",
+    "60": "3c<<<<",
+    "61": 0.49410000449631003,
+    "62": {
+        "62": {
+            "62": [null, 62, null]
+        }
+    }, "63": false,
+    "64": false,
+    "65": [65, null, [65, -0.52650000479115, 65]],
+    "66": 66,
+    "67": -67,
+    "68": {
+        "68": -68
+    }, "69": 69,
+    "70": null,
+    "71": false,
+    "72": -0.58320000530712,
+    "73": 0.59130000538083,
+    "74": {
+        "74": true
+    }, "75": {
+        "75": {
+            "75": false
+        }
+    }, "76": 0.61560000560196,
+    "77": 0.6237000056756701,
+    "78": null,
+    "79": 79,
+    "80": "50PPPP",
+    "81": {
+        "81": {
+            "81": 0.6561000059705101
+        }
+    }, "82": -82,
+    "83": [true, ["53SSSS", {
+        "83": {
+            "83": {
+                "83": [{
+                    "83": 83
+                }, 83, {
+                    "83": -83
+                }]
+            }
+        }
+    }, -0.6723000061179301], 83],
+    "84": {
+        "84": "54TTTT"
+    }, "85": true,
+    "86": -86,
+    "87": null,
+    "88": {
+        "88": 0.71280000648648
+    }, "89": 89,
+    "90": {
+        "90": false
+    }, "91": true,
+    "92": -92,
+    "93": 93,
+    "94": -0.7614000069287401,
+    "95": {
+        "95": 0.76950000700245
+    }, "96": null,
+    "97": [
+        ["61aaaa", {
+            "97": null
+        }, -0.7857000071498701], {
+            "97": [0.7857000071498701, 0.7857000071498701, [0.7857000071498701, -0.7857000071498701, "61aaaa"]]
+        }, {
+            "97": 97
+        }],
+    "98": {
+        "98": true
+    }, "99": null, 
+};
+function test_stream_unpack() {
+    var b1 = msgpack.pack([0,1,2]);
+    var b2 = msgpack.pack([3,4,5,6,7,8,9]);
+    var u = msgpack.stream_unpacker();
+    var buf = new Uint8Array(b1.length + b2.length);
+    buf.set(b1);
+    buf.set(b2, b1.length);
+    console.log("buflen:" + buf.length + "|" + JSON.stringify(buf));
+    var p1 = u.unpack(buf.subarray(0, buf.length / 2));
+    if (equals(p1, [0,1,2])) { 
+    }
+    else { 
+       console.log("unpack fails:" + JSON.stringify(p1) + " is not same as " + JSON.stringify([0,1,2])); 
+       return false;
+    }
+    var p2 = u.unpack(null);
+    if (p2 === undefined) {
+    }
+    else {
+       console.log("unpack fails:" + JSON.stringify(p2) + " should be same as null"); 
+       return false;
+    }
+    var p3 = u.unpack(buf.subarray(buf.length / 2));
+    if (equals(p3, [3,4,5,6,7,8,9])) { 
+    }
+    else { 
+       console.log("unpack fails:" + JSON.stringify(p3) + " is not same as " + JSON.stringify([3,4,5,6,7,8,9]));
+       return false; 
+    }
+    return true;
+}
+//stream unpack packed-one big hash 
+function test_long_chunked_data(name, data) {
+    var b = msgpack.pack(data);
+    var chunk = 64; //devide b as 64byte chunk
+    var n_chunk = Math.ceil(b.length / chunk);
+    var buf = new Uint8Array(b);
+    var u = msgpack.stream_unpacker();
+    //console.log(JSON.stringify("whole buffer:" + JSON.stringify(buf)));
+    for (i = 0; i < n_chunk; i++) {
+        var begin = i * chunk, end = (i + 1) * chunk;
+        if (end >= buf.length) { 
+            end = buf.length; 
+        }
+        var r = u.unpack(buf.subarray(begin, end));
+        if (end == buf.length) {
+            console.log("stream finished: size=" + buf.length);
+            if (!equals(r, data)) {
+                console.log(name + ":invalid result: r=" + JSON.stringify(r).substr(0,100) + "(*snip*), original=" + JSON.stringify(data).substr(0,100) + "(*snip*)");
+                return false;
+            }
+        }
+        else if (r !== undefined) {
+            console.log(name + ":unpack should on going but r is not null:" + JSON.stringify(r));
+            return false;
+        }
+    }
+    console.log(name + ": stream unpack OK!");
+    return true;
+}
+//stream unpack batched msgpack records
+function test_batched_chunk_data(name, data) {
+    var buf = new Uint8Array(256 * 1024), blen = 0;   //256k
+    for (k in data) {
+        var b = msgpack.pack(data[k]);
+        buf.set(b, blen);
+        blen += b.length;
+    }
+    var buf = buf.subarray(0, blen);
+    var n_elem = Object.keys(data).length, parsed = [];
+    var chunk = 64; //devide b as 64byte chunk
+    var n_chunk = Math.ceil(buf.length / chunk);
+    var u = msgpack.stream_unpacker();
+    //console.log(JSON.stringify("whole buffer:" + JSON.stringify(buf)));
+    for (i = 0; i < n_chunk; i++) {
+        var begin = i * chunk, end = (i + 1) * chunk;
+        if (end >= buf.length) { 
+            end = buf.length; 
+        }
+        var r = u.unpack(buf.subarray(begin, end));
+        while (r !== undefined) {
+            parsed.push(r);
+            r = u.unpack(null);
+        }
+        if (end == buf.length) {
+            console.log("stream finished: size=" + buf.length);
+            if (n_elem != parsed.length) {
+                console.log("parsed result incorrect(num) expect:" + n_elem + "/actually:" + parsed.length);
+                return false;
+            }
+            for (var j = 0; j < parsed.length; j++) {
+                if (!equals(data[String(j)], parsed[j])) {
+                    console.log(String(j) + ":parsed result incorrect");
+                    console.log("src:" + JSON.stringify(data[String(j)]));
+                    console.log("result:" + JSON.stringify(parsed[j]));
+                    return false;
+                }
+            }
+        }
+    }
+    console.log(name + ": batched stream unpack OK!");
+    return true;
+}
+
+uu.test = function(hash) {
+    var r, i;
+
+    for (i in hash) {
+        v = hash[i];
+        if (v === "") { // caption
+            uu.ok(i);
+        } else {
+            r = v();
+            uu.ok(i, r[0], r[1], r[2], r[3]);
+        }
+    }
+    uu.ok();
+};
+uu.ready(function(uu) {
+    uu.test({
+            'simple stream unpack': function() {
+                var rv = test_stream_unpack();
+                return [rv, "is", true, null];
+            },
+            'big hash unpack (solid100)': function() {
+                var rv = test_long_chunked_data('solid100', solid100);
+                return [rv, "is", true, null];
+            },
+            'big hash unpack (solid10000)': function() {
+                var rv = test_long_chunked_data('solid10000', solid10000);
+                return [rv, "is", true, null];
+            },
+            'batched record unpack (solid100)': function() {
+                var rv = test_batched_chunk_data('solid100', solid100);
+                return [rv, "is", true, null];
+            },
+            'batched record unpack (solid10000)': function() {
+                var rv = test_batched_chunk_data('solid10000', solid10000);
+                return [rv, "is", true, null];
+            },
+    });
+});
+</script>
+</head><body></body></html>