From d4c38f05adbc1beb2aaed28de381a2de8582238c Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Sat, 12 Oct 2019 23:00:55 -0700 Subject: [PATCH 1/9] push null on arraystream end --- lib/understream.coffee | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/understream.coffee b/lib/understream.coffee index bc80b64..8314449 100644 --- a/lib/understream.coffee +++ b/lib/understream.coffee @@ -92,8 +92,12 @@ class ArrayStream extends Readable constructor: (@options, @arr, @index=0) -> super _(@options).extend objectMode: true _read: (size) => - debug "_read #{size} #{JSON.stringify @arr[@index]}" - @push @arr[@index] # Note: push(undefined) signals the end of the stream, so this just works^tm + data = @arr[@index] + if data is undefined + @push null + return + debug "_read #{size} #{JSON.stringify data}" + @push data @index += 1 class DevNull extends Writable From ab5bc0104a4200eb16be16d9a1b268a7cf7ff809 Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Sat, 12 Oct 2019 23:02:48 -0700 Subject: [PATCH 2/9] Bump version and node engine --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 335053c..418fe2e 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "understream", - "version": "0.10.14", + "version": "0.11.0", "description": "stream helpers", "engines": { - "node": ">=0.10.x" + "node": ">=10.x" }, "main": "index.js", "scripts": { From 72651b0c73b75f458a4b8346c0895b3b1f86737f Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Sat, 12 Oct 2019 22:58:25 -0700 Subject: [PATCH 3/9] fix tests --- Makefile | 6 +++--- lib/helpers.coffee | 2 ++ lib/transforms/join.coffee | 2 +- lib/understream.coffee | 4 +++- test/combine.coffee | 4 ++++ test/errors.coffee | 3 ++- test/join.coffee | 6 +++++- test/split.coffee | 2 +- test/wrap.coffee | 2 +- 9 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index ef1fc79..5b5fe3f 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,9 @@ # `make test` runs all the tests # `make test/each.coffee` runs just that test .PHONY: test test-cov -TESTS=$(shell cd test && ls *.coffee | sed s/\.coffee$$//) +TESTS=$(shell ls test/*.coffee | sed s/\.coffee$$//) LIBS=$(shell find . -regex "^./lib\/.*\.coffee\$$" | sed s/\.coffee$$/\.js/ | sed s/lib/lib-js/) -MONGO_URL ?= localhost # needed for tests +MONGO_URL ?= mongodb://127.0.0.1:27017 # needed for tests build: $(LIBS) @@ -16,7 +16,7 @@ lib-js/%.js : lib/%.coffee test: $(TESTS) $(TESTS): build - MONGO_URL=$(MONGO_URL) DEBUG=* NODE_ENV=test node_modules/mocha/bin/mocha --timeout 60000 --bail --compilers coffee:coffee-script test/$@.coffee + MONGO_URL=$(MONGO_URL) DEBUG=* NODE_ENV=test node_modules/mocha/bin/mocha --timeout 60000 --bail --compilers coffee:coffee-script $@.coffee test-cov: build # jscoverage only accepts directory arguments so have to rebuild everything diff --git a/lib/helpers.coffee b/lib/helpers.coffee index 17d3d69..5075d07 100644 --- a/lib/helpers.coffee +++ b/lib/helpers.coffee @@ -8,3 +8,5 @@ module.exports = instance instanceof EventEmitter and instance.pipe? and (instance._read? or instance.read? or instance.readable) + + DEFAULT_MAX_LISTENERS: 10 diff --git a/lib/transforms/join.coffee b/lib/transforms/join.coffee index 34243ae..1c323c0 100644 --- a/lib/transforms/join.coffee +++ b/lib/transforms/join.coffee @@ -106,7 +106,7 @@ class SortedMergeJoin extends Transform # Hack to find out if a stream has no more data. This is totally not kosher # since it relies on undocumented internals. ended = (stream) -> - stream._readableState.ended and _.isEmpty stream._readableState.buffer + stream._readableState.ended and stream._readableState.buffer.length == 0 constructor: (stream_opts, {@right, @key}) -> super stream_opts diff --git a/lib/understream.coffee b/lib/understream.coffee index 8314449..fd174b0 100644 --- a/lib/understream.coffee +++ b/lib/understream.coffee @@ -5,7 +5,7 @@ _.mixin require 'underscore.deep' debug = require('debug') 'us:' domain = require 'domain' {EventEmitter} = require 'events' -{is_readable} = require './helpers' +{is_readable, DEFAULT_MAX_LISTENERS} = require './helpers' # Adds a listener to an EventEmitter but first bumps the max listeners limit # for that emitter. The limit is meant to prevent memory leaks, so this should @@ -14,6 +14,8 @@ domain = require 'domain' add_listener_unsafe = (emitter, event, listener) -> if emitter._maxListeners? emitter.setMaxListeners emitter._maxListeners + 1 + else + emitter._maxListeners = DEFAULT_MAX_LISTENERS + 1 emitter.addListener event, listener # Wraps a stream's _transform method in a domain, catching any thrown errors diff --git a/test/combine.coffee b/test/combine.coffee index dc12f75..041bee4 100644 --- a/test/combine.coffee +++ b/test/combine.coffee @@ -16,6 +16,10 @@ lazy_stream_from_array = (arr) -> i = 0 rs._read = -> setImmediate => + data = arr[i] + if data is undefined + @push null + return @push arr[i] i += 1 rs diff --git a/test/errors.coffee b/test/errors.coffee index 678cf88..a0d8216 100644 --- a/test/errors.coffee +++ b/test/errors.coffee @@ -3,6 +3,7 @@ async = require 'async' _ = require 'underscore' Understream = require "#{__dirname}/../index" {Readable} = require 'stream' +{DEFAULT_MAX_LISTENERS} = require '../lib/helpers' # domain_thrown (0,8) vs domainThrown (0.10) was_thrown = (domain_err) -> @@ -119,7 +120,7 @@ describe '_.stream error handling', -> stream = new Understream([]).stream() _.each ['once', 'again'], (time) -> it time, (done) -> - starting_max_listeners = stream._maxListeners + starting_max_listeners = stream._maxListeners || DEFAULT_MAX_LISTENERS starting_listeners = num_listeners stream new Understream(stream).each(-> ).run (err) -> assert.ifError err diff --git a/test/join.coffee b/test/join.coffee index 5c3a08a..8089d2e 100644 --- a/test/join.coffee +++ b/test/join.coffee @@ -226,7 +226,11 @@ describe '_.join', -> # Simulate some data being loaded after a delay else setTimeout -> - right.push right_input[i] + data = right_input[i] + if data? + right.push right_input[i] + else + right.push null i += 1 , 100 diff --git a/test/split.coffee b/test/split.coffee index 20e2695..7b943aa 100644 --- a/test/split.coffee +++ b/test/split.coffee @@ -17,7 +17,7 @@ describe '_.split', -> ] async.forEachSeries test_inputs, (test_input, cb_fe) -> new Understream(test_input).split("test").run (err) -> - assert err.message.match /non-string\/buffer chunk/ + assert err.message.match /argument must be one of type string or Buffer/ cb_fe() , done diff --git a/test/wrap.coffee b/test/wrap.coffee index 721f713..4e94e29 100644 --- a/test/wrap.coffee +++ b/test/wrap.coffee @@ -19,7 +19,7 @@ describe '_.stream', -> done() it 'wraps a mongoose stream', (done) -> - mongoose.connect "#{process.env.MONGO_URL}/test-understream" + mongoose.connect "#{process.env.MONGO_URL}/test-understream", {useNewUrlParser: true} Doc = mongoose.model "Doc", new mongoose.Schema { foo: String } input = ['a', 'b', 'c'] async.waterfall [ From 99cfa22d2e942af9d7a4761e01c8a56badd389f6 Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Sat, 12 Oct 2019 22:58:42 -0700 Subject: [PATCH 4/9] dev dependencies update --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 418fe2e..4bc1847 100644 --- a/package.json +++ b/package.json @@ -23,12 +23,12 @@ "underscore.deep": "^0.5.1" }, "devDependencies": { + "JSONStream": "~0.7.1", "coffee-script": "~1.6.2", - "sinon": "~1.5.2", "mocha": "~1.9.0", - "through": "~2.3.4", - "mongoose": "~3.6.19", - "JSONStream": "~0.7.1" + "mongoose": "^5.7.4", + "sinon": "~1.5.2", + "through": "~2.3.4" }, "publishConfig": { "registry": "https://registry.npmjs.org" From 29badcfaae1fc0e1452864181d37836fec78f5ff Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Sat, 12 Oct 2019 22:59:00 -0700 Subject: [PATCH 5/9] update circle image and run tests --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96845c3..c45d5d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ jobs: build: working_directory: ~/Clever/understream docker: - - image: circleci/node:8.11.3-stretch + - image: circleci/node:10.15.1-stretch environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results @@ -19,3 +19,6 @@ jobs: - run: command: npm install name: npm install + - run: + command: make test + name: make test From d3372090ee6785ab0bfd41e53199a2f6a01769bf Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Mon, 14 Oct 2019 07:20:41 -0700 Subject: [PATCH 6/9] fix circle config --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c45d5d5..475ea6d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,5 +20,5 @@ jobs: command: npm install name: npm install - run: - command: make test - name: make test + command: make test + name: make test From 5fec127237f9587d09b86724f4856158d069e2fd Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Mon, 14 Oct 2019 07:24:51 -0700 Subject: [PATCH 7/9] add mongo image for circle --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 475ea6d..fb0c53f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,7 @@ jobs: working_directory: ~/Clever/understream docker: - image: circleci/node:10.15.1-stretch + - image: circleci/mongo:3.2.20-jessie-ram environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts CIRCLE_TEST_REPORTS: /tmp/circleci-test-results From 00a0795af52149b9ee1b06d9e7ce47221eb0867d Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Mon, 14 Oct 2019 12:16:53 -0700 Subject: [PATCH 8/9] Allow undefined values in the stream --- lib/understream.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/understream.coffee b/lib/understream.coffee index fd174b0..0e1d3a1 100644 --- a/lib/understream.coffee +++ b/lib/understream.coffee @@ -94,10 +94,10 @@ class ArrayStream extends Readable constructor: (@options, @arr, @index=0) -> super _(@options).extend objectMode: true _read: (size) => - data = @arr[@index] - if data is undefined + if @index > (@arr.length - 1) @push null return + data = @arr[@index] debug "_read #{size} #{JSON.stringify data}" @push data @index += 1 From 244406b7a38c749c493e3c490fd39b8bc97d79bf Mon Sep 17 00:00:00 2001 From: Amelia Jones Date: Mon, 14 Oct 2019 12:17:02 -0700 Subject: [PATCH 9/9] fix tests --- test/batch.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/batch.coffee b/test/batch.coffee index 708b963..40af0ad 100644 --- a/test/batch.coffee +++ b/test/batch.coffee @@ -4,7 +4,7 @@ Understream = require "#{__dirname}/../index" charRange = (start, stop) -> if not stop? then stop = start; start = 0 - _(_.range(start, stop)) + _.range(start, stop) .map (i) -> 'abcdefghijklmnopqrstuvwxyz'[i] describe '_.batch(n)', ->