diff --git a/packages/models/src/util.js b/packages/models/src/util.js index f51edc4a58..c5ceb8f939 100644 --- a/packages/models/src/util.js +++ b/packages/models/src/util.js @@ -350,7 +350,86 @@ export function buildLabels(classMap, events) { // sizeof returns a naive byte count for an object when serialized. // I was using an external library for this (object-sizeof), but getting results off by a factor of // ~2. This is awfully wasteful, slow and inaccurate but it works for now. -DB -export const sizeof = (obj) => JSON.stringify(obj).length; +export const sizeof = (obj) => { + try { + return JSON.stringify(obj).length; + } catch (e) { + // In case of large objects (e.g. ~500MB), JSON.stringify might fail with RangeError: Invalid string length. + // In that case, we fall back to a recursive calculation. + if (e instanceof RangeError) { + const seen = new WeakSet(); + + const recursiveSize = (value) => { + try { + const s = JSON.stringify(value); + // undefined returns undefined. The loops shouldn't pass undefined, but if they do, handling it is safe. + if (s === undefined) return 0; + return s.length; + } catch (err) { + if (!(err instanceof RangeError)) throw err; + } + + if (value === null) return 4; + if (typeof value !== 'object') { + // Fallback for huge strings that failed stringify + if (typeof value === 'string') { + return value.length + 2; // +2 for quotes + } + return String(value).length; + } + + if (seen.has(value)) { + throw new TypeError('Converting circular structure to JSON'); + } + seen.add(value); + + if (Array.isArray(value)) { + let size = 2; // [] + if (value.length > 0) { + size += value.length - 1; // commas + for (let i = 0; i < value.length; i += 1) { + const item = value[i]; + // In arrays, undefined, functions, and symbols are converted to null + if (item === undefined || typeof item === 'function' || typeof item === 'symbol') { + size += 4; // "null" + } else { + size += recursiveSize(item); + } + } + } + return size; + } + + // Generic Object + let size = 2; // {} + const keys = Object.keys(value); + let addedProps = 0; + for (let i = 0; i < keys.length; i += 1) { + const k = keys[i]; + const v = value[k]; + + // In objects, properties with undefined, function, or symbol values are omitted + if (v === undefined || typeof v === 'function' || typeof v === 'symbol') { + continue; + } + + if (addedProps > 0) { + size += 1; // comma + } + + size += JSON.stringify(k).length; // "key" + size += 1; // : + size += recursiveSize(v); + addedProps += 1; + } + return size; + }; + + return recursiveSize(obj); + } + throw e; + } +}; // Returns a unique 'hash' (or really, a key) tied to the event's core identity: SQL, HTTP, or a // specific method on a specific class. This is _really_ naive. The idea is that this better finds diff --git a/packages/models/tests/unit/sizeof.spec.js b/packages/models/tests/unit/sizeof.spec.js new file mode 100644 index 0000000000..eb4a927837 --- /dev/null +++ b/packages/models/tests/unit/sizeof.spec.js @@ -0,0 +1,110 @@ +import { sizeof } from '../../src/util'; + +describe('sizeof', () => { + it('calculates size of simple objects', () => { + const obj = { a: 1 }; + // {"a":1} -> 7 chars + expect(sizeof(obj)).toEqual(7); + }); + + it('calculates size of arrays', () => { + const arr = [1, 2]; + // [1,2] -> 5 chars + expect(sizeof(arr)).toEqual(5); + }); + + it('handles nested objects', () => { + const obj = { a: { b: 1 } }; + // {"a":{"b":1}} -> 13 chars + expect(sizeof(obj)).toEqual(13); + }); + + describe('fallback behavior', () => { + let stringifySpy; + const originalStringify = JSON.stringify; + + beforeEach(() => { + // Mock JSON.stringify to throw RangeError for objects with a specific flag + stringifySpy = jest.spyOn(JSON, 'stringify').mockImplementation((val) => { + if (val && typeof val === 'object' && val !== null && val.forceFail) { + throw new RangeError('Invalid string length: forced by test'); + } + return originalStringify(val); + }); + }); + + afterEach(() => { + stringifySpy.mockRestore(); + }); + + it('falls back to recursive calculation on RangeError', () => { + const hugeObj = { + forceFail: true, + a: 1, + b: [2, 3], + }; + // We manually calculate expected size or temporarily bypass the mock + // {"forceFail":true,"a":1,"b":[2,3]} length is 34 + expect(sizeof(hugeObj)).toEqual(34); + expect(stringifySpy).toHaveBeenCalled(); + }); + + it('correctly calculates size of strings with escaped characters', () => { + const obj = { + forceFail: true, + str: 'hello "world"\n', + }; + + const expected = originalStringify(obj).length; + + expect(sizeof(obj)).toEqual(expected); + }); + + it('handles arrays with undefined, null, functions', () => { + const obj = { + forceFail: true, + arr: [undefined, null, () => {}, 123], + }; + + const expected = originalStringify(obj).length; + expect(sizeof(obj)).toEqual(expected); + }); + + it('handles objects with undefined, null, functions', () => { + const obj = { + forceFail: true, + a: undefined, + b: null, + c: () => {}, + d: 123, + }; + + const expected = originalStringify(obj).length; + expect(sizeof(obj)).toEqual(expected); + }); + + it('detects circular references and throws TypeError', () => { + const obj = { + forceFail: true, + }; + obj.self = obj; + + expect(() => sizeof(obj)).toThrow(TypeError); + expect(() => sizeof(obj)).toThrow('Converting circular structure to JSON'); + }); + + it('handles deep nesting', () => { + const obj = { + forceFail: true, + level1: { + level2: { + level3: [1, 2, 3], + }, + }, + }; + + const expected = originalStringify(obj).length; + expect(sizeof(obj)).toEqual(expected); + }); + }); +}); diff --git a/packages/validate/src/build.js b/packages/validate/src/build.js index 7fcb09d853..64801678c1 100644 --- a/packages/validate/src/build.js +++ b/packages/validate/src/build.js @@ -2,7 +2,7 @@ const FileSystem = require('fs'); const Semver = require('semver'); const YAML = require('yaml'); -const lines = FileSystem.readFileSync(`${__dirname}/macro.yml`, 'utf8').split('\n'); +const lines = FileSystem.readFileSync(`${__dirname}/macro.yml`, 'utf8').split(/\r?\n/); const versions = ['1.13.1', '1.13.0', '1.12.0', '1.11.0', '1.10.0', '1.9.0', '1.8.0', '1.7.0', '1.6.0', '1.5.1', '1.5.0', '1.4.1', '1.4.0', '1.3.0', '1.2.0'];