From 8084b559d6ff6e1090a9808a2d5129f93f54afd7 Mon Sep 17 00:00:00 2001 From: Dmitry Khitrin Date: Fri, 20 Dec 2024 22:41:53 +0200 Subject: [PATCH 1/5] feat: support new separated type scheme for js port --- port/js/src/Codec.ts | 11 ++++++----- port/js/src/protocol/Protocols.types.ts | 8 +++++++- port/js/tests/Codec.test.ts | 4 ++-- port/js/tests/Integration.test.ts | 16 ++++++++-------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/port/js/src/Codec.ts b/port/js/src/Codec.ts index 9c223c9..8786cbe 100644 --- a/port/js/src/Codec.ts +++ b/port/js/src/Codec.ts @@ -13,17 +13,18 @@ export class Codec { this.protocols.load(rawTypes); const converterFactory = new ConverterFactory(this.protocols); - for (const { name, proto_id: protoId, types } of protocols) { + for (const { name: protoName, proto_id: protoId, messages } of protocols) { const typeMap = new Map(); const idMap = new Map(); - for (const [messageId, typeName] of Object.entries(types)) { + for (const message of Object.values(messages)) { + const { message_id: messageId, name, type: typeName } = message; const converter = converterFactory.toConverter(typeName); - typeMap.set(typeName, converter); - idMap.set(parseInt(messageId, 10), converter); + typeMap.set(name, converter); + idMap.set(messageId, converter); } - this.typesByName.set(name, typeMap); + this.typesByName.set(protoName, typeMap); this.typesById.set(protoId, idMap); } } diff --git a/port/js/src/protocol/Protocols.types.ts b/port/js/src/protocol/Protocols.types.ts index e3df371..60e6bdb 100644 --- a/port/js/src/protocol/Protocols.types.ts +++ b/port/js/src/protocol/Protocols.types.ts @@ -40,8 +40,14 @@ export interface EnumTypeClass { export type StructureType = StructTypeClass | EnumTypeClass; +export interface RawMessage { + message_id: number; + name: IName; + type: string; +} + export interface Protocol { name: string; proto_id: number; - types: Record; + messages: Record; } diff --git a/port/js/tests/Codec.test.ts b/port/js/tests/Codec.test.ts index 7e08954..a9865e5 100644 --- a/port/js/tests/Codec.test.ts +++ b/port/js/tests/Codec.test.ts @@ -38,7 +38,7 @@ describe('Codec', () => { f9: true, }; - const message = codec.serialize('test_proto', 'messgen/test/simple_struct', rawData); + const message = codec.serialize('test_proto', 'simple_struct_msg', rawData); expect(codec.deserialize(1, 0, message.buffer)).toEqual({ ...rawData, @@ -54,7 +54,7 @@ describe('Codec', () => { const message = codec.serialize( 'another_proto', - 'cross_proto', + 'cross_proto_msg', rawData, ); diff --git a/port/js/tests/Integration.test.ts b/port/js/tests/Integration.test.ts index 1876317..20c7f87 100644 --- a/port/js/tests/Integration.test.ts +++ b/port/js/tests/Integration.test.ts @@ -33,7 +33,7 @@ describe('integration', () => { }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/simple_struct.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/simple_struct', rawData); + const buffer = codec.serialize('test_proto', 'simple_struct_msg', rawData); const result = codec.deserialize(1, 0, new Uint8Array(rawDataBit).buffer); expect(result).toEqual({ ...rawData, f5: expect.closeTo(rawData.f5, 5) }); @@ -49,7 +49,7 @@ describe('integration', () => { }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/var_size_struct.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/var_size_struct', rawData); + const buffer = codec.serialize('test_proto', 'var_size_struct_msg', rawData); const result = codec.deserialize(1, 2, new Uint8Array(rawDataBit).buffer); expect(result).toEqual(rawData); @@ -61,7 +61,7 @@ describe('integration', () => { const rawData = { f0: bigint, f1: bigint, e0: 1 }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/struct_with_enum.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/struct_with_enum', rawData); + const buffer = codec.serialize('test_proto', 'struct_with_enum_msg', rawData); const result = codec.deserialize(1, 3, new Uint8Array(rawDataBit).buffer); expect(result).toEqual(rawData); @@ -73,7 +73,7 @@ describe('integration', () => { const rawData = {}; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/empty_struct.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/empty_struct', rawData); + const buffer = codec.serialize('test_proto', 'empty_struct_msg', rawData); const result = codec.deserialize(1, 4, new Uint8Array(rawDataBit).buffer); expect(result).toEqual(rawData); @@ -105,7 +105,7 @@ describe('integration', () => { }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/complex_struct_with_empty.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/complex_struct_with_empty', rawData); + const buffer = codec.serialize('test_proto', 'complex_struct_with_empty_msg', rawData); const result = codec.deserialize(1, 5, new Uint8Array(rawDataBit).buffer); expect(result).toEqual(rawData); @@ -162,7 +162,7 @@ describe('integration', () => { }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/complex_struct_nostl.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/complex_struct_nostl', rawData); + const buffer = codec.serialize('test_proto', 'complex_struct_nostl_msg', rawData); const result = codec.deserialize(1, 6, new Uint8Array(rawDataBit).buffer); simpleStruct.f5 = expect.closeTo(simpleStruct.f5, 4); @@ -220,7 +220,7 @@ describe('integration', () => { }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/complex_struct.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/complex_struct', rawData); + const buffer = codec.serialize('test_proto', 'complex_struct_msg', rawData); const result = codec.deserialize(1, 1, new Uint8Array(rawDataBit).buffer); simpleStruct.f5 = expect.closeTo(simpleStruct.f5, 4); @@ -245,7 +245,7 @@ describe('integration', () => { }; const rawDataBit = uploadBinary('../../../tests/data/serialized/bin/flat_struct.bin'); - const buffer = codec.serialize('test_proto', 'messgen/test/flat_struct', rawData); + const buffer = codec.serialize('test_proto', 'flat_struct_msg', rawData); const result = codec.deserialize(1, 7, new Uint8Array(rawDataBit).buffer); rawData.f5 = expect.closeTo(rawData.f5, 5); From df4b02a52cec8ac1c013819236167a20a6bd6a1a Mon Sep 17 00:00:00 2001 From: Dmitry Khitrin Date: Fri, 20 Dec 2024 22:43:19 +0200 Subject: [PATCH 2/5] chore: enable js ci pipeline --- .github/workflows/js.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml index a4c94e8..16ed872 100644 --- a/.github/workflows/js.yml +++ b/.github/workflows/js.yml @@ -8,7 +8,6 @@ on: jobs: build: - if: false # disable the entire workflow runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From 26bd727b61d479f67f1510248ee55b6ab953eef3 Mon Sep 17 00:00:00 2001 From: Dmitry Khitrin Date: Fri, 20 Dec 2024 22:46:03 +0200 Subject: [PATCH 3/5] fix: fix ts generator --- messgen/ts_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messgen/ts_generator.py b/messgen/ts_generator.py index c9cd2b4..0c8cbee 100644 --- a/messgen/ts_generator.py +++ b/messgen/ts_generator.py @@ -76,7 +76,7 @@ def generate_protocols(self, out_dir: Path, protocols: dict[str, Protocol]) -> N code.append(f" '{proto_name}': {{") for message in proto_def.messages.values(): ts_struct_name = self._to_camel_case(message.type) - code.append(f" '{message.name}': {message.type};") + code.append(f" '{message.name}': {ts_struct_name};") types.add(ts_struct_name) code.append(' }') code.append('}') From 448352aaf0a7c6cd18437533b3ab9bb6566a558c Mon Sep 17 00:00:00 2001 From: Dmitry Khitrin Date: Fri, 20 Dec 2024 22:48:55 +0200 Subject: [PATCH 4/5] fix: fix mypy error for the yaml parser --- messgen/yaml_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messgen/yaml_parser.py b/messgen/yaml_parser.py index f6eb2ed..2a65ec1 100644 --- a/messgen/yaml_parser.py +++ b/messgen/yaml_parser.py @@ -222,7 +222,7 @@ def _get_enum_type(type_name: str, type_descriptors: dict[str, dict[str, Any]], type_desc = type_descriptors.get(type_name) assert type_desc - base_type = type_desc.get("base_type") + base_type: str = type_desc.get("base_type") if base_type: type_dependencies.add(_get_dependency_type(type_name, base_type, type_descriptors, type_dependencies)[0]) From 764717ea6a794810082e953932014a7e8df4f640 Mon Sep 17 00:00:00 2001 From: Dmitry Khitrin Date: Fri, 20 Dec 2024 22:52:25 +0200 Subject: [PATCH 5/5] fix: fix mypy error for the yaml parser --- messgen/yaml_parser.py | 2 +- port/js/src/Codec.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/messgen/yaml_parser.py b/messgen/yaml_parser.py index 2a65ec1..7bf65b9 100644 --- a/messgen/yaml_parser.py +++ b/messgen/yaml_parser.py @@ -222,7 +222,7 @@ def _get_enum_type(type_name: str, type_descriptors: dict[str, dict[str, Any]], type_desc = type_descriptors.get(type_name) assert type_desc - base_type: str = type_desc.get("base_type") + base_type = type_desc.get("base_type", '') if base_type: type_dependencies.add(_get_dependency_type(type_name, base_type, type_descriptors, type_dependencies)[0]) diff --git a/port/js/src/Codec.ts b/port/js/src/Codec.ts index 8786cbe..3595a8c 100644 --- a/port/js/src/Codec.ts +++ b/port/js/src/Codec.ts @@ -18,10 +18,10 @@ export class Codec { const idMap = new Map(); for (const message of Object.values(messages)) { - const { message_id: messageId, name, type: typeName } = message; + const { message_id: messageId, name: messageName, type: typeName } = message; const converter = converterFactory.toConverter(typeName); - typeMap.set(name, converter); + typeMap.set(messageName, converter); idMap.set(messageId, converter); } this.typesByName.set(protoName, typeMap);