From 80e72137c27a0bf42e7604976e868cb291a1c6ad Mon Sep 17 00:00:00 2001 From: Arondondon Date: Tue, 4 Mar 2025 12:58:07 +0300 Subject: [PATCH 1/4] Changed archive with .proto files from .tar to .tar.gz --- snet/cli/utils/ipfs_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snet/cli/utils/ipfs_utils.py b/snet/cli/utils/ipfs_utils.py index f627075d..49fccfe5 100644 --- a/snet/cli/utils/ipfs_utils.py +++ b/snet/cli/utils/ipfs_utils.py @@ -54,7 +54,7 @@ def publish_proto_in_ipfs(ipfs_client, protodir): files.sort() tarbytes = io.BytesIO() - tar = tarfile.open(fileobj=tarbytes, mode="w") + tar = tarfile.open(fileobj=tarbytes, mode="w:gz") for f in files: tar.add(f, os.path.basename(f)) tar.close() @@ -79,7 +79,7 @@ def publish_proto_in_filecoin(filecoin_client, protodir): tarbytes = io.BytesIO() - with tarfile.open(fileobj=tarbytes, mode="w") as tar: + with tarfile.open(fileobj=tarbytes, mode="w:gz") as tar: for f in files: tar.add(f, os.path.basename(f)) tarbytes.seek(0) From d926cac7b46635bdd1895b1e46f0fda5b3dcc1ab Mon Sep 17 00:00:00 2001 From: Arondondon Date: Tue, 4 Mar 2025 15:47:57 +0300 Subject: [PATCH 2/4] Fixed generating service stubs with training import. --- snet/cli/commands/mpe_channel.py | 6 +- snet/cli/resources/proto/training.proto | 182 ++++++++++-------- .../resources/proto/training/training.proto | 128 ++++++++++++ snet/cli/utils/utils.py | 23 ++- 4 files changed, 252 insertions(+), 87 deletions(-) create mode 100644 snet/cli/resources/proto/training/training.proto diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py index 62fdad3f..063dc039 100644 --- a/snet/cli/commands/mpe_channel.py +++ b/snet/cli/commands/mpe_channel.py @@ -17,7 +17,8 @@ from snet.cli.utils.token2cogs import cogs2strtoken from snet.cli.utils.ipfs_utils import get_from_ipfs_and_checkhash from snet.cli.utils.utils import abi_decode_struct_to_dict, abi_get_element_by_name, \ - compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto + compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto, \ + check_training_in_proto # we inherit MPEServiceCommand because we need _get_service_metadata_from_registry @@ -602,9 +603,10 @@ def _init_or_update_service_if_needed(self, metadata, service_registration): os.makedirs(spec_dir, mode=0o700) service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash") download_and_safe_extract_proto(service_api_source, spec_dir, self._get_ipfs_client()) + training_added = check_training_in_proto(spec_dir) # compile .proto files - if not compile_proto(Path(spec_dir), service_dir): + if not compile_proto(Path(spec_dir), service_dir, add_training = training_added): raise Exception("Fail to compile %s/*.proto" % spec_dir) # save service_metadata.json in channel_dir diff --git a/snet/cli/resources/proto/training.proto b/snet/cli/resources/proto/training.proto index 86b3074c..8582c666 100644 --- a/snet/cli/resources/proto/training.proto +++ b/snet/cli/resources/proto/training.proto @@ -1,112 +1,128 @@ syntax = "proto3"; -import "google/protobuf/descriptor.proto"; +import "google/protobuf/descriptor.proto"; // Required for indicators to work package training; -option go_package = "../training"; -//Please note that the AI developers need to provide a server implementation of the gprc server of this proto. -message ModelDetails { - //This Id will be generated when you invoke the create_model method and hence doesnt need to be filled when you - //invoke the create model - string model_id = 1; - //define the training method name - string grpc_method_name = 2; - //define the grpc service name , under which the method is defined - string grpc_service_name = 3; - string description = 4; +option go_package = "github.com/singnet/snet-daemon/v5/training"; - string status = 6; - string updated_date = 7; - //List of all the addresses that will have access to this model - repeated string address_list = 8; - // this is optional - string training_data_link = 9; - string model_name = 10; +// Methods that the service provider must implement +service Model { + // Free + // Can pass the address of the model creator + rpc create_model(NewModel) returns (ModelID) {} - string organization_id = 11; - string service_id = 12 ; - string group_id = 13; + // Free + rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {} - //set this to true if you want your model to be used by other AI consumers - bool is_publicly_accessible = 14; + // Paid + rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {} -} + // Paid + rpc validate_model(ValidateRequest) returns (StatusResponse) {} -message AuthorizationDetails { - uint64 current_block = 1; - //Signer can fill in any message here - string message = 2; - //signature of the following message: - //("user specified message", user_address, current_block_number) - bytes signature = 3; - string signer_address = 4; + // Free, one signature for both train_model_price & train_model methods + rpc train_model_price(ModelID) returns (PriceInBaseUnit) {} -} + // Paid + rpc train_model(ModelID) returns (StatusResponse) {} -enum Status { - CREATED = 0; - IN_PROGRESS = 1; - ERRORED = 2; - COMPLETED = 3; - DELETED = 4; -} + // Free + rpc delete_model(ModelID) returns (StatusResponse) { + // After model deletion, the status becomes DELETED in etcd + } -message CreateModelRequest { - AuthorizationDetails authorization = 1; - ModelDetails model_details = 2; + // Free + rpc get_model_status(ModelID) returns (StatusResponse) {} } -//the signer address will get to know all the models associated with this address. -message AccessibleModelsRequest { - string grpc_method_name = 1; - string grpc_service_name = 2; - AuthorizationDetails authorization = 3; -} +message ModelResponse { + string model_id = 1; + Status status = 2; + string created_date = 3; + string updated_date = 4; + string name = 5; + string description = 6; + string grpc_method_name = 7; + string grpc_service_name = 8; -message AccessibleModelsResponse { - repeated ModelDetails list_of_models = 1; -} + // List of all addresses that will have access to this model + repeated string address_list = 9; + + // Access to the model is granted only for use and viewing + bool is_public = 10; + + string training_data_link = 11; -message ModelDetailsRequest { - ModelDetails model_details = 1 ; - AuthorizationDetails authorization = 2; + string created_by_address = 12; + string updated_by_address = 13; } -//helps determine which service end point to call for model training -//format is of type "packageName/serviceName/MethodName", Example :"/example_service.Calculator/estimate_add" -//Daemon will invoke the model training end point , when the below method option is specified -message TrainingMethodOption { - string trainingMethodIndicator = 1; +// Used as input for new_model requests +// The service provider decides whether to use these fields; returning model_id is mandatory +message NewModel { + string name = 1; + string description = 2; + string grpc_method_name = 3; + string grpc_service_name = 4; + + // List of all addresses that will have access to this model + repeated string address_list = 5; + + // Set this to true if you want your model to be accessible by other AI consumers + bool is_public = 6; + + // These parameters will be passed by the daemon + string organization_id = 7; + string service_id = 8; + string group_id = 9; } -extend google.protobuf.MethodOptions { - TrainingMethodOption my_method_option = 9999197; +// This structure must be used by the service provider +message ModelID { + string model_id = 1; } -message UpdateModelRequest { - ModelDetails update_model_details = 1 ; - AuthorizationDetails authorization = 2; +// This structure must be used by the service provider +// Used in the train_model_price method to get the training/validation price +message PriceInBaseUnit { + uint64 price = 1; // cogs, weis, afet, aasi, etc. } +enum Status { + CREATED = 0; + VALIDATING = 1; + VALIDATED = 2; + TRAINING = 3; + READY_TO_USE = 4; // After training is completed + ERRORED = 5; + DELETED = 6; +} -message ModelDetailsResponse { +message StatusResponse { Status status = 1; - ModelDetails model_details = 2; - } -service Model { - - // The AI developer needs to Implement this service and Daemon will call these - // There will be no cost borne by the consumer in calling these methods, - // Pricing will apply when you actually call the training methods defined. - // AI consumer will call all these methods - rpc create_model(CreateModelRequest) returns (ModelDetailsResponse) {} - rpc delete_model(UpdateModelRequest) returns (ModelDetailsResponse) {} - rpc get_model_status(ModelDetailsRequest) returns (ModelDetailsResponse) {} - - // Daemon will implement , however the AI developer should skip implementing these and just provide dummy code. - rpc update_model_access(UpdateModelRequest) returns (ModelDetailsResponse) {} - rpc get_all_models(AccessibleModelsRequest) returns (AccessibleModelsResponse) {} +message UploadInput { + string model_id = 1; + bytes data = 2; + string file_name = 3; + uint64 file_size = 4; // in bytes + uint64 batch_size = 5; + uint64 batch_number = 6; + uint64 batch_count = 7; +} +message ValidateRequest { + string model_id = 2; + string training_data_link = 3; +} -} \ No newline at end of file +extend google.protobuf.MethodOptions { + string default_model_id = 50001; + uint64 max_models_per_user = 50002; // max models per method & user + uint64 dataset_max_size_mb = 50003; // max size of dataset + uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset + uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset + string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3 + string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar + string dataset_description = 50008; // additional free-form requirements +} diff --git a/snet/cli/resources/proto/training/training.proto b/snet/cli/resources/proto/training/training.proto new file mode 100644 index 00000000..8582c666 --- /dev/null +++ b/snet/cli/resources/proto/training/training.proto @@ -0,0 +1,128 @@ +syntax = "proto3"; +import "google/protobuf/descriptor.proto"; // Required for indicators to work +package training; +option go_package = "github.com/singnet/snet-daemon/v5/training"; + +// Methods that the service provider must implement +service Model { + + // Free + // Can pass the address of the model creator + rpc create_model(NewModel) returns (ModelID) {} + + // Free + rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {} + + // Paid + rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {} + + // Paid + rpc validate_model(ValidateRequest) returns (StatusResponse) {} + + // Free, one signature for both train_model_price & train_model methods + rpc train_model_price(ModelID) returns (PriceInBaseUnit) {} + + // Paid + rpc train_model(ModelID) returns (StatusResponse) {} + + // Free + rpc delete_model(ModelID) returns (StatusResponse) { + // After model deletion, the status becomes DELETED in etcd + } + + // Free + rpc get_model_status(ModelID) returns (StatusResponse) {} +} + +message ModelResponse { + string model_id = 1; + Status status = 2; + string created_date = 3; + string updated_date = 4; + string name = 5; + string description = 6; + string grpc_method_name = 7; + string grpc_service_name = 8; + + // List of all addresses that will have access to this model + repeated string address_list = 9; + + // Access to the model is granted only for use and viewing + bool is_public = 10; + + string training_data_link = 11; + + string created_by_address = 12; + string updated_by_address = 13; +} + +// Used as input for new_model requests +// The service provider decides whether to use these fields; returning model_id is mandatory +message NewModel { + string name = 1; + string description = 2; + string grpc_method_name = 3; + string grpc_service_name = 4; + + // List of all addresses that will have access to this model + repeated string address_list = 5; + + // Set this to true if you want your model to be accessible by other AI consumers + bool is_public = 6; + + // These parameters will be passed by the daemon + string organization_id = 7; + string service_id = 8; + string group_id = 9; +} + +// This structure must be used by the service provider +message ModelID { + string model_id = 1; +} + +// This structure must be used by the service provider +// Used in the train_model_price method to get the training/validation price +message PriceInBaseUnit { + uint64 price = 1; // cogs, weis, afet, aasi, etc. +} + +enum Status { + CREATED = 0; + VALIDATING = 1; + VALIDATED = 2; + TRAINING = 3; + READY_TO_USE = 4; // After training is completed + ERRORED = 5; + DELETED = 6; +} + +message StatusResponse { + Status status = 1; +} + +message UploadInput { + string model_id = 1; + bytes data = 2; + string file_name = 3; + uint64 file_size = 4; // in bytes + uint64 batch_size = 5; + uint64 batch_number = 6; + uint64 batch_count = 7; +} + +message ValidateRequest { + string model_id = 2; + string training_data_link = 3; +} + +extend google.protobuf.MethodOptions { + string default_model_id = 50001; + uint64 max_models_per_user = 50002; // max models per method & user + uint64 dataset_max_size_mb = 50003; // max size of dataset + uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset + uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset + string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3 + string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar + string dataset_description = 50008; // additional free-form requirements +} diff --git a/snet/cli/utils/utils.py b/snet/cli/utils/utils.py index b2b38479..0efc8d20 100644 --- a/snet/cli/utils/utils.py +++ b/snet/cli/utils/utils.py @@ -146,7 +146,7 @@ def get_cli_version(): return distribution("snet.cli").version -def compile_proto(entry_path, codegen_dir, proto_file=None): +def compile_proto(entry_path, codegen_dir, proto_file=None, add_training=False): try: if not os.path.exists(codegen_dir): os.makedirs(codegen_dir) @@ -157,6 +157,10 @@ def compile_proto(entry_path, codegen_dir, proto_file=None): "-I{}".format(proto_include) ] + if add_training: + training_include = RESOURCES_PATH.joinpath("proto", "training") + compiler_args.append("-I{}".format(training_include)) + compiler_args.insert(0, "protoc") compiler_args.append("--python_out={}".format(codegen_dir)) compiler_args.append("--grpc_python_out={}".format(codegen_dir)) @@ -167,6 +171,9 @@ def compile_proto(entry_path, codegen_dir, proto_file=None): else: compiler_args.extend([str(p) for p in entry_path.glob("**/*.proto")]) + if add_training: + compiler_args.append(str(training_include.joinpath("training.proto"))) + if not compiler(compiler_args): return True else: @@ -346,4 +353,16 @@ def download_and_safe_extract_proto(service_api_source, protodir, ipfs_client): os.remove(fullname) print("%s removed." % fullname) # now it is safe to call extractall - f.extractall(protodir) \ No newline at end of file + f.extractall(protodir) + + +def check_training_in_proto(protodir) -> bool: + files = os.listdir(protodir) + for file in files: + if ".proto" not in file: + continue + with open(protodir.joinpath(file), "r") as f: + proto_text = f.read() + if 'import "training.proto";' in proto_text: + return True + return False \ No newline at end of file From a453fe5ba97e9278afccb2f78645d7dbe3f3c2b2 Mon Sep 17 00:00:00 2001 From: Arondondon Date: Tue, 4 Mar 2025 18:14:36 +0300 Subject: [PATCH 3/4] Fixed generate-client-library command --- snet/cli/commands/sdk_command.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/snet/cli/commands/sdk_command.py b/snet/cli/commands/sdk_command.py index fae953ac..0f8b05d0 100644 --- a/snet/cli/commands/sdk_command.py +++ b/snet/cli/commands/sdk_command.py @@ -1,7 +1,7 @@ import os from pathlib import Path, PurePath -from snet.cli.utils.utils import compile_proto, download_and_safe_extract_proto +from snet.cli.utils.utils import compile_proto, download_and_safe_extract_proto, check_training_in_proto from snet.cli.commands.mpe_service import MPEServiceCommand @@ -28,8 +28,10 @@ def generate_client_library(self): # Receive proto files download_and_safe_extract_proto(service_api_source, library_dir_path, self._get_ipfs_client()) + training_added = check_training_in_proto(library_dir_path) + # Compile proto files - compile_proto(Path(library_dir_path), library_dir_path) + compile_proto(Path(library_dir_path), library_dir_path, add_training = training_added) self._printout( 'client libraries for service with id "{}" in org with id "{}" generated at {}'.format(library_service_id, From 854787d33d44039d508b4e1ed7680c125808c7cd Mon Sep 17 00:00:00 2001 From: Arondondon Date: Tue, 4 Mar 2025 18:58:28 +0300 Subject: [PATCH 4/4] Fixed manifest --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 11fc35c8..dc34359b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -include snet/cli/resources/* +recursive-include snet/cli/resources * recursive-exclude * .git recursive-exclude * node_modules