From ed3c6dcbe775cf114e187753ced8adc3da6a79d1 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 16:12:31 +0200 Subject: [PATCH 01/44] rpc config migration v7 to v8 --- node_management/migration/rpc/README.md | 3 + node_management/migration/rpc/config.toml | 136 ++++++++++++++++++ .../migration/rpc/config_v7.1.8.toml | 131 +++++++++++++++++ .../migration/rpc/config_v8.0.2.toml | 118 +++++++++++++++ .../rpc/migrate-rpc-config-v7-to-v8.py | 118 +++++++++++++++ 5 files changed, 506 insertions(+) create mode 100644 node_management/migration/rpc/README.md create mode 100644 node_management/migration/rpc/config.toml create mode 100644 node_management/migration/rpc/config_v7.1.8.toml create mode 100644 node_management/migration/rpc/config_v8.0.2.toml create mode 100644 node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py diff --git a/node_management/migration/rpc/README.md b/node_management/migration/rpc/README.md new file mode 100644 index 00000000..53cfc4c8 --- /dev/null +++ b/node_management/migration/rpc/README.md @@ -0,0 +1,3 @@ +https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml + +https://mainnet-data.supra.com/configs/config.toml \ No newline at end of file diff --git a/node_management/migration/rpc/config.toml b/node_management/migration/rpc/config.toml new file mode 100644 index 00000000..2b67889b --- /dev/null +++ b/node_management/migration/rpc/config.toml @@ -0,0 +1,136 @@ +# Version: v7.1.8 +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_instance.chain_id = 8 +# The length of an epoch in seconds. +chain_instance.epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +chain_instance.recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +chain_instance.voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +chain_instance.is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +chain_instance.genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:30000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = false +# The path to the TLS certificate for the connection with the attached validator. +# The path to the private key to be used when negotiating TLS connections. +# The path to the TLS root certificate authority certificate. +# The websocket address of the attached validator. +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +# The number of seconds to wait before retrying a block sync request. + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +consensus_access_tokens = [] +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retring a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:27000" +description = "LocalNet" +mode = "Server" +[synchronization.ws] +consensus_rpc = "ws://:26000" +consensus_client_cert_path = "./configs/client_supra_certificate.pem" +consensus_client_private_key_path = "./configs/client_supra_key.pem" +consensus_root_ca_cert_path = "./configs/ca_certificate.pem" + +[chain_state_assembler] +sync_retry_interval_in_secs = 1 +certified_block_cache_bucket_size = 50 diff --git a/node_management/migration/rpc/config_v7.1.8.toml b/node_management/migration/rpc/config_v7.1.8.toml new file mode 100644 index 00000000..8e73ca30 --- /dev/null +++ b/node_management/migration/rpc/config_v7.1.8.toml @@ -0,0 +1,131 @@ +# Version: v7.1.8 +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_instance.chain_id = 8 +# The length of an epoch in seconds. +chain_instance.epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +chain_instance.recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +chain_instance.voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +chain_instance.is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +chain_instance.genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:30000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = false +# The path to the TLS certificate for the connection with the attached validator. +consensus_client_cert_path = "./configs/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +consensus_client_private_key_path = "./configs/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +consensus_root_ca_cert_path = "./configs/ca_certificate.pem" +# The websocket address of the attached validator. +consensus_rpc = "ws://:26000" +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +# The number of seconds to wait before retrying a block sync request. +sync_retry_interval_in_secs = 1 + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retring a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:27000" +description = "LocalNet" +mode = "Server" \ No newline at end of file diff --git a/node_management/migration/rpc/config_v8.0.2.toml b/node_management/migration/rpc/config_v8.0.2.toml new file mode 100644 index 00000000..2331537b --- /dev/null +++ b/node_management/migration/rpc/config_v8.0.2.toml @@ -0,0 +1,118 @@ +# Version: v8.0.2 +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 6 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = true +# Tuesday, September 17, 2024 12:00:00.000 PM (UTC) +genesis_timestamp_microseconds = 1726574400000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +certified_block_cache_bucket_size = 50 +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The path to the TLS certificate for the connection with the attached validator. +consensus_client_cert_path = "./configs/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +consensus_client_private_key_path = "./configs/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +consensus_root_ca_cert_path = "./configs/ca_certificate.pem" +# The websocket address of the attached validator. +consensus_rpc = "ws://:26000" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. The below settings are the default values required for use in RPC nodes run by validator node operators. They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-testnet.supra.com" +description = "RPC For Supra Scan and Faucet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] + +url = "https://rpc-testnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:27000" +description = "LocalNet" +mode = "Server" \ No newline at end of file diff --git a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py b/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py new file mode 100644 index 00000000..83dc0545 --- /dev/null +++ b/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py @@ -0,0 +1,118 @@ +import tomlkit + + +def migrate_sync_ws_parameters(toml_data): + # v7 config should not have a [synchronization] table + if "synchronization" in toml_data: + raise SystemExit( + "Error: [synchronization] table should not exist in v7 config." + ) + + # List of keys to move, in the order they should appear + keys_to_move = [ + "consensus_rpc", + "consensus_client_cert_path", + "consensus_client_private_key_path", + "consensus_root_ca_cert_path", + ] + + # Create [synchronization.ws] table + sync_table = tomlkit.table() + # We'll collect the items to move, preserving order and comments + items_to_remove = [] + for item in toml_data.value.body: + if item.key and item.key in keys_to_move: + # Move the entire item (with comments) + sync_table.add(item.key, item.value) + items_to_remove.append(item.key) + print(f"Moving `{item.key}` from root level to [synchronization.ws]") + # Remove the items from the root (after iteration to avoid mutation during loop) + for key in items_to_remove: + del toml_data[key] + # Warn for missing keys + for key in keys_to_move: + if key not in sync_table: + print(f"Warning: `{key}` not found in root level") + sync_table.add(key, None) + toml_data["synchronization"] = tomlkit.table() + toml_data["synchronization"]["ws"] = sync_table + + +def migrate_chain_state_assembler_parameters(toml_data): + # v7 config should not have a [chain_state_assembler] table + if "chain_state_assembler" in toml_data: + raise SystemExit( + "Error: [chain_state_assembler] table should not exist in v7 config." + ) + + # Explicitly fetch and move these keys from the root + # level to the [chain_state_assembler] table + chain_assembler_keys = { + "sync_retry_interval_in_secs": None, + } + for key in chain_assembler_keys: + if key in toml_data: + value = toml_data.pop(key) + if value is None: + print( + f"Warning: `{key}` does not exist or its value is None, your config must be invalid and you should check it manually after migration" + ) + continue + else: + print(f"Moving `{key}` from root level to [chain_state_assembler]") + if value != 1: + print( + f"Warning: `{key}` value is {value}, will be set to `1` as recommended by Supra." + ) + value = 1 + chain_assembler_keys[key] = value + else: + print(f"Warning: `{key}` not found in root level") + + # Create [chain_state_assembler] table + chain_assembler_table = tomlkit.table() + for key, value in chain_assembler_keys.items(): + chain_assembler_table[key] = value + new_parameters = {"certified_block_cache_bucket_size": 50} + for key, value in new_parameters.items(): + print(f"Adding `{key} = {value}` to [chain_state_assembler]") + chain_assembler_table[key] = value + toml_data["chain_state_assembler"] = chain_assembler_table + + +def add_new_root_parameters(toml_data): + new_parameters = {"consensus_access_tokens": []} + for key, value in new_parameters.items(): + print(f"Adding `{key} = {value}` to root level") + toml_data[key] = value + + +def migrate_config(old_path: str, new_path: str): + with open(old_path, "r") as f: + toml_data = tomlkit.parse(f.read()) + + add_new_root_parameters(toml_data) + + migrate_sync_ws_parameters(toml_data) + + migrate_chain_state_assembler_parameters(toml_data) + + # Write new config + with open(new_path, "w") as f: + f.write(tomlkit.dumps(toml_data)) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Migrate RPC config from v7 to v8") + parser.add_argument( + "--v7", required=True, type=str, help="Path to the v7 config file" + ) + parser.add_argument( + "--v8", required=True, type=str, help="Path to save the v8 config file" + ) + args = parser.parse_args() + assert args.v7 != args.v8, "Error: v7 and v8 config paths must be different" + migrate_config(args.v7, args.v8) + print(f"Config migrated from {args.v7} to {args.v8}") From bdc4e3cbfeedd7c89633ba09f22b9b0028caf95c Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 16:23:15 +0200 Subject: [PATCH 02/44] rpc config migration v7 to v8 --- .../rpc/migrate-rpc-config-v7-to-v8.py | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py b/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py index 83dc0545..8df3208f 100644 --- a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py +++ b/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py @@ -8,32 +8,31 @@ def migrate_sync_ws_parameters(toml_data): "Error: [synchronization] table should not exist in v7 config." ) - # List of keys to move, in the order they should appear - keys_to_move = [ - "consensus_rpc", - "consensus_client_cert_path", - "consensus_client_private_key_path", - "consensus_root_ca_cert_path", - ] - + # Explicitly fetch and move these keys from the root + # level to the [synchronization.ws] table + sync_ws_parameters = { + "consensus_rpc": None, + "consensus_client_cert_path": None, + "consensus_client_private_key_path": None, + "consensus_root_ca_cert_path": None, + } + for key in sync_ws_parameters: + if key in toml_data: + value = toml_data.pop(key) + if value is None: + print( + f"Warning: `{key}` does not exist or its value is None, your config must be invalid and you should check it manually after migration" + ) + continue + else: + print(f"Moving `{key}` from root level to [synchronization.ws]") + sync_ws_parameters[key] = value + else: + print(f"Warning: `{key}` not found in root level") # Create [synchronization.ws] table sync_table = tomlkit.table() - # We'll collect the items to move, preserving order and comments - items_to_remove = [] - for item in toml_data.value.body: - if item.key and item.key in keys_to_move: - # Move the entire item (with comments) - sync_table.add(item.key, item.value) - items_to_remove.append(item.key) - print(f"Moving `{item.key}` from root level to [synchronization.ws]") - # Remove the items from the root (after iteration to avoid mutation during loop) - for key in items_to_remove: - del toml_data[key] - # Warn for missing keys - for key in keys_to_move: - if key not in sync_table: - print(f"Warning: `{key}` not found in root level") - sync_table.add(key, None) + for key, value in sync_ws_parameters.items(): + sync_table[key] = value toml_data["synchronization"] = tomlkit.table() toml_data["synchronization"]["ws"] = sync_table From 8af24f4bc1110168e8c959630ad6d69eeef59a2b Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 16:44:45 +0200 Subject: [PATCH 03/44] show diff after migration --- node_management/migration/rpc/config.toml | 1 + .../migration/rpc/config_v7.1.8.toml | 2 +- .../rpc/migrate-rpc-config-v7-to-v8.py | 33 +++++++++++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/node_management/migration/rpc/config.toml b/node_management/migration/rpc/config.toml index 2b67889b..d397310e 100644 --- a/node_management/migration/rpc/config.toml +++ b/node_management/migration/rpc/config.toml @@ -125,6 +125,7 @@ mode = "Server" url = "http://localhost:27000" description = "LocalNet" mode = "Server" + [synchronization.ws] consensus_rpc = "ws://:26000" consensus_client_cert_path = "./configs/client_supra_certificate.pem" diff --git a/node_management/migration/rpc/config_v7.1.8.toml b/node_management/migration/rpc/config_v7.1.8.toml index 8e73ca30..5f0eeb6a 100644 --- a/node_management/migration/rpc/config_v7.1.8.toml +++ b/node_management/migration/rpc/config_v7.1.8.toml @@ -128,4 +128,4 @@ mode = "Server" [[allowed_origin]] url = "http://localhost:27000" description = "LocalNet" -mode = "Server" \ No newline at end of file +mode = "Server" diff --git a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py b/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py index 8df3208f..bc744f9a 100644 --- a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py +++ b/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py @@ -1,5 +1,18 @@ +from copy import deepcopy + import tomlkit +def print_colored_diff(diff): + # The color is added here manually using ANSI escape codes. + for line in diff: + if line.startswith('+') and not line.startswith('+++'): + print(f"\033[32m{line}\033[0m", end='') # Green for additions + elif line.startswith('-') and not line.startswith('---'): + print(f"\033[31m{line}\033[0m", end='') # Red for deletions + elif line.startswith('@@'): + print(f"\033[36m{line}\033[0m", end='') # Cyan for hunk headers + else: + print(line, end='') def migrate_sync_ws_parameters(toml_data): # v7 config should not have a [synchronization] table @@ -90,16 +103,31 @@ def migrate_config(old_path: str, new_path: str): with open(old_path, "r") as f: toml_data = tomlkit.parse(f.read()) - add_new_root_parameters(toml_data) + # Backup old config + toml_data_v7 = deepcopy(toml_data) + # Do migrations migrate_sync_ws_parameters(toml_data) - migrate_chain_state_assembler_parameters(toml_data) + add_new_root_parameters(toml_data) # Write new config with open(new_path, "w") as f: f.write(tomlkit.dumps(toml_data)) + print( + f"Config migrated from {args.v7} to {args.v8}.\nPlease check the new config file for any additional changes.\n" + ) + + # Print the diff + print("Diff between v7 and v8 config:") + v7_str = tomlkit.dumps(toml_data_v7).splitlines(keepends=True) + v8_str = tomlkit.dumps(toml_data).splitlines(keepends=True) + import difflib + + diff = difflib.unified_diff(v7_str, v8_str, fromfile="v7", tofile="v8") + print_colored_diff(diff) + if __name__ == "__main__": import argparse @@ -114,4 +142,3 @@ def migrate_config(old_path: str, new_path: str): args = parser.parse_args() assert args.v7 != args.v8, "Error: v7 and v8 config paths must be different" migrate_config(args.v7, args.v8) - print(f"Config migrated from {args.v7} to {args.v8}") From bf02ca37ff5a584a579a173d87110425cb268876 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 17:29:46 +0200 Subject: [PATCH 04/44] packaging script --- node_management/migration/.gitignore | 7 ++ ...o-v8.py => migrate_rpc_config_v7_to_v8.py} | 72 ++++++++++++++----- node_management/migration/rpc/pyproject.toml | 15 ++++ 3 files changed, 75 insertions(+), 19 deletions(-) create mode 100644 node_management/migration/.gitignore rename node_management/migration/rpc/{migrate-rpc-config-v7-to-v8.py => migrate_rpc_config_v7_to_v8.py} (71%) create mode 100644 node_management/migration/rpc/pyproject.toml diff --git a/node_management/migration/.gitignore b/node_management/migration/.gitignore new file mode 100644 index 00000000..7bbb8acc --- /dev/null +++ b/node_management/migration/.gitignore @@ -0,0 +1,7 @@ +# Ignore all files in the ruff cache directory +**/.ruff_cache + +**/build/ + +# egg-info +**/*.egg-info/ \ No newline at end of file diff --git a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py b/node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py similarity index 71% rename from node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py rename to node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py index bc744f9a..4ae8ff17 100644 --- a/node_management/migration/rpc/migrate-rpc-config-v7-to-v8.py +++ b/node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py @@ -1,18 +1,21 @@ from copy import deepcopy import tomlkit +import difflib + def print_colored_diff(diff): # The color is added here manually using ANSI escape codes. for line in diff: - if line.startswith('+') and not line.startswith('+++'): - print(f"\033[32m{line}\033[0m", end='') # Green for additions - elif line.startswith('-') and not line.startswith('---'): - print(f"\033[31m{line}\033[0m", end='') # Red for deletions - elif line.startswith('@@'): - print(f"\033[36m{line}\033[0m", end='') # Cyan for hunk headers + if line.startswith("+") and not line.startswith("+++"): + print(f"\033[32m{line}\033[0m", end="") # Green for additions + elif line.startswith("-") and not line.startswith("---"): + print(f"\033[31m{line}\033[0m", end="") # Red for deletions + elif line.startswith("@@"): + print(f"\033[36m{line}\033[0m", end="") # Cyan for hunk headers else: - print(line, end='') + print(line, end="") + def migrate_sync_ws_parameters(toml_data): # v7 config should not have a [synchronization] table @@ -74,7 +77,7 @@ def migrate_chain_state_assembler_parameters(toml_data): print(f"Moving `{key}` from root level to [chain_state_assembler]") if value != 1: print( - f"Warning: `{key}` value is {value}, will be set to `1` as recommended by Supra." + f"Warning: `{key} = {value}`, will be set to `1` as recommended by Supra." ) value = 1 chain_assembler_keys[key] = value @@ -100,45 +103,76 @@ def add_new_root_parameters(toml_data): def migrate_config(old_path: str, new_path: str): + default_backup_path = old_path + ".bak" + if old_path == new_path: + print(f"Warning: The source and destination paths are the same ({old_path}).") + print(f"A backup of your original config will be saved to: {default_backup_path}") + confirm = input( + "This will overwrite your original config file. Continue? [y/N]: " + ) + if confirm.lower() != "y": + print("Aborted by user.") + return + with open(old_path, "r") as f: toml_data = tomlkit.parse(f.read()) # Backup old config toml_data_v7 = deepcopy(toml_data) - # Do migrations + print("Starting migration from v7 to v8...") migrate_sync_ws_parameters(toml_data) migrate_chain_state_assembler_parameters(toml_data) add_new_root_parameters(toml_data) + print(f"Backing up old config to {default_backup_path}") + tomlkit.dump(toml_data_v7, open(default_backup_path, "w")) + # Write new config + print(f"Writing new config to {new_path}") with open(new_path, "w") as f: f.write(tomlkit.dumps(toml_data)) - print( - f"Config migrated from {args.v7} to {args.v8}.\nPlease check the new config file for any additional changes.\n" - ) - # Print the diff print("Diff between v7 and v8 config:") v7_str = tomlkit.dumps(toml_data_v7).splitlines(keepends=True) v8_str = tomlkit.dumps(toml_data).splitlines(keepends=True) - import difflib diff = difflib.unified_diff(v7_str, v8_str, fromfile="v7", tofile="v8") print_colored_diff(diff) + print( + f""" + Config migrated from {old_path} to {new_path}. + Please double check above for the diff between v7 and v8 config. + Please ensure to use the new config file for v8 binary. + """ + ) -if __name__ == "__main__": + +def main(): import argparse parser = argparse.ArgumentParser(description="Migrate RPC config from v7 to v8") parser.add_argument( - "--v7", required=True, type=str, help="Path to the v7 config file" + "-f", + "--from-file", + required=True, + type=str, + metavar="", + help="Path to the v7 config file", ) parser.add_argument( - "--v8", required=True, type=str, help="Path to save the v8 config file" + "-t", + "--to-file", + required=True, + type=str, + metavar="", + help="Path to save the v8 config file", ) args = parser.parse_args() - assert args.v7 != args.v8, "Error: v7 and v8 config paths must be different" - migrate_config(args.v7, args.v8) + migrate_config(args.from_file, args.to_file) + + +if __name__ == "__main__": + main() diff --git a/node_management/migration/rpc/pyproject.toml b/node_management/migration/rpc/pyproject.toml new file mode 100644 index 00000000..61353550 --- /dev/null +++ b/node_management/migration/rpc/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "supra-rpc-migrator" +version = "0.1.0" +description = "A CLI tool to migrate Supra RPC config from v7 to v8." +authors = [ + { name = "Kaiqi Chen" } +] +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "tomlkit>=0.11.0" +] + +[project.scripts] +migrate-rpc-config-v7-to-v8 = "migrate_rpc_config_v7_to_v8:main" From 5367ec0afc259b34c7afd078cbd93cc5b06d03fc Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 21:26:26 +0200 Subject: [PATCH 05/44] support migrate path choices --- node_management/migration/rpc/README.md | 6 + .../migration/rpc/config_v9.0.7.toml | 120 ++++++++++++ .../rpc/migrate_rpc_config_v7_to_v8.py | 178 ------------------ node_management/migration/rpc/pyproject.toml | 6 +- .../rpc/src/migrate_rpc_config/__init__.py | 1 + .../rpc/src/migrate_rpc_config/cli.py | 108 +++++++++++ .../src/migrate_rpc_config/from_v7_to_v8.py | 93 +++++++++ .../src/migrate_rpc_config/from_v8_to_v9.py | 48 +++++ .../rpc/src/migrate_rpc_config/utils.py | 19 ++ 9 files changed, 398 insertions(+), 181 deletions(-) create mode 100644 node_management/migration/rpc/config_v9.0.7.toml delete mode 100644 node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py create mode 100644 node_management/migration/rpc/src/migrate_rpc_config/__init__.py create mode 100644 node_management/migration/rpc/src/migrate_rpc_config/cli.py create mode 100644 node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py create mode 100644 node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py create mode 100644 node_management/migration/rpc/src/migrate_rpc_config/utils.py diff --git a/node_management/migration/rpc/README.md b/node_management/migration/rpc/README.md index 53cfc4c8..493eff69 100644 --- a/node_management/migration/rpc/README.md +++ b/node_management/migration/rpc/README.md @@ -1,3 +1,9 @@ + +#### v9 +https://testnet-snapshot.supra.com/configs/config_v9.0.7.toml + +#### v8 https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml +#### v7 https://mainnet-data.supra.com/configs/config.toml \ No newline at end of file diff --git a/node_management/migration/rpc/config_v9.0.7.toml b/node_management/migration/rpc/config_v9.0.7.toml new file mode 100644 index 00000000..9718ec34 --- /dev/null +++ b/node_management/migration/rpc/config_v9.0.7.toml @@ -0,0 +1,120 @@ +# Version: v9.0.7 +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 6 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = true +# Tuesday, September 17, 2024 12:00:00.000 PM (UTC) +genesis_timestamp_microseconds = 1726574400000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +certified_block_cache_bucket_size = 50 +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The websocket address of the attached validator. +consensus_rpc = "ws://:26000" + +[synchronization.ws.certificates] +# The path to the TLS certificate for the connection with the attached validator. +cert_path = "./configs/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./configs/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. The below settings are the default values required for use in RPC nodes run by validator node operators. They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-testnet.supra.com" +description = "RPC For Supra Scan and Faucet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] + +url = "https://rpc-testnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-testnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:27000" +description = "LocalNet" +mode = "Server" diff --git a/node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py b/node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py deleted file mode 100644 index 4ae8ff17..00000000 --- a/node_management/migration/rpc/migrate_rpc_config_v7_to_v8.py +++ /dev/null @@ -1,178 +0,0 @@ -from copy import deepcopy - -import tomlkit -import difflib - - -def print_colored_diff(diff): - # The color is added here manually using ANSI escape codes. - for line in diff: - if line.startswith("+") and not line.startswith("+++"): - print(f"\033[32m{line}\033[0m", end="") # Green for additions - elif line.startswith("-") and not line.startswith("---"): - print(f"\033[31m{line}\033[0m", end="") # Red for deletions - elif line.startswith("@@"): - print(f"\033[36m{line}\033[0m", end="") # Cyan for hunk headers - else: - print(line, end="") - - -def migrate_sync_ws_parameters(toml_data): - # v7 config should not have a [synchronization] table - if "synchronization" in toml_data: - raise SystemExit( - "Error: [synchronization] table should not exist in v7 config." - ) - - # Explicitly fetch and move these keys from the root - # level to the [synchronization.ws] table - sync_ws_parameters = { - "consensus_rpc": None, - "consensus_client_cert_path": None, - "consensus_client_private_key_path": None, - "consensus_root_ca_cert_path": None, - } - for key in sync_ws_parameters: - if key in toml_data: - value = toml_data.pop(key) - if value is None: - print( - f"Warning: `{key}` does not exist or its value is None, your config must be invalid and you should check it manually after migration" - ) - continue - else: - print(f"Moving `{key}` from root level to [synchronization.ws]") - sync_ws_parameters[key] = value - else: - print(f"Warning: `{key}` not found in root level") - # Create [synchronization.ws] table - sync_table = tomlkit.table() - for key, value in sync_ws_parameters.items(): - sync_table[key] = value - toml_data["synchronization"] = tomlkit.table() - toml_data["synchronization"]["ws"] = sync_table - - -def migrate_chain_state_assembler_parameters(toml_data): - # v7 config should not have a [chain_state_assembler] table - if "chain_state_assembler" in toml_data: - raise SystemExit( - "Error: [chain_state_assembler] table should not exist in v7 config." - ) - - # Explicitly fetch and move these keys from the root - # level to the [chain_state_assembler] table - chain_assembler_keys = { - "sync_retry_interval_in_secs": None, - } - for key in chain_assembler_keys: - if key in toml_data: - value = toml_data.pop(key) - if value is None: - print( - f"Warning: `{key}` does not exist or its value is None, your config must be invalid and you should check it manually after migration" - ) - continue - else: - print(f"Moving `{key}` from root level to [chain_state_assembler]") - if value != 1: - print( - f"Warning: `{key} = {value}`, will be set to `1` as recommended by Supra." - ) - value = 1 - chain_assembler_keys[key] = value - else: - print(f"Warning: `{key}` not found in root level") - - # Create [chain_state_assembler] table - chain_assembler_table = tomlkit.table() - for key, value in chain_assembler_keys.items(): - chain_assembler_table[key] = value - new_parameters = {"certified_block_cache_bucket_size": 50} - for key, value in new_parameters.items(): - print(f"Adding `{key} = {value}` to [chain_state_assembler]") - chain_assembler_table[key] = value - toml_data["chain_state_assembler"] = chain_assembler_table - - -def add_new_root_parameters(toml_data): - new_parameters = {"consensus_access_tokens": []} - for key, value in new_parameters.items(): - print(f"Adding `{key} = {value}` to root level") - toml_data[key] = value - - -def migrate_config(old_path: str, new_path: str): - default_backup_path = old_path + ".bak" - if old_path == new_path: - print(f"Warning: The source and destination paths are the same ({old_path}).") - print(f"A backup of your original config will be saved to: {default_backup_path}") - confirm = input( - "This will overwrite your original config file. Continue? [y/N]: " - ) - if confirm.lower() != "y": - print("Aborted by user.") - return - - with open(old_path, "r") as f: - toml_data = tomlkit.parse(f.read()) - - # Backup old config - toml_data_v7 = deepcopy(toml_data) - - print("Starting migration from v7 to v8...") - migrate_sync_ws_parameters(toml_data) - migrate_chain_state_assembler_parameters(toml_data) - add_new_root_parameters(toml_data) - - print(f"Backing up old config to {default_backup_path}") - tomlkit.dump(toml_data_v7, open(default_backup_path, "w")) - - # Write new config - print(f"Writing new config to {new_path}") - with open(new_path, "w") as f: - f.write(tomlkit.dumps(toml_data)) - - # Print the diff - print("Diff between v7 and v8 config:") - v7_str = tomlkit.dumps(toml_data_v7).splitlines(keepends=True) - v8_str = tomlkit.dumps(toml_data).splitlines(keepends=True) - - diff = difflib.unified_diff(v7_str, v8_str, fromfile="v7", tofile="v8") - print_colored_diff(diff) - - print( - f""" - Config migrated from {old_path} to {new_path}. - Please double check above for the diff between v7 and v8 config. - Please ensure to use the new config file for v8 binary. - """ - ) - - -def main(): - import argparse - - parser = argparse.ArgumentParser(description="Migrate RPC config from v7 to v8") - parser.add_argument( - "-f", - "--from-file", - required=True, - type=str, - metavar="", - help="Path to the v7 config file", - ) - parser.add_argument( - "-t", - "--to-file", - required=True, - type=str, - metavar="", - help="Path to save the v8 config file", - ) - args = parser.parse_args() - migrate_config(args.from_file, args.to_file) - - -if __name__ == "__main__": - main() diff --git a/node_management/migration/rpc/pyproject.toml b/node_management/migration/rpc/pyproject.toml index 61353550..09af0498 100644 --- a/node_management/migration/rpc/pyproject.toml +++ b/node_management/migration/rpc/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "supra-rpc-migrator" +name = "migrate-rpc-config" version = "0.1.0" -description = "A CLI tool to migrate Supra RPC config from v7 to v8." +description = "A CLI tool to migrate Supra RPC config." authors = [ { name = "Kaiqi Chen" } ] @@ -12,4 +12,4 @@ dependencies = [ ] [project.scripts] -migrate-rpc-config-v7-to-v8 = "migrate_rpc_config_v7_to_v8:main" +migrate-rpc-config = "migrate_rpc_config.cli:main" diff --git a/node_management/migration/rpc/src/migrate_rpc_config/__init__.py b/node_management/migration/rpc/src/migrate_rpc_config/__init__.py new file mode 100644 index 00000000..6ed3e492 --- /dev/null +++ b/node_management/migration/rpc/src/migrate_rpc_config/__init__.py @@ -0,0 +1 @@ +# This file is intentionally left blank to mark this directory as a package. diff --git a/node_management/migration/rpc/src/migrate_rpc_config/cli.py b/node_management/migration/rpc/src/migrate_rpc_config/cli.py new file mode 100644 index 00000000..13b597e5 --- /dev/null +++ b/node_management/migration/rpc/src/migrate_rpc_config/cli.py @@ -0,0 +1,108 @@ +from .from_v7_to_v8 import migrate_v7_to_v8 +from .from_v8_to_v9 import migrate_v8_to_v9 + +from . import utils +import tomlkit +from copy import deepcopy + + +RPC_CONFIG_MIGRATE_PATH = { + "v7->v8": [migrate_v7_to_v8], + "v8->v9": [migrate_v8_to_v9], + "v7->v9": [migrate_v7_to_v8, migrate_v8_to_v9], +} + + +def migrate_config(migrate_path: str, from_path: str, to_path: str): + # Migrate config + migrate_functions = RPC_CONFIG_MIGRATE_PATH.pop(migrate_path) + if not migrate_functions: + raise SystemExit(f"Error: unsupported migration path {migrate_path}.") + from_version = migrate_path.split("->")[0] + to_version = migrate_path.split("->")[1] + + default_backup_path = f"{from_path}_{from_version}.bak" + if from_path == to_path: + print(f"Warning: The source and destination paths are the same ({from_path}).") + print( + f"A backup of your original config will be saved to: {default_backup_path}" + ) + confirm = input( + "This will overwrite your original config file. Continue? [y/N]: " + ) + if confirm.lower() != "y": + print("Aborted by user.") + return + + with open(from_path, "r") as f: + toml_data = tomlkit.parse(f.read()) + + # Backup old config + original_toml_data = deepcopy(toml_data) + + + for fn in migrate_functions: + print(f"Running migration function: {fn.__name__}") + fn(toml_data) + + print(f"Backing up old config to {default_backup_path}") + tomlkit.dump(original_toml_data, open(default_backup_path, "w")) + + # Write new config + print(f"Writing new config to {to_path}") + with open(to_path, "w") as f: + f.write(tomlkit.dumps(toml_data)) + + # Print the diff + from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) + to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) + + utils.unified_diff( + from_str, + to_str, + fromfile=from_version, + tofile=to_version, + ) + + print( + f""" + Config migrated from {from_path} to {to_path}. + Please double check above for the diff between old and new config. + Please ensure to use the new config file for target binary version. + """ + ) + + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description=f"Migrate RPC config: {RPC_CONFIG_MIGRATE_PATH.keys()}" + ) + parser.add_argument( + "-p", + "--migrate-path", + required=True, + type=str, + choices=RPC_CONFIG_MIGRATE_PATH.keys(), + metavar="", + help="Path to migrate the config", + ) + parser.add_argument( + "-f", + "--from-file", + required=True, + type=str, + metavar="", + help="Path to the config file to migrate", + ) + parser.add_argument( + "-t", + "--to-file", + required=True, + type=str, + metavar="", + help="Path to the new config file", + ) + args = parser.parse_args() + migrate_config(args.migrate_path, args.from_file, args.to_file) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py new file mode 100644 index 00000000..ade30c5d --- /dev/null +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py @@ -0,0 +1,93 @@ +""" +v7 to v8 migration functions for Supra's RPC configuration. +""" + +import tomlkit + + +def __migrate_sync_ws_parameters(toml_data): + # v7 config should not have a [synchronization] table + if "synchronization" in toml_data: + raise SystemExit( + "Error: [synchronization] table should not exist in v7 config. Please check your migration path matching the version of your config file." + ) + + # Explicitly fetch and move these keys from the root + # level to the [synchronization.ws] table + sync_ws_keys = { + "consensus_rpc": None, + "consensus_client_cert_path": None, + "consensus_client_private_key_path": None, + "consensus_root_ca_cert_path": None, + } + for key in sync_ws_keys: + if key in toml_data: + value = toml_data.pop(key) + if not value: + print( + f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" + ) + print(f"Moving `{key}` from root level to [synchronization.ws]") + sync_ws_keys[key] = value + else: + print(f"Warning: `{key}` not found in root level") + # Create [synchronization.ws] table + sync_table = tomlkit.table() + for key, value in sync_ws_keys.items(): + sync_table[key] = value + toml_data["synchronization"] = tomlkit.table() + toml_data["synchronization"]["ws"] = sync_table + + +def __migrate_chain_state_assembler_parameters(toml_data): + # v7 config should not have a [chain_state_assembler] table + if "chain_state_assembler" in toml_data: + raise SystemExit( + "Error: [chain_state_assembler] table should not exist in v7 config." + ) + + # Explicitly fetch and move these keys from the root + # level to the [chain_state_assembler] table + chain_assembler_keys = { + "sync_retry_interval_in_secs": None, + } + for key in chain_assembler_keys: + if key in toml_data: + value = toml_data.pop(key) + if not value: + print( + f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" + ) + print(f"Moving `{key}` from root level to [chain_state_assembler]") + if value != 1: + print( + f"Warning: `{key} = {value}`, will be set to `1` as recommended by Supra." + ) + value = 1 + chain_assembler_keys[key] = value + else: + print(f"Warning: `{key}` not found in root level") + + # Create [chain_state_assembler] table + chain_assembler_table = tomlkit.table() + for key, value in chain_assembler_keys.items(): + chain_assembler_table[key] = value + new_parameters = {"certified_block_cache_bucket_size": 50} + for key, value in new_parameters.items(): + print(f"Adding `{key} = {value}` to [chain_state_assembler]") + chain_assembler_table[key] = value + toml_data["chain_state_assembler"] = chain_assembler_table + + +def __add_consensus_access_tokens(toml_data): + new_parameters = {"consensus_access_tokens": []} + for key, value in new_parameters.items(): + print(f"Adding `{key} = {value}` to root level") + toml_data[key] = value + + +def migrate_v7_to_v8(toml_data): + # Migrate the config + __migrate_sync_ws_parameters(toml_data) + __migrate_chain_state_assembler_parameters(toml_data) + __add_consensus_access_tokens(toml_data) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py new file mode 100644 index 00000000..1416d4c9 --- /dev/null +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py @@ -0,0 +1,48 @@ +""" +v8 to v9 migration functions for Supra RPC config file. +""" + +import tomlkit + + +def __migrate_sync_ws_certificates(toml_data): + if "synchronization" not in toml_data: + raise SystemExit( + "Error: [synchronization] table should exist in v8 config. Please check your migration path matching the version of your config file." + ) + if "certificates" in toml_data["synchronization"]["ws"]: + raise SystemExit( + "Error: [synchronization.ws.certificates] table should not exist in v8 config. Please check your migration path matching the version of your config file." + ) + + # Explicitly move and rename these keys from the [synchronization.ws] table + # to the [synchronization.ws.certificates] table + sync_ws_certificates_keys_v8_to_v9 = { + "consensus_client_cert_path": "cert_path", + "consensus_client_private_key_path": "private_key_path", + "consensus_root_ca_cert_path": "root_ca_cert_path", + } + + sync_ws_certificates_v9 = {} + + for key, new_key in sync_ws_certificates_keys_v8_to_v9.items(): + if key in toml_data["synchronization"]["ws"]: + value = toml_data["synchronization"]["ws"].pop(key) + if not value: + print( + f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" + ) + print( + f"Moving `{key}` from [synchronization.ws] to {new_key} in [synchronization.ws.certificates] " + ) + sync_ws_certificates_v9[new_key] = value + else: + print(f"Warning: `{key}` not found in [synchronization.ws]") + + toml_data["synchronization"]["ws"]["certificates"] = tomlkit.table() + for key, value in sync_ws_certificates_v9.items(): + toml_data["synchronization"]["ws"]["certificates"][key] = value + + +def migrate_v8_to_v9(toml_data): + __migrate_sync_ws_certificates(toml_data) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/utils.py b/node_management/migration/rpc/src/migrate_rpc_config/utils.py new file mode 100644 index 00000000..1506fa99 --- /dev/null +++ b/node_management/migration/rpc/src/migrate_rpc_config/utils.py @@ -0,0 +1,19 @@ +import difflib + + +def unified_diff(from_str, to_str, fromfile, tofile): + diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) + print_colored_diff(diff) + + +def print_colored_diff(diff): + # The color is added here manually using ANSI escape codes. + for line in diff: + if line.startswith("+") and not line.startswith("+++"): + print(f"\033[32m{line}\033[0m", end="") # Green for additions + elif line.startswith("-") and not line.startswith("---"): + print(f"\033[31m{line}\033[0m", end="") # Red for deletions + elif line.startswith("@@"): + print(f"\033[36m{line}\033[0m", end="") # Cyan for hunk headers + else: + print(line, end="") From ce4392a29d80eb1f6916e7c9aea2238c6f8bcca8 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 21:26:46 +0200 Subject: [PATCH 06/44] support migrate path choices --- node_management/migration/rpc/src/migrate_rpc_config/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/__init__.py b/node_management/migration/rpc/src/migrate_rpc_config/__init__.py index 6ed3e492..e69de29b 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/__init__.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/__init__.py @@ -1 +0,0 @@ -# This file is intentionally left blank to mark this directory as a package. From 9e2d0e3670013e3aff4a1c143319525e6dd2ce22 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 21:28:06 +0200 Subject: [PATCH 07/44] support migrate path choices --- node_management/migration/rpc/src/migrate_rpc_config/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/cli.py b/node_management/migration/rpc/src/migrate_rpc_config/cli.py index 13b597e5..cff7d508 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/cli.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/cli.py @@ -14,7 +14,7 @@ def migrate_config(migrate_path: str, from_path: str, to_path: str): - # Migrate config + # Get the migration functions for the given path migrate_functions = RPC_CONFIG_MIGRATE_PATH.pop(migrate_path) if not migrate_functions: raise SystemExit(f"Error: unsupported migration path {migrate_path}.") From 714fbce9e8fa9fd2faf10917597ab2ee946ae050 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 21:56:17 +0200 Subject: [PATCH 08/44] add pytest to check diff --- node_management/migration/.gitignore | 8 +- node_management/migration/rpc/config.toml | 137 ------------------ .../migration/rpc/config_v7.1.8.toml | 2 +- .../src/migrate_rpc_config/from_v7_to_v8.py | 5 + .../migration/rpc/test_migration.py | 46 ++++++ 5 files changed, 59 insertions(+), 139 deletions(-) delete mode 100644 node_management/migration/rpc/config.toml create mode 100644 node_management/migration/rpc/test_migration.py diff --git a/node_management/migration/.gitignore b/node_management/migration/.gitignore index 7bbb8acc..33a954f4 100644 --- a/node_management/migration/.gitignore +++ b/node_management/migration/.gitignore @@ -4,4 +4,10 @@ **/build/ # egg-info -**/*.egg-info/ \ No newline at end of file +**/*.egg-info/ + +# python cache +**/__pycache__/ + +# pytest +**/.hypothesis/ \ No newline at end of file diff --git a/node_management/migration/rpc/config.toml b/node_management/migration/rpc/config.toml deleted file mode 100644 index d397310e..00000000 --- a/node_management/migration/rpc/config.toml +++ /dev/null @@ -1,137 +0,0 @@ -# Version: v7.1.8 -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_instance.chain_id = 8 -# The length of an epoch in seconds. -chain_instance.epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 48 hours. -chain_instance.recurring_lockup_duration_secs = 172800 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 46 hours. -chain_instance.voting_duration_secs = 165600 -# Determines whether the network will start with a faucet, amongst other things. -chain_instance.is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -chain_instance.genesis_timestamp_microseconds = 1732060800000000 - - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:30000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = false -# The path to the TLS certificate for the connection with the attached validator. -# The path to the private key to be used when negotiating TLS connections. -# The path to the TLS root certificate authority certificate. -# The websocket address of the attached validator. -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" -# The number of seconds to wait before retrying a block sync request. - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -consensus_access_tokens = [] -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_archive" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_ledger" - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_store" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retring a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. -[[allowed_origin]] -url = "https://rpc-mainnet.supra.com" -description = "RPC For Supra" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-wallet-mainnet.supra.com" -description = "RPC For Supra Wallet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-suprascan-mainnet.supra.com" -description = "RPC For suprascan" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:27000" -description = "LocalNet" -mode = "Server" - -[synchronization.ws] -consensus_rpc = "ws://:26000" -consensus_client_cert_path = "./configs/client_supra_certificate.pem" -consensus_client_private_key_path = "./configs/client_supra_key.pem" -consensus_root_ca_cert_path = "./configs/ca_certificate.pem" - -[chain_state_assembler] -sync_retry_interval_in_secs = 1 -certified_block_cache_bucket_size = 50 diff --git a/node_management/migration/rpc/config_v7.1.8.toml b/node_management/migration/rpc/config_v7.1.8.toml index 5f0eeb6a..d5971402 100644 --- a/node_management/migration/rpc/config_v7.1.8.toml +++ b/node_management/migration/rpc/config_v7.1.8.toml @@ -31,7 +31,7 @@ chain_instance.genesis_timestamp_microseconds = 1732060800000000 # The below parameters are node-specific and may be configured as required by the operator. # The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:30000" +bind_addr = "0.0.0.0:26000" # If `true` then blocks will not be verified before execution. This value should be `false` # unless you also control the node from which this RPC node is receiving blocks. block_provider_is_trusted = false diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py index ade30c5d..10c8bdda 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py @@ -86,8 +86,13 @@ def __add_consensus_access_tokens(toml_data): toml_data[key] = value +def __update_block_provider_is_trusted(toml_data): + toml_data["block_provider_is_trusted"] = True + + def migrate_v7_to_v8(toml_data): # Migrate the config __migrate_sync_ws_parameters(toml_data) __migrate_chain_state_assembler_parameters(toml_data) __add_consensus_access_tokens(toml_data) + __update_block_provider_is_trusted(toml_data) diff --git a/node_management/migration/rpc/test_migration.py b/node_management/migration/rpc/test_migration.py new file mode 100644 index 00000000..6c295c96 --- /dev/null +++ b/node_management/migration/rpc/test_migration.py @@ -0,0 +1,46 @@ +import os +import shutil +import tempfile +import tomlkit +import pytest +from migrate_rpc_config.cli import migrate_config + +CONFIGS = [ + "config_v7.1.8.toml", + "config_v8.0.2.toml", + "config_v9.0.7.toml", +] + +@pytest.mark.parametrize("from_file,to_file,migrate_path", [ + ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7->v8"), + ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8->v9"), + ("config_v7.1.8.toml", "config_v9.0.7.toml", "v7->v9"), +]) +def test_migration(tmp_path, from_file, to_file, migrate_path): + # Copy config files to temp dir + src_dir = os.path.dirname(os.path.abspath(__file__)) + from_path = os.path.join(src_dir, from_file) + to_path = os.path.join(src_dir, to_file) + tmp_from = tmp_path / from_file + tmp_to = tmp_path / to_file + shutil.copy(from_path, tmp_from) + # Run migration + migrate_config(migrate_path, str(tmp_from), str(tmp_to)) + # Load both files + with open(tmp_to, "r") as f: + migrated = tomlkit.parse(f.read()) + with open(to_path, "r") as f: + expected = tomlkit.parse(f.read()) + + # if "allowed_origin" in migrated: + # del migrated["allowed_origin"] + # if "allowed_origin" in expected: + # del expected["allowed_origin"] + + + # if "chain_instance" in migrated: + # del migrated["chain_instance"] + # if "chain_instance" in expected: + # del expected["chain_instance"] + # Compare TOML dicts + assert migrated == expected, f"Migration {migrate_path} failed: {from_file} -> {to_file}" From d882201edfd123e870d82c4d283607393016b6db Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 21 May 2025 22:41:33 +0200 Subject: [PATCH 09/44] improve ux --- .../rpc/src/migrate_rpc_config/cli.py | 1 - .../src/migrate_rpc_config/from_v7_to_v8.py | 19 ++++++++++++++----- .../src/migrate_rpc_config/from_v8_to_v9.py | 3 ++- .../rpc/src/migrate_rpc_config/utils.py | 11 +++++++++-- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/cli.py b/node_management/migration/rpc/src/migrate_rpc_config/cli.py index cff7d508..d2e408bf 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/cli.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/cli.py @@ -40,7 +40,6 @@ def migrate_config(migrate_path: str, from_path: str, to_path: str): # Backup old config original_toml_data = deepcopy(toml_data) - for fn in migrate_functions: print(f"Running migration function: {fn.__name__}") fn(toml_data) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py index 10c8bdda..ab4f391d 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py @@ -3,6 +3,7 @@ """ import tomlkit +from . import utils def __migrate_sync_ws_parameters(toml_data): @@ -27,7 +28,9 @@ def __migrate_sync_ws_parameters(toml_data): print( f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" ) - print(f"Moving `{key}` from root level to [synchronization.ws]") + utils.print_with_checkmark( + f"Moving `{key}` from root level to [synchronization.ws]" + ) sync_ws_keys[key] = value else: print(f"Warning: `{key}` not found in root level") @@ -58,7 +61,9 @@ def __migrate_chain_state_assembler_parameters(toml_data): print( f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" ) - print(f"Moving `{key}` from root level to [chain_state_assembler]") + utils.print_with_checkmark( + f"Moving `{key}` from root level to [chain_state_assembler]" + ) if value != 1: print( f"Warning: `{key} = {value}`, will be set to `1` as recommended by Supra." @@ -74,7 +79,7 @@ def __migrate_chain_state_assembler_parameters(toml_data): chain_assembler_table[key] = value new_parameters = {"certified_block_cache_bucket_size": 50} for key, value in new_parameters.items(): - print(f"Adding `{key} = {value}` to [chain_state_assembler]") + utils.print_with_checkmark(f"Adding `{key} = {value}` to [chain_state_assembler]") chain_assembler_table[key] = value toml_data["chain_state_assembler"] = chain_assembler_table @@ -82,12 +87,16 @@ def __migrate_chain_state_assembler_parameters(toml_data): def __add_consensus_access_tokens(toml_data): new_parameters = {"consensus_access_tokens": []} for key, value in new_parameters.items(): - print(f"Adding `{key} = {value}` to root level") + utils.print_with_checkmark(f"Adding `{key} = {value}` to root level") toml_data[key] = value def __update_block_provider_is_trusted(toml_data): - toml_data["block_provider_is_trusted"] = True + value = True + utils.print_with_checkmark( + f"Adding `block_provider_is_trusted = {value}` to root level" + ) + toml_data["block_provider_is_trusted"] = value def migrate_v7_to_v8(toml_data): diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py index 1416d4c9..faefb325 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py @@ -3,6 +3,7 @@ """ import tomlkit +from . import utils def __migrate_sync_ws_certificates(toml_data): @@ -32,7 +33,7 @@ def __migrate_sync_ws_certificates(toml_data): print( f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" ) - print( + utils.print_with_checkmark( f"Moving `{key}` from [synchronization.ws] to {new_key} in [synchronization.ws.certificates] " ) sync_ws_certificates_v9[new_key] = value diff --git a/node_management/migration/rpc/src/migrate_rpc_config/utils.py b/node_management/migration/rpc/src/migrate_rpc_config/utils.py index 1506fa99..1eb20c2a 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/utils.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/utils.py @@ -3,10 +3,10 @@ def unified_diff(from_str, to_str, fromfile, tofile): diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) - print_colored_diff(diff) + __print_colored_diff(diff) -def print_colored_diff(diff): +def __print_colored_diff(diff): # The color is added here manually using ANSI escape codes. for line in diff: if line.startswith("+") and not line.startswith("+++"): @@ -17,3 +17,10 @@ def print_colored_diff(diff): print(f"\033[36m{line}\033[0m", end="") # Cyan for hunk headers else: print(line, end="") + + +def print_with_checkmark(message): + """ + Print a message with a checkmark. + """ + print(f"✓ {message}") From 9f0d6752f6f87ffcd4d663cb668ad00598f5214c Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 22 May 2025 08:43:54 +0200 Subject: [PATCH 10/44] restore v7.1.8 template --- node_management/migration/rpc/config_v7.1.8.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_management/migration/rpc/config_v7.1.8.toml b/node_management/migration/rpc/config_v7.1.8.toml index d5971402..8e73ca30 100644 --- a/node_management/migration/rpc/config_v7.1.8.toml +++ b/node_management/migration/rpc/config_v7.1.8.toml @@ -31,7 +31,7 @@ chain_instance.genesis_timestamp_microseconds = 1732060800000000 # The below parameters are node-specific and may be configured as required by the operator. # The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:26000" +bind_addr = "0.0.0.0:30000" # If `true` then blocks will not be verified before execution. This value should be `false` # unless you also control the node from which this RPC node is receiving blocks. block_provider_is_trusted = false @@ -128,4 +128,4 @@ mode = "Server" [[allowed_origin]] url = "http://localhost:27000" description = "LocalNet" -mode = "Server" +mode = "Server" \ No newline at end of file From f1aa83d2c3a90591f41a2228fe8d4ea63cb0a26e Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 22 May 2025 08:57:47 +0200 Subject: [PATCH 11/44] update cli help ui --- .../migration/rpc/src/migrate_rpc_config/cli.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/cli.py b/node_management/migration/rpc/src/migrate_rpc_config/cli.py index d2e408bf..c9b805f6 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/cli.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/cli.py @@ -75,17 +75,19 @@ def migrate_config(migrate_path: str, from_path: str, to_path: str): def main(): import argparse + parser = argparse.ArgumentParser( - description=f"Migrate RPC config: {RPC_CONFIG_MIGRATE_PATH.keys()}" + description="Migrate RPC config.toml file from old version to new one." ) + choices_str = ", ".join([f"'{k}'" for k in RPC_CONFIG_MIGRATE_PATH.keys()]) parser.add_argument( "-p", "--migrate-path", required=True, type=str, - choices=RPC_CONFIG_MIGRATE_PATH.keys(), metavar="", - help="Path to migrate the config", + choices=list(RPC_CONFIG_MIGRATE_PATH.keys()), + help=f"Path to migrate the config. Choices: {choices_str}", ) parser.add_argument( "-f", @@ -93,7 +95,7 @@ def main(): required=True, type=str, metavar="", - help="Path to the config file to migrate", + help="Path to the old version config file to migrate", ) parser.add_argument( "-t", @@ -101,7 +103,7 @@ def main(): required=True, type=str, metavar="", - help="Path to the new config file", + help="Path to save the migrated config file", ) args = parser.parse_args() migrate_config(args.migrate_path, args.from_file, args.to_file) From e37634f8742f6ff28ee73c3c6e4abb7b81471679 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 22 May 2025 09:00:58 +0200 Subject: [PATCH 12/44] add readme --- node_management/migration/rpc/README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/node_management/migration/rpc/README.md b/node_management/migration/rpc/README.md index 493eff69..9ae1766d 100644 --- a/node_management/migration/rpc/README.md +++ b/node_management/migration/rpc/README.md @@ -1,9 +1,25 @@ -#### v9 +### Usage + +```sh +pip install . + +# From v7 to v8 +migrate-rpc-config -f config.toml -t config.toml -p 'v7->v8' + +# From v8 to v9 +migrate-rpc-config -f config.toml -t config.toml -p 'v8->v9' + +# From v7 to v9 +migrate-rpc-config -f config.toml -t config.toml -p 'v7->v9' + +``` + +#### v9 template https://testnet-snapshot.supra.com/configs/config_v9.0.7.toml -#### v8 +#### v8 template https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml -#### v7 +#### v7 template https://mainnet-data.supra.com/configs/config.toml \ No newline at end of file From 711bbd2141bfb3c3cf3c0ca5c5a3acbbdd025651 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 22 May 2025 20:08:01 +0200 Subject: [PATCH 13/44] add migrate smrsetting --- node_management/migration/README.md | 87 +++++++++ .../migration/common/pyproject.toml | 7 + .../migration/common/src/common/__init__ | 0 .../migration/common/src/common/migration.py | 77 ++++++++ .../src/common}/utils.py | 0 node_management/migration/rpc/README.md | 25 --- node_management/migration/rpc/poetry.lock | 32 ++++ node_management/migration/rpc/pyproject.toml | 19 +- .../rpc/src/migrate_rpc_config/cli.py | 66 +------ .../src/migrate_rpc_config/from_v7_to_v8.py | 2 +- .../src/migrate_rpc_config/from_v8_to_v9.py | 2 +- node_management/migration/smr/pyproject.toml | 13 ++ .../migration/smr/smr_settings.toml | 113 ++++++++++++ .../migration/smr/smr_settings_v7.1.x.toml | 167 ++++++++++++++++++ .../smr/smr_settings_v7.1.x.toml_v7.bak | 167 ++++++++++++++++++ .../migration/smr/smr_settings_v9.0.x.toml | 125 +++++++++++++ .../smr/src/migrate_smr_settings/__init__.py | 0 .../smr/src/migrate_smr_settings/cli.py | 53 ++++++ .../migrate_smr_settings/migrate_v7_to_v9.py | 72 ++++++++ .../migration/smr/test_migration.py | 31 ++++ 20 files changed, 960 insertions(+), 98 deletions(-) create mode 100644 node_management/migration/README.md create mode 100644 node_management/migration/common/pyproject.toml create mode 100644 node_management/migration/common/src/common/__init__ create mode 100644 node_management/migration/common/src/common/migration.py rename node_management/migration/{rpc/src/migrate_rpc_config => common/src/common}/utils.py (100%) delete mode 100644 node_management/migration/rpc/README.md create mode 100644 node_management/migration/rpc/poetry.lock create mode 100644 node_management/migration/smr/pyproject.toml create mode 100644 node_management/migration/smr/smr_settings.toml create mode 100644 node_management/migration/smr/smr_settings_v7.1.x.toml create mode 100644 node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak create mode 100644 node_management/migration/smr/smr_settings_v9.0.x.toml create mode 100644 node_management/migration/smr/src/migrate_smr_settings/__init__.py create mode 100644 node_management/migration/smr/src/migrate_smr_settings/cli.py create mode 100644 node_management/migration/smr/src/migrate_smr_settings/migrate_v7_to_v9.py create mode 100644 node_management/migration/smr/test_migration.py diff --git a/node_management/migration/README.md b/node_management/migration/README.md new file mode 100644 index 00000000..ebb2b765 --- /dev/null +++ b/node_management/migration/README.md @@ -0,0 +1,87 @@ + +### Usage + +```sh +# If you have not installed poetry, install below +pip install poetry + +# Install migrate-rpc-config +cd rpc +poetry install + +# From v7 to v8 +poetry run migrate-rpc-config -f config.toml -t config.toml -p 'v7->v8' + +# From v8 to v9 +poetry run migrate-rpc-config -f config.toml -t config.toml -p 'v8->v9' + +# From v7 to v9 +poetry run migrate-rpc-config -f config.toml -t config.toml -p 'v7->v9' + +# Install migrate-rpc-config +cd smr +poetry install + +poetry run migrate-smr-settings +``` + +#### v9 template +https://testnet-snapshot.supra.com/configs/config_v9.0.7.toml +https://testnet-snapshot.supra.com/configs/smr_settings.toml + +#### v8 template +https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml + +#### v7 template +https://mainnet-data.supra.com/configs/config.toml + + + +### Migrate from v7 to v9 + + +#### rpc +migrate-rpc-config -f config.toml -t config.toml -p 'v7->v9' + +(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v8 migrate-db config.toml + [======================================================================================================================================================================] 100/100MigrationReport { drop_cf: ["tx_block_info"], migrate_kv: {"tx_block_info__txn_hash_to_block_hash": 8537} } +(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v9 migrate-db config.toml +Counting the number of entries to remove from prune_index... +Cleaning up prune index: [00:00:00] ████████████████████ 0/0 00:00:00 Counting the number of entries in block_to_transaction... +Migrating block_to_transaction: [00:00:00] ████████████████████ 8537/8537 00:00:00 dropped: +- block_to_tx +migrated: + block_to_tx -> block_to_tx_ordered: Migrated 8537 records, up to 239 block height +databases_checked: +- chain_store +- archive + + +#### smr + +v7 -> v8 only cli profile and identity file migration + +TO BE CONFIRMED: no change in smr_settings.toml and database in v8. + +v7 -> v9 smr_settings migration + + + +v7 -> v9 database migration + + +(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/node_0$ supra-v9 data migrate -p smr_settings.toml +Counting the number of entries in certified_block... +Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 69/69 00:00:00 Counting the number of entries in uncommitted_block... +Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 74/74 00:00:00 +Cleaning up uncommitted_block: [00:00:00] ████████████████████ 4/4 00:00:00 Counting the number of entries in certified_block... +Counting the number of entries in certified_block_dehydrated... +Counting the number of entries in uncommitted_block... +Counting the number of entries in qc... +Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 70/70 00:00:00 Counting the number of entries to remove from prune_index... +Cleaning up prune index: [00:00:00] ████████████████████ 70/70 00:00:00 dropped: +- certified_block +migrated: + certified_block -> certified_block_dehydrated: Migrated 70 records, up to 244 block height +databases_checked: +- chain_store diff --git a/node_management/migration/common/pyproject.toml b/node_management/migration/common/pyproject.toml new file mode 100644 index 00000000..7d361b43 --- /dev/null +++ b/node_management/migration/common/pyproject.toml @@ -0,0 +1,7 @@ +[tool.poetry] +name = "common" +version = "0.1.0" +description = "Common migration utilities" + +[tool.poetry.dependencies] +python = ">=3.10" \ No newline at end of file diff --git a/node_management/migration/common/src/common/__init__ b/node_management/migration/common/src/common/__init__ new file mode 100644 index 00000000..e69de29b diff --git a/node_management/migration/common/src/common/migration.py b/node_management/migration/common/src/common/migration.py new file mode 100644 index 00000000..da5383d1 --- /dev/null +++ b/node_management/migration/common/src/common/migration.py @@ -0,0 +1,77 @@ + +import typing as ty +import tomlkit +from copy import deepcopy +from . import utils + +class Migration: + def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]): + self.migrate_path = migrate_path + + + def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): + + from_and_to_version = migrate_choice.split("->") + if len(from_and_to_version) != 2: + raise SystemExit( + f"Error: invalid migration choice {migrate_choice}. It should be in the format 'vX->vY'." + ) + + # Get the migration functions for the given path + migrate_functions = self.migrate_path.pop(migrate_choice) + if not migrate_functions: + raise SystemExit(f"Error: unsupported migration path {migrate_choice}.") + from_version = from_and_to_version[0] + to_version = from_and_to_version[1] + + default_backup_path = f"{from_path}_{from_version}.bak" + if from_path == to_path: + print(f"Warning: The source and destination paths are the same ({from_path}).") + print( + f"A backup of your original config will be saved to: {default_backup_path}" + ) + confirm = input( + "This will overwrite your original config file. Continue? [y/N]: " + ) + if confirm.lower() != "y": + print("Aborted by user.") + return + + with open(from_path, "r") as f: + toml_data = tomlkit.parse(f.read()) + + # Backup old config + original_toml_data = deepcopy(toml_data) + + for fn in migrate_functions: + print(f"Running migration function: {fn.__name__}") + fn(toml_data) + + print(f"Backing up old config to {default_backup_path}") + tomlkit.dump(original_toml_data, open(default_backup_path, "w")) + + # Write new config + print(f"Writing new config to {to_path}") + with open(to_path, "w") as f: + f.write(tomlkit.dumps(toml_data)) + + # Print the diff + from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) + to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) + + utils.unified_diff( + from_str, + to_str, + fromfile=from_version, + tofile=to_version, + ) + + print( + f""" + Config migrated from {from_path} to {to_path}. + Please double check above for the diff between old and new config. + Please ensure to use the new config file for target binary version. + NOTE: the comments may not be preserved in the new config file, so + you would need to fix the comments manually. + """ + ) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/utils.py b/node_management/migration/common/src/common/utils.py similarity index 100% rename from node_management/migration/rpc/src/migrate_rpc_config/utils.py rename to node_management/migration/common/src/common/utils.py diff --git a/node_management/migration/rpc/README.md b/node_management/migration/rpc/README.md deleted file mode 100644 index 9ae1766d..00000000 --- a/node_management/migration/rpc/README.md +++ /dev/null @@ -1,25 +0,0 @@ - -### Usage - -```sh -pip install . - -# From v7 to v8 -migrate-rpc-config -f config.toml -t config.toml -p 'v7->v8' - -# From v8 to v9 -migrate-rpc-config -f config.toml -t config.toml -p 'v8->v9' - -# From v7 to v9 -migrate-rpc-config -f config.toml -t config.toml -p 'v7->v9' - -``` - -#### v9 template -https://testnet-snapshot.supra.com/configs/config_v9.0.7.toml - -#### v8 template -https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml - -#### v7 template -https://mainnet-data.supra.com/configs/config.toml \ No newline at end of file diff --git a/node_management/migration/rpc/poetry.lock b/node_management/migration/rpc/poetry.lock new file mode 100644 index 00000000..c580ca8a --- /dev/null +++ b/node_management/migration/rpc/poetry.lock @@ -0,0 +1,32 @@ +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. + +[[package]] +name = "common" +version = "0.1.0" +description = "Common migration utilities" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [] +develop = false + +[package.source] +type = "directory" +url = "../common" + +[[package]] +name = "tomlkit" +version = "0.13.2" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, + {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, +] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.10" +content-hash = "ac45718893aff931c57841618c9240195f4a12b10d6c2be50c1e7229478e7877" diff --git a/node_management/migration/rpc/pyproject.toml b/node_management/migration/rpc/pyproject.toml index 09af0498..1260eb29 100644 --- a/node_management/migration/rpc/pyproject.toml +++ b/node_management/migration/rpc/pyproject.toml @@ -1,15 +1,12 @@ -[project] +[tool.poetry] name = "migrate-rpc-config" version = "0.1.0" description = "A CLI tool to migrate Supra RPC config." -authors = [ - { name = "Kaiqi Chen" } -] -readme = "README.md" -requires-python = ">=3.10" -dependencies = [ - "tomlkit>=0.11.0" -] -[project.scripts] -migrate-rpc-config = "migrate_rpc_config.cli:main" +[tool.poetry.dependencies] +python = ">=3.10" +tomlkit = ">=0.13.2" +common = { path = "../common" } + +[tool.poetry.scripts] +migrate-rpc-config = "migrate_rpc_config.cli:main" \ No newline at end of file diff --git a/node_management/migration/rpc/src/migrate_rpc_config/cli.py b/node_management/migration/rpc/src/migrate_rpc_config/cli.py index c9b805f6..87ca4205 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/cli.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/cli.py @@ -1,9 +1,8 @@ from .from_v7_to_v8 import migrate_v7_to_v8 from .from_v8_to_v9 import migrate_v8_to_v9 -from . import utils -import tomlkit -from copy import deepcopy +from common import migration + RPC_CONFIG_MIGRATE_PATH = { @@ -14,64 +13,11 @@ def migrate_config(migrate_path: str, from_path: str, to_path: str): - # Get the migration functions for the given path - migrate_functions = RPC_CONFIG_MIGRATE_PATH.pop(migrate_path) - if not migrate_functions: - raise SystemExit(f"Error: unsupported migration path {migrate_path}.") - from_version = migrate_path.split("->")[0] - to_version = migrate_path.split("->")[1] - - default_backup_path = f"{from_path}_{from_version}.bak" - if from_path == to_path: - print(f"Warning: The source and destination paths are the same ({from_path}).") - print( - f"A backup of your original config will be saved to: {default_backup_path}" - ) - confirm = input( - "This will overwrite your original config file. Continue? [y/N]: " - ) - if confirm.lower() != "y": - print("Aborted by user.") - return - - with open(from_path, "r") as f: - toml_data = tomlkit.parse(f.read()) - - # Backup old config - original_toml_data = deepcopy(toml_data) - - for fn in migrate_functions: - print(f"Running migration function: {fn.__name__}") - fn(toml_data) - - print(f"Backing up old config to {default_backup_path}") - tomlkit.dump(original_toml_data, open(default_backup_path, "w")) - - # Write new config - print(f"Writing new config to {to_path}") - with open(to_path, "w") as f: - f.write(tomlkit.dumps(toml_data)) - - # Print the diff - from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) - to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) - - utils.unified_diff( - from_str, - to_str, - fromfile=from_version, - tofile=to_version, + + migration.Migration(RPC_CONFIG_MIGRATE_PATH).migrate_config( + migrate_path, from_path, to_path ) - print( - f""" - Config migrated from {from_path} to {to_path}. - Please double check above for the diff between old and new config. - Please ensure to use the new config file for target binary version. - """ - ) - - def main(): import argparse @@ -87,7 +33,7 @@ def main(): type=str, metavar="", choices=list(RPC_CONFIG_MIGRATE_PATH.keys()), - help=f"Path to migrate the config. Choices: {choices_str}", + help=f"Migration path choices: {choices_str}", ) parser.add_argument( "-f", diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py index ab4f391d..9f07e611 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py @@ -3,7 +3,7 @@ """ import tomlkit -from . import utils +from common import utils def __migrate_sync_ws_parameters(toml_data): diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py b/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py index faefb325..bdc5e9d5 100644 --- a/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py +++ b/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py @@ -3,7 +3,7 @@ """ import tomlkit -from . import utils +from common import utils def __migrate_sync_ws_certificates(toml_data): diff --git a/node_management/migration/smr/pyproject.toml b/node_management/migration/smr/pyproject.toml new file mode 100644 index 00000000..701b363c --- /dev/null +++ b/node_management/migration/smr/pyproject.toml @@ -0,0 +1,13 @@ +[tool.poetry] +name = "migrate-smr-settings" +version = "0.1.0" +description = "A CLI tool to migrate Supra SMR settings." +authors = ["Kaiqi Chen"] + +[tool.poetry.dependencies] +python = ">=3.10" +tomlkit = ">=0.13.2" +common = { path = "../common" } + +[tool.poetry.scripts] +migrate-smr-settings = "migrate_smr_settings.cli:main" \ No newline at end of file diff --git a/node_management/migration/smr/smr_settings.toml b/node_management/migration/smr/smr_settings.toml new file mode 100644 index 00000000..13b02539 --- /dev/null +++ b/node_management/migration/smr/smr_settings.toml @@ -0,0 +1,113 @@ +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +# Parameters related to the MoveVM. Primarily related to governance features. +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to the TLS root certificate authority certificate. +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 +# The path to the TLS certificate for this node. +# The path to the private key to be used when negotiating TLS connections. + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 +[node.ws_server.certificates] +root_ca_cert_path = "./configs/ca_certificate.pem" +cert_path = "./configs/server_supra_certificate.pem" +private_key_path = "./configs/server_supra_key.pem" diff --git a/node_management/migration/smr/smr_settings_v7.1.x.toml b/node_management/migration/smr/smr_settings_v7.1.x.toml new file mode 100644 index 00000000..ef62f316 --- /dev/null +++ b/node_management/migration/smr/smr_settings_v7.1.x.toml @@ -0,0 +1,167 @@ +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +# Parameters related to the MoveVM. Primarily related to governance features. +[move_vm] +# Initially `false` until the network matures. +allow_new_validators = false +# The maximum stake that may be allocated to a Supra Validator. We are not currently doing +# stake-weighted voting, so this value does not impact our decentralization quotient. This +# may change in the future. Initially set to the total supply. +# +# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 100_000_000_000 SUPRA. +max_stake = "10000000000000000000" +# The minimum stake required to run a Supra Validator. There is no minimum at genesis +# because The Foundation's stake is added after the stake pools are created. We will increase +# it to its intended value of 55M SUPRA via governance. New nodes will not be able to join the +# validator set before the update is made. Measured in Quants (1 Quant = 10^-8 SUPRA). +min_stake = 0 +# The number of tokens initially allocated to node operators. Tokens will be earned through block +# rewards. +operator_account_balance = 0 +# The amount of Quants to transfer from each validator owner account to the corresponding stake pool +# after it has been created, during the genesis transaction. Adding the stake after creating the +# pool allows us to ensure that The Foundation's stake is not subject to the PBO locking schedule, +# which is only intended to apply to winners of the Project Blast Off campaign. +# +# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 55_000_000 SUPRA. +pbo_owner_stake = 5500000000000000 +# The number of seconds after `genesis_timestamp_microseconds` at which all accounts with +# allocations at genesis will be able to unlock their initial amounts. These amounts are stored +# in vesting contracts. +# +# Corresponds to Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +remaining_balance_lockup_cliff_period_in_seconds = 0 +# The amount of SUPRA required to qualify as a proposer (this parameter is currently unused). +required_proposer_stake = 0 +# The annual percent yield for validators, proportional to their stake. Specified as a percentage +# with 2 decimals of precision in u64 format due to limitations in the MoveVM. The below value +# represents 12.85%. +rewards_apy_percentage = 1285 +# The percentage of staking rewards earned by Supra Foundation controlled nodes that will be paid +# to the corresponding node operators. Specified as a percentage with 2 decimals of precision in +# u64 format due to limitations in the MoveVM. The below value represents 37.74%. +validator_commission_rate_percentage = 3774 +# The percentage of new stake relative to the current total stake that can join the validator +# set or be added to existing validators within a single epoch. +voting_power_increase_limit = 33 + + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 +# The path to the TLS certificate for this node. +server_cert_path = "./configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +server_private_key_path = "./configs/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 \ No newline at end of file diff --git a/node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak b/node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak new file mode 100644 index 00000000..ef62f316 --- /dev/null +++ b/node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak @@ -0,0 +1,167 @@ +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +# Parameters related to the MoveVM. Primarily related to governance features. +[move_vm] +# Initially `false` until the network matures. +allow_new_validators = false +# The maximum stake that may be allocated to a Supra Validator. We are not currently doing +# stake-weighted voting, so this value does not impact our decentralization quotient. This +# may change in the future. Initially set to the total supply. +# +# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 100_000_000_000 SUPRA. +max_stake = "10000000000000000000" +# The minimum stake required to run a Supra Validator. There is no minimum at genesis +# because The Foundation's stake is added after the stake pools are created. We will increase +# it to its intended value of 55M SUPRA via governance. New nodes will not be able to join the +# validator set before the update is made. Measured in Quants (1 Quant = 10^-8 SUPRA). +min_stake = 0 +# The number of tokens initially allocated to node operators. Tokens will be earned through block +# rewards. +operator_account_balance = 0 +# The amount of Quants to transfer from each validator owner account to the corresponding stake pool +# after it has been created, during the genesis transaction. Adding the stake after creating the +# pool allows us to ensure that The Foundation's stake is not subject to the PBO locking schedule, +# which is only intended to apply to winners of the Project Blast Off campaign. +# +# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 55_000_000 SUPRA. +pbo_owner_stake = 5500000000000000 +# The number of seconds after `genesis_timestamp_microseconds` at which all accounts with +# allocations at genesis will be able to unlock their initial amounts. These amounts are stored +# in vesting contracts. +# +# Corresponds to Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +remaining_balance_lockup_cliff_period_in_seconds = 0 +# The amount of SUPRA required to qualify as a proposer (this parameter is currently unused). +required_proposer_stake = 0 +# The annual percent yield for validators, proportional to their stake. Specified as a percentage +# with 2 decimals of precision in u64 format due to limitations in the MoveVM. The below value +# represents 12.85%. +rewards_apy_percentage = 1285 +# The percentage of staking rewards earned by Supra Foundation controlled nodes that will be paid +# to the corresponding node operators. Specified as a percentage with 2 decimals of precision in +# u64 format due to limitations in the MoveVM. The below value represents 37.74%. +validator_commission_rate_percentage = 3774 +# The percentage of new stake relative to the current total stake that can join the validator +# set or be added to existing validators within a single epoch. +voting_power_increase_limit = 33 + + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 +# The path to the TLS certificate for this node. +server_cert_path = "./configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +server_private_key_path = "./configs/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 \ No newline at end of file diff --git a/node_management/migration/smr/smr_settings_v9.0.x.toml b/node_management/migration/smr/smr_settings_v9.0.x.toml new file mode 100644 index 00000000..09390f16 --- /dev/null +++ b/node_management/migration/smr/smr_settings_v9.0.x.toml @@ -0,0 +1,125 @@ +# Version: v9.0.x + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 6 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 4 hours. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 2 hours. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = true +# Tuesday, September 17, 2024 12:00:00 PM +genesis_timestamp_microseconds = 1726574400000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 2500 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 20 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 5000 + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 + +[node.ws_server.certificates] +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "configs/ca_certificate.pem" +# The path to the TLS certificate for this node. +cert_path = "configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "configs/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 diff --git a/node_management/migration/smr/src/migrate_smr_settings/__init__.py b/node_management/migration/smr/src/migrate_smr_settings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/node_management/migration/smr/src/migrate_smr_settings/cli.py b/node_management/migration/smr/src/migrate_smr_settings/cli.py new file mode 100644 index 00000000..2e5cdc98 --- /dev/null +++ b/node_management/migration/smr/src/migrate_smr_settings/cli.py @@ -0,0 +1,53 @@ +from .migrate_v7_to_v9 import migrate_v7_to_v9 + +from common import migration + + + +SMR_SETTINGS_MIGRATE_PATH = { + "v7->v9": [migrate_v7_to_v9], +} + + +def migrate_config(migrate_path: str, from_path: str, to_path: str): + + migration.Migration(SMR_SETTINGS_MIGRATE_PATH).migrate_config( + migrate_path, from_path, to_path + ) + +def main(): + import argparse + + + parser = argparse.ArgumentParser( + description="Migrate validator smr_settings.toml file from old version to new one." + ) + choices_str = ", ".join([f"'{k}'" for k in SMR_SETTINGS_MIGRATE_PATH.keys()]) + parser.add_argument( + "-p", + "--migrate-path", + required=True, + type=str, + metavar="", + choices=list(SMR_SETTINGS_MIGRATE_PATH.keys()), + help=f"Migration path choices: {choices_str}", + ) + + parser.add_argument( + "-f", + "--from-file", + required=True, + type=str, + metavar="", + help="Path to the old version config file to migrate", + ) + parser.add_argument( + "-t", + "--to-file", + required=True, + type=str, + metavar="", + help="Path to save the migrated config file", + ) + args = parser.parse_args() + migrate_config(args.migrate_path, args.from_file, args.to_file) diff --git a/node_management/migration/smr/src/migrate_smr_settings/migrate_v7_to_v9.py b/node_management/migration/smr/src/migrate_smr_settings/migrate_v7_to_v9.py new file mode 100644 index 00000000..59ec7f6f --- /dev/null +++ b/node_management/migration/smr/src/migrate_smr_settings/migrate_v7_to_v9.py @@ -0,0 +1,72 @@ +""" +remove [move_vm] table + +move below keys in [node] + +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" +# The path to the TLS certificate for this node. +server_cert_path = "./configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +server_private_key_path = "./configs/server_supra_key.pem" + +==> + +[node.ws_server.certificates] +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "configs/ca_certificate.pem" +# The path to the TLS certificate for this node. +cert_path = "configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "configs/server_supra_key.pem" + +""" + +import tomlkit +from common import utils + + +def __migrate_move_vm_parameters(toml_data): + utils.print_with_checkmark( + "Removing [move_vm] table" + ) + toml_data.pop("move_vm") + + +def __migrate_node_ws_certificates(toml_data): + if "ws_server" in toml_data["node"]: + raise SystemExit( + "Error: [node.ws_server] table should not exist in before v9 config. Please check your migration path matching the version of your config file." + ) + + # Explicitly move and rename these keys from the [node] table + # to the [node.ws_server.certificates] table + node_ws_certificates_keys_v7_to_v9 = { + "root_ca_cert_path": "root_ca_cert_path", + "server_cert_path": "cert_path", + "server_private_key_path": "private_key_path", + } + ws_certificates_v9 = {} + + for old_key, new_key in node_ws_certificates_keys_v7_to_v9.items(): + if old_key in toml_data["node"]: + value = toml_data["node"].pop(old_key) + if not value: + print( + f"Warning: `{old_key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" + ) + utils.print_with_checkmark(f"Moving `{old_key}` from [node] to {new_key} in [node.ws_server.certificates]") + ws_certificates_v9[new_key] = value + else: + print(f"Warning: `{old_key}` not found in [node]") + + # Create [node.ws_server.certificates] table + toml_data["node"]["ws_server"] = tomlkit.table() + toml_data["node"]["ws_server"]["certificates"] = tomlkit.table() + for key, value in ws_certificates_v9.items(): + toml_data["node"]["ws_server"]["certificates"][key] = value + + +def migrate_v7_to_v9(toml_data): + __migrate_move_vm_parameters(toml_data) + __migrate_node_ws_certificates(toml_data) \ No newline at end of file diff --git a/node_management/migration/smr/test_migration.py b/node_management/migration/smr/test_migration.py new file mode 100644 index 00000000..2d8a6245 --- /dev/null +++ b/node_management/migration/smr/test_migration.py @@ -0,0 +1,31 @@ +import os +import shutil +import tomlkit +import pytest +from migrate_smr_settings.cli import migrate_config + +CONFIGS = [ + "smr_settings_v7.1.x.toml", + "smr_settings_v9.0.x.toml", +] + +@pytest.mark.parametrize("from_file,to_file,migrate_path", [ + ("smr_settings_v7.1.x.toml", "smr_settings_v9.0.x.toml", "v7->v9"), +]) +def test_migration(tmp_path, from_file, to_file, migrate_path): + # Copy config files to temp dir + src_dir = os.path.dirname(os.path.abspath(__file__)) + from_path = os.path.join(src_dir, from_file) + to_path = os.path.join(src_dir, to_file) + tmp_from = tmp_path / from_file + tmp_to = tmp_path / to_file + shutil.copy(from_path, tmp_from) + # Run migration + migrate_config(migrate_path, str(tmp_from), str(tmp_to)) + # Load both files + with open(tmp_to, "r") as f: + migrated = tomlkit.parse(f.read()) + with open(to_path, "r") as f: + expected = tomlkit.parse(f.read()) + # Compare TOML dicts + assert migrated == expected, f"Migration {migrate_path} failed: {from_file} -> {to_file}" From 67c8501d0f161398df671ac4e4b335c812a13428 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 22 May 2025 21:37:23 +0200 Subject: [PATCH 14/44] refactor --- node_management/migration/README.md | 59 +++---- .../migration/common/pyproject.toml | 7 - node_management/migration/pyproject.toml | 17 ++ node_management/migration/rpc/poetry.lock | 32 ---- node_management/migration/rpc/pyproject.toml | 12 -- .../rpc/src/migrate_rpc_config/cli.py | 55 ------ node_management/migration/smr/pyproject.toml | 13 -- .../migration/smr/smr_settings.toml | 113 ------------ .../smr/smr_settings_v7.1.x.toml_v7.bak | 167 ------------------ .../smr/src/migrate_smr_settings/cli.py | 53 ------ .../cli}/__init__.py | 0 node_management/migration/src/cli/main.py | 60 +++++++ .../{common => }/src/common/__init__ | 0 .../{common => }/src/common/migration.py | 40 +++-- .../{common => }/src/common/utils.py | 4 +- .../rpc_config}/__init__.py | 0 .../rpc_config}/from_v7_to_v8.py | 0 .../rpc_config}/from_v8_to_v9.py | 0 .../migration/src/rpc_config/migrate_path.py | 20 +++ .../migration/src/smr_settings/__init__.py | 0 .../smr_settings/from_v7_to_v9.py} | 0 .../src/smr_settings/migrate_path.py | 17 ++ .../{rpc => tests}/config_v7.1.8.toml | 0 .../{rpc => tests}/config_v8.0.2.toml | 0 .../{rpc => tests}/config_v9.0.7.toml | 0 .../{smr => tests}/smr_settings_v7.1.x.toml | 0 .../{smr => tests}/smr_settings_v9.0.x.toml | 0 .../test_rpc_migration.py} | 10 +- .../test_smr_migration.py} | 6 +- 29 files changed, 175 insertions(+), 510 deletions(-) delete mode 100644 node_management/migration/common/pyproject.toml create mode 100644 node_management/migration/pyproject.toml delete mode 100644 node_management/migration/rpc/poetry.lock delete mode 100644 node_management/migration/rpc/pyproject.toml delete mode 100644 node_management/migration/rpc/src/migrate_rpc_config/cli.py delete mode 100644 node_management/migration/smr/pyproject.toml delete mode 100644 node_management/migration/smr/smr_settings.toml delete mode 100644 node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak delete mode 100644 node_management/migration/smr/src/migrate_smr_settings/cli.py rename node_management/migration/{rpc/src/migrate_rpc_config => src/cli}/__init__.py (100%) create mode 100644 node_management/migration/src/cli/main.py rename node_management/migration/{common => }/src/common/__init__ (100%) rename node_management/migration/{common => }/src/common/migration.py (65%) rename node_management/migration/{common => }/src/common/utils.py (97%) rename node_management/migration/{smr/src/migrate_smr_settings => src/rpc_config}/__init__.py (100%) rename node_management/migration/{rpc/src/migrate_rpc_config => src/rpc_config}/from_v7_to_v8.py (100%) rename node_management/migration/{rpc/src/migrate_rpc_config => src/rpc_config}/from_v8_to_v9.py (100%) create mode 100644 node_management/migration/src/rpc_config/migrate_path.py create mode 100644 node_management/migration/src/smr_settings/__init__.py rename node_management/migration/{smr/src/migrate_smr_settings/migrate_v7_to_v9.py => src/smr_settings/from_v7_to_v9.py} (100%) create mode 100644 node_management/migration/src/smr_settings/migrate_path.py rename node_management/migration/{rpc => tests}/config_v7.1.8.toml (100%) rename node_management/migration/{rpc => tests}/config_v8.0.2.toml (100%) rename node_management/migration/{rpc => tests}/config_v9.0.7.toml (100%) rename node_management/migration/{smr => tests}/smr_settings_v7.1.x.toml (100%) rename node_management/migration/{smr => tests}/smr_settings_v9.0.x.toml (100%) rename node_management/migration/{rpc/test_migration.py => tests/test_rpc_migration.py} (79%) rename node_management/migration/{smr/test_migration.py => tests/test_smr_migration.py} (86%) diff --git a/node_management/migration/README.md b/node_management/migration/README.md index ebb2b765..8d31e5db 100644 --- a/node_management/migration/README.md +++ b/node_management/migration/README.md @@ -1,28 +1,25 @@ -### Usage +### Usage Example ```sh -# If you have not installed poetry, install below -pip install poetry - -# Install migrate-rpc-config -cd rpc -poetry install - -# From v7 to v8 -poetry run migrate-rpc-config -f config.toml -t config.toml -p 'v7->v8' - -# From v8 to v9 -poetry run migrate-rpc-config -f config.toml -t config.toml -p 'v8->v9' - -# From v7 to v9 -poetry run migrate-rpc-config -f config.toml -t config.toml -p 'v7->v9' - -# Install migrate-rpc-config -cd smr -poetry install - -poetry run migrate-smr-settings +pip install . + +# Migrate rpc config from v7 to v9 +migrate-config rpc -p v7-v9 -f config.toml -t config.toml +# Migrate db from v7 to v8 +rpc-v8 migrate-db config.toml +# Migrate db from v8 to v9 +rpc-v9 migrate-db config.toml + +# Migrate cli profile from v7 to v8 +supra-v8 migrate --network localnet +# Migrate identity from v7 to v8 +cp validator_identity.pem node_identity.pem + +# Migrate smr_settings from v7 to v9 +migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml +# Migrate db from v7 to v9 +supra-v9 data migrate -p smr_settings.toml ``` #### v9 template @@ -40,11 +37,12 @@ https://mainnet-data.supra.com/configs/config.toml ### Migrate from v7 to v9 -#### rpc -migrate-rpc-config -f config.toml -t config.toml -p 'v7->v9' +#### rpc db migration (e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v8 migrate-db config.toml [======================================================================================================================================================================] 100/100MigrationReport { drop_cf: ["tx_block_info"], migrate_kv: {"tx_block_info__txn_hash_to_block_hash": 8537} } + + (e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v9 migrate-db config.toml Counting the number of entries to remove from prune_index... Cleaning up prune index: [00:00:00] ████████████████████ 0/0 00:00:00 Counting the number of entries in block_to_transaction... @@ -57,18 +55,7 @@ databases_checked: - archive -#### smr - -v7 -> v8 only cli profile and identity file migration - -TO BE CONFIRMED: no change in smr_settings.toml and database in v8. - -v7 -> v9 smr_settings migration - - - -v7 -> v9 database migration - +#### smr db migration (e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/node_0$ supra-v9 data migrate -p smr_settings.toml Counting the number of entries in certified_block... diff --git a/node_management/migration/common/pyproject.toml b/node_management/migration/common/pyproject.toml deleted file mode 100644 index 7d361b43..00000000 --- a/node_management/migration/common/pyproject.toml +++ /dev/null @@ -1,7 +0,0 @@ -[tool.poetry] -name = "common" -version = "0.1.0" -description = "Common migration utilities" - -[tool.poetry.dependencies] -python = ">=3.10" \ No newline at end of file diff --git a/node_management/migration/pyproject.toml b/node_management/migration/pyproject.toml new file mode 100644 index 00000000..6c791cb8 --- /dev/null +++ b/node_management/migration/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "migrate-config" +version = "0.1.0" +description = "Unified CLI tool to migrate Supra RPC and SMR configs." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "tomlkit>=0.13.2", + "click>=8.0.0" +] + +[project.scripts] +migrate-config = "cli.main:main" + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/node_management/migration/rpc/poetry.lock b/node_management/migration/rpc/poetry.lock deleted file mode 100644 index c580ca8a..00000000 --- a/node_management/migration/rpc/poetry.lock +++ /dev/null @@ -1,32 +0,0 @@ -# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. - -[[package]] -name = "common" -version = "0.1.0" -description = "Common migration utilities" -optional = false -python-versions = ">=3.10" -groups = ["main"] -files = [] -develop = false - -[package.source] -type = "directory" -url = "../common" - -[[package]] -name = "tomlkit" -version = "0.13.2" -description = "Style preserving TOML library" -optional = false -python-versions = ">=3.8" -groups = ["main"] -files = [ - {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, - {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, -] - -[metadata] -lock-version = "2.1" -python-versions = ">=3.10" -content-hash = "ac45718893aff931c57841618c9240195f4a12b10d6c2be50c1e7229478e7877" diff --git a/node_management/migration/rpc/pyproject.toml b/node_management/migration/rpc/pyproject.toml deleted file mode 100644 index 1260eb29..00000000 --- a/node_management/migration/rpc/pyproject.toml +++ /dev/null @@ -1,12 +0,0 @@ -[tool.poetry] -name = "migrate-rpc-config" -version = "0.1.0" -description = "A CLI tool to migrate Supra RPC config." - -[tool.poetry.dependencies] -python = ">=3.10" -tomlkit = ">=0.13.2" -common = { path = "../common" } - -[tool.poetry.scripts] -migrate-rpc-config = "migrate_rpc_config.cli:main" \ No newline at end of file diff --git a/node_management/migration/rpc/src/migrate_rpc_config/cli.py b/node_management/migration/rpc/src/migrate_rpc_config/cli.py deleted file mode 100644 index 87ca4205..00000000 --- a/node_management/migration/rpc/src/migrate_rpc_config/cli.py +++ /dev/null @@ -1,55 +0,0 @@ -from .from_v7_to_v8 import migrate_v7_to_v8 -from .from_v8_to_v9 import migrate_v8_to_v9 - -from common import migration - - - -RPC_CONFIG_MIGRATE_PATH = { - "v7->v8": [migrate_v7_to_v8], - "v8->v9": [migrate_v8_to_v9], - "v7->v9": [migrate_v7_to_v8, migrate_v8_to_v9], -} - - -def migrate_config(migrate_path: str, from_path: str, to_path: str): - - migration.Migration(RPC_CONFIG_MIGRATE_PATH).migrate_config( - migrate_path, from_path, to_path - ) - -def main(): - import argparse - - - parser = argparse.ArgumentParser( - description="Migrate RPC config.toml file from old version to new one." - ) - choices_str = ", ".join([f"'{k}'" for k in RPC_CONFIG_MIGRATE_PATH.keys()]) - parser.add_argument( - "-p", - "--migrate-path", - required=True, - type=str, - metavar="", - choices=list(RPC_CONFIG_MIGRATE_PATH.keys()), - help=f"Migration path choices: {choices_str}", - ) - parser.add_argument( - "-f", - "--from-file", - required=True, - type=str, - metavar="", - help="Path to the old version config file to migrate", - ) - parser.add_argument( - "-t", - "--to-file", - required=True, - type=str, - metavar="", - help="Path to save the migrated config file", - ) - args = parser.parse_args() - migrate_config(args.migrate_path, args.from_file, args.to_file) diff --git a/node_management/migration/smr/pyproject.toml b/node_management/migration/smr/pyproject.toml deleted file mode 100644 index 701b363c..00000000 --- a/node_management/migration/smr/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[tool.poetry] -name = "migrate-smr-settings" -version = "0.1.0" -description = "A CLI tool to migrate Supra SMR settings." -authors = ["Kaiqi Chen"] - -[tool.poetry.dependencies] -python = ">=3.10" -tomlkit = ">=0.13.2" -common = { path = "../common" } - -[tool.poetry.scripts] -migrate-smr-settings = "migrate_smr_settings.cli:main" \ No newline at end of file diff --git a/node_management/migration/smr/smr_settings.toml b/node_management/migration/smr/smr_settings.toml deleted file mode 100644 index 13b02539..00000000 --- a/node_management/migration/smr/smr_settings.toml +++ /dev/null @@ -1,113 +0,0 @@ -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. Paths are set relative -# to $SUPRA_HOME. - -# Core protocol parameters. -[instance] -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 48 hours. -recurring_lockup_duration_secs = 172800 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 46 hours. -voting_duration_secs = 165600 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - -# Parameters related to the mempool. -[mempool] -# The maximum number of milliseconds that a node will wait before proposing a batch when it has -# at least one transaction to process. -max_batch_delay_ms = 500 -# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` -# then a batch will be proposed immediately. -max_batch_size_bytes = 5000000 -# The amount of time that a node will wait before repeating a sync request for a batch that it -# is missing. -sync_retry_delay_ms = 2000 -# The number of signers of the related batch certificate that a node should ask for a batch -# attempting to retry a sync request. -sync_retry_nodes = 3 - -# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. -[moonshot] -# The maximum number of milliseconds that the timestamp of a proposed block may be -# ahead of a node's local time when it attempts to vote for the block. Validators -# must wait until the timestamp of a certified block has passed before advancing to -# the next round and leaders must wait until the timestamp of the parent block has -# passed before proposing, so this limit prevents Byzantine leaders from forcing -# honest nodes to wait indefinitely by proposing blocks with timestamps that are -# arbitrarily far in the future. -block_recency_bound_ms = 500 -# Causes the node to stop producing blocks when there are no transactions to be -# processed. If all nodes set this value to `true` then the chain will not produce -# new blocks when there are no transactions to process, conserving disk space. -halt_block_production_when_no_txs = false -# The type of leader election function to use. This function generates a schedule that ensures -# that each node eventually succeeds every other. -leader_elector = "FairSuccession" -# The delay after which the block proposer will create a new block despite not having any -# payload items to propose. Denominated in ms. -max_block_delay_ms = 1250 -# The maximum number of batch availability certificates that may be included in a single -# consensus block. -max_payload_items_per_block = 50 -# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, -# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount -# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can -# become. This trade-off must be balanced. -message_recency_bound_rounds = 1000 -# The delay after which the node will try to repeat sync requests for missing blocks. -# Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 1000 -# The delay after which the node will send a Timeout message for its current Moonshot round, -# measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 3500 - -# Parameters related to the MoveVM. Primarily related to governance features. -[node] -# The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 1 -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The path to the TLS root certificate authority certificate. -# The port on which to listen for connections from the associated RPC node. Each validator -# may serve at most one RPC node. -rpc_access_port = 26000 -# The path to the TLS certificate for this node. -# The path to the private key to be used when negotiating TLS connections. - -# Parameters for the blockchain database. -[node.database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/smr_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true - -# Parameters for the DKG database. -[node.database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/ledger_storage" - -# Parameters related to database pruning. -[node.database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 -[node.ws_server.certificates] -root_ca_cert_path = "./configs/ca_certificate.pem" -cert_path = "./configs/server_supra_certificate.pem" -private_key_path = "./configs/server_supra_key.pem" diff --git a/node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak b/node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak deleted file mode 100644 index ef62f316..00000000 --- a/node_management/migration/smr/smr_settings_v7.1.x.toml_v7.bak +++ /dev/null @@ -1,167 +0,0 @@ -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. Paths are set relative -# to $SUPRA_HOME. - -# Core protocol parameters. -[instance] -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 48 hours. -recurring_lockup_duration_secs = 172800 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 46 hours. -voting_duration_secs = 165600 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - -# Parameters related to the mempool. -[mempool] -# The maximum number of milliseconds that a node will wait before proposing a batch when it has -# at least one transaction to process. -max_batch_delay_ms = 500 -# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` -# then a batch will be proposed immediately. -max_batch_size_bytes = 5000000 -# The amount of time that a node will wait before repeating a sync request for a batch that it -# is missing. -sync_retry_delay_ms = 2000 -# The number of signers of the related batch certificate that a node should ask for a batch -# attempting to retry a sync request. -sync_retry_nodes = 3 - -# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. -[moonshot] -# The maximum number of milliseconds that the timestamp of a proposed block may be -# ahead of a node's local time when it attempts to vote for the block. Validators -# must wait until the timestamp of a certified block has passed before advancing to -# the next round and leaders must wait until the timestamp of the parent block has -# passed before proposing, so this limit prevents Byzantine leaders from forcing -# honest nodes to wait indefinitely by proposing blocks with timestamps that are -# arbitrarily far in the future. -block_recency_bound_ms = 500 -# Causes the node to stop producing blocks when there are no transactions to be -# processed. If all nodes set this value to `true` then the chain will not produce -# new blocks when there are no transactions to process, conserving disk space. -halt_block_production_when_no_txs = false -# The type of leader election function to use. This function generates a schedule that ensures -# that each node eventually succeeds every other. -leader_elector = "FairSuccession" -# The delay after which the block proposer will create a new block despite not having any -# payload items to propose. Denominated in ms. -max_block_delay_ms = 1250 -# The maximum number of batch availability certificates that may be included in a single -# consensus block. -max_payload_items_per_block = 50 -# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, -# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount -# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can -# become. This trade-off must be balanced. -message_recency_bound_rounds = 1000 -# The delay after which the node will try to repeat sync requests for missing blocks. -# Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 1000 -# The delay after which the node will send a Timeout message for its current Moonshot round, -# measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 3500 - -# Parameters related to the MoveVM. Primarily related to governance features. -[move_vm] -# Initially `false` until the network matures. -allow_new_validators = false -# The maximum stake that may be allocated to a Supra Validator. We are not currently doing -# stake-weighted voting, so this value does not impact our decentralization quotient. This -# may change in the future. Initially set to the total supply. -# -# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 100_000_000_000 SUPRA. -max_stake = "10000000000000000000" -# The minimum stake required to run a Supra Validator. There is no minimum at genesis -# because The Foundation's stake is added after the stake pools are created. We will increase -# it to its intended value of 55M SUPRA via governance. New nodes will not be able to join the -# validator set before the update is made. Measured in Quants (1 Quant = 10^-8 SUPRA). -min_stake = 0 -# The number of tokens initially allocated to node operators. Tokens will be earned through block -# rewards. -operator_account_balance = 0 -# The amount of Quants to transfer from each validator owner account to the corresponding stake pool -# after it has been created, during the genesis transaction. Adding the stake after creating the -# pool allows us to ensure that The Foundation's stake is not subject to the PBO locking schedule, -# which is only intended to apply to winners of the Project Blast Off campaign. -# -# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 55_000_000 SUPRA. -pbo_owner_stake = 5500000000000000 -# The number of seconds after `genesis_timestamp_microseconds` at which all accounts with -# allocations at genesis will be able to unlock their initial amounts. These amounts are stored -# in vesting contracts. -# -# Corresponds to Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -remaining_balance_lockup_cliff_period_in_seconds = 0 -# The amount of SUPRA required to qualify as a proposer (this parameter is currently unused). -required_proposer_stake = 0 -# The annual percent yield for validators, proportional to their stake. Specified as a percentage -# with 2 decimals of precision in u64 format due to limitations in the MoveVM. The below value -# represents 12.85%. -rewards_apy_percentage = 1285 -# The percentage of staking rewards earned by Supra Foundation controlled nodes that will be paid -# to the corresponding node operators. Specified as a percentage with 2 decimals of precision in -# u64 format due to limitations in the MoveVM. The below value represents 37.74%. -validator_commission_rate_percentage = 3774 -# The percentage of new stake relative to the current total stake that can join the validator -# set or be added to existing validators within a single epoch. -voting_power_increase_limit = 33 - - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. Paths -# should be specified either as absolute paths or as relative to the `supra` binary. -# -# When running the node in Docker and working with a mounted directory, the paths must be specified -# with reference to the location of the mounter directory in the Docker filesystem rather than the -# host filesystem. The default settings provided below assume the default Docker configuration -# provided by the `manage_supra_nodes.sh` script. - -[node] -# The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 1 -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" -# The port on which to listen for connections from the associated RPC node. Each validator -# may serve at most one RPC node. -rpc_access_port = 26000 -# The path to the TLS certificate for this node. -server_cert_path = "./configs/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -server_private_key_path = "./configs/server_supra_key.pem" - -# Parameters for the blockchain database. -[node.database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/smr_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true - -# Parameters for the DKG database. -[node.database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/ledger_storage" - -# Parameters related to database pruning. -[node.database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 \ No newline at end of file diff --git a/node_management/migration/smr/src/migrate_smr_settings/cli.py b/node_management/migration/smr/src/migrate_smr_settings/cli.py deleted file mode 100644 index 2e5cdc98..00000000 --- a/node_management/migration/smr/src/migrate_smr_settings/cli.py +++ /dev/null @@ -1,53 +0,0 @@ -from .migrate_v7_to_v9 import migrate_v7_to_v9 - -from common import migration - - - -SMR_SETTINGS_MIGRATE_PATH = { - "v7->v9": [migrate_v7_to_v9], -} - - -def migrate_config(migrate_path: str, from_path: str, to_path: str): - - migration.Migration(SMR_SETTINGS_MIGRATE_PATH).migrate_config( - migrate_path, from_path, to_path - ) - -def main(): - import argparse - - - parser = argparse.ArgumentParser( - description="Migrate validator smr_settings.toml file from old version to new one." - ) - choices_str = ", ".join([f"'{k}'" for k in SMR_SETTINGS_MIGRATE_PATH.keys()]) - parser.add_argument( - "-p", - "--migrate-path", - required=True, - type=str, - metavar="", - choices=list(SMR_SETTINGS_MIGRATE_PATH.keys()), - help=f"Migration path choices: {choices_str}", - ) - - parser.add_argument( - "-f", - "--from-file", - required=True, - type=str, - metavar="", - help="Path to the old version config file to migrate", - ) - parser.add_argument( - "-t", - "--to-file", - required=True, - type=str, - metavar="", - help="Path to save the migrated config file", - ) - args = parser.parse_args() - migrate_config(args.migrate_path, args.from_file, args.to_file) diff --git a/node_management/migration/rpc/src/migrate_rpc_config/__init__.py b/node_management/migration/src/cli/__init__.py similarity index 100% rename from node_management/migration/rpc/src/migrate_rpc_config/__init__.py rename to node_management/migration/src/cli/__init__.py diff --git a/node_management/migration/src/cli/main.py b/node_management/migration/src/cli/main.py new file mode 100644 index 00000000..23c2015d --- /dev/null +++ b/node_management/migration/src/cli/main.py @@ -0,0 +1,60 @@ +import click +from rpc_config.migrate_path import RPC_CONFIG_MIGRATE_PATH +from rpc_config.migrate_path import run_migration as migrate_rpc_config +from smr_settings.migrate_path import SMR_SETTINGS_MIGRATE_PATH +from smr_settings.migrate_path import run_migration as migrate_smr_config + + +@click.group() +def main(): + """Migration CLI for Supra configs.""" + + +@main.command() +@click.option( + "--migrate-path", + "-p", + required=True, + type=click.Choice(RPC_CONFIG_MIGRATE_PATH, case_sensitive=True), + help=f"Migration path (choices: {', '.join(RPC_CONFIG_MIGRATE_PATH)})", +) +@click.option( + "--from-file", + "-f", + required=True, + type=click.Path(exists=True), + help="Source config file", +) +@click.option( + "--to-file", "-t", required=True, type=click.Path(), help="Output config file" +) +def rpc(migrate_path, from_file, to_file): + """Migrate RPC config.""" + migrate_rpc_config(migrate_path, from_file, to_file) + + +@main.command() +@click.option( + "--migrate-path", + "-p", + required=True, + type=click.Choice(SMR_SETTINGS_MIGRATE_PATH, case_sensitive=True), + help=f"Migration path (choices: {', '.join(SMR_SETTINGS_MIGRATE_PATH)})", +) +@click.option( + "--from-file", + "-f", + required=True, + type=click.Path(exists=True), + help="Source config file", +) +@click.option( + "--to-file", "-t", required=True, type=click.Path(), help="Output config file" +) +def smr(migrate_path, from_file, to_file): + """Migrate SMR config.""" + migrate_smr_config(migrate_path, from_file, to_file) + + +if __name__ == "__main__": + main() diff --git a/node_management/migration/common/src/common/__init__ b/node_management/migration/src/common/__init__ similarity index 100% rename from node_management/migration/common/src/common/__init__ rename to node_management/migration/src/common/__init__ diff --git a/node_management/migration/common/src/common/migration.py b/node_management/migration/src/common/migration.py similarity index 65% rename from node_management/migration/common/src/common/migration.py rename to node_management/migration/src/common/migration.py index da5383d1..9619bade 100644 --- a/node_management/migration/common/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -4,26 +4,40 @@ from copy import deepcopy from . import utils +class MigrationPathSet: + """ + Base class for migration paths. + """ + def __init__(self, migrate_paths: ty.Dict[str, ty.List[ty.Callable]]): + self.migrate_paths = migrate_paths + + def get_versions(self, key: str) -> ty.Tuple[str, str]: + """Split the key into from_version and to_version.""" + if key not in self.migrate_paths: + raise ValueError(f"Invalid key: {key}") + from_version, to_version = key.split('-', 1) + return from_version, to_version + + def get_migration_functions(self, key: str) -> ty.List[ty.Callable]: + """Get the migration functions for the given key.""" + if key not in self.migrate_paths: + raise ValueError(f"Unknown migration path: {key}") + return self.migrate_paths[key] + + class Migration: + """ + Top level migration class that handles the migration of config files. + """ def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]): - self.migrate_path = migrate_path + self.migrate_path = MigrationPathSet(migrate_path) def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): - from_and_to_version = migrate_choice.split("->") - if len(from_and_to_version) != 2: - raise SystemExit( - f"Error: invalid migration choice {migrate_choice}. It should be in the format 'vX->vY'." - ) - - # Get the migration functions for the given path - migrate_functions = self.migrate_path.pop(migrate_choice) - if not migrate_functions: - raise SystemExit(f"Error: unsupported migration path {migrate_choice}.") - from_version = from_and_to_version[0] - to_version = from_and_to_version[1] + migrate_functions = self.migrate_path.get_migration_functions(migrate_choice) + from_version, to_version = self.migrate_path.get_versions(migrate_choice) default_backup_path = f"{from_path}_{from_version}.bak" if from_path == to_path: print(f"Warning: The source and destination paths are the same ({from_path}).") diff --git a/node_management/migration/common/src/common/utils.py b/node_management/migration/src/common/utils.py similarity index 97% rename from node_management/migration/common/src/common/utils.py rename to node_management/migration/src/common/utils.py index 1eb20c2a..35824476 100644 --- a/node_management/migration/common/src/common/utils.py +++ b/node_management/migration/src/common/utils.py @@ -1,5 +1,5 @@ import difflib - +import typing as ty def unified_diff(from_str, to_str, fromfile, tofile): diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) @@ -24,3 +24,5 @@ def print_with_checkmark(message): Print a message with a checkmark. """ print(f"✓ {message}") + + diff --git a/node_management/migration/smr/src/migrate_smr_settings/__init__.py b/node_management/migration/src/rpc_config/__init__.py similarity index 100% rename from node_management/migration/smr/src/migrate_smr_settings/__init__.py rename to node_management/migration/src/rpc_config/__init__.py diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py b/node_management/migration/src/rpc_config/from_v7_to_v8.py similarity index 100% rename from node_management/migration/rpc/src/migrate_rpc_config/from_v7_to_v8.py rename to node_management/migration/src/rpc_config/from_v7_to_v8.py diff --git a/node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py b/node_management/migration/src/rpc_config/from_v8_to_v9.py similarity index 100% rename from node_management/migration/rpc/src/migrate_rpc_config/from_v8_to_v9.py rename to node_management/migration/src/rpc_config/from_v8_to_v9.py diff --git a/node_management/migration/src/rpc_config/migrate_path.py b/node_management/migration/src/rpc_config/migrate_path.py new file mode 100644 index 00000000..dfcbeebd --- /dev/null +++ b/node_management/migration/src/rpc_config/migrate_path.py @@ -0,0 +1,20 @@ +from .from_v7_to_v8 import migrate_v7_to_v8 +from .from_v8_to_v9 import migrate_v8_to_v9 + +from common import migration + + + +RPC_CONFIG_MIGRATE_PATH = { + "v7-v8": [migrate_v7_to_v8], + "v8-v9": [migrate_v8_to_v9], + "v7-v9": [migrate_v7_to_v8, migrate_v8_to_v9], +} + + +def run_migration(migrate_path: str, from_path: str, to_path: str): + + migration.Migration(RPC_CONFIG_MIGRATE_PATH).migrate_config( + migrate_path, from_path, to_path + ) + diff --git a/node_management/migration/src/smr_settings/__init__.py b/node_management/migration/src/smr_settings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/node_management/migration/smr/src/migrate_smr_settings/migrate_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py similarity index 100% rename from node_management/migration/smr/src/migrate_smr_settings/migrate_v7_to_v9.py rename to node_management/migration/src/smr_settings/from_v7_to_v9.py diff --git a/node_management/migration/src/smr_settings/migrate_path.py b/node_management/migration/src/smr_settings/migrate_path.py new file mode 100644 index 00000000..1ca63870 --- /dev/null +++ b/node_management/migration/src/smr_settings/migrate_path.py @@ -0,0 +1,17 @@ +from .from_v7_to_v9 import migrate_v7_to_v9 + +from common import migration + + + +SMR_SETTINGS_MIGRATE_PATH = { + "v7-v9": [migrate_v7_to_v9], +} + + +def run_migration(migrate_path: str, from_path: str, to_path: str): + + migration.Migration(SMR_SETTINGS_MIGRATE_PATH).migrate_config( + migrate_path, from_path, to_path + ) + diff --git a/node_management/migration/rpc/config_v7.1.8.toml b/node_management/migration/tests/config_v7.1.8.toml similarity index 100% rename from node_management/migration/rpc/config_v7.1.8.toml rename to node_management/migration/tests/config_v7.1.8.toml diff --git a/node_management/migration/rpc/config_v8.0.2.toml b/node_management/migration/tests/config_v8.0.2.toml similarity index 100% rename from node_management/migration/rpc/config_v8.0.2.toml rename to node_management/migration/tests/config_v8.0.2.toml diff --git a/node_management/migration/rpc/config_v9.0.7.toml b/node_management/migration/tests/config_v9.0.7.toml similarity index 100% rename from node_management/migration/rpc/config_v9.0.7.toml rename to node_management/migration/tests/config_v9.0.7.toml diff --git a/node_management/migration/smr/smr_settings_v7.1.x.toml b/node_management/migration/tests/smr_settings_v7.1.x.toml similarity index 100% rename from node_management/migration/smr/smr_settings_v7.1.x.toml rename to node_management/migration/tests/smr_settings_v7.1.x.toml diff --git a/node_management/migration/smr/smr_settings_v9.0.x.toml b/node_management/migration/tests/smr_settings_v9.0.x.toml similarity index 100% rename from node_management/migration/smr/smr_settings_v9.0.x.toml rename to node_management/migration/tests/smr_settings_v9.0.x.toml diff --git a/node_management/migration/rpc/test_migration.py b/node_management/migration/tests/test_rpc_migration.py similarity index 79% rename from node_management/migration/rpc/test_migration.py rename to node_management/migration/tests/test_rpc_migration.py index 6c295c96..a6ee01eb 100644 --- a/node_management/migration/rpc/test_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -3,7 +3,7 @@ import tempfile import tomlkit import pytest -from migrate_rpc_config.cli import migrate_config +from rpc_config.migrate_path import run_migration as rpc_run_migration CONFIGS = [ "config_v7.1.8.toml", @@ -12,9 +12,9 @@ ] @pytest.mark.parametrize("from_file,to_file,migrate_path", [ - ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7->v8"), - ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8->v9"), - ("config_v7.1.8.toml", "config_v9.0.7.toml", "v7->v9"), + ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7-v8"), + ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8-v9"), + ("config_v7.1.8.toml", "config_v9.0.7.toml", "v7-v9"), ]) def test_migration(tmp_path, from_file, to_file, migrate_path): # Copy config files to temp dir @@ -25,7 +25,7 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): tmp_to = tmp_path / to_file shutil.copy(from_path, tmp_from) # Run migration - migrate_config(migrate_path, str(tmp_from), str(tmp_to)) + rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files with open(tmp_to, "r") as f: migrated = tomlkit.parse(f.read()) diff --git a/node_management/migration/smr/test_migration.py b/node_management/migration/tests/test_smr_migration.py similarity index 86% rename from node_management/migration/smr/test_migration.py rename to node_management/migration/tests/test_smr_migration.py index 2d8a6245..701edee8 100644 --- a/node_management/migration/smr/test_migration.py +++ b/node_management/migration/tests/test_smr_migration.py @@ -2,7 +2,7 @@ import shutil import tomlkit import pytest -from migrate_smr_settings.cli import migrate_config +from smr_settings.migrate_path import run_migration as smr_run_migration CONFIGS = [ "smr_settings_v7.1.x.toml", @@ -10,7 +10,7 @@ ] @pytest.mark.parametrize("from_file,to_file,migrate_path", [ - ("smr_settings_v7.1.x.toml", "smr_settings_v9.0.x.toml", "v7->v9"), + ("smr_settings_v7.1.x.toml", "smr_settings_v9.0.x.toml", "v7-v9"), ]) def test_migration(tmp_path, from_file, to_file, migrate_path): # Copy config files to temp dir @@ -21,7 +21,7 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): tmp_to = tmp_path / to_file shutil.copy(from_path, tmp_from) # Run migration - migrate_config(migrate_path, str(tmp_from), str(tmp_to)) + smr_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files with open(tmp_to, "r") as f: migrated = tomlkit.parse(f.read()) From b4cc37e008ea2dc59e0f3a09977dfe77fb422f1f Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 22 May 2025 22:47:19 +0200 Subject: [PATCH 15/44] update readme --- node_management/migration/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/node_management/migration/README.md b/node_management/migration/README.md index 8d31e5db..c267fb0e 100644 --- a/node_management/migration/README.md +++ b/node_management/migration/README.md @@ -1,6 +1,13 @@ ### Usage Example +alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node +alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node + +alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra" +alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra" + + ```sh pip install . @@ -13,8 +20,9 @@ rpc-v9 migrate-db config.toml # Migrate cli profile from v7 to v8 supra-v8 migrate --network localnet -# Migrate identity from v7 to v8 cp validator_identity.pem node_identity.pem +# Migrate cli profile from v8 to v9 +supra-v9 profile migrate # Migrate smr_settings from v7 to v9 migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml From b221ad35b050fee996aac25b4fe2629bb92ea206 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Fri, 23 May 2025 15:17:57 +0200 Subject: [PATCH 16/44] address comments --- .../migration/src/common/migration.py | 16 ++++---- node_management/migration/src/common/utils.py | 4 +- .../migration/src/rpc_config/from_v7_to_v8.py | 36 +++++++++++++++++- .../migration/src/rpc_config/from_v8_to_v9.py | 12 +++++- .../migration/src/rpc_config/migrate_path.py | 3 -- .../src/smr_settings/from_v7_to_v9.py | 37 +++++++------------ .../src/smr_settings/migrate_path.py | 3 -- .../migration/tests/test_rpc_migration.py | 22 ++++++----- .../migration/tests/test_smr_migration.py | 14 +++++-- 9 files changed, 92 insertions(+), 55 deletions(-) diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py index 9619bade..ff3d7d79 100644 --- a/node_management/migration/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -1,13 +1,14 @@ - import typing as ty import tomlkit from copy import deepcopy from . import utils + class MigrationPathSet: """ Base class for migration paths. """ + def __init__(self, migrate_paths: ty.Dict[str, ty.List[ty.Callable]]): self.migrate_paths = migrate_paths @@ -15,32 +16,33 @@ def get_versions(self, key: str) -> ty.Tuple[str, str]: """Split the key into from_version and to_version.""" if key not in self.migrate_paths: raise ValueError(f"Invalid key: {key}") - from_version, to_version = key.split('-', 1) + from_version, to_version = key.split("-", 1) return from_version, to_version - + def get_migration_functions(self, key: str) -> ty.List[ty.Callable]: """Get the migration functions for the given key.""" if key not in self.migrate_paths: raise ValueError(f"Unknown migration path: {key}") return self.migrate_paths[key] - + class Migration: """ Top level migration class that handles the migration of config files. """ + def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]): self.migrate_path = MigrationPathSet(migrate_path) - def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): - migrate_functions = self.migrate_path.get_migration_functions(migrate_choice) from_version, to_version = self.migrate_path.get_versions(migrate_choice) default_backup_path = f"{from_path}_{from_version}.bak" if from_path == to_path: - print(f"Warning: The source and destination paths are the same ({from_path}).") + print( + f"Warning: The source and destination paths are the same ({from_path})." + ) print( f"A backup of your original config will be saved to: {default_backup_path}" ) diff --git a/node_management/migration/src/common/utils.py b/node_management/migration/src/common/utils.py index 35824476..1eb20c2a 100644 --- a/node_management/migration/src/common/utils.py +++ b/node_management/migration/src/common/utils.py @@ -1,5 +1,5 @@ import difflib -import typing as ty + def unified_diff(from_str, to_str, fromfile, tofile): diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) @@ -24,5 +24,3 @@ def print_with_checkmark(message): Print a message with a checkmark. """ print(f"✓ {message}") - - diff --git a/node_management/migration/src/rpc_config/from_v7_to_v8.py b/node_management/migration/src/rpc_config/from_v7_to_v8.py index 9f07e611..ece7f1ee 100644 --- a/node_management/migration/src/rpc_config/from_v7_to_v8.py +++ b/node_management/migration/src/rpc_config/from_v7_to_v8.py @@ -1,5 +1,35 @@ """ -v7 to v8 migration functions for Supra's RPC configuration. +This module provides migration functions to upgrade Supra's RPC configuration from version 7 (v7) to version 8 (v8). + +Migration Overview: +------------------- +The migration process involves restructuring and updating the configuration TOML data to match the v8 schema. The key changes performed by this migration are: + +1. Synchronization Parameters: + - Moves the following keys from the root level to a new `[synchronization.ws]` table: + - `consensus_rpc` + - `consensus_client_cert_path` + - `consensus_client_private_key_path` + - `consensus_root_ca_cert_path` + - Raises an error if a `[synchronization]` table already exists in the v7 config. + +2. Chain State Assembler Parameters: + - Moves `sync_retry_interval_in_secs` from the root level to a new `[chain_state_assembler]` table. + - Ensures `sync_retry_interval_in_secs` is set to `1` as recommended. + - Adds a new parameter `certified_block_cache_bucket_size = 50` to `[chain_state_assembler]`. + - Raises an error if a `[chain_state_assembler]` table already exists in the v7 config. + +3. Consensus Access Tokens: + - Adds a new root-level parameter: `consensus_access_tokens = []`. + +4. Block Provider Trust: + - Adds a new root-level parameter: `block_provider_is_trusted = True`. + +Warnings and Checks: +-------------------- +- Issues warnings if expected keys are missing or have empty values. +- Uses utility functions to print migration steps with checkmarks for better traceability. + """ import tomlkit @@ -79,7 +109,9 @@ def __migrate_chain_state_assembler_parameters(toml_data): chain_assembler_table[key] = value new_parameters = {"certified_block_cache_bucket_size": 50} for key, value in new_parameters.items(): - utils.print_with_checkmark(f"Adding `{key} = {value}` to [chain_state_assembler]") + utils.print_with_checkmark( + f"Adding `{key} = {value}` to [chain_state_assembler]" + ) chain_assembler_table[key] = value toml_data["chain_state_assembler"] = chain_assembler_table diff --git a/node_management/migration/src/rpc_config/from_v8_to_v9.py b/node_management/migration/src/rpc_config/from_v8_to_v9.py index bdc5e9d5..c108f749 100644 --- a/node_management/migration/src/rpc_config/from_v8_to_v9.py +++ b/node_management/migration/src/rpc_config/from_v8_to_v9.py @@ -1,5 +1,15 @@ """ -v8 to v9 migration functions for Supra RPC config file. +This module provides migration functions to upgrade a Supra RPC config file from version v8 to v9. + +Key changes handled by this migration: +- Moves and renames certificate-related keys from the [synchronization.ws] table to a new [synchronization.ws.certificates] table. + - "consensus_client_cert_path" is renamed to "cert_path" + - "consensus_client_private_key_path" is renamed to "private_key_path" + - "consensus_root_ca_cert_path" is renamed to "root_ca_cert_path" +- Ensures that the [synchronization] table exists and that the [synchronization.ws.certificates] table does not already exist in the v8 config. +- Warns if any expected keys are missing or have empty values, indicating potential issues with the input config. +- Provides utility output to indicate the migration steps being performed. + """ import tomlkit diff --git a/node_management/migration/src/rpc_config/migrate_path.py b/node_management/migration/src/rpc_config/migrate_path.py index dfcbeebd..2778caac 100644 --- a/node_management/migration/src/rpc_config/migrate_path.py +++ b/node_management/migration/src/rpc_config/migrate_path.py @@ -4,7 +4,6 @@ from common import migration - RPC_CONFIG_MIGRATE_PATH = { "v7-v8": [migrate_v7_to_v8], "v8-v9": [migrate_v8_to_v9], @@ -13,8 +12,6 @@ def run_migration(migrate_path: str, from_path: str, to_path: str): - migration.Migration(RPC_CONFIG_MIGRATE_PATH).migrate_config( migrate_path, from_path, to_path ) - diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index 59ec7f6f..98edd802 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -1,24 +1,15 @@ """ -remove [move_vm] table +Migration script for upgrading Supra smr_settings config files from version v7 to v9. -move below keys in [node] +This module provides functions to transform configuration files in TOML format to be compatible with v9. +The migration includes the following changes: -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" -# The path to the TLS certificate for this node. -server_cert_path = "./configs/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -server_private_key_path = "./configs/server_supra_key.pem" - -==> - -[node.ws_server.certificates] -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "configs/ca_certificate.pem" -# The path to the TLS certificate for this node. -cert_path = "configs/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "configs/server_supra_key.pem" +1. Removal of the `[move_vm]` table from the configuration. +2. Migration of WebSocket server certificate paths: + - Moves `root_ca_cert_path`, `server_cert_path`, and `server_private_key_path` from the `[node]` table. + - Renames `server_cert_path` to `cert_path` and `server_private_key_path` to `private_key_path`. + - Creates a new `[node.ws_server.certificates]` table to store these certificate paths. + - Ensures that the `[node.ws_server]` table does not exist prior to migration, enforcing correct migration order. """ @@ -27,9 +18,7 @@ def __migrate_move_vm_parameters(toml_data): - utils.print_with_checkmark( - "Removing [move_vm] table" - ) + utils.print_with_checkmark("Removing [move_vm] table") toml_data.pop("move_vm") @@ -55,7 +44,9 @@ def __migrate_node_ws_certificates(toml_data): print( f"Warning: `{old_key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" ) - utils.print_with_checkmark(f"Moving `{old_key}` from [node] to {new_key} in [node.ws_server.certificates]") + utils.print_with_checkmark( + f"Moving `{old_key}` from [node] to {new_key} in [node.ws_server.certificates]" + ) ws_certificates_v9[new_key] = value else: print(f"Warning: `{old_key}` not found in [node]") @@ -69,4 +60,4 @@ def __migrate_node_ws_certificates(toml_data): def migrate_v7_to_v9(toml_data): __migrate_move_vm_parameters(toml_data) - __migrate_node_ws_certificates(toml_data) \ No newline at end of file + __migrate_node_ws_certificates(toml_data) diff --git a/node_management/migration/src/smr_settings/migrate_path.py b/node_management/migration/src/smr_settings/migrate_path.py index 1ca63870..829da3a5 100644 --- a/node_management/migration/src/smr_settings/migrate_path.py +++ b/node_management/migration/src/smr_settings/migrate_path.py @@ -3,15 +3,12 @@ from common import migration - SMR_SETTINGS_MIGRATE_PATH = { "v7-v9": [migrate_v7_to_v9], } def run_migration(migrate_path: str, from_path: str, to_path: str): - migration.Migration(SMR_SETTINGS_MIGRATE_PATH).migrate_config( migrate_path, from_path, to_path ) - diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py index a6ee01eb..54eccc6c 100644 --- a/node_management/migration/tests/test_rpc_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -1,6 +1,5 @@ import os import shutil -import tempfile import tomlkit import pytest from rpc_config.migrate_path import run_migration as rpc_run_migration @@ -11,11 +10,15 @@ "config_v9.0.7.toml", ] -@pytest.mark.parametrize("from_file,to_file,migrate_path", [ - ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7-v8"), - ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8-v9"), - ("config_v7.1.8.toml", "config_v9.0.7.toml", "v7-v9"), -]) + +@pytest.mark.parametrize( + "from_file,to_file,migrate_path", + [ + ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7-v8"), + ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8-v9"), + ("config_v7.1.8.toml", "config_v9.0.7.toml", "v7-v9"), + ], +) def test_migration(tmp_path, from_file, to_file, migrate_path): # Copy config files to temp dir src_dir = os.path.dirname(os.path.abspath(__file__)) @@ -31,16 +34,17 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): migrated = tomlkit.parse(f.read()) with open(to_path, "r") as f: expected = tomlkit.parse(f.read()) - + # if "allowed_origin" in migrated: # del migrated["allowed_origin"] # if "allowed_origin" in expected: # del expected["allowed_origin"] - # if "chain_instance" in migrated: # del migrated["chain_instance"] # if "chain_instance" in expected: # del expected["chain_instance"] # Compare TOML dicts - assert migrated == expected, f"Migration {migrate_path} failed: {from_file} -> {to_file}" + assert migrated == expected, ( + f"Migration {migrate_path} failed: {from_file} -> {to_file}" + ) diff --git a/node_management/migration/tests/test_smr_migration.py b/node_management/migration/tests/test_smr_migration.py index 701edee8..b402d285 100644 --- a/node_management/migration/tests/test_smr_migration.py +++ b/node_management/migration/tests/test_smr_migration.py @@ -9,9 +9,13 @@ "smr_settings_v9.0.x.toml", ] -@pytest.mark.parametrize("from_file,to_file,migrate_path", [ - ("smr_settings_v7.1.x.toml", "smr_settings_v9.0.x.toml", "v7-v9"), -]) + +@pytest.mark.parametrize( + "from_file,to_file,migrate_path", + [ + ("smr_settings_v7.1.x.toml", "smr_settings_v9.0.x.toml", "v7-v9"), + ], +) def test_migration(tmp_path, from_file, to_file, migrate_path): # Copy config files to temp dir src_dir = os.path.dirname(os.path.abspath(__file__)) @@ -28,4 +32,6 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): with open(to_path, "r") as f: expected = tomlkit.parse(f.read()) # Compare TOML dicts - assert migrated == expected, f"Migration {migrate_path} failed: {from_file} -> {to_file}" + assert migrated == expected, ( + f"Migration {migrate_path} failed: {from_file} -> {to_file}" + ) From 710b4dfb618d5a5cbd87f2d4440fc702b774d60c Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 26 May 2025 16:26:55 +0200 Subject: [PATCH 17/44] prepare v9 mainnet rpc config template --- .../migration/tests/config_v9_mainnet.toml | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 node_management/migration/tests/config_v9_mainnet.toml diff --git a/node_management/migration/tests/config_v9_mainnet.toml b/node_management/migration/tests/config_v9_mainnet.toml new file mode 100644 index 00000000..55a06520 --- /dev/null +++ b/node_management/migration/tests/config_v9_mainnet.toml @@ -0,0 +1,132 @@ +# Version: v9 mainnet +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +# The access tokens used to authenticate public RPC requests to this RPC node. +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +certified_block_cache_bucket_size = 50 +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The websocket address of the attached validator. +consensus_rpc = "ws://:26000" + +[synchronization.ws.certificates] +# The path to the TLS certificate for the connection with the attached validator. +cert_path = "./configs/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./configs/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +# The below settings are the default values required for use in RPC nodes run by validator node operators. +# They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:26000" +description = "LocalNet" +mode = "Server" \ No newline at end of file From 21e08affb7803782649a6e2674112cbbda80ed6c Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 26 May 2025 16:32:35 +0200 Subject: [PATCH 18/44] prepare v9 mainnet smr settings template --- .../tests/smr_settings_v9_mainnet.toml | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 node_management/migration/tests/smr_settings_v9_mainnet.toml diff --git a/node_management/migration/tests/smr_settings_v9_mainnet.toml b/node_management/migration/tests/smr_settings_v9_mainnet.toml new file mode 100644 index 00000000..40440880 --- /dev/null +++ b/node_management/migration/tests/smr_settings_v9_mainnet.toml @@ -0,0 +1,125 @@ +# Version: v9.0.x + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 4 hours. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 2 hours. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 + +[node.ws_server.certificates] +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "configs/ca_certificate.pem" +# The path to the TLS certificate for this node. +cert_path = "configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "configs/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 From 8bd12e8e32078a8b929ae180bec9513f13600d31 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 26 May 2025 16:35:13 +0200 Subject: [PATCH 19/44] update docs --- node_management/migration/tests/smr_settings_v9_mainnet.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_management/migration/tests/smr_settings_v9_mainnet.toml b/node_management/migration/tests/smr_settings_v9_mainnet.toml index 40440880..1940ff3a 100644 --- a/node_management/migration/tests/smr_settings_v9_mainnet.toml +++ b/node_management/migration/tests/smr_settings_v9_mainnet.toml @@ -1,4 +1,4 @@ -# Version: v9.0.x +# Version: v9 mainnet ####################################### PROTOCOL PARAMETERS ####################################### From 1ecfe570300aea53b8461af3fce0f985f7d91da3 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 26 May 2025 17:51:58 +0200 Subject: [PATCH 20/44] update docs --- .../{config_v9_mainnet.toml => config_v9.1_mainnet.toml} | 8 +++++++- ...ngs_v9_mainnet.toml => smr_settings_v9.1_mainnet.toml} | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) rename node_management/migration/tests/{config_v9_mainnet.toml => config_v9.1_mainnet.toml} (91%) rename node_management/migration/tests/{smr_settings_v9_mainnet.toml => smr_settings_v9.1_mainnet.toml} (99%) diff --git a/node_management/migration/tests/config_v9_mainnet.toml b/node_management/migration/tests/config_v9.1_mainnet.toml similarity index 91% rename from node_management/migration/tests/config_v9_mainnet.toml rename to node_management/migration/tests/config_v9.1_mainnet.toml index 55a06520..45167d5e 100644 --- a/node_management/migration/tests/config_v9_mainnet.toml +++ b/node_management/migration/tests/config_v9.1_mainnet.toml @@ -1,4 +1,5 @@ -# Version: v9 mainnet +# Version: v9.1 mainnet + ####################################### PROTOCOL PARAMETERS ####################################### # The below parameters are fixed for the protocol and must be agreed upon by all node operators @@ -12,6 +13,8 @@ bind_addr = "0.0.0.0:26000" # If `true` then blocks will not be verified before execution. This value should be `false` # unless you also control the node from which this RPC node is receiving blocks. block_provider_is_trusted = true +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. resume = true # The path to `supra_committees.json`. supra_committees_config = "./configs/supra_committees.json" @@ -36,7 +39,10 @@ genesis_timestamp_microseconds = 1732060800000000 ######################################### NODE PARAMETERS ######################################### [chain_state_assembler] +# Number of certified blocks stored in memory as reference to pending blocks to be executed. +# Only this amount of certified blocks are stored in memory, for the rest memo is kept. certified_block_cache_bucket_size = 50 +# Retry interval for the sync requests for which no response yet available. sync_retry_interval_in_secs = 1 [synchronization.ws] diff --git a/node_management/migration/tests/smr_settings_v9_mainnet.toml b/node_management/migration/tests/smr_settings_v9.1_mainnet.toml similarity index 99% rename from node_management/migration/tests/smr_settings_v9_mainnet.toml rename to node_management/migration/tests/smr_settings_v9.1_mainnet.toml index 1940ff3a..6cdd891f 100644 --- a/node_management/migration/tests/smr_settings_v9_mainnet.toml +++ b/node_management/migration/tests/smr_settings_v9.1_mainnet.toml @@ -1,4 +1,4 @@ -# Version: v9 mainnet +# Version: v9.1 mainnet ####################################### PROTOCOL PARAMETERS ####################################### From ce726ac1c6c6d8449384e82ad57e15054015df2c Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 26 May 2025 22:13:59 +0200 Subject: [PATCH 21/44] migrate to v9 using templating --- .../migration/src/common/migration.py | 4 +- .../migration/src/rpc_config/from_v7_to_v9.py | 78 ++++++++++ .../migration/src/rpc_config/migrate_path.py | 4 +- .../src/rpc_config/v9_1_x_mainnet_template.py | 140 ++++++++++++++++++ .../migration/tests/config_v7.1.8.toml | 22 +-- .../tests/config_v7_to_v9_expected.toml | 138 +++++++++++++++++ .../migration/tests/test_rpc_migration.py | 17 ++- 7 files changed, 380 insertions(+), 23 deletions(-) create mode 100644 node_management/migration/src/rpc_config/from_v7_to_v9.py create mode 100644 node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py create mode 100644 node_management/migration/tests/config_v7_to_v9_expected.toml diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py index ff3d7d79..01089fec 100644 --- a/node_management/migration/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -61,7 +61,8 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): for fn in migrate_functions: print(f"Running migration function: {fn.__name__}") - fn(toml_data) + toml_data = fn(toml_data) + print("!!!!", toml_data, "!!!!") print(f"Backing up old config to {default_backup_path}") tomlkit.dump(original_toml_data, open(default_backup_path, "w")) @@ -70,7 +71,6 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): print(f"Writing new config to {to_path}") with open(to_path, "w") as f: f.write(tomlkit.dumps(toml_data)) - # Print the diff from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py new file mode 100644 index 00000000..411a34d4 --- /dev/null +++ b/node_management/migration/src/rpc_config/from_v7_to_v9.py @@ -0,0 +1,78 @@ +""" +This module provides migration functions to upgrade Supra's RPC configuration from version v7 to v9. + +Migration Overview: +------------------- +The migration process involves restructuring and updating the configuration TOML data to match the v9 schema. The key changes performed by this migration are: + +1. Synchronization Parameters: + - Copy the following key value from v7 root level to a new `[synchronization.ws]` table in the v9 template: + - `consensus_rpc` + - Copy the following keys' values from v7 root level to the `[synchronization.ws.certificates]` table in v9 template: + - from `consensus_client_cert_path` to `cert_path` + - from `consensus_client_private_key_path` to `private_key_path` + - from `consensus_root_ca_cert_path` to `root_ca_cert_path` + - Raises an error if a `[synchronization]` table already exists in the v7 config. + +2. Chain State Assembler Parameters: + - Raises an error if a `[chain_state_assembler]` table already exists in the v7 config. + +3. Copy value from 7 to v9 template in same table: + - bind_addr + - supra_committees_config + - [database_setup.dbs.archive.rocks_db].path + - [database_setup.dbs.ledger.rocks_db].path + - [database_setup.dbs.chain_store.rocks_db].path + - [[database_setup.snapshot_config]].path + +""" +import tomlkit + +from .v9_1_x_mainnet_template import RPC_CONFIG_V9_1_MAINNET_TEMPLATE + + +def __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data): + if "synchronization" in v7_toml_data: + raise SystemExit( + "Error: [synchronization] table should not exist in v7 config. Please check your migration path matching the version of your config file." + ) + v9_toml_data["synchronization"]["ws"]["consensus_rpc"] = v7_toml_data["consensus_rpc"] + + v9_toml_data["synchronization"]["ws"]["certificates"]["cert_path"] = v7_toml_data["consensus_client_cert_path"] + v9_toml_data["synchronization"]["ws"]["certificates"]["private_key_path"] = v7_toml_data["consensus_client_private_key_path"] + v9_toml_data["synchronization"]["ws"]["certificates"]["root_ca_cert_path"] = v7_toml_data["consensus_root_ca_cert_path"] + + +def __migrate_chain_state_assembler_parameters(v7_toml_data, v9_toml_data): + if "chain_state_assembler" in v7_toml_data: + raise SystemExit( + "Error: [chain_state_assembler] table should not exist in v7 config." + ) + # No changes needed for chain state assembler parameters in v9 + pass + +def __migrate_bind_addr(v7_toml_data, v9_toml_data): + v9_toml_data["bind_addr"] = v7_toml_data["bind_addr"] + + +def __migrate_supra_committees_config(v7_toml_data, v9_toml_data): + v9_toml_data["supra_committees_config"] = v7_toml_data["supra_committees_config"] + +def __migrate_database_paths(v7_toml_data, v9_toml_data): + v9_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"]["path"] + v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] + v9_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] + +def __migrate_snapshot_paths(v7_toml_data, v9_toml_data): + v9_toml_data["database_setup"]["snapshot_config"]["path"] = v7_toml_data["database_setup"]["snapshot_config"]["path"] + + +def migrate_v7_to_v9(v7_toml_data): + v9_toml_data = tomlkit.parse(RPC_CONFIG_V9_1_MAINNET_TEMPLATE) + __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data) + __migrate_chain_state_assembler_parameters(v7_toml_data, v9_toml_data) + __migrate_bind_addr(v7_toml_data, v9_toml_data) + __migrate_supra_committees_config(v7_toml_data, v9_toml_data) + __migrate_database_paths(v7_toml_data, v9_toml_data) + __migrate_snapshot_paths(v7_toml_data, v9_toml_data) + return v9_toml_data diff --git a/node_management/migration/src/rpc_config/migrate_path.py b/node_management/migration/src/rpc_config/migrate_path.py index 2778caac..dc783be6 100644 --- a/node_management/migration/src/rpc_config/migrate_path.py +++ b/node_management/migration/src/rpc_config/migrate_path.py @@ -1,13 +1,13 @@ from .from_v7_to_v8 import migrate_v7_to_v8 from .from_v8_to_v9 import migrate_v8_to_v9 - +from .from_v7_to_v9 import migrate_v7_to_v9 from common import migration RPC_CONFIG_MIGRATE_PATH = { "v7-v8": [migrate_v7_to_v8], "v8-v9": [migrate_v8_to_v9], - "v7-v9": [migrate_v7_to_v8, migrate_v8_to_v9], + "v7-v9": [migrate_v7_to_v9], } diff --git a/node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py b/node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py new file mode 100644 index 00000000..68891a42 --- /dev/null +++ b/node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py @@ -0,0 +1,140 @@ +RPC_CONFIG_V9_1_MAINNET_TEMPLATE = """ +# Version: v9.1 mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +# The access tokens used to authenticate public RPC requests to this RPC node. +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +# Number of certified blocks stored in memory as reference to pending blocks to be executed. +# Only this amount of certified blocks are stored in memory, for the rest memo is kept. +certified_block_cache_bucket_size = 50 +# Retry interval for the sync requests for which no response yet available. +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The websocket address of the attached validator. +consensus_rpc = "ws://:26000" + +[synchronization.ws.certificates] +# The path to the TLS certificate for the connection with the attached validator. +cert_path = "./configs/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./configs/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +# The below settings are the default values required for use in RPC nodes run by validator node operators. +# They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:26000" +description = "LocalNet" +mode = "Server" +""" diff --git a/node_management/migration/tests/config_v7.1.8.toml b/node_management/migration/tests/config_v7.1.8.toml index 8e73ca30..79649d19 100644 --- a/node_management/migration/tests/config_v7.1.8.toml +++ b/node_management/migration/tests/config_v7.1.8.toml @@ -31,43 +31,43 @@ chain_instance.genesis_timestamp_microseconds = 1732060800000000 # The below parameters are node-specific and may be configured as required by the operator. # The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:30000" +bind_addr = "1.1.1.1:26000" # If `true` then blocks will not be verified before execution. This value should be `false` # unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = false +block_provider_is_trusted = true # The path to the TLS certificate for the connection with the attached validator. -consensus_client_cert_path = "./configs/client_supra_certificate.pem" +consensus_client_cert_path = "./xyz/client_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -consensus_client_private_key_path = "./configs/client_supra_key.pem" +consensus_client_private_key_path = "./xyz/client_supra_key.pem" # The path to the TLS root certificate authority certificate. -consensus_root_ca_cert_path = "./configs/ca_certificate.pem" +consensus_root_ca_cert_path = "./xyz/ca_certificate.pem" # The websocket address of the attached validator. -consensus_rpc = "ws://:26000" +consensus_rpc = "ws://2.2.2.2:26000" # If true, all components will attempt to load their previous state from disk. Otherwise, # all components will start in their default state. Should always be `true` for testnet and # mainnet. resume = true # The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" +supra_committees_config = "./xyz/supra_committees.json" # The number of seconds to wait before retrying a block sync request. sync_retry_interval_in_secs = 1 # Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. [database_setup.dbs.archive.rocks_db] # The path at which the database should be created. -path = "./configs/rpc_archive" +path = "./xyz/rpc_archive" # Whether snapshots should be taken of the database. enable_snapshots = true # Parameters for the DKG database. [database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. -path = "./configs/rpc_ledger" +path = "./xyz/rpc_ledger" # Parameters for the blockchain database. [database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. -path = "./configs/rpc_store" +path = "./xyz/rpc_store" # Whether snapshots should be taken of the database. enable_snapshots = true @@ -78,7 +78,7 @@ depth = 2 # The interval between snapshots in seconds. interval_in_seconds = 1800 # The path at which the snapshots should be stored. -path = "./configs/snapshot" +path = "./xyz/snapshot" # The number of times to retry a snapshot in the event that it fails unexpectedly. retry_count = 3 # The interval in seconds to wait before retring a snapshot. diff --git a/node_management/migration/tests/config_v7_to_v9_expected.toml b/node_management/migration/tests/config_v7_to_v9_expected.toml new file mode 100644 index 00000000..74925eee --- /dev/null +++ b/node_management/migration/tests/config_v7_to_v9_expected.toml @@ -0,0 +1,138 @@ +# Version: v9.1 mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "1.1.1.1:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./xyz/supra_committees.json" +# The access tokens used to authenticate public RPC requests to this RPC node. +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +# Number of certified blocks stored in memory as reference to pending blocks to be executed. +# Only this amount of certified blocks are stored in memory, for the rest memo is kept. +certified_block_cache_bucket_size = 50 +# Retry interval for the sync requests for which no response yet available. +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The websocket address of the attached validator. +consensus_rpc = "ws://2.2.2.2:26000" + +[synchronization.ws.certificates] +# The path to the TLS certificate for the connection with the attached validator. +cert_path = "./xyz/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./xyz/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./xyz/ca_certificate.pem" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./xyz/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +# The below settings are the default values required for use in RPC nodes run by validator node operators. +# They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:26000" +description = "LocalNet" +mode = "Server" \ No newline at end of file diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py index 54eccc6c..779dcbe9 100644 --- a/node_management/migration/tests/test_rpc_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -4,19 +4,20 @@ import pytest from rpc_config.migrate_path import run_migration as rpc_run_migration -CONFIGS = [ - "config_v7.1.8.toml", - "config_v8.0.2.toml", - "config_v9.0.7.toml", -] +# CONFIGS = [ +# "config_v7.1.8.toml", +# "config_v8.0.2.toml", +# "config_v9.0.7.toml", +# "config_v9.1_mainnet.toml", +# ] @pytest.mark.parametrize( "from_file,to_file,migrate_path", [ - ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7-v8"), - ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8-v9"), - ("config_v7.1.8.toml", "config_v9.0.7.toml", "v7-v9"), + # ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7-v8"), + # ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8-v9"), + ("config_v7.1.8.toml", "config_v7_to_v9_expected.toml", "v7-v9"), ], ) def test_migration(tmp_path, from_file, to_file, migrate_path): From f79fe039c21514a953a3bda196cad93d5cb57f59 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Tue, 27 May 2025 10:29:42 +0200 Subject: [PATCH 22/44] migrate to smr settings v9 using templating --- .../migration/src/common/migration.py | 4 - .../migration/src/rpc_config/from_v7_to_v8.py | 139 ------------------ .../migration/src/rpc_config/from_v7_to_v9.py | 7 +- .../migration/src/rpc_config/from_v8_to_v9.py | 59 -------- .../migration/src/rpc_config/migrate_path.py | 4 - ... => rpc_config_v9_1_x_mainnet_template.py} | 22 +-- .../src/smr_settings/from_v7_to_v9.py | 78 +++++----- .../smr_settings_v9_1_x_mainnet_template.py | 128 ++++++++++++++++ .../rpc_config_v9.1.x_mainnet.toml} | 2 +- .../smr_settings_v9.1.x_mainnet.toml} | 2 +- .../tests/config_v7_to_v9_expected.toml | 2 +- .../migration/tests/config_v8.0.2.toml | 118 --------------- .../migration/tests/config_v9.0.7.toml | 120 --------------- ...ml => smr_settings_v7_to_v9_expected.toml} | 22 +-- .../migration/tests/test_rpc_migration.py | 9 -- .../migration/tests/test_smr_migration.py | 6 +- 16 files changed, 192 insertions(+), 530 deletions(-) delete mode 100644 node_management/migration/src/rpc_config/from_v7_to_v8.py delete mode 100644 node_management/migration/src/rpc_config/from_v8_to_v9.py rename node_management/migration/src/rpc_config/{v9_1_x_mainnet_template.py => rpc_config_v9_1_x_mainnet_template.py} (91%) create mode 100644 node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py rename node_management/migration/{tests/config_v9.1_mainnet.toml => templates/rpc_config_v9.1.x_mainnet.toml} (99%) rename node_management/migration/{tests/smr_settings_v9.1_mainnet.toml => templates/smr_settings_v9.1.x_mainnet.toml} (99%) delete mode 100644 node_management/migration/tests/config_v8.0.2.toml delete mode 100644 node_management/migration/tests/config_v9.0.7.toml rename node_management/migration/tests/{smr_settings_v9.0.x.toml => smr_settings_v7_to_v9_expected.toml} (93%) diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py index 01089fec..e4f542e9 100644 --- a/node_management/migration/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -62,7 +62,6 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): for fn in migrate_functions: print(f"Running migration function: {fn.__name__}") toml_data = fn(toml_data) - print("!!!!", toml_data, "!!!!") print(f"Backing up old config to {default_backup_path}") tomlkit.dump(original_toml_data, open(default_backup_path, "w")) @@ -85,9 +84,6 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): print( f""" Config migrated from {from_path} to {to_path}. - Please double check above for the diff between old and new config. Please ensure to use the new config file for target binary version. - NOTE: the comments may not be preserved in the new config file, so - you would need to fix the comments manually. """ ) diff --git a/node_management/migration/src/rpc_config/from_v7_to_v8.py b/node_management/migration/src/rpc_config/from_v7_to_v8.py deleted file mode 100644 index ece7f1ee..00000000 --- a/node_management/migration/src/rpc_config/from_v7_to_v8.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -This module provides migration functions to upgrade Supra's RPC configuration from version 7 (v7) to version 8 (v8). - -Migration Overview: -------------------- -The migration process involves restructuring and updating the configuration TOML data to match the v8 schema. The key changes performed by this migration are: - -1. Synchronization Parameters: - - Moves the following keys from the root level to a new `[synchronization.ws]` table: - - `consensus_rpc` - - `consensus_client_cert_path` - - `consensus_client_private_key_path` - - `consensus_root_ca_cert_path` - - Raises an error if a `[synchronization]` table already exists in the v7 config. - -2. Chain State Assembler Parameters: - - Moves `sync_retry_interval_in_secs` from the root level to a new `[chain_state_assembler]` table. - - Ensures `sync_retry_interval_in_secs` is set to `1` as recommended. - - Adds a new parameter `certified_block_cache_bucket_size = 50` to `[chain_state_assembler]`. - - Raises an error if a `[chain_state_assembler]` table already exists in the v7 config. - -3. Consensus Access Tokens: - - Adds a new root-level parameter: `consensus_access_tokens = []`. - -4. Block Provider Trust: - - Adds a new root-level parameter: `block_provider_is_trusted = True`. - -Warnings and Checks: --------------------- -- Issues warnings if expected keys are missing or have empty values. -- Uses utility functions to print migration steps with checkmarks for better traceability. - -""" - -import tomlkit -from common import utils - - -def __migrate_sync_ws_parameters(toml_data): - # v7 config should not have a [synchronization] table - if "synchronization" in toml_data: - raise SystemExit( - "Error: [synchronization] table should not exist in v7 config. Please check your migration path matching the version of your config file." - ) - - # Explicitly fetch and move these keys from the root - # level to the [synchronization.ws] table - sync_ws_keys = { - "consensus_rpc": None, - "consensus_client_cert_path": None, - "consensus_client_private_key_path": None, - "consensus_root_ca_cert_path": None, - } - for key in sync_ws_keys: - if key in toml_data: - value = toml_data.pop(key) - if not value: - print( - f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" - ) - utils.print_with_checkmark( - f"Moving `{key}` from root level to [synchronization.ws]" - ) - sync_ws_keys[key] = value - else: - print(f"Warning: `{key}` not found in root level") - # Create [synchronization.ws] table - sync_table = tomlkit.table() - for key, value in sync_ws_keys.items(): - sync_table[key] = value - toml_data["synchronization"] = tomlkit.table() - toml_data["synchronization"]["ws"] = sync_table - - -def __migrate_chain_state_assembler_parameters(toml_data): - # v7 config should not have a [chain_state_assembler] table - if "chain_state_assembler" in toml_data: - raise SystemExit( - "Error: [chain_state_assembler] table should not exist in v7 config." - ) - - # Explicitly fetch and move these keys from the root - # level to the [chain_state_assembler] table - chain_assembler_keys = { - "sync_retry_interval_in_secs": None, - } - for key in chain_assembler_keys: - if key in toml_data: - value = toml_data.pop(key) - if not value: - print( - f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" - ) - utils.print_with_checkmark( - f"Moving `{key}` from root level to [chain_state_assembler]" - ) - if value != 1: - print( - f"Warning: `{key} = {value}`, will be set to `1` as recommended by Supra." - ) - value = 1 - chain_assembler_keys[key] = value - else: - print(f"Warning: `{key}` not found in root level") - - # Create [chain_state_assembler] table - chain_assembler_table = tomlkit.table() - for key, value in chain_assembler_keys.items(): - chain_assembler_table[key] = value - new_parameters = {"certified_block_cache_bucket_size": 50} - for key, value in new_parameters.items(): - utils.print_with_checkmark( - f"Adding `{key} = {value}` to [chain_state_assembler]" - ) - chain_assembler_table[key] = value - toml_data["chain_state_assembler"] = chain_assembler_table - - -def __add_consensus_access_tokens(toml_data): - new_parameters = {"consensus_access_tokens": []} - for key, value in new_parameters.items(): - utils.print_with_checkmark(f"Adding `{key} = {value}` to root level") - toml_data[key] = value - - -def __update_block_provider_is_trusted(toml_data): - value = True - utils.print_with_checkmark( - f"Adding `block_provider_is_trusted = {value}` to root level" - ) - toml_data["block_provider_is_trusted"] = value - - -def migrate_v7_to_v8(toml_data): - # Migrate the config - __migrate_sync_ws_parameters(toml_data) - __migrate_chain_state_assembler_parameters(toml_data) - __add_consensus_access_tokens(toml_data) - __update_block_provider_is_trusted(toml_data) diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py index 411a34d4..9be888b0 100644 --- a/node_management/migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/migration/src/rpc_config/from_v7_to_v9.py @@ -28,7 +28,7 @@ """ import tomlkit -from .v9_1_x_mainnet_template import RPC_CONFIG_V9_1_MAINNET_TEMPLATE +from .rpc_config_v9_1_x_mainnet_template import RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE def __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data): @@ -68,7 +68,10 @@ def __migrate_snapshot_paths(v7_toml_data, v9_toml_data): def migrate_v7_to_v9(v7_toml_data): - v9_toml_data = tomlkit.parse(RPC_CONFIG_V9_1_MAINNET_TEMPLATE) + """ + Returns a new TOML data structure that is compatible with RPC config v9. + """ + v9_toml_data = tomlkit.parse(RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE) __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data) __migrate_chain_state_assembler_parameters(v7_toml_data, v9_toml_data) __migrate_bind_addr(v7_toml_data, v9_toml_data) diff --git a/node_management/migration/src/rpc_config/from_v8_to_v9.py b/node_management/migration/src/rpc_config/from_v8_to_v9.py deleted file mode 100644 index c108f749..00000000 --- a/node_management/migration/src/rpc_config/from_v8_to_v9.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -This module provides migration functions to upgrade a Supra RPC config file from version v8 to v9. - -Key changes handled by this migration: -- Moves and renames certificate-related keys from the [synchronization.ws] table to a new [synchronization.ws.certificates] table. - - "consensus_client_cert_path" is renamed to "cert_path" - - "consensus_client_private_key_path" is renamed to "private_key_path" - - "consensus_root_ca_cert_path" is renamed to "root_ca_cert_path" -- Ensures that the [synchronization] table exists and that the [synchronization.ws.certificates] table does not already exist in the v8 config. -- Warns if any expected keys are missing or have empty values, indicating potential issues with the input config. -- Provides utility output to indicate the migration steps being performed. - -""" - -import tomlkit -from common import utils - - -def __migrate_sync_ws_certificates(toml_data): - if "synchronization" not in toml_data: - raise SystemExit( - "Error: [synchronization] table should exist in v8 config. Please check your migration path matching the version of your config file." - ) - if "certificates" in toml_data["synchronization"]["ws"]: - raise SystemExit( - "Error: [synchronization.ws.certificates] table should not exist in v8 config. Please check your migration path matching the version of your config file." - ) - - # Explicitly move and rename these keys from the [synchronization.ws] table - # to the [synchronization.ws.certificates] table - sync_ws_certificates_keys_v8_to_v9 = { - "consensus_client_cert_path": "cert_path", - "consensus_client_private_key_path": "private_key_path", - "consensus_root_ca_cert_path": "root_ca_cert_path", - } - - sync_ws_certificates_v9 = {} - - for key, new_key in sync_ws_certificates_keys_v8_to_v9.items(): - if key in toml_data["synchronization"]["ws"]: - value = toml_data["synchronization"]["ws"].pop(key) - if not value: - print( - f"Warning: `{key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" - ) - utils.print_with_checkmark( - f"Moving `{key}` from [synchronization.ws] to {new_key} in [synchronization.ws.certificates] " - ) - sync_ws_certificates_v9[new_key] = value - else: - print(f"Warning: `{key}` not found in [synchronization.ws]") - - toml_data["synchronization"]["ws"]["certificates"] = tomlkit.table() - for key, value in sync_ws_certificates_v9.items(): - toml_data["synchronization"]["ws"]["certificates"][key] = value - - -def migrate_v8_to_v9(toml_data): - __migrate_sync_ws_certificates(toml_data) diff --git a/node_management/migration/src/rpc_config/migrate_path.py b/node_management/migration/src/rpc_config/migrate_path.py index dc783be6..00018344 100644 --- a/node_management/migration/src/rpc_config/migrate_path.py +++ b/node_management/migration/src/rpc_config/migrate_path.py @@ -1,12 +1,8 @@ -from .from_v7_to_v8 import migrate_v7_to_v8 -from .from_v8_to_v9 import migrate_v8_to_v9 from .from_v7_to_v9 import migrate_v7_to_v9 from common import migration RPC_CONFIG_MIGRATE_PATH = { - "v7-v8": [migrate_v7_to_v8], - "v8-v9": [migrate_v8_to_v9], "v7-v9": [migrate_v7_to_v9], } diff --git a/node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py similarity index 91% rename from node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py rename to node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py index 68891a42..157a1f77 100644 --- a/node_management/migration/src/rpc_config/v9_1_x_mainnet_template.py +++ b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py @@ -1,5 +1,5 @@ -RPC_CONFIG_V9_1_MAINNET_TEMPLATE = """ -# Version: v9.1 mainnet +RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE = """ +# Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### @@ -10,7 +10,7 @@ # The below parameters are node-specific and may be configured as required by the operator. # The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:26000" +bind_addr = ":26000" # If `true` then blocks will not be verified before execution. This value should be `false` # unless you also control the node from which this RPC node is receiving blocks. block_provider_is_trusted = true @@ -18,7 +18,7 @@ # all components will start in their default state. resume = true # The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" +supra_committees_config = "/supra_committees.json" # The access tokens used to authenticate public RPC requests to this RPC node. consensus_access_tokens = [] @@ -52,28 +52,28 @@ [synchronization.ws.certificates] # The path to the TLS certificate for the connection with the attached validator. -cert_path = "./configs/client_supra_certificate.pem" +cert_path = "/client_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -private_key_path = "./configs/client_supra_key.pem" +private_key_path = "/client_supra_key.pem" # The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" +root_ca_cert_path = "/ca_certificate.pem" # Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. [database_setup.dbs.archive.rocks_db] # The path at which the database should be created. -path = "./configs/rpc_archive" +path = "/rpc_archive" # Whether snapshots should be taken of the database. enable_snapshots = true # Parameters for the DKG database. [database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. -path = "./configs/rpc_ledger" +path = "/rpc_ledger" # Parameters for the blockchain database. [database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. -path = "./configs/rpc_store" +path = "/rpc_store" # Whether snapshots should be taken of the database. enable_snapshots = true @@ -84,7 +84,7 @@ # The interval between snapshots in seconds. interval_in_seconds = 1800 # The path at which the snapshots should be stored. -path = "./configs/snapshot" +path = "/configs/snapshot" # The number of times to retry a snapshot in the event that it fails unexpectedly. retry_count = 3 # The interval in seconds to wait before retrying a snapshot. diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index 98edd802..8e57a189 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -4,60 +4,48 @@ This module provides functions to transform configuration files in TOML format to be compatible with v9. The migration includes the following changes: -1. Removal of the `[move_vm]` table from the configuration. -2. Migration of WebSocket server certificate paths: - - Moves `root_ca_cert_path`, `server_cert_path`, and `server_private_key_path` from the `[node]` table. - - Renames `server_cert_path` to `cert_path` and `server_private_key_path` to `private_key_path`. - - Creates a new `[node.ws_server.certificates]` table to store these certificate paths. +1. Migration of WebSocket server certificate paths: + - Copy below keys in v7 [node] table to the `[node.ws_server.certificates]` table in v9: + - from `root_ca_cert_path` to `root_ca_cert_path` + - from `server_cert_path` to `cert_path` + - from `server_private_key_path` to `private_key_path` - Ensures that the `[node.ws_server]` table does not exist prior to migration, enforcing correct migration order. - +2. Copy value from 7 to v9 template in same table: + - [node].rpc_access_port + - [node.database_setup.dbs.chain_store.rocks_db].path + - [node.database_setup.dbs.ledger.rocks_db].path """ import tomlkit from common import utils +from .smr_settings_v9_1_x_mainnet_template import SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE - -def __migrate_move_vm_parameters(toml_data): - utils.print_with_checkmark("Removing [move_vm] table") - toml_data.pop("move_vm") - - -def __migrate_node_ws_certificates(toml_data): - if "ws_server" in toml_data["node"]: +def __migrate_ws_certificates(v7_toml_data, v9_toml_data): + if "ws_server" in v7_toml_data["node"]: raise SystemExit( - "Error: [node.ws_server] table should not exist in before v9 config. Please check your migration path matching the version of your config file." + "Error: [node.ws_server] table should not exist in v7 config. Please check your migration path matching the version of your config file." ) - # Explicitly move and rename these keys from the [node] table - # to the [node.ws_server.certificates] table - node_ws_certificates_keys_v7_to_v9 = { - "root_ca_cert_path": "root_ca_cert_path", - "server_cert_path": "cert_path", - "server_private_key_path": "private_key_path", - } - ws_certificates_v9 = {} - - for old_key, new_key in node_ws_certificates_keys_v7_to_v9.items(): - if old_key in toml_data["node"]: - value = toml_data["node"].pop(old_key) - if not value: - print( - f"Warning: `{old_key}` does not exist or its value is empty. Your config must be invalid and you should check it manually after migration" - ) - utils.print_with_checkmark( - f"Moving `{old_key}` from [node] to {new_key} in [node.ws_server.certificates]" - ) - ws_certificates_v9[new_key] = value - else: - print(f"Warning: `{old_key}` not found in [node]") - - # Create [node.ws_server.certificates] table - toml_data["node"]["ws_server"] = tomlkit.table() - toml_data["node"]["ws_server"]["certificates"] = tomlkit.table() - for key, value in ws_certificates_v9.items(): - toml_data["node"]["ws_server"]["certificates"][key] = value + v7_node_data = v7_toml_data["node"] + + v9_toml_data["node"]["ws_server"]["certificates"]["root_ca_cert_path"] = v7_node_data.get("root_ca_cert_path") + v9_toml_data["node"]["ws_server"]["certificates"]["cert_path"] = v7_node_data.get("server_cert_path") + v9_toml_data["node"]["ws_server"]["certificates"]["private_key_path"] = v7_node_data.get("server_private_key_path") + + +def __migrate_rpc_access_port(v7_toml_data, v9_toml_data): + v9_toml_data["node"]["rpc_access_port"] = v7_toml_data["node"].get("rpc_access_port") +def __migrate_database_paths(v7_toml_data, v9_toml_data): + v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"].get("path") + v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"].get("path") def migrate_v7_to_v9(toml_data): - __migrate_move_vm_parameters(toml_data) - __migrate_node_ws_certificates(toml_data) + """ + Returns a new TOML data structure that is compatible with SMR settings v9. + """ + v9_toml_data = tomlkit.parse(SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE) + __migrate_ws_certificates(toml_data, v9_toml_data) + __migrate_rpc_access_port(toml_data, v9_toml_data) + __migrate_database_paths(toml_data, v9_toml_data) + return v9_toml_data \ No newline at end of file diff --git a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py new file mode 100644 index 00000000..7fdf2a2b --- /dev/null +++ b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py @@ -0,0 +1,128 @@ +SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE = """ +# Version: v9.1.x mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 4 hours. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 2 hours. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = "" + +[node.ws_server.certificates] +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "/ca_certificate.pem" +# The path to the TLS certificate for this node. +cert_path = "/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 + +""" \ No newline at end of file diff --git a/node_management/migration/tests/config_v9.1_mainnet.toml b/node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml similarity index 99% rename from node_management/migration/tests/config_v9.1_mainnet.toml rename to node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml index 45167d5e..839a410f 100644 --- a/node_management/migration/tests/config_v9.1_mainnet.toml +++ b/node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml @@ -1,4 +1,4 @@ -# Version: v9.1 mainnet +# Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### diff --git a/node_management/migration/tests/smr_settings_v9.1_mainnet.toml b/node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml similarity index 99% rename from node_management/migration/tests/smr_settings_v9.1_mainnet.toml rename to node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml index 6cdd891f..7caa4784 100644 --- a/node_management/migration/tests/smr_settings_v9.1_mainnet.toml +++ b/node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml @@ -1,4 +1,4 @@ -# Version: v9.1 mainnet +# Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### diff --git a/node_management/migration/tests/config_v7_to_v9_expected.toml b/node_management/migration/tests/config_v7_to_v9_expected.toml index 74925eee..fdca0dbd 100644 --- a/node_management/migration/tests/config_v7_to_v9_expected.toml +++ b/node_management/migration/tests/config_v7_to_v9_expected.toml @@ -1,4 +1,4 @@ -# Version: v9.1 mainnet +# Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### diff --git a/node_management/migration/tests/config_v8.0.2.toml b/node_management/migration/tests/config_v8.0.2.toml deleted file mode 100644 index 2331537b..00000000 --- a/node_management/migration/tests/config_v8.0.2.toml +++ /dev/null @@ -1,118 +0,0 @@ -# Version: v8.0.2 -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:26000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = true -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" -consensus_access_tokens = [] - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -[chain_instance] -chain_id = 6 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = true -# Tuesday, September 17, 2024 12:00:00.000 PM (UTC) -genesis_timestamp_microseconds = 1726574400000000 - - -######################################### NODE PARAMETERS ######################################### -[chain_state_assembler] -certified_block_cache_bucket_size = 50 -sync_retry_interval_in_secs = 1 - -[synchronization.ws] -# The path to the TLS certificate for the connection with the attached validator. -consensus_client_cert_path = "./configs/client_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -consensus_client_private_key_path = "./configs/client_supra_key.pem" -# The path to the TLS root certificate authority certificate. -consensus_root_ca_cert_path = "./configs/ca_certificate.pem" -# The websocket address of the attached validator. -consensus_rpc = "ws://:26000" - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_archive" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_ledger" - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_store" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. The below settings are the default values required for use in RPC nodes run by validator node operators. They are optional for non-validators. -[[allowed_origin]] -url = "https://rpc-testnet.supra.com" -description = "RPC For Supra Scan and Faucet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] - -url = "https://rpc-testnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:27000" -description = "LocalNet" -mode = "Server" \ No newline at end of file diff --git a/node_management/migration/tests/config_v9.0.7.toml b/node_management/migration/tests/config_v9.0.7.toml deleted file mode 100644 index 9718ec34..00000000 --- a/node_management/migration/tests/config_v9.0.7.toml +++ /dev/null @@ -1,120 +0,0 @@ -# Version: v9.0.7 -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:26000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = true -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" -consensus_access_tokens = [] - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -[chain_instance] -chain_id = 6 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = true -# Tuesday, September 17, 2024 12:00:00.000 PM (UTC) -genesis_timestamp_microseconds = 1726574400000000 - - -######################################### NODE PARAMETERS ######################################### -[chain_state_assembler] -certified_block_cache_bucket_size = 50 -sync_retry_interval_in_secs = 1 - -[synchronization.ws] -# The websocket address of the attached validator. -consensus_rpc = "ws://:26000" - -[synchronization.ws.certificates] -# The path to the TLS certificate for the connection with the attached validator. -cert_path = "./configs/client_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "./configs/client_supra_key.pem" -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_archive" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_ledger" - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_store" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. The below settings are the default values required for use in RPC nodes run by validator node operators. They are optional for non-validators. -[[allowed_origin]] -url = "https://rpc-testnet.supra.com" -description = "RPC For Supra Scan and Faucet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] - -url = "https://rpc-testnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-testnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:27000" -description = "LocalNet" -mode = "Server" diff --git a/node_management/migration/tests/smr_settings_v9.0.x.toml b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml similarity index 93% rename from node_management/migration/tests/smr_settings_v9.0.x.toml rename to node_management/migration/tests/smr_settings_v7_to_v9_expected.toml index 09390f16..0288d8d7 100644 --- a/node_management/migration/tests/smr_settings_v9.0.x.toml +++ b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml @@ -1,4 +1,4 @@ -# Version: v9.0.x +# Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### @@ -9,7 +9,7 @@ # Core protocol parameters. [instance] # A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 6 +chain_id = 8 # The length of an epoch in seconds. epoch_duration_secs = 7200 # The number of seconds that stake locked in a Stake Pool will automatically be locked up for when @@ -23,9 +23,9 @@ recurring_lockup_duration_secs = 14400 # 2 hours. voting_duration_secs = 7200 # Determines whether the network will start with a faucet, amongst other things. -is_testnet = true -# Tuesday, September 17, 2024 12:00:00 PM -genesis_timestamp_microseconds = 1726574400000000 +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 # Parameters related to the mempool. [mempool] @@ -61,7 +61,7 @@ halt_block_production_when_no_txs = false leader_elector = "FairSuccession" # The delay after which the block proposer will create a new block despite not having any # payload items to propose. Denominated in ms. -max_block_delay_ms = 2500 +max_block_delay_ms = 1250 # The maximum number of batch availability certificates that may be included in a single # consensus block. max_payload_items_per_block = 50 @@ -69,13 +69,13 @@ max_payload_items_per_block = 50 # Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount # of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can # become. This trade-off must be balanced. -message_recency_bound_rounds = 20 +message_recency_bound_rounds = 1000 # The delay after which the node will try to repeat sync requests for missing blocks. # Denominated in ms. Should be the same for all nodes. sync_retry_delay_ms = 1000 # The delay after which the node will send a Timeout message for its current Moonshot round, # measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 5000 +timeout_delay_ms = 3500 ######################################### NODE PARAMETERS ######################################### @@ -100,11 +100,11 @@ rpc_access_port = 26000 [node.ws_server.certificates] # The path to the TLS root certificate authority certificate. -root_ca_cert_path = "configs/ca_certificate.pem" +root_ca_cert_path = "./configs/ca_certificate.pem" # The path to the TLS certificate for this node. -cert_path = "configs/server_supra_certificate.pem" +cert_path = "./configs/server_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -private_key_path = "configs/server_supra_key.pem" +private_key_path = "./configs/server_supra_key.pem" # Parameters for the blockchain database. [node.database_setup.dbs.chain_store.rocks_db] diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py index 779dcbe9..20a97df5 100644 --- a/node_management/migration/tests/test_rpc_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -36,15 +36,6 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): with open(to_path, "r") as f: expected = tomlkit.parse(f.read()) - # if "allowed_origin" in migrated: - # del migrated["allowed_origin"] - # if "allowed_origin" in expected: - # del expected["allowed_origin"] - - # if "chain_instance" in migrated: - # del migrated["chain_instance"] - # if "chain_instance" in expected: - # del expected["chain_instance"] # Compare TOML dicts assert migrated == expected, ( f"Migration {migrate_path} failed: {from_file} -> {to_file}" diff --git a/node_management/migration/tests/test_smr_migration.py b/node_management/migration/tests/test_smr_migration.py index b402d285..889f315f 100644 --- a/node_management/migration/tests/test_smr_migration.py +++ b/node_management/migration/tests/test_smr_migration.py @@ -4,16 +4,12 @@ import pytest from smr_settings.migrate_path import run_migration as smr_run_migration -CONFIGS = [ - "smr_settings_v7.1.x.toml", - "smr_settings_v9.0.x.toml", -] @pytest.mark.parametrize( "from_file,to_file,migrate_path", [ - ("smr_settings_v7.1.x.toml", "smr_settings_v9.0.x.toml", "v7-v9"), + ("smr_settings_v7.1.x.toml", "smr_settings_v7_to_v9_expected.toml", "v7-v9"), ], ) def test_migration(tmp_path, from_file, to_file, migrate_path): From b20842fab74a69dff92fa4ff400515637a1b25da Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Tue, 27 May 2025 11:15:33 +0200 Subject: [PATCH 23/44] update readme --- node_management/migration/README.md | 125 ++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 37 deletions(-) diff --git a/node_management/migration/README.md b/node_management/migration/README.md index c267fb0e..3f5dbb6b 100644 --- a/node_management/migration/README.md +++ b/node_management/migration/README.md @@ -1,6 +1,7 @@ ### Usage Example + alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node @@ -8,16 +9,26 @@ alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra" alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra" +1. Install the `migrate-config` tool + ```sh pip install . +``` +2. Migrate rpc + +```sh # Migrate rpc config from v7 to v9 migrate-config rpc -p v7-v9 -f config.toml -t config.toml # Migrate db from v7 to v8 rpc-v8 migrate-db config.toml # Migrate db from v8 to v9 rpc-v9 migrate-db config.toml +``` +3. Migrate smr/validator + +```sh # Migrate cli profile from v7 to v8 supra-v8 migrate --network localnet cp validator_identity.pem node_identity.pem @@ -30,53 +41,93 @@ migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml supra-v9 data migrate -p smr_settings.toml ``` -#### v9 template -https://testnet-snapshot.supra.com/configs/config_v9.0.7.toml -https://testnet-snapshot.supra.com/configs/smr_settings.toml -#### v8 template -https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml +### Migrate from v7 to v9 example flow + +#### RPC + +1. Config migration + +$ migrate-config rpc -p v7-v9 -f config.toml -t config.toml + + +2. DB migration + +**You need to have both v8 and v9 `rpc_node` binary to run the db migration.** + +$ rpc-v8 migrate-db config.toml + + [================================================================================================================================================================================================] 100/100 + MigrationReport { drop_cf: ["tx_block_info"], migrate_kv: {"tx_block_info__txn_hash_to_block_hash": 8537} } + +$ rpc-v9 migrate-db config.toml + + Counting the number of entries in certified_block... + Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 243/243 00:00:00 + Counting the number of entries in uncommitted_block... + Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 244/244 00:00:00 + Cleaning up uncommitted_block: [00:00:00] ████████████████████ 0/0 00:00:00 + Counting the number of entries in certified_block... + Counting the number of entries in certified_block_dehydrated... + Counting the number of entries in uncommitted_block... + Counting the number of entries in qc... + Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 244/244 00:00:00 + Counting the number of entries to remove from prune_index... + Cleaning up prune index: [00:00:00] ████████████████████ 244/244 00:00:00 + Counting the number of entries in block_to_transaction... + Migrating block_to_transaction: [00:00:00] ████████████████████ 8537/8537 00:00:00 + dropped: + - certified_block + - block_to_tx + migrated: + block_to_tx -> block_to_tx_ordered: Migrated 8537 records, up to 239 block height + certified_block -> certified_block_dehydrated: Migrated 244 records, up to 244 block height + databases_checked: + - chain_store + - archive + +#### SMR + +1. Config migration + +$ migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml -#### v7 template -https://mainnet-data.supra.com/configs/config.toml +2. Profile/Identity migration +**You need both v8 and v9 supra binary to run the profile migratino.** +$ supra-v8 migrate --network mainnet -### Migrate from v7 to v9 +$ cp validator_identity.pem node_identity.pem +$ supra-v9 profile migrate -#### rpc db migration +3. DB migration -(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v8 migrate-db config.toml - [======================================================================================================================================================================] 100/100MigrationReport { drop_cf: ["tx_block_info"], migrate_kv: {"tx_block_info__txn_hash_to_block_hash": 8537} } +**You need v9 supra binary to run the db migration.** +$ supra-v9 data migrate -p smr_settings.toml -(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v9 migrate-db config.toml -Counting the number of entries to remove from prune_index... -Cleaning up prune index: [00:00:00] ████████████████████ 0/0 00:00:00 Counting the number of entries in block_to_transaction... -Migrating block_to_transaction: [00:00:00] ████████████████████ 8537/8537 00:00:00 dropped: -- block_to_tx -migrated: - block_to_tx -> block_to_tx_ordered: Migrated 8537 records, up to 239 block height -databases_checked: -- chain_store -- archive + Counting the number of entries in certified_block... + Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 69/69 00:00:00 + Counting the number of entries in uncommitted_block... + Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 74/74 00:00:00 + Cleaning up uncommitted_block: [00:00:00] ████████████████████ 4/4 00:00:00 + Counting the number of entries in certified_block... + Counting the number of entries in certified_block_dehydrated... + Counting the number of entries in uncommitted_block... + Counting the number of entries in qc... + Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 70/70 00:00:00 + Counting the number of entries to remove from prune_index... + Cleaning up prune index: [00:00:00] ████████████████████ 70/70 00:00:00 + dropped: + - certified_block + migrated: + certified_block -> certified_block_dehydrated: Migrated 70 records, up to 244 block height + databases_checked: + - chain_store -#### smr db migration +### Test harness of migrate-config script -(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/node_0$ supra-v9 data migrate -p smr_settings.toml -Counting the number of entries in certified_block... -Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 69/69 00:00:00 Counting the number of entries in uncommitted_block... -Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 74/74 00:00:00 -Cleaning up uncommitted_block: [00:00:00] ████████████████████ 4/4 00:00:00 Counting the number of entries in certified_block... -Counting the number of entries in certified_block_dehydrated... -Counting the number of entries in uncommitted_block... -Counting the number of entries in qc... -Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 70/70 00:00:00 Counting the number of entries to remove from prune_index... -Cleaning up prune index: [00:00:00] ████████████████████ 70/70 00:00:00 dropped: -- certified_block -migrated: - certified_block -> certified_block_dehydrated: Migrated 70 records, up to 244 block height -databases_checked: -- chain_store +`PYTHONPATH=src pytest` \ No newline at end of file From fa0774a5cbe966ad8b216452eb2a78bd4950e141 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 28 May 2025 10:51:26 +0200 Subject: [PATCH 24/44] support dump template to toml file in cli --- node_management/manage_supra_nodes.sh | 10 +- node_management/migration/src/cli/main.py | 11 ++ .../rpc_config_v9_1_x_mainnet_template.py | 14 +- .../src/smr_settings/from_v7_to_v9.py | 8 +- .../smr_settings_v9_1_x_mainnet_template.py | 16 +- .../templates/rpc_config_v9.1.x_mainnet.toml | 138 ------------------ .../smr_settings_v9.1.x_mainnet.toml | 125 ---------------- .../tests/config_v7_to_v9_expected.toml | 4 +- .../tests/smr_settings_v7_to_v9_expected.toml | 2 + 9 files changed, 57 insertions(+), 271 deletions(-) delete mode 100644 node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml delete mode 100644 node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml diff --git a/node_management/manage_supra_nodes.sh b/node_management/manage_supra_nodes.sh index 2ea178ee..a770bd5f 100755 --- a/node_management/manage_supra_nodes.sh +++ b/node_management/manage_supra_nodes.sh @@ -222,8 +222,8 @@ function start_validator_docker_container() { --name "$CONTAINER_NAME" \ --user "${user_id}:${group_id}" \ -v "$HOST_SUPRA_HOME:/supra/configs" \ - -e "RUST_LOG=debug,sop2p=info,multistream_select=off,libp2p_swarm=off,yamux=off" \ -e "SUPRA_HOME=/supra/configs/" \ + -e "RUST_LOG=debug,sop2p=info,multistream_select=off,libp2p_swarm=off,yamux=off" \ -e "SUPRA_LOG_DIR=/supra/configs/supra_node_logs" \ -e "SUPRA_MAX_LOG_FILE_SIZE=500000000" \ -e "SUPRA_MAX_UNCOMPRESSED_LOGS=20" \ @@ -240,8 +240,8 @@ function start_rpc_docker_container() { --name "$CONTAINER_NAME" \ --user "${user_id}:${group_id}" \ -v "$HOST_SUPRA_HOME:/supra/configs" \ - -e "RUST_LOG=debug,sop2p=info,multistream_select=off,libp2p_swarm=off,yamux=off" \ -e "SUPRA_HOME=/supra/configs/" \ + -e "RUST_LOG=debug,sop2p=info,multistream_select=off,libp2p_swarm=off,yamux=off" \ -e "SUPRA_LOG_DIR=/supra/configs/rpc_node_logs" \ -e "SUPRA_MAX_LOG_FILE_SIZE=500000000" \ -e "SUPRA_MAX_UNCOMPRESSED_LOGS=20" \ @@ -615,6 +615,12 @@ EOF if [ "$NETWORK" == "mainnet" ]; then export AWS_ACCESS_KEY_ID="c64bed98a85ccd3197169bf7363ce94f" export AWS_SECRET_ACCESS_KEY="0b7f15dbeef4ebe871ee8ce483e3fc8bab97be0da6a362b2c4d80f020cae9df7" + + if is_validator; then + bucket_name="mainnet-validator-snapshot" + elif is_rpc; then + bucket_name="mainnet-snapshot" + fi elif [ "$NETWORK" == "testnet" ]; then export AWS_ACCESS_KEY_ID="229502d7eedd0007640348c057869c90" export AWS_SECRET_ACCESS_KEY="799d15f4fd23c57cd0f182f2ab85a19d885887d745e2391975bb27853e2db949" diff --git a/node_management/migration/src/cli/main.py b/node_management/migration/src/cli/main.py index 23c2015d..30da3d35 100644 --- a/node_management/migration/src/cli/main.py +++ b/node_management/migration/src/cli/main.py @@ -56,5 +56,16 @@ def smr(migrate_path, from_file, to_file): migrate_smr_config(migrate_path, from_file, to_file) +@main.command() +def dump_templates(): + """Dump built-in template to TOML files.""" + from rpc_config.rpc_config_v9_1_x_mainnet_template import dump_template as dump_rpc_template + from smr_settings.smr_settings_v9_1_x_mainnet_template import dump_template as dump_smr_template + + dump_rpc_template() + dump_smr_template() + print("Templates dumped successfully.") + + if __name__ == "__main__": main() diff --git a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py index 157a1f77..d5794ec8 100644 --- a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py +++ b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py @@ -1,3 +1,6 @@ +import tomlkit + + RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE = """ # Version: v9.1.x mainnet @@ -63,7 +66,7 @@ # The path at which the database should be created. path = "/rpc_archive" # Whether snapshots should be taken of the database. -enable_snapshots = true +enable_snapshots = false # Parameters for the DKG database. [database_setup.dbs.ledger.rocks_db] @@ -75,7 +78,7 @@ # The path at which the database should be created. path = "/rpc_store" # Whether snapshots should be taken of the database. -enable_snapshots = true +enable_snapshots = false # Parameters for the database snapshot service. [database_setup.snapshot_config] @@ -138,3 +141,10 @@ description = "LocalNet" mode = "Server" """ + + +def dump_template(): + v9_toml_data = tomlkit.parse(RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE) + tomlkit.dump( + v9_toml_data, open("rpc_config_v9_1_x_mainnet_template.toml", "w") + ) \ No newline at end of file diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index 8e57a189..535476b4 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -48,4 +48,10 @@ def migrate_v7_to_v9(toml_data): __migrate_ws_certificates(toml_data, v9_toml_data) __migrate_rpc_access_port(toml_data, v9_toml_data) __migrate_database_paths(toml_data, v9_toml_data) - return v9_toml_data \ No newline at end of file + return v9_toml_data + + +def dump_template(v9_toml_data): + tomlkit.dump( + v9_toml_data, open("../templates/smr_settings_v9_1_x_mainnet_template.toml", "w") + ) \ No newline at end of file diff --git a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py index 7fdf2a2b..bcf0109e 100644 --- a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py +++ b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py @@ -1,3 +1,6 @@ +import tomlkit + + SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE = """ # Version: v9.1.x mainnet @@ -114,6 +117,8 @@ # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false # Parameters for the DKG database. [node.database_setup.dbs.ledger.rocks_db] @@ -125,4 +130,13 @@ # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. epochs_to_retain = 84 -""" \ No newline at end of file +""" + + + + +def dump_template(): + v9_toml_data = tomlkit.parse(SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE) + tomlkit.dump( + v9_toml_data, open("smr_settings_v9_1_x_mainnet_template.toml", "w") + ) \ No newline at end of file diff --git a/node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml b/node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml deleted file mode 100644 index 839a410f..00000000 --- a/node_management/migration/templates/rpc_config_v9.1.x_mainnet.toml +++ /dev/null @@ -1,138 +0,0 @@ -# Version: v9.1.x mainnet - -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:26000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = true -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" -# The access tokens used to authenticate public RPC requests to this RPC node. -consensus_access_tokens = [] - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -[chain_instance] -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - - -######################################### NODE PARAMETERS ######################################### -[chain_state_assembler] -# Number of certified blocks stored in memory as reference to pending blocks to be executed. -# Only this amount of certified blocks are stored in memory, for the rest memo is kept. -certified_block_cache_bucket_size = 50 -# Retry interval for the sync requests for which no response yet available. -sync_retry_interval_in_secs = 1 - -[synchronization.ws] -# The websocket address of the attached validator. -consensus_rpc = "ws://:26000" - -[synchronization.ws.certificates] -# The path to the TLS certificate for the connection with the attached validator. -cert_path = "./configs/client_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "./configs/client_supra_key.pem" -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_archive" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_ledger" - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_store" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. -# The below settings are the default values required for use in RPC nodes run by validator node operators. -# They are optional for non-validators. -[[allowed_origin]] -url = "https://rpc-mainnet.supra.com" -description = "RPC For Supra" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-wallet-mainnet.supra.com" -description = "RPC For Supra Wallet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-suprascan-mainnet.supra.com" -description = "RPC For suprascan" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:26000" -description = "LocalNet" -mode = "Server" \ No newline at end of file diff --git a/node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml b/node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml deleted file mode 100644 index 7caa4784..00000000 --- a/node_management/migration/templates/smr_settings_v9.1.x_mainnet.toml +++ /dev/null @@ -1,125 +0,0 @@ -# Version: v9.1.x mainnet - -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. Paths are set relative -# to $SUPRA_HOME. - -# Core protocol parameters. -[instance] -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 4 hours. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 2 hours. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - -# Parameters related to the mempool. -[mempool] -# The maximum number of milliseconds that a node will wait before proposing a batch when it has -# at least one transaction to process. -max_batch_delay_ms = 500 -# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` -# then a batch will be proposed immediately. -max_batch_size_bytes = 5000000 -# The amount of time that a node will wait before repeating a sync request for a batch that it -# is missing. -sync_retry_delay_ms = 2000 -# The number of signers of the related batch certificate that a node should ask for a batch -# attempting to retry a sync request. -sync_retry_nodes = 3 - -# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. -[moonshot] -# The maximum number of milliseconds that the timestamp of a proposed block may be -# ahead of a node's local time when it attempts to vote for the block. Validators -# must wait until the timestamp of a certified block has passed before advancing to -# the next round and leaders must wait until the timestamp of the parent block has -# passed before proposing, so this limit prevents Byzantine leaders from forcing -# honest nodes to wait indefinitely by proposing blocks with timestamps that are -# arbitrarily far in the future. -block_recency_bound_ms = 500 -# Causes the node to stop producing blocks when there are no transactions to be -# processed. If all nodes set this value to `true` then the chain will not produce -# new blocks when there are no transactions to process, conserving disk space. -halt_block_production_when_no_txs = false -# The type of leader election function to use. This function generates a schedule that ensures -# that each node eventually succeeds every other. -leader_elector = "FairSuccession" -# The delay after which the block proposer will create a new block despite not having any -# payload items to propose. Denominated in ms. -max_block_delay_ms = 1250 -# The maximum number of batch availability certificates that may be included in a single -# consensus block. -max_payload_items_per_block = 50 -# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, -# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount -# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can -# become. This trade-off must be balanced. -message_recency_bound_rounds = 1000 -# The delay after which the node will try to repeat sync requests for missing blocks. -# Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 1000 -# The delay after which the node will send a Timeout message for its current Moonshot round, -# measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 3500 - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. Paths -# should be specified either as absolute paths or as relative to the `supra` binary. -# -# When running the node in Docker and working with a mounted directory, the paths must be specified -# with reference to the location of the mounter directory in the Docker filesystem rather than the -# host filesystem. The default settings provided below assume the default Docker configuration -# provided by the `manage_supra_nodes.sh` script. - -[node] -# The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 1 -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The port on which to listen for connections from the associated RPC node. Each validator -# may serve at most one RPC node. -rpc_access_port = 26000 - -[node.ws_server.certificates] -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "configs/ca_certificate.pem" -# The path to the TLS certificate for this node. -cert_path = "configs/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "configs/server_supra_key.pem" - -# Parameters for the blockchain database. -[node.database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/smr_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true - -# Parameters for the DKG database. -[node.database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/ledger_storage" - -# Parameters related to database pruning. -[node.database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 diff --git a/node_management/migration/tests/config_v7_to_v9_expected.toml b/node_management/migration/tests/config_v7_to_v9_expected.toml index fdca0dbd..64129e34 100644 --- a/node_management/migration/tests/config_v7_to_v9_expected.toml +++ b/node_management/migration/tests/config_v7_to_v9_expected.toml @@ -62,7 +62,7 @@ root_ca_cert_path = "./xyz/ca_certificate.pem" # The path at which the database should be created. path = "./xyz/rpc_archive" # Whether snapshots should be taken of the database. -enable_snapshots = true +enable_snapshots = false # Parameters for the DKG database. [database_setup.dbs.ledger.rocks_db] @@ -74,7 +74,7 @@ path = "./xyz/rpc_ledger" # The path at which the database should be created. path = "./xyz/rpc_store" # Whether snapshots should be taken of the database. -enable_snapshots = true +enable_snapshots = false # Parameters for the database snapshot service. [database_setup.snapshot_config] diff --git a/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml index 0288d8d7..c66b40bd 100644 --- a/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml +++ b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml @@ -113,6 +113,8 @@ path = "./configs/smr_storage" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false # Parameters for the DKG database. [node.database_setup.dbs.ledger.rocks_db] From 1007cb1ae7cf99d03a7ad017decf815aaaf3dc39 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 28 May 2025 14:11:47 +0200 Subject: [PATCH 25/44] wip: packaing toml file --- node_management/migration/pyproject.toml | 12 +++++++++- node_management/migration/src/cli/main.py | 11 --------- .../migration/src/common/migration.py | 23 +++++++++++-------- node_management/migration/src/common/utils.py | 2 ++ .../migration/src/rpc_config/from_v7_to_v9.py | 6 ++++- .../src/smr_settings/from_v7_to_v9.py | 6 ----- 6 files changed, 32 insertions(+), 28 deletions(-) diff --git a/node_management/migration/pyproject.toml b/node_management/migration/pyproject.toml index 6c791cb8..abc3a956 100644 --- a/node_management/migration/pyproject.toml +++ b/node_management/migration/pyproject.toml @@ -14,4 +14,14 @@ migrate-config = "cli.main:main" [build-system] requires = ["setuptools>=61.0", "wheel"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +rpc_config = ["rpc_config_v9_1_x_mainnet_template.toml"] +smr_settings = ["smr_settings_v9_1_x_mainnet_template.toml"] diff --git a/node_management/migration/src/cli/main.py b/node_management/migration/src/cli/main.py index 30da3d35..23c2015d 100644 --- a/node_management/migration/src/cli/main.py +++ b/node_management/migration/src/cli/main.py @@ -56,16 +56,5 @@ def smr(migrate_path, from_file, to_file): migrate_smr_config(migrate_path, from_file, to_file) -@main.command() -def dump_templates(): - """Dump built-in template to TOML files.""" - from rpc_config.rpc_config_v9_1_x_mainnet_template import dump_template as dump_rpc_template - from smr_settings.smr_settings_v9_1_x_mainnet_template import dump_template as dump_smr_template - - dump_rpc_template() - dump_smr_template() - print("Templates dumped successfully.") - - if __name__ == "__main__": main() diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py index e4f542e9..0c3ee49e 100644 --- a/node_management/migration/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -2,7 +2,7 @@ import tomlkit from copy import deepcopy from . import utils - +import shutil class MigrationPathSet: """ @@ -36,9 +36,9 @@ def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]): def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): migrate_functions = self.migrate_path.get_migration_functions(migrate_choice) - from_version, to_version = self.migrate_path.get_versions(migrate_choice) default_backup_path = f"{from_path}_{from_version}.bak" + if from_path == to_path: print( f"Warning: The source and destination paths are the same ({from_path})." @@ -52,24 +52,24 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): if confirm.lower() != "y": print("Aborted by user.") return + # Backup old config + print(f"Backing up old config to {default_backup_path}") + shutil.copyfile(from_path, default_backup_path) with open(from_path, "r") as f: - toml_data = tomlkit.parse(f.read()) + toml_data = tomlkit.load(f) - # Backup old config original_toml_data = deepcopy(toml_data) for fn in migrate_functions: print(f"Running migration function: {fn.__name__}") toml_data = fn(toml_data) - print(f"Backing up old config to {default_backup_path}") - tomlkit.dump(original_toml_data, open(default_backup_path, "w")) # Write new config print(f"Writing new config to {to_path}") with open(to_path, "w") as f: - f.write(tomlkit.dumps(toml_data)) + tomlkit.dump(toml_data, f) # Print the diff from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) @@ -83,7 +83,12 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): print( f""" - Config migrated from {from_path} to {to_path}. - Please ensure to use the new config file for target binary version. + ###################################################################### + # Config migrated from {from_path} to {to_path}. + # + # Please review the diff above for changes made during migration. + # + # Please ensure to use the new config file for target binary version. + ###################################################################### """ ) diff --git a/node_management/migration/src/common/utils.py b/node_management/migration/src/common/utils.py index 1eb20c2a..af45eb76 100644 --- a/node_management/migration/src/common/utils.py +++ b/node_management/migration/src/common/utils.py @@ -2,8 +2,10 @@ def unified_diff(from_str, to_str, fromfile, tofile): + print(f"|----------------- Begin diff {fromfile} vs {tofile} -----------------|") diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) __print_colored_diff(diff) + print(f"|----------------- End diff {fromfile} vs {tofile} -----------------|") def __print_colored_diff(diff): diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py index 9be888b0..e8674399 100644 --- a/node_management/migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/migration/src/rpc_config/from_v7_to_v9.py @@ -27,6 +27,7 @@ """ import tomlkit +import importlib.resources from .rpc_config_v9_1_x_mainnet_template import RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE @@ -71,7 +72,10 @@ def migrate_v7_to_v9(v7_toml_data): """ Returns a new TOML data structure that is compatible with RPC config v9. """ - v9_toml_data = tomlkit.parse(RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE) + + with importlib.resources.files(__package__).joinpath("rpc_config_v9_1_x_mainnet_template.toml").open("r") as f: + template = f.read() + v9_toml_data = tomlkit.parse(template) __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data) __migrate_chain_state_assembler_parameters(v7_toml_data, v9_toml_data) __migrate_bind_addr(v7_toml_data, v9_toml_data) diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index 535476b4..c4ce035c 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -49,9 +49,3 @@ def migrate_v7_to_v9(toml_data): __migrate_rpc_access_port(toml_data, v9_toml_data) __migrate_database_paths(toml_data, v9_toml_data) return v9_toml_data - - -def dump_template(v9_toml_data): - tomlkit.dump( - v9_toml_data, open("../templates/smr_settings_v9_1_x_mainnet_template.toml", "w") - ) \ No newline at end of file From 700e56dab46c92ba16a450f8ca9f8ecff81e1730 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 28 May 2025 17:46:03 +0200 Subject: [PATCH 26/44] package toml in pip, impl scan updates --- node_management/migration/src/cli/main.py | 22 ++++- .../migration/src/common/globals.py | 1 + .../migration/src/common/migration.py | 9 ++- node_management/migration/src/common/utils.py | 48 ++++++++++- .../migration/src/rpc_config/from_v7_to_v9.py | 81 +++++++++++++------ ...> rpc_config_v9_1_x_mainnet_template.toml} | 11 --- .../src/smr_settings/from_v7_to_v9.py | 7 +- ...smr_settings_v9_1_x_mainnet_template.toml} | 13 --- .../migration/tests/test_rpc_migration.py | 2 + 9 files changed, 135 insertions(+), 59 deletions(-) create mode 100644 node_management/migration/src/common/globals.py rename node_management/migration/src/rpc_config/{rpc_config_v9_1_x_mainnet_template.py => rpc_config_v9_1_x_mainnet_template.toml} (95%) rename node_management/migration/src/smr_settings/{smr_settings_v9_1_x_mainnet_template.py => smr_settings_v9_1_x_mainnet_template.toml} (96%) diff --git a/node_management/migration/src/cli/main.py b/node_management/migration/src/cli/main.py index 23c2015d..6f5eee73 100644 --- a/node_management/migration/src/cli/main.py +++ b/node_management/migration/src/cli/main.py @@ -3,7 +3,7 @@ from rpc_config.migrate_path import run_migration as migrate_rpc_config from smr_settings.migrate_path import SMR_SETTINGS_MIGRATE_PATH from smr_settings.migrate_path import run_migration as migrate_smr_config - +import common.globals @click.group() def main(): @@ -28,8 +28,16 @@ def main(): @click.option( "--to-file", "-t", required=True, type=click.Path(), help="Output config file" ) -def rpc(migrate_path, from_file, to_file): +@click.option( + "--assume-yes", + "-y", + is_flag=True, + default=False, + help="Assume yes for all prompts (default: False)", +) +def rpc(migrate_path, from_file, to_file, assume_yes): """Migrate RPC config.""" + common.globals.ASSUME_YES = assume_yes migrate_rpc_config(migrate_path, from_file, to_file) @@ -51,8 +59,16 @@ def rpc(migrate_path, from_file, to_file): @click.option( "--to-file", "-t", required=True, type=click.Path(), help="Output config file" ) -def smr(migrate_path, from_file, to_file): +@click.option( + "--assume-yes", + "-y", + is_flag=True, + default=False, + help="Assume yes for all prompts (default: False)", +) +def smr(migrate_path, from_file, to_file, assume_yes): """Migrate SMR config.""" + common.globals.ASSUME_YES = assume_yes migrate_smr_config(migrate_path, from_file, to_file) diff --git a/node_management/migration/src/common/globals.py b/node_management/migration/src/common/globals.py new file mode 100644 index 00000000..7f2baecd --- /dev/null +++ b/node_management/migration/src/common/globals.py @@ -0,0 +1 @@ +ASSUME_YES = False \ No newline at end of file diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py index 0c3ee49e..3c2f4090 100644 --- a/node_management/migration/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -3,7 +3,7 @@ from copy import deepcopy from . import utils import shutil - +from common.globals import ASSUME_YES class MigrationPathSet: """ Base class for migration paths. @@ -46,10 +46,11 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): print( f"A backup of your original config will be saved to: {default_backup_path}" ) - confirm = input( - "This will overwrite your original config file. Continue? [y/N]: " + confirm = utils.prompt_or_assume_yes( + "This will overwrite your original config file. Continue?", + ASSUME_YES ) - if confirm.lower() != "y": + if not confirm: print("Aborted by user.") return # Backup old config diff --git a/node_management/migration/src/common/utils.py b/node_management/migration/src/common/utils.py index af45eb76..9663094f 100644 --- a/node_management/migration/src/common/utils.py +++ b/node_management/migration/src/common/utils.py @@ -1,5 +1,5 @@ import difflib - +import tomlkit def unified_diff(from_str, to_str, fromfile, tofile): print(f"|----------------- Begin diff {fromfile} vs {tofile} -----------------|") @@ -26,3 +26,49 @@ def print_with_checkmark(message): Print a message with a checkmark. """ print(f"✓ {message}") + + + +def prompt_or_assume_yes(message, assume_yes=False) -> bool: + """ + Prompt the user for confirmation or assume 'yes' for all prompts + """ + if not assume_yes: + response = input(f"{message} [y/N]: ").strip().lower() + return response in ('y', 'yes') + else: + print(f"{message} (assuming yes)") + return True + +def truncate(val, max_len=50): + """ + Truncate a string representation of a value to a maximum length. + """ + s = str(val) + return s if len(s) <= max_len else s[:max_len - 3] + '...' + + +def scan_and_recommend_updates(original_data: tomlkit.items.Table, to_data: tomlkit.items.Table): + """ + Scan the original data and recommend updates to the new version's data. + """ + from .globals import ASSUME_YES + + + + for k, v in to_data.items(): + if not isinstance(v, tomlkit.items.AbstractTable): + if k in original_data: + if to_data[k] != original_data[k]: + use_recommended = prompt_or_assume_yes( + f"`{k}={truncate(original_data[k])}` is not recommended for new version.\n" + f"Do you want to apply the recommended config: `{k}={truncate(to_data[k])}`?", + ASSUME_YES, + ) + if use_recommended: + print_with_checkmark(f"Apply recommended config: `{k}={truncate(to_data[k])}`") + else: + print_with_checkmark(f"Keep original config: `{k}={truncate(original_data[k])}`") + to_data[k] = original_data[k] + else: + print_with_checkmark(f"`{k}` not found in original config, using new version's default value: {truncate(v)}") diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py index e8674399..f3aa2c4c 100644 --- a/node_management/migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/migration/src/rpc_config/from_v7_to_v9.py @@ -25,47 +25,79 @@ - [database_setup.dbs.chain_store.rocks_db].path - [[database_setup.snapshot_config]].path +4. recommended updates + - `sync_retry_interval_in_secs` + - `block_provider_is_trusted` + - `enable_snapshots` + - `enable_pruning` """ import tomlkit import importlib.resources +from common.globals import ASSUME_YES +from common.utils import prompt_or_assume_yes, print_with_checkmark, scan_and_recommend_updates -from .rpc_config_v9_1_x_mainnet_template import RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE +def __migrate_root_config(v7_toml_data, v9_toml_data): + v9_toml_data["supra_committees_config"] = v7_toml_data["supra_committees_config"] + + v9_toml_data["bind_addr"] = v7_toml_data["bind_addr"] + + print("\nScanning root level configuration ...") + scan_and_recommend_updates(v7_toml_data, v9_toml_data) -def __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data): +def __migrate_sync_ws_config(v7_toml_data, v9_toml_data): if "synchronization" in v7_toml_data: raise SystemExit( "Error: [synchronization] table should not exist in v7 config. Please check your migration path matching the version of your config file." ) - v9_toml_data["synchronization"]["ws"]["consensus_rpc"] = v7_toml_data["consensus_rpc"] + v9_sync_ws_config = v9_toml_data["synchronization"]["ws"] + v9_sync_ws_config["consensus_rpc"] = v7_toml_data["consensus_rpc"] - v9_toml_data["synchronization"]["ws"]["certificates"]["cert_path"] = v7_toml_data["consensus_client_cert_path"] - v9_toml_data["synchronization"]["ws"]["certificates"]["private_key_path"] = v7_toml_data["consensus_client_private_key_path"] - v9_toml_data["synchronization"]["ws"]["certificates"]["root_ca_cert_path"] = v7_toml_data["consensus_root_ca_cert_path"] + v9_sync_ws_certificates = v9_sync_ws_config["certificates"] + v9_sync_ws_certificates["cert_path"] = v7_toml_data["consensus_client_cert_path"] + v9_sync_ws_certificates["private_key_path"] = v7_toml_data["consensus_client_private_key_path"] + v9_sync_ws_certificates["root_ca_cert_path"] = v7_toml_data["consensus_root_ca_cert_path"] -def __migrate_chain_state_assembler_parameters(v7_toml_data, v9_toml_data): +def __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data): if "chain_state_assembler" in v7_toml_data: raise SystemExit( "Error: [chain_state_assembler] table should not exist in v7 config." ) - # No changes needed for chain state assembler parameters in v9 - pass -def __migrate_bind_addr(v7_toml_data, v9_toml_data): - v9_toml_data["bind_addr"] = v7_toml_data["bind_addr"] + print("\nScanning chain state assembler configuration ...") + scan_and_recommend_updates(v7_toml_data, v9_toml_data["chain_state_assembler"]) -def __migrate_supra_committees_config(v7_toml_data, v9_toml_data): - v9_toml_data["supra_committees_config"] = v7_toml_data["supra_committees_config"] +def __migrate_db_archive_config(v7_toml_data, v9_toml_data): + v9_db_archive_config = v9_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"] + v7_db_archive_config = v7_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"] + + v9_db_archive_config["path"] = v7_db_archive_config["path"] -def __migrate_database_paths(v7_toml_data, v9_toml_data): - v9_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"]["path"] + print("\nScanning archive configuration ...") + scan_and_recommend_updates(v7_db_archive_config, v9_db_archive_config) + +def __migrate_db_chain_store_config(v7_toml_data, v9_toml_data): + v9_db_chain_store_config = v9_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v7_db_chain_store_config = v7_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"] + + v9_db_chain_store_config["path"] = v7_db_chain_store_config["path"] + + print("\nScanning chain store configuration ...") + scan_and_recommend_updates(v7_db_chain_store_config, v9_db_chain_store_config) + +def __migrate_db_ledger_config(v7_toml_data, v9_toml_data): v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] - v9_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] -def __migrate_snapshot_paths(v7_toml_data, v9_toml_data): - v9_toml_data["database_setup"]["snapshot_config"]["path"] = v7_toml_data["database_setup"]["snapshot_config"]["path"] +def __migrate_snapshot_config(v7_toml_data, v9_toml_data): + v9_snapshot_config = v9_toml_data["database_setup"]["snapshot_config"] + v7_snapshot_config = v7_toml_data["database_setup"]["snapshot_config"] + + v9_snapshot_config["path"] = v7_snapshot_config["path"] + + print("\nScanning snapshot configuration ...") + scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) def migrate_v7_to_v9(v7_toml_data): @@ -76,10 +108,11 @@ def migrate_v7_to_v9(v7_toml_data): with importlib.resources.files(__package__).joinpath("rpc_config_v9_1_x_mainnet_template.toml").open("r") as f: template = f.read() v9_toml_data = tomlkit.parse(template) - __migrate_sync_ws_parameters(v7_toml_data, v9_toml_data) - __migrate_chain_state_assembler_parameters(v7_toml_data, v9_toml_data) - __migrate_bind_addr(v7_toml_data, v9_toml_data) - __migrate_supra_committees_config(v7_toml_data, v9_toml_data) - __migrate_database_paths(v7_toml_data, v9_toml_data) - __migrate_snapshot_paths(v7_toml_data, v9_toml_data) + __migrate_root_config(v7_toml_data, v9_toml_data) + __migrate_sync_ws_config(v7_toml_data, v9_toml_data) + __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data) + __migrate_db_archive_config(v7_toml_data, v9_toml_data) + __migrate_db_ledger_config(v7_toml_data, v9_toml_data) + __migrate_db_chain_store_config(v7_toml_data, v9_toml_data) + __migrate_snapshot_config(v7_toml_data, v9_toml_data) return v9_toml_data diff --git a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml similarity index 95% rename from node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py rename to node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml index d5794ec8..7c91be95 100644 --- a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.py +++ b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -1,7 +1,4 @@ -import tomlkit - -RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE = """ # Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### @@ -140,11 +137,3 @@ url = "http://localhost:26000" description = "LocalNet" mode = "Server" -""" - - -def dump_template(): - v9_toml_data = tomlkit.parse(RPC_CONFIG_V9_1_X_MAINNET_TEMPLATE) - tomlkit.dump( - v9_toml_data, open("rpc_config_v9_1_x_mainnet_template.toml", "w") - ) \ No newline at end of file diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index c4ce035c..9cad1881 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -17,8 +17,7 @@ """ import tomlkit -from common import utils -from .smr_settings_v9_1_x_mainnet_template import SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE +import importlib.resources def __migrate_ws_certificates(v7_toml_data, v9_toml_data): if "ws_server" in v7_toml_data["node"]: @@ -44,7 +43,9 @@ def migrate_v7_to_v9(toml_data): """ Returns a new TOML data structure that is compatible with SMR settings v9. """ - v9_toml_data = tomlkit.parse(SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE) + with importlib.resources.files(__package__).joinpath("smr_settings_v9_1_x_mainnet_template.toml").open("r") as f: + template = f.read() + v9_toml_data = tomlkit.parse(template) __migrate_ws_certificates(toml_data, v9_toml_data) __migrate_rpc_access_port(toml_data, v9_toml_data) __migrate_database_paths(toml_data, v9_toml_data) diff --git a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml similarity index 96% rename from node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py rename to node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml index bcf0109e..172ce8cb 100644 --- a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.py +++ b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml @@ -1,7 +1,4 @@ -import tomlkit - -SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE = """ # Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### @@ -130,13 +127,3 @@ # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. epochs_to_retain = 84 -""" - - - - -def dump_template(): - v9_toml_data = tomlkit.parse(SMR_SETTINGS_V9_1_X_MAINNET_TEMPLATE) - tomlkit.dump( - v9_toml_data, open("smr_settings_v9_1_x_mainnet_template.toml", "w") - ) \ No newline at end of file diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py index 20a97df5..cdeff510 100644 --- a/node_management/migration/tests/test_rpc_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -29,6 +29,8 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): tmp_to = tmp_path / to_file shutil.copy(from_path, tmp_from) # Run migration + import common.globals + common.globals.ASSUME_YES = True rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files with open(tmp_to, "r") as f: From 5161f9e712f3b22b2bd6256496ba62c64e20b21a Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 28 May 2025 17:50:20 +0200 Subject: [PATCH 27/44] fmt --- node_management/migration/src/cli/main.py | 1 + .../migration/src/common/globals.py | 2 +- .../migration/src/common/migration.py | 6 +-- node_management/migration/src/common/utils.py | 31 +++++++++------ .../migration/src/rpc_config/from_v7_to_v9.py | 38 ++++++++++++++----- .../src/smr_settings/from_v7_to_v9.py | 37 ++++++++++++++---- .../migration/tests/test_rpc_migration.py | 1 + .../migration/tests/test_smr_migration.py | 1 - 8 files changed, 83 insertions(+), 34 deletions(-) diff --git a/node_management/migration/src/cli/main.py b/node_management/migration/src/cli/main.py index 6f5eee73..29efbfe5 100644 --- a/node_management/migration/src/cli/main.py +++ b/node_management/migration/src/cli/main.py @@ -5,6 +5,7 @@ from smr_settings.migrate_path import run_migration as migrate_smr_config import common.globals + @click.group() def main(): """Migration CLI for Supra configs.""" diff --git a/node_management/migration/src/common/globals.py b/node_management/migration/src/common/globals.py index 7f2baecd..9b8bd68b 100644 --- a/node_management/migration/src/common/globals.py +++ b/node_management/migration/src/common/globals.py @@ -1 +1 @@ -ASSUME_YES = False \ No newline at end of file +ASSUME_YES = False diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py index 3c2f4090..18efc502 100644 --- a/node_management/migration/src/common/migration.py +++ b/node_management/migration/src/common/migration.py @@ -4,6 +4,8 @@ from . import utils import shutil from common.globals import ASSUME_YES + + class MigrationPathSet: """ Base class for migration paths. @@ -47,8 +49,7 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): f"A backup of your original config will be saved to: {default_backup_path}" ) confirm = utils.prompt_or_assume_yes( - "This will overwrite your original config file. Continue?", - ASSUME_YES + "This will overwrite your original config file. Continue?", ASSUME_YES ) if not confirm: print("Aborted by user.") @@ -66,7 +67,6 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): print(f"Running migration function: {fn.__name__}") toml_data = fn(toml_data) - # Write new config print(f"Writing new config to {to_path}") with open(to_path, "w") as f: diff --git a/node_management/migration/src/common/utils.py b/node_management/migration/src/common/utils.py index 9663094f..b61e3ef4 100644 --- a/node_management/migration/src/common/utils.py +++ b/node_management/migration/src/common/utils.py @@ -1,6 +1,7 @@ import difflib import tomlkit + def unified_diff(from_str, to_str, fromfile, tofile): print(f"|----------------- Begin diff {fromfile} vs {tofile} -----------------|") diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) @@ -28,47 +29,53 @@ def print_with_checkmark(message): print(f"✓ {message}") - def prompt_or_assume_yes(message, assume_yes=False) -> bool: """ Prompt the user for confirmation or assume 'yes' for all prompts """ if not assume_yes: response = input(f"{message} [y/N]: ").strip().lower() - return response in ('y', 'yes') + return response in ("y", "yes") else: print(f"{message} (assuming yes)") return True - + + def truncate(val, max_len=50): """ Truncate a string representation of a value to a maximum length. """ s = str(val) - return s if len(s) <= max_len else s[:max_len - 3] + '...' + return s if len(s) <= max_len else s[: max_len - 3] + "..." -def scan_and_recommend_updates(original_data: tomlkit.items.Table, to_data: tomlkit.items.Table): +def scan_and_recommend_updates( + original_data: tomlkit.items.Table, to_data: tomlkit.items.Table +): """ Scan the original data and recommend updates to the new version's data. """ from .globals import ASSUME_YES - - for k, v in to_data.items(): if not isinstance(v, tomlkit.items.AbstractTable): if k in original_data: if to_data[k] != original_data[k]: use_recommended = prompt_or_assume_yes( - f"`{k}={truncate(original_data[k])}` is not recommended for new version.\n" - f"Do you want to apply the recommended config: `{k}={truncate(to_data[k])}`?", + f"`{k} = {truncate(original_data[k])}` is not recommended for new version.\n" + f"Do you want to apply the recommended config: `{k} = {truncate(to_data[k])}`?", ASSUME_YES, ) if use_recommended: - print_with_checkmark(f"Apply recommended config: `{k}={truncate(to_data[k])}`") + print_with_checkmark( + f"Apply recommended config: `{k} = {truncate(to_data[k])}`" + ) else: - print_with_checkmark(f"Keep original config: `{k}={truncate(original_data[k])}`") + print_with_checkmark( + f"Keep original config: `{k} = {truncate(original_data[k])}`" + ) to_data[k] = original_data[k] else: - print_with_checkmark(f"`{k}` not found in original config, using new version's default value: {truncate(v)}") + print_with_checkmark( + f"`{k}` not found in original config, using new version's default value: {truncate(v)}" + ) diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py index f3aa2c4c..ced9d56d 100644 --- a/node_management/migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/migration/src/rpc_config/from_v7_to_v9.py @@ -26,15 +26,17 @@ - [[database_setup.snapshot_config]].path 4. recommended updates - - `sync_retry_interval_in_secs` + - `sync_retry_interval_in_secs` - `block_provider_is_trusted` - `enable_snapshots` - `enable_pruning` """ + import tomlkit import importlib.resources -from common.globals import ASSUME_YES -from common.utils import prompt_or_assume_yes, print_with_checkmark, scan_and_recommend_updates +from common.utils import ( + scan_and_recommend_updates, +) def __migrate_root_config(v7_toml_data, v9_toml_data): @@ -45,6 +47,7 @@ def __migrate_root_config(v7_toml_data, v9_toml_data): print("\nScanning root level configuration ...") scan_and_recommend_updates(v7_toml_data, v9_toml_data) + def __migrate_sync_ws_config(v7_toml_data, v9_toml_data): if "synchronization" in v7_toml_data: raise SystemExit( @@ -55,8 +58,12 @@ def __migrate_sync_ws_config(v7_toml_data, v9_toml_data): v9_sync_ws_certificates = v9_sync_ws_config["certificates"] v9_sync_ws_certificates["cert_path"] = v7_toml_data["consensus_client_cert_path"] - v9_sync_ws_certificates["private_key_path"] = v7_toml_data["consensus_client_private_key_path"] - v9_sync_ws_certificates["root_ca_cert_path"] = v7_toml_data["consensus_root_ca_cert_path"] + v9_sync_ws_certificates["private_key_path"] = v7_toml_data[ + "consensus_client_private_key_path" + ] + v9_sync_ws_certificates["root_ca_cert_path"] = v7_toml_data[ + "consensus_root_ca_cert_path" + ] def __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data): @@ -78,17 +85,26 @@ def __migrate_db_archive_config(v7_toml_data, v9_toml_data): print("\nScanning archive configuration ...") scan_and_recommend_updates(v7_db_archive_config, v9_db_archive_config) + def __migrate_db_chain_store_config(v7_toml_data, v9_toml_data): - v9_db_chain_store_config = v9_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"] - v7_db_chain_store_config = v7_toml_data["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v9_db_chain_store_config = v9_toml_data["database_setup"]["dbs"]["chain_store"][ + "rocks_db" + ] + v7_db_chain_store_config = v7_toml_data["database_setup"]["dbs"]["chain_store"][ + "rocks_db" + ] v9_db_chain_store_config["path"] = v7_db_chain_store_config["path"] print("\nScanning chain store configuration ...") scan_and_recommend_updates(v7_db_chain_store_config, v9_db_chain_store_config) + def __migrate_db_ledger_config(v7_toml_data, v9_toml_data): - v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] + v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data[ + "database_setup" + ]["dbs"]["ledger"]["rocks_db"]["path"] + def __migrate_snapshot_config(v7_toml_data, v9_toml_data): v9_snapshot_config = v9_toml_data["database_setup"]["snapshot_config"] @@ -105,7 +121,11 @@ def migrate_v7_to_v9(v7_toml_data): Returns a new TOML data structure that is compatible with RPC config v9. """ - with importlib.resources.files(__package__).joinpath("rpc_config_v9_1_x_mainnet_template.toml").open("r") as f: + with ( + importlib.resources.files(__package__) + .joinpath("rpc_config_v9_1_x_mainnet_template.toml") + .open("r") as f + ): template = f.read() v9_toml_data = tomlkit.parse(template) __migrate_root_config(v7_toml_data, v9_toml_data) diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index 9cad1881..b8984ddc 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -19,6 +19,7 @@ import tomlkit import importlib.resources + def __migrate_ws_certificates(v7_toml_data, v9_toml_data): if "ws_server" in v7_toml_data["node"]: raise SystemExit( @@ -26,24 +27,44 @@ def __migrate_ws_certificates(v7_toml_data, v9_toml_data): ) v7_node_data = v7_toml_data["node"] - - v9_toml_data["node"]["ws_server"]["certificates"]["root_ca_cert_path"] = v7_node_data.get("root_ca_cert_path") - v9_toml_data["node"]["ws_server"]["certificates"]["cert_path"] = v7_node_data.get("server_cert_path") - v9_toml_data["node"]["ws_server"]["certificates"]["private_key_path"] = v7_node_data.get("server_private_key_path") + + v9_toml_data["node"]["ws_server"]["certificates"]["root_ca_cert_path"] = ( + v7_node_data.get("root_ca_cert_path") + ) + v9_toml_data["node"]["ws_server"]["certificates"]["cert_path"] = v7_node_data.get( + "server_cert_path" + ) + v9_toml_data["node"]["ws_server"]["certificates"]["private_key_path"] = ( + v7_node_data.get("server_private_key_path") + ) def __migrate_rpc_access_port(v7_toml_data, v9_toml_data): - v9_toml_data["node"]["rpc_access_port"] = v7_toml_data["node"].get("rpc_access_port") + v9_toml_data["node"]["rpc_access_port"] = v7_toml_data["node"].get( + "rpc_access_port" + ) + def __migrate_database_paths(v7_toml_data, v9_toml_data): - v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"].get("path") - v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"].get("path") + v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] = ( + v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"].get( + "path" + ) + ) + v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = ( + v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"].get("path") + ) + def migrate_v7_to_v9(toml_data): """ Returns a new TOML data structure that is compatible with SMR settings v9. """ - with importlib.resources.files(__package__).joinpath("smr_settings_v9_1_x_mainnet_template.toml").open("r") as f: + with ( + importlib.resources.files(__package__) + .joinpath("smr_settings_v9_1_x_mainnet_template.toml") + .open("r") as f + ): template = f.read() v9_toml_data = tomlkit.parse(template) __migrate_ws_certificates(toml_data, v9_toml_data) diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py index cdeff510..f458cddf 100644 --- a/node_management/migration/tests/test_rpc_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -30,6 +30,7 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): shutil.copy(from_path, tmp_from) # Run migration import common.globals + common.globals.ASSUME_YES = True rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files diff --git a/node_management/migration/tests/test_smr_migration.py b/node_management/migration/tests/test_smr_migration.py index 889f315f..cf9c52b6 100644 --- a/node_management/migration/tests/test_smr_migration.py +++ b/node_management/migration/tests/test_smr_migration.py @@ -5,7 +5,6 @@ from smr_settings.migrate_path import run_migration as smr_run_migration - @pytest.mark.parametrize( "from_file,to_file,migrate_path", [ From 0a2c4b654806f0ad7dfb41306f77e9e129fbb498 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 28 May 2025 19:17:30 +0200 Subject: [PATCH 28/44] scanning all parmaeters --- node_management/migration/src/common/utils.py | 25 ++-- .../migration/src/rpc_config/from_v7_to_v9.py | 81 ++++++++----- .../rpc_config_v9_1_x_mainnet_template.toml | 34 ++++-- .../src/smr_settings/from_v7_to_v9.py | 110 +++++++++++++----- .../smr_settings_v9_1_x_mainnet_template.toml | 29 ++++- .../tests/config_v7_to_v9_expected.toml | 16 +++ .../migration/tests/smr_settings_v7.1.x.toml | 22 ++-- .../tests/smr_settings_v7_to_v9_expected.toml | 28 ++++- .../migration/tests/test_rpc_migration.py | 3 - .../migration/tests/test_smr_migration.py | 3 + 10 files changed, 249 insertions(+), 102 deletions(-) diff --git a/node_management/migration/src/common/utils.py b/node_management/migration/src/common/utils.py index b61e3ef4..f10c7130 100644 --- a/node_management/migration/src/common/utils.py +++ b/node_management/migration/src/common/utils.py @@ -50,31 +50,36 @@ def truncate(val, max_len=50): def scan_and_recommend_updates( - original_data: tomlkit.items.Table, to_data: tomlkit.items.Table + original_table: tomlkit.items.Table, to_table: tomlkit.items.Table ): """ - Scan the original data and recommend updates to the new version's data. + Scan the original table and recommend updates to the new version's table. + + Only scan top level keys and values (i.e. skip nested tables). + + If a key exists in both tables, compare their values and prompt the user + to either keep the original value or use the new version's recommended value. """ from .globals import ASSUME_YES - for k, v in to_data.items(): + for k, v in to_table.items(): if not isinstance(v, tomlkit.items.AbstractTable): - if k in original_data: - if to_data[k] != original_data[k]: + if k in original_table: + if to_table[k] != original_table[k]: use_recommended = prompt_or_assume_yes( - f"`{k} = {truncate(original_data[k])}` is not recommended for new version.\n" - f"Do you want to apply the recommended config: `{k} = {truncate(to_data[k])}`?", + f"`{k} = {truncate(original_table[k])}` is not recommended for new version.\n" + f"Do you want to apply the recommended config: `{k} = {truncate(to_table[k])}`?", ASSUME_YES, ) if use_recommended: print_with_checkmark( - f"Apply recommended config: `{k} = {truncate(to_data[k])}`" + f"Apply recommended config: `{k} = {truncate(to_table[k])}`" ) else: print_with_checkmark( - f"Keep original config: `{k} = {truncate(original_data[k])}`" + f"Keep original config: `{k} = {truncate(original_table[k])}`" ) - to_data[k] = original_data[k] + to_table[k] = original_table[k] else: print_with_checkmark( f"`{k}` not found in original config, using new version's default value: {truncate(v)}" diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py index ced9d56d..43da1458 100644 --- a/node_management/migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/migration/src/rpc_config/from_v7_to_v9.py @@ -1,35 +1,20 @@ """ +This module provides migration utilities to upgrade Supra's RPC configuration files from version v7 to v9. + +Migration Steps: +---------------- +- Loads a v9 template configuration as the migration base. +- Migrates root-level fields such as 'supra_committees_config' and 'bind_addr'. +- Migrates synchronization WebSocket settings, including consensus RPC and certificate paths. +- Migrates chain state assembler configuration. +- Migrates database paths for archive, chain store, and ledger components. +- Migrates snapshot configuration paths. +- For each section, scans and recommends updates for any legacy fields. +- Ensures that deprecated or unexpected sections in v7 (e.g., 'synchronization', 'chain_state_assembler') are flagged as errors. + +The main entrypoint is `migrate_v7_to_v9(v7_toml_data)`, which returns a new TOML data structure compatible with v9. This module provides migration functions to upgrade Supra's RPC configuration from version v7 to v9. -Migration Overview: -------------------- -The migration process involves restructuring and updating the configuration TOML data to match the v9 schema. The key changes performed by this migration are: - -1. Synchronization Parameters: - - Copy the following key value from v7 root level to a new `[synchronization.ws]` table in the v9 template: - - `consensus_rpc` - - Copy the following keys' values from v7 root level to the `[synchronization.ws.certificates]` table in v9 template: - - from `consensus_client_cert_path` to `cert_path` - - from `consensus_client_private_key_path` to `private_key_path` - - from `consensus_root_ca_cert_path` to `root_ca_cert_path` - - Raises an error if a `[synchronization]` table already exists in the v7 config. - -2. Chain State Assembler Parameters: - - Raises an error if a `[chain_state_assembler]` table already exists in the v7 config. - -3. Copy value from 7 to v9 template in same table: - - bind_addr - - supra_committees_config - - [database_setup.dbs.archive.rocks_db].path - - [database_setup.dbs.ledger.rocks_db].path - - [database_setup.dbs.chain_store.rocks_db].path - - [[database_setup.snapshot_config]].path - -4. recommended updates - - `sync_retry_interval_in_secs` - - `block_provider_is_trusted` - - `enable_snapshots` - - `enable_pruning` """ import tomlkit @@ -101,12 +86,25 @@ def __migrate_db_chain_store_config(v7_toml_data, v9_toml_data): def __migrate_db_ledger_config(v7_toml_data, v9_toml_data): - v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = v7_toml_data[ - "database_setup" - ]["dbs"]["ledger"]["rocks_db"]["path"] + v9_db_ledger_config = v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"] + v7_db_ledger_config = v7_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"] + v9_db_ledger_config["path"] = v7_db_ledger_config["path"] + print("\nScanning ledger configuration ...") + scan_and_recommend_updates(v7_db_ledger_config, v9_db_ledger_config) def __migrate_snapshot_config(v7_toml_data, v9_toml_data): + # Optional config + if "snapshot_config" not in v7_toml_data["database_setup"]: + print( + "Warning: [database_setup.snapshot_config] table not found in v7 config. Skipping migration." + ) + return + if "snapshot_config" not in v9_toml_data["database_setup"]: + raise SystemExit( + "Error: [database_setup.snapshot_config] table should exist in v9 template." + ) + v9_snapshot_config = v9_toml_data["database_setup"]["snapshot_config"] v7_snapshot_config = v7_toml_data["database_setup"]["snapshot_config"] @@ -116,6 +114,24 @@ def __migrate_snapshot_config(v7_toml_data, v9_toml_data): scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) +def __migrate_prune_config(v7_toml_data, v9_toml_data): + # Optional config + if "prune_config" not in v7_toml_data["database_setup"]: + print( + "Warning: [database_setup.prune_config] table not found in v7 config. Skipping migration." + ) + return + if "prune_config" not in v9_toml_data["database_setup"]: + raise SystemExit( + "Error: [database_setup.prune_config] table should exist in v9 template." + ) + v9_prune_config = v9_toml_data["database_setup"]["prune_config"] + v7_prune_config = v7_toml_data["database_setup"]["prune_config"] + + print("\nScanning prune configuration ...") + scan_and_recommend_updates(v7_prune_config, v9_prune_config) + + def migrate_v7_to_v9(v7_toml_data): """ Returns a new TOML data structure that is compatible with RPC config v9. @@ -135,4 +151,5 @@ def migrate_v7_to_v9(v7_toml_data): __migrate_db_ledger_config(v7_toml_data, v9_toml_data) __migrate_db_chain_store_config(v7_toml_data, v9_toml_data) __migrate_snapshot_config(v7_toml_data, v9_toml_data) + __migrate_prune_config(v7_toml_data, v9_toml_data) return v9_toml_data diff --git a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml index 7c91be95..f8953d03 100644 --- a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml +++ b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -10,7 +10,7 @@ # The below parameters are node-specific and may be configured as required by the operator. # The port on which the node should listen for incoming RPC requests. -bind_addr = ":26000" +bind_addr = "0.0.0.0:26000" # If `true` then blocks will not be verified before execution. This value should be `false` # unless you also control the node from which this RPC node is receiving blocks. block_provider_is_trusted = true @@ -18,7 +18,7 @@ block_provider_is_trusted = true # all components will start in their default state. resume = true # The path to `supra_committees.json`. -supra_committees_config = "/supra_committees.json" +supra_committees_config = "./configs/supra_committees.json" # The access tokens used to authenticate public RPC requests to this RPC node. consensus_access_tokens = [] @@ -52,31 +52,47 @@ consensus_rpc = "ws://:26000" [synchronization.ws.certificates] # The path to the TLS certificate for the connection with the attached validator. -cert_path = "/client_supra_certificate.pem" +cert_path = "./configs/client_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -private_key_path = "/client_supra_key.pem" +private_key_path = "./configs/client_supra_key.pem" # The path to the TLS root certificate authority certificate. -root_ca_cert_path = "/ca_certificate.pem" +root_ca_cert_path = "./configs/ca_certificate.pem" # Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. [database_setup.dbs.archive.rocks_db] # The path at which the database should be created. -path = "/rpc_archive" +path = "./configs/rpc_archive" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false # Whether snapshots should be taken of the database. enable_snapshots = false # Parameters for the DKG database. [database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. -path = "/rpc_ledger" +path = "./configs/rpc_ledger" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false # Parameters for the blockchain database. [database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. -path = "/rpc_store" +path = "./configs/rpc_store" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false # Whether snapshots should be taken of the database. enable_snapshots = false +# Parameters related to database pruning. +[database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 336 + # Parameters for the database snapshot service. [database_setup.snapshot_config] # The number of snapshots to retain, including the latest. @@ -84,7 +100,7 @@ depth = 2 # The interval between snapshots in seconds. interval_in_seconds = 1800 # The path at which the snapshots should be stored. -path = "/configs/snapshot" +path = "./configs/snapshot" # The number of times to retry a snapshot in the event that it fails unexpectedly. retry_count = 3 # The interval in seconds to wait before retrying a snapshot. diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index b8984ddc..2b1da2c8 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -18,43 +18,97 @@ import tomlkit import importlib.resources +from common.utils import scan_and_recommend_updates - -def __migrate_ws_certificates(v7_toml_data, v9_toml_data): +def __migrate_node_root_config(v7_toml_data, v9_toml_data): if "ws_server" in v7_toml_data["node"]: raise SystemExit( "Error: [node.ws_server] table should not exist in v7 config. Please check your migration path matching the version of your config file." ) + v9_node_data = v9_toml_data["node"] v7_node_data = v7_toml_data["node"] + v9_node_data["rpc_access_port"] = v7_node_data["rpc_access_port"] + + v9_node_ws_certificates = v9_node_data["ws_server"]["certificates"] + v9_node_ws_certificates["root_ca_cert_path"] = v7_node_data["root_ca_cert_path"] + v9_node_ws_certificates["cert_path"] = v7_node_data["server_cert_path"] + v9_node_ws_certificates["private_key_path"] = v7_node_data["server_private_key_path"] + + print("\nScanning node root configuration ...") + scan_and_recommend_updates(v7_node_data, v9_node_data) + + +def __migrate_db_chain_store(v7_toml_data, v9_toml_data): + v9_db_chain_store = v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v7_db_chain_store = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v9_db_chain_store["path"] = v7_db_chain_store["path"] + + print("\nScanning chain store configuration ...") + scan_and_recommend_updates(v7_db_chain_store, v9_db_chain_store) - v9_toml_data["node"]["ws_server"]["certificates"]["root_ca_cert_path"] = ( - v7_node_data.get("root_ca_cert_path") - ) - v9_toml_data["node"]["ws_server"]["certificates"]["cert_path"] = v7_node_data.get( - "server_cert_path" - ) - v9_toml_data["node"]["ws_server"]["certificates"]["private_key_path"] = ( - v7_node_data.get("server_private_key_path") - ) +def __migrate_db_ledger(v7_toml_data, v9_toml_data): + v9_db_ledger = v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] + v7_db_ledger = v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] + v9_db_ledger["path"] = v7_db_ledger["path"] + + print("\nScanning ledger configuration ...") + scan_and_recommend_updates(v7_db_ledger, v9_db_ledger) + +def __migrate_snapshot_config(v7_toml_data, v9_toml_data): + """ + snapshot_config is optional, + - if absent in v7, the v9 config template will be used as is. + """ + if "snapshot_config" not in v7_toml_data["node"]["database_setup"]: + return + + if "snapshot_config" not in v9_toml_data["node"]["database_setup"]: + raise SystemExit( + "Error: [node.database_setup.snapshot_config] table should exist in v9 template." + ) + v9_snapshot_config = v9_toml_data["node"]["database_setup"]["snapshot_config"] + v7_snapshot_config = v7_toml_data["node"]["database_setup"]["snapshot_config"] -def __migrate_rpc_access_port(v7_toml_data, v9_toml_data): - v9_toml_data["node"]["rpc_access_port"] = v7_toml_data["node"].get( - "rpc_access_port" - ) + v9_snapshot_config["path"] = v7_snapshot_config["path"] + print("\nScanning snapshot configuration ...") + scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) -def __migrate_database_paths(v7_toml_data, v9_toml_data): - v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"]["path"] = ( - v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"].get( - "path" +def __migrate_prune_config(v7_toml_data, v9_toml_data): + """ + prune_config is optional, so we skip if it does not exist in v7. + """ + if "prune_config" not in v7_toml_data["node"]["database_setup"]: + return + + if "prune_config" not in v9_toml_data["node"]["database_setup"]: + raise SystemExit( + "Error: [node.database_setup.prune_config] table should exist in v9 template." ) - ) - v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"]["path"] = ( - v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"].get("path") - ) + v9_prune_config = v9_toml_data["node"]["database_setup"]["prune_config"] + v7_prune_config = v7_toml_data["node"]["database_setup"]["prune_config"] + + print("\nScanning prune configuration ...") + scan_and_recommend_updates(v7_prune_config, v9_prune_config) + + +def __migrate_mempool_config(v7_toml_data, v9_toml_data): + + v9_mempool_config = v9_toml_data["mempool"] + v7_mempool_config = v7_toml_data["mempool"] + + print("\nScanning mempool configuration ...") + scan_and_recommend_updates(v7_mempool_config, v9_mempool_config) + +def __migrate_moonshot_config(v7_toml_data, v9_toml_data): + v9_moonshot_config = v9_toml_data["moonshot"] + v7_moonshot_config = v7_toml_data["moonshot"] + + print("\nScanning moonshot configuration ...") + scan_and_recommend_updates(v7_moonshot_config, v9_moonshot_config) def migrate_v7_to_v9(toml_data): """ @@ -67,7 +121,11 @@ def migrate_v7_to_v9(toml_data): ): template = f.read() v9_toml_data = tomlkit.parse(template) - __migrate_ws_certificates(toml_data, v9_toml_data) - __migrate_rpc_access_port(toml_data, v9_toml_data) - __migrate_database_paths(toml_data, v9_toml_data) + __migrate_node_root_config(toml_data, v9_toml_data) + __migrate_db_ledger(toml_data, v9_toml_data) + __migrate_db_chain_store(toml_data, v9_toml_data) + __migrate_snapshot_config(toml_data, v9_toml_data) + __migrate_prune_config(toml_data, v9_toml_data) + __migrate_mempool_config(toml_data, v9_toml_data) + __migrate_moonshot_config(toml_data, v9_toml_data) return v9_toml_data diff --git a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml index 172ce8cb..3b65a2f5 100644 --- a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml +++ b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml @@ -97,20 +97,20 @@ connection_refresh_timeout_sec = 1 resume = true # The port on which to listen for connections from the associated RPC node. Each validator # may serve at most one RPC node. -rpc_access_port = "" +rpc_access_port = 26000 [node.ws_server.certificates] # The path to the TLS root certificate authority certificate. -root_ca_cert_path = "/ca_certificate.pem" +root_ca_cert_path = "./configs/ca_certificate.pem" # The path to the TLS certificate for this node. -cert_path = "/server_supra_certificate.pem" +cert_path = "./configs/server_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -private_key_path = "/server_supra_key.pem" +private_key_path = "./configs/server_supra_key.pem" # Parameters for the blockchain database. [node.database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. -path = "/smr_storage" +path = "./configs/smr_storage" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. enable_pruning = true @@ -120,10 +120,27 @@ enable_snapshots = false # Parameters for the DKG database. [node.database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. -path = "/ledger_storage" +path = "./configs/ledger_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false # Parameters related to database pruning. [node.database_setup.prune_config] # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. epochs_to_retain = 84 +# Parameters for the database snapshot service. +[node.database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 diff --git a/node_management/migration/tests/config_v7_to_v9_expected.toml b/node_management/migration/tests/config_v7_to_v9_expected.toml index 64129e34..71b4040c 100644 --- a/node_management/migration/tests/config_v7_to_v9_expected.toml +++ b/node_management/migration/tests/config_v7_to_v9_expected.toml @@ -61,6 +61,9 @@ root_ca_cert_path = "./xyz/ca_certificate.pem" [database_setup.dbs.archive.rocks_db] # The path at which the database should be created. path = "./xyz/rpc_archive" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false # Whether snapshots should be taken of the database. enable_snapshots = false @@ -68,14 +71,27 @@ enable_snapshots = false [database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. path = "./xyz/rpc_ledger" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false # Parameters for the blockchain database. [database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. path = "./xyz/rpc_store" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false # Whether snapshots should be taken of the database. enable_snapshots = false +# Parameters related to database pruning. +[database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 336 + # Parameters for the database snapshot service. [database_setup.snapshot_config] # The number of snapshots to retain, including the latest. diff --git a/node_management/migration/tests/smr_settings_v7.1.x.toml b/node_management/migration/tests/smr_settings_v7.1.x.toml index ef62f316..355aa86a 100644 --- a/node_management/migration/tests/smr_settings_v7.1.x.toml +++ b/node_management/migration/tests/smr_settings_v7.1.x.toml @@ -29,7 +29,7 @@ genesis_timestamp_microseconds = 1732060800000000 [mempool] # The maximum number of milliseconds that a node will wait before proposing a batch when it has # at least one transaction to process. -max_batch_delay_ms = 500 +max_batch_delay_ms = 1500 # The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` # then a batch will be proposed immediately. max_batch_size_bytes = 5000000 @@ -67,13 +67,13 @@ max_payload_items_per_block = 50 # Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount # of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can # become. This trade-off must be balanced. -message_recency_bound_rounds = 1000 +message_recency_bound_rounds = 20 # The delay after which the node will try to repeat sync requests for missing blocks. # Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 1000 +sync_retry_delay_ms = 2000 # The delay after which the node will send a Timeout message for its current Moonshot round, # measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 3500 +timeout_delay_ms = 5000 # Parameters related to the MoveVM. Primarily related to governance features. [move_vm] @@ -133,25 +133,25 @@ voting_power_increase_limit = 33 [node] # The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 1 +connection_refresh_timeout_sec = 2 # If true, all components will attempt to load their previous state from disk. Otherwise, # all components will start in their default state. Should always be `true` for testnet and # mainnet. resume = true # The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" +root_ca_cert_path = "./tests/ca_certificate.pem" # The port on which to listen for connections from the associated RPC node. Each validator # may serve at most one RPC node. rpc_access_port = 26000 # The path to the TLS certificate for this node. -server_cert_path = "./configs/server_supra_certificate.pem" +server_cert_path = "./tests/server_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -server_private_key_path = "./configs/server_supra_key.pem" +server_private_key_path = "./tests/server_supra_key.pem" # Parameters for the blockchain database. [node.database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. -path = "./configs/smr_storage" +path = "./tests/smr_storage" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. enable_pruning = true @@ -159,9 +159,9 @@ enable_pruning = true # Parameters for the DKG database. [node.database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. -path = "./configs/ledger_storage" +path = "./tests/ledger_storage" # Parameters related to database pruning. [node.database_setup.prune_config] # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 \ No newline at end of file +epochs_to_retain = 84 diff --git a/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml index c66b40bd..eef5ffb5 100644 --- a/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml +++ b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml @@ -100,16 +100,16 @@ rpc_access_port = 26000 [node.ws_server.certificates] # The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" +root_ca_cert_path = "./tests/ca_certificate.pem" # The path to the TLS certificate for this node. -cert_path = "./configs/server_supra_certificate.pem" +cert_path = "./tests/server_supra_certificate.pem" # The path to the private key to be used when negotiating TLS connections. -private_key_path = "./configs/server_supra_key.pem" +private_key_path = "./tests/server_supra_key.pem" # Parameters for the blockchain database. [node.database_setup.dbs.chain_store.rocks_db] # The path at which the database should be created. -path = "./configs/smr_storage" +path = "./tests/smr_storage" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. enable_pruning = true @@ -119,9 +119,27 @@ enable_snapshots = false # Parameters for the DKG database. [node.database_setup.dbs.ledger.rocks_db] # The path at which the database should be created. -path = "./configs/ledger_storage" +path = "./tests/ledger_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false # Parameters related to database pruning. [node.database_setup.prune_config] # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. epochs_to_retain = 84 + +# Parameters for the database snapshot service. +[node.database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py index f458cddf..1344a480 100644 --- a/node_management/migration/tests/test_rpc_migration.py +++ b/node_management/migration/tests/test_rpc_migration.py @@ -15,8 +15,6 @@ @pytest.mark.parametrize( "from_file,to_file,migrate_path", [ - # ("config_v7.1.8.toml", "config_v8.0.2.toml", "v7-v8"), - # ("config_v8.0.2.toml", "config_v9.0.7.toml", "v8-v9"), ("config_v7.1.8.toml", "config_v7_to_v9_expected.toml", "v7-v9"), ], ) @@ -30,7 +28,6 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): shutil.copy(from_path, tmp_from) # Run migration import common.globals - common.globals.ASSUME_YES = True rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files diff --git a/node_management/migration/tests/test_smr_migration.py b/node_management/migration/tests/test_smr_migration.py index cf9c52b6..ef6eef8b 100644 --- a/node_management/migration/tests/test_smr_migration.py +++ b/node_management/migration/tests/test_smr_migration.py @@ -20,6 +20,9 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): tmp_to = tmp_path / to_file shutil.copy(from_path, tmp_from) # Run migration + import common.globals + common.globals.ASSUME_YES = True + smr_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files with open(tmp_to, "r") as f: From 1f7af5b46b75191ba853674eea4668a239fa3725 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 28 May 2025 19:21:53 +0200 Subject: [PATCH 29/44] add docs --- .../src/smr_settings/from_v7_to_v9.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py index 2b1da2c8..c96b371a 100644 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/migration/src/smr_settings/from_v7_to_v9.py @@ -1,19 +1,17 @@ """ +This script reads a v7 configuration and migrates relevant settings into a v9 template, +ensuring compatibility with the new version. The migration process includes: +- Validating the absence of deprecated tables in v7 configs. +- Copying and updating node root configuration, including RPC and certificate paths. +- Migrating database paths for chain_store and ledger RocksDB instances. +- Optionally migrating snapshot and prune configurations if present. +- Migrating mempool and moonshot configurations. +- Scanning each migrated section for recommended updates. + +The migration preserves user-specific paths and settings where applicable, while adopting +the structure and defaults of the v9 template. Migration script for upgrading Supra smr_settings config files from version v7 to v9. -This module provides functions to transform configuration files in TOML format to be compatible with v9. -The migration includes the following changes: - -1. Migration of WebSocket server certificate paths: - - Copy below keys in v7 [node] table to the `[node.ws_server.certificates]` table in v9: - - from `root_ca_cert_path` to `root_ca_cert_path` - - from `server_cert_path` to `cert_path` - - from `server_private_key_path` to `private_key_path` - - Ensures that the `[node.ws_server]` table does not exist prior to migration, enforcing correct migration order. -2. Copy value from 7 to v9 template in same table: - - [node].rpc_access_port - - [node.database_setup.dbs.chain_store.rocks_db].path - - [node.database_setup.dbs.ledger.rocks_db].path """ import tomlkit From f884b17b0ff7256e994644094b4ed2d0750d6a77 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 29 May 2025 10:41:41 +0200 Subject: [PATCH 30/44] update readme example --- node_management/migration/README.md | 97 +++++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 5 deletions(-) diff --git a/node_management/migration/README.md b/node_management/migration/README.md index 3f5dbb6b..aae89263 100644 --- a/node_management/migration/README.md +++ b/node_management/migration/README.md @@ -50,6 +50,47 @@ supra-v9 data migrate -p smr_settings.toml $ migrate-config rpc -p v7-v9 -f config.toml -t config.toml + Running migration function: migrate_v7_to_v9 + + Scanning root level configuration ... + ✓ `consensus_access_tokens` not found in original config, using new version's default value: [] + `allowed_origin = Date: Thu, 29 May 2025 10:50:29 +0200 Subject: [PATCH 31/44] rename dir --- node_management/migration/.gitignore | 13 -- node_management/migration/README.md | 220 ------------------ node_management/migration/pyproject.toml | 27 --- node_management/migration/src/cli/__init__.py | 0 node_management/migration/src/cli/main.py | 77 ------ node_management/migration/src/common/__init__ | 0 .../migration/src/common/globals.py | 1 - .../migration/src/common/migration.py | 95 -------- node_management/migration/src/common/utils.py | 86 ------- .../migration/src/rpc_config/__init__.py | 0 .../migration/src/rpc_config/from_v7_to_v9.py | 155 ------------ .../migration/src/rpc_config/migrate_path.py | 13 -- .../rpc_config_v9_1_x_mainnet_template.toml | 155 ------------ .../migration/src/smr_settings/__init__.py | 0 .../src/smr_settings/from_v7_to_v9.py | 129 ---------- .../src/smr_settings/migrate_path.py | 14 -- .../smr_settings_v9_1_x_mainnet_template.toml | 146 ------------ .../migration/tests/config_v7.1.8.toml | 131 ----------- .../tests/config_v7_to_v9_expected.toml | 154 ------------ .../migration/tests/smr_settings_v7.1.x.toml | 167 ------------- .../tests/smr_settings_v7_to_v9_expected.toml | 145 ------------ .../migration/tests/test_rpc_migration.py | 42 ---- .../migration/tests/test_smr_migration.py | 35 --- 23 files changed, 1805 deletions(-) delete mode 100644 node_management/migration/.gitignore delete mode 100644 node_management/migration/README.md delete mode 100644 node_management/migration/pyproject.toml delete mode 100644 node_management/migration/src/cli/__init__.py delete mode 100644 node_management/migration/src/cli/main.py delete mode 100644 node_management/migration/src/common/__init__ delete mode 100644 node_management/migration/src/common/globals.py delete mode 100644 node_management/migration/src/common/migration.py delete mode 100644 node_management/migration/src/common/utils.py delete mode 100644 node_management/migration/src/rpc_config/__init__.py delete mode 100644 node_management/migration/src/rpc_config/from_v7_to_v9.py delete mode 100644 node_management/migration/src/rpc_config/migrate_path.py delete mode 100644 node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml delete mode 100644 node_management/migration/src/smr_settings/__init__.py delete mode 100644 node_management/migration/src/smr_settings/from_v7_to_v9.py delete mode 100644 node_management/migration/src/smr_settings/migrate_path.py delete mode 100644 node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml delete mode 100644 node_management/migration/tests/config_v7.1.8.toml delete mode 100644 node_management/migration/tests/config_v7_to_v9_expected.toml delete mode 100644 node_management/migration/tests/smr_settings_v7.1.x.toml delete mode 100644 node_management/migration/tests/smr_settings_v7_to_v9_expected.toml delete mode 100644 node_management/migration/tests/test_rpc_migration.py delete mode 100644 node_management/migration/tests/test_smr_migration.py diff --git a/node_management/migration/.gitignore b/node_management/migration/.gitignore deleted file mode 100644 index 33a954f4..00000000 --- a/node_management/migration/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -# Ignore all files in the ruff cache directory -**/.ruff_cache - -**/build/ - -# egg-info -**/*.egg-info/ - -# python cache -**/__pycache__/ - -# pytest -**/.hypothesis/ \ No newline at end of file diff --git a/node_management/migration/README.md b/node_management/migration/README.md deleted file mode 100644 index aae89263..00000000 --- a/node_management/migration/README.md +++ /dev/null @@ -1,220 +0,0 @@ - -### Usage Example - - -alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node -alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node - -alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra" -alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra" - - -1. Install the `migrate-config` tool - -```sh -pip install . -``` - -2. Migrate rpc - -```sh -# Migrate rpc config from v7 to v9 -migrate-config rpc -p v7-v9 -f config.toml -t config.toml -# Migrate db from v7 to v8 -rpc-v8 migrate-db config.toml -# Migrate db from v8 to v9 -rpc-v9 migrate-db config.toml -``` - -3. Migrate smr/validator - -```sh -# Migrate cli profile from v7 to v8 -supra-v8 migrate --network localnet -cp validator_identity.pem node_identity.pem -# Migrate cli profile from v8 to v9 -supra-v9 profile migrate - -# Migrate smr_settings from v7 to v9 -migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml -# Migrate db from v7 to v9 -supra-v9 data migrate -p smr_settings.toml -``` - - -### Migrate from v7 to v9 example flow - -#### RPC - -1. Config migration - -$ migrate-config rpc -p v7-v9 -f config.toml -t config.toml - - Running migration function: migrate_v7_to_v9 - - Scanning root level configuration ... - ✓ `consensus_access_tokens` not found in original config, using new version's default value: [] - `allowed_origin = block_to_tx_ordered: Migrated 8537 records, up to 239 block height - certified_block -> certified_block_dehydrated: Migrated 244 records, up to 244 block height - databases_checked: - - chain_store - - archive - -#### SMR - -1. Profile/Identity migration - -**You need both v8 and v9 supra binary to run the profile migratino.** - -$ supra-v8 migrate --network mainnet - -$ cp validator_identity.pem node_identity.pem - -$ supra-v9 profile migrate - -2. Config migration - -$ migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml - - Running migration function: migrate_v7_to_v9 - - Scanning node root configuration ... - `connection_refresh_timeout_sec = 2` is not recommended for new version. - Do you want to apply the recommended config: `connection_refresh_timeout_sec = 1`? [y/N]: y - ✓ Apply recommended config: `connection_refresh_timeout_sec = 1` - - Scanning ledger configuration ... - ✓ `enable_pruning` not found in original config, using new version's default value: True - ✓ `enable_snapshots` not found in original config, using new version's default value: False - - Scanning chain store configuration ... - ✓ `enable_snapshots` not found in original config, using new version's default value: False - - Scanning prune configuration ... - - Scanning mempool configuration ... - `max_batch_delay_ms = 1500` is not recommended for new version. - Do you want to apply the recommended config: `max_batch_delay_ms = 500`? [y/N]: y - ✓ Apply recommended config: `max_batch_delay_ms = 500` - - Scanning moonshot configuration ... - `message_recency_bound_rounds = 20` is not recommended for new version. - Do you want to apply the recommended config: `message_recency_bound_rounds = 1000`? [y/N]: y - ✓ Apply recommended config: `message_recency_bound_rounds = 1000` - `sync_retry_delay_ms = 2000` is not recommended for new version. - Do you want to apply the recommended config: `sync_retry_delay_ms = 1000`? [y/N]: y - ✓ Apply recommended config: `sync_retry_delay_ms = 1000` - `timeout_delay_ms = 5000` is not recommended for new version. - Do you want to apply the recommended config: `timeout_delay_ms = 3500`? [y/N]: y - ✓ Apply recommended config: `timeout_delay_ms = 3500` - Writing new config to /tmp/new_smr.toml - |----------------- Begin diff v7 vs v9 -----------------| - --- v7 - +++ v9 - .. - |----------------- End diff v7 vs v9 -----------------| - - ###################################################################### - # Config migrated from tests/smr_settings_v7.1.x.toml to /tmp/new_smr.toml. - # - # Please review the diff above for changes made during migration. - # - # Please ensure to use the new config file for target binary version. - ###################################################################### - -3. DB migration - -**You need v9 supra binary to run the db migration.** - -$ supra-v9 data migrate -p smr_settings.toml - - Counting the number of entries in certified_block... - Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 69/69 00:00:00 - Counting the number of entries in uncommitted_block... - Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 74/74 00:00:00 - Cleaning up uncommitted_block: [00:00:00] ████████████████████ 4/4 00:00:00 - Counting the number of entries in certified_block... - Counting the number of entries in certified_block_dehydrated... - Counting the number of entries in uncommitted_block... - Counting the number of entries in qc... - Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 70/70 00:00:00 - Counting the number of entries to remove from prune_index... - Cleaning up prune index: [00:00:00] ████████████████████ 70/70 00:00:00 - dropped: - - certified_block - migrated: - certified_block -> certified_block_dehydrated: Migrated 70 records, up to 244 block height - databases_checked: - - chain_store - - -### Test harness of migrate-config script - -`PYTHONPATH=src pytest` \ No newline at end of file diff --git a/node_management/migration/pyproject.toml b/node_management/migration/pyproject.toml deleted file mode 100644 index abc3a956..00000000 --- a/node_management/migration/pyproject.toml +++ /dev/null @@ -1,27 +0,0 @@ -[project] -name = "migrate-config" -version = "0.1.0" -description = "Unified CLI tool to migrate Supra RPC and SMR configs." -readme = "README.md" -requires-python = ">=3.10" -dependencies = [ - "tomlkit>=0.13.2", - "click>=8.0.0" -] - -[project.scripts] -migrate-config = "cli.main:main" - -[build-system] -requires = ["setuptools>=61.0", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.setuptools] -package-dir = {"" = "src"} - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -rpc_config = ["rpc_config_v9_1_x_mainnet_template.toml"] -smr_settings = ["smr_settings_v9_1_x_mainnet_template.toml"] diff --git a/node_management/migration/src/cli/__init__.py b/node_management/migration/src/cli/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/node_management/migration/src/cli/main.py b/node_management/migration/src/cli/main.py deleted file mode 100644 index 29efbfe5..00000000 --- a/node_management/migration/src/cli/main.py +++ /dev/null @@ -1,77 +0,0 @@ -import click -from rpc_config.migrate_path import RPC_CONFIG_MIGRATE_PATH -from rpc_config.migrate_path import run_migration as migrate_rpc_config -from smr_settings.migrate_path import SMR_SETTINGS_MIGRATE_PATH -from smr_settings.migrate_path import run_migration as migrate_smr_config -import common.globals - - -@click.group() -def main(): - """Migration CLI for Supra configs.""" - - -@main.command() -@click.option( - "--migrate-path", - "-p", - required=True, - type=click.Choice(RPC_CONFIG_MIGRATE_PATH, case_sensitive=True), - help=f"Migration path (choices: {', '.join(RPC_CONFIG_MIGRATE_PATH)})", -) -@click.option( - "--from-file", - "-f", - required=True, - type=click.Path(exists=True), - help="Source config file", -) -@click.option( - "--to-file", "-t", required=True, type=click.Path(), help="Output config file" -) -@click.option( - "--assume-yes", - "-y", - is_flag=True, - default=False, - help="Assume yes for all prompts (default: False)", -) -def rpc(migrate_path, from_file, to_file, assume_yes): - """Migrate RPC config.""" - common.globals.ASSUME_YES = assume_yes - migrate_rpc_config(migrate_path, from_file, to_file) - - -@main.command() -@click.option( - "--migrate-path", - "-p", - required=True, - type=click.Choice(SMR_SETTINGS_MIGRATE_PATH, case_sensitive=True), - help=f"Migration path (choices: {', '.join(SMR_SETTINGS_MIGRATE_PATH)})", -) -@click.option( - "--from-file", - "-f", - required=True, - type=click.Path(exists=True), - help="Source config file", -) -@click.option( - "--to-file", "-t", required=True, type=click.Path(), help="Output config file" -) -@click.option( - "--assume-yes", - "-y", - is_flag=True, - default=False, - help="Assume yes for all prompts (default: False)", -) -def smr(migrate_path, from_file, to_file, assume_yes): - """Migrate SMR config.""" - common.globals.ASSUME_YES = assume_yes - migrate_smr_config(migrate_path, from_file, to_file) - - -if __name__ == "__main__": - main() diff --git a/node_management/migration/src/common/__init__ b/node_management/migration/src/common/__init__ deleted file mode 100644 index e69de29b..00000000 diff --git a/node_management/migration/src/common/globals.py b/node_management/migration/src/common/globals.py deleted file mode 100644 index 9b8bd68b..00000000 --- a/node_management/migration/src/common/globals.py +++ /dev/null @@ -1 +0,0 @@ -ASSUME_YES = False diff --git a/node_management/migration/src/common/migration.py b/node_management/migration/src/common/migration.py deleted file mode 100644 index 18efc502..00000000 --- a/node_management/migration/src/common/migration.py +++ /dev/null @@ -1,95 +0,0 @@ -import typing as ty -import tomlkit -from copy import deepcopy -from . import utils -import shutil -from common.globals import ASSUME_YES - - -class MigrationPathSet: - """ - Base class for migration paths. - """ - - def __init__(self, migrate_paths: ty.Dict[str, ty.List[ty.Callable]]): - self.migrate_paths = migrate_paths - - def get_versions(self, key: str) -> ty.Tuple[str, str]: - """Split the key into from_version and to_version.""" - if key not in self.migrate_paths: - raise ValueError(f"Invalid key: {key}") - from_version, to_version = key.split("-", 1) - return from_version, to_version - - def get_migration_functions(self, key: str) -> ty.List[ty.Callable]: - """Get the migration functions for the given key.""" - if key not in self.migrate_paths: - raise ValueError(f"Unknown migration path: {key}") - return self.migrate_paths[key] - - -class Migration: - """ - Top level migration class that handles the migration of config files. - """ - - def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]): - self.migrate_path = MigrationPathSet(migrate_path) - - def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): - migrate_functions = self.migrate_path.get_migration_functions(migrate_choice) - from_version, to_version = self.migrate_path.get_versions(migrate_choice) - default_backup_path = f"{from_path}_{from_version}.bak" - - if from_path == to_path: - print( - f"Warning: The source and destination paths are the same ({from_path})." - ) - print( - f"A backup of your original config will be saved to: {default_backup_path}" - ) - confirm = utils.prompt_or_assume_yes( - "This will overwrite your original config file. Continue?", ASSUME_YES - ) - if not confirm: - print("Aborted by user.") - return - # Backup old config - print(f"Backing up old config to {default_backup_path}") - shutil.copyfile(from_path, default_backup_path) - - with open(from_path, "r") as f: - toml_data = tomlkit.load(f) - - original_toml_data = deepcopy(toml_data) - - for fn in migrate_functions: - print(f"Running migration function: {fn.__name__}") - toml_data = fn(toml_data) - - # Write new config - print(f"Writing new config to {to_path}") - with open(to_path, "w") as f: - tomlkit.dump(toml_data, f) - # Print the diff - from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) - to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) - - utils.unified_diff( - from_str, - to_str, - fromfile=from_version, - tofile=to_version, - ) - - print( - f""" - ###################################################################### - # Config migrated from {from_path} to {to_path}. - # - # Please review the diff above for changes made during migration. - # - # Please ensure to use the new config file for target binary version. - ###################################################################### - """ - ) diff --git a/node_management/migration/src/common/utils.py b/node_management/migration/src/common/utils.py deleted file mode 100644 index f10c7130..00000000 --- a/node_management/migration/src/common/utils.py +++ /dev/null @@ -1,86 +0,0 @@ -import difflib -import tomlkit - - -def unified_diff(from_str, to_str, fromfile, tofile): - print(f"|----------------- Begin diff {fromfile} vs {tofile} -----------------|") - diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) - __print_colored_diff(diff) - print(f"|----------------- End diff {fromfile} vs {tofile} -----------------|") - - -def __print_colored_diff(diff): - # The color is added here manually using ANSI escape codes. - for line in diff: - if line.startswith("+") and not line.startswith("+++"): - print(f"\033[32m{line}\033[0m", end="") # Green for additions - elif line.startswith("-") and not line.startswith("---"): - print(f"\033[31m{line}\033[0m", end="") # Red for deletions - elif line.startswith("@@"): - print(f"\033[36m{line}\033[0m", end="") # Cyan for hunk headers - else: - print(line, end="") - - -def print_with_checkmark(message): - """ - Print a message with a checkmark. - """ - print(f"✓ {message}") - - -def prompt_or_assume_yes(message, assume_yes=False) -> bool: - """ - Prompt the user for confirmation or assume 'yes' for all prompts - """ - if not assume_yes: - response = input(f"{message} [y/N]: ").strip().lower() - return response in ("y", "yes") - else: - print(f"{message} (assuming yes)") - return True - - -def truncate(val, max_len=50): - """ - Truncate a string representation of a value to a maximum length. - """ - s = str(val) - return s if len(s) <= max_len else s[: max_len - 3] + "..." - - -def scan_and_recommend_updates( - original_table: tomlkit.items.Table, to_table: tomlkit.items.Table -): - """ - Scan the original table and recommend updates to the new version's table. - - Only scan top level keys and values (i.e. skip nested tables). - - If a key exists in both tables, compare their values and prompt the user - to either keep the original value or use the new version's recommended value. - """ - from .globals import ASSUME_YES - - for k, v in to_table.items(): - if not isinstance(v, tomlkit.items.AbstractTable): - if k in original_table: - if to_table[k] != original_table[k]: - use_recommended = prompt_or_assume_yes( - f"`{k} = {truncate(original_table[k])}` is not recommended for new version.\n" - f"Do you want to apply the recommended config: `{k} = {truncate(to_table[k])}`?", - ASSUME_YES, - ) - if use_recommended: - print_with_checkmark( - f"Apply recommended config: `{k} = {truncate(to_table[k])}`" - ) - else: - print_with_checkmark( - f"Keep original config: `{k} = {truncate(original_table[k])}`" - ) - to_table[k] = original_table[k] - else: - print_with_checkmark( - f"`{k}` not found in original config, using new version's default value: {truncate(v)}" - ) diff --git a/node_management/migration/src/rpc_config/__init__.py b/node_management/migration/src/rpc_config/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/node_management/migration/src/rpc_config/from_v7_to_v9.py b/node_management/migration/src/rpc_config/from_v7_to_v9.py deleted file mode 100644 index 43da1458..00000000 --- a/node_management/migration/src/rpc_config/from_v7_to_v9.py +++ /dev/null @@ -1,155 +0,0 @@ -""" -This module provides migration utilities to upgrade Supra's RPC configuration files from version v7 to v9. - -Migration Steps: ----------------- -- Loads a v9 template configuration as the migration base. -- Migrates root-level fields such as 'supra_committees_config' and 'bind_addr'. -- Migrates synchronization WebSocket settings, including consensus RPC and certificate paths. -- Migrates chain state assembler configuration. -- Migrates database paths for archive, chain store, and ledger components. -- Migrates snapshot configuration paths. -- For each section, scans and recommends updates for any legacy fields. -- Ensures that deprecated or unexpected sections in v7 (e.g., 'synchronization', 'chain_state_assembler') are flagged as errors. - -The main entrypoint is `migrate_v7_to_v9(v7_toml_data)`, which returns a new TOML data structure compatible with v9. -This module provides migration functions to upgrade Supra's RPC configuration from version v7 to v9. - -""" - -import tomlkit -import importlib.resources -from common.utils import ( - scan_and_recommend_updates, -) - - -def __migrate_root_config(v7_toml_data, v9_toml_data): - v9_toml_data["supra_committees_config"] = v7_toml_data["supra_committees_config"] - - v9_toml_data["bind_addr"] = v7_toml_data["bind_addr"] - - print("\nScanning root level configuration ...") - scan_and_recommend_updates(v7_toml_data, v9_toml_data) - - -def __migrate_sync_ws_config(v7_toml_data, v9_toml_data): - if "synchronization" in v7_toml_data: - raise SystemExit( - "Error: [synchronization] table should not exist in v7 config. Please check your migration path matching the version of your config file." - ) - v9_sync_ws_config = v9_toml_data["synchronization"]["ws"] - v9_sync_ws_config["consensus_rpc"] = v7_toml_data["consensus_rpc"] - - v9_sync_ws_certificates = v9_sync_ws_config["certificates"] - v9_sync_ws_certificates["cert_path"] = v7_toml_data["consensus_client_cert_path"] - v9_sync_ws_certificates["private_key_path"] = v7_toml_data[ - "consensus_client_private_key_path" - ] - v9_sync_ws_certificates["root_ca_cert_path"] = v7_toml_data[ - "consensus_root_ca_cert_path" - ] - - -def __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data): - if "chain_state_assembler" in v7_toml_data: - raise SystemExit( - "Error: [chain_state_assembler] table should not exist in v7 config." - ) - - print("\nScanning chain state assembler configuration ...") - scan_and_recommend_updates(v7_toml_data, v9_toml_data["chain_state_assembler"]) - - -def __migrate_db_archive_config(v7_toml_data, v9_toml_data): - v9_db_archive_config = v9_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"] - v7_db_archive_config = v7_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"] - - v9_db_archive_config["path"] = v7_db_archive_config["path"] - - print("\nScanning archive configuration ...") - scan_and_recommend_updates(v7_db_archive_config, v9_db_archive_config) - - -def __migrate_db_chain_store_config(v7_toml_data, v9_toml_data): - v9_db_chain_store_config = v9_toml_data["database_setup"]["dbs"]["chain_store"][ - "rocks_db" - ] - v7_db_chain_store_config = v7_toml_data["database_setup"]["dbs"]["chain_store"][ - "rocks_db" - ] - - v9_db_chain_store_config["path"] = v7_db_chain_store_config["path"] - - print("\nScanning chain store configuration ...") - scan_and_recommend_updates(v7_db_chain_store_config, v9_db_chain_store_config) - - -def __migrate_db_ledger_config(v7_toml_data, v9_toml_data): - v9_db_ledger_config = v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"] - v7_db_ledger_config = v7_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"] - v9_db_ledger_config["path"] = v7_db_ledger_config["path"] - print("\nScanning ledger configuration ...") - scan_and_recommend_updates(v7_db_ledger_config, v9_db_ledger_config) - - -def __migrate_snapshot_config(v7_toml_data, v9_toml_data): - # Optional config - if "snapshot_config" not in v7_toml_data["database_setup"]: - print( - "Warning: [database_setup.snapshot_config] table not found in v7 config. Skipping migration." - ) - return - if "snapshot_config" not in v9_toml_data["database_setup"]: - raise SystemExit( - "Error: [database_setup.snapshot_config] table should exist in v9 template." - ) - - v9_snapshot_config = v9_toml_data["database_setup"]["snapshot_config"] - v7_snapshot_config = v7_toml_data["database_setup"]["snapshot_config"] - - v9_snapshot_config["path"] = v7_snapshot_config["path"] - - print("\nScanning snapshot configuration ...") - scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) - - -def __migrate_prune_config(v7_toml_data, v9_toml_data): - # Optional config - if "prune_config" not in v7_toml_data["database_setup"]: - print( - "Warning: [database_setup.prune_config] table not found in v7 config. Skipping migration." - ) - return - if "prune_config" not in v9_toml_data["database_setup"]: - raise SystemExit( - "Error: [database_setup.prune_config] table should exist in v9 template." - ) - v9_prune_config = v9_toml_data["database_setup"]["prune_config"] - v7_prune_config = v7_toml_data["database_setup"]["prune_config"] - - print("\nScanning prune configuration ...") - scan_and_recommend_updates(v7_prune_config, v9_prune_config) - - -def migrate_v7_to_v9(v7_toml_data): - """ - Returns a new TOML data structure that is compatible with RPC config v9. - """ - - with ( - importlib.resources.files(__package__) - .joinpath("rpc_config_v9_1_x_mainnet_template.toml") - .open("r") as f - ): - template = f.read() - v9_toml_data = tomlkit.parse(template) - __migrate_root_config(v7_toml_data, v9_toml_data) - __migrate_sync_ws_config(v7_toml_data, v9_toml_data) - __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data) - __migrate_db_archive_config(v7_toml_data, v9_toml_data) - __migrate_db_ledger_config(v7_toml_data, v9_toml_data) - __migrate_db_chain_store_config(v7_toml_data, v9_toml_data) - __migrate_snapshot_config(v7_toml_data, v9_toml_data) - __migrate_prune_config(v7_toml_data, v9_toml_data) - return v9_toml_data diff --git a/node_management/migration/src/rpc_config/migrate_path.py b/node_management/migration/src/rpc_config/migrate_path.py deleted file mode 100644 index 00018344..00000000 --- a/node_management/migration/src/rpc_config/migrate_path.py +++ /dev/null @@ -1,13 +0,0 @@ -from .from_v7_to_v9 import migrate_v7_to_v9 -from common import migration - - -RPC_CONFIG_MIGRATE_PATH = { - "v7-v9": [migrate_v7_to_v9], -} - - -def run_migration(migrate_path: str, from_path: str, to_path: str): - migration.Migration(RPC_CONFIG_MIGRATE_PATH).migrate_config( - migrate_path, from_path, to_path - ) diff --git a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml deleted file mode 100644 index f8953d03..00000000 --- a/node_management/migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml +++ /dev/null @@ -1,155 +0,0 @@ - -# Version: v9.1.x mainnet - -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "0.0.0.0:26000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = true -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./configs/supra_committees.json" -# The access tokens used to authenticate public RPC requests to this RPC node. -consensus_access_tokens = [] - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -[chain_instance] -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - - -######################################### NODE PARAMETERS ######################################### -[chain_state_assembler] -# Number of certified blocks stored in memory as reference to pending blocks to be executed. -# Only this amount of certified blocks are stored in memory, for the rest memo is kept. -certified_block_cache_bucket_size = 50 -# Retry interval for the sync requests for which no response yet available. -sync_retry_interval_in_secs = 1 - -[synchronization.ws] -# The websocket address of the attached validator. -consensus_rpc = "ws://:26000" - -[synchronization.ws.certificates] -# The path to the TLS certificate for the connection with the attached validator. -cert_path = "./configs/client_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "./configs/client_supra_key.pem" -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_archive" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = false -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_ledger" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = false -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/rpc_store" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = false -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters related to database pruning. -[database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 336 - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. -# The below settings are the default values required for use in RPC nodes run by validator node operators. -# They are optional for non-validators. -[[allowed_origin]] -url = "https://rpc-mainnet.supra.com" -description = "RPC For Supra" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-wallet-mainnet.supra.com" -description = "RPC For Supra Wallet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-suprascan-mainnet.supra.com" -description = "RPC For suprascan" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:26000" -description = "LocalNet" -mode = "Server" diff --git a/node_management/migration/src/smr_settings/__init__.py b/node_management/migration/src/smr_settings/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/node_management/migration/src/smr_settings/from_v7_to_v9.py b/node_management/migration/src/smr_settings/from_v7_to_v9.py deleted file mode 100644 index c96b371a..00000000 --- a/node_management/migration/src/smr_settings/from_v7_to_v9.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -This script reads a v7 configuration and migrates relevant settings into a v9 template, -ensuring compatibility with the new version. The migration process includes: -- Validating the absence of deprecated tables in v7 configs. -- Copying and updating node root configuration, including RPC and certificate paths. -- Migrating database paths for chain_store and ledger RocksDB instances. -- Optionally migrating snapshot and prune configurations if present. -- Migrating mempool and moonshot configurations. -- Scanning each migrated section for recommended updates. - -The migration preserves user-specific paths and settings where applicable, while adopting -the structure and defaults of the v9 template. -Migration script for upgrading Supra smr_settings config files from version v7 to v9. - -""" - -import tomlkit -import importlib.resources -from common.utils import scan_and_recommend_updates - -def __migrate_node_root_config(v7_toml_data, v9_toml_data): - if "ws_server" in v7_toml_data["node"]: - raise SystemExit( - "Error: [node.ws_server] table should not exist in v7 config. Please check your migration path matching the version of your config file." - ) - - v9_node_data = v9_toml_data["node"] - v7_node_data = v7_toml_data["node"] - v9_node_data["rpc_access_port"] = v7_node_data["rpc_access_port"] - - v9_node_ws_certificates = v9_node_data["ws_server"]["certificates"] - v9_node_ws_certificates["root_ca_cert_path"] = v7_node_data["root_ca_cert_path"] - v9_node_ws_certificates["cert_path"] = v7_node_data["server_cert_path"] - v9_node_ws_certificates["private_key_path"] = v7_node_data["server_private_key_path"] - - print("\nScanning node root configuration ...") - scan_and_recommend_updates(v7_node_data, v9_node_data) - - -def __migrate_db_chain_store(v7_toml_data, v9_toml_data): - v9_db_chain_store = v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] - v7_db_chain_store = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] - v9_db_chain_store["path"] = v7_db_chain_store["path"] - - print("\nScanning chain store configuration ...") - scan_and_recommend_updates(v7_db_chain_store, v9_db_chain_store) - -def __migrate_db_ledger(v7_toml_data, v9_toml_data): - v9_db_ledger = v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] - v7_db_ledger = v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] - v9_db_ledger["path"] = v7_db_ledger["path"] - - print("\nScanning ledger configuration ...") - scan_and_recommend_updates(v7_db_ledger, v9_db_ledger) - -def __migrate_snapshot_config(v7_toml_data, v9_toml_data): - """ - snapshot_config is optional, - - if absent in v7, the v9 config template will be used as is. - """ - if "snapshot_config" not in v7_toml_data["node"]["database_setup"]: - return - - if "snapshot_config" not in v9_toml_data["node"]["database_setup"]: - raise SystemExit( - "Error: [node.database_setup.snapshot_config] table should exist in v9 template." - ) - - v9_snapshot_config = v9_toml_data["node"]["database_setup"]["snapshot_config"] - v7_snapshot_config = v7_toml_data["node"]["database_setup"]["snapshot_config"] - - v9_snapshot_config["path"] = v7_snapshot_config["path"] - - print("\nScanning snapshot configuration ...") - scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) - -def __migrate_prune_config(v7_toml_data, v9_toml_data): - """ - prune_config is optional, so we skip if it does not exist in v7. - """ - if "prune_config" not in v7_toml_data["node"]["database_setup"]: - return - - if "prune_config" not in v9_toml_data["node"]["database_setup"]: - raise SystemExit( - "Error: [node.database_setup.prune_config] table should exist in v9 template." - ) - - v9_prune_config = v9_toml_data["node"]["database_setup"]["prune_config"] - v7_prune_config = v7_toml_data["node"]["database_setup"]["prune_config"] - - print("\nScanning prune configuration ...") - scan_and_recommend_updates(v7_prune_config, v9_prune_config) - - -def __migrate_mempool_config(v7_toml_data, v9_toml_data): - - v9_mempool_config = v9_toml_data["mempool"] - v7_mempool_config = v7_toml_data["mempool"] - - print("\nScanning mempool configuration ...") - scan_and_recommend_updates(v7_mempool_config, v9_mempool_config) - -def __migrate_moonshot_config(v7_toml_data, v9_toml_data): - v9_moonshot_config = v9_toml_data["moonshot"] - v7_moonshot_config = v7_toml_data["moonshot"] - - print("\nScanning moonshot configuration ...") - scan_and_recommend_updates(v7_moonshot_config, v9_moonshot_config) - -def migrate_v7_to_v9(toml_data): - """ - Returns a new TOML data structure that is compatible with SMR settings v9. - """ - with ( - importlib.resources.files(__package__) - .joinpath("smr_settings_v9_1_x_mainnet_template.toml") - .open("r") as f - ): - template = f.read() - v9_toml_data = tomlkit.parse(template) - __migrate_node_root_config(toml_data, v9_toml_data) - __migrate_db_ledger(toml_data, v9_toml_data) - __migrate_db_chain_store(toml_data, v9_toml_data) - __migrate_snapshot_config(toml_data, v9_toml_data) - __migrate_prune_config(toml_data, v9_toml_data) - __migrate_mempool_config(toml_data, v9_toml_data) - __migrate_moonshot_config(toml_data, v9_toml_data) - return v9_toml_data diff --git a/node_management/migration/src/smr_settings/migrate_path.py b/node_management/migration/src/smr_settings/migrate_path.py deleted file mode 100644 index 829da3a5..00000000 --- a/node_management/migration/src/smr_settings/migrate_path.py +++ /dev/null @@ -1,14 +0,0 @@ -from .from_v7_to_v9 import migrate_v7_to_v9 - -from common import migration - - -SMR_SETTINGS_MIGRATE_PATH = { - "v7-v9": [migrate_v7_to_v9], -} - - -def run_migration(migrate_path: str, from_path: str, to_path: str): - migration.Migration(SMR_SETTINGS_MIGRATE_PATH).migrate_config( - migrate_path, from_path, to_path - ) diff --git a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml b/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml deleted file mode 100644 index 3b65a2f5..00000000 --- a/node_management/migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml +++ /dev/null @@ -1,146 +0,0 @@ - -# Version: v9.1.x mainnet - -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. Paths are set relative -# to $SUPRA_HOME. - -# Core protocol parameters. -[instance] -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 4 hours. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 2 hours. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - -# Parameters related to the mempool. -[mempool] -# The maximum number of milliseconds that a node will wait before proposing a batch when it has -# at least one transaction to process. -max_batch_delay_ms = 500 -# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` -# then a batch will be proposed immediately. -max_batch_size_bytes = 5000000 -# The amount of time that a node will wait before repeating a sync request for a batch that it -# is missing. -sync_retry_delay_ms = 2000 -# The number of signers of the related batch certificate that a node should ask for a batch -# attempting to retry a sync request. -sync_retry_nodes = 3 - -# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. -[moonshot] -# The maximum number of milliseconds that the timestamp of a proposed block may be -# ahead of a node's local time when it attempts to vote for the block. Validators -# must wait until the timestamp of a certified block has passed before advancing to -# the next round and leaders must wait until the timestamp of the parent block has -# passed before proposing, so this limit prevents Byzantine leaders from forcing -# honest nodes to wait indefinitely by proposing blocks with timestamps that are -# arbitrarily far in the future. -block_recency_bound_ms = 500 -# Causes the node to stop producing blocks when there are no transactions to be -# processed. If all nodes set this value to `true` then the chain will not produce -# new blocks when there are no transactions to process, conserving disk space. -halt_block_production_when_no_txs = false -# The type of leader election function to use. This function generates a schedule that ensures -# that each node eventually succeeds every other. -leader_elector = "FairSuccession" -# The delay after which the block proposer will create a new block despite not having any -# payload items to propose. Denominated in ms. -max_block_delay_ms = 1250 -# The maximum number of batch availability certificates that may be included in a single -# consensus block. -max_payload_items_per_block = 50 -# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, -# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount -# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can -# become. This trade-off must be balanced. -message_recency_bound_rounds = 1000 -# The delay after which the node will try to repeat sync requests for missing blocks. -# Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 1000 -# The delay after which the node will send a Timeout message for its current Moonshot round, -# measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 3500 - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. Paths -# should be specified either as absolute paths or as relative to the `supra` binary. -# -# When running the node in Docker and working with a mounted directory, the paths must be specified -# with reference to the location of the mounter directory in the Docker filesystem rather than the -# host filesystem. The default settings provided below assume the default Docker configuration -# provided by the `manage_supra_nodes.sh` script. - -[node] -# The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 1 -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The port on which to listen for connections from the associated RPC node. Each validator -# may serve at most one RPC node. -rpc_access_port = 26000 - -[node.ws_server.certificates] -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./configs/ca_certificate.pem" -# The path to the TLS certificate for this node. -cert_path = "./configs/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "./configs/server_supra_key.pem" - -# Parameters for the blockchain database. -[node.database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./configs/smr_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters for the DKG database. -[node.database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./configs/ledger_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters related to database pruning. -[node.database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 - -# Parameters for the database snapshot service. -[node.database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 diff --git a/node_management/migration/tests/config_v7.1.8.toml b/node_management/migration/tests/config_v7.1.8.toml deleted file mode 100644 index 79649d19..00000000 --- a/node_management/migration/tests/config_v7.1.8.toml +++ /dev/null @@ -1,131 +0,0 @@ -# Version: v7.1.8 -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_instance.chain_id = 8 -# The length of an epoch in seconds. -chain_instance.epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 48 hours. -chain_instance.recurring_lockup_duration_secs = 172800 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 46 hours. -chain_instance.voting_duration_secs = 165600 -# Determines whether the network will start with a faucet, amongst other things. -chain_instance.is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -chain_instance.genesis_timestamp_microseconds = 1732060800000000 - - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "1.1.1.1:26000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = true -# The path to the TLS certificate for the connection with the attached validator. -consensus_client_cert_path = "./xyz/client_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -consensus_client_private_key_path = "./xyz/client_supra_key.pem" -# The path to the TLS root certificate authority certificate. -consensus_root_ca_cert_path = "./xyz/ca_certificate.pem" -# The websocket address of the attached validator. -consensus_rpc = "ws://2.2.2.2:26000" -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./xyz/supra_committees.json" -# The number of seconds to wait before retrying a block sync request. -sync_retry_interval_in_secs = 1 - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./xyz/rpc_archive" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./xyz/rpc_ledger" - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./xyz/rpc_store" -# Whether snapshots should be taken of the database. -enable_snapshots = true - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./xyz/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retring a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. -[[allowed_origin]] -url = "https://rpc-mainnet.supra.com" -description = "RPC For Supra" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-wallet-mainnet.supra.com" -description = "RPC For Supra Wallet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-suprascan-mainnet.supra.com" -description = "RPC For suprascan" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:27000" -description = "LocalNet" -mode = "Server" \ No newline at end of file diff --git a/node_management/migration/tests/config_v7_to_v9_expected.toml b/node_management/migration/tests/config_v7_to_v9_expected.toml deleted file mode 100644 index 71b4040c..00000000 --- a/node_management/migration/tests/config_v7_to_v9_expected.toml +++ /dev/null @@ -1,154 +0,0 @@ -# Version: v9.1.x mainnet - -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. - -# Core protocol parameters. -# The below parameters are node-specific and may be configured as required by the operator. - -# The port on which the node should listen for incoming RPC requests. -bind_addr = "1.1.1.1:26000" -# If `true` then blocks will not be verified before execution. This value should be `false` -# unless you also control the node from which this RPC node is receiving blocks. -block_provider_is_trusted = true -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. -resume = true -# The path to `supra_committees.json`. -supra_committees_config = "./xyz/supra_committees.json" -# The access tokens used to authenticate public RPC requests to this RPC node. -consensus_access_tokens = [] - -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -[chain_instance] -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - - -######################################### NODE PARAMETERS ######################################### -[chain_state_assembler] -# Number of certified blocks stored in memory as reference to pending blocks to be executed. -# Only this amount of certified blocks are stored in memory, for the rest memo is kept. -certified_block_cache_bucket_size = 50 -# Retry interval for the sync requests for which no response yet available. -sync_retry_interval_in_secs = 1 - -[synchronization.ws] -# The websocket address of the attached validator. -consensus_rpc = "ws://2.2.2.2:26000" - -[synchronization.ws.certificates] -# The path to the TLS certificate for the connection with the attached validator. -cert_path = "./xyz/client_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "./xyz/client_supra_key.pem" -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./xyz/ca_certificate.pem" - -# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. -[database_setup.dbs.archive.rocks_db] -# The path at which the database should be created. -path = "./xyz/rpc_archive" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = false -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters for the DKG database. -[database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./xyz/rpc_ledger" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = false -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters for the blockchain database. -[database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./xyz/rpc_store" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = false -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters related to database pruning. -[database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 336 - -# Parameters for the database snapshot service. -[database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./xyz/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 - -# CORS settings for RPC API requests. -# The below settings are the default values required for use in RPC nodes run by validator node operators. -# They are optional for non-validators. -[[allowed_origin]] -url = "https://rpc-mainnet.supra.com" -description = "RPC For Supra" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet1.supra.com" -description = "RPC For nodeops group1" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet2.supra.com" -description = "RPC For nodeops group2" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet3.supra.com" -description = "RPC For nodeops group3" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet4.supra.com" -description = "RPC For nodeops group4" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-mainnet5.supra.com" -description = "RPC For nodeops group5" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-wallet-mainnet.supra.com" -description = "RPC For Supra Wallet" -mode = "Server" - -[[allowed_origin]] -url = "https://rpc-suprascan-mainnet.supra.com" -description = "RPC For suprascan" -mode = "Server" - -[[allowed_origin]] -url = "http://localhost:26000" -description = "LocalNet" -mode = "Server" \ No newline at end of file diff --git a/node_management/migration/tests/smr_settings_v7.1.x.toml b/node_management/migration/tests/smr_settings_v7.1.x.toml deleted file mode 100644 index 355aa86a..00000000 --- a/node_management/migration/tests/smr_settings_v7.1.x.toml +++ /dev/null @@ -1,167 +0,0 @@ -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. Paths are set relative -# to $SUPRA_HOME. - -# Core protocol parameters. -[instance] -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 48 hours. -recurring_lockup_duration_secs = 172800 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 46 hours. -voting_duration_secs = 165600 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - -# Parameters related to the mempool. -[mempool] -# The maximum number of milliseconds that a node will wait before proposing a batch when it has -# at least one transaction to process. -max_batch_delay_ms = 1500 -# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` -# then a batch will be proposed immediately. -max_batch_size_bytes = 5000000 -# The amount of time that a node will wait before repeating a sync request for a batch that it -# is missing. -sync_retry_delay_ms = 2000 -# The number of signers of the related batch certificate that a node should ask for a batch -# attempting to retry a sync request. -sync_retry_nodes = 3 - -# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. -[moonshot] -# The maximum number of milliseconds that the timestamp of a proposed block may be -# ahead of a node's local time when it attempts to vote for the block. Validators -# must wait until the timestamp of a certified block has passed before advancing to -# the next round and leaders must wait until the timestamp of the parent block has -# passed before proposing, so this limit prevents Byzantine leaders from forcing -# honest nodes to wait indefinitely by proposing blocks with timestamps that are -# arbitrarily far in the future. -block_recency_bound_ms = 500 -# Causes the node to stop producing blocks when there are no transactions to be -# processed. If all nodes set this value to `true` then the chain will not produce -# new blocks when there are no transactions to process, conserving disk space. -halt_block_production_when_no_txs = false -# The type of leader election function to use. This function generates a schedule that ensures -# that each node eventually succeeds every other. -leader_elector = "FairSuccession" -# The delay after which the block proposer will create a new block despite not having any -# payload items to propose. Denominated in ms. -max_block_delay_ms = 1250 -# The maximum number of batch availability certificates that may be included in a single -# consensus block. -max_payload_items_per_block = 50 -# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, -# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount -# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can -# become. This trade-off must be balanced. -message_recency_bound_rounds = 20 -# The delay after which the node will try to repeat sync requests for missing blocks. -# Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 2000 -# The delay after which the node will send a Timeout message for its current Moonshot round, -# measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 5000 - -# Parameters related to the MoveVM. Primarily related to governance features. -[move_vm] -# Initially `false` until the network matures. -allow_new_validators = false -# The maximum stake that may be allocated to a Supra Validator. We are not currently doing -# stake-weighted voting, so this value does not impact our decentralization quotient. This -# may change in the future. Initially set to the total supply. -# -# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 100_000_000_000 SUPRA. -max_stake = "10000000000000000000" -# The minimum stake required to run a Supra Validator. There is no minimum at genesis -# because The Foundation's stake is added after the stake pools are created. We will increase -# it to its intended value of 55M SUPRA via governance. New nodes will not be able to join the -# validator set before the update is made. Measured in Quants (1 Quant = 10^-8 SUPRA). -min_stake = 0 -# The number of tokens initially allocated to node operators. Tokens will be earned through block -# rewards. -operator_account_balance = 0 -# The amount of Quants to transfer from each validator owner account to the corresponding stake pool -# after it has been created, during the genesis transaction. Adding the stake after creating the -# pool allows us to ensure that The Foundation's stake is not subject to the PBO locking schedule, -# which is only intended to apply to winners of the Project Blast Off campaign. -# -# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 55_000_000 SUPRA. -pbo_owner_stake = 5500000000000000 -# The number of seconds after `genesis_timestamp_microseconds` at which all accounts with -# allocations at genesis will be able to unlock their initial amounts. These amounts are stored -# in vesting contracts. -# -# Corresponds to Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -remaining_balance_lockup_cliff_period_in_seconds = 0 -# The amount of SUPRA required to qualify as a proposer (this parameter is currently unused). -required_proposer_stake = 0 -# The annual percent yield for validators, proportional to their stake. Specified as a percentage -# with 2 decimals of precision in u64 format due to limitations in the MoveVM. The below value -# represents 12.85%. -rewards_apy_percentage = 1285 -# The percentage of staking rewards earned by Supra Foundation controlled nodes that will be paid -# to the corresponding node operators. Specified as a percentage with 2 decimals of precision in -# u64 format due to limitations in the MoveVM. The below value represents 37.74%. -validator_commission_rate_percentage = 3774 -# The percentage of new stake relative to the current total stake that can join the validator -# set or be added to existing validators within a single epoch. -voting_power_increase_limit = 33 - - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. Paths -# should be specified either as absolute paths or as relative to the `supra` binary. -# -# When running the node in Docker and working with a mounted directory, the paths must be specified -# with reference to the location of the mounter directory in the Docker filesystem rather than the -# host filesystem. The default settings provided below assume the default Docker configuration -# provided by the `manage_supra_nodes.sh` script. - -[node] -# The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 2 -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./tests/ca_certificate.pem" -# The port on which to listen for connections from the associated RPC node. Each validator -# may serve at most one RPC node. -rpc_access_port = 26000 -# The path to the TLS certificate for this node. -server_cert_path = "./tests/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -server_private_key_path = "./tests/server_supra_key.pem" - -# Parameters for the blockchain database. -[node.database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./tests/smr_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true - -# Parameters for the DKG database. -[node.database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./tests/ledger_storage" - -# Parameters related to database pruning. -[node.database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 diff --git a/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml b/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml deleted file mode 100644 index eef5ffb5..00000000 --- a/node_management/migration/tests/smr_settings_v7_to_v9_expected.toml +++ /dev/null @@ -1,145 +0,0 @@ -# Version: v9.1.x mainnet - -####################################### PROTOCOL PARAMETERS ####################################### - -# The below parameters are fixed for the protocol and must be agreed upon by all node operators -# at genesis. They may subsequently be updated via governance decisions. Paths are set relative -# to $SUPRA_HOME. - -# Core protocol parameters. -[instance] -# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. -chain_id = 8 -# The length of an epoch in seconds. -epoch_duration_secs = 7200 -# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when -# its current lockup expires, if no request is made to unlock it. -# -# 4 hours. -recurring_lockup_duration_secs = 14400 -# The number of seconds allocated for voting on governance proposals. Governance will initially be -# controlled by The Supra Foundation. -# -# 2 hours. -voting_duration_secs = 7200 -# Determines whether the network will start with a faucet, amongst other things. -is_testnet = false -# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). -genesis_timestamp_microseconds = 1732060800000000 - -# Parameters related to the mempool. -[mempool] -# The maximum number of milliseconds that a node will wait before proposing a batch when it has -# at least one transaction to process. -max_batch_delay_ms = 500 -# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` -# then a batch will be proposed immediately. -max_batch_size_bytes = 5000000 -# The amount of time that a node will wait before repeating a sync request for a batch that it -# is missing. -sync_retry_delay_ms = 2000 -# The number of signers of the related batch certificate that a node should ask for a batch -# attempting to retry a sync request. -sync_retry_nodes = 3 - -# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. -[moonshot] -# The maximum number of milliseconds that the timestamp of a proposed block may be -# ahead of a node's local time when it attempts to vote for the block. Validators -# must wait until the timestamp of a certified block has passed before advancing to -# the next round and leaders must wait until the timestamp of the parent block has -# passed before proposing, so this limit prevents Byzantine leaders from forcing -# honest nodes to wait indefinitely by proposing blocks with timestamps that are -# arbitrarily far in the future. -block_recency_bound_ms = 500 -# Causes the node to stop producing blocks when there are no transactions to be -# processed. If all nodes set this value to `true` then the chain will not produce -# new blocks when there are no transactions to process, conserving disk space. -halt_block_production_when_no_txs = false -# The type of leader election function to use. This function generates a schedule that ensures -# that each node eventually succeeds every other. -leader_elector = "FairSuccession" -# The delay after which the block proposer will create a new block despite not having any -# payload items to propose. Denominated in ms. -max_block_delay_ms = 1250 -# The maximum number of batch availability certificates that may be included in a single -# consensus block. -max_payload_items_per_block = 50 -# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, -# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount -# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can -# become. This trade-off must be balanced. -message_recency_bound_rounds = 1000 -# The delay after which the node will try to repeat sync requests for missing blocks. -# Denominated in ms. Should be the same for all nodes. -sync_retry_delay_ms = 1000 -# The delay after which the node will send a Timeout message for its current Moonshot round, -# measured from the start of the round. Denominated in ms. Must be the same for all nodes. -timeout_delay_ms = 3500 - -######################################### NODE PARAMETERS ######################################### - -# The below parameters are node-specific and may be configured as required by the operator. Paths -# should be specified either as absolute paths or as relative to the `supra` binary. -# -# When running the node in Docker and working with a mounted directory, the paths must be specified -# with reference to the location of the mounter directory in the Docker filesystem rather than the -# host filesystem. The default settings provided below assume the default Docker configuration -# provided by the `manage_supra_nodes.sh` script. - -[node] -# The duration in seconds that a node waits between polling its connections to its peers. -connection_refresh_timeout_sec = 1 -# If true, all components will attempt to load their previous state from disk. Otherwise, -# all components will start in their default state. Should always be `true` for testnet and -# mainnet. -resume = true -# The port on which to listen for connections from the associated RPC node. Each validator -# may serve at most one RPC node. -rpc_access_port = 26000 - -[node.ws_server.certificates] -# The path to the TLS root certificate authority certificate. -root_ca_cert_path = "./tests/ca_certificate.pem" -# The path to the TLS certificate for this node. -cert_path = "./tests/server_supra_certificate.pem" -# The path to the private key to be used when negotiating TLS connections. -private_key_path = "./tests/server_supra_key.pem" - -# Parameters for the blockchain database. -[node.database_setup.dbs.chain_store.rocks_db] -# The path at which the database should be created. -path = "./tests/smr_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters for the DKG database. -[node.database_setup.dbs.ledger.rocks_db] -# The path at which the database should be created. -path = "./tests/ledger_storage" -# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` -# old will be deleted. -enable_pruning = true -# Whether snapshots should be taken of the database. -enable_snapshots = false - -# Parameters related to database pruning. -[node.database_setup.prune_config] -# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 84 - -# Parameters for the database snapshot service. -[node.database_setup.snapshot_config] -# The number of snapshots to retain, including the latest. -depth = 2 -# The interval between snapshots in seconds. -interval_in_seconds = 1800 -# The path at which the snapshots should be stored. -path = "./configs/snapshot" -# The number of times to retry a snapshot in the event that it fails unexpectedly. -retry_count = 3 -# The interval in seconds to wait before retrying a snapshot. -retry_interval_in_seconds = 5 diff --git a/node_management/migration/tests/test_rpc_migration.py b/node_management/migration/tests/test_rpc_migration.py deleted file mode 100644 index 1344a480..00000000 --- a/node_management/migration/tests/test_rpc_migration.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import shutil -import tomlkit -import pytest -from rpc_config.migrate_path import run_migration as rpc_run_migration - -# CONFIGS = [ -# "config_v7.1.8.toml", -# "config_v8.0.2.toml", -# "config_v9.0.7.toml", -# "config_v9.1_mainnet.toml", -# ] - - -@pytest.mark.parametrize( - "from_file,to_file,migrate_path", - [ - ("config_v7.1.8.toml", "config_v7_to_v9_expected.toml", "v7-v9"), - ], -) -def test_migration(tmp_path, from_file, to_file, migrate_path): - # Copy config files to temp dir - src_dir = os.path.dirname(os.path.abspath(__file__)) - from_path = os.path.join(src_dir, from_file) - to_path = os.path.join(src_dir, to_file) - tmp_from = tmp_path / from_file - tmp_to = tmp_path / to_file - shutil.copy(from_path, tmp_from) - # Run migration - import common.globals - common.globals.ASSUME_YES = True - rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) - # Load both files - with open(tmp_to, "r") as f: - migrated = tomlkit.parse(f.read()) - with open(to_path, "r") as f: - expected = tomlkit.parse(f.read()) - - # Compare TOML dicts - assert migrated == expected, ( - f"Migration {migrate_path} failed: {from_file} -> {to_file}" - ) diff --git a/node_management/migration/tests/test_smr_migration.py b/node_management/migration/tests/test_smr_migration.py deleted file mode 100644 index ef6eef8b..00000000 --- a/node_management/migration/tests/test_smr_migration.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import shutil -import tomlkit -import pytest -from smr_settings.migrate_path import run_migration as smr_run_migration - - -@pytest.mark.parametrize( - "from_file,to_file,migrate_path", - [ - ("smr_settings_v7.1.x.toml", "smr_settings_v7_to_v9_expected.toml", "v7-v9"), - ], -) -def test_migration(tmp_path, from_file, to_file, migrate_path): - # Copy config files to temp dir - src_dir = os.path.dirname(os.path.abspath(__file__)) - from_path = os.path.join(src_dir, from_file) - to_path = os.path.join(src_dir, to_file) - tmp_from = tmp_path / from_file - tmp_to = tmp_path / to_file - shutil.copy(from_path, tmp_from) - # Run migration - import common.globals - common.globals.ASSUME_YES = True - - smr_run_migration(migrate_path, str(tmp_from), str(tmp_to)) - # Load both files - with open(tmp_to, "r") as f: - migrated = tomlkit.parse(f.read()) - with open(to_path, "r") as f: - expected = tomlkit.parse(f.read()) - # Compare TOML dicts - assert migrated == expected, ( - f"Migration {migrate_path} failed: {from_file} -> {to_file}" - ) From 9c7f76af5462bf3007adf96e79a4b0210a9e428d Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 29 May 2025 10:52:08 +0200 Subject: [PATCH 32/44] add back --- node_management/config_migration/.gitignore | 13 ++ node_management/config_migration/README.md | 220 ++++++++++++++++++ node_management/config_migration/lazydog.md | 35 +++ .../config_migration/pyproject.toml | 27 +++ .../config_migration/src/cli/__init__.py | 0 .../config_migration/src/cli/main.py | 77 ++++++ .../config_migration/src/common/__init__ | 0 .../config_migration/src/common/globals.py | 1 + .../config_migration/src/common/migration.py | 95 ++++++++ .../config_migration/src/common/utils.py | 86 +++++++ .../src/rpc_config/__init__.py | 0 .../src/rpc_config/from_v7_to_v9.py | 155 ++++++++++++ .../src/rpc_config/migrate_path.py | 13 ++ .../rpc_config_v9_1_x_mainnet_template.toml | 155 ++++++++++++ .../src/smr_settings/__init__.py | 0 .../src/smr_settings/from_v7_to_v9.py | 129 ++++++++++ .../src/smr_settings/migrate_path.py | 14 ++ .../smr_settings_v9_1_x_mainnet_template.toml | 146 ++++++++++++ .../config_migration/tests/config_v7.1.8.toml | 131 +++++++++++ .../tests/config_v7_to_v9_expected.toml | 154 ++++++++++++ .../tests/smr_settings_v7.1.x.toml | 167 +++++++++++++ .../tests/smr_settings_v7_to_v9_expected.toml | 145 ++++++++++++ .../tests/test_rpc_migration.py | 42 ++++ .../tests/test_smr_migration.py | 35 +++ 24 files changed, 1840 insertions(+) create mode 100644 node_management/config_migration/.gitignore create mode 100644 node_management/config_migration/README.md create mode 100644 node_management/config_migration/lazydog.md create mode 100644 node_management/config_migration/pyproject.toml create mode 100644 node_management/config_migration/src/cli/__init__.py create mode 100644 node_management/config_migration/src/cli/main.py create mode 100644 node_management/config_migration/src/common/__init__ create mode 100644 node_management/config_migration/src/common/globals.py create mode 100644 node_management/config_migration/src/common/migration.py create mode 100644 node_management/config_migration/src/common/utils.py create mode 100644 node_management/config_migration/src/rpc_config/__init__.py create mode 100644 node_management/config_migration/src/rpc_config/from_v7_to_v9.py create mode 100644 node_management/config_migration/src/rpc_config/migrate_path.py create mode 100644 node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml create mode 100644 node_management/config_migration/src/smr_settings/__init__.py create mode 100644 node_management/config_migration/src/smr_settings/from_v7_to_v9.py create mode 100644 node_management/config_migration/src/smr_settings/migrate_path.py create mode 100644 node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml create mode 100644 node_management/config_migration/tests/config_v7.1.8.toml create mode 100644 node_management/config_migration/tests/config_v7_to_v9_expected.toml create mode 100644 node_management/config_migration/tests/smr_settings_v7.1.x.toml create mode 100644 node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml create mode 100644 node_management/config_migration/tests/test_rpc_migration.py create mode 100644 node_management/config_migration/tests/test_smr_migration.py diff --git a/node_management/config_migration/.gitignore b/node_management/config_migration/.gitignore new file mode 100644 index 00000000..33a954f4 --- /dev/null +++ b/node_management/config_migration/.gitignore @@ -0,0 +1,13 @@ +# Ignore all files in the ruff cache directory +**/.ruff_cache + +**/build/ + +# egg-info +**/*.egg-info/ + +# python cache +**/__pycache__/ + +# pytest +**/.hypothesis/ \ No newline at end of file diff --git a/node_management/config_migration/README.md b/node_management/config_migration/README.md new file mode 100644 index 00000000..aae89263 --- /dev/null +++ b/node_management/config_migration/README.md @@ -0,0 +1,220 @@ + +### Usage Example + + +alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node +alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node + +alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra" +alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra" + + +1. Install the `migrate-config` tool + +```sh +pip install . +``` + +2. Migrate rpc + +```sh +# Migrate rpc config from v7 to v9 +migrate-config rpc -p v7-v9 -f config.toml -t config.toml +# Migrate db from v7 to v8 +rpc-v8 migrate-db config.toml +# Migrate db from v8 to v9 +rpc-v9 migrate-db config.toml +``` + +3. Migrate smr/validator + +```sh +# Migrate cli profile from v7 to v8 +supra-v8 migrate --network localnet +cp validator_identity.pem node_identity.pem +# Migrate cli profile from v8 to v9 +supra-v9 profile migrate + +# Migrate smr_settings from v7 to v9 +migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml +# Migrate db from v7 to v9 +supra-v9 data migrate -p smr_settings.toml +``` + + +### Migrate from v7 to v9 example flow + +#### RPC + +1. Config migration + +$ migrate-config rpc -p v7-v9 -f config.toml -t config.toml + + Running migration function: migrate_v7_to_v9 + + Scanning root level configuration ... + ✓ `consensus_access_tokens` not found in original config, using new version's default value: [] + `allowed_origin = block_to_tx_ordered: Migrated 8537 records, up to 239 block height + certified_block -> certified_block_dehydrated: Migrated 244 records, up to 244 block height + databases_checked: + - chain_store + - archive + +#### SMR + +1. Profile/Identity migration + +**You need both v8 and v9 supra binary to run the profile migratino.** + +$ supra-v8 migrate --network mainnet + +$ cp validator_identity.pem node_identity.pem + +$ supra-v9 profile migrate + +2. Config migration + +$ migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml + + Running migration function: migrate_v7_to_v9 + + Scanning node root configuration ... + `connection_refresh_timeout_sec = 2` is not recommended for new version. + Do you want to apply the recommended config: `connection_refresh_timeout_sec = 1`? [y/N]: y + ✓ Apply recommended config: `connection_refresh_timeout_sec = 1` + + Scanning ledger configuration ... + ✓ `enable_pruning` not found in original config, using new version's default value: True + ✓ `enable_snapshots` not found in original config, using new version's default value: False + + Scanning chain store configuration ... + ✓ `enable_snapshots` not found in original config, using new version's default value: False + + Scanning prune configuration ... + + Scanning mempool configuration ... + `max_batch_delay_ms = 1500` is not recommended for new version. + Do you want to apply the recommended config: `max_batch_delay_ms = 500`? [y/N]: y + ✓ Apply recommended config: `max_batch_delay_ms = 500` + + Scanning moonshot configuration ... + `message_recency_bound_rounds = 20` is not recommended for new version. + Do you want to apply the recommended config: `message_recency_bound_rounds = 1000`? [y/N]: y + ✓ Apply recommended config: `message_recency_bound_rounds = 1000` + `sync_retry_delay_ms = 2000` is not recommended for new version. + Do you want to apply the recommended config: `sync_retry_delay_ms = 1000`? [y/N]: y + ✓ Apply recommended config: `sync_retry_delay_ms = 1000` + `timeout_delay_ms = 5000` is not recommended for new version. + Do you want to apply the recommended config: `timeout_delay_ms = 3500`? [y/N]: y + ✓ Apply recommended config: `timeout_delay_ms = 3500` + Writing new config to /tmp/new_smr.toml + |----------------- Begin diff v7 vs v9 -----------------| + --- v7 + +++ v9 + .. + |----------------- End diff v7 vs v9 -----------------| + + ###################################################################### + # Config migrated from tests/smr_settings_v7.1.x.toml to /tmp/new_smr.toml. + # + # Please review the diff above for changes made during migration. + # + # Please ensure to use the new config file for target binary version. + ###################################################################### + +3. DB migration + +**You need v9 supra binary to run the db migration.** + +$ supra-v9 data migrate -p smr_settings.toml + + Counting the number of entries in certified_block... + Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 69/69 00:00:00 + Counting the number of entries in uncommitted_block... + Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 74/74 00:00:00 + Cleaning up uncommitted_block: [00:00:00] ████████████████████ 4/4 00:00:00 + Counting the number of entries in certified_block... + Counting the number of entries in certified_block_dehydrated... + Counting the number of entries in uncommitted_block... + Counting the number of entries in qc... + Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 70/70 00:00:00 + Counting the number of entries to remove from prune_index... + Cleaning up prune index: [00:00:00] ████████████████████ 70/70 00:00:00 + dropped: + - certified_block + migrated: + certified_block -> certified_block_dehydrated: Migrated 70 records, up to 244 block height + databases_checked: + - chain_store + + +### Test harness of migrate-config script + +`PYTHONPATH=src pytest` \ No newline at end of file diff --git a/node_management/config_migration/lazydog.md b/node_management/config_migration/lazydog.md new file mode 100644 index 00000000..d80c8dde --- /dev/null +++ b/node_management/config_migration/lazydog.md @@ -0,0 +1,35 @@ +alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node +alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node +alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra" +alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra" +export NODE_OPERATOR_KEY_PASSWORD="88888888" + + +# SMR +supra-v8 migrate --network localnet + + +cp validator_identity.pem node_identity.pem +supra-v9 profile migrate +sed -i '' 's/is_testnet = true/is_testnet = false/' smr_settings.toml +migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml + + +lsof -n -P -iTCP:26002-26006 -sTCP:LISTEN + +supra-v9 data migrate -p smr_settings.toml + + +supra-v9 node smr run + + +### RPC +sed -i '' 's/is_testnet = true/is_testnet = false/' config.toml +migrate-config rpc -p v7-v9 -f config.toml -t config.toml + + + +rpc-v8 migrate-db config.toml + +rpc-v9 migrate-db config.toml +rpc-v9 start & \ No newline at end of file diff --git a/node_management/config_migration/pyproject.toml b/node_management/config_migration/pyproject.toml new file mode 100644 index 00000000..abc3a956 --- /dev/null +++ b/node_management/config_migration/pyproject.toml @@ -0,0 +1,27 @@ +[project] +name = "migrate-config" +version = "0.1.0" +description = "Unified CLI tool to migrate Supra RPC and SMR configs." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "tomlkit>=0.13.2", + "click>=8.0.0" +] + +[project.scripts] +migrate-config = "cli.main:main" + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +rpc_config = ["rpc_config_v9_1_x_mainnet_template.toml"] +smr_settings = ["smr_settings_v9_1_x_mainnet_template.toml"] diff --git a/node_management/config_migration/src/cli/__init__.py b/node_management/config_migration/src/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/node_management/config_migration/src/cli/main.py b/node_management/config_migration/src/cli/main.py new file mode 100644 index 00000000..29efbfe5 --- /dev/null +++ b/node_management/config_migration/src/cli/main.py @@ -0,0 +1,77 @@ +import click +from rpc_config.migrate_path import RPC_CONFIG_MIGRATE_PATH +from rpc_config.migrate_path import run_migration as migrate_rpc_config +from smr_settings.migrate_path import SMR_SETTINGS_MIGRATE_PATH +from smr_settings.migrate_path import run_migration as migrate_smr_config +import common.globals + + +@click.group() +def main(): + """Migration CLI for Supra configs.""" + + +@main.command() +@click.option( + "--migrate-path", + "-p", + required=True, + type=click.Choice(RPC_CONFIG_MIGRATE_PATH, case_sensitive=True), + help=f"Migration path (choices: {', '.join(RPC_CONFIG_MIGRATE_PATH)})", +) +@click.option( + "--from-file", + "-f", + required=True, + type=click.Path(exists=True), + help="Source config file", +) +@click.option( + "--to-file", "-t", required=True, type=click.Path(), help="Output config file" +) +@click.option( + "--assume-yes", + "-y", + is_flag=True, + default=False, + help="Assume yes for all prompts (default: False)", +) +def rpc(migrate_path, from_file, to_file, assume_yes): + """Migrate RPC config.""" + common.globals.ASSUME_YES = assume_yes + migrate_rpc_config(migrate_path, from_file, to_file) + + +@main.command() +@click.option( + "--migrate-path", + "-p", + required=True, + type=click.Choice(SMR_SETTINGS_MIGRATE_PATH, case_sensitive=True), + help=f"Migration path (choices: {', '.join(SMR_SETTINGS_MIGRATE_PATH)})", +) +@click.option( + "--from-file", + "-f", + required=True, + type=click.Path(exists=True), + help="Source config file", +) +@click.option( + "--to-file", "-t", required=True, type=click.Path(), help="Output config file" +) +@click.option( + "--assume-yes", + "-y", + is_flag=True, + default=False, + help="Assume yes for all prompts (default: False)", +) +def smr(migrate_path, from_file, to_file, assume_yes): + """Migrate SMR config.""" + common.globals.ASSUME_YES = assume_yes + migrate_smr_config(migrate_path, from_file, to_file) + + +if __name__ == "__main__": + main() diff --git a/node_management/config_migration/src/common/__init__ b/node_management/config_migration/src/common/__init__ new file mode 100644 index 00000000..e69de29b diff --git a/node_management/config_migration/src/common/globals.py b/node_management/config_migration/src/common/globals.py new file mode 100644 index 00000000..9b8bd68b --- /dev/null +++ b/node_management/config_migration/src/common/globals.py @@ -0,0 +1 @@ +ASSUME_YES = False diff --git a/node_management/config_migration/src/common/migration.py b/node_management/config_migration/src/common/migration.py new file mode 100644 index 00000000..18efc502 --- /dev/null +++ b/node_management/config_migration/src/common/migration.py @@ -0,0 +1,95 @@ +import typing as ty +import tomlkit +from copy import deepcopy +from . import utils +import shutil +from common.globals import ASSUME_YES + + +class MigrationPathSet: + """ + Base class for migration paths. + """ + + def __init__(self, migrate_paths: ty.Dict[str, ty.List[ty.Callable]]): + self.migrate_paths = migrate_paths + + def get_versions(self, key: str) -> ty.Tuple[str, str]: + """Split the key into from_version and to_version.""" + if key not in self.migrate_paths: + raise ValueError(f"Invalid key: {key}") + from_version, to_version = key.split("-", 1) + return from_version, to_version + + def get_migration_functions(self, key: str) -> ty.List[ty.Callable]: + """Get the migration functions for the given key.""" + if key not in self.migrate_paths: + raise ValueError(f"Unknown migration path: {key}") + return self.migrate_paths[key] + + +class Migration: + """ + Top level migration class that handles the migration of config files. + """ + + def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]): + self.migrate_path = MigrationPathSet(migrate_path) + + def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): + migrate_functions = self.migrate_path.get_migration_functions(migrate_choice) + from_version, to_version = self.migrate_path.get_versions(migrate_choice) + default_backup_path = f"{from_path}_{from_version}.bak" + + if from_path == to_path: + print( + f"Warning: The source and destination paths are the same ({from_path})." + ) + print( + f"A backup of your original config will be saved to: {default_backup_path}" + ) + confirm = utils.prompt_or_assume_yes( + "This will overwrite your original config file. Continue?", ASSUME_YES + ) + if not confirm: + print("Aborted by user.") + return + # Backup old config + print(f"Backing up old config to {default_backup_path}") + shutil.copyfile(from_path, default_backup_path) + + with open(from_path, "r") as f: + toml_data = tomlkit.load(f) + + original_toml_data = deepcopy(toml_data) + + for fn in migrate_functions: + print(f"Running migration function: {fn.__name__}") + toml_data = fn(toml_data) + + # Write new config + print(f"Writing new config to {to_path}") + with open(to_path, "w") as f: + tomlkit.dump(toml_data, f) + # Print the diff + from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True) + to_str = tomlkit.dumps(toml_data).splitlines(keepends=True) + + utils.unified_diff( + from_str, + to_str, + fromfile=from_version, + tofile=to_version, + ) + + print( + f""" + ###################################################################### + # Config migrated from {from_path} to {to_path}. + # + # Please review the diff above for changes made during migration. + # + # Please ensure to use the new config file for target binary version. + ###################################################################### + """ + ) diff --git a/node_management/config_migration/src/common/utils.py b/node_management/config_migration/src/common/utils.py new file mode 100644 index 00000000..f10c7130 --- /dev/null +++ b/node_management/config_migration/src/common/utils.py @@ -0,0 +1,86 @@ +import difflib +import tomlkit + + +def unified_diff(from_str, to_str, fromfile, tofile): + print(f"|----------------- Begin diff {fromfile} vs {tofile} -----------------|") + diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile) + __print_colored_diff(diff) + print(f"|----------------- End diff {fromfile} vs {tofile} -----------------|") + + +def __print_colored_diff(diff): + # The color is added here manually using ANSI escape codes. + for line in diff: + if line.startswith("+") and not line.startswith("+++"): + print(f"\033[32m{line}\033[0m", end="") # Green for additions + elif line.startswith("-") and not line.startswith("---"): + print(f"\033[31m{line}\033[0m", end="") # Red for deletions + elif line.startswith("@@"): + print(f"\033[36m{line}\033[0m", end="") # Cyan for hunk headers + else: + print(line, end="") + + +def print_with_checkmark(message): + """ + Print a message with a checkmark. + """ + print(f"✓ {message}") + + +def prompt_or_assume_yes(message, assume_yes=False) -> bool: + """ + Prompt the user for confirmation or assume 'yes' for all prompts + """ + if not assume_yes: + response = input(f"{message} [y/N]: ").strip().lower() + return response in ("y", "yes") + else: + print(f"{message} (assuming yes)") + return True + + +def truncate(val, max_len=50): + """ + Truncate a string representation of a value to a maximum length. + """ + s = str(val) + return s if len(s) <= max_len else s[: max_len - 3] + "..." + + +def scan_and_recommend_updates( + original_table: tomlkit.items.Table, to_table: tomlkit.items.Table +): + """ + Scan the original table and recommend updates to the new version's table. + + Only scan top level keys and values (i.e. skip nested tables). + + If a key exists in both tables, compare their values and prompt the user + to either keep the original value or use the new version's recommended value. + """ + from .globals import ASSUME_YES + + for k, v in to_table.items(): + if not isinstance(v, tomlkit.items.AbstractTable): + if k in original_table: + if to_table[k] != original_table[k]: + use_recommended = prompt_or_assume_yes( + f"`{k} = {truncate(original_table[k])}` is not recommended for new version.\n" + f"Do you want to apply the recommended config: `{k} = {truncate(to_table[k])}`?", + ASSUME_YES, + ) + if use_recommended: + print_with_checkmark( + f"Apply recommended config: `{k} = {truncate(to_table[k])}`" + ) + else: + print_with_checkmark( + f"Keep original config: `{k} = {truncate(original_table[k])}`" + ) + to_table[k] = original_table[k] + else: + print_with_checkmark( + f"`{k}` not found in original config, using new version's default value: {truncate(v)}" + ) diff --git a/node_management/config_migration/src/rpc_config/__init__.py b/node_management/config_migration/src/rpc_config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/node_management/config_migration/src/rpc_config/from_v7_to_v9.py b/node_management/config_migration/src/rpc_config/from_v7_to_v9.py new file mode 100644 index 00000000..43da1458 --- /dev/null +++ b/node_management/config_migration/src/rpc_config/from_v7_to_v9.py @@ -0,0 +1,155 @@ +""" +This module provides migration utilities to upgrade Supra's RPC configuration files from version v7 to v9. + +Migration Steps: +---------------- +- Loads a v9 template configuration as the migration base. +- Migrates root-level fields such as 'supra_committees_config' and 'bind_addr'. +- Migrates synchronization WebSocket settings, including consensus RPC and certificate paths. +- Migrates chain state assembler configuration. +- Migrates database paths for archive, chain store, and ledger components. +- Migrates snapshot configuration paths. +- For each section, scans and recommends updates for any legacy fields. +- Ensures that deprecated or unexpected sections in v7 (e.g., 'synchronization', 'chain_state_assembler') are flagged as errors. + +The main entrypoint is `migrate_v7_to_v9(v7_toml_data)`, which returns a new TOML data structure compatible with v9. +This module provides migration functions to upgrade Supra's RPC configuration from version v7 to v9. + +""" + +import tomlkit +import importlib.resources +from common.utils import ( + scan_and_recommend_updates, +) + + +def __migrate_root_config(v7_toml_data, v9_toml_data): + v9_toml_data["supra_committees_config"] = v7_toml_data["supra_committees_config"] + + v9_toml_data["bind_addr"] = v7_toml_data["bind_addr"] + + print("\nScanning root level configuration ...") + scan_and_recommend_updates(v7_toml_data, v9_toml_data) + + +def __migrate_sync_ws_config(v7_toml_data, v9_toml_data): + if "synchronization" in v7_toml_data: + raise SystemExit( + "Error: [synchronization] table should not exist in v7 config. Please check your migration path matching the version of your config file." + ) + v9_sync_ws_config = v9_toml_data["synchronization"]["ws"] + v9_sync_ws_config["consensus_rpc"] = v7_toml_data["consensus_rpc"] + + v9_sync_ws_certificates = v9_sync_ws_config["certificates"] + v9_sync_ws_certificates["cert_path"] = v7_toml_data["consensus_client_cert_path"] + v9_sync_ws_certificates["private_key_path"] = v7_toml_data[ + "consensus_client_private_key_path" + ] + v9_sync_ws_certificates["root_ca_cert_path"] = v7_toml_data[ + "consensus_root_ca_cert_path" + ] + + +def __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data): + if "chain_state_assembler" in v7_toml_data: + raise SystemExit( + "Error: [chain_state_assembler] table should not exist in v7 config." + ) + + print("\nScanning chain state assembler configuration ...") + scan_and_recommend_updates(v7_toml_data, v9_toml_data["chain_state_assembler"]) + + +def __migrate_db_archive_config(v7_toml_data, v9_toml_data): + v9_db_archive_config = v9_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"] + v7_db_archive_config = v7_toml_data["database_setup"]["dbs"]["archive"]["rocks_db"] + + v9_db_archive_config["path"] = v7_db_archive_config["path"] + + print("\nScanning archive configuration ...") + scan_and_recommend_updates(v7_db_archive_config, v9_db_archive_config) + + +def __migrate_db_chain_store_config(v7_toml_data, v9_toml_data): + v9_db_chain_store_config = v9_toml_data["database_setup"]["dbs"]["chain_store"][ + "rocks_db" + ] + v7_db_chain_store_config = v7_toml_data["database_setup"]["dbs"]["chain_store"][ + "rocks_db" + ] + + v9_db_chain_store_config["path"] = v7_db_chain_store_config["path"] + + print("\nScanning chain store configuration ...") + scan_and_recommend_updates(v7_db_chain_store_config, v9_db_chain_store_config) + + +def __migrate_db_ledger_config(v7_toml_data, v9_toml_data): + v9_db_ledger_config = v9_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"] + v7_db_ledger_config = v7_toml_data["database_setup"]["dbs"]["ledger"]["rocks_db"] + v9_db_ledger_config["path"] = v7_db_ledger_config["path"] + print("\nScanning ledger configuration ...") + scan_and_recommend_updates(v7_db_ledger_config, v9_db_ledger_config) + + +def __migrate_snapshot_config(v7_toml_data, v9_toml_data): + # Optional config + if "snapshot_config" not in v7_toml_data["database_setup"]: + print( + "Warning: [database_setup.snapshot_config] table not found in v7 config. Skipping migration." + ) + return + if "snapshot_config" not in v9_toml_data["database_setup"]: + raise SystemExit( + "Error: [database_setup.snapshot_config] table should exist in v9 template." + ) + + v9_snapshot_config = v9_toml_data["database_setup"]["snapshot_config"] + v7_snapshot_config = v7_toml_data["database_setup"]["snapshot_config"] + + v9_snapshot_config["path"] = v7_snapshot_config["path"] + + print("\nScanning snapshot configuration ...") + scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) + + +def __migrate_prune_config(v7_toml_data, v9_toml_data): + # Optional config + if "prune_config" not in v7_toml_data["database_setup"]: + print( + "Warning: [database_setup.prune_config] table not found in v7 config. Skipping migration." + ) + return + if "prune_config" not in v9_toml_data["database_setup"]: + raise SystemExit( + "Error: [database_setup.prune_config] table should exist in v9 template." + ) + v9_prune_config = v9_toml_data["database_setup"]["prune_config"] + v7_prune_config = v7_toml_data["database_setup"]["prune_config"] + + print("\nScanning prune configuration ...") + scan_and_recommend_updates(v7_prune_config, v9_prune_config) + + +def migrate_v7_to_v9(v7_toml_data): + """ + Returns a new TOML data structure that is compatible with RPC config v9. + """ + + with ( + importlib.resources.files(__package__) + .joinpath("rpc_config_v9_1_x_mainnet_template.toml") + .open("r") as f + ): + template = f.read() + v9_toml_data = tomlkit.parse(template) + __migrate_root_config(v7_toml_data, v9_toml_data) + __migrate_sync_ws_config(v7_toml_data, v9_toml_data) + __migrate_chain_state_assembler_config(v7_toml_data, v9_toml_data) + __migrate_db_archive_config(v7_toml_data, v9_toml_data) + __migrate_db_ledger_config(v7_toml_data, v9_toml_data) + __migrate_db_chain_store_config(v7_toml_data, v9_toml_data) + __migrate_snapshot_config(v7_toml_data, v9_toml_data) + __migrate_prune_config(v7_toml_data, v9_toml_data) + return v9_toml_data diff --git a/node_management/config_migration/src/rpc_config/migrate_path.py b/node_management/config_migration/src/rpc_config/migrate_path.py new file mode 100644 index 00000000..00018344 --- /dev/null +++ b/node_management/config_migration/src/rpc_config/migrate_path.py @@ -0,0 +1,13 @@ +from .from_v7_to_v9 import migrate_v7_to_v9 +from common import migration + + +RPC_CONFIG_MIGRATE_PATH = { + "v7-v9": [migrate_v7_to_v9], +} + + +def run_migration(migrate_path: str, from_path: str, to_path: str): + migration.Migration(RPC_CONFIG_MIGRATE_PATH).migrate_config( + migrate_path, from_path, to_path + ) diff --git a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml new file mode 100644 index 00000000..f8953d03 --- /dev/null +++ b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -0,0 +1,155 @@ + +# Version: v9.1.x mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "0.0.0.0:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./configs/supra_committees.json" +# The access tokens used to authenticate public RPC requests to this RPC node. +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +# Number of certified blocks stored in memory as reference to pending blocks to be executed. +# Only this amount of certified blocks are stored in memory, for the rest memo is kept. +certified_block_cache_bucket_size = 50 +# Retry interval for the sync requests for which no response yet available. +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The websocket address of the attached validator. +consensus_rpc = "ws://:26000" + +[synchronization.ws.certificates] +# The path to the TLS certificate for the connection with the attached validator. +cert_path = "./configs/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./configs/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_archive" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_ledger" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/rpc_store" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters related to database pruning. +[database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 336 + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +# The below settings are the default values required for use in RPC nodes run by validator node operators. +# They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:26000" +description = "LocalNet" +mode = "Server" diff --git a/node_management/config_migration/src/smr_settings/__init__.py b/node_management/config_migration/src/smr_settings/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/node_management/config_migration/src/smr_settings/from_v7_to_v9.py b/node_management/config_migration/src/smr_settings/from_v7_to_v9.py new file mode 100644 index 00000000..c96b371a --- /dev/null +++ b/node_management/config_migration/src/smr_settings/from_v7_to_v9.py @@ -0,0 +1,129 @@ +""" +This script reads a v7 configuration and migrates relevant settings into a v9 template, +ensuring compatibility with the new version. The migration process includes: +- Validating the absence of deprecated tables in v7 configs. +- Copying and updating node root configuration, including RPC and certificate paths. +- Migrating database paths for chain_store and ledger RocksDB instances. +- Optionally migrating snapshot and prune configurations if present. +- Migrating mempool and moonshot configurations. +- Scanning each migrated section for recommended updates. + +The migration preserves user-specific paths and settings where applicable, while adopting +the structure and defaults of the v9 template. +Migration script for upgrading Supra smr_settings config files from version v7 to v9. + +""" + +import tomlkit +import importlib.resources +from common.utils import scan_and_recommend_updates + +def __migrate_node_root_config(v7_toml_data, v9_toml_data): + if "ws_server" in v7_toml_data["node"]: + raise SystemExit( + "Error: [node.ws_server] table should not exist in v7 config. Please check your migration path matching the version of your config file." + ) + + v9_node_data = v9_toml_data["node"] + v7_node_data = v7_toml_data["node"] + v9_node_data["rpc_access_port"] = v7_node_data["rpc_access_port"] + + v9_node_ws_certificates = v9_node_data["ws_server"]["certificates"] + v9_node_ws_certificates["root_ca_cert_path"] = v7_node_data["root_ca_cert_path"] + v9_node_ws_certificates["cert_path"] = v7_node_data["server_cert_path"] + v9_node_ws_certificates["private_key_path"] = v7_node_data["server_private_key_path"] + + print("\nScanning node root configuration ...") + scan_and_recommend_updates(v7_node_data, v9_node_data) + + +def __migrate_db_chain_store(v7_toml_data, v9_toml_data): + v9_db_chain_store = v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v7_db_chain_store = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v9_db_chain_store["path"] = v7_db_chain_store["path"] + + print("\nScanning chain store configuration ...") + scan_and_recommend_updates(v7_db_chain_store, v9_db_chain_store) + +def __migrate_db_ledger(v7_toml_data, v9_toml_data): + v9_db_ledger = v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] + v7_db_ledger = v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] + v9_db_ledger["path"] = v7_db_ledger["path"] + + print("\nScanning ledger configuration ...") + scan_and_recommend_updates(v7_db_ledger, v9_db_ledger) + +def __migrate_snapshot_config(v7_toml_data, v9_toml_data): + """ + snapshot_config is optional, + - if absent in v7, the v9 config template will be used as is. + """ + if "snapshot_config" not in v7_toml_data["node"]["database_setup"]: + return + + if "snapshot_config" not in v9_toml_data["node"]["database_setup"]: + raise SystemExit( + "Error: [node.database_setup.snapshot_config] table should exist in v9 template." + ) + + v9_snapshot_config = v9_toml_data["node"]["database_setup"]["snapshot_config"] + v7_snapshot_config = v7_toml_data["node"]["database_setup"]["snapshot_config"] + + v9_snapshot_config["path"] = v7_snapshot_config["path"] + + print("\nScanning snapshot configuration ...") + scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) + +def __migrate_prune_config(v7_toml_data, v9_toml_data): + """ + prune_config is optional, so we skip if it does not exist in v7. + """ + if "prune_config" not in v7_toml_data["node"]["database_setup"]: + return + + if "prune_config" not in v9_toml_data["node"]["database_setup"]: + raise SystemExit( + "Error: [node.database_setup.prune_config] table should exist in v9 template." + ) + + v9_prune_config = v9_toml_data["node"]["database_setup"]["prune_config"] + v7_prune_config = v7_toml_data["node"]["database_setup"]["prune_config"] + + print("\nScanning prune configuration ...") + scan_and_recommend_updates(v7_prune_config, v9_prune_config) + + +def __migrate_mempool_config(v7_toml_data, v9_toml_data): + + v9_mempool_config = v9_toml_data["mempool"] + v7_mempool_config = v7_toml_data["mempool"] + + print("\nScanning mempool configuration ...") + scan_and_recommend_updates(v7_mempool_config, v9_mempool_config) + +def __migrate_moonshot_config(v7_toml_data, v9_toml_data): + v9_moonshot_config = v9_toml_data["moonshot"] + v7_moonshot_config = v7_toml_data["moonshot"] + + print("\nScanning moonshot configuration ...") + scan_and_recommend_updates(v7_moonshot_config, v9_moonshot_config) + +def migrate_v7_to_v9(toml_data): + """ + Returns a new TOML data structure that is compatible with SMR settings v9. + """ + with ( + importlib.resources.files(__package__) + .joinpath("smr_settings_v9_1_x_mainnet_template.toml") + .open("r") as f + ): + template = f.read() + v9_toml_data = tomlkit.parse(template) + __migrate_node_root_config(toml_data, v9_toml_data) + __migrate_db_ledger(toml_data, v9_toml_data) + __migrate_db_chain_store(toml_data, v9_toml_data) + __migrate_snapshot_config(toml_data, v9_toml_data) + __migrate_prune_config(toml_data, v9_toml_data) + __migrate_mempool_config(toml_data, v9_toml_data) + __migrate_moonshot_config(toml_data, v9_toml_data) + return v9_toml_data diff --git a/node_management/config_migration/src/smr_settings/migrate_path.py b/node_management/config_migration/src/smr_settings/migrate_path.py new file mode 100644 index 00000000..829da3a5 --- /dev/null +++ b/node_management/config_migration/src/smr_settings/migrate_path.py @@ -0,0 +1,14 @@ +from .from_v7_to_v9 import migrate_v7_to_v9 + +from common import migration + + +SMR_SETTINGS_MIGRATE_PATH = { + "v7-v9": [migrate_v7_to_v9], +} + + +def run_migration(migrate_path: str, from_path: str, to_path: str): + migration.Migration(SMR_SETTINGS_MIGRATE_PATH).migrate_config( + migrate_path, from_path, to_path + ) diff --git a/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml new file mode 100644 index 00000000..3b65a2f5 --- /dev/null +++ b/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml @@ -0,0 +1,146 @@ + +# Version: v9.1.x mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 4 hours. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 2 hours. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 + +[node.ws_server.certificates] +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./configs/ca_certificate.pem" +# The path to the TLS certificate for this node. +cert_path = "./configs/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./configs/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./configs/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./configs/ledger_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 + +# Parameters for the database snapshot service. +[node.database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 diff --git a/node_management/config_migration/tests/config_v7.1.8.toml b/node_management/config_migration/tests/config_v7.1.8.toml new file mode 100644 index 00000000..79649d19 --- /dev/null +++ b/node_management/config_migration/tests/config_v7.1.8.toml @@ -0,0 +1,131 @@ +# Version: v7.1.8 +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_instance.chain_id = 8 +# The length of an epoch in seconds. +chain_instance.epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +chain_instance.recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +chain_instance.voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +chain_instance.is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +chain_instance.genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "1.1.1.1:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +# The path to the TLS certificate for the connection with the attached validator. +consensus_client_cert_path = "./xyz/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +consensus_client_private_key_path = "./xyz/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +consensus_root_ca_cert_path = "./xyz/ca_certificate.pem" +# The websocket address of the attached validator. +consensus_rpc = "ws://2.2.2.2:26000" +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./xyz/supra_committees.json" +# The number of seconds to wait before retrying a block sync request. +sync_retry_interval_in_secs = 1 + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_archive" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_ledger" + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_store" +# Whether snapshots should be taken of the database. +enable_snapshots = true + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./xyz/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retring a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:27000" +description = "LocalNet" +mode = "Server" \ No newline at end of file diff --git a/node_management/config_migration/tests/config_v7_to_v9_expected.toml b/node_management/config_migration/tests/config_v7_to_v9_expected.toml new file mode 100644 index 00000000..71b4040c --- /dev/null +++ b/node_management/config_migration/tests/config_v7_to_v9_expected.toml @@ -0,0 +1,154 @@ +# Version: v9.1.x mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. + +# Core protocol parameters. +# The below parameters are node-specific and may be configured as required by the operator. + +# The port on which the node should listen for incoming RPC requests. +bind_addr = "1.1.1.1:26000" +# If `true` then blocks will not be verified before execution. This value should be `false` +# unless you also control the node from which this RPC node is receiving blocks. +block_provider_is_trusted = true +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. +resume = true +# The path to `supra_committees.json`. +supra_committees_config = "./xyz/supra_committees.json" +# The access tokens used to authenticate public RPC requests to this RPC node. +consensus_access_tokens = [] + +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +[chain_instance] +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + + +######################################### NODE PARAMETERS ######################################### +[chain_state_assembler] +# Number of certified blocks stored in memory as reference to pending blocks to be executed. +# Only this amount of certified blocks are stored in memory, for the rest memo is kept. +certified_block_cache_bucket_size = 50 +# Retry interval for the sync requests for which no response yet available. +sync_retry_interval_in_secs = 1 + +[synchronization.ws] +# The websocket address of the attached validator. +consensus_rpc = "ws://2.2.2.2:26000" + +[synchronization.ws.certificates] +# The path to the TLS certificate for the connection with the attached validator. +cert_path = "./xyz/client_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./xyz/client_supra_key.pem" +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./xyz/ca_certificate.pem" + +# Parameters for the RPC Archive database. This database stores the indexes used to serve RPC API calls. +[database_setup.dbs.archive.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_archive" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters for the DKG database. +[database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_ledger" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters for the blockchain database. +[database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./xyz/rpc_store" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = false +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters related to database pruning. +[database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 336 + +# Parameters for the database snapshot service. +[database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./xyz/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 + +# CORS settings for RPC API requests. +# The below settings are the default values required for use in RPC nodes run by validator node operators. +# They are optional for non-validators. +[[allowed_origin]] +url = "https://rpc-mainnet.supra.com" +description = "RPC For Supra" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet1.supra.com" +description = "RPC For nodeops group1" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet2.supra.com" +description = "RPC For nodeops group2" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet3.supra.com" +description = "RPC For nodeops group3" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet4.supra.com" +description = "RPC For nodeops group4" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-mainnet5.supra.com" +description = "RPC For nodeops group5" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-wallet-mainnet.supra.com" +description = "RPC For Supra Wallet" +mode = "Server" + +[[allowed_origin]] +url = "https://rpc-suprascan-mainnet.supra.com" +description = "RPC For suprascan" +mode = "Server" + +[[allowed_origin]] +url = "http://localhost:26000" +description = "LocalNet" +mode = "Server" \ No newline at end of file diff --git a/node_management/config_migration/tests/smr_settings_v7.1.x.toml b/node_management/config_migration/tests/smr_settings_v7.1.x.toml new file mode 100644 index 00000000..355aa86a --- /dev/null +++ b/node_management/config_migration/tests/smr_settings_v7.1.x.toml @@ -0,0 +1,167 @@ +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 48 hours. +recurring_lockup_duration_secs = 172800 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 46 hours. +voting_duration_secs = 165600 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 1500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 20 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 2000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 5000 + +# Parameters related to the MoveVM. Primarily related to governance features. +[move_vm] +# Initially `false` until the network matures. +allow_new_validators = false +# The maximum stake that may be allocated to a Supra Validator. We are not currently doing +# stake-weighted voting, so this value does not impact our decentralization quotient. This +# may change in the future. Initially set to the total supply. +# +# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 100_000_000_000 SUPRA. +max_stake = "10000000000000000000" +# The minimum stake required to run a Supra Validator. There is no minimum at genesis +# because The Foundation's stake is added after the stake pools are created. We will increase +# it to its intended value of 55M SUPRA via governance. New nodes will not be able to join the +# validator set before the update is made. Measured in Quants (1 Quant = 10^-8 SUPRA). +min_stake = 0 +# The number of tokens initially allocated to node operators. Tokens will be earned through block +# rewards. +operator_account_balance = 0 +# The amount of Quants to transfer from each validator owner account to the corresponding stake pool +# after it has been created, during the genesis transaction. Adding the stake after creating the +# pool allows us to ensure that The Foundation's stake is not subject to the PBO locking schedule, +# which is only intended to apply to winners of the Project Blast Off campaign. +# +# Measured in Quants (1 Quant = 10^-8 SUPRA). Equal to 55_000_000 SUPRA. +pbo_owner_stake = 5500000000000000 +# The number of seconds after `genesis_timestamp_microseconds` at which all accounts with +# allocations at genesis will be able to unlock their initial amounts. These amounts are stored +# in vesting contracts. +# +# Corresponds to Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +remaining_balance_lockup_cliff_period_in_seconds = 0 +# The amount of SUPRA required to qualify as a proposer (this parameter is currently unused). +required_proposer_stake = 0 +# The annual percent yield for validators, proportional to their stake. Specified as a percentage +# with 2 decimals of precision in u64 format due to limitations in the MoveVM. The below value +# represents 12.85%. +rewards_apy_percentage = 1285 +# The percentage of staking rewards earned by Supra Foundation controlled nodes that will be paid +# to the corresponding node operators. Specified as a percentage with 2 decimals of precision in +# u64 format due to limitations in the MoveVM. The below value represents 37.74%. +validator_commission_rate_percentage = 3774 +# The percentage of new stake relative to the current total stake that can join the validator +# set or be added to existing validators within a single epoch. +voting_power_increase_limit = 33 + + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 2 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./tests/ca_certificate.pem" +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 +# The path to the TLS certificate for this node. +server_cert_path = "./tests/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +server_private_key_path = "./tests/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./tests/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./tests/ledger_storage" + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 diff --git a/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml b/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml new file mode 100644 index 00000000..eef5ffb5 --- /dev/null +++ b/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml @@ -0,0 +1,145 @@ +# Version: v9.1.x mainnet + +####################################### PROTOCOL PARAMETERS ####################################### + +# The below parameters are fixed for the protocol and must be agreed upon by all node operators +# at genesis. They may subsequently be updated via governance decisions. Paths are set relative +# to $SUPRA_HOME. + +# Core protocol parameters. +[instance] +# A unique identifier for this instance of the Supra protocol. Prevents replay attacks across chains. +chain_id = 8 +# The length of an epoch in seconds. +epoch_duration_secs = 7200 +# The number of seconds that stake locked in a Stake Pool will automatically be locked up for when +# its current lockup expires, if no request is made to unlock it. +# +# 4 hours. +recurring_lockup_duration_secs = 14400 +# The number of seconds allocated for voting on governance proposals. Governance will initially be +# controlled by The Supra Foundation. +# +# 2 hours. +voting_duration_secs = 7200 +# Determines whether the network will start with a faucet, amongst other things. +is_testnet = false +# Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). +genesis_timestamp_microseconds = 1732060800000000 + +# Parameters related to the mempool. +[mempool] +# The maximum number of milliseconds that a node will wait before proposing a batch when it has +# at least one transaction to process. +max_batch_delay_ms = 500 +# The maximum size of a batch. If `max_batch_size_bytes` is reached before `max_batch_delay_ms` +# then a batch will be proposed immediately. +max_batch_size_bytes = 5000000 +# The amount of time that a node will wait before repeating a sync request for a batch that it +# is missing. +sync_retry_delay_ms = 2000 +# The number of signers of the related batch certificate that a node should ask for a batch +# attempting to retry a sync request. +sync_retry_nodes = 3 + +# Parameters related to the Moonshot consensus protocol. See https:#arxiv.org/abs/2401.01791. +[moonshot] +# The maximum number of milliseconds that the timestamp of a proposed block may be +# ahead of a node's local time when it attempts to vote for the block. Validators +# must wait until the timestamp of a certified block has passed before advancing to +# the next round and leaders must wait until the timestamp of the parent block has +# passed before proposing, so this limit prevents Byzantine leaders from forcing +# honest nodes to wait indefinitely by proposing blocks with timestamps that are +# arbitrarily far in the future. +block_recency_bound_ms = 500 +# Causes the node to stop producing blocks when there are no transactions to be +# processed. If all nodes set this value to `true` then the chain will not produce +# new blocks when there are no transactions to process, conserving disk space. +halt_block_production_when_no_txs = false +# The type of leader election function to use. This function generates a schedule that ensures +# that each node eventually succeeds every other. +leader_elector = "FairSuccession" +# The delay after which the block proposer will create a new block despite not having any +# payload items to propose. Denominated in ms. +max_block_delay_ms = 1250 +# The maximum number of batch availability certificates that may be included in a single +# consensus block. +max_payload_items_per_block = 50 +# The number of rounds ahead of self.round for which this node should accept Optimistic Proposals, +# Votes and Timeouts. Must be the same for all nodes. This parameter helps to limit the amount +# of memory that Byzantine nodes can consume, but the larger it is the more efficient syncing can +# become. This trade-off must be balanced. +message_recency_bound_rounds = 1000 +# The delay after which the node will try to repeat sync requests for missing blocks. +# Denominated in ms. Should be the same for all nodes. +sync_retry_delay_ms = 1000 +# The delay after which the node will send a Timeout message for its current Moonshot round, +# measured from the start of the round. Denominated in ms. Must be the same for all nodes. +timeout_delay_ms = 3500 + +######################################### NODE PARAMETERS ######################################### + +# The below parameters are node-specific and may be configured as required by the operator. Paths +# should be specified either as absolute paths or as relative to the `supra` binary. +# +# When running the node in Docker and working with a mounted directory, the paths must be specified +# with reference to the location of the mounter directory in the Docker filesystem rather than the +# host filesystem. The default settings provided below assume the default Docker configuration +# provided by the `manage_supra_nodes.sh` script. + +[node] +# The duration in seconds that a node waits between polling its connections to its peers. +connection_refresh_timeout_sec = 1 +# If true, all components will attempt to load their previous state from disk. Otherwise, +# all components will start in their default state. Should always be `true` for testnet and +# mainnet. +resume = true +# The port on which to listen for connections from the associated RPC node. Each validator +# may serve at most one RPC node. +rpc_access_port = 26000 + +[node.ws_server.certificates] +# The path to the TLS root certificate authority certificate. +root_ca_cert_path = "./tests/ca_certificate.pem" +# The path to the TLS certificate for this node. +cert_path = "./tests/server_supra_certificate.pem" +# The path to the private key to be used when negotiating TLS connections. +private_key_path = "./tests/server_supra_key.pem" + +# Parameters for the blockchain database. +[node.database_setup.dbs.chain_store.rocks_db] +# The path at which the database should be created. +path = "./tests/smr_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters for the DKG database. +[node.database_setup.dbs.ledger.rocks_db] +# The path at which the database should be created. +path = "./tests/ledger_storage" +# Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` +# old will be deleted. +enable_pruning = true +# Whether snapshots should be taken of the database. +enable_snapshots = false + +# Parameters related to database pruning. +[node.database_setup.prune_config] +# Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. +epochs_to_retain = 84 + +# Parameters for the database snapshot service. +[node.database_setup.snapshot_config] +# The number of snapshots to retain, including the latest. +depth = 2 +# The interval between snapshots in seconds. +interval_in_seconds = 1800 +# The path at which the snapshots should be stored. +path = "./configs/snapshot" +# The number of times to retry a snapshot in the event that it fails unexpectedly. +retry_count = 3 +# The interval in seconds to wait before retrying a snapshot. +retry_interval_in_seconds = 5 diff --git a/node_management/config_migration/tests/test_rpc_migration.py b/node_management/config_migration/tests/test_rpc_migration.py new file mode 100644 index 00000000..1344a480 --- /dev/null +++ b/node_management/config_migration/tests/test_rpc_migration.py @@ -0,0 +1,42 @@ +import os +import shutil +import tomlkit +import pytest +from rpc_config.migrate_path import run_migration as rpc_run_migration + +# CONFIGS = [ +# "config_v7.1.8.toml", +# "config_v8.0.2.toml", +# "config_v9.0.7.toml", +# "config_v9.1_mainnet.toml", +# ] + + +@pytest.mark.parametrize( + "from_file,to_file,migrate_path", + [ + ("config_v7.1.8.toml", "config_v7_to_v9_expected.toml", "v7-v9"), + ], +) +def test_migration(tmp_path, from_file, to_file, migrate_path): + # Copy config files to temp dir + src_dir = os.path.dirname(os.path.abspath(__file__)) + from_path = os.path.join(src_dir, from_file) + to_path = os.path.join(src_dir, to_file) + tmp_from = tmp_path / from_file + tmp_to = tmp_path / to_file + shutil.copy(from_path, tmp_from) + # Run migration + import common.globals + common.globals.ASSUME_YES = True + rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) + # Load both files + with open(tmp_to, "r") as f: + migrated = tomlkit.parse(f.read()) + with open(to_path, "r") as f: + expected = tomlkit.parse(f.read()) + + # Compare TOML dicts + assert migrated == expected, ( + f"Migration {migrate_path} failed: {from_file} -> {to_file}" + ) diff --git a/node_management/config_migration/tests/test_smr_migration.py b/node_management/config_migration/tests/test_smr_migration.py new file mode 100644 index 00000000..ef6eef8b --- /dev/null +++ b/node_management/config_migration/tests/test_smr_migration.py @@ -0,0 +1,35 @@ +import os +import shutil +import tomlkit +import pytest +from smr_settings.migrate_path import run_migration as smr_run_migration + + +@pytest.mark.parametrize( + "from_file,to_file,migrate_path", + [ + ("smr_settings_v7.1.x.toml", "smr_settings_v7_to_v9_expected.toml", "v7-v9"), + ], +) +def test_migration(tmp_path, from_file, to_file, migrate_path): + # Copy config files to temp dir + src_dir = os.path.dirname(os.path.abspath(__file__)) + from_path = os.path.join(src_dir, from_file) + to_path = os.path.join(src_dir, to_file) + tmp_from = tmp_path / from_file + tmp_to = tmp_path / to_file + shutil.copy(from_path, tmp_from) + # Run migration + import common.globals + common.globals.ASSUME_YES = True + + smr_run_migration(migrate_path, str(tmp_from), str(tmp_to)) + # Load both files + with open(tmp_to, "r") as f: + migrated = tomlkit.parse(f.read()) + with open(to_path, "r") as f: + expected = tomlkit.parse(f.read()) + # Compare TOML dicts + assert migrated == expected, ( + f"Migration {migrate_path} failed: {from_file} -> {to_file}" + ) From 4ba2e840dc0f603e1fc994c02e8ff42dfe16d6ce Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 29 May 2025 10:53:55 +0200 Subject: [PATCH 33/44] add --- node_management/config_migration/lazydog.md | 35 --------------------- 1 file changed, 35 deletions(-) delete mode 100644 node_management/config_migration/lazydog.md diff --git a/node_management/config_migration/lazydog.md b/node_management/config_migration/lazydog.md deleted file mode 100644 index d80c8dde..00000000 --- a/node_management/config_migration/lazydog.md +++ /dev/null @@ -1,35 +0,0 @@ -alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node -alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node -alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra" -alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra" -export NODE_OPERATOR_KEY_PASSWORD="88888888" - - -# SMR -supra-v8 migrate --network localnet - - -cp validator_identity.pem node_identity.pem -supra-v9 profile migrate -sed -i '' 's/is_testnet = true/is_testnet = false/' smr_settings.toml -migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml - - -lsof -n -P -iTCP:26002-26006 -sTCP:LISTEN - -supra-v9 data migrate -p smr_settings.toml - - -supra-v9 node smr run - - -### RPC -sed -i '' 's/is_testnet = true/is_testnet = false/' config.toml -migrate-config rpc -p v7-v9 -f config.toml -t config.toml - - - -rpc-v8 migrate-db config.toml - -rpc-v9 migrate-db config.toml -rpc-v9 start & \ No newline at end of file From 5d366dcbfa2f2814effd6803433fac72d44da54c Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 29 May 2025 20:26:48 +0200 Subject: [PATCH 34/44] update docs --- .../src/rpc_config/from_v7_to_v9.py | 7 ++-- .../src/smr_settings/from_v7_to_v9.py | 42 ++++++++++--------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/node_management/config_migration/src/rpc_config/from_v7_to_v9.py b/node_management/config_migration/src/rpc_config/from_v7_to_v9.py index 43da1458..7f7344d6 100644 --- a/node_management/config_migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/config_migration/src/rpc_config/from_v7_to_v9.py @@ -8,13 +8,12 @@ - Migrates synchronization WebSocket settings, including consensus RPC and certificate paths. - Migrates chain state assembler configuration. - Migrates database paths for archive, chain store, and ledger components. -- Migrates snapshot configuration paths. +- Migrates snapshot configuration paths if present; skips if not found in v7. +- Migrates prune configuration if present; skips if not found in v7. - For each section, scans and recommends updates for any legacy fields. -- Ensures that deprecated or unexpected sections in v7 (e.g., 'synchronization', 'chain_state_assembler') are flagged as errors. +- Exit early if any unexpected sections in v7 (e.g., 'synchronization', 'chain_state_assembler') are present before migration. The main entrypoint is `migrate_v7_to_v9(v7_toml_data)`, which returns a new TOML data structure compatible with v9. -This module provides migration functions to upgrade Supra's RPC configuration from version v7 to v9. - """ import tomlkit diff --git a/node_management/config_migration/src/smr_settings/from_v7_to_v9.py b/node_management/config_migration/src/smr_settings/from_v7_to_v9.py index c96b371a..510c97cc 100644 --- a/node_management/config_migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/config_migration/src/smr_settings/from_v7_to_v9.py @@ -1,16 +1,18 @@ """ -This script reads a v7 configuration and migrates relevant settings into a v9 template, -ensuring compatibility with the new version. The migration process includes: -- Validating the absence of deprecated tables in v7 configs. -- Copying and updating node root configuration, including RPC and certificate paths. -- Migrating database paths for chain_store and ledger RocksDB instances. -- Optionally migrating snapshot and prune configurations if present. -- Migrating mempool and moonshot configurations. -- Scanning each migrated section for recommended updates. - -The migration preserves user-specific paths and settings where applicable, while adopting -the structure and defaults of the v9 template. -Migration script for upgrading Supra smr_settings config files from version v7 to v9. +This module provides migration utilities to upgrade Supra's Validator configuration files from version v7 to v9. + +Migration Steps: +---------------- +- Loads a v9 template configuration as the migration base. +- Migrates root-level fields such as rpc_access_port and certificate paths. +- Migrates database paths for chain store and ledger components. +- Migrates snapshot configuration paths if present; skips if not found in v7. +- Migrates prune configuration if present; skips if not found in v7. +- Migrates mempool and moonshot sections, preserving all settings. +- For each section, scans and recommends updates for any legacy fields. +- Exit early if any unexpected sections in v7 (e.g., [node.ws_server]) are present before migration. + +The main entrypoint is `migrate_v7_to_v9(v7_toml_data)`, which returns a new TOML data structure compatible with v9. """ @@ -108,7 +110,7 @@ def __migrate_moonshot_config(v7_toml_data, v9_toml_data): print("\nScanning moonshot configuration ...") scan_and_recommend_updates(v7_moonshot_config, v9_moonshot_config) -def migrate_v7_to_v9(toml_data): +def migrate_v7_to_v9(v7_toml_data): """ Returns a new TOML data structure that is compatible with SMR settings v9. """ @@ -119,11 +121,11 @@ def migrate_v7_to_v9(toml_data): ): template = f.read() v9_toml_data = tomlkit.parse(template) - __migrate_node_root_config(toml_data, v9_toml_data) - __migrate_db_ledger(toml_data, v9_toml_data) - __migrate_db_chain_store(toml_data, v9_toml_data) - __migrate_snapshot_config(toml_data, v9_toml_data) - __migrate_prune_config(toml_data, v9_toml_data) - __migrate_mempool_config(toml_data, v9_toml_data) - __migrate_moonshot_config(toml_data, v9_toml_data) + __migrate_node_root_config(v7_toml_data, v9_toml_data) + __migrate_db_ledger(v7_toml_data, v9_toml_data) + __migrate_db_chain_store(v7_toml_data, v9_toml_data) + __migrate_snapshot_config(v7_toml_data, v9_toml_data) + __migrate_prune_config(v7_toml_data, v9_toml_data) + __migrate_mempool_config(v7_toml_data, v9_toml_data) + __migrate_moonshot_config(v7_toml_data, v9_toml_data) return v9_toml_data From 1fc2cb9b350f1e211d6893f358c449fc2993c180 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Fri, 30 May 2025 13:31:57 +0200 Subject: [PATCH 35/44] update err handling --- .../config_migration/src/common/migration.py | 24 ++- .../migrate_mainnet_from_v7_to_v9.sh | 196 ++++++++++++++++++ 2 files changed, 209 insertions(+), 11 deletions(-) create mode 100755 node_management/migrate_mainnet_from_v7_to_v9.sh diff --git a/node_management/config_migration/src/common/migration.py b/node_management/config_migration/src/common/migration.py index 18efc502..759fbf65 100644 --- a/node_management/config_migration/src/common/migration.py +++ b/node_management/config_migration/src/common/migration.py @@ -41,6 +41,16 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): from_version, to_version = self.migrate_path.get_versions(migrate_choice) default_backup_path = f"{from_path}_{from_version}.bak" + with open(from_path, "r") as f: + toml_data = tomlkit.load(f) + + original_toml_data = deepcopy(toml_data) + + for fn in migrate_functions: + print(f"Running migration function: {fn.__name__}") + toml_data = fn(toml_data) + + # Check before overwriting the same file. if from_path == to_path: print( f"Warning: The source and destination paths are the same ({from_path})." @@ -52,21 +62,13 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): "This will overwrite your original config file. Continue?", ASSUME_YES ) if not confirm: - print("Aborted by user.") - return + raise SystemExit( + "Migration aborted by user. No changes were made." + ) # Backup old config print(f"Backing up old config to {default_backup_path}") shutil.copyfile(from_path, default_backup_path) - with open(from_path, "r") as f: - toml_data = tomlkit.load(f) - - original_toml_data = deepcopy(toml_data) - - for fn in migrate_functions: - print(f"Running migration function: {fn.__name__}") - toml_data = fn(toml_data) - # Write new config print(f"Writing new config to {to_path}") with open(to_path, "w") as f: diff --git a/node_management/migrate_mainnet_from_v7_to_v9.sh b/node_management/migrate_mainnet_from_v7_to_v9.sh new file mode 100755 index 00000000..eb723e72 --- /dev/null +++ b/node_management/migrate_mainnet_from_v7_to_v9.sh @@ -0,0 +1,196 @@ +#!/bin/bash + + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +SCRIPT_NAME="migrate_from_v7_to_v9" + +# This script is expected to be installed with `install_management_scripts.sh`, which +# creates the `.supra` directory and retrieves the `node_management` directory. +source "$SCRIPT_DIR/.supra/node_management/utils.sh" + +function parse_args() { + NODE_TYPE="$1" + CONTAINER_NAME="$2" + HOST_SUPRA_HOME="$3" +} + +function basic_usage() { + echo "Usage: ./$SCRIPT_NAME.sh " >&2 + echo "Parameters:" >&2 + node_type_usage + container_name_usage + host_supra_home_usage + exit 1 +} + + +function verify_rpc() { + if ! verify_container_name || ! verify_host_supra_home; then + basic_usage + fi +} + +function verify_validator() { + if ! verify_container_name || ! verify_host_supra_home; then + basic_usage + fi +} + +function verify_args() { + if [[ "$NODE_TYPE" == "rpc" ]]; then + verify_rpc + elif [[ "$NODE_TYPE" == "validator" ]]; then + verify_validator + else + basic_usage + fi +} + + +#---------------------------------------------------------- RPC ---------------------------------------------------------- + +function migrate_rpc() { + echo "Migrating RPC $CONTAINER_NAME at $HOST_SUPRA_HOME to v9" + # TODO(sc): replace with mainnet image + RPC_V9_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-testnet/rpc-node:v9.0.12 + RPC_V8_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-mainnet/rpc-node:v8.0.3 + + # Define shell functions instead of aliases, so they are available in the script + rpc-v8() { docker exec -it rpc-v8 /supra/rpc_node "$@"; } + rpc-v9() { docker exec -it rpc-v9 /supra/rpc_node "$@"; } + # Stop the Docker container if it is running. + # TODO(SC) stop running container so that releasing database lock + docker stop "$CONTAINER_NAME" || : + + # Stop all rpc containers + docker rm -f rpc-v8 || : + docker rm -f rpc-v9 || : + docker container ls + + docker run --name rpc-v8 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$RPC_V8_IMAGE" + + docker run --name rpc-v9 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$RPC_V9_IMAGE" + + echo "Migrate rpc config from v7 to v9" + # TODO(SC) to run in docker context, update path to `./configs/config.toml` + migrate-config rpc -p v7-v9 -f $HOST_SUPRA_HOME/config.toml -t $HOST_SUPRA_HOME/config.toml + # Localnet only (Optional: local env path is different from docker env path, need to be modified to use docker env path) + sed -i "" "s#${HOST_SUPRA_HOME}#configs#g" ${HOST_SUPRA_HOME}/config.toml + + # Run in docker context. + echo "Migrate db from v7 to v8" + rpc-v8 migrate-db configs/config.toml + echo "Migrate db from v8 to v9" + rpc-v9 migrate-db configs/config.toml + + # Remove containers again because we do not need them after migration + docker rm -f rpc-v8 || : + docker rm -f rpc-v9 || : + docker container ls + + # Remove any existing snapshots. If we don't do this then they will start to take + # up a large amount of disk space during the sync. + # TODO(sc): WHY? + rm -rf "$HOST_SUPRA_HOME/snapshot" + rm -rf "$HOST_SUPRA_HOME/snapshots" + + # Finished migration, and follow guide that start the node with the new image with new config and sync + # ------ + + # ./manage_supra_nodes.sh \ + # sync \ + # --exact-timestamps \ + # --snapshot-source testnet-archive-snapshot \ + # rpc \ + # "$HOST_SUPRA_HOME" \ + # testnet + # echo "Migration complete. Please transfer all custom settings from $v8_config_toml to " + # echo -n "$config_toml before starting your node." +} + +#---------------------------------------------------------- Validator ---------------------------------------------------------- + +function migrate_validator() { + echo "Migrating validator $CONTAINER_NAME at $HOST_SUPRA_HOME to v9" + + SUPRA_V9_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-mainnet/validator-node:v9.0.5 + SUPRA_V8_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-mainnet/validator-node:v8.0.3 + supra-v8() { docker exec -it supra-v8 /supra/supra "$@"; } + supra-v9() { docker exec -it supra-v9 /supra/supra "$@"; } + + + # Stop the Docker container if it is running. + # TODO(SC) stop running container so that releasing database lock + docker stop "$CONTAINER_NAME" || : + + docker rm -f supra-v8 || : + docker rm -f supra-v9 || : + docker container ls + + docker run --name supra-v8 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$SUPRA_V8_IMAGE" + + docker run --name supra-v9 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$SUPRA_V9_IMAGE" + + + + echo "Migrate cli profile from v7 to v8" + supra-v8 migrate --network mainnet + cp $HOST_SUPRA_HOME/validator_identity.pem $HOST_SUPRA_HOME/node_identity.pem + echo "Migrate cli profile from v8 to v9" + supra-v9 profile migrate + echo "Migrate smr_settings from v7 to v9" + # TODO(SC) to be run in docker context, update path to `./configs/config.toml` + migrate-config smr -p v7-v9 -f $HOST_SUPRA_HOME/smr_settings.toml -t $HOST_SUPRA_HOME/smr_settings.toml + + # Localnet only (Optional: local env path is different from docker env path, need to be modified to use docker env path) + sed -i "" "s#${HOST_SUPRA_HOME}#configs#g" ${HOST_SUPRA_HOME}/smr_settings.toml + echo "Migrate db from v7 to v9" + supra-v9 data migrate -p configs/smr_settings.toml + + + # Remove containers again because we do not need them after migration + docker rm -f supra-v8 || : + docker rm -f supra-v9 || : + docker container ls + + # Remove any existing snapshots. If we don't do this then they will start to take + # up a large amount of disk space during the sync. + # TODO(sc): WHY? + rm -rf "$HOST_SUPRA_HOME/snapshot" + rm -rf "$HOST_SUPRA_HOME/snapshots" + + # Finished migration, and follow guide that start the node with the new image with new config and sync + + echo "Migration complete." +} + +function main() { + if [ "$#" -lt 3 ]; then + basic_usage + fi + parse_args "$@" + verify_args + ensure_supra_home_is_absolute_path + + if [ "$NODE_TYPE" == "validator" ]; then + migrate_validator + elif [ "$NODE_TYPE" == "rpc" ]; then + migrate_rpc + fi +} + +main "$@" From 1dc5842ba25f86dcd7b886d035ced2ee9fb13e46 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 2 Jun 2025 09:42:46 +0200 Subject: [PATCH 36/44] update docs --- node_management/migrate_mainnet_from_v7_to_v9.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/node_management/migrate_mainnet_from_v7_to_v9.sh b/node_management/migrate_mainnet_from_v7_to_v9.sh index eb723e72..483a3387 100755 --- a/node_management/migrate_mainnet_from_v7_to_v9.sh +++ b/node_management/migrate_mainnet_from_v7_to_v9.sh @@ -57,18 +57,19 @@ function migrate_rpc() { RPC_V9_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-testnet/rpc-node:v9.0.12 RPC_V8_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-mainnet/rpc-node:v8.0.3 - # Define shell functions instead of aliases, so they are available in the script rpc-v8() { docker exec -it rpc-v8 /supra/rpc_node "$@"; } rpc-v9() { docker exec -it rpc-v9 /supra/rpc_node "$@"; } - # Stop the Docker container if it is running. - # TODO(SC) stop running container so that releasing database lock + + echo "Stop the User's container if it is running." docker stop "$CONTAINER_NAME" || : - # Stop all rpc containers + echo "Prepare containers needed for running migration." + # Stop+Remove rpc containers for migration if they exist. docker rm -f rpc-v8 || : docker rm -f rpc-v9 || : docker container ls + # Start rpc containers with proper env and volume mounts. docker run --name rpc-v8 \ -v "$HOST_SUPRA_HOME:/supra/configs" \ -e "SUPRA_HOME=/supra/configs/" \ From 829f5e5040413c3f7891959098a0eb80a9219ec1 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Mon, 2 Jun 2025 09:46:12 +0200 Subject: [PATCH 37/44] add docs and print --- node_management/migrate_mainnet_from_v7_to_v9.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/node_management/migrate_mainnet_from_v7_to_v9.sh b/node_management/migrate_mainnet_from_v7_to_v9.sh index 483a3387..348383ea 100755 --- a/node_management/migrate_mainnet_from_v7_to_v9.sh +++ b/node_management/migrate_mainnet_from_v7_to_v9.sh @@ -60,7 +60,7 @@ function migrate_rpc() { rpc-v8() { docker exec -it rpc-v8 /supra/rpc_node "$@"; } rpc-v9() { docker exec -it rpc-v9 /supra/rpc_node "$@"; } - echo "Stop the User's container if it is running." + echo "Stop the user's container if it is running." docker stop "$CONTAINER_NAME" || : echo "Prepare containers needed for running migration." @@ -92,11 +92,13 @@ function migrate_rpc() { echo "Migrate db from v8 to v9" rpc-v9 migrate-db configs/config.toml + echo "Cleanup containers used for migration." # Remove containers again because we do not need them after migration docker rm -f rpc-v8 || : docker rm -f rpc-v9 || : docker container ls + echo "Remove snapshot and snapshots directories" # Remove any existing snapshots. If we don't do this then they will start to take # up a large amount of disk space during the sync. # TODO(sc): WHY? @@ -128,14 +130,16 @@ function migrate_validator() { supra-v9() { docker exec -it supra-v9 /supra/supra "$@"; } - # Stop the Docker container if it is running. - # TODO(SC) stop running container so that releasing database lock + echo "Stop the users' container if it is running." docker stop "$CONTAINER_NAME" || : + echo "Prepare containers needed for running migration." + # Stop+Remove supra containers for migration if they exist. docker rm -f supra-v8 || : docker rm -f supra-v9 || : docker container ls + # Start supra containers with proper env and volume mounts. docker run --name supra-v8 \ -v "$HOST_SUPRA_HOME:/supra/configs" \ -e "SUPRA_HOME=/supra/configs/" \ @@ -163,11 +167,12 @@ function migrate_validator() { supra-v9 data migrate -p configs/smr_settings.toml - # Remove containers again because we do not need them after migration + echo "Cleanup containers used for migration." docker rm -f supra-v8 || : docker rm -f supra-v9 || : docker container ls + echo "Remove snapshot and snapshots directories" # Remove any existing snapshots. If we don't do this then they will start to take # up a large amount of disk space during the sync. # TODO(sc): WHY? From 5bc69c78df2ca8a37e91fa344a4aa42ad4c9abd2 Mon Sep 17 00:00:00 2001 From: simonchen-supra Date: Wed, 4 Jun 2025 13:43:58 +0200 Subject: [PATCH 38/44] Update node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml Co-authored-by: Isaac Doidge <30425649+isaacdoidge@users.noreply.github.com> --- .../src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml index f8953d03..6e709640 100644 --- a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml +++ b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -91,7 +91,7 @@ enable_snapshots = false # Parameters related to database pruning. [database_setup.prune_config] # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 336 +epochs_to_retain = 1008 # Parameters for the database snapshot service. [database_setup.snapshot_config] From db185eaa763bae479851dfdfac9be391301ef7a4 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 4 Jun 2025 13:43:53 +0200 Subject: [PATCH 39/44] chore: remove codes --- .../config_migration/tests/test_rpc_migration.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/node_management/config_migration/tests/test_rpc_migration.py b/node_management/config_migration/tests/test_rpc_migration.py index 1344a480..a7b4e0dc 100644 --- a/node_management/config_migration/tests/test_rpc_migration.py +++ b/node_management/config_migration/tests/test_rpc_migration.py @@ -4,13 +4,6 @@ import pytest from rpc_config.migrate_path import run_migration as rpc_run_migration -# CONFIGS = [ -# "config_v7.1.8.toml", -# "config_v8.0.2.toml", -# "config_v9.0.7.toml", -# "config_v9.1_mainnet.toml", -# ] - @pytest.mark.parametrize( "from_file,to_file,migrate_path", From db87a3b098eac9fe1552a0984b55538e242d5a39 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 4 Jun 2025 14:01:16 +0200 Subject: [PATCH 40/44] address comments --- .../config_migration/src/common/migration.py | 6 ++--- .../config_migration/src/common/utils.py | 4 ++-- .../src/rpc_config/from_v7_to_v9.py | 2 +- .../src/smr_settings/from_v7_to_v9.py | 23 ++++++++++++++----- .../smr_settings_v9_1_x_mainnet_template.toml | 1 - .../tests/test_rpc_migration.py | 1 + .../tests/test_smr_migration.py | 1 + 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/node_management/config_migration/src/common/migration.py b/node_management/config_migration/src/common/migration.py index 759fbf65..9c145136 100644 --- a/node_management/config_migration/src/common/migration.py +++ b/node_management/config_migration/src/common/migration.py @@ -50,7 +50,7 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): print(f"Running migration function: {fn.__name__}") toml_data = fn(toml_data) - # Check before overwriting the same file. + # Check before overwriting the same file. if from_path == to_path: print( f"Warning: The source and destination paths are the same ({from_path})." @@ -62,9 +62,7 @@ def migrate_config(self, migrate_choice: str, from_path: str, to_path: str): "This will overwrite your original config file. Continue?", ASSUME_YES ) if not confirm: - raise SystemExit( - "Migration aborted by user. No changes were made." - ) + raise SystemExit("Migration aborted by user. No changes were made.") # Backup old config print(f"Backing up old config to {default_backup_path}") shutil.copyfile(from_path, default_backup_path) diff --git a/node_management/config_migration/src/common/utils.py b/node_management/config_migration/src/common/utils.py index f10c7130..675a5afb 100644 --- a/node_management/config_migration/src/common/utils.py +++ b/node_management/config_migration/src/common/utils.py @@ -54,9 +54,9 @@ def scan_and_recommend_updates( ): """ Scan the original table and recommend updates to the new version's table. - + Only scan top level keys and values (i.e. skip nested tables). - + If a key exists in both tables, compare their values and prompt the user to either keep the original value or use the new version's recommended value. """ diff --git a/node_management/config_migration/src/rpc_config/from_v7_to_v9.py b/node_management/config_migration/src/rpc_config/from_v7_to_v9.py index 7f7344d6..cb4a7866 100644 --- a/node_management/config_migration/src/rpc_config/from_v7_to_v9.py +++ b/node_management/config_migration/src/rpc_config/from_v7_to_v9.py @@ -103,7 +103,7 @@ def __migrate_snapshot_config(v7_toml_data, v9_toml_data): raise SystemExit( "Error: [database_setup.snapshot_config] table should exist in v9 template." ) - + v9_snapshot_config = v9_toml_data["database_setup"]["snapshot_config"] v7_snapshot_config = v7_toml_data["database_setup"]["snapshot_config"] diff --git a/node_management/config_migration/src/smr_settings/from_v7_to_v9.py b/node_management/config_migration/src/smr_settings/from_v7_to_v9.py index 510c97cc..376c2557 100644 --- a/node_management/config_migration/src/smr_settings/from_v7_to_v9.py +++ b/node_management/config_migration/src/smr_settings/from_v7_to_v9.py @@ -20,6 +20,7 @@ import importlib.resources from common.utils import scan_and_recommend_updates + def __migrate_node_root_config(v7_toml_data, v9_toml_data): if "ws_server" in v7_toml_data["node"]: raise SystemExit( @@ -33,20 +34,27 @@ def __migrate_node_root_config(v7_toml_data, v9_toml_data): v9_node_ws_certificates = v9_node_data["ws_server"]["certificates"] v9_node_ws_certificates["root_ca_cert_path"] = v7_node_data["root_ca_cert_path"] v9_node_ws_certificates["cert_path"] = v7_node_data["server_cert_path"] - v9_node_ws_certificates["private_key_path"] = v7_node_data["server_private_key_path"] + v9_node_ws_certificates["private_key_path"] = v7_node_data[ + "server_private_key_path" + ] print("\nScanning node root configuration ...") scan_and_recommend_updates(v7_node_data, v9_node_data) def __migrate_db_chain_store(v7_toml_data, v9_toml_data): - v9_db_chain_store = v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] - v7_db_chain_store = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"]["rocks_db"] + v9_db_chain_store = v9_toml_data["node"]["database_setup"]["dbs"]["chain_store"][ + "rocks_db" + ] + v7_db_chain_store = v7_toml_data["node"]["database_setup"]["dbs"]["chain_store"][ + "rocks_db" + ] v9_db_chain_store["path"] = v7_db_chain_store["path"] print("\nScanning chain store configuration ...") scan_and_recommend_updates(v7_db_chain_store, v9_db_chain_store) + def __migrate_db_ledger(v7_toml_data, v9_toml_data): v9_db_ledger = v9_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] v7_db_ledger = v7_toml_data["node"]["database_setup"]["dbs"]["ledger"]["rocks_db"] @@ -55,6 +63,7 @@ def __migrate_db_ledger(v7_toml_data, v9_toml_data): print("\nScanning ledger configuration ...") scan_and_recommend_updates(v7_db_ledger, v9_db_ledger) + def __migrate_snapshot_config(v7_toml_data, v9_toml_data): """ snapshot_config is optional, @@ -62,7 +71,7 @@ def __migrate_snapshot_config(v7_toml_data, v9_toml_data): """ if "snapshot_config" not in v7_toml_data["node"]["database_setup"]: return - + if "snapshot_config" not in v9_toml_data["node"]["database_setup"]: raise SystemExit( "Error: [node.database_setup.snapshot_config] table should exist in v9 template." @@ -76,13 +85,14 @@ def __migrate_snapshot_config(v7_toml_data, v9_toml_data): print("\nScanning snapshot configuration ...") scan_and_recommend_updates(v7_snapshot_config, v9_snapshot_config) + def __migrate_prune_config(v7_toml_data, v9_toml_data): """ prune_config is optional, so we skip if it does not exist in v7. """ if "prune_config" not in v7_toml_data["node"]["database_setup"]: return - + if "prune_config" not in v9_toml_data["node"]["database_setup"]: raise SystemExit( "Error: [node.database_setup.prune_config] table should exist in v9 template." @@ -96,13 +106,13 @@ def __migrate_prune_config(v7_toml_data, v9_toml_data): def __migrate_mempool_config(v7_toml_data, v9_toml_data): - v9_mempool_config = v9_toml_data["mempool"] v7_mempool_config = v7_toml_data["mempool"] print("\nScanning mempool configuration ...") scan_and_recommend_updates(v7_mempool_config, v9_mempool_config) + def __migrate_moonshot_config(v7_toml_data, v9_toml_data): v9_moonshot_config = v9_toml_data["moonshot"] v7_moonshot_config = v7_toml_data["moonshot"] @@ -110,6 +120,7 @@ def __migrate_moonshot_config(v7_toml_data, v9_toml_data): print("\nScanning moonshot configuration ...") scan_and_recommend_updates(v7_moonshot_config, v9_moonshot_config) + def migrate_v7_to_v9(v7_toml_data): """ Returns a new TOML data structure that is compatible with SMR settings v9. diff --git a/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml index 3b65a2f5..8d1aaf4f 100644 --- a/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml +++ b/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml @@ -1,4 +1,3 @@ - # Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### diff --git a/node_management/config_migration/tests/test_rpc_migration.py b/node_management/config_migration/tests/test_rpc_migration.py index a7b4e0dc..8795ad71 100644 --- a/node_management/config_migration/tests/test_rpc_migration.py +++ b/node_management/config_migration/tests/test_rpc_migration.py @@ -21,6 +21,7 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): shutil.copy(from_path, tmp_from) # Run migration import common.globals + common.globals.ASSUME_YES = True rpc_run_migration(migrate_path, str(tmp_from), str(tmp_to)) # Load both files diff --git a/node_management/config_migration/tests/test_smr_migration.py b/node_management/config_migration/tests/test_smr_migration.py index ef6eef8b..4f9b0c9a 100644 --- a/node_management/config_migration/tests/test_smr_migration.py +++ b/node_management/config_migration/tests/test_smr_migration.py @@ -21,6 +21,7 @@ def test_migration(tmp_path, from_file, to_file, migrate_path): shutil.copy(from_path, tmp_from) # Run migration import common.globals + common.globals.ASSUME_YES = True smr_run_migration(migrate_path, str(tmp_from), str(tmp_to)) From a496619e7060a60fc06895dc55937753948a021a Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Wed, 4 Jun 2025 15:44:05 +0200 Subject: [PATCH 41/44] enable_pruning = true for rpc --- .../rpc_config/rpc_config_v9_1_x_mainnet_template.toml | 7 +++---- .../config_migration/tests/config_v7_to_v9_expected.toml | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml index 6e709640..1d649adf 100644 --- a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml +++ b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -1,4 +1,3 @@ - # Version: v9.1.x mainnet ####################################### PROTOCOL PARAMETERS ####################################### @@ -64,7 +63,7 @@ root_ca_cert_path = "./configs/ca_certificate.pem" path = "./configs/rpc_archive" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. -enable_pruning = false +enable_pruning = true # Whether snapshots should be taken of the database. enable_snapshots = false @@ -74,7 +73,7 @@ enable_snapshots = false path = "./configs/rpc_ledger" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. -enable_pruning = false +enable_pruning = true # Whether snapshots should be taken of the database. enable_snapshots = false @@ -84,7 +83,7 @@ enable_snapshots = false path = "./configs/rpc_store" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. -enable_pruning = false +enable_pruning = true # Whether snapshots should be taken of the database. enable_snapshots = false diff --git a/node_management/config_migration/tests/config_v7_to_v9_expected.toml b/node_management/config_migration/tests/config_v7_to_v9_expected.toml index 71b4040c..7a9ef8f7 100644 --- a/node_management/config_migration/tests/config_v7_to_v9_expected.toml +++ b/node_management/config_migration/tests/config_v7_to_v9_expected.toml @@ -63,7 +63,7 @@ root_ca_cert_path = "./xyz/ca_certificate.pem" path = "./xyz/rpc_archive" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. -enable_pruning = false +enable_pruning = true # Whether snapshots should be taken of the database. enable_snapshots = false @@ -73,7 +73,7 @@ enable_snapshots = false path = "./xyz/rpc_ledger" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. -enable_pruning = false +enable_pruning = true # Whether snapshots should be taken of the database. enable_snapshots = false @@ -83,14 +83,14 @@ enable_snapshots = false path = "./xyz/rpc_store" # Whether the database should be pruned. If `true`, data that is more than `epochs_to_retain` # old will be deleted. -enable_pruning = false +enable_pruning = true # Whether snapshots should be taken of the database. enable_snapshots = false # Parameters related to database pruning. [database_setup.prune_config] # Data stored more than `epochs_to_retain` ago will be pruned if `enable_pruning = true`. -epochs_to_retain = 336 +epochs_to_retain = 1008 # Parameters for the database snapshot service. [database_setup.snapshot_config] From 7f1330e9a67497641c67e20fe94fd353a2b52109 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 5 Jun 2025 09:47:27 +0200 Subject: [PATCH 42/44] add config only migration script for operator --- ...migrate_config_and_db_mainnet_v7_to_v9.sh} | 0 .../migrate_config_mainnet_v7_to_v9_docker.sh | 181 ++++++++++++++++++ 2 files changed, 181 insertions(+) rename node_management/{migrate_mainnet_from_v7_to_v9.sh => migrate_config_and_db_mainnet_v7_to_v9.sh} (100%) create mode 100755 node_management/migrate_config_mainnet_v7_to_v9_docker.sh diff --git a/node_management/migrate_mainnet_from_v7_to_v9.sh b/node_management/migrate_config_and_db_mainnet_v7_to_v9.sh similarity index 100% rename from node_management/migrate_mainnet_from_v7_to_v9.sh rename to node_management/migrate_config_and_db_mainnet_v7_to_v9.sh diff --git a/node_management/migrate_config_mainnet_v7_to_v9_docker.sh b/node_management/migrate_config_mainnet_v7_to_v9_docker.sh new file mode 100755 index 00000000..cfc6d09b --- /dev/null +++ b/node_management/migrate_config_mainnet_v7_to_v9_docker.sh @@ -0,0 +1,181 @@ +#!/bin/bash + + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +SCRIPT_NAME="migrate_from_v7_to_v9" + +# This script is expected to be installed with `install_management_scripts.sh`, which +# creates the `.supra` directory and retrieves the `node_management` directory. +source "$SCRIPT_DIR/.supra/node_management/utils.sh" + +function parse_args() { + NODE_TYPE="$1" + CONTAINER_NAME="$2" + HOST_SUPRA_HOME="$3" +} + +function basic_usage() { + echo "Usage: ./$SCRIPT_NAME.sh " >&2 + echo "Parameters:" >&2 + node_type_usage + container_name_usage + host_supra_home_usage + exit 1 +} + + +function verify_rpc() { + if ! verify_container_name || ! verify_host_supra_home; then + basic_usage + fi +} + +function verify_validator() { + if ! verify_container_name || ! verify_host_supra_home; then + basic_usage + fi +} + +function verify_args() { + if [[ "$NODE_TYPE" == "rpc" ]]; then + verify_rpc + elif [[ "$NODE_TYPE" == "validator" ]]; then + verify_validator + else + basic_usage + fi +} + + +#---------------------------------------------------------- RPC ---------------------------------------------------------- + +function migrate_rpc() { + echo "Migrating RPC $CONTAINER_NAME at $HOST_SUPRA_HOME to v9" + # TODO(sc): replace with mainnet image + RPC_V9_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-testnet/rpc-node:v9.0.12 + + rpc-v9() { docker exec -it rpc-v9 /supra/rpc_node "$@"; } + + echo "Stop the user's container if it is running." + docker rm -f "$CONTAINER_NAME" || : + + echo "Prepare containers needed for running migration." + # Stop+Remove rpc containers for migration if they exist. + docker rm -f rpc-v9 || : + + docker run --name rpc-v9 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$RPC_V9_IMAGE" + + echo "Migrate rpc config from v7 to v9" + # TODO(SC) to run in docker context, update path to `./configs/config.toml` + migrate-config rpc -p v7-v9 -f $HOST_SUPRA_HOME/config.toml -t $HOST_SUPRA_HOME/config.toml + + + echo "Cleanup containers used for migration." + docker rm -f rpc-v9 || : + docker container ls + + echo "Remove snapshot and snapshots directories" + # Remove any existing snapshots. If we don't do this then they will start to take + # up a large amount of disk space during the sync. + # TODO(sc): WHY? + rm -rf "$HOST_SUPRA_HOME/snapshot" + rm -rf "$HOST_SUPRA_HOME/snapshots" + + # Finished migration, and follow guide that start the node with the new image with new config and sync + # ------ + + # ./manage_supra_nodes.sh \ + # sync \ + # --exact-timestamps \ + # --snapshot-source testnet-archive-snapshot \ + # rpc \ + # "$HOST_SUPRA_HOME" \ + # testnet + # echo "Migration complete. Please transfer all custom settings from $v8_config_toml to " + # echo -n "$config_toml before starting your node." + + echo "Migration rpc config complete." + +} + +#---------------------------------------------------------- Validator ---------------------------------------------------------- + +function migrate_validator() { + echo "Migrating validator $CONTAINER_NAME at $HOST_SUPRA_HOME to v9" + + SUPRA_V9_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-mainnet/validator-node:v9.0.5 + SUPRA_V8_IMAGE=asia-docker.pkg.dev/supra-devnet-misc/supra-mainnet/validator-node:v8.0.3 + supra-v8() { docker exec -it supra-v8 /supra/supra "$@"; } + supra-v9() { docker exec -it supra-v9 /supra/supra "$@"; } + + + echo "Stop the users' container if it is running." + docker stop "$CONTAINER_NAME" || : + + echo "Prepare containers needed for running migration." + docker rm -f supra-v8 || : + docker rm -f supra-v9 || : + + # Start supra containers with proper env and volume mounts. + docker run --name supra-v8 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$SUPRA_V8_IMAGE" + + docker run --name supra-v9 \ + -v "$HOST_SUPRA_HOME:/supra/configs" \ + -e "SUPRA_HOME=/supra/configs/" \ + -itd "$SUPRA_V9_IMAGE" + + + + echo "Migrate cli profile from v7 to v8" + supra-v8 migrate --network mainnet + cp $HOST_SUPRA_HOME/validator_identity.pem $HOST_SUPRA_HOME/node_identity.pem + echo "Migrate cli profile from v8 to v9" + supra-v9 profile migrate + echo "Migrate smr_settings from v7 to v9" + # TODO(SC) to be run in docker context, update path to `./configs/config.toml` + migrate-config smr -p v7-v9 -f $HOST_SUPRA_HOME/smr_settings.toml -t $HOST_SUPRA_HOME/smr_settings.toml + + # # Localnet only (Optional: local env path is different from docker env path, need to be modified to use docker env path) + # sed -i "" "s#${HOST_SUPRA_HOME}#configs#g" ${HOST_SUPRA_HOME}/smr_settings.toml + + echo "Cleanup containers used for migration." + docker rm -f supra-v8 || : + docker rm -f supra-v9 || : + docker container ls + + echo "Remove snapshot and snapshots directories" + # Remove any existing snapshots. If we don't do this then they will start to take + # up a large amount of disk space during the sync. + # TODO(sc): WHY? + rm -rf "$HOST_SUPRA_HOME/snapshot" + rm -rf "$HOST_SUPRA_HOME/snapshots" + + # Finished migration, and follow guide that start the node with the new image with new config and sync + + echo "Migration validator config complete." +} + +function main() { + if [ "$#" -lt 3 ]; then + basic_usage + fi + parse_args "$@" + verify_args + ensure_supra_home_is_absolute_path + + if [ "$NODE_TYPE" == "validator" ]; then + migrate_validator + elif [ "$NODE_TYPE" == "rpc" ]; then + migrate_rpc + fi +} + +main "$@" From 3f7c5ecffa0e7f2315d9cff0ac14f9e596e25c54 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 12 Jun 2025 15:15:24 +0200 Subject: [PATCH 43/44] keep originl value in chain instance --- .../rpc_config_v9_1_x_mainnet_template.toml | 8 ++++++-- .../smr_settings_v9_1_x_mainnet_template.toml | 8 ++++---- node_management/manage_supra_nodes.sh | 12 ------------ 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml index 1d649adf..e7817f1b 100644 --- a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml +++ b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -28,9 +28,13 @@ chain_id = 8 epoch_duration_secs = 7200 # The number of seconds that stake locked in a Stake Pool will automatically be locked up for when # its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 +# +# 48 hours. +chain_instance.recurring_lockup_duration_secs = 172800 # The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 +# +# 46 hours. +chain_instance.voting_duration_secs = 165600 # Determines whether the network will start with a faucet, amongst other things. is_testnet = false # Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). diff --git a/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml index 8d1aaf4f..2cb73209 100644 --- a/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml +++ b/node_management/config_migration/src/smr_settings/smr_settings_v9_1_x_mainnet_template.toml @@ -15,13 +15,13 @@ epoch_duration_secs = 7200 # The number of seconds that stake locked in a Stake Pool will automatically be locked up for when # its current lockup expires, if no request is made to unlock it. # -# 4 hours. -recurring_lockup_duration_secs = 14400 +# 48 hours. +recurring_lockup_duration_secs = 172800 # The number of seconds allocated for voting on governance proposals. Governance will initially be # controlled by The Supra Foundation. # -# 2 hours. -voting_duration_secs = 7200 +# 46 hours. +voting_duration_secs = 165600 # Determines whether the network will start with a faucet, amongst other things. is_testnet = false # Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). diff --git a/node_management/manage_supra_nodes.sh b/node_management/manage_supra_nodes.sh index a770bd5f..7fc3dedc 100755 --- a/node_management/manage_supra_nodes.sh +++ b/node_management/manage_supra_nodes.sh @@ -512,24 +512,12 @@ function update() { #---------------------------------------------------------- Start ---------------------------------------------------------- -function copy_rpc_root_config_files() { - docker cp "$HOST_SUPRA_HOME"/config.toml "$CONTAINER_NAME:/supra/" - docker cp "$HOST_SUPRA_HOME"/genesis.blob "$CONTAINER_NAME:/supra/" -} - -function copy_validator_root_config_files() { - docker cp "$HOST_SUPRA_HOME"/smr_settings.toml "$CONTAINER_NAME:/supra/" - docker cp "$HOST_SUPRA_HOME"/genesis.blob "$CONTAINER_NAME:/supra/" -} - function start_rpc_node() { - copy_rpc_root_config_files start_rpc_docker_container docker exec -itd $CONTAINER_NAME /supra/rpc_node start } function start_validator_node() { - copy_validator_root_config_files start_validator_docker_container prompt_for_cli_password From 72be37daaac095f51b61aeb6f49855c5956be811 Mon Sep 17 00:00:00 2001 From: Simon Chen Date: Thu, 12 Jun 2025 15:40:44 +0200 Subject: [PATCH 44/44] fix test --- .../rpc_config/rpc_config_v9_1_x_mainnet_template.toml | 4 ++-- .../config_migration/tests/config_v7_to_v9_expected.toml | 8 ++++++-- .../tests/smr_settings_v7_to_v9_expected.toml | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml index e7817f1b..8d599e43 100644 --- a/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml +++ b/node_management/config_migration/src/rpc_config/rpc_config_v9_1_x_mainnet_template.toml @@ -30,11 +30,11 @@ epoch_duration_secs = 7200 # its current lockup expires, if no request is made to unlock it. # # 48 hours. -chain_instance.recurring_lockup_duration_secs = 172800 +recurring_lockup_duration_secs = 172800 # The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. # # 46 hours. -chain_instance.voting_duration_secs = 165600 +voting_duration_secs = 165600 # Determines whether the network will start with a faucet, amongst other things. is_testnet = false # Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). diff --git a/node_management/config_migration/tests/config_v7_to_v9_expected.toml b/node_management/config_migration/tests/config_v7_to_v9_expected.toml index 7a9ef8f7..420f6bab 100644 --- a/node_management/config_migration/tests/config_v7_to_v9_expected.toml +++ b/node_management/config_migration/tests/config_v7_to_v9_expected.toml @@ -28,9 +28,13 @@ chain_id = 8 epoch_duration_secs = 7200 # The number of seconds that stake locked in a Stake Pool will automatically be locked up for when # its current lockup expires, if no request is made to unlock it. -recurring_lockup_duration_secs = 14400 +# +# 48 hours. +recurring_lockup_duration_secs = 172800 # The number of seconds allocated for voting on governance proposals. Governance will initially be controlled by The Supra Foundation. -voting_duration_secs = 7200 +# +# 46 hours. +voting_duration_secs = 165600 # Determines whether the network will start with a faucet, amongst other things. is_testnet = false # Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC). diff --git a/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml b/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml index eef5ffb5..d9c3e73f 100644 --- a/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml +++ b/node_management/config_migration/tests/smr_settings_v7_to_v9_expected.toml @@ -15,13 +15,13 @@ epoch_duration_secs = 7200 # The number of seconds that stake locked in a Stake Pool will automatically be locked up for when # its current lockup expires, if no request is made to unlock it. # -# 4 hours. -recurring_lockup_duration_secs = 14400 +# 48 hours. +recurring_lockup_duration_secs = 172800 # The number of seconds allocated for voting on governance proposals. Governance will initially be # controlled by The Supra Foundation. # -# 2 hours. -voting_duration_secs = 7200 +# 46 hours. +voting_duration_secs = 165600 # Determines whether the network will start with a faucet, amongst other things. is_testnet = false # Wednesday, Nov 20, 2024 12:00:00.000 AM (UTC).