From 5740595506885a0c4ce41c51f4175a861c670db9 Mon Sep 17 00:00:00 2001 From: ping-ke Date: Mon, 29 Sep 2025 17:47:34 +0800 Subject: [PATCH 1/7] refactor --- cmd/es-node/config.go | 263 ------------------------------ cmd/es-node/main.go | 20 +-- cmd/es-node/utils.go | 149 ++--------------- cmd/es-node/utils_test.go | 9 +- ethstorage/eth/polling_client.go | 28 ++++ ethstorage/miner/worker.go | 24 +-- ethstorage/node/config.go | 271 +++++++++++++++++++++++++++++++ 7 files changed, 342 insertions(+), 422 deletions(-) delete mode 100644 cmd/es-node/config.go diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go deleted file mode 100644 index 9c778939..00000000 --- a/cmd/es-node/config.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright 2022-2023, EthStorage. -// For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE - -package main - -import ( - "context" - "fmt" - "math/big" - - oppprof "github.com/ethereum-optimism/optimism/op-service/pprof" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/log" - "github.com/ethstorage/go-ethstorage/ethstorage/archiver" - "github.com/ethstorage/go-ethstorage/ethstorage/db" - "github.com/ethstorage/go-ethstorage/ethstorage/downloader" - "github.com/ethstorage/go-ethstorage/ethstorage/email" - "github.com/ethstorage/go-ethstorage/ethstorage/eth" - "github.com/ethstorage/go-ethstorage/ethstorage/flags" - "github.com/ethstorage/go-ethstorage/ethstorage/miner" - "github.com/ethstorage/go-ethstorage/ethstorage/node" - p2pcli "github.com/ethstorage/go-ethstorage/ethstorage/p2p/cli" - "github.com/ethstorage/go-ethstorage/ethstorage/scanner" - "github.com/ethstorage/go-ethstorage/ethstorage/signer" - "github.com/ethstorage/go-ethstorage/ethstorage/storage" - "github.com/urfave/cli" -) - -// NewConfig creates a Config from the provided flags or environment variables. -func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { - if err := flags.CheckRequired(ctx); err != nil { - return nil, err - } - - datadir := ctx.GlobalString(flags.DataDir.Name) - - // TODO: blocktime is set to zero, need to update the value - p2pConfig, err := p2pcli.NewConfig(ctx, 0) - if err != nil { - return nil, fmt.Errorf("failed to load p2p config: %w", err) - } - - l1Endpoint, client, err := NewL1EndpointConfig(ctx, lg) - if err != nil { - return nil, err - } - lg.Info("Read L1 config", flags.L1NodeAddr.Name, l1Endpoint.L1NodeAddr) - lg.Info("Read L1 config", flags.L1BeaconAddr.Name, l1Endpoint.L1BeaconURL) - defer client.Close() - - storageConfig, err := NewStorageConfig(ctx, client, lg) - if err != nil { - return nil, fmt.Errorf("failed to load storage config: %w", err) - } - - dlConfig := NewDownloaderConfig(ctx) - minerConfig, err := NewMinerConfig(ctx, client, storageConfig.L1Contract, storageConfig.Miner, lg) - if err != nil { - return nil, fmt.Errorf("failed to load miner config: %w", err) - } - chainId := new(big.Int).SetUint64(ctx.GlobalUint64(flags.ChainId.Name)) - lg.Info("Read chain ID of EthStorage network", "chainID", chainId) - if minerConfig != nil { - minerConfig.ChainID = chainId - } - archiverConfig := archiver.NewConfig(ctx) - // l2Endpoint, err := NewL2EndpointConfig(ctx, lg) - // if err != nil { - // return nil, fmt.Errorf("failed to load l2 endpoints info: %w", err) - // } - - // l2SyncEndpoint := NewL2SyncEndpointConfig(ctx) - cfg := &node.Config{ - L1: *l1Endpoint, - ChainID: chainId, - Downloader: *dlConfig, - - DataDir: datadir, - StateUploadURL: ctx.GlobalString(flags.StateUploadURL.Name), - DBConfig: db.DefaultDBConfig(), - // rpc url to get randao from - RandaoSourceURL: ctx.GlobalString(flags.RandaoURL.Name), - // Driver: *driverConfig, - RPC: node.RPCConfig{ - ListenAddr: ctx.GlobalString(flags.RPCListenAddr.Name), - ListenPort: ctx.GlobalInt(flags.RPCListenPort.Name), - ESCallURL: ctx.GlobalString(flags.RPCESCallURL.Name), - }, - Metrics: node.MetricsConfig{ - Enabled: ctx.GlobalBool(flags.MetricsEnabledFlag.Name), - ListenAddr: ctx.GlobalString(flags.MetricsAddrFlag.Name), - ListenPort: ctx.GlobalInt(flags.MetricsPortFlag.Name), - }, - Pprof: oppprof.CLIConfig{ - Enabled: ctx.GlobalBool(flags.PprofEnabledFlag.Name), - ListenAddr: ctx.GlobalString(flags.PprofAddrFlag.Name), - ListenPort: ctx.GlobalInt(flags.PprofPortFlag.Name), - }, - P2P: p2pConfig, - - L1EpochPollInterval: ctx.GlobalDuration(flags.L1EpochPollIntervalFlag.Name), - // Heartbeat: node.HeartbeatConfig{ - // Enabled: ctx.GlobalBool(flags.HeartbeatEnabledFlag.Name), - // Moniker: ctx.GlobalString(flags.HeartbeatMonikerFlag.Name), - // URL: ctx.GlobalString(flags.HeartbeatURLFlag.Name), - // }, - Storage: *storageConfig, - Mining: minerConfig, - Archiver: archiverConfig, - Scanner: scanner.NewConfig(ctx), - } - if err := cfg.Check(); err != nil { - return nil, err - } - return cfg, nil -} - -func NewMinerConfig(ctx *cli.Context, client *ethclient.Client, l1Contract, minerAddr common.Address, lg log.Logger) (*miner.Config, error) { - cliConfig := miner.ReadCLIConfig(ctx) - if !cliConfig.Enabled { - lg.Info("Miner is not enabled.") - return nil, nil - } - if minerAddr == (common.Address{}) { - return nil, fmt.Errorf("miner address cannot be empty") - } - lg.Debug("Read mining config from cli", "config", fmt.Sprintf("%+v", cliConfig)) - err := cliConfig.Check() - if err != nil { - return nil, fmt.Errorf("invalid miner flags: %w", err) - } - minerConfig, err := cliConfig.ToMinerConfig() - if err != nil { - return nil, err - } - if minerConfig.EmailEnabled { - emailConfig, err := email.GetEmailConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get email config: %w", err) - } - minerConfig.EmailConfig = *emailConfig - } - - cctx := context.Background() - cr := newContractReader(cctx, client, l1Contract, lg) - - randomChecks, err := cr.readUint("randomChecks") - if err != nil { - return nil, err - } - minerConfig.RandomChecks = randomChecks - nonceLimit, err := cr.readUint("nonceLimit") - if err != nil { - return nil, err - } - minerConfig.NonceLimit = nonceLimit - minimumDiff, err := cr.readBig("minimumDiff") - if err != nil { - return nil, err - } - minerConfig.MinimumDiff = minimumDiff - cutoff, err := cr.readBig("cutoff") - if err != nil { - return nil, err - } - minerConfig.Cutoff = cutoff - diffAdjDivisor, err := cr.readBig("diffAdjDivisor") - if err != nil { - return nil, err - } - minerConfig.DiffAdjDivisor = diffAdjDivisor - dcf, err := cr.readBig("dcfFactor") - if err != nil { - return nil, err - } - minerConfig.DcfFactor = dcf - - startTime, err := cr.readUint("startTime") - if err != nil { - return nil, err - } - minerConfig.StartTime = startTime - shardEntryBits, err := cr.readUint("shardEntryBits") - if err != nil { - return nil, err - } - minerConfig.ShardEntry = 1 << shardEntryBits - treasuryShare, err := cr.readUint("treasuryShare") - if err != nil { - return nil, err - } - minerConfig.TreasuryShare = treasuryShare - storageCost, err := cr.readBig("storageCost") - if err != nil { - return nil, err - } - minerConfig.StorageCost = storageCost - prepaidAmount, err := cr.readBig("prepaidAmount") - if err != nil { - return nil, err - } - minerConfig.PrepaidAmount = prepaidAmount - signerFnFactory, signerAddr, err := NewSignerConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get signer: %w", err) - } - minerConfig.SignerFnFactory = signerFnFactory - minerConfig.SignerAddr = signerAddr - return &minerConfig, nil -} - -func NewSignerConfig(ctx *cli.Context) (signer.SignerFactory, common.Address, error) { - signerConfig := signer.ReadCLIConfig(ctx) - if err := signerConfig.Check(); err != nil { - return nil, common.Address{}, fmt.Errorf("invalid siger flags: %w", err) - } - return signer.SignerFactoryFromConfig(signerConfig) -} - -func NewStorageConfig(ctx *cli.Context, client *ethclient.Client, lg log.Logger) (*storage.StorageConfig, error) { - l1Contract := common.HexToAddress(ctx.GlobalString(flags.StorageL1Contract.Name)) - miner := common.HexToAddress(ctx.GlobalString(flags.StorageMiner.Name)) - lg.Info("Loaded storage config", "l1Contract", l1Contract, "miner", miner) - storageCfg, err := initStorageConfig(context.Background(), client, l1Contract, miner, lg) - if err != nil { - lg.Error("Failed to load storage config from contract", "error", err) - return nil, err - } - storageCfg.Filenames = ctx.GlobalStringSlice(flags.StorageFiles.Name) - return storageCfg, nil -} - -func NewL1EndpointConfig(ctx *cli.Context, lg log.Logger) (*eth.L1EndpointConfig, *ethclient.Client, error) { - l1NodeAddr := ctx.GlobalString(flags.L1NodeAddr.Name) - client, err := ethclient.DialContext(context.Background(), l1NodeAddr) - if err != nil { - lg.Error("Failed to connect to the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) - return nil, nil, err - } - chainid, err := client.ChainID(context.Background()) - if err != nil { - lg.Error("Failed to fetch chain id from the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) - return nil, nil, err - } - return ð.L1EndpointConfig{ - L1ChainID: chainid.Uint64(), - L1NodeAddr: l1NodeAddr, - L1BlockTime: ctx.GlobalUint64(flags.L1BlockTime.Name), - L1BeaconURL: ctx.GlobalString(flags.L1BeaconAddr.Name), - L1BeaconSlotTime: ctx.GlobalUint64(flags.L1BeaconSlotTime.Name), - DAURL: ctx.GlobalString(flags.DAURL.Name), - L1MinDurationForBlobsRequest: ctx.GlobalUint64(flags.L1MinDurationForBlobsRequest.Name), - }, client, nil -} - -func NewDownloaderConfig(ctx *cli.Context) *downloader.Config { - return &downloader.Config{ - DownloadStart: ctx.GlobalInt64(flags.DownloadStart.Name), - DownloadDump: ctx.GlobalString(flags.DownloadDump.Name), - DownloadThreadNum: ctx.GlobalInt(flags.DownloadThreadNum.Name), - } -} diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 74cbc5e1..59413810 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -16,7 +16,6 @@ import ( oppprof "github.com/ethereum-optimism/optimism/op-service/pprof" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" "github.com/ethstorage/go-ethstorage/ethstorage" "github.com/ethstorage/go-ethstorage/ethstorage/email" @@ -154,7 +153,7 @@ func EsNodeMain(ctx *cli.Context) error { lgCfg := log.ReadCLIConfig(ctx) lg.Info("Loading log config", "config", lgCfg) clog := log.NewLogger(lgCfg) - cfg, err := NewConfig(ctx, clog) + cfg, err := node.NewConfig(ctx, clog) if err != nil { lg.Error("Unable to create the rollup node config", "error", err) return err @@ -249,16 +248,17 @@ func EsNodeInit(ctx *cli.Context) error { } shardLen = shards } + cctx := context.Background() - client, err := ethclient.DialContext(cctx, l1Rpc) + l1Contract := common.HexToAddress(contract) + client, err := eth.DialContext(cctx, l1Rpc, l1Contract, 12, lg) if err != nil { lg.Error("Failed to connect to the Ethereum client", "error", err, "l1Rpc", l1Rpc) return err } defer client.Close() - l1Contract := common.HexToAddress(contract) - storageCfg, err := initStorageConfig(cctx, client, l1Contract, common.HexToAddress(miner), lg) + storageCfg, err := node.InitStorageConfig(client, common.HexToAddress(miner), lg) if err != nil { lg.Error("Failed to load storage config", "error", err) return err @@ -268,18 +268,18 @@ func EsNodeInit(ctx *cli.Context) error { if len(shardIndexes) > 0 { out: for i := 0; i < len(shardIndexes); i++ { - new := uint64(shardIndexes[i]) + n := uint64(shardIndexes[i]) // prevent duplicated for _, s := range shardIdxList { - if s == new { + if s == n { continue out } } - shardIdxList = append(shardIdxList, new) + shardIdxList = append(shardIdxList, n) } } else { // get shard indexes of length shardLen from contract - shardList, err := getShardList(cctx, client, l1Contract, shardLen) + shardList, err := getTopNShardListSortByDiff(cctx, client, shardLen) if err != nil { lg.Error("Failed to get shard indexes from contract", "error", err) return err @@ -339,7 +339,7 @@ func EsNodeSync(ctx *cli.Context) error { } lg.Info("Download blob from RPC done", "kvIndex", kvIndex, "commit", commit.Hex()) // write blob and meta - shardManager, err := initShardManager(ctx, l1Rpc, l1contract) + shardManager, err := initShardManager(ctx, l1Rpc, l1contract, lg) if err != nil { return fmt.Errorf("failed to init shard manager: %w", err) } diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index 443fcd88..803aecc3 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -14,17 +14,16 @@ import ( "path/filepath" "sort" - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/kzg4844" - "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" es "github.com/ethstorage/go-ethstorage/ethstorage" + "github.com/ethstorage/go-ethstorage/ethstorage/eth" "github.com/ethstorage/go-ethstorage/ethstorage/flags" + "github.com/ethstorage/go-ethstorage/ethstorage/miner" + "github.com/ethstorage/go-ethstorage/ethstorage/node" "github.com/ethstorage/go-ethstorage/ethstorage/storage" "github.com/urfave/cli" ) @@ -38,105 +37,22 @@ const ( esRpcFlagName = "es_rpc" ) -type ContractReader struct { - ctx context.Context - client *ethclient.Client - contract common.Address - lg log.Logger -} - -func newContractReader(ctx context.Context, client *ethclient.Client, contract common.Address, logger log.Logger) *ContractReader { - return &ContractReader{ - ctx: ctx, - client: client, - contract: contract, - lg: logger, - } -} - -func (c *ContractReader) readSlot(fieldName string) ([]byte, error) { - h := crypto.Keccak256Hash([]byte(fieldName + "()")) - msg := ethereum.CallMsg{ - To: &c.contract, - Data: h[0:4], - } - bs, err := c.client.CallContract(c.ctx, msg, nil) - if err != nil { - return nil, fmt.Errorf("failed to get %s from contract: %v", fieldName, err) - } - return bs, nil -} - -func (c *ContractReader) readUint(fieldName string) (uint64, error) { - bs, err := c.readSlot(fieldName) - if err != nil { - return 0, err - } - value := new(big.Int).SetBytes(bs).Uint64() - c.lg.Info("Read uint from contract", "field", fieldName, "value", value) - return value, nil -} - -func (c *ContractReader) readBig(fieldName string) (*big.Int, error) { - bs, err := c.readSlot(fieldName) - if err != nil { - return nil, err - } - value := new(big.Int).SetBytes(bs) - c.lg.Info("Read big int from contract", "field", fieldName, "value", value) - return new(big.Int).SetBytes(bs), nil -} - -func (c *ContractReader) contractExists() (bool, error) { - code, err := c.client.CodeAt(c.ctx, c.contract, nil) - if err != nil { - return false, fmt.Errorf("failed to get code at %s: %v", c.contract.Hex(), err) - } - return len(code) > 0, nil -} - -func initStorageConfig(ctx context.Context, client *ethclient.Client, l1Contract, miner common.Address, lg log.Logger) (*storage.StorageConfig, error) { - cr := newContractReader(ctx, client, l1Contract, lg) - exist, err := cr.contractExists() - if err != nil { - return nil, err - } - if !exist { - return nil, fmt.Errorf("contract does not exist") - } - maxKvSizeBits, err := cr.readUint("maxKvSizeBits") - if err != nil { - return nil, err - } - chunkSizeBits := maxKvSizeBits - shardEntryBits, err := cr.readUint("shardEntryBits") - if err != nil { - return nil, err - } - return &storage.StorageConfig{ - L1Contract: l1Contract, - Miner: miner, - KvSize: 1 << maxKvSizeBits, - ChunkSize: 1 << chunkSizeBits, - KvEntriesPerShard: 1 << shardEntryBits, - }, nil -} - -func getShardList(ctx context.Context, client *ethclient.Client, contract common.Address, shardLen int) ([]uint64, error) { +func getTopNShardListSortByDiff(ctx context.Context, pClient *eth.PollingClient, n int) ([]uint64, error) { var shardId uint64 = 0 var diffs []*big.Int + api := miner.NewL1MiningAPI(pClient, nil, nil) for { - diff, err := getDifficulty(ctx, client, contract, shardId) + info, err := api.GetMiningInfo(ctx, pClient.ContractAddress(), shardId) if err != nil { lg.Error("Query difficulty by shard", "error", err) break } - if diff != nil && diff.Cmp(big.NewInt(0)) == 0 { + if info.Difficulty != nil && info.Difficulty.Cmp(big.NewInt(0)) == 0 { // shardId not exist yet break } - lg.Info("Query difficulty by shard", "shard", shardId, "difficulty", diff) - diffs = append(diffs, diff) + lg.Info("Query difficulty by shard", "shard", shardId, "difficulty", info.Difficulty) + diffs = append(diffs, info.Difficulty) shardId++ } // get the shards with lowest difficulty @@ -146,10 +62,10 @@ func getShardList(ctx context.Context, client *ethclient.Client, contract common // Will create at least one data file result = []uint64{0} } else { - if len(sortedShardIds) < shardLen { - shardLen = len(sortedShardIds) + if len(sortedShardIds) < n { + n = len(sortedShardIds) } - for i := 0; i < shardLen; i++ { + for i := 0; i < n; i++ { result = append(result, uint64(sortedShardIds[i])) } } @@ -157,40 +73,6 @@ func getShardList(ctx context.Context, client *ethclient.Client, contract common return result, nil } -func getDifficulty(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) (*big.Int, error) { - res, err := getMiningInfo(ctx, client, contract, shardIdx) - if err != nil { - return nil, err - } - return res[1].(*big.Int), nil -} - -func getMiningInfo(ctx context.Context, client *ethclient.Client, contract common.Address, shardIdx uint64) ([]interface{}, error) { - uint256Type, _ := abi.NewType("uint256", "", nil) - dataField, _ := abi.Arguments{{Type: uint256Type}}.Pack(new(big.Int).SetUint64(shardIdx)) - h := crypto.Keccak256Hash([]byte(`infos(uint256)`)) - calldata := append(h[0:4], dataField...) - msg := ethereum.CallMsg{ - To: &contract, - Data: calldata, - } - bs, err := client.CallContract(ctx, msg, nil) - if err != nil { - lg.Error("Failed to call contract", "error", err.Error()) - return nil, err - } - res, _ := abi.Arguments{ - {Type: uint256Type}, - {Type: uint256Type}, - {Type: uint256Type}, - }.UnpackValues(bs) - if res == nil || len(res) < 3 { - lg.Error("Query mining info by shard", "error", "invalid result", "result", res) - return nil, fmt.Errorf("invalid result: %v", res) - } - return res, nil -} - func createDataFile(cfg *storage.StorageConfig, shardIdxList []uint64, datadir string, encodingType int) ([]string, error) { lg.Info("Creating data files", "shardIdxList", shardIdxList, "dataDir", datadir) if _, err := os.Stat(datadir); os.IsNotExist(err) { @@ -276,19 +158,18 @@ func downloadBlobFromRPC(endpoint string, kvIndex uint64, hash common.Hash) ([]b return result, nil } -func initShardManager(ctx *cli.Context, l1Rpc string, l1contract common.Address) (*es.ShardManager, error) { +func initShardManager(ctx *cli.Context, l1Rpc string, l1contract common.Address, lg log.Logger) (*es.ShardManager, error) { miner := readRequiredFlag(ctx, flags.StorageMiner) if !common.IsHexAddress(miner) { return nil, fmt.Errorf("invalid miner address %s", miner) } - cctx := context.Background() - client, err := ethclient.DialContext(cctx, l1Rpc) + client, err := eth.Dial(l1Rpc, l1contract, 0, lg) if err != nil { return nil, fmt.Errorf("failed to connect to the Ethereum client: %w", err) } defer client.Close() - storageCfg, err := initStorageConfig(cctx, client, l1contract, common.HexToAddress(miner), lg) + storageCfg, err := node.InitStorageConfig(client, common.HexToAddress(miner), lg) if err != nil { return nil, fmt.Errorf("failed to load storage config: %w", err) } diff --git a/cmd/es-node/utils_test.go b/cmd/es-node/utils_test.go index f91842af..99979ece 100644 --- a/cmd/es-node/utils_test.go +++ b/cmd/es-node/utils_test.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethstorage/go-ethstorage/ethstorage" + "github.com/ethstorage/go-ethstorage/ethstorage/eth" "github.com/ethstorage/go-ethstorage/ethstorage/storage" ) @@ -53,16 +54,18 @@ func TestCreateDataFile(t *testing.T) { false, }, } - client, err := ethclient.DialContext(context.Background(), "http://65.108.236.27:8545") + ctx := context.Background() + client, err := ethclient.DialContext(ctx, "http://65.108.236.27:8545") if err != nil { t.Fatalf("connect to L1 error: %v ", err) } defer client.Close() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - shardList, err := getShardList(context.Background(), client, tt.args.cfg.L1Contract, tt.args.sLen) + pClient := eth.NewClient(ctx, client, true, tt.args.cfg.L1Contract, 0, nil, lg) + shardList, err := getTopNShardListSortByDiff(ctx, pClient, tt.args.sLen) if err != nil { - t.Fatalf("getShardList() error: %v ", err) + t.Fatalf("getTopNShardListSortByDiff() error: %v ", err) } files, err := createDataFile(tt.args.cfg, shardList, ".", ethstorage.ENCODE_BLOB_POSEIDON) if err != nil { diff --git a/ethstorage/eth/polling_client.go b/ethstorage/eth/polling_client.go index 95169cb7..114a2b6c 100644 --- a/ethstorage/eth/polling_client.go +++ b/ethstorage/eth/polling_client.go @@ -334,6 +334,22 @@ func (w *PollingClient) ReadContractField(fieldName string, blockNumber *big.Int return bs, nil } +func (w *PollingClient) ReadContractBigIntField(fieldName string, blockNumber *big.Int) (*big.Int, error) { + bs, err := w.ReadContractField(fieldName, blockNumber) + if err != nil { + return nil, err + } + return new(big.Int).SetBytes(bs), nil +} + +func (w *PollingClient) ReadContractUint64Field(fieldName string, blockNumber *big.Int) (uint64, error) { + bs, err := w.ReadContractField(fieldName, blockNumber) + if err != nil { + return 0, err + } + return new(big.Int).SetBytes(bs).Uint64(), nil +} + func (w *PollingClient) GetContractVersion() (string, error) { bs, err := w.ReadContractField("version", nil) if err != nil { @@ -350,6 +366,18 @@ func (w *PollingClient) GetContractVersion() (string, error) { return version, nil } +func (w *PollingClient) IsContractExist() (bool, error) { + code, err := w.CodeAt(w.ctx, w.esContract, nil) + if err != nil { + return false, fmt.Errorf("failed to get code at %s: %v", w.esContract.Hex(), err) + } + return len(code) > 0, nil +} + +func (w *PollingClient) ContractAddress() common.Address { + return w.esContract +} + func decodeString(data []byte) (string, error) { if len(data) < 64 { return "", fmt.Errorf("data too short") diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index 47b9257e..561dbbc4 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -423,35 +423,35 @@ func (w *worker) resultLoop() { for { select { case <-w.resultCh: - result := w.getResult() - if result == nil { + r := w.getResult() + if r == nil { continue } - w.lg.Info("Mining result loop get result", "shard", result.startShardId, "block", result.blockNumber, "nonce", result.nonce) + w.lg.Info("Mining result loop get r", "shard", r.startShardId, "block", r.blockNumber, "nonce", r.nonce) txHash, err := w.l1API.SubmitMinedResult( context.Background(), w.storageMgr.ContractAddress(), - *result, + *r, w.config, ) - if s, ok := w.submissionStates[result.startShardId]; ok { + if s, ok := w.submissionStates[r.startShardId]; ok { if err != nil { var dropErr errDropped if errors.As(err, &dropErr) { s.Dropped++ w.lg.Warn("Mining transaction dropped", - "shard", result.startShardId, - "block", result.blockNumber, + "shard", r.startShardId, + "block", r.blockNumber, "reason", dropErr.reason) } else { s.Failed++ - errorCache = append(errorCache, miningError{result.startShardId, result.blockNumber, err}) + errorCache = append(errorCache, miningError{r.startShardId, r.blockNumber, err}) var diff *big.Int if strings.Contains(err.Error(), "StorageContract_DifficultyNotMet") { info, err := w.l1API.GetMiningInfo( context.Background(), w.storageMgr.ContractAddress(), - result.startShardId, + r.startShardId, ) if err != nil { w.lg.Warn("Failed to get es mining info", "error", err.Error()) @@ -460,9 +460,9 @@ func (w *worker) resultLoop() { } } if diff != nil { - w.lg.Error("Failed to submit mined result", "shard", result.startShardId, "block", result.blockNumber, "difficulty", diff, "error", err.Error()) + w.lg.Error("Failed to submit mined r", "shard", r.startShardId, "block", r.blockNumber, "difficulty", diff, "error", err.Error()) } else { - w.lg.Error("Failed to submit mined result", "shard", result.startShardId, "block", result.blockNumber, "error", err.Error()) + w.lg.Error("Failed to submit mined r", "shard", r.startShardId, "block", r.blockNumber, "error", err.Error()) } } } else { @@ -470,7 +470,7 @@ func (w *worker) resultLoop() { s.LastSubmittedTime = time.Now().UnixMilli() } } - w.reportMiningResult(result, txHash, err) + w.reportMiningResult(r, txHash, err) // optimistically check next result if exists w.notifyResultLoop() case <-ticker.C: diff --git a/ethstorage/node/config.go b/ethstorage/node/config.go index 40cb9f77..56262fb6 100644 --- a/ethstorage/node/config.go +++ b/ethstorage/node/config.go @@ -4,6 +4,7 @@ package node import ( + "context" "errors" "fmt" "math" @@ -12,14 +13,22 @@ import ( "time" oppprof "github.com/ethereum-optimism/optimism/op-service/pprof" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" "github.com/ethstorage/go-ethstorage/ethstorage/archiver" "github.com/ethstorage/go-ethstorage/ethstorage/db" "github.com/ethstorage/go-ethstorage/ethstorage/downloader" + "github.com/ethstorage/go-ethstorage/ethstorage/email" "github.com/ethstorage/go-ethstorage/ethstorage/eth" + "github.com/ethstorage/go-ethstorage/ethstorage/flags" "github.com/ethstorage/go-ethstorage/ethstorage/miner" "github.com/ethstorage/go-ethstorage/ethstorage/p2p" + p2pcli "github.com/ethstorage/go-ethstorage/ethstorage/p2p/cli" "github.com/ethstorage/go-ethstorage/ethstorage/scanner" + "github.com/ethstorage/go-ethstorage/ethstorage/signer" "github.com/ethstorage/go-ethstorage/ethstorage/storage" + "github.com/urfave/cli" ) type Config struct { @@ -132,3 +141,265 @@ func (c *Config) ResolveAncient(name string, ancient string) string { } return ancient } + +// NewConfig creates a Config from the provided flags or environment variables. +func NewConfig(ctx *cli.Context, lg log.Logger) (*Config, error) { + if err := flags.CheckRequired(ctx); err != nil { + return nil, err + } + + datadir := ctx.GlobalString(flags.DataDir.Name) + + // TODO: blocktime is set to zero, need to update the value + p2pConfig, err := p2pcli.NewConfig(ctx, 0) + if err != nil { + return nil, fmt.Errorf("failed to load p2p config: %w", err) + } + + l1Endpoint, client, err := NewL1EndpointConfig(ctx, lg) + if err != nil { + return nil, err + } + lg.Info("Read L1 config", flags.L1NodeAddr.Name, l1Endpoint.L1NodeAddr) + lg.Info("Read L1 config", flags.L1BeaconAddr.Name, l1Endpoint.L1BeaconURL) + defer client.Close() + + l1Contract := common.HexToAddress(ctx.GlobalString(flags.StorageL1Contract.Name)) + pClient := eth.NewClient(context.Background(), client, true, l1Contract, 0, nil, lg) + storageConfig, err := NewStorageConfig(ctx, pClient, lg) + if err != nil { + return nil, fmt.Errorf("failed to load storage config: %w", err) + } + + dlConfig := NewDownloaderConfig(ctx) + minerConfig, err := NewMinerConfig(ctx, pClient, storageConfig.Miner, lg) + if err != nil { + return nil, fmt.Errorf("failed to load miner config: %w", err) + } + chainId := new(big.Int).SetUint64(ctx.GlobalUint64(flags.ChainId.Name)) + lg.Info("Read chain ID of EthStorage network", "chainID", chainId) + if minerConfig != nil { + minerConfig.ChainID = chainId + } + archiverConfig := archiver.NewConfig(ctx) + // l2Endpoint, err := NewL2EndpointConfig(ctx, lg) + // if err != nil { + // return nil, fmt.Errorf("failed to load l2 endpoints info: %w", err) + // } + + // l2SyncEndpoint := NewL2SyncEndpointConfig(ctx) + cfg := &Config{ + L1: *l1Endpoint, + ChainID: chainId, + Downloader: *dlConfig, + + DataDir: datadir, + StateUploadURL: ctx.GlobalString(flags.StateUploadURL.Name), + DBConfig: db.DefaultDBConfig(), + // rpc url to get randao from + RandaoSourceURL: ctx.GlobalString(flags.RandaoURL.Name), + // Driver: *driverConfig, + RPC: RPCConfig{ + ListenAddr: ctx.GlobalString(flags.RPCListenAddr.Name), + ListenPort: ctx.GlobalInt(flags.RPCListenPort.Name), + ESCallURL: ctx.GlobalString(flags.RPCESCallURL.Name), + }, + Metrics: MetricsConfig{ + Enabled: ctx.GlobalBool(flags.MetricsEnabledFlag.Name), + ListenAddr: ctx.GlobalString(flags.MetricsAddrFlag.Name), + ListenPort: ctx.GlobalInt(flags.MetricsPortFlag.Name), + }, + Pprof: oppprof.CLIConfig{ + Enabled: ctx.GlobalBool(flags.PprofEnabledFlag.Name), + ListenAddr: ctx.GlobalString(flags.PprofAddrFlag.Name), + ListenPort: ctx.GlobalInt(flags.PprofPortFlag.Name), + }, + P2P: p2pConfig, + + L1EpochPollInterval: ctx.GlobalDuration(flags.L1EpochPollIntervalFlag.Name), + // Heartbeat: node.HeartbeatConfig{ + // Enabled: ctx.GlobalBool(flags.HeartbeatEnabledFlag.Name), + // Moniker: ctx.GlobalString(flags.HeartbeatMonikerFlag.Name), + // URL: ctx.GlobalString(flags.HeartbeatURLFlag.Name), + // }, + Storage: *storageConfig, + Mining: minerConfig, + Archiver: archiverConfig, + Scanner: scanner.NewConfig(ctx), + } + if err := cfg.Check(); err != nil { + return nil, err + } + return cfg, nil +} + +func NewMinerConfig(ctx *cli.Context, pClient *eth.PollingClient, minerAddr common.Address, lg log.Logger) (*miner.Config, error) { + cliConfig := miner.ReadCLIConfig(ctx) + if !cliConfig.Enabled { + lg.Info("Miner is not enabled.") + return nil, nil + } + if minerAddr == (common.Address{}) { + return nil, fmt.Errorf("miner address cannot be empty") + } + lg.Debug("Read mining config from cli", "config", fmt.Sprintf("%+v", cliConfig)) + err := cliConfig.Check() + if err != nil { + return nil, fmt.Errorf("invalid miner flags: %w", err) + } + minerConfig, err := cliConfig.ToMinerConfig() + if err != nil { + return nil, err + } + if minerConfig.EmailEnabled { + emailConfig, err := email.GetEmailConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get email config: %w", err) + } + minerConfig.EmailConfig = *emailConfig + } + + randomChecks, err := pClient.ReadContractUint64Field("randomChecks", nil) + if err != nil { + return nil, err + } + minerConfig.RandomChecks = randomChecks + nonceLimit, err := pClient.ReadContractUint64Field("nonceLimit", nil) + if err != nil { + return nil, err + } + minerConfig.NonceLimit = nonceLimit + minimumDiff, err := pClient.ReadContractBigIntField("minimumDiff", nil) + if err != nil { + return nil, err + } + minerConfig.MinimumDiff = minimumDiff + cutoff, err := pClient.ReadContractBigIntField("cutoff", nil) + if err != nil { + return nil, err + } + minerConfig.Cutoff = cutoff + diffAdjDivisor, err := pClient.ReadContractBigIntField("diffAdjDivisor", nil) + if err != nil { + return nil, err + } + minerConfig.DiffAdjDivisor = diffAdjDivisor + dcf, err := pClient.ReadContractBigIntField("dcfFactor", nil) + if err != nil { + return nil, err + } + minerConfig.DcfFactor = dcf + + startTime, err := pClient.ReadContractUint64Field("startTime", nil) + if err != nil { + return nil, err + } + minerConfig.StartTime = startTime + shardEntryBits, err := pClient.ReadContractUint64Field("shardEntryBits", nil) + if err != nil { + return nil, err + } + minerConfig.ShardEntry = 1 << shardEntryBits + treasuryShare, err := pClient.ReadContractUint64Field("treasuryShare", nil) + if err != nil { + return nil, err + } + minerConfig.TreasuryShare = treasuryShare + storageCost, err := pClient.ReadContractBigIntField("storageCost", nil) + if err != nil { + return nil, err + } + minerConfig.StorageCost = storageCost + prepaidAmount, err := pClient.ReadContractBigIntField("prepaidAmount", nil) + if err != nil { + return nil, err + } + minerConfig.PrepaidAmount = prepaidAmount + signerFnFactory, signerAddr, err := NewSignerConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get signer: %w", err) + } + minerConfig.SignerFnFactory = signerFnFactory + minerConfig.SignerAddr = signerAddr + return &minerConfig, nil +} + +func NewSignerConfig(ctx *cli.Context) (signer.SignerFactory, common.Address, error) { + signerConfig := signer.ReadCLIConfig(ctx) + if err := signerConfig.Check(); err != nil { + return nil, common.Address{}, fmt.Errorf("invalid siger flags: %w", err) + } + return signer.SignerFactoryFromConfig(signerConfig) +} + +func NewStorageConfig(ctx *cli.Context, pClient *eth.PollingClient, lg log.Logger) (*storage.StorageConfig, error) { + miner := common.HexToAddress(ctx.GlobalString(flags.StorageMiner.Name)) + lg.Info("Loaded storage config", "l1Contract", pClient.ContractAddress(), "miner", miner) + storageCfg, err := InitStorageConfig(pClient, miner, lg) + if err != nil { + lg.Error("Failed to load storage config from contract", "error", err) + return nil, err + } + storageCfg.Filenames = ctx.GlobalStringSlice(flags.StorageFiles.Name) + return storageCfg, nil +} + +func NewL1EndpointConfig(ctx *cli.Context, lg log.Logger) (*eth.L1EndpointConfig, *ethclient.Client, error) { + l1NodeAddr := ctx.GlobalString(flags.L1NodeAddr.Name) + client, err := ethclient.DialContext(context.Background(), l1NodeAddr) + if err != nil { + lg.Error("Failed to connect to the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) + return nil, nil, err + } + chainid, err := client.ChainID(context.Background()) + if err != nil { + lg.Error("Failed to fetch chain id from the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) + return nil, nil, err + } + return ð.L1EndpointConfig{ + L1ChainID: chainid.Uint64(), + L1NodeAddr: l1NodeAddr, + L1BlockTime: ctx.GlobalUint64(flags.L1BlockTime.Name), + L1BeaconURL: ctx.GlobalString(flags.L1BeaconAddr.Name), + L1BeaconSlotTime: ctx.GlobalUint64(flags.L1BeaconSlotTime.Name), + DAURL: ctx.GlobalString(flags.DAURL.Name), + L1MinDurationForBlobsRequest: ctx.GlobalUint64(flags.L1MinDurationForBlobsRequest.Name), + }, client, nil +} + +func NewDownloaderConfig(ctx *cli.Context) *downloader.Config { + return &downloader.Config{ + DownloadStart: ctx.GlobalInt64(flags.DownloadStart.Name), + DownloadDump: ctx.GlobalString(flags.DownloadDump.Name), + DownloadThreadNum: ctx.GlobalInt(flags.DownloadThreadNum.Name), + } +} + +func InitStorageConfig(client *eth.PollingClient, miner common.Address, lg log.Logger) (*storage.StorageConfig, error) { + exist, err := client.IsContractExist() + if err != nil { + return nil, fmt.Errorf("check contract exist fail: %s", err.Error()) + } + if !exist { + return nil, fmt.Errorf("contract does not exist") + } + result, err := client.ReadContractField("maxKvSizeBits", nil) + if err != nil { + return nil, fmt.Errorf("get maxKvSizeBits: %s", err.Error()) + } + maxKvSizeBits := new(big.Int).SetBytes(result).Uint64() + chunkSizeBits := maxKvSizeBits + result, err = client.ReadContractField("shardEntryBits", nil) + if err != nil { + return nil, fmt.Errorf("get shardEntryBits: %s", err.Error()) + } + shardEntryBits := new(big.Int).SetBytes(result).Uint64() + + return &storage.StorageConfig{ + L1Contract: client.ContractAddress(), + Miner: miner, + KvSize: 1 << maxKvSizeBits, + ChunkSize: 1 << chunkSizeBits, + KvEntriesPerShard: 1 << shardEntryBits, + }, nil +} From 0d7860aabb5d5e2b27ef6a2c3f524fd464fabea5 Mon Sep 17 00:00:00 2001 From: ping-ke Date: Tue, 30 Sep 2025 00:23:26 +0800 Subject: [PATCH 2/7] update --- cmd/es-node/main.go | 4 +- cmd/es-node/utils.go | 3 +- ethstorage/eth/polling_client.go | 30 ++++ ethstorage/miner/cli.go | 104 ++++++++++++ ethstorage/node/config.go | 271 ------------------------------- 5 files changed, 137 insertions(+), 275 deletions(-) diff --git a/cmd/es-node/main.go b/cmd/es-node/main.go index 59413810..b7140766 100644 --- a/cmd/es-node/main.go +++ b/cmd/es-node/main.go @@ -153,7 +153,7 @@ func EsNodeMain(ctx *cli.Context) error { lgCfg := log.ReadCLIConfig(ctx) lg.Info("Loading log config", "config", lgCfg) clog := log.NewLogger(lgCfg) - cfg, err := node.NewConfig(ctx, clog) + cfg, err := NewConfig(ctx, clog) if err != nil { lg.Error("Unable to create the rollup node config", "error", err) return err @@ -258,7 +258,7 @@ func EsNodeInit(ctx *cli.Context) error { } defer client.Close() - storageCfg, err := node.InitStorageConfig(client, common.HexToAddress(miner), lg) + storageCfg, err := client.LoadStorageConfigFromContract(common.HexToAddress(miner)) if err != nil { lg.Error("Failed to load storage config", "error", err) return err diff --git a/cmd/es-node/utils.go b/cmd/es-node/utils.go index 803aecc3..16e78451 100644 --- a/cmd/es-node/utils.go +++ b/cmd/es-node/utils.go @@ -23,7 +23,6 @@ import ( "github.com/ethstorage/go-ethstorage/ethstorage/eth" "github.com/ethstorage/go-ethstorage/ethstorage/flags" "github.com/ethstorage/go-ethstorage/ethstorage/miner" - "github.com/ethstorage/go-ethstorage/ethstorage/node" "github.com/ethstorage/go-ethstorage/ethstorage/storage" "github.com/urfave/cli" ) @@ -169,7 +168,7 @@ func initShardManager(ctx *cli.Context, l1Rpc string, l1contract common.Address, } defer client.Close() - storageCfg, err := node.InitStorageConfig(client, common.HexToAddress(miner), lg) + storageCfg, err := client.LoadStorageConfigFromContract(common.HexToAddress(miner)) if err != nil { return nil, fmt.Errorf("failed to load storage config: %w", err) } diff --git a/ethstorage/eth/polling_client.go b/ethstorage/eth/polling_client.go index 114a2b6c..ea2e7274 100644 --- a/ethstorage/eth/polling_client.go +++ b/ethstorage/eth/polling_client.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/ethstorage/go-ethstorage/ethstorage/storage" "math/big" "regexp" "sync" @@ -378,6 +379,35 @@ func (w *PollingClient) ContractAddress() common.Address { return w.esContract } +func (w *PollingClient) LoadStorageConfigFromContract(miner common.Address) (*storage.StorageConfig, error) { + exist, err := w.IsContractExist() + if err != nil { + return nil, fmt.Errorf("check contract exist fail: %s", err.Error()) + } + if !exist { + return nil, fmt.Errorf("contract does not exist") + } + result, err := w.ReadContractField("maxKvSizeBits", nil) + if err != nil { + return nil, fmt.Errorf("get maxKvSizeBits: %s", err.Error()) + } + maxKvSizeBits := new(big.Int).SetBytes(result).Uint64() + chunkSizeBits := maxKvSizeBits + result, err = w.ReadContractField("shardEntryBits", nil) + if err != nil { + return nil, fmt.Errorf("get shardEntryBits: %s", err.Error()) + } + shardEntryBits := new(big.Int).SetBytes(result).Uint64() + + return &storage.StorageConfig{ + L1Contract: w.esContract, + Miner: miner, + KvSize: 1 << maxKvSizeBits, + ChunkSize: 1 << chunkSizeBits, + KvEntriesPerShard: 1 << shardEntryBits, + }, nil +} + func decodeString(data []byte) (string, error) { if len(data) < 64 { return "", fmt.Errorf("data too short") diff --git a/ethstorage/miner/cli.go b/ethstorage/miner/cli.go index 374a4dcd..7c28af42 100644 --- a/ethstorage/miner/cli.go +++ b/ethstorage/miner/cli.go @@ -9,9 +9,14 @@ import ( "os" "path/filepath" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethstorage/go-ethstorage/ethstorage/email" + "github.com/ethstorage/go-ethstorage/ethstorage/eth" "github.com/ethstorage/go-ethstorage/ethstorage/flags/types" "github.com/ethstorage/go-ethstorage/ethstorage/flags/utils" "github.com/ethstorage/go-ethstorage/ethstorage/prover" + "github.com/ethstorage/go-ethstorage/ethstorage/signer" "github.com/urfave/cli" ) @@ -169,3 +174,102 @@ func ReadCLIConfig(ctx *cli.Context) CLIConfig { } return cfg } + +func NewMinerConfig(ctx *cli.Context, pClient *eth.PollingClient, minerAddr common.Address, lg log.Logger) (*Config, error) { + cliConfig := ReadCLIConfig(ctx) + if !cliConfig.Enabled { + lg.Info("Miner is not enabled.") + return nil, nil + } + if minerAddr == (common.Address{}) { + return nil, fmt.Errorf("miner address cannot be empty") + } + lg.Debug("Read mining config from cli", "config", fmt.Sprintf("%+v", cliConfig)) + err := cliConfig.Check() + if err != nil { + return nil, fmt.Errorf("invalid miner flags: %w", err) + } + minerConfig, err := cliConfig.ToMinerConfig() + if err != nil { + return nil, err + } + if minerConfig.EmailEnabled { + emailConfig, err := email.GetEmailConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get email config: %w", err) + } + minerConfig.EmailConfig = *emailConfig + } + + randomChecks, err := pClient.ReadContractUint64Field("randomChecks", nil) + if err != nil { + return nil, err + } + minerConfig.RandomChecks = randomChecks + nonceLimit, err := pClient.ReadContractUint64Field("nonceLimit", nil) + if err != nil { + return nil, err + } + minerConfig.NonceLimit = nonceLimit + minimumDiff, err := pClient.ReadContractBigIntField("minimumDiff", nil) + if err != nil { + return nil, err + } + minerConfig.MinimumDiff = minimumDiff + cutoff, err := pClient.ReadContractBigIntField("cutoff", nil) + if err != nil { + return nil, err + } + minerConfig.Cutoff = cutoff + diffAdjDivisor, err := pClient.ReadContractBigIntField("diffAdjDivisor", nil) + if err != nil { + return nil, err + } + minerConfig.DiffAdjDivisor = diffAdjDivisor + dcf, err := pClient.ReadContractBigIntField("dcfFactor", nil) + if err != nil { + return nil, err + } + minerConfig.DcfFactor = dcf + + startTime, err := pClient.ReadContractUint64Field("startTime", nil) + if err != nil { + return nil, err + } + minerConfig.StartTime = startTime + shardEntryBits, err := pClient.ReadContractUint64Field("shardEntryBits", nil) + if err != nil { + return nil, err + } + minerConfig.ShardEntry = 1 << shardEntryBits + treasuryShare, err := pClient.ReadContractUint64Field("treasuryShare", nil) + if err != nil { + return nil, err + } + minerConfig.TreasuryShare = treasuryShare + storageCost, err := pClient.ReadContractBigIntField("storageCost", nil) + if err != nil { + return nil, err + } + minerConfig.StorageCost = storageCost + prepaidAmount, err := pClient.ReadContractBigIntField("prepaidAmount", nil) + if err != nil { + return nil, err + } + minerConfig.PrepaidAmount = prepaidAmount + signerFnFactory, signerAddr, err := NewSignerConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get signer: %w", err) + } + minerConfig.SignerFnFactory = signerFnFactory + minerConfig.SignerAddr = signerAddr + return &minerConfig, nil +} + +func NewSignerConfig(ctx *cli.Context) (signer.SignerFactory, common.Address, error) { + signerConfig := signer.ReadCLIConfig(ctx) + if err := signerConfig.Check(); err != nil { + return nil, common.Address{}, fmt.Errorf("invalid siger flags: %w", err) + } + return signer.SignerFactoryFromConfig(signerConfig) +} diff --git a/ethstorage/node/config.go b/ethstorage/node/config.go index 56262fb6..40cb9f77 100644 --- a/ethstorage/node/config.go +++ b/ethstorage/node/config.go @@ -4,7 +4,6 @@ package node import ( - "context" "errors" "fmt" "math" @@ -13,22 +12,14 @@ import ( "time" oppprof "github.com/ethereum-optimism/optimism/op-service/pprof" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/log" "github.com/ethstorage/go-ethstorage/ethstorage/archiver" "github.com/ethstorage/go-ethstorage/ethstorage/db" "github.com/ethstorage/go-ethstorage/ethstorage/downloader" - "github.com/ethstorage/go-ethstorage/ethstorage/email" "github.com/ethstorage/go-ethstorage/ethstorage/eth" - "github.com/ethstorage/go-ethstorage/ethstorage/flags" "github.com/ethstorage/go-ethstorage/ethstorage/miner" "github.com/ethstorage/go-ethstorage/ethstorage/p2p" - p2pcli "github.com/ethstorage/go-ethstorage/ethstorage/p2p/cli" "github.com/ethstorage/go-ethstorage/ethstorage/scanner" - "github.com/ethstorage/go-ethstorage/ethstorage/signer" "github.com/ethstorage/go-ethstorage/ethstorage/storage" - "github.com/urfave/cli" ) type Config struct { @@ -141,265 +132,3 @@ func (c *Config) ResolveAncient(name string, ancient string) string { } return ancient } - -// NewConfig creates a Config from the provided flags or environment variables. -func NewConfig(ctx *cli.Context, lg log.Logger) (*Config, error) { - if err := flags.CheckRequired(ctx); err != nil { - return nil, err - } - - datadir := ctx.GlobalString(flags.DataDir.Name) - - // TODO: blocktime is set to zero, need to update the value - p2pConfig, err := p2pcli.NewConfig(ctx, 0) - if err != nil { - return nil, fmt.Errorf("failed to load p2p config: %w", err) - } - - l1Endpoint, client, err := NewL1EndpointConfig(ctx, lg) - if err != nil { - return nil, err - } - lg.Info("Read L1 config", flags.L1NodeAddr.Name, l1Endpoint.L1NodeAddr) - lg.Info("Read L1 config", flags.L1BeaconAddr.Name, l1Endpoint.L1BeaconURL) - defer client.Close() - - l1Contract := common.HexToAddress(ctx.GlobalString(flags.StorageL1Contract.Name)) - pClient := eth.NewClient(context.Background(), client, true, l1Contract, 0, nil, lg) - storageConfig, err := NewStorageConfig(ctx, pClient, lg) - if err != nil { - return nil, fmt.Errorf("failed to load storage config: %w", err) - } - - dlConfig := NewDownloaderConfig(ctx) - minerConfig, err := NewMinerConfig(ctx, pClient, storageConfig.Miner, lg) - if err != nil { - return nil, fmt.Errorf("failed to load miner config: %w", err) - } - chainId := new(big.Int).SetUint64(ctx.GlobalUint64(flags.ChainId.Name)) - lg.Info("Read chain ID of EthStorage network", "chainID", chainId) - if minerConfig != nil { - minerConfig.ChainID = chainId - } - archiverConfig := archiver.NewConfig(ctx) - // l2Endpoint, err := NewL2EndpointConfig(ctx, lg) - // if err != nil { - // return nil, fmt.Errorf("failed to load l2 endpoints info: %w", err) - // } - - // l2SyncEndpoint := NewL2SyncEndpointConfig(ctx) - cfg := &Config{ - L1: *l1Endpoint, - ChainID: chainId, - Downloader: *dlConfig, - - DataDir: datadir, - StateUploadURL: ctx.GlobalString(flags.StateUploadURL.Name), - DBConfig: db.DefaultDBConfig(), - // rpc url to get randao from - RandaoSourceURL: ctx.GlobalString(flags.RandaoURL.Name), - // Driver: *driverConfig, - RPC: RPCConfig{ - ListenAddr: ctx.GlobalString(flags.RPCListenAddr.Name), - ListenPort: ctx.GlobalInt(flags.RPCListenPort.Name), - ESCallURL: ctx.GlobalString(flags.RPCESCallURL.Name), - }, - Metrics: MetricsConfig{ - Enabled: ctx.GlobalBool(flags.MetricsEnabledFlag.Name), - ListenAddr: ctx.GlobalString(flags.MetricsAddrFlag.Name), - ListenPort: ctx.GlobalInt(flags.MetricsPortFlag.Name), - }, - Pprof: oppprof.CLIConfig{ - Enabled: ctx.GlobalBool(flags.PprofEnabledFlag.Name), - ListenAddr: ctx.GlobalString(flags.PprofAddrFlag.Name), - ListenPort: ctx.GlobalInt(flags.PprofPortFlag.Name), - }, - P2P: p2pConfig, - - L1EpochPollInterval: ctx.GlobalDuration(flags.L1EpochPollIntervalFlag.Name), - // Heartbeat: node.HeartbeatConfig{ - // Enabled: ctx.GlobalBool(flags.HeartbeatEnabledFlag.Name), - // Moniker: ctx.GlobalString(flags.HeartbeatMonikerFlag.Name), - // URL: ctx.GlobalString(flags.HeartbeatURLFlag.Name), - // }, - Storage: *storageConfig, - Mining: minerConfig, - Archiver: archiverConfig, - Scanner: scanner.NewConfig(ctx), - } - if err := cfg.Check(); err != nil { - return nil, err - } - return cfg, nil -} - -func NewMinerConfig(ctx *cli.Context, pClient *eth.PollingClient, minerAddr common.Address, lg log.Logger) (*miner.Config, error) { - cliConfig := miner.ReadCLIConfig(ctx) - if !cliConfig.Enabled { - lg.Info("Miner is not enabled.") - return nil, nil - } - if minerAddr == (common.Address{}) { - return nil, fmt.Errorf("miner address cannot be empty") - } - lg.Debug("Read mining config from cli", "config", fmt.Sprintf("%+v", cliConfig)) - err := cliConfig.Check() - if err != nil { - return nil, fmt.Errorf("invalid miner flags: %w", err) - } - minerConfig, err := cliConfig.ToMinerConfig() - if err != nil { - return nil, err - } - if minerConfig.EmailEnabled { - emailConfig, err := email.GetEmailConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get email config: %w", err) - } - minerConfig.EmailConfig = *emailConfig - } - - randomChecks, err := pClient.ReadContractUint64Field("randomChecks", nil) - if err != nil { - return nil, err - } - minerConfig.RandomChecks = randomChecks - nonceLimit, err := pClient.ReadContractUint64Field("nonceLimit", nil) - if err != nil { - return nil, err - } - minerConfig.NonceLimit = nonceLimit - minimumDiff, err := pClient.ReadContractBigIntField("minimumDiff", nil) - if err != nil { - return nil, err - } - minerConfig.MinimumDiff = minimumDiff - cutoff, err := pClient.ReadContractBigIntField("cutoff", nil) - if err != nil { - return nil, err - } - minerConfig.Cutoff = cutoff - diffAdjDivisor, err := pClient.ReadContractBigIntField("diffAdjDivisor", nil) - if err != nil { - return nil, err - } - minerConfig.DiffAdjDivisor = diffAdjDivisor - dcf, err := pClient.ReadContractBigIntField("dcfFactor", nil) - if err != nil { - return nil, err - } - minerConfig.DcfFactor = dcf - - startTime, err := pClient.ReadContractUint64Field("startTime", nil) - if err != nil { - return nil, err - } - minerConfig.StartTime = startTime - shardEntryBits, err := pClient.ReadContractUint64Field("shardEntryBits", nil) - if err != nil { - return nil, err - } - minerConfig.ShardEntry = 1 << shardEntryBits - treasuryShare, err := pClient.ReadContractUint64Field("treasuryShare", nil) - if err != nil { - return nil, err - } - minerConfig.TreasuryShare = treasuryShare - storageCost, err := pClient.ReadContractBigIntField("storageCost", nil) - if err != nil { - return nil, err - } - minerConfig.StorageCost = storageCost - prepaidAmount, err := pClient.ReadContractBigIntField("prepaidAmount", nil) - if err != nil { - return nil, err - } - minerConfig.PrepaidAmount = prepaidAmount - signerFnFactory, signerAddr, err := NewSignerConfig(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get signer: %w", err) - } - minerConfig.SignerFnFactory = signerFnFactory - minerConfig.SignerAddr = signerAddr - return &minerConfig, nil -} - -func NewSignerConfig(ctx *cli.Context) (signer.SignerFactory, common.Address, error) { - signerConfig := signer.ReadCLIConfig(ctx) - if err := signerConfig.Check(); err != nil { - return nil, common.Address{}, fmt.Errorf("invalid siger flags: %w", err) - } - return signer.SignerFactoryFromConfig(signerConfig) -} - -func NewStorageConfig(ctx *cli.Context, pClient *eth.PollingClient, lg log.Logger) (*storage.StorageConfig, error) { - miner := common.HexToAddress(ctx.GlobalString(flags.StorageMiner.Name)) - lg.Info("Loaded storage config", "l1Contract", pClient.ContractAddress(), "miner", miner) - storageCfg, err := InitStorageConfig(pClient, miner, lg) - if err != nil { - lg.Error("Failed to load storage config from contract", "error", err) - return nil, err - } - storageCfg.Filenames = ctx.GlobalStringSlice(flags.StorageFiles.Name) - return storageCfg, nil -} - -func NewL1EndpointConfig(ctx *cli.Context, lg log.Logger) (*eth.L1EndpointConfig, *ethclient.Client, error) { - l1NodeAddr := ctx.GlobalString(flags.L1NodeAddr.Name) - client, err := ethclient.DialContext(context.Background(), l1NodeAddr) - if err != nil { - lg.Error("Failed to connect to the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) - return nil, nil, err - } - chainid, err := client.ChainID(context.Background()) - if err != nil { - lg.Error("Failed to fetch chain id from the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) - return nil, nil, err - } - return ð.L1EndpointConfig{ - L1ChainID: chainid.Uint64(), - L1NodeAddr: l1NodeAddr, - L1BlockTime: ctx.GlobalUint64(flags.L1BlockTime.Name), - L1BeaconURL: ctx.GlobalString(flags.L1BeaconAddr.Name), - L1BeaconSlotTime: ctx.GlobalUint64(flags.L1BeaconSlotTime.Name), - DAURL: ctx.GlobalString(flags.DAURL.Name), - L1MinDurationForBlobsRequest: ctx.GlobalUint64(flags.L1MinDurationForBlobsRequest.Name), - }, client, nil -} - -func NewDownloaderConfig(ctx *cli.Context) *downloader.Config { - return &downloader.Config{ - DownloadStart: ctx.GlobalInt64(flags.DownloadStart.Name), - DownloadDump: ctx.GlobalString(flags.DownloadDump.Name), - DownloadThreadNum: ctx.GlobalInt(flags.DownloadThreadNum.Name), - } -} - -func InitStorageConfig(client *eth.PollingClient, miner common.Address, lg log.Logger) (*storage.StorageConfig, error) { - exist, err := client.IsContractExist() - if err != nil { - return nil, fmt.Errorf("check contract exist fail: %s", err.Error()) - } - if !exist { - return nil, fmt.Errorf("contract does not exist") - } - result, err := client.ReadContractField("maxKvSizeBits", nil) - if err != nil { - return nil, fmt.Errorf("get maxKvSizeBits: %s", err.Error()) - } - maxKvSizeBits := new(big.Int).SetBytes(result).Uint64() - chunkSizeBits := maxKvSizeBits - result, err = client.ReadContractField("shardEntryBits", nil) - if err != nil { - return nil, fmt.Errorf("get shardEntryBits: %s", err.Error()) - } - shardEntryBits := new(big.Int).SetBytes(result).Uint64() - - return &storage.StorageConfig{ - L1Contract: client.ContractAddress(), - Miner: miner, - KvSize: 1 << maxKvSizeBits, - ChunkSize: 1 << chunkSizeBits, - KvEntriesPerShard: 1 << shardEntryBits, - }, nil -} From 423ea7d0ebd2d918318fc820342d6899d08b16ef Mon Sep 17 00:00:00 2001 From: ping-ke Date: Tue, 30 Sep 2025 00:38:57 +0800 Subject: [PATCH 3/7] add missing change --- cmd/es-node/config.go | 142 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 cmd/es-node/config.go diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go new file mode 100644 index 00000000..578327b2 --- /dev/null +++ b/cmd/es-node/config.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "fmt" + "math/big" + + oppprof "github.com/ethereum-optimism/optimism/op-service/pprof" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/log" + "github.com/ethstorage/go-ethstorage/ethstorage/archiver" + "github.com/ethstorage/go-ethstorage/ethstorage/db" + "github.com/ethstorage/go-ethstorage/ethstorage/downloader" + "github.com/ethstorage/go-ethstorage/ethstorage/eth" + "github.com/ethstorage/go-ethstorage/ethstorage/flags" + "github.com/ethstorage/go-ethstorage/ethstorage/miner" + "github.com/ethstorage/go-ethstorage/ethstorage/node" + p2pcli "github.com/ethstorage/go-ethstorage/ethstorage/p2p/cli" + "github.com/ethstorage/go-ethstorage/ethstorage/scanner" + "github.com/ethstorage/go-ethstorage/ethstorage/storage" + "github.com/urfave/cli" +) + +// NewConfig creates a Config from the provided flags or environment variables. +func NewConfig(ctx *cli.Context, lg log.Logger) (*node.Config, error) { + if err := flags.CheckRequired(ctx); err != nil { + return nil, err + } + datadir := ctx.GlobalString(flags.DataDir.Name) + + // TODO: blocktime is set to zero, need to update the value + p2pConfig, err := p2pcli.NewConfig(ctx, 0) + if err != nil { + return nil, fmt.Errorf("failed to load p2p config: %w", err) + } + + l1Endpoint, client, err := NewL1EndpointConfig(ctx, lg) + if err != nil { + return nil, err + } + lg.Info("Read L1 config", flags.L1NodeAddr.Name, l1Endpoint.L1NodeAddr) + lg.Info("Read L1 config", flags.L1BeaconAddr.Name, l1Endpoint.L1BeaconURL) + defer client.Close() + + l1Contract := common.HexToAddress(ctx.GlobalString(flags.StorageL1Contract.Name)) + pClient := eth.NewClient(context.Background(), client, true, l1Contract, 0, nil, lg) + storageConfig, err := NewStorageConfig(ctx, pClient, lg) + if err != nil { + return nil, fmt.Errorf("failed to load storage config: %w", err) + } + + minerConfig, err := miner.NewMinerConfig(ctx, pClient, storageConfig.Miner, lg) + if err != nil { + return nil, fmt.Errorf("failed to load miner config: %w", err) + } + chainId := new(big.Int).SetUint64(ctx.GlobalUint64(flags.ChainId.Name)) + lg.Info("Read chain ID of EthStorage network", "chainID", chainId) + if minerConfig != nil { + minerConfig.ChainID = chainId + } + archiverConfig := archiver.NewConfig(ctx) + + cfg := &node.Config{ + L1: *l1Endpoint, + ChainID: chainId, + Downloader: downloader.Config{ + DownloadStart: ctx.GlobalInt64(flags.DownloadStart.Name), + DownloadDump: ctx.GlobalString(flags.DownloadDump.Name), + DownloadThreadNum: ctx.GlobalInt(flags.DownloadThreadNum.Name), + }, + + DataDir: datadir, + StateUploadURL: ctx.GlobalString(flags.StateUploadURL.Name), + DBConfig: db.DefaultDBConfig(), + // rpc url to get randao from + RandaoSourceURL: ctx.GlobalString(flags.RandaoURL.Name), + // Driver: *driverConfig, + RPC: node.RPCConfig{ + ListenAddr: ctx.GlobalString(flags.RPCListenAddr.Name), + ListenPort: ctx.GlobalInt(flags.RPCListenPort.Name), + ESCallURL: ctx.GlobalString(flags.RPCESCallURL.Name), + }, + Metrics: node.MetricsConfig{ + Enabled: ctx.GlobalBool(flags.MetricsEnabledFlag.Name), + ListenAddr: ctx.GlobalString(flags.MetricsAddrFlag.Name), + ListenPort: ctx.GlobalInt(flags.MetricsPortFlag.Name), + }, + Pprof: oppprof.CLIConfig{ + Enabled: ctx.GlobalBool(flags.PprofEnabledFlag.Name), + ListenAddr: ctx.GlobalString(flags.PprofAddrFlag.Name), + ListenPort: ctx.GlobalInt(flags.PprofPortFlag.Name), + }, + P2P: p2pConfig, + + L1EpochPollInterval: ctx.GlobalDuration(flags.L1EpochPollIntervalFlag.Name), + + Storage: *storageConfig, + Mining: minerConfig, + Archiver: archiverConfig, + Scanner: scanner.NewConfig(ctx), + } + if err := cfg.Check(); err != nil { + return nil, err + } + return cfg, nil +} + +func NewStorageConfig(ctx *cli.Context, pClient *eth.PollingClient, lg log.Logger) (*storage.StorageConfig, error) { + miner := common.HexToAddress(ctx.GlobalString(flags.StorageMiner.Name)) + lg.Info("Loaded storage config", "l1Contract", pClient.ContractAddress(), "miner", miner) + storageCfg, err := pClient.LoadStorageConfigFromContract(miner) + if err != nil { + lg.Error("Failed to load storage config from contract", "error", err) + return nil, err + } + storageCfg.Filenames = ctx.GlobalStringSlice(flags.StorageFiles.Name) + return storageCfg, nil +} + +func NewL1EndpointConfig(ctx *cli.Context, lg log.Logger) (*eth.L1EndpointConfig, *ethclient.Client, error) { + l1NodeAddr := ctx.GlobalString(flags.L1NodeAddr.Name) + client, err := ethclient.DialContext(context.Background(), l1NodeAddr) + if err != nil { + lg.Error("Failed to connect to the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) + return nil, nil, err + } + chainid, err := client.ChainID(context.Background()) + if err != nil { + lg.Error("Failed to fetch chain id from the L1 RPC", "error", err, "l1Rpc", l1NodeAddr) + return nil, nil, err + } + return ð.L1EndpointConfig{ + L1ChainID: chainid.Uint64(), + L1NodeAddr: l1NodeAddr, + L1BlockTime: ctx.GlobalUint64(flags.L1BlockTime.Name), + L1BeaconURL: ctx.GlobalString(flags.L1BeaconAddr.Name), + L1BeaconSlotTime: ctx.GlobalUint64(flags.L1BeaconSlotTime.Name), + DAURL: ctx.GlobalString(flags.DAURL.Name), + L1MinDurationForBlobsRequest: ctx.GlobalUint64(flags.L1MinDurationForBlobsRequest.Name), + }, client, nil +} From bf34d1c4eff2b965c2a11e2c7f3bf7be51df6b03 Mon Sep 17 00:00:00 2001 From: ping-ke Date: Tue, 30 Sep 2025 00:39:50 +0800 Subject: [PATCH 4/7] update --- cmd/es-node/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/es-node/config.go b/cmd/es-node/config.go index 578327b2..f636ddef 100644 --- a/cmd/es-node/config.go +++ b/cmd/es-node/config.go @@ -1,3 +1,6 @@ +// Copyright 2022-2023, EthStorage. +// For license information, see https://github.com/ethstorage/es-node/blob/main/LICENSE + package main import ( From 5e7ee6aefc4ff07b7bdd58aa7514944ff4b2bc3a Mon Sep 17 00:00:00 2001 From: ping-ke Date: Wed, 1 Oct 2025 09:49:09 +0800 Subject: [PATCH 5/7] resolve comment --- ethstorage/miner/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index 561dbbc4..ddf13374 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -427,7 +427,7 @@ func (w *worker) resultLoop() { if r == nil { continue } - w.lg.Info("Mining result loop get r", "shard", r.startShardId, "block", r.blockNumber, "nonce", r.nonce) + w.lg.Info("Mining result loop get result", "shard", r.startShardId, "block", r.blockNumber, "nonce", r.nonce) txHash, err := w.l1API.SubmitMinedResult( context.Background(), w.storageMgr.ContractAddress(), From d64a3d6d2bf7824c4e6faa2f7d3079ea9ab3c29f Mon Sep 17 00:00:00 2001 From: ping-ke Date: Wed, 8 Oct 2025 12:01:56 +0800 Subject: [PATCH 6/7] update --- ethstorage/eth/polling_client.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ethstorage/eth/polling_client.go b/ethstorage/eth/polling_client.go index ea2e7274..c1a983d1 100644 --- a/ethstorage/eth/polling_client.go +++ b/ethstorage/eth/polling_client.go @@ -387,17 +387,15 @@ func (w *PollingClient) LoadStorageConfigFromContract(miner common.Address) (*st if !exist { return nil, fmt.Errorf("contract does not exist") } - result, err := w.ReadContractField("maxKvSizeBits", nil) + maxKvSizeBits, err := w.ReadContractUint64Field("maxKvSizeBits", nil) if err != nil { return nil, fmt.Errorf("get maxKvSizeBits: %s", err.Error()) } - maxKvSizeBits := new(big.Int).SetBytes(result).Uint64() chunkSizeBits := maxKvSizeBits - result, err = w.ReadContractField("shardEntryBits", nil) + shardEntryBits, err := w.ReadContractUint64Field("shardEntryBits", nil) if err != nil { return nil, fmt.Errorf("get shardEntryBits: %s", err.Error()) } - shardEntryBits := new(big.Int).SetBytes(result).Uint64() return &storage.StorageConfig{ L1Contract: w.esContract, From 9a8df45e324664f1f01239f3d575e6c95eaf12d5 Mon Sep 17 00:00:00 2001 From: ping-ke Date: Tue, 14 Oct 2025 11:05:25 +0800 Subject: [PATCH 7/7] resolve comments --- ethstorage/miner/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ethstorage/miner/worker.go b/ethstorage/miner/worker.go index ddf13374..a9c1d9d0 100644 --- a/ethstorage/miner/worker.go +++ b/ethstorage/miner/worker.go @@ -460,9 +460,9 @@ func (w *worker) resultLoop() { } } if diff != nil { - w.lg.Error("Failed to submit mined r", "shard", r.startShardId, "block", r.blockNumber, "difficulty", diff, "error", err.Error()) + w.lg.Error("Failed to submit mined result", "shard", r.startShardId, "block", r.blockNumber, "difficulty", diff, "error", err.Error()) } else { - w.lg.Error("Failed to submit mined r", "shard", r.startShardId, "block", r.blockNumber, "error", err.Error()) + w.lg.Error("Failed to submit mined result", "shard", r.startShardId, "block", r.blockNumber, "error", err.Error()) } } } else {