From 24670c2251a69841932e78a334531ee24fe9c220 Mon Sep 17 00:00:00 2001 From: ericlee Date: Tue, 9 Dec 2025 20:51:24 +0800 Subject: [PATCH 01/18] batch: use origin ethers.js 6.16.0 --- package.json | 2 +- packages/batch-submitter/package.json | 2 +- .../batch-submitter/state-batch-submitter.ts | 2 +- .../tx-batch-submitter-inbox.ts | 23 ++--- .../src/batch-submitter/tx-batch-submitter.ts | 2 +- packages/batch-submitter/src/da/blob.ts | 27 +++--- packages/batch-submitter/src/da/channel.ts | 16 +++- packages/batch-submitter/src/da/types.ts | 19 ++-- .../src/storage/inbox-storage.ts | 34 +++++-- .../batch-submitter/src/utils/mpc-client.ts | 94 ++++++------------- packages/core-utils/package.json | 2 +- packages/data-transport-layer/package.json | 2 +- yarn.lock | 16 ++-- 13 files changed, 106 insertions(+), 135 deletions(-) diff --git a/package.json b/package.json index 80180cab0397c..1c568e79290e9 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "release": "yarn build && yarn changeset publish" }, "dependencies": { - "ethersv6": "github:ericlee42/ethers#metis" + "ethersv6": "npm:ethers@^6.16.0" }, "resolutions": { "node-gyp": "^9.4.0" diff --git a/packages/batch-submitter/package.json b/packages/batch-submitter/package.json index 69e44a79bca4e..f317b582a3b1e 100755 --- a/packages/batch-submitter/package.json +++ b/packages/batch-submitter/package.json @@ -44,7 +44,7 @@ "bluebird": "^3.7.2", "c-kzg": "^4.0.1", "dotenv": "^10.0.0", - "ethersv6": "github:ericlee42/ethers#metis", + "ethersv6": "npm:ethers@^6.16.0", "lodash": "^4.17.21", "old-contracts": "npm:@eth-optimism/contracts@^0.0.2-alpha.7", "prom-client": "^13.1.0", diff --git a/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts b/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts index 96b6f486b0140..55ca6eb26e55f 100755 --- a/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts +++ b/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts @@ -323,7 +323,7 @@ export class StateBatchSubmitter extends BatchSubmitter { if (!mpcInfo || !mpcInfo.mpc_address) { throw new Error('MPC 1 info get failed') } - const txUnsign: ethers.TransactionRequest = { + const txUnsign: ethers.TransactionLike = { type: 2, to: tx.to, data: tx.data, diff --git a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts index 187c758c413f8..a0fd13a08e974 100644 --- a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts +++ b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts @@ -12,7 +12,6 @@ import { zlibCompressHexString, } from '@metis.io/core-utils' import { Promise as bPromise } from 'bluebird' -import * as kzg from 'c-kzg' import { ethers, Provider, @@ -20,7 +19,6 @@ import { toBeHex, toBigInt, TransactionReceipt, - TransactionRequest, } from 'ethersv6' /* Internal Imports */ @@ -146,7 +144,6 @@ export class TransactionBatchSubmitterInbox { } metrics.numTxPerBatch.observe(endBlock - startBlock) this.logger.info('Submitting batch to inbox', { - meta: batchParams.inputMeta, useBlob, blobTxes: batchParams.blobTxData.length, batchSizeInBytes, @@ -157,7 +154,7 @@ export class TransactionBatchSubmitterInbox { nextBatchIndex, { input: batchParams.inputData, - blobs: batchParams.blobTxData.map((tx) => tx.blobs.map((b) => b.data)), + blobs: batchParams.blobTxData.map((tx) => tx.blobs), txHashes: [], }, signer, @@ -197,7 +194,7 @@ export class TransactionBatchSubmitterInbox { ) => Promise ): Promise { const { chainId } = await signer.provider.getNetwork() - const inboxTx: TransactionRequest = { + const inboxTx: ethers.TransactionLike = { type: 2, chainId, to: this.inboxAddress, @@ -244,7 +241,7 @@ export class TransactionBatchSubmitterInbox { continue } - const blobTx: ethers.TransactionRequest = { + const blobTx: ethers.TransactionLike = { type: 3, // 3 for blob tx type to: this.inboxAddress, // since we are using blob tx, call data will be empty, @@ -253,7 +250,7 @@ export class TransactionBatchSubmitterInbox { chainId, nonce: await signer.provider.getTransactionCount(signerAddress), blobs, - blobVersion: 1, // Osaka is enabled on all the chains + blobWrapperVersion: 1, // Osaka is enabled on all the chains } const replaced = await setTxEIP1559Fees( @@ -285,9 +282,8 @@ export class TransactionBatchSubmitterInbox { // need to append the blob sidecar to the signed tx const signedTxUnmarshaled = ethers.Transaction.from(signedTx) - signedTxUnmarshaled.type = 3 - signedTxUnmarshaled.kzg = kzg - signedTxUnmarshaled.blobVersion = blobTx.blobVersion + signedTxUnmarshaled.blobWrapperVersion = + blobTx.blobWrapperVersion signedTxUnmarshaled.blobs = blobTx.blobs // repack the tx return signedTxUnmarshaled.serialized @@ -661,13 +657,6 @@ export class TransactionBatchSubmitterInbox { encoded = `${da}${compressType}${batchIndex}${l2Start}${totalElements}${compressedEncoded}` return { - inputMeta: { - da, - compressType, - batchIndex, - l2Start, - totalElements, - }, inputData: encoded, batch: blocks, blobTxData, diff --git a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts index 2eb8a32e3e3e8..578c697ec8065 100755 --- a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts +++ b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts @@ -581,7 +581,7 @@ export class TransactionBatchSubmitter extends BatchSubmitter { // this.encodeSequencerBatchOptions // ) // unsigned tx - const tx: ethers.TransactionRequest = { + const tx: ethers.TransactionLike = { to: this.useMinio ? await this.mvmCtcContract.getAddress() : await this.chainContract.getAddress(), diff --git a/packages/batch-submitter/src/da/blob.ts b/packages/batch-submitter/src/da/blob.ts index c8d46fde18530..ccd891d9774b7 100644 --- a/packages/batch-submitter/src/da/blob.ts +++ b/packages/batch-submitter/src/da/blob.ts @@ -1,9 +1,9 @@ import { blobToKzgCommitment, - Bytes48, Blob as CBlob, - computeBlobKzgProof, - verifyBlobKzgProof, + computeCellsAndKzgProofs, + KZGCommitment, + KZGProof, } from 'c-kzg' import { createHash } from 'crypto' import { Frame } from './types' @@ -11,30 +11,23 @@ import { Frame } from './types' const BlobSize = 4096 * 32 const MaxBlobDataSize = (4 * 31 + 3) * 1024 - 4 const EncodingVersion = 0 -const VersionOffset = 1 +// const VersionOffset = 1 const Rounds = 1024 export class Blob { public readonly data: Uint8Array = new Uint8Array(BlobSize) - public readonly commitment: Bytes48 = new Uint8Array(48) - public readonly proof: Bytes48 = new Uint8Array(48) + public readonly commitment: KZGCommitment = new Uint8Array(48) + // cell proofs + public readonly proof: KZGProof = new Uint8Array(48 * 128) public versionedHash: string = '' - static kzgToVersionedHash(commitment: Bytes48): string { + static kzgToVersionedHash(commitment: KZGCommitment): string { const hasher = createHash('sha256') hasher.update(commitment) // versioned hash = [1 byte version][31 byte hash] return '0x01' + hasher.digest('hex').substring(2) } - static verifyBlobProof( - blob: Blob, - commitment: Bytes48, - proof: Bytes48 - ): boolean { - return verifyBlobKzgProof(blob.data as CBlob, commitment, proof) - } - marshalFrame(frame: Frame): Uint8Array { const buffer = new ArrayBuffer(16 + 2 + 4 + frame.data.length + 1) const view = new DataView(buffer) @@ -159,7 +152,9 @@ export class Blob { } this.commitment.set(blobToKzgCommitment(this.data as CBlob)) - this.proof.set(computeBlobKzgProof(this.data as CBlob, this.commitment)) + this.proof.set( + Buffer.concat(computeCellsAndKzgProofs(this.data as CBlob)[1]) + ) this.versionedHash = Blob.kzgToVersionedHash(this.commitment) return this diff --git a/packages/batch-submitter/src/da/channel.ts b/packages/batch-submitter/src/da/channel.ts index a15de09b7d90e..b0add8f002371 100644 --- a/packages/batch-submitter/src/da/channel.ts +++ b/packages/batch-submitter/src/da/channel.ts @@ -1,15 +1,16 @@ // channel.ts +import { Logger } from '@eth-optimism/common-ts' import { ethers } from 'ethersv6' +import { Blob } from './blob' import { ChannelBuilder } from './channel-builder' import { BatchToInboxElement, + BlobLike, ChannelConfig, Frame, RollupConfig, TxData, } from './types' -import { Blob } from './blob' -import { Logger } from '@eth-optimism/common-ts' export class Channel { private channelBuilder: ChannelBuilder @@ -82,8 +83,15 @@ export class Channel { return sb }, - get blobs(): Blob[] { - return this.frames.map((f: Frame) => new Blob().fromFrame(f)) + get blobs(): BlobLike[] { + return this.frames.map((f: Frame) => { + const blob = new Blob().fromFrame(f) + return { + data: blob.data, + proof: blob.proof, + commitment: blob.commitment, + } + }) }, } this.pendingTransactions.set(txData.id, txData) diff --git a/packages/batch-submitter/src/da/types.ts b/packages/batch-submitter/src/da/types.ts index ecd0460dd4c8b..5bacaca42edf4 100644 --- a/packages/batch-submitter/src/da/types.ts +++ b/packages/batch-submitter/src/da/types.ts @@ -1,6 +1,5 @@ // types.ts import { BytesLike, ethers, getBytes } from 'ethersv6' -import { Blob } from './blob' export interface RollupConfig { l1ChainID: bigint @@ -49,12 +48,18 @@ export interface Frame { isLast: boolean } +export type BlobLike = { + data: BytesLike + proof: BytesLike + commitment: BytesLike +} + export interface TxData { frames: Frame[] asBlob: boolean get id(): string - get blobs(): Blob[] + get blobs(): BlobLike[] } export interface L1BlockInfo { @@ -153,18 +158,10 @@ export interface BatchToInboxElement { extraData: string txs: BatchToInboxRawTx[] } -export declare type BatchToInbox = BatchToInboxElement[] -export interface InboxInputMeta { - da: string - compressType: string - batchIndex: string - l2Start: string - totalElements: string -} +export declare type BatchToInbox = BatchToInboxElement[] export interface InboxBatchParams { - inputMeta: InboxInputMeta inputData: string batch: BatchToInbox blobTxData: TxData[] diff --git a/packages/batch-submitter/src/storage/inbox-storage.ts b/packages/batch-submitter/src/storage/inbox-storage.ts index cc8e0e4915dff..1a1edfba66d4e 100644 --- a/packages/batch-submitter/src/storage/inbox-storage.ts +++ b/packages/batch-submitter/src/storage/inbox-storage.ts @@ -1,8 +1,9 @@ /* Imports: External */ import { Logger } from '@eth-optimism/common-ts' -import { BytesLike, toNumber } from 'ethersv6' +import { toNumber } from 'ethersv6' import * as fs from 'fs/promises' import * as path from 'path' +import { BlobLike } from '../da/types' const INBOX_OK_FILE = 'inbox_ok.json' const INBOX_FAIL_FILE = 'inbox_fail.json' @@ -16,7 +17,7 @@ export interface InboxRecordInfo { export interface InboxSteps { input: string // the inbox tx data input - blobs: Array> // array of blob tx data + blobs: Array> // array of blob tx data txHashes: Array // blob tx hashes + inbox tx hash } @@ -75,15 +76,25 @@ export class InboxStorage { } public async insertStep(jsonData: InboxSteps) { - const data = { + const data: InboxSteps = { input: jsonData.input, txHashes: jsonData.txHashes, blobs: jsonData.blobs.map((blobArray) => blobArray.map((blob) => { - if (typeof blob === 'string') { - return blob + return { + data: + typeof blob.data === 'string' + ? blob.data + : '0x' + Buffer.from(blob.data).toString('hex'), + proof: + typeof blob.proof === 'string' + ? blob.proof + : '0x' + Buffer.from(blob.proof).toString('hex'), + commitment: + typeof blob.commitment === 'string' + ? blob.commitment + : '0x' + Buffer.from(blob.commitment).toString('hex'), } - return '0x' + Buffer.from(blob).toString('hex') }) ), } @@ -97,8 +108,15 @@ export class InboxStorage { if (!(await this.fileExists(filePath))) { return null } - const data = await fs.readFile(filePath, 'utf-8') - return JSON.parse(data) + const raw = await fs.readFile(filePath, 'utf-8') + const parsed: InboxSteps = JSON.parse(raw) + if (!Array.isArray(parsed.blobs)) { + throw new Error('Invalid steps file format: blobs is not an array') + } + if (!Array.isArray(parsed.txHashes)) { + throw new Error('Invalid steps file format: txHashes is not an array') + } + return parsed } private async fileExists(filePath) { diff --git a/packages/batch-submitter/src/utils/mpc-client.ts b/packages/batch-submitter/src/utils/mpc-client.ts index 5456d3986cb3e..50d9c16194a57 100644 --- a/packages/batch-submitter/src/utils/mpc-client.ts +++ b/packages/batch-submitter/src/utils/mpc-client.ts @@ -1,8 +1,6 @@ import { Logger } from '@eth-optimism/common-ts' -import '@metis.io/core-utils' -import * as kzg from 'c-kzg' import { randomUUID } from 'crypto' -import { ethers, toBigInt, toNumber, TransactionLike } from 'ethersv6' +import { ethers } from 'ethersv6' import * as http from 'http' import * as https from 'https' import { URL } from 'url' @@ -168,93 +166,59 @@ export class MpcClient { // call this public async signTx( - tx: any, - mpcId: any, + unsigned: ethers.TransactionLike, + mpcId: string, timeoutMilli: number ): Promise { // check tx - if (!tx.gasLimit) { + if (!unsigned.gasLimit) { throw new Error('tx gasLimit is required') } - if (tx.nonce === undefined || tx.nonce === null) { + if (unsigned.nonce === undefined || unsigned.nonce === null) { throw new Error('tx nonce is required') } - - this.logger.info('signing tx with mpc', { - mpcId, - timeout: timeoutMilli / 1e3 + 's', - }) - - // call mpc to sign tx - const unsignedTx: TransactionLike = { - data: tx.data, - nonce: toNumber(tx.nonce), - to: tx.to, - value: tx.value ? toBigInt(tx.value) : toBigInt(0), - gasLimit: toBigInt(tx.gasLimit), - chainId: tx.chainId, - } - if (!tx.type) { - // populate legacy tx - if (!tx.gasPrice) { - throw new Error('gasPrice is required for legacy tx') + if (unsigned.type > 1) { + // check for post-EIP1559 tx + if (!unsigned.maxFeePerGas) { + throw new Error('maxFeePerGas is required for post-EIP1559 tx') } - unsignedTx.gasPrice = toBigInt(tx.gasPrice) - } else { - unsignedTx.accessList = tx.accessList + if (!unsigned.maxPriorityFeePerGas) { + throw new Error('maxPriorityFeePerGas is required for post-EIP1559 tx') + } - // populate typed tx - const txType = toNumber(tx.type) - unsignedTx.type = txType - if (txType === 1) { - // check for access list tx - if (!tx.gasPrice) { - throw new Error('gasPrice is required for access list tx') - } - unsignedTx.gasPrice = toBigInt(tx.gasPrice) - } else if (txType > 1) { - // check for post-EIP1559 tx - if (!tx.maxFeePerGas) { - throw new Error('maxFeePerGas is required for post-EIP1559 tx') + if (unsigned.type === 3) { + // extra checks for blob tx + if (!unsigned.maxFeePerBlobGas) { + throw new Error('maxFeePerBlobGas is required for blob tx') } - if (!tx.maxPriorityFeePerGas) { - throw new Error( - 'maxPriorityFeePerGas is required for post-EIP1559 tx' - ) - } - - unsignedTx.maxFeePerGas = toBigInt(tx.maxFeePerGas) - unsignedTx.maxPriorityFeePerGas = toBigInt(tx.maxPriorityFeePerGas) - if (txType === 3) { - // extra checks for blob tx - if (!tx.maxFeePerBlobGas) { - throw new Error('maxFeePerBlobGas is required for blob tx') - } - - if (!tx.blobs) { - throw new Error('blobs are required for blob tx') - } - - unsignedTx.maxFeePerBlobGas = toBigInt(tx.maxFeePerBlobGas) - unsignedTx.kzg = kzg - unsignedTx.blobVersion = tx.blobVersion - unsignedTx.blobs = tx.blobs + if (!unsigned.blobs || !unsigned.blobs.length) { + throw new Error('blobs are required for blob tx') } } } + this.logger.info('signing tx with mpc', { + mpcId, + timeout: timeoutMilli / 1e3 + 's', + }) + const signId = randomUUID() const postData = { sign_id: signId, mpc_id: mpcId, sign_type: 0, - sign_data: ethers.Transaction.from(unsignedTx).unsignedSerialized, + sign_data: ethers.Transaction.from(unsigned).unsignedSerialized, sign_msg: '', } const signResp = await this.proposeMpcSign(postData) if (!signResp) { + this.logger.error('mpc propose sign failed', { + mpcId, + signId, + signResp, + }) throw new Error(`MPC ${mpcId} propose sign failed`) } diff --git a/packages/core-utils/package.json b/packages/core-utils/package.json index 220f1cfbb3d5e..ea2c4295d7428 100755 --- a/packages/core-utils/package.json +++ b/packages/core-utils/package.json @@ -50,7 +50,7 @@ "dependencies": { "@metis.io/minio": "^7.0.28", "chai": "^4.3.4", - "ethersv6": "github:ericlee42/ethers#metis", + "ethersv6": "npm:ethers@^6.16.0", "lodash": "^4.17.21", "merkletreejs": "^0.2.32" } diff --git a/packages/data-transport-layer/package.json b/packages/data-transport-layer/package.json index 1dc728f92b094..139fbedfda9da 100755 --- a/packages/data-transport-layer/package.json +++ b/packages/data-transport-layer/package.json @@ -40,7 +40,7 @@ "c-kzg": "^4.0.1", "cors": "^2.8.5", "dotenv": "^10.0.0", - "ethersv6": "github:ericlee42/ethers#metis", + "ethersv6": "npm:ethers@^6.16.0", "express": "^4.17.1", "express-prom-bundle": "^6.3.6", "level": "^6.0.1", diff --git a/yarn.lock b/yarn.lock index 814debff3354f..47dcb0290fca0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -415,7 +415,7 @@ __metadata: eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" ethereum-waffle: "npm:^3.3.0" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" ganache-core: "npm:^2.13.2" hardhat: "npm:^2.3.0" lint-staged: "npm:11.0.0" @@ -548,7 +548,7 @@ __metadata: eslint-plugin-prettier: "npm:^3.4.0" eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" express: "npm:^4.17.1" express-prom-bundle: "npm:^6.3.6" hardhat: "npm:^2.3.0" @@ -2808,7 +2808,7 @@ __metadata: eslint-plugin-prettier: "npm:^3.4.0" eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" lint-staged: "npm:11.0.0" lodash: "npm:^4.17.21" merkletreejs: "npm:^0.2.32" @@ -10342,9 +10342,9 @@ __metadata: languageName: node linkType: hard -"ethersv6@github:ericlee42/ethers#metis": - version: 6.15.0 - resolution: "ethersv6@https://github.com/ericlee42/ethers.git#commit=5f376edc77c815bf8c992c426bafc4faaad7876a" +"ethersv6@npm:ethers@^6.16.0": + version: 6.16.0 + resolution: "ethers@npm:6.16.0" dependencies: "@adraffy/ens-normalize": "npm:1.10.1" "@noble/curves": "npm:1.2.0" @@ -10353,7 +10353,7 @@ __metadata: aes-js: "npm:4.0.0-beta.5" tslib: "npm:2.7.0" ws: "npm:8.17.1" - checksum: 10c0/3505e9c9ae3a9e3a4191c3131327b4adfa2f82bcb7ea5de1365e369d02195abb8efa4c04763407b3f3181309b437875af7a86509d37561360f4d15a9fc437bac + checksum: 10c0/65c0ff7016b1592b1961c3b68508902b89fc75e1ad025bab3a722df1b5699fd077551875a7285e65b2d0cfea9a85b2459bb84010a0fa4e4494d231848aee3a73 languageName: node linkType: hard @@ -15716,7 +15716,7 @@ __metadata: eslint-plugin-prettier: "npm:^3.4.0" eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" lerna: "npm:^4.0.0" patch-package: "npm:^6.4.7" prettier: "npm:^2.3.1" From c4291ab805d0c4f184a870b2337be3f87b599d4f Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 10:47:06 +0800 Subject: [PATCH 02/18] dtl: refactor and use axios v1 --- packages/data-transport-layer/package.json | 56 +++---- .../data-transport-layer/src/da/blob/index.ts | 148 ++++++++---------- .../src/da/blob/l1-beacon-client.ts | 49 ++++-- .../data-transport-layer/src/services/run.ts | 4 +- yarn.lock | 19 ++- 5 files changed, 148 insertions(+), 128 deletions(-) diff --git a/packages/data-transport-layer/package.json b/packages/data-transport-layer/package.json index 139fbedfda9da..16a0c57f1e040 100755 --- a/packages/data-transport-layer/package.json +++ b/packages/data-transport-layer/package.json @@ -1,30 +1,5 @@ { - "private": true, - "name": "@eth-optimism/data-transport-layer", - "version": "0.5.0", - "description": "Service for shuttling data from L1 into L2", - "main": "dist/index", - "types": "dist/index", - "files": [ - "dist/index" - ], "author": "Metis.io", - "license": "MIT", - "scripts": { - "clean": "rimraf ./dist ./tsconfig.build.tsbuildinfo", - "clean:db": "rimraf ./db", - "lint": "yarn run lint:fix && yarn run lint:check", - "lint:fix": "yarn lint:check --fix", - "format": "prettier --write '**/*.ts'", - "lint:check": "eslint .", - "start": "ts-node ./src/services/run.ts", - "start:local": "ts-node ./src/services/run.ts | pino-pretty", - "test": "hardhat --config test/config/hardhat.config.ts test", - "test:coverage": "nyc hardhat --config test/config/hardhat.config.ts test && nyc merge .nyc_output coverage.json", - "test:blob": "ts-mocha test/unit-tests/da/blob.spec.ts --timeout 320000 --show-stack-traces", - "build": "tsc -p tsconfig.build.json", - "pre-commit": "lint-staged" - }, "dependencies": { "@eth-optimism/common-ts": "0.1.5", "@eth-optimism/contracts": "^0.4.13", @@ -33,7 +8,7 @@ "@sentry/node": "^6.3.1", "@sentry/tracing": "^6.3.1", "@types/express": "^4.17.12", - "axios": "^0.21.1", + "axios": "^1.13.2", "bcfg": "^0.1.6", "bfj": "^7.0.2", "browser-or-node": "^1.3.0", @@ -48,8 +23,10 @@ "merkletreejs": "^0.3.10", "node-fetch": "^2.6.1", "pako": "^2.1.0", + "qs": "^6.14.0", "rlp": "^3.0.0" }, + "description": "Service for shuttling data from L1 into L2", "devDependencies": { "@types/browser-or-node": "^1.3.0", "@types/chai": "^4.2.18", @@ -58,6 +35,7 @@ "@types/levelup": "^4.3.0", "@types/mocha": "^8.2.2", "@types/node-fetch": "^2.5.10", + "@types/qs": "^6.14.0", "@types/workerpool": "^6.0.0", "@typescript-eslint/eslint-plugin": "^4.26.0", "@typescript-eslint/parser": "^4.26.0", @@ -82,5 +60,29 @@ "rimraf": "^3.0.2", "ts-node": "^10.0.0", "typescript": "^4.3.5" - } + }, + "files": [ + "dist/index" + ], + "license": "MIT", + "main": "dist/index", + "name": "@eth-optimism/data-transport-layer", + "private": true, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "clean": "rimraf ./dist ./tsconfig.build.tsbuildinfo", + "clean:db": "rimraf ./db", + "format": "prettier --write '**/*.ts'", + "lint": "yarn run lint:fix && yarn run lint:check", + "lint:check": "eslint .", + "lint:fix": "yarn lint:check --fix", + "pre-commit": "lint-staged", + "start": "ts-node ./src/services/run.ts", + "start:local": "ts-node ./src/services/run.ts | pino-pretty", + "test": "hardhat --config test/config/hardhat.config.ts test", + "test:blob": "ts-mocha test/unit-tests/da/blob.spec.ts --timeout 320000 --show-stack-traces", + "test:coverage": "nyc hardhat --config test/config/hardhat.config.ts test && nyc merge .nyc_output coverage.json" + }, + "types": "dist/index", + "version": "0.5.0" } diff --git a/packages/data-transport-layer/src/da/blob/index.ts b/packages/data-transport-layer/src/da/blob/index.ts index c67363fc8ff8e..81d0fc41c35da 100644 --- a/packages/data-transport-layer/src/da/blob/index.ts +++ b/packages/data-transport-layer/src/da/blob/index.ts @@ -59,98 +59,82 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { throw new BlobDataExpiredError(blobTxHash) } - // TODO: We might be able to cache this somewhere, no need to retrieve this every time. - // But due to potential chain reorgs, just retrieve the data everytime for now. - // Might need to think of a better solution in the future. - const block = await l1RpcProvider.getBlock(receipt.blockNumber, true) + const block = await l1RpcProvider.getBlock(receipt.blockHash, true) if (!block) { throw new Error(`Block ${receipt.blockNumber} not found`) } - const txs = block.prefetchedTransactions - - // Even we got the hash of the blob tx, we still need to traverse through the blocks - // since we need to count the blob index in the block - let blobIndex = 0 - for (const tx of txs) { - if (!tx) { - continue - } - - // only process the blob tx hash recorded in the commitment - if (blobTxHash.toLowerCase() === tx.hash.toLowerCase()) { - const sender = tx.from - if (!fetchConf.batchSenders.includes(sender.toLowerCase())) { - continue - } - - const datas: Uint8Array[] = [] - if (tx.type !== BlobTxType) { - // We are not processing old transactions those are using call data, - // this should not happen. - throw new Error( - `Found inbox transaction ${tx.hash} that is not using blob, ignore` - ) - } else { - if (!tx.blobVersionedHashes) { - // no blob in this blob tx, ignore - continue - } + const tx = block.prefetchedTransactions[receipt.index] + if (!tx || tx.hash !== blobTxHash) { + throw new Error(`Transaction ${blobTxHash} not found in block`) + } - // get blob hashes and indices - const hashes = tx.blobVersionedHashes.map((hash, index) => ({ - index: blobIndex + index, - hash, - })) - blobIndex += hashes.length - - // fetch blob data from beacon chain - const blobs = await l1BeaconProvider.getBlobs( - block.timestamp, - hashes.map((h) => h.index) - ) - - for (const blob of blobs) { - datas.push(blob.data) - } - } + // only process the blob tx hash recorded in the commitment + const sender = tx.from + if (!fetchConf.batchSenders.includes(sender.toLowerCase())) { + continue + } - let frames: Frame[] = [] - for (const data of datas) { - try { - // parse the frames from the blob data - const parsedFrames = parseFrames(data, block.number) - frames = frames.concat(parsedFrames) - } catch (err) { - // invalid frame data in the blob, stop and throw error - throw new Error(`Failed to parse frames: ${err}`) - } - } + const datas: Uint8Array[] = [] + if (tx.type !== BlobTxType) { + // We are not processing old transactions those are using call data, + // this should not happen. + throw new Error( + `Found inbox transaction ${tx.hash} that is not using blob, ignore` + ) + } else { + if (!tx.blobVersionedHashes || tx.blobVersionedHashes.length === 0) { + // no blob in this blob tx + throw new Error( + `No blobVersionedHashes found in transaction ${tx.hash}` + ) + } - const txMetadata = { - txIndex: tx.index, - inboxAddr: tx.to, - blockNumber: block.number, - blockHash: block.hash, - blockTime: block.timestamp, - chainId: fetchConf.chainId, - sender, - validSender: true, - tx, - frames: frames.map((frame) => ({ - id: Buffer.from(frame.id).toString('hex'), - data: frame.data, - isLast: frame.isLast, - frameNumber: frame.frameNumber, - inclusionBlock: frame.inclusionBlock, - })), - } + // fetch blob data from beacon chain + const blobs = await l1BeaconProvider.getBlobs( + block.timestamp, + tx.blobVersionedHashes + ) + if (blobs.length !== tx.blobVersionedHashes.length) { + throw new Error( + `Blob count mismatch in tx ${tx.hash}: expected ${tx.blobVersionedHashes.length}, got ${blobs.length}` + ) + } + datas.push(...blobs) + } - txsMetadata.push(txMetadata) - } else { - blobIndex += tx.blobVersionedHashes?.length || 0 + let frames: Frame[] = [] + for (const data of datas) { + try { + // parse the frames from the blob data + const parsedFrames = parseFrames(data, receipt.blockNumber) + frames = frames.concat(parsedFrames) + } catch (err) { + // invalid frame data in the blob, stop and throw error + throw new Error(`Failed to parse frames: ${err}`) } } + + const txMetadata = { + txIndex: tx.index, + inboxAddr: tx.to, + blockNumber: receipt.blockNumber, + blockHash: receipt.blockHash, + blockTime: block.timestamp, + chainId: fetchConf.chainId, + sender, + validSender: true, + tx, + frames: frames.map((frame) => ({ + id: Buffer.from(frame.id).toString('hex'), + data: frame.data, + isLast: frame.isLast, + frameNumber: frame.frameNumber, + inclusionBlock: frame.inclusionBlock, + })), + } + + txsMetadata.push(txMetadata) } const channelMap: { [channelId: string]: Channel } = {} diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index c8250aa3771de..04f9e68fd0791 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -1,5 +1,8 @@ -import axios, { AxiosInstance } from 'axios' -import { Blob } from './blob' +import axios, { type AxiosInstance } from 'axios' +import { blobToKzgCommitment, type Blob as CBlob } from 'c-kzg' +import { createHash } from 'crypto' +import { ethers } from 'ethersv6' +import qs from 'qs' export class L1BeaconClient { private readonly http: AxiosInstance @@ -7,7 +10,7 @@ export class L1BeaconClient { public readonly beaconChainGenesisPromise: Promise public readonly beaconChainConfigPromise: Promise - constructor(endpoint: string) { + constructor(endpoint: string, timeoutMs: number = 30000) { const parsed = new URL(endpoint) // extract baseURL (origin + pathname, trim trailing slash) @@ -20,6 +23,7 @@ export class L1BeaconClient { const defaultParams = Object.fromEntries(parsed.searchParams) this.http = axios.create({ baseURL, + timeout: timeoutMs, params: defaultParams, }) @@ -33,25 +37,38 @@ export class L1BeaconClient { } // retrieve blobs from the beacon chain - async getBlobs(timestamp: number, indices: number[]): Promise { + async getBlobs(timestamp: number, indices: string[]): Promise { // calculate the beacon chain slot from the given timestamp const slot = (await this.getTimeToSlotFn())(timestamp) - const sidecars = await this.getBlobSidecars(slot, indices) - const blobs = sidecars.map((sidecar: any) => { - const blob = new Blob(sidecar.blob) - return { - data: blob.toData(), - kzgCommitment: sidecar.kzg_commitment, - kzgProof: sidecar.kzg_proof, + const data = await this.getBlobsByVerHashs(slot, indices) + const blobs = data.map((b) => ethers.toBeArray(b)) + if (blobs.length !== indices.length) { + throw new Error( + `Expected ${indices.length} blobs, but got ${blobs.length}` + ) + } + // verify that the retrieved blobs match the requested versioned hashes + for (const [index, blob] of blobs.entries()) { + const hasher = createHash('sha256') + hasher.update(blobToKzgCommitment(blob as CBlob)) + const versionedHash = '0x01' + hasher.digest('hex').substring(2) + const expectedIndex = indices[index]!.toLowerCase() + if (versionedHash !== expectedIndex) { + throw new Error( + `Blob at index ${index} has invalid versioned hash. Expected ${expectedIndex}, got ${versionedHash}` + ) } - }) + } return blobs } // retrieve blob sidecars from the beacon chain - async getBlobSidecars(slot: number, indices: number[]): Promise { - const response = await this.request(`eth/v1/beacon/blob_sidecars/${slot}`, { - indices: indices.join(','), + async getBlobsByVerHashs( + slot: number, + versioned_hashes: string[] + ): Promise { + const response = await this.request(`eth/v1/beacon/blobs/${slot}`, { + versioned_hashes, }) return response.data } @@ -92,6 +109,8 @@ export class L1BeaconClient { method: 'GET', params: params ?? undefined, validateStatus: () => true, // handle status manually below + paramsSerializer: (params) => + qs.stringify(params, { arrayFormat: 'repeat' }), }) // accept any 2xx as success diff --git a/packages/data-transport-layer/src/services/run.ts b/packages/data-transport-layer/src/services/run.ts index 21452dd851870..5176266e25f06 100755 --- a/packages/data-transport-layer/src/services/run.ts +++ b/packages/data-transport-layer/src/services/run.ts @@ -1,7 +1,8 @@ /* Imports: External */ -import * as dotenv from 'dotenv' import { Bcfg } from '@metis.io/core-utils' import Config from 'bcfg' +import { loadTrustedSetup } from 'c-kzg' +import * as dotenv from 'dotenv' /* Imports: Internal */ import { L1DataTransportService } from './main/service' @@ -10,6 +11,7 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli' ;(async () => { try { dotenv.config() + loadTrustedSetup(0) const config: Bcfg = new Config('data-transport-layer') config.load({ diff --git a/yarn.lock b/yarn.lock index 47dcb0290fca0..b328f2c134204 100644 --- a/yarn.lock +++ b/yarn.lock @@ -527,10 +527,11 @@ __metadata: "@types/levelup": "npm:^4.3.0" "@types/mocha": "npm:^8.2.2" "@types/node-fetch": "npm:^2.5.10" + "@types/qs": "npm:^6.14.0" "@types/workerpool": "npm:^6.0.0" "@typescript-eslint/eslint-plugin": "npm:^4.26.0" "@typescript-eslint/parser": "npm:^4.26.0" - axios: "npm:^0.21.1" + axios: "npm:^1.13.2" bcfg: "npm:^0.1.6" bfj: "npm:^7.0.2" browser-or-node: "npm:^1.3.0" @@ -562,6 +563,7 @@ __metadata: pino-pretty: "npm:^4.7.1" prettier: "npm:^2.3.1" prom-client: "npm:^13.1.0" + qs: "npm:^6.14.0" rimraf: "npm:^3.0.2" rlp: "npm:^3.0.0" ts-node: "npm:^10.0.0" @@ -4564,7 +4566,7 @@ __metadata: languageName: node linkType: hard -"@types/qs@npm:*, @types/qs@npm:^6.2.31, @types/qs@npm:^6.9.7": +"@types/qs@npm:*, @types/qs@npm:^6.14.0, @types/qs@npm:^6.2.31, @types/qs@npm:^6.9.7": version: 6.14.0 resolution: "@types/qs@npm:6.14.0" checksum: 10c0/5b3036df6e507483869cdb3858201b2e0b64b4793dc4974f188caa5b5732f2333ab9db45c08157975054d3b070788b35088b4bc60257ae263885016ee2131310 @@ -5709,6 +5711,17 @@ __metadata: languageName: node linkType: hard +"axios@npm:^1.13.2": + version: 1.13.2 + resolution: "axios@npm:1.13.2" + dependencies: + follow-redirects: "npm:^1.15.6" + form-data: "npm:^4.0.4" + proxy-from-env: "npm:^1.1.0" + checksum: 10c0/e8a42e37e5568ae9c7a28c348db0e8cf3e43d06fcbef73f0048669edfe4f71219664da7b6cc991b0c0f01c28a48f037c515263cb79be1f1ae8ff034cd813867b + languageName: node + linkType: hard + "axios@npm:^1.4.0, axios@npm:^1.5.1": version: 1.13.0 resolution: "axios@npm:1.13.0" @@ -17817,7 +17830,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.11.0, qs@npm:^6.12.3, qs@npm:^6.4.0, qs@npm:^6.9.4": +"qs@npm:^6.11.0, qs@npm:^6.12.3, qs@npm:^6.14.0, qs@npm:^6.4.0, qs@npm:^6.9.4": version: 6.14.0 resolution: "qs@npm:6.14.0" dependencies: From f7b0b81bf2d86fb792def1edb469118faedcd53f Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 10:47:35 +0800 Subject: [PATCH 03/18] refactor: simplify paramsSerializer function in L1BeaconClient --- packages/data-transport-layer/src/da/blob/l1-beacon-client.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index 04f9e68fd0791..1560922b18f3a 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -109,8 +109,7 @@ export class L1BeaconClient { method: 'GET', params: params ?? undefined, validateStatus: () => true, // handle status manually below - paramsSerializer: (params) => - qs.stringify(params, { arrayFormat: 'repeat' }), + paramsSerializer: (p) => qs.stringify(p, { arrayFormat: 'repeat' }), }) // accept any 2xx as success From ae6937abc29c86817804a83b602567effe62c12a Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 16:28:44 +0800 Subject: [PATCH 04/18] batch: refactor Blob class and L1BeaconClient for improved data handling --- .../data-transport-layer/src/da/blob/blob.ts | 35 ++++++------------- .../data-transport-layer/src/da/blob/index.ts | 16 ++------- .../src/da/blob/l1-beacon-client.ts | 31 +++++++++------- 3 files changed, 31 insertions(+), 51 deletions(-) diff --git a/packages/data-transport-layer/src/da/blob/blob.ts b/packages/data-transport-layer/src/da/blob/blob.ts index f7c76e33ee321..69f77a5636f6f 100644 --- a/packages/data-transport-layer/src/da/blob/blob.ts +++ b/packages/data-transport-layer/src/da/blob/blob.ts @@ -1,35 +1,14 @@ +import { blobToKzgCommitment } from 'c-kzg' +import { createHash } from 'crypto' + const BlobSize = 4096 * 32 const MaxBlobDataSize = (4 * 31 + 3) * 1024 - 4 const EncodingVersion = 0 const VersionOffset = 1 const Rounds = 1024 -const hexStringToUint8Array = (hexString: string): Uint8Array => { - hexString = hexString.replace(/^0x/, '').replace(/\s/g, '') - - if (hexString.length % 2 !== 0) { - throw new Error('Invalid hex string') - } - - const arrayBuffer = new Uint8Array(hexString.length / 2) - - for (let i = 0; i < hexString.length; i += 2) { - const byteValue = parseInt(hexString.substring(i, i + 2), 16) - if (isNaN(byteValue)) { - throw new Error('Invalid hex string') - } - arrayBuffer[i / 2] = byteValue - } - - return arrayBuffer -} - export class Blob { - private readonly data: Uint8Array - - constructor(hex: string) { - this.data = hexStringToUint8Array(hex) - } + constructor(public readonly data: Uint8Array) {} toString(): string { return Buffer.from(this.data).toString('hex') @@ -100,6 +79,12 @@ export class Blob { this.data.fill(0) } + versionedHash(): string { + const hasher = createHash('sha256') + hasher.update(blobToKzgCommitment(this.data)) + return '0x01' + hasher.digest('hex').substring(2) + } + private decodeFieldElement( opos: number, ipos: number, diff --git a/packages/data-transport-layer/src/da/blob/index.ts b/packages/data-transport-layer/src/da/blob/index.ts index 81d0fc41c35da..3b571045c5c02 100644 --- a/packages/data-transport-layer/src/da/blob/index.ts +++ b/packages/data-transport-layer/src/da/blob/index.ts @@ -75,7 +75,7 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { continue } - const datas: Uint8Array[] = [] + const frames: Frame[] = [] if (tx.type !== BlobTxType) { // We are not processing old transactions those are using call data, // this should not happen. @@ -100,18 +100,8 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { `Blob count mismatch in tx ${tx.hash}: expected ${tx.blobVersionedHashes.length}, got ${blobs.length}` ) } - datas.push(...blobs) - } - - let frames: Frame[] = [] - for (const data of datas) { - try { - // parse the frames from the blob data - const parsedFrames = parseFrames(data, receipt.blockNumber) - frames = frames.concat(parsedFrames) - } catch (err) { - // invalid frame data in the blob, stop and throw error - throw new Error(`Failed to parse frames: ${err}`) + for (const blob of blobs) { + frames.push(...parseFrames(blob, receipt.blockNumber)) } } diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index 1560922b18f3a..75cbadd12b763 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -1,8 +1,7 @@ import axios, { type AxiosInstance } from 'axios' -import { blobToKzgCommitment, type Blob as CBlob } from 'c-kzg' -import { createHash } from 'crypto' import { ethers } from 'ethersv6' import qs from 'qs' +import { Blob } from './blob' export class L1BeaconClient { private readonly http: AxiosInstance @@ -41,17 +40,10 @@ export class L1BeaconClient { // calculate the beacon chain slot from the given timestamp const slot = (await this.getTimeToSlotFn())(timestamp) const data = await this.getBlobsByVerHashs(slot, indices) - const blobs = data.map((b) => ethers.toBeArray(b)) - if (blobs.length !== indices.length) { - throw new Error( - `Expected ${indices.length} blobs, but got ${blobs.length}` - ) - } + const blobs = data.map((b) => new Blob(ethers.toBeArray(b))) // verify that the retrieved blobs match the requested versioned hashes for (const [index, blob] of blobs.entries()) { - const hasher = createHash('sha256') - hasher.update(blobToKzgCommitment(blob as CBlob)) - const versionedHash = '0x01' + hasher.digest('hex').substring(2) + const versionedHash = blob.versionedHash() const expectedIndex = indices[index]!.toLowerCase() if (versionedHash !== expectedIndex) { throw new Error( @@ -59,7 +51,7 @@ export class L1BeaconClient { ) } } - return blobs + return blobs.map((b) => b.toData()) } // retrieve blob sidecars from the beacon chain @@ -70,7 +62,20 @@ export class L1BeaconClient { const response = await this.request(`eth/v1/beacon/blobs/${slot}`, { versioned_hashes, }) - return response.data + const res = response.data + if (!Array.isArray(res)) { + throw new Error( + `Invalid response for blobs at slot ${slot} with versioned_hashes ${versioned_hashes}: ${JSON.stringify( + res + )}` + ) + } + if (res.length !== versioned_hashes.length) { + throw new Error( + `Blob count mismatch: expected ${versioned_hashes.length}, got ${res.length}` + ) + } + return res } // calculate the slot number from a given timestamp From e104545154b321a2affbfe0b0a67f662ced26861 Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 17:05:11 +0800 Subject: [PATCH 05/18] batch: enhance Blob class for hex string initialization and update L1BeaconClient and tests --- .../data-transport-layer/src/da/blob/blob.ts | 31 +++++++++++++++++-- .../src/da/blob/l1-beacon-client.ts | 5 ++- .../src/services/l1-ingestion/service.ts | 6 ++-- .../test/unit-tests/da/blob.spec.ts | 10 +++--- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/packages/data-transport-layer/src/da/blob/blob.ts b/packages/data-transport-layer/src/da/blob/blob.ts index 69f77a5636f6f..420eb4cff3fa2 100644 --- a/packages/data-transport-layer/src/da/blob/blob.ts +++ b/packages/data-transport-layer/src/da/blob/blob.ts @@ -8,7 +8,34 @@ const VersionOffset = 1 const Rounds = 1024 export class Blob { - constructor(public readonly data: Uint8Array) {} + private readonly data: Uint8Array + + constructor(hexString: string) { + if (hexString.startsWith('0x') || hexString.startsWith('0X')) { + hexString = hexString.slice(2) + } + + if (hexString.length % 2 !== 0) { + throw new Error('Invalid hex string') + } + + const buffer = new Uint8Array(hexString.length / 2) + for (let i = 0; i < hexString.length; i += 2) { + const byteValue = parseInt(hexString.substring(i, i + 2), 16) + if (isNaN(byteValue)) { + throw new Error('Invalid hex string') + } + buffer[i / 2] = byteValue + } + + if (buffer.length !== BlobSize) { + throw new Error( + `Invalid blob size: expected ${BlobSize}, got ${buffer.length}` + ) + } + + this.data = buffer + } toString(): string { return Buffer.from(this.data).toString('hex') @@ -20,7 +47,7 @@ export class Blob { )}..${Buffer.from(this.data.slice(BlobSize - 3)).toString('hex')}` } - toData(): Uint8Array { + resolve(): Uint8Array { if (this.data[VersionOffset] !== EncodingVersion) { throw new Error( `Invalid encoding version, expected: ${EncodingVersion}, got: ${this.data[VersionOffset]}` diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index 75cbadd12b763..1b6830e32175b 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -1,5 +1,4 @@ import axios, { type AxiosInstance } from 'axios' -import { ethers } from 'ethersv6' import qs from 'qs' import { Blob } from './blob' @@ -40,7 +39,7 @@ export class L1BeaconClient { // calculate the beacon chain slot from the given timestamp const slot = (await this.getTimeToSlotFn())(timestamp) const data = await this.getBlobsByVerHashs(slot, indices) - const blobs = data.map((b) => new Blob(ethers.toBeArray(b))) + const blobs = data.map((b) => new Blob(b)) // verify that the retrieved blobs match the requested versioned hashes for (const [index, blob] of blobs.entries()) { const versionedHash = blob.versionedHash() @@ -51,7 +50,7 @@ export class L1BeaconClient { ) } } - return blobs.map((b) => b.toData()) + return blobs.map((b) => b.resolve()) } // retrieve blob sidecars from the beacon chain diff --git a/packages/data-transport-layer/src/services/l1-ingestion/service.ts b/packages/data-transport-layer/src/services/l1-ingestion/service.ts index 27d234f3e6fb1..0defe05f9021b 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/service.ts @@ -803,7 +803,7 @@ export class L1IngestionService extends BaseService { ): Promise { const chainId = this.state.l1ChainId.toString() if (addressEvent[chainId]) { - this.logger.info( + this.logger.debug( `Reading from local ${contractName}, chainId is ${chainId}` ) const addressDict = addressEvent[chainId] @@ -816,7 +816,7 @@ export class L1IngestionService extends BaseService { const addr = arr[i] if (blockNumber >= addr.Start) { findAddress = addr.Address - this.logger.info( + this.logger.debug( `Read cached contract address for ${contractName} from ${addr.Start} to ${blockNumber}, get ${findAddress}` ) break @@ -824,7 +824,7 @@ export class L1IngestionService extends BaseService { } return findAddress } - this.logger.info(`Searching from RPC ${contractName}`) + this.logger.debug(`Searching from RPC ${contractName}`) const events = await this.state.contracts.Lib_AddressManager.queryFilter( this.state.contracts.Lib_AddressManager.filters.AddressSet(contractName), this.state.startingL1BlockNumber, diff --git a/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts b/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts index d541971e242ed..6dd961e19a42f 100644 --- a/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts +++ b/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts @@ -1,5 +1,5 @@ -import { parseFrames } from '../../../src/da/blob/frame' -import { l1BlobData } from '../examples/l1-data' +import { ethers, hexlify } from 'ethersv6' +import { Blob } from '../../../src/da/blob/blob' import { BatchData, batchReader, @@ -7,15 +7,15 @@ import { RawSpanBatch, SpanBatchType, } from '../../../src/da/blob/channel' -import { Blob } from '../../../src/da/blob/blob' -import { ethers, hexlify } from 'ethersv6' +import { parseFrames } from '../../../src/da/blob/frame' +import { l1BlobData } from '../examples/l1-data' describe('Decode Blob Transaction', function () { this.timeout(60000) it('should decode blob data and restore transactions', async () => { const blob = new Blob(l1BlobData) - const frames = parseFrames(blob.toData(), 0) + const frames = parseFrames(blob.resolve(), 0) const channel = new Channel(hexlify(frames[0].id), frames[0].inclusionBlock) for (const item of frames) { From 67b9cafa8847ba47a55d78166298a6bac6fd18a7 Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 17:24:54 +0800 Subject: [PATCH 06/18] batch: rename resolve method to toData in Blob class and update L1BeaconClient and tests --- packages/data-transport-layer/src/da/blob/blob.ts | 2 +- packages/data-transport-layer/src/da/blob/l1-beacon-client.ts | 2 +- packages/data-transport-layer/test/unit-tests/da/blob.spec.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/data-transport-layer/src/da/blob/blob.ts b/packages/data-transport-layer/src/da/blob/blob.ts index 420eb4cff3fa2..59163af154253 100644 --- a/packages/data-transport-layer/src/da/blob/blob.ts +++ b/packages/data-transport-layer/src/da/blob/blob.ts @@ -47,7 +47,7 @@ export class Blob { )}..${Buffer.from(this.data.slice(BlobSize - 3)).toString('hex')}` } - resolve(): Uint8Array { + toData(): Uint8Array { if (this.data[VersionOffset] !== EncodingVersion) { throw new Error( `Invalid encoding version, expected: ${EncodingVersion}, got: ${this.data[VersionOffset]}` diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index 1b6830e32175b..9553bf5c17d2f 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -50,7 +50,7 @@ export class L1BeaconClient { ) } } - return blobs.map((b) => b.resolve()) + return blobs.map((b) => b.toData()) } // retrieve blob sidecars from the beacon chain diff --git a/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts b/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts index 6dd961e19a42f..cc6ac28e1d9a1 100644 --- a/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts +++ b/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts @@ -15,7 +15,7 @@ describe('Decode Blob Transaction', function () { it('should decode blob data and restore transactions', async () => { const blob = new Blob(l1BlobData) - const frames = parseFrames(blob.resolve(), 0) + const frames = parseFrames(blob.toData(), 0) const channel = new Channel(hexlify(frames[0].id), frames[0].inclusionBlock) for (const item of frames) { From 57590d3588770b72595605a82d44c785422869ef Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 17:28:58 +0800 Subject: [PATCH 07/18] dtl: sort package.json --- packages/data-transport-layer/package.json | 52 +++++++++++----------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/data-transport-layer/package.json b/packages/data-transport-layer/package.json index 16a0c57f1e040..88b08960397ab 100755 --- a/packages/data-transport-layer/package.json +++ b/packages/data-transport-layer/package.json @@ -1,5 +1,30 @@ { + "private": true, + "name": "@eth-optimism/data-transport-layer", + "version": "0.5.0", + "description": "Service for shuttling data from L1 into L2", + "main": "dist/index", + "types": "dist/index", + "files": [ + "dist/index" + ], "author": "Metis.io", + "license": "MIT", + "scripts": { + "clean": "rimraf ./dist ./tsconfig.build.tsbuildinfo", + "clean:db": "rimraf ./db", + "lint": "yarn run lint:fix && yarn run lint:check", + "lint:fix": "yarn lint:check --fix", + "format": "prettier --write '**/*.ts'", + "lint:check": "eslint .", + "start": "ts-node ./src/services/run.ts", + "start:local": "ts-node ./src/services/run.ts | pino-pretty", + "test": "hardhat --config test/config/hardhat.config.ts test", + "test:coverage": "nyc hardhat --config test/config/hardhat.config.ts test && nyc merge .nyc_output coverage.json", + "test:blob": "ts-mocha test/unit-tests/da/blob.spec.ts --timeout 320000 --show-stack-traces", + "build": "tsc -p tsconfig.build.json", + "pre-commit": "lint-staged" + }, "dependencies": { "@eth-optimism/common-ts": "0.1.5", "@eth-optimism/contracts": "^0.4.13", @@ -26,7 +51,6 @@ "qs": "^6.14.0", "rlp": "^3.0.0" }, - "description": "Service for shuttling data from L1 into L2", "devDependencies": { "@types/browser-or-node": "^1.3.0", "@types/chai": "^4.2.18", @@ -60,29 +84,5 @@ "rimraf": "^3.0.2", "ts-node": "^10.0.0", "typescript": "^4.3.5" - }, - "files": [ - "dist/index" - ], - "license": "MIT", - "main": "dist/index", - "name": "@eth-optimism/data-transport-layer", - "private": true, - "scripts": { - "build": "tsc -p tsconfig.build.json", - "clean": "rimraf ./dist ./tsconfig.build.tsbuildinfo", - "clean:db": "rimraf ./db", - "format": "prettier --write '**/*.ts'", - "lint": "yarn run lint:fix && yarn run lint:check", - "lint:check": "eslint .", - "lint:fix": "yarn lint:check --fix", - "pre-commit": "lint-staged", - "start": "ts-node ./src/services/run.ts", - "start:local": "ts-node ./src/services/run.ts | pino-pretty", - "test": "hardhat --config test/config/hardhat.config.ts test", - "test:blob": "ts-mocha test/unit-tests/da/blob.spec.ts --timeout 320000 --show-stack-traces", - "test:coverage": "nyc hardhat --config test/config/hardhat.config.ts test && nyc merge .nyc_output coverage.json" - }, - "types": "dist/index", - "version": "0.5.0" + } } From 91cebda341f6b34522fe75a156899411b5af6985 Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 17:33:08 +0800 Subject: [PATCH 08/18] fix: update MAX_BLOB_NUM_PER_TX to 6 for consistency in transaction limits --- packages/batch-submitter/src/da/consts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/batch-submitter/src/da/consts.ts b/packages/batch-submitter/src/da/consts.ts index 4ba41c2dbf863..449604c01ac2c 100644 --- a/packages/batch-submitter/src/da/consts.ts +++ b/packages/batch-submitter/src/da/consts.ts @@ -1,6 +1,6 @@ export const FRAME_OVERHEAD_SIZE = 200 export const MAX_RLP_BYTES_PER_CHANNEL = 100_000_000 export const MAX_BLOB_SIZE = (4 * 31 + 3) * 1024 - 4 -export const MAX_BLOB_NUM_PER_TX = 7 +export const MAX_BLOB_NUM_PER_TX = 6 export const TX_GAS = 21_000 export const CHANNEL_FULL_ERR = new Error('Channel is full') From 5d4bdb2b085442363676783fb55600fa705cf36f Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 20:00:18 +0800 Subject: [PATCH 09/18] batch: refactor fetchBatches and L1BeaconClient integration for improved clarity and performance --- .../data-transport-layer/src/da/blob/index.ts | 76 +++++-------------- .../src/da/blob/l1-beacon-client.ts | 4 +- .../handlers/sequencer-batch-inbox.ts | 50 ++++-------- .../src/services/l1-ingestion/service.ts | 45 ++++++----- .../src/services/l2-ingestion/service.ts | 13 ++-- .../src/services/main/service.ts | 14 ++-- .../data-transport-layer/src/services/run.ts | 6 +- .../src/services/server/service.ts | 34 +++------ 8 files changed, 92 insertions(+), 150 deletions(-) diff --git a/packages/data-transport-layer/src/da/blob/index.ts b/packages/data-transport-layer/src/da/blob/index.ts index 3b571045c5c02..cf630278fa768 100644 --- a/packages/data-transport-layer/src/da/blob/index.ts +++ b/packages/data-transport-layer/src/da/blob/index.ts @@ -21,17 +21,15 @@ interface FetchBatchesConfig { concurrentRequests: number // concurrent requests number l2ChainId: number // l2 chain id - l1Rpc: string // l1 rpc url - l1Beacon: string // l1 beacon chain url + l1RpcProvider: ethers.JsonRpcProvider + l1BeaconProvider: L1BeaconClient } let chainIdHasChecked = false // whether chain id has been checked // fetch l2 batches from l1 chain export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { - const l1RpcProvider = new ethers.JsonRpcProvider(fetchConf.l1Rpc) - const l1BeaconProvider = new L1BeaconClient(fetchConf.l1Beacon) - + const { l1RpcProvider, l1BeaconProvider } = fetchConf if (!chainIdHasChecked) { const checkId = await l1BeaconProvider.getChainId() if (Number(checkId) !== fetchConf.chainId) { @@ -128,13 +126,11 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { } const channelMap: { [channelId: string]: Channel } = {} - const frameDataValidity: { [channelId: string]: boolean } = {} // process downloaded tx metadata for (const txMetadata of txsMetadata) { const framesData = txMetadata.frames - const invalidFrames = false for (const frameData of framesData) { const frame: Frame = { id: Buffer.from(frameData.id, 'hex'), @@ -150,17 +146,9 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { } // add frames to channel - try { - channelMap[channelId].addFrame(frame) - frameDataValidity[channelId] = true - } catch (e) { - frameDataValidity[channelId] = false - } + channelMap[channelId].addFrame(frame) } - if (invalidFrames) { - continue - } for (const channelId in channelMap) { if (!channelMap.hasOwnProperty(channelId)) { // ignore object prototype properties @@ -182,21 +170,6 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { } ) - // short circuit if frame data is invalid - if (!frameDataValidity[channelId]) { - channelsMetadata.push({ - id: channelId, - isReady: channel.isReady(), - invalidFrames: true, - invalidBatches: false, - frames: framesMetadata, - batches: [], - batchTypes: [], - comprAlgos: [], - }) - continue - } - if (!channel || !channel.isReady()) { continue } @@ -207,37 +180,28 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { const batches = [] const batchTypes = [] const comprAlgos = [] - let invalidBatches = false - - try { - // By default, this is after fjord, since we are directly upgrade to fjord, - // so no need to keep compatibility for old op versions - const readBatch = await batchReader(reader) - let batchData: BatchData | null - while ((batchData = await readBatch())) { - if (batchData.batchType === SpanBatchType) { - const spanBatch = batchData.inner as RawSpanBatch - batchData.inner = await spanBatch.derive( - ethers.toBigInt(fetchConf.l2ChainId) - ) - } - batches.push(batchData.inner) - batchTypes.push(batchData.batchType) - if (batchData.comprAlgo) { - comprAlgos.push(batchData.comprAlgo) - } + + // By default, this is after fjord, since we are directly upgrade to fjord, + // so no need to keep compatibility for old op versions + const readBatch = await batchReader(reader) + let batchData: BatchData | null + while ((batchData = await readBatch())) { + if (batchData.batchType === SpanBatchType) { + const spanBatch = batchData.inner as RawSpanBatch + batchData.inner = await spanBatch.derive( + ethers.toBigInt(fetchConf.l2ChainId) + ) + } + batches.push(batchData.inner) + batchTypes.push(batchData.batchType) + if (batchData.comprAlgo) { + comprAlgos.push(batchData.comprAlgo) } - } catch (err) { - // mark batches as invalid - console.log(`Failed to read batches: ${err}`) - invalidBatches = true } const channelMetadata = { id: channelId, isReady: channel.isReady(), - invalidFrames: false, - invalidBatches, frames: framesMetadata, batches, batchTypes, diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index 9553bf5c17d2f..afb84b71947d1 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -97,9 +97,9 @@ export class L1BeaconClient { } } - async getChainId(): Promise { + async getChainId(): Promise { const response = await this.beaconChainConfigPromise - return response.data.DEPOSIT_NETWORK_ID + return Number(response.data.DEPOSIT_NETWORK_ID) } private async request( diff --git a/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts b/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts index 0ec8ebe6c4a57..e7add29b0ab03 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts @@ -23,7 +23,7 @@ import { TransactionEntry, } from '../../../types' import { parseSignatureVParam, SEQUENCER_GAS_LIMIT } from '../../../utils' -import { BlobDataExpiredError, MissingElementError } from './errors' +import { MissingElementError } from './errors' export const handleEventsSequencerBatchInbox: EventHandlerSetAny< SequencerBatchAppendedExtraData, @@ -96,7 +96,7 @@ export const handleEventsSequencerBatchInbox: EventHandlerSetAny< const da = toNumber(calldata.subarray(0, 1)) const compressType = toNumber(calldata.subarray(1, 2)) let contextData = calldata.subarray(70) - let channels = [] + const channels = [] // da first if (da === 1) { const storageObject = remove0x(toHexString(contextData)) @@ -143,38 +143,20 @@ export const handleEventsSequencerBatchInbox: EventHandlerSetAny< blobTxHashes.push(ethers.hexlify(contextData.subarray(i, i + 32))) } - try { - channels = channels.concat( - await fetchBatches({ - blobTxHashes, - chainId: toNumber(l1ChainId), - batchInbox: options.batchInboxAddress, - batchSenders: - extraData.context && extraData.context.inboxBlobSenderAddress - ? [extraData.context.inboxBlobSenderAddress.toLowerCase()] - : [], - concurrentRequests: 0, - l1Rpc: options.l1RpcProvider, - l1Beacon: options.l1BeaconProvider, - l2ChainId: options.l2ChainId, - }) - ) - } catch (e) { - if (e instanceof BlobDataExpiredError) { - // if blob data has already expired, we must recover from a snapshot - console.error( - `Blob data has already expired, please recover from a most recent snapshot, error: ${e}` - ) - process.exit(1) - } - throw e - } - - for (const channel of channels) { - if (channel.invalidBatches || channel.invalidFrames) { - throw new Error(`Invalid batches found: ${channel.id}`) - } - } + const result = await fetchBatches({ + blobTxHashes, + chainId: toNumber(l1ChainId), + l2ChainId: options.l2ChainId, + batchInbox: options.batchInboxAddress, + batchSenders: + extraData.context && extraData.context.inboxBlobSenderAddress + ? [extraData.context.inboxBlobSenderAddress.toLowerCase()] + : [], + concurrentRequests: 0, + l1RpcProvider: options.l1RpcProvider, + l1BeaconProvider: options.l1BeaconProvider, + }) + channels.push(...result) } if (compressType === 11) { contextData = await zlibDecompress(contextData) diff --git a/packages/data-transport-layer/src/services/l1-ingestion/service.ts b/packages/data-transport-layer/src/services/l1-ingestion/service.ts index 0defe05f9021b..6bc17c310569c 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/service.ts @@ -1,6 +1,6 @@ /* Imports: External */ import { BaseService, Metrics } from '@eth-optimism/common-ts' -import { FallbackProvider, fromHexString } from '@metis.io/core-utils' +import { fromHexString } from '@metis.io/core-utils' import { Block, ethers, @@ -13,6 +13,7 @@ import { LevelUp } from 'levelup' import { Counter, Gauge } from 'prom-client' /* Imports: Internal */ +import { L1BeaconClient } from '../../da/blob/l1-beacon-client' import { TransportDB, TransportDBMap, @@ -122,6 +123,7 @@ export class L1IngestionService extends BaseService { dbOfL2: TransportDB contracts: OptimismContracts l1RpcProvider: Provider + l1BeaconProvider: L1BeaconClient l1ChainId: number startingL1BlockNumber: number startingL1BatchIndex: number @@ -143,29 +145,32 @@ export class L1IngestionService extends BaseService { this.state.dbs = {} this.l1IngestionMetrics = registerMetrics(this.metrics) - if (typeof this.options.l1RpcProvider === 'string') { - // FIXME: ethers v6's fallback provider has a bug that it will lost the prefetched transaction data while - // converting the json rpc data to ethers transaction object, so we will only enable it when we configured - // multiple rpc providers, but this will cause a significant performance downgrade. Because we need to fetch - // the transactions one by one. - if (this.options.l1RpcProvider.indexOf(',') >= 0) { - this.logger.info('Using FallbackProvider for L1 RPC Provider') - this.state.l1RpcProvider = FallbackProvider(this.options.l1RpcProvider) - } else { - this.state.l1RpcProvider = new ethers.JsonRpcProvider( - this.options.l1RpcProvider - ) - } - } else { - this.state.l1RpcProvider = this.options.l1RpcProvider - } + this.state.l1RpcProvider = new ethers.JsonRpcProvider( + this.options.l1RpcEndpoint + ) const network = await this.state.l1RpcProvider.getNetwork() this.state.l1ChainId = toNumber(network.chainId) this.logger.info('Using L1 RPC Provider', { l1ChainId: this.state.l1ChainId, + l1RpcEndpoint: this.options.l1RpcEndpoint, }) + console.log( + 'Initializing L1 Beacon Client...', + this.options.l1BeaconEndpoint + ) + this.state.l1BeaconProvider = new L1BeaconClient( + this.options.l1BeaconEndpoint + ) + + const l1BeaconChainId = await this.state.l1BeaconProvider.getChainId() + if (l1BeaconChainId !== this.state.l1ChainId) { + throw new Error( + `Chain ID mismatch: Beacon ${l1BeaconChainId} !== L1 RPC ${this.state.l1ChainId}` + ) + } + this.logger.info('Using AddressManager', { addressManager: this.options.addressManager, }) @@ -652,7 +657,11 @@ export class L1IngestionService extends BaseService { extraData, this.options.l2ChainId, this.state.l1ChainId, - this.options + { + ...this.options, + l1RpcProvider: this.state.l1RpcProvider, + l1BeaconProvider: this.state.l1BeaconProvider, + } ) this.logger.info('Storing Inbox Batch:', { chainId: this.options.l2ChainId, diff --git a/packages/data-transport-layer/src/services/l2-ingestion/service.ts b/packages/data-transport-layer/src/services/l2-ingestion/service.ts index f2c530f3fafe4..a88e92b6660f7 100755 --- a/packages/data-transport-layer/src/services/l2-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l2-ingestion/service.ts @@ -1,17 +1,17 @@ /* Imports: External */ import { BaseService, Metrics } from '@eth-optimism/common-ts' -import { LevelUp } from 'levelup' import axios from 'axios' import bfj from 'bfj' -import { Gauge } from 'prom-client' +import { JsonRpcProvider, toNumber } from 'ethersv6' +import { LevelUp } from 'levelup' import path from 'path' -import { toNumber, JsonRpcProvider } from 'ethersv6' +import { Gauge } from 'prom-client' /* Imports: Internal */ import { TransportDB, - TransportDBMapHolder, TransportDBMap, + TransportDBMapHolder, } from '../../db/transport-db' import { sleep, toRpcHexString, validators } from '../../utils' import { L1DataTransportServiceOptions } from '../main/service' @@ -105,10 +105,7 @@ export class L2IngestionService extends BaseService { `${this.options.l2ChainId}` ) } else { - this.state.l2RpcProvider = - typeof this.options.l2RpcProvider === 'string' - ? new JsonRpcProvider(this.options.l2RpcProvider) - : this.options.l2RpcProvider + this.state.l2RpcProvider = new JsonRpcProvider(this.options.l2RpcEndpoint) } } diff --git a/packages/data-transport-layer/src/services/main/service.ts b/packages/data-transport-layer/src/services/main/service.ts index 44d93c6908078..0c59882dce431 100755 --- a/packages/data-transport-layer/src/services/main/service.ts +++ b/packages/data-transport-layer/src/services/main/service.ts @@ -1,15 +1,15 @@ /* Imports: External */ import { BaseService, Metrics } from '@eth-optimism/common-ts' -import { LevelUp } from 'levelup' import level from 'level' +import { LevelUp } from 'levelup' import { Counter } from 'prom-client' /* Imports: Internal */ -import { L1IngestionService } from '../l1-ingestion/service' -import { L1TransportServer } from '../server/service' +import { TransportDBMapHolder } from '../../db/transport-db' import { validators } from '../../utils' +import { L1IngestionService } from '../l1-ingestion/service' import { L2IngestionService } from '../l2-ingestion/service' -import { TransportDBMapHolder } from '../../db/transport-db' +import { L1TransportServer } from '../server/service' export interface L1DataTransportServiceOptions { nodeEnv: string @@ -19,10 +19,10 @@ export interface L1DataTransportServiceOptions { confirmations: number dangerouslyCatchAllErrors?: boolean hostname: string - l1RpcProvider: string - l1BeaconProvider: string + l1RpcEndpoint: string + l1BeaconEndpoint: string l2ChainId: number - l2RpcProvider: string + l2RpcEndpoint: string metrics?: Metrics dbPath: string logsPerPollingInterval: number diff --git a/packages/data-transport-layer/src/services/run.ts b/packages/data-transport-layer/src/services/run.ts index 5176266e25f06..c4bf1f1789419 100755 --- a/packages/data-transport-layer/src/services/run.ts +++ b/packages/data-transport-layer/src/services/run.ts @@ -27,7 +27,7 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli' port: config.uint('server-port', 7878), hostname: config.str('server-hostname', 'localhost'), confirmations: config.uint('confirmations', 35), - l1RpcProvider: config.str('l1-rpc-endpoint'), + l1RpcEndpoint: config.str('l1-rpc-endpoint'), addressManager: config.str('address-manager'), pollingInterval: config.uint('polling-interval', 5000), logsPerPollingInterval: config.uint('logs-per-polling-interval', 2000), @@ -35,8 +35,8 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli' 'dangerously-catch-all-errors', false ), - l1BeaconProvider: config.str('l1-beacon-endpoint'), - l2RpcProvider: config.str('l2-rpc-endpoint'), + l1BeaconEndpoint: config.str('l1-beacon-endpoint'), + l2RpcEndpoint: config.str('l2-rpc-endpoint'), l2ChainId: config.uint('l2-chain-id'), syncFromL1: config.bool('sync-from-l1', true), syncFromL2: config.bool('sync-from-l2', false), diff --git a/packages/data-transport-layer/src/services/server/service.ts b/packages/data-transport-layer/src/services/server/service.ts index 25a6836e82411..869e4a57c9383 100755 --- a/packages/data-transport-layer/src/services/server/service.ts +++ b/packages/data-transport-layer/src/services/server/service.ts @@ -1,34 +1,31 @@ /* Imports: External */ import { BaseService, Logger, Metrics } from '@eth-optimism/common-ts' +import * as Sentry from '@sentry/node' +import * as Tracing from '@sentry/tracing' +import cors from 'cors' +import { JsonRpcProvider, toBigInt, toNumber } from 'ethersv6' import express, { Request, Response } from 'express' import promBundle from 'express-prom-bundle' -import cors from 'cors' import { LevelUp } from 'levelup' -import * as Sentry from '@sentry/node' -import * as Tracing from '@sentry/tracing' -import { toBigInt, toNumber, JsonRpcProvider, Block } from 'ethersv6' /* Imports: Internal */ import { TransportDB, TransportDBMapHolder } from '../../db/transport-db' import { + AppendBatchElementResponse, + BlockBatchResponse, + BlockResponse, ContextResponse, - GasPriceResponse, EnqueueResponse, + GasPriceResponse, + HighestResponse, StateRootBatchResponse, StateRootResponse, SyncingResponse, TransactionBatchResponse, TransactionResponse, - BlockBatchResponse, - BlockResponse, - VerifierResultResponse, VerifierResultEntry, + VerifierResultResponse, VerifierStakeResponse, - AppendBatchElementResponse, - HighestResponse, - SyncStatusResponse, - L1BlockRef, - L2BlockRef, } from '../../types' import { validators } from '../../utils' import { L1DataTransportServiceOptions } from '../main/service' @@ -101,15 +98,8 @@ export class L1TransportServer extends BaseService { this.options.db, this.options.l2ChainId === 1088 ) - this.state.l1RpcProvider = - typeof this.options.l1RpcProvider === 'string' - ? new JsonRpcProvider(this.options.l1RpcProvider) - : this.options.l1RpcProvider - - this.state.l2RpcProvider = - typeof this.options.l2RpcProvider === 'string' - ? new JsonRpcProvider(this.options.l2RpcProvider) - : this.options.l2RpcProvider + this.state.l1RpcProvider = new JsonRpcProvider(this.options.l1RpcEndpoint) + this.state.l2RpcProvider = new JsonRpcProvider(this.options.l2RpcEndpoint) this._initializeApp() } From 9bf2393733da9adb54492e649b6f46049c4635dc Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 20:01:39 +0800 Subject: [PATCH 10/18] revert sort diff --- .../data-transport-layer/test/unit-tests/da/blob.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts b/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts index cc6ac28e1d9a1..d541971e242ed 100644 --- a/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts +++ b/packages/data-transport-layer/test/unit-tests/da/blob.spec.ts @@ -1,5 +1,5 @@ -import { ethers, hexlify } from 'ethersv6' -import { Blob } from '../../../src/da/blob/blob' +import { parseFrames } from '../../../src/da/blob/frame' +import { l1BlobData } from '../examples/l1-data' import { BatchData, batchReader, @@ -7,8 +7,8 @@ import { RawSpanBatch, SpanBatchType, } from '../../../src/da/blob/channel' -import { parseFrames } from '../../../src/da/blob/frame' -import { l1BlobData } from '../examples/l1-data' +import { Blob } from '../../../src/da/blob/blob' +import { ethers, hexlify } from 'ethersv6' describe('Decode Blob Transaction', function () { this.timeout(60000) From 3f2da0f70dbfdea376b7551e23649d3443915ec8 Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 20:08:23 +0800 Subject: [PATCH 11/18] batch: improve proof validation in Blob class and remove unused import --- packages/batch-submitter/src/da/blob.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/batch-submitter/src/da/blob.ts b/packages/batch-submitter/src/da/blob.ts index ccd891d9774b7..6a5d4ccc7bf9e 100644 --- a/packages/batch-submitter/src/da/blob.ts +++ b/packages/batch-submitter/src/da/blob.ts @@ -1,6 +1,5 @@ import { blobToKzgCommitment, - Blob as CBlob, computeCellsAndKzgProofs, KZGCommitment, KZGProof, @@ -151,10 +150,14 @@ export class Blob { ) } - this.commitment.set(blobToKzgCommitment(this.data as CBlob)) - this.proof.set( - Buffer.concat(computeCellsAndKzgProofs(this.data as CBlob)[1]) - ) + this.commitment.set(blobToKzgCommitment(this.data)) + const proofs = Buffer.concat(computeCellsAndKzgProofs(this.data)[1]) + if (proofs.length !== this.proof.length) { + throw new Error( + `Invalid proof length: expected ${this.proof.length}, got ${proofs.length}` + ) + } + this.proof.set(proofs) this.versionedHash = Blob.kzgToVersionedHash(this.commitment) return this From 44828ab720f5fbe3bad45696ed43ff0a814bb234 Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 20:09:33 +0800 Subject: [PATCH 12/18] batch: update STEPS_FILE constant to use 'inbox_blob_txs.json' for consistency --- packages/batch-submitter/src/storage/inbox-storage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/batch-submitter/src/storage/inbox-storage.ts b/packages/batch-submitter/src/storage/inbox-storage.ts index 1a1edfba66d4e..d32cc5c6e60bf 100644 --- a/packages/batch-submitter/src/storage/inbox-storage.ts +++ b/packages/batch-submitter/src/storage/inbox-storage.ts @@ -7,7 +7,7 @@ import { BlobLike } from '../da/types' const INBOX_OK_FILE = 'inbox_ok.json' const INBOX_FAIL_FILE = 'inbox_fail.json' -const STEPS_FILE = 'steps.json' +const STEPS_FILE = 'inbox_blobs.json' export interface InboxRecordInfo { batchIndex: number | bigint From 2015fb8a07b81d750101f183c6db30585c34badc Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 20:20:38 +0800 Subject: [PATCH 13/18] batch: change log level from info to debug for event storage --- .../data-transport-layer/src/services/l1-ingestion/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/data-transport-layer/src/services/l1-ingestion/service.ts b/packages/data-transport-layer/src/services/l1-ingestion/service.ts index 6bc17c310569c..4e3d4b812b1f7 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/service.ts @@ -781,7 +781,7 @@ export class L1IngestionService extends BaseService { db = await this.options.dbs.getTransportDbByChainId(chainId) } - this.logger.info('Storing Event:', { + this.logger.debug('Storing Event:', { chainId, parsedEvent, }) From af0ca09910d852737d62e11a5df915290779af3d Mon Sep 17 00:00:00 2001 From: ericlee Date: Wed, 10 Dec 2025 22:39:13 +0800 Subject: [PATCH 14/18] batch: streamline logging for Layer 1 synchronization by removing redundant information --- .../src/services/l1-ingestion/service.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/data-transport-layer/src/services/l1-ingestion/service.ts b/packages/data-transport-layer/src/services/l1-ingestion/service.ts index 4e3d4b812b1f7..220c88b5c6028 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/service.ts @@ -315,12 +315,6 @@ export class L1IngestionService extends BaseService { const highestSyncedL1BatchIndex = latestBatch === null ? -1 : latestBatch.index - this.logger.info('Synchronizing events from Layer 1 (Ethereum)', { - usingL2ChainId: this.options.l2ChainId, - latestBatch, - stateLatestBatch: await this.state.db.getLatestTransactionBatch(), - }) - const inboxAddress = this.options.batchInboxAddress const inboxBatchStart = this.options.batchInboxStartIndex const inboxSender = this.options.batchInboxSender @@ -340,6 +334,7 @@ export class L1IngestionService extends BaseService { this.state.startingL1BatchIndex <= highestSyncedL1BatchIndex + 1 this.logger.info('Synchronizing events from Layer 1 (Ethereum)', { + latestBatch, highestSyncedL1Block, targetL1Block, highestSyncedL1BatchIndex, From e708b5c2869981b9828acc703f898ada629990a6 Mon Sep 17 00:00:00 2001 From: ericlee Date: Thu, 11 Dec 2025 07:14:34 +0800 Subject: [PATCH 15/18] batch: add logging for syncInboxBatch method to track block range --- .../src/services/l1-ingestion/service.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/data-transport-layer/src/services/l1-ingestion/service.ts b/packages/data-transport-layer/src/services/l1-ingestion/service.ts index 220c88b5c6028..e1dcef37d266e 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/service.ts @@ -524,6 +524,11 @@ export class L1IngestionService extends BaseService { toL1Block: number, handlers: EventHandlerSetAny ): Promise { + this.logger.info('syncInboxBatch', { + fromL1Block, + toL1Block, + }) + const blockPromises = [] for (let i = fromL1Block; i <= toL1Block; i++) { blockPromises.push(this.state.l1RpcProvider.getBlock(i, true)) @@ -531,10 +536,6 @@ export class L1IngestionService extends BaseService { // Just making sure that the blocks will come back in increasing order. const blocks = (await Promise.all(blockPromises)) as Block[] - this.logger.info('_syncInboxBatch get blocks', { - fromL1Block, - toL1Block, - }) const extraMap: Record = {} for (const block of blocks) { From 84c9398b369ed15ddf20a487a2dff14295a1da52 Mon Sep 17 00:00:00 2001 From: ericlee Date: Thu, 11 Dec 2025 07:25:24 +0800 Subject: [PATCH 16/18] batch: refactor timestamp handling in transaction-enqueued to use event data directly --- .../src/db/transport-db.ts | 20 ++++++++----------- .../handlers/transaction-enqueued.ts | 5 +---- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/packages/data-transport-layer/src/db/transport-db.ts b/packages/data-transport-layer/src/db/transport-db.ts index 8c0a2c7d07bff..9b222acfbfb68 100755 --- a/packages/data-transport-layer/src/db/transport-db.ts +++ b/packages/data-transport-layer/src/db/transport-db.ts @@ -1,25 +1,25 @@ /* Imports: External */ -import { LevelUp } from 'levelup' import level from 'level' +import { LevelUp } from 'levelup' // 1088 patch only import patch01 from './patch-01' /* Imports: Internal */ +import { toBigInt, toNumber } from 'ethersv6' import { + AppendBatchElementEntry, + BlockEntry, EnqueueEntry, + InboxSenderSetEntry, + SenderType, StateRootBatchEntry, StateRootEntry, TransactionBatchEntry, TransactionEntry, + Upgrades, VerifierResultEntry, VerifierStakeEntry, - AppendBatchElementEntry, - BlockEntry, - InboxSenderSetEntry, - SenderType, - Upgrades, } from '../types/database-types' import { SimpleDB } from './simple-db' -import { toBigInt, toNumber } from 'ethersv6' const TRANSPORT_DB_KEYS = { ENQUEUE: `enqueue`, @@ -572,11 +572,7 @@ export class TransportDB { transaction.timestamp = patch01[txBlockNumber][1] } if (transaction.queueOrigin === 'l1') { - // Andromeda failed 20397 queue, skip one for verifier batch only - let queueIndex = transaction.queueIndex - if (queueIndex >= 20397) { - queueIndex++ - } + const queueIndex = transaction.queueIndex const enqueue = await this.getEnqueueByIndex(queueIndex) if (enqueue === null) { return null diff --git a/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts b/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts index c0aa49426eb98..8275ef99bda95 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts @@ -21,10 +21,7 @@ export const handleEventsTransactionEnqueued: EventHandlerSet< gasLimit: event.args._gasLimit.toString(), origin: event.args._l1TxOrigin, blockNumber: toNumber(event.blockNumber), - timestamp: - toNumber(event.blockNumber) >= 14570938 - ? Math.floor(new Date().getTime() / 1000) - : toNumber(event.args._timestamp), + timestamp: toNumber(event.args._timestamp), ctcIndex: null, } }, From df4d3739bb1693db331a60cd183ea16c6d8e5ad9 Mon Sep 17 00:00:00 2001 From: ericlee Date: Thu, 11 Dec 2025 08:09:06 +0800 Subject: [PATCH 17/18] batch: improve error handling in proposeMpcSign method by simplifying error logging --- packages/batch-submitter/src/utils/mpc-client.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/batch-submitter/src/utils/mpc-client.ts b/packages/batch-submitter/src/utils/mpc-client.ts index 50d9c16194a57..af16f0f19eb73 100644 --- a/packages/batch-submitter/src/utils/mpc-client.ts +++ b/packages/batch-submitter/src/utils/mpc-client.ts @@ -214,12 +214,11 @@ export class MpcClient { } const signResp = await this.proposeMpcSign(postData) if (!signResp) { - this.logger.error('mpc propose sign failed', { - mpcId, - signId, - signResp, - }) - throw new Error(`MPC ${mpcId} propose sign failed`) + throw new Error( + `MPC ${mpcId} propose sign ${signId} failed: ${JSON.stringify( + signResp + )}` + ) } const signedTx = await this.getMpcSignWithTimeout( From ea6ba75d11f34907a6fa01b2d74a684a508285f4 Mon Sep 17 00:00:00 2001 From: ericlee Date: Thu, 11 Dec 2025 08:15:55 +0800 Subject: [PATCH 18/18] batch: enhance error handling in fetchBatches for blob transactions --- .../data-transport-layer/src/da/blob/index.ts | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/data-transport-layer/src/da/blob/index.ts b/packages/data-transport-layer/src/da/blob/index.ts index cf630278fa768..aa33fb1d0509d 100644 --- a/packages/data-transport-layer/src/da/blob/index.ts +++ b/packages/data-transport-layer/src/da/blob/index.ts @@ -66,6 +66,17 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { if (!tx || tx.hash !== blobTxHash) { throw new Error(`Transaction ${blobTxHash} not found in block`) } + if (tx.type !== BlobTxType) { + // We are not processing old transactions those are using call data, + // this should not happen. + throw new Error( + `Found inbox transaction ${tx.hash} that is not using blob, ignore` + ) + } + // no blob in this blob tx + if (!tx.blobVersionedHashes || tx.blobVersionedHashes.length === 0) { + throw new Error(`No blobVersionedHashes found in transaction ${tx.hash}`) + } // only process the blob tx hash recorded in the commitment const sender = tx.from @@ -74,20 +85,7 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { } const frames: Frame[] = [] - if (tx.type !== BlobTxType) { - // We are not processing old transactions those are using call data, - // this should not happen. - throw new Error( - `Found inbox transaction ${tx.hash} that is not using blob, ignore` - ) - } else { - if (!tx.blobVersionedHashes || tx.blobVersionedHashes.length === 0) { - // no blob in this blob tx - throw new Error( - `No blobVersionedHashes found in transaction ${tx.hash}` - ) - } - + { // fetch blob data from beacon chain const blobs = await l1BeaconProvider.getBlobs( block.timestamp,