diff --git a/package.json b/package.json index 80180cab0397c..1c568e79290e9 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "release": "yarn build && yarn changeset publish" }, "dependencies": { - "ethersv6": "github:ericlee42/ethers#metis" + "ethersv6": "npm:ethers@^6.16.0" }, "resolutions": { "node-gyp": "^9.4.0" diff --git a/packages/batch-submitter/package.json b/packages/batch-submitter/package.json index 69e44a79bca4e..f317b582a3b1e 100755 --- a/packages/batch-submitter/package.json +++ b/packages/batch-submitter/package.json @@ -44,7 +44,7 @@ "bluebird": "^3.7.2", "c-kzg": "^4.0.1", "dotenv": "^10.0.0", - "ethersv6": "github:ericlee42/ethers#metis", + "ethersv6": "npm:ethers@^6.16.0", "lodash": "^4.17.21", "old-contracts": "npm:@eth-optimism/contracts@^0.0.2-alpha.7", "prom-client": "^13.1.0", diff --git a/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts b/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts index 96b6f486b0140..55ca6eb26e55f 100755 --- a/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts +++ b/packages/batch-submitter/src/batch-submitter/state-batch-submitter.ts @@ -323,7 +323,7 @@ export class StateBatchSubmitter extends BatchSubmitter { if (!mpcInfo || !mpcInfo.mpc_address) { throw new Error('MPC 1 info get failed') } - const txUnsign: ethers.TransactionRequest = { + const txUnsign: ethers.TransactionLike = { type: 2, to: tx.to, data: tx.data, diff --git a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts index 187c758c413f8..a0fd13a08e974 100644 --- a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts +++ b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter-inbox.ts @@ -12,7 +12,6 @@ import { zlibCompressHexString, } from '@metis.io/core-utils' import { Promise as bPromise } from 'bluebird' -import * as kzg from 'c-kzg' import { ethers, Provider, @@ -20,7 +19,6 @@ import { toBeHex, toBigInt, TransactionReceipt, - TransactionRequest, } from 'ethersv6' /* Internal Imports */ @@ -146,7 +144,6 @@ export class TransactionBatchSubmitterInbox { } metrics.numTxPerBatch.observe(endBlock - startBlock) this.logger.info('Submitting batch to inbox', { - meta: batchParams.inputMeta, useBlob, blobTxes: batchParams.blobTxData.length, batchSizeInBytes, @@ -157,7 +154,7 @@ export class TransactionBatchSubmitterInbox { nextBatchIndex, { input: batchParams.inputData, - blobs: batchParams.blobTxData.map((tx) => tx.blobs.map((b) => b.data)), + blobs: batchParams.blobTxData.map((tx) => tx.blobs), txHashes: [], }, signer, @@ -197,7 +194,7 @@ export class TransactionBatchSubmitterInbox { ) => Promise ): Promise { const { chainId } = await signer.provider.getNetwork() - const inboxTx: TransactionRequest = { + const inboxTx: ethers.TransactionLike = { type: 2, chainId, to: this.inboxAddress, @@ -244,7 +241,7 @@ export class TransactionBatchSubmitterInbox { continue } - const blobTx: ethers.TransactionRequest = { + const blobTx: ethers.TransactionLike = { type: 3, // 3 for blob tx type to: this.inboxAddress, // since we are using blob tx, call data will be empty, @@ -253,7 +250,7 @@ export class TransactionBatchSubmitterInbox { chainId, nonce: await signer.provider.getTransactionCount(signerAddress), blobs, - blobVersion: 1, // Osaka is enabled on all the chains + blobWrapperVersion: 1, // Osaka is enabled on all the chains } const replaced = await setTxEIP1559Fees( @@ -285,9 +282,8 @@ export class TransactionBatchSubmitterInbox { // need to append the blob sidecar to the signed tx const signedTxUnmarshaled = ethers.Transaction.from(signedTx) - signedTxUnmarshaled.type = 3 - signedTxUnmarshaled.kzg = kzg - signedTxUnmarshaled.blobVersion = blobTx.blobVersion + signedTxUnmarshaled.blobWrapperVersion = + blobTx.blobWrapperVersion signedTxUnmarshaled.blobs = blobTx.blobs // repack the tx return signedTxUnmarshaled.serialized @@ -661,13 +657,6 @@ export class TransactionBatchSubmitterInbox { encoded = `${da}${compressType}${batchIndex}${l2Start}${totalElements}${compressedEncoded}` return { - inputMeta: { - da, - compressType, - batchIndex, - l2Start, - totalElements, - }, inputData: encoded, batch: blocks, blobTxData, diff --git a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts index 2eb8a32e3e3e8..578c697ec8065 100755 --- a/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts +++ b/packages/batch-submitter/src/batch-submitter/tx-batch-submitter.ts @@ -581,7 +581,7 @@ export class TransactionBatchSubmitter extends BatchSubmitter { // this.encodeSequencerBatchOptions // ) // unsigned tx - const tx: ethers.TransactionRequest = { + const tx: ethers.TransactionLike = { to: this.useMinio ? await this.mvmCtcContract.getAddress() : await this.chainContract.getAddress(), diff --git a/packages/batch-submitter/src/da/blob.ts b/packages/batch-submitter/src/da/blob.ts index c8d46fde18530..6a5d4ccc7bf9e 100644 --- a/packages/batch-submitter/src/da/blob.ts +++ b/packages/batch-submitter/src/da/blob.ts @@ -1,9 +1,8 @@ import { blobToKzgCommitment, - Bytes48, - Blob as CBlob, - computeBlobKzgProof, - verifyBlobKzgProof, + computeCellsAndKzgProofs, + KZGCommitment, + KZGProof, } from 'c-kzg' import { createHash } from 'crypto' import { Frame } from './types' @@ -11,30 +10,23 @@ import { Frame } from './types' const BlobSize = 4096 * 32 const MaxBlobDataSize = (4 * 31 + 3) * 1024 - 4 const EncodingVersion = 0 -const VersionOffset = 1 +// const VersionOffset = 1 const Rounds = 1024 export class Blob { public readonly data: Uint8Array = new Uint8Array(BlobSize) - public readonly commitment: Bytes48 = new Uint8Array(48) - public readonly proof: Bytes48 = new Uint8Array(48) + public readonly commitment: KZGCommitment = new Uint8Array(48) + // cell proofs + public readonly proof: KZGProof = new Uint8Array(48 * 128) public versionedHash: string = '' - static kzgToVersionedHash(commitment: Bytes48): string { + static kzgToVersionedHash(commitment: KZGCommitment): string { const hasher = createHash('sha256') hasher.update(commitment) // versioned hash = [1 byte version][31 byte hash] return '0x01' + hasher.digest('hex').substring(2) } - static verifyBlobProof( - blob: Blob, - commitment: Bytes48, - proof: Bytes48 - ): boolean { - return verifyBlobKzgProof(blob.data as CBlob, commitment, proof) - } - marshalFrame(frame: Frame): Uint8Array { const buffer = new ArrayBuffer(16 + 2 + 4 + frame.data.length + 1) const view = new DataView(buffer) @@ -158,8 +150,14 @@ export class Blob { ) } - this.commitment.set(blobToKzgCommitment(this.data as CBlob)) - this.proof.set(computeBlobKzgProof(this.data as CBlob, this.commitment)) + this.commitment.set(blobToKzgCommitment(this.data)) + const proofs = Buffer.concat(computeCellsAndKzgProofs(this.data)[1]) + if (proofs.length !== this.proof.length) { + throw new Error( + `Invalid proof length: expected ${this.proof.length}, got ${proofs.length}` + ) + } + this.proof.set(proofs) this.versionedHash = Blob.kzgToVersionedHash(this.commitment) return this diff --git a/packages/batch-submitter/src/da/channel.ts b/packages/batch-submitter/src/da/channel.ts index a15de09b7d90e..b0add8f002371 100644 --- a/packages/batch-submitter/src/da/channel.ts +++ b/packages/batch-submitter/src/da/channel.ts @@ -1,15 +1,16 @@ // channel.ts +import { Logger } from '@eth-optimism/common-ts' import { ethers } from 'ethersv6' +import { Blob } from './blob' import { ChannelBuilder } from './channel-builder' import { BatchToInboxElement, + BlobLike, ChannelConfig, Frame, RollupConfig, TxData, } from './types' -import { Blob } from './blob' -import { Logger } from '@eth-optimism/common-ts' export class Channel { private channelBuilder: ChannelBuilder @@ -82,8 +83,15 @@ export class Channel { return sb }, - get blobs(): Blob[] { - return this.frames.map((f: Frame) => new Blob().fromFrame(f)) + get blobs(): BlobLike[] { + return this.frames.map((f: Frame) => { + const blob = new Blob().fromFrame(f) + return { + data: blob.data, + proof: blob.proof, + commitment: blob.commitment, + } + }) }, } this.pendingTransactions.set(txData.id, txData) diff --git a/packages/batch-submitter/src/da/consts.ts b/packages/batch-submitter/src/da/consts.ts index 4ba41c2dbf863..449604c01ac2c 100644 --- a/packages/batch-submitter/src/da/consts.ts +++ b/packages/batch-submitter/src/da/consts.ts @@ -1,6 +1,6 @@ export const FRAME_OVERHEAD_SIZE = 200 export const MAX_RLP_BYTES_PER_CHANNEL = 100_000_000 export const MAX_BLOB_SIZE = (4 * 31 + 3) * 1024 - 4 -export const MAX_BLOB_NUM_PER_TX = 7 +export const MAX_BLOB_NUM_PER_TX = 6 export const TX_GAS = 21_000 export const CHANNEL_FULL_ERR = new Error('Channel is full') diff --git a/packages/batch-submitter/src/da/types.ts b/packages/batch-submitter/src/da/types.ts index ecd0460dd4c8b..5bacaca42edf4 100644 --- a/packages/batch-submitter/src/da/types.ts +++ b/packages/batch-submitter/src/da/types.ts @@ -1,6 +1,5 @@ // types.ts import { BytesLike, ethers, getBytes } from 'ethersv6' -import { Blob } from './blob' export interface RollupConfig { l1ChainID: bigint @@ -49,12 +48,18 @@ export interface Frame { isLast: boolean } +export type BlobLike = { + data: BytesLike + proof: BytesLike + commitment: BytesLike +} + export interface TxData { frames: Frame[] asBlob: boolean get id(): string - get blobs(): Blob[] + get blobs(): BlobLike[] } export interface L1BlockInfo { @@ -153,18 +158,10 @@ export interface BatchToInboxElement { extraData: string txs: BatchToInboxRawTx[] } -export declare type BatchToInbox = BatchToInboxElement[] -export interface InboxInputMeta { - da: string - compressType: string - batchIndex: string - l2Start: string - totalElements: string -} +export declare type BatchToInbox = BatchToInboxElement[] export interface InboxBatchParams { - inputMeta: InboxInputMeta inputData: string batch: BatchToInbox blobTxData: TxData[] diff --git a/packages/batch-submitter/src/storage/inbox-storage.ts b/packages/batch-submitter/src/storage/inbox-storage.ts index cc8e0e4915dff..d32cc5c6e60bf 100644 --- a/packages/batch-submitter/src/storage/inbox-storage.ts +++ b/packages/batch-submitter/src/storage/inbox-storage.ts @@ -1,12 +1,13 @@ /* Imports: External */ import { Logger } from '@eth-optimism/common-ts' -import { BytesLike, toNumber } from 'ethersv6' +import { toNumber } from 'ethersv6' import * as fs from 'fs/promises' import * as path from 'path' +import { BlobLike } from '../da/types' const INBOX_OK_FILE = 'inbox_ok.json' const INBOX_FAIL_FILE = 'inbox_fail.json' -const STEPS_FILE = 'steps.json' +const STEPS_FILE = 'inbox_blobs.json' export interface InboxRecordInfo { batchIndex: number | bigint @@ -16,7 +17,7 @@ export interface InboxRecordInfo { export interface InboxSteps { input: string // the inbox tx data input - blobs: Array> // array of blob tx data + blobs: Array> // array of blob tx data txHashes: Array // blob tx hashes + inbox tx hash } @@ -75,15 +76,25 @@ export class InboxStorage { } public async insertStep(jsonData: InboxSteps) { - const data = { + const data: InboxSteps = { input: jsonData.input, txHashes: jsonData.txHashes, blobs: jsonData.blobs.map((blobArray) => blobArray.map((blob) => { - if (typeof blob === 'string') { - return blob + return { + data: + typeof blob.data === 'string' + ? blob.data + : '0x' + Buffer.from(blob.data).toString('hex'), + proof: + typeof blob.proof === 'string' + ? blob.proof + : '0x' + Buffer.from(blob.proof).toString('hex'), + commitment: + typeof blob.commitment === 'string' + ? blob.commitment + : '0x' + Buffer.from(blob.commitment).toString('hex'), } - return '0x' + Buffer.from(blob).toString('hex') }) ), } @@ -97,8 +108,15 @@ export class InboxStorage { if (!(await this.fileExists(filePath))) { return null } - const data = await fs.readFile(filePath, 'utf-8') - return JSON.parse(data) + const raw = await fs.readFile(filePath, 'utf-8') + const parsed: InboxSteps = JSON.parse(raw) + if (!Array.isArray(parsed.blobs)) { + throw new Error('Invalid steps file format: blobs is not an array') + } + if (!Array.isArray(parsed.txHashes)) { + throw new Error('Invalid steps file format: txHashes is not an array') + } + return parsed } private async fileExists(filePath) { diff --git a/packages/batch-submitter/src/utils/mpc-client.ts b/packages/batch-submitter/src/utils/mpc-client.ts index 5456d3986cb3e..af16f0f19eb73 100644 --- a/packages/batch-submitter/src/utils/mpc-client.ts +++ b/packages/batch-submitter/src/utils/mpc-client.ts @@ -1,8 +1,6 @@ import { Logger } from '@eth-optimism/common-ts' -import '@metis.io/core-utils' -import * as kzg from 'c-kzg' import { randomUUID } from 'crypto' -import { ethers, toBigInt, toNumber, TransactionLike } from 'ethersv6' +import { ethers } from 'ethersv6' import * as http from 'http' import * as https from 'https' import { URL } from 'url' @@ -168,94 +166,59 @@ export class MpcClient { // call this public async signTx( - tx: any, - mpcId: any, + unsigned: ethers.TransactionLike, + mpcId: string, timeoutMilli: number ): Promise { // check tx - if (!tx.gasLimit) { + if (!unsigned.gasLimit) { throw new Error('tx gasLimit is required') } - if (tx.nonce === undefined || tx.nonce === null) { + if (unsigned.nonce === undefined || unsigned.nonce === null) { throw new Error('tx nonce is required') } - - this.logger.info('signing tx with mpc', { - mpcId, - timeout: timeoutMilli / 1e3 + 's', - }) - - // call mpc to sign tx - const unsignedTx: TransactionLike = { - data: tx.data, - nonce: toNumber(tx.nonce), - to: tx.to, - value: tx.value ? toBigInt(tx.value) : toBigInt(0), - gasLimit: toBigInt(tx.gasLimit), - chainId: tx.chainId, - } - if (!tx.type) { - // populate legacy tx - if (!tx.gasPrice) { - throw new Error('gasPrice is required for legacy tx') + if (unsigned.type > 1) { + // check for post-EIP1559 tx + if (!unsigned.maxFeePerGas) { + throw new Error('maxFeePerGas is required for post-EIP1559 tx') } - unsignedTx.gasPrice = toBigInt(tx.gasPrice) - } else { - unsignedTx.accessList = tx.accessList + if (!unsigned.maxPriorityFeePerGas) { + throw new Error('maxPriorityFeePerGas is required for post-EIP1559 tx') + } - // populate typed tx - const txType = toNumber(tx.type) - unsignedTx.type = txType - if (txType === 1) { - // check for access list tx - if (!tx.gasPrice) { - throw new Error('gasPrice is required for access list tx') - } - unsignedTx.gasPrice = toBigInt(tx.gasPrice) - } else if (txType > 1) { - // check for post-EIP1559 tx - if (!tx.maxFeePerGas) { - throw new Error('maxFeePerGas is required for post-EIP1559 tx') - } - if (!tx.maxPriorityFeePerGas) { - throw new Error( - 'maxPriorityFeePerGas is required for post-EIP1559 tx' - ) + if (unsigned.type === 3) { + // extra checks for blob tx + if (!unsigned.maxFeePerBlobGas) { + throw new Error('maxFeePerBlobGas is required for blob tx') } - unsignedTx.maxFeePerGas = toBigInt(tx.maxFeePerGas) - unsignedTx.maxPriorityFeePerGas = toBigInt(tx.maxPriorityFeePerGas) - - if (txType === 3) { - // extra checks for blob tx - if (!tx.maxFeePerBlobGas) { - throw new Error('maxFeePerBlobGas is required for blob tx') - } - - if (!tx.blobs) { - throw new Error('blobs are required for blob tx') - } - - unsignedTx.maxFeePerBlobGas = toBigInt(tx.maxFeePerBlobGas) - unsignedTx.kzg = kzg - unsignedTx.blobVersion = tx.blobVersion - unsignedTx.blobs = tx.blobs + if (!unsigned.blobs || !unsigned.blobs.length) { + throw new Error('blobs are required for blob tx') } } } + this.logger.info('signing tx with mpc', { + mpcId, + timeout: timeoutMilli / 1e3 + 's', + }) + const signId = randomUUID() const postData = { sign_id: signId, mpc_id: mpcId, sign_type: 0, - sign_data: ethers.Transaction.from(unsignedTx).unsignedSerialized, + sign_data: ethers.Transaction.from(unsigned).unsignedSerialized, sign_msg: '', } const signResp = await this.proposeMpcSign(postData) if (!signResp) { - throw new Error(`MPC ${mpcId} propose sign failed`) + throw new Error( + `MPC ${mpcId} propose sign ${signId} failed: ${JSON.stringify( + signResp + )}` + ) } const signedTx = await this.getMpcSignWithTimeout( diff --git a/packages/core-utils/package.json b/packages/core-utils/package.json index 220f1cfbb3d5e..ea2c4295d7428 100755 --- a/packages/core-utils/package.json +++ b/packages/core-utils/package.json @@ -50,7 +50,7 @@ "dependencies": { "@metis.io/minio": "^7.0.28", "chai": "^4.3.4", - "ethersv6": "github:ericlee42/ethers#metis", + "ethersv6": "npm:ethers@^6.16.0", "lodash": "^4.17.21", "merkletreejs": "^0.2.32" } diff --git a/packages/data-transport-layer/package.json b/packages/data-transport-layer/package.json index 1dc728f92b094..88b08960397ab 100755 --- a/packages/data-transport-layer/package.json +++ b/packages/data-transport-layer/package.json @@ -33,14 +33,14 @@ "@sentry/node": "^6.3.1", "@sentry/tracing": "^6.3.1", "@types/express": "^4.17.12", - "axios": "^0.21.1", + "axios": "^1.13.2", "bcfg": "^0.1.6", "bfj": "^7.0.2", "browser-or-node": "^1.3.0", "c-kzg": "^4.0.1", "cors": "^2.8.5", "dotenv": "^10.0.0", - "ethersv6": "github:ericlee42/ethers#metis", + "ethersv6": "npm:ethers@^6.16.0", "express": "^4.17.1", "express-prom-bundle": "^6.3.6", "level": "^6.0.1", @@ -48,6 +48,7 @@ "merkletreejs": "^0.3.10", "node-fetch": "^2.6.1", "pako": "^2.1.0", + "qs": "^6.14.0", "rlp": "^3.0.0" }, "devDependencies": { @@ -58,6 +59,7 @@ "@types/levelup": "^4.3.0", "@types/mocha": "^8.2.2", "@types/node-fetch": "^2.5.10", + "@types/qs": "^6.14.0", "@types/workerpool": "^6.0.0", "@typescript-eslint/eslint-plugin": "^4.26.0", "@typescript-eslint/parser": "^4.26.0", diff --git a/packages/data-transport-layer/src/da/blob/blob.ts b/packages/data-transport-layer/src/da/blob/blob.ts index f7c76e33ee321..59163af154253 100644 --- a/packages/data-transport-layer/src/da/blob/blob.ts +++ b/packages/data-transport-layer/src/da/blob/blob.ts @@ -1,34 +1,40 @@ +import { blobToKzgCommitment } from 'c-kzg' +import { createHash } from 'crypto' + const BlobSize = 4096 * 32 const MaxBlobDataSize = (4 * 31 + 3) * 1024 - 4 const EncodingVersion = 0 const VersionOffset = 1 const Rounds = 1024 -const hexStringToUint8Array = (hexString: string): Uint8Array => { - hexString = hexString.replace(/^0x/, '').replace(/\s/g, '') - - if (hexString.length % 2 !== 0) { - throw new Error('Invalid hex string') - } +export class Blob { + private readonly data: Uint8Array - const arrayBuffer = new Uint8Array(hexString.length / 2) + constructor(hexString: string) { + if (hexString.startsWith('0x') || hexString.startsWith('0X')) { + hexString = hexString.slice(2) + } - for (let i = 0; i < hexString.length; i += 2) { - const byteValue = parseInt(hexString.substring(i, i + 2), 16) - if (isNaN(byteValue)) { + if (hexString.length % 2 !== 0) { throw new Error('Invalid hex string') } - arrayBuffer[i / 2] = byteValue - } - return arrayBuffer -} + const buffer = new Uint8Array(hexString.length / 2) + for (let i = 0; i < hexString.length; i += 2) { + const byteValue = parseInt(hexString.substring(i, i + 2), 16) + if (isNaN(byteValue)) { + throw new Error('Invalid hex string') + } + buffer[i / 2] = byteValue + } -export class Blob { - private readonly data: Uint8Array + if (buffer.length !== BlobSize) { + throw new Error( + `Invalid blob size: expected ${BlobSize}, got ${buffer.length}` + ) + } - constructor(hex: string) { - this.data = hexStringToUint8Array(hex) + this.data = buffer } toString(): string { @@ -100,6 +106,12 @@ export class Blob { this.data.fill(0) } + versionedHash(): string { + const hasher = createHash('sha256') + hasher.update(blobToKzgCommitment(this.data)) + return '0x01' + hasher.digest('hex').substring(2) + } + private decodeFieldElement( opos: number, ipos: number, diff --git a/packages/data-transport-layer/src/da/blob/index.ts b/packages/data-transport-layer/src/da/blob/index.ts index c67363fc8ff8e..aa33fb1d0509d 100644 --- a/packages/data-transport-layer/src/da/blob/index.ts +++ b/packages/data-transport-layer/src/da/blob/index.ts @@ -21,17 +21,15 @@ interface FetchBatchesConfig { concurrentRequests: number // concurrent requests number l2ChainId: number // l2 chain id - l1Rpc: string // l1 rpc url - l1Beacon: string // l1 beacon chain url + l1RpcProvider: ethers.JsonRpcProvider + l1BeaconProvider: L1BeaconClient } let chainIdHasChecked = false // whether chain id has been checked // fetch l2 batches from l1 chain export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { - const l1RpcProvider = new ethers.JsonRpcProvider(fetchConf.l1Rpc) - const l1BeaconProvider = new L1BeaconClient(fetchConf.l1Beacon) - + const { l1RpcProvider, l1BeaconProvider } = fetchConf if (!chainIdHasChecked) { const checkId = await l1BeaconProvider.getChainId() if (Number(checkId) !== fetchConf.chainId) { @@ -59,108 +57,78 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { throw new BlobDataExpiredError(blobTxHash) } - // TODO: We might be able to cache this somewhere, no need to retrieve this every time. - // But due to potential chain reorgs, just retrieve the data everytime for now. - // Might need to think of a better solution in the future. - const block = await l1RpcProvider.getBlock(receipt.blockNumber, true) + const block = await l1RpcProvider.getBlock(receipt.blockHash, true) if (!block) { throw new Error(`Block ${receipt.blockNumber} not found`) } - const txs = block.prefetchedTransactions - - // Even we got the hash of the blob tx, we still need to traverse through the blocks - // since we need to count the blob index in the block - let blobIndex = 0 - for (const tx of txs) { - if (!tx) { - continue - } - - // only process the blob tx hash recorded in the commitment - if (blobTxHash.toLowerCase() === tx.hash.toLowerCase()) { - const sender = tx.from - if (!fetchConf.batchSenders.includes(sender.toLowerCase())) { - continue - } - - const datas: Uint8Array[] = [] - if (tx.type !== BlobTxType) { - // We are not processing old transactions those are using call data, - // this should not happen. - throw new Error( - `Found inbox transaction ${tx.hash} that is not using blob, ignore` - ) - } else { - if (!tx.blobVersionedHashes) { - // no blob in this blob tx, ignore - continue - } - - // get blob hashes and indices - const hashes = tx.blobVersionedHashes.map((hash, index) => ({ - index: blobIndex + index, - hash, - })) - blobIndex += hashes.length - - // fetch blob data from beacon chain - const blobs = await l1BeaconProvider.getBlobs( - block.timestamp, - hashes.map((h) => h.index) - ) - - for (const blob of blobs) { - datas.push(blob.data) - } - } - - let frames: Frame[] = [] - for (const data of datas) { - try { - // parse the frames from the blob data - const parsedFrames = parseFrames(data, block.number) - frames = frames.concat(parsedFrames) - } catch (err) { - // invalid frame data in the blob, stop and throw error - throw new Error(`Failed to parse frames: ${err}`) - } - } + const tx = block.prefetchedTransactions[receipt.index] + if (!tx || tx.hash !== blobTxHash) { + throw new Error(`Transaction ${blobTxHash} not found in block`) + } + if (tx.type !== BlobTxType) { + // We are not processing old transactions those are using call data, + // this should not happen. + throw new Error( + `Found inbox transaction ${tx.hash} that is not using blob, ignore` + ) + } + // no blob in this blob tx + if (!tx.blobVersionedHashes || tx.blobVersionedHashes.length === 0) { + throw new Error(`No blobVersionedHashes found in transaction ${tx.hash}`) + } - const txMetadata = { - txIndex: tx.index, - inboxAddr: tx.to, - blockNumber: block.number, - blockHash: block.hash, - blockTime: block.timestamp, - chainId: fetchConf.chainId, - sender, - validSender: true, - tx, - frames: frames.map((frame) => ({ - id: Buffer.from(frame.id).toString('hex'), - data: frame.data, - isLast: frame.isLast, - frameNumber: frame.frameNumber, - inclusionBlock: frame.inclusionBlock, - })), - } + // only process the blob tx hash recorded in the commitment + const sender = tx.from + if (!fetchConf.batchSenders.includes(sender.toLowerCase())) { + continue + } - txsMetadata.push(txMetadata) - } else { - blobIndex += tx.blobVersionedHashes?.length || 0 + const frames: Frame[] = [] + { + // fetch blob data from beacon chain + const blobs = await l1BeaconProvider.getBlobs( + block.timestamp, + tx.blobVersionedHashes + ) + if (blobs.length !== tx.blobVersionedHashes.length) { + throw new Error( + `Blob count mismatch in tx ${tx.hash}: expected ${tx.blobVersionedHashes.length}, got ${blobs.length}` + ) + } + for (const blob of blobs) { + frames.push(...parseFrames(blob, receipt.blockNumber)) } } + + const txMetadata = { + txIndex: tx.index, + inboxAddr: tx.to, + blockNumber: receipt.blockNumber, + blockHash: receipt.blockHash, + blockTime: block.timestamp, + chainId: fetchConf.chainId, + sender, + validSender: true, + tx, + frames: frames.map((frame) => ({ + id: Buffer.from(frame.id).toString('hex'), + data: frame.data, + isLast: frame.isLast, + frameNumber: frame.frameNumber, + inclusionBlock: frame.inclusionBlock, + })), + } + + txsMetadata.push(txMetadata) } const channelMap: { [channelId: string]: Channel } = {} - const frameDataValidity: { [channelId: string]: boolean } = {} // process downloaded tx metadata for (const txMetadata of txsMetadata) { const framesData = txMetadata.frames - const invalidFrames = false for (const frameData of framesData) { const frame: Frame = { id: Buffer.from(frameData.id, 'hex'), @@ -176,17 +144,9 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { } // add frames to channel - try { - channelMap[channelId].addFrame(frame) - frameDataValidity[channelId] = true - } catch (e) { - frameDataValidity[channelId] = false - } + channelMap[channelId].addFrame(frame) } - if (invalidFrames) { - continue - } for (const channelId in channelMap) { if (!channelMap.hasOwnProperty(channelId)) { // ignore object prototype properties @@ -208,21 +168,6 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { } ) - // short circuit if frame data is invalid - if (!frameDataValidity[channelId]) { - channelsMetadata.push({ - id: channelId, - isReady: channel.isReady(), - invalidFrames: true, - invalidBatches: false, - frames: framesMetadata, - batches: [], - batchTypes: [], - comprAlgos: [], - }) - continue - } - if (!channel || !channel.isReady()) { continue } @@ -233,37 +178,28 @@ export const fetchBatches = async (fetchConf: FetchBatchesConfig) => { const batches = [] const batchTypes = [] const comprAlgos = [] - let invalidBatches = false - try { - // By default, this is after fjord, since we are directly upgrade to fjord, - // so no need to keep compatibility for old op versions - const readBatch = await batchReader(reader) - let batchData: BatchData | null - while ((batchData = await readBatch())) { - if (batchData.batchType === SpanBatchType) { - const spanBatch = batchData.inner as RawSpanBatch - batchData.inner = await spanBatch.derive( - ethers.toBigInt(fetchConf.l2ChainId) - ) - } - batches.push(batchData.inner) - batchTypes.push(batchData.batchType) - if (batchData.comprAlgo) { - comprAlgos.push(batchData.comprAlgo) - } + // By default, this is after fjord, since we are directly upgrade to fjord, + // so no need to keep compatibility for old op versions + const readBatch = await batchReader(reader) + let batchData: BatchData | null + while ((batchData = await readBatch())) { + if (batchData.batchType === SpanBatchType) { + const spanBatch = batchData.inner as RawSpanBatch + batchData.inner = await spanBatch.derive( + ethers.toBigInt(fetchConf.l2ChainId) + ) + } + batches.push(batchData.inner) + batchTypes.push(batchData.batchType) + if (batchData.comprAlgo) { + comprAlgos.push(batchData.comprAlgo) } - } catch (err) { - // mark batches as invalid - console.log(`Failed to read batches: ${err}`) - invalidBatches = true } const channelMetadata = { id: channelId, isReady: channel.isReady(), - invalidFrames: false, - invalidBatches, frames: framesMetadata, batches, batchTypes, diff --git a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts index c8250aa3771de..afb84b71947d1 100644 --- a/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts +++ b/packages/data-transport-layer/src/da/blob/l1-beacon-client.ts @@ -1,4 +1,5 @@ -import axios, { AxiosInstance } from 'axios' +import axios, { type AxiosInstance } from 'axios' +import qs from 'qs' import { Blob } from './blob' export class L1BeaconClient { @@ -7,7 +8,7 @@ export class L1BeaconClient { public readonly beaconChainGenesisPromise: Promise public readonly beaconChainConfigPromise: Promise - constructor(endpoint: string) { + constructor(endpoint: string, timeoutMs: number = 30000) { const parsed = new URL(endpoint) // extract baseURL (origin + pathname, trim trailing slash) @@ -20,6 +21,7 @@ export class L1BeaconClient { const defaultParams = Object.fromEntries(parsed.searchParams) this.http = axios.create({ baseURL, + timeout: timeoutMs, params: defaultParams, }) @@ -33,27 +35,46 @@ export class L1BeaconClient { } // retrieve blobs from the beacon chain - async getBlobs(timestamp: number, indices: number[]): Promise { + async getBlobs(timestamp: number, indices: string[]): Promise { // calculate the beacon chain slot from the given timestamp const slot = (await this.getTimeToSlotFn())(timestamp) - const sidecars = await this.getBlobSidecars(slot, indices) - const blobs = sidecars.map((sidecar: any) => { - const blob = new Blob(sidecar.blob) - return { - data: blob.toData(), - kzgCommitment: sidecar.kzg_commitment, - kzgProof: sidecar.kzg_proof, + const data = await this.getBlobsByVerHashs(slot, indices) + const blobs = data.map((b) => new Blob(b)) + // verify that the retrieved blobs match the requested versioned hashes + for (const [index, blob] of blobs.entries()) { + const versionedHash = blob.versionedHash() + const expectedIndex = indices[index]!.toLowerCase() + if (versionedHash !== expectedIndex) { + throw new Error( + `Blob at index ${index} has invalid versioned hash. Expected ${expectedIndex}, got ${versionedHash}` + ) } - }) - return blobs + } + return blobs.map((b) => b.toData()) } // retrieve blob sidecars from the beacon chain - async getBlobSidecars(slot: number, indices: number[]): Promise { - const response = await this.request(`eth/v1/beacon/blob_sidecars/${slot}`, { - indices: indices.join(','), + async getBlobsByVerHashs( + slot: number, + versioned_hashes: string[] + ): Promise { + const response = await this.request(`eth/v1/beacon/blobs/${slot}`, { + versioned_hashes, }) - return response.data + const res = response.data + if (!Array.isArray(res)) { + throw new Error( + `Invalid response for blobs at slot ${slot} with versioned_hashes ${versioned_hashes}: ${JSON.stringify( + res + )}` + ) + } + if (res.length !== versioned_hashes.length) { + throw new Error( + `Blob count mismatch: expected ${versioned_hashes.length}, got ${res.length}` + ) + } + return res } // calculate the slot number from a given timestamp @@ -76,9 +97,9 @@ export class L1BeaconClient { } } - async getChainId(): Promise { + async getChainId(): Promise { const response = await this.beaconChainConfigPromise - return response.data.DEPOSIT_NETWORK_ID + return Number(response.data.DEPOSIT_NETWORK_ID) } private async request( @@ -92,6 +113,7 @@ export class L1BeaconClient { method: 'GET', params: params ?? undefined, validateStatus: () => true, // handle status manually below + paramsSerializer: (p) => qs.stringify(p, { arrayFormat: 'repeat' }), }) // accept any 2xx as success diff --git a/packages/data-transport-layer/src/db/transport-db.ts b/packages/data-transport-layer/src/db/transport-db.ts index 8c0a2c7d07bff..9b222acfbfb68 100755 --- a/packages/data-transport-layer/src/db/transport-db.ts +++ b/packages/data-transport-layer/src/db/transport-db.ts @@ -1,25 +1,25 @@ /* Imports: External */ -import { LevelUp } from 'levelup' import level from 'level' +import { LevelUp } from 'levelup' // 1088 patch only import patch01 from './patch-01' /* Imports: Internal */ +import { toBigInt, toNumber } from 'ethersv6' import { + AppendBatchElementEntry, + BlockEntry, EnqueueEntry, + InboxSenderSetEntry, + SenderType, StateRootBatchEntry, StateRootEntry, TransactionBatchEntry, TransactionEntry, + Upgrades, VerifierResultEntry, VerifierStakeEntry, - AppendBatchElementEntry, - BlockEntry, - InboxSenderSetEntry, - SenderType, - Upgrades, } from '../types/database-types' import { SimpleDB } from './simple-db' -import { toBigInt, toNumber } from 'ethersv6' const TRANSPORT_DB_KEYS = { ENQUEUE: `enqueue`, @@ -572,11 +572,7 @@ export class TransportDB { transaction.timestamp = patch01[txBlockNumber][1] } if (transaction.queueOrigin === 'l1') { - // Andromeda failed 20397 queue, skip one for verifier batch only - let queueIndex = transaction.queueIndex - if (queueIndex >= 20397) { - queueIndex++ - } + const queueIndex = transaction.queueIndex const enqueue = await this.getEnqueueByIndex(queueIndex) if (enqueue === null) { return null diff --git a/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts b/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts index 0ec8ebe6c4a57..e7add29b0ab03 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/handlers/sequencer-batch-inbox.ts @@ -23,7 +23,7 @@ import { TransactionEntry, } from '../../../types' import { parseSignatureVParam, SEQUENCER_GAS_LIMIT } from '../../../utils' -import { BlobDataExpiredError, MissingElementError } from './errors' +import { MissingElementError } from './errors' export const handleEventsSequencerBatchInbox: EventHandlerSetAny< SequencerBatchAppendedExtraData, @@ -96,7 +96,7 @@ export const handleEventsSequencerBatchInbox: EventHandlerSetAny< const da = toNumber(calldata.subarray(0, 1)) const compressType = toNumber(calldata.subarray(1, 2)) let contextData = calldata.subarray(70) - let channels = [] + const channels = [] // da first if (da === 1) { const storageObject = remove0x(toHexString(contextData)) @@ -143,38 +143,20 @@ export const handleEventsSequencerBatchInbox: EventHandlerSetAny< blobTxHashes.push(ethers.hexlify(contextData.subarray(i, i + 32))) } - try { - channels = channels.concat( - await fetchBatches({ - blobTxHashes, - chainId: toNumber(l1ChainId), - batchInbox: options.batchInboxAddress, - batchSenders: - extraData.context && extraData.context.inboxBlobSenderAddress - ? [extraData.context.inboxBlobSenderAddress.toLowerCase()] - : [], - concurrentRequests: 0, - l1Rpc: options.l1RpcProvider, - l1Beacon: options.l1BeaconProvider, - l2ChainId: options.l2ChainId, - }) - ) - } catch (e) { - if (e instanceof BlobDataExpiredError) { - // if blob data has already expired, we must recover from a snapshot - console.error( - `Blob data has already expired, please recover from a most recent snapshot, error: ${e}` - ) - process.exit(1) - } - throw e - } - - for (const channel of channels) { - if (channel.invalidBatches || channel.invalidFrames) { - throw new Error(`Invalid batches found: ${channel.id}`) - } - } + const result = await fetchBatches({ + blobTxHashes, + chainId: toNumber(l1ChainId), + l2ChainId: options.l2ChainId, + batchInbox: options.batchInboxAddress, + batchSenders: + extraData.context && extraData.context.inboxBlobSenderAddress + ? [extraData.context.inboxBlobSenderAddress.toLowerCase()] + : [], + concurrentRequests: 0, + l1RpcProvider: options.l1RpcProvider, + l1BeaconProvider: options.l1BeaconProvider, + }) + channels.push(...result) } if (compressType === 11) { contextData = await zlibDecompress(contextData) diff --git a/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts b/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts index c0aa49426eb98..8275ef99bda95 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/handlers/transaction-enqueued.ts @@ -21,10 +21,7 @@ export const handleEventsTransactionEnqueued: EventHandlerSet< gasLimit: event.args._gasLimit.toString(), origin: event.args._l1TxOrigin, blockNumber: toNumber(event.blockNumber), - timestamp: - toNumber(event.blockNumber) >= 14570938 - ? Math.floor(new Date().getTime() / 1000) - : toNumber(event.args._timestamp), + timestamp: toNumber(event.args._timestamp), ctcIndex: null, } }, diff --git a/packages/data-transport-layer/src/services/l1-ingestion/service.ts b/packages/data-transport-layer/src/services/l1-ingestion/service.ts index 27d234f3e6fb1..e1dcef37d266e 100755 --- a/packages/data-transport-layer/src/services/l1-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l1-ingestion/service.ts @@ -1,6 +1,6 @@ /* Imports: External */ import { BaseService, Metrics } from '@eth-optimism/common-ts' -import { FallbackProvider, fromHexString } from '@metis.io/core-utils' +import { fromHexString } from '@metis.io/core-utils' import { Block, ethers, @@ -13,6 +13,7 @@ import { LevelUp } from 'levelup' import { Counter, Gauge } from 'prom-client' /* Imports: Internal */ +import { L1BeaconClient } from '../../da/blob/l1-beacon-client' import { TransportDB, TransportDBMap, @@ -122,6 +123,7 @@ export class L1IngestionService extends BaseService { dbOfL2: TransportDB contracts: OptimismContracts l1RpcProvider: Provider + l1BeaconProvider: L1BeaconClient l1ChainId: number startingL1BlockNumber: number startingL1BatchIndex: number @@ -143,29 +145,32 @@ export class L1IngestionService extends BaseService { this.state.dbs = {} this.l1IngestionMetrics = registerMetrics(this.metrics) - if (typeof this.options.l1RpcProvider === 'string') { - // FIXME: ethers v6's fallback provider has a bug that it will lost the prefetched transaction data while - // converting the json rpc data to ethers transaction object, so we will only enable it when we configured - // multiple rpc providers, but this will cause a significant performance downgrade. Because we need to fetch - // the transactions one by one. - if (this.options.l1RpcProvider.indexOf(',') >= 0) { - this.logger.info('Using FallbackProvider for L1 RPC Provider') - this.state.l1RpcProvider = FallbackProvider(this.options.l1RpcProvider) - } else { - this.state.l1RpcProvider = new ethers.JsonRpcProvider( - this.options.l1RpcProvider - ) - } - } else { - this.state.l1RpcProvider = this.options.l1RpcProvider - } + this.state.l1RpcProvider = new ethers.JsonRpcProvider( + this.options.l1RpcEndpoint + ) const network = await this.state.l1RpcProvider.getNetwork() this.state.l1ChainId = toNumber(network.chainId) this.logger.info('Using L1 RPC Provider', { l1ChainId: this.state.l1ChainId, + l1RpcEndpoint: this.options.l1RpcEndpoint, }) + console.log( + 'Initializing L1 Beacon Client...', + this.options.l1BeaconEndpoint + ) + this.state.l1BeaconProvider = new L1BeaconClient( + this.options.l1BeaconEndpoint + ) + + const l1BeaconChainId = await this.state.l1BeaconProvider.getChainId() + if (l1BeaconChainId !== this.state.l1ChainId) { + throw new Error( + `Chain ID mismatch: Beacon ${l1BeaconChainId} !== L1 RPC ${this.state.l1ChainId}` + ) + } + this.logger.info('Using AddressManager', { addressManager: this.options.addressManager, }) @@ -310,12 +315,6 @@ export class L1IngestionService extends BaseService { const highestSyncedL1BatchIndex = latestBatch === null ? -1 : latestBatch.index - this.logger.info('Synchronizing events from Layer 1 (Ethereum)', { - usingL2ChainId: this.options.l2ChainId, - latestBatch, - stateLatestBatch: await this.state.db.getLatestTransactionBatch(), - }) - const inboxAddress = this.options.batchInboxAddress const inboxBatchStart = this.options.batchInboxStartIndex const inboxSender = this.options.batchInboxSender @@ -335,6 +334,7 @@ export class L1IngestionService extends BaseService { this.state.startingL1BatchIndex <= highestSyncedL1BatchIndex + 1 this.logger.info('Synchronizing events from Layer 1 (Ethereum)', { + latestBatch, highestSyncedL1Block, targetL1Block, highestSyncedL1BatchIndex, @@ -524,6 +524,11 @@ export class L1IngestionService extends BaseService { toL1Block: number, handlers: EventHandlerSetAny ): Promise { + this.logger.info('syncInboxBatch', { + fromL1Block, + toL1Block, + }) + const blockPromises = [] for (let i = fromL1Block; i <= toL1Block; i++) { blockPromises.push(this.state.l1RpcProvider.getBlock(i, true)) @@ -531,10 +536,6 @@ export class L1IngestionService extends BaseService { // Just making sure that the blocks will come back in increasing order. const blocks = (await Promise.all(blockPromises)) as Block[] - this.logger.info('_syncInboxBatch get blocks', { - fromL1Block, - toL1Block, - }) const extraMap: Record = {} for (const block of blocks) { @@ -652,7 +653,11 @@ export class L1IngestionService extends BaseService { extraData, this.options.l2ChainId, this.state.l1ChainId, - this.options + { + ...this.options, + l1RpcProvider: this.state.l1RpcProvider, + l1BeaconProvider: this.state.l1BeaconProvider, + } ) this.logger.info('Storing Inbox Batch:', { chainId: this.options.l2ChainId, @@ -772,7 +777,7 @@ export class L1IngestionService extends BaseService { db = await this.options.dbs.getTransportDbByChainId(chainId) } - this.logger.info('Storing Event:', { + this.logger.debug('Storing Event:', { chainId, parsedEvent, }) @@ -803,7 +808,7 @@ export class L1IngestionService extends BaseService { ): Promise { const chainId = this.state.l1ChainId.toString() if (addressEvent[chainId]) { - this.logger.info( + this.logger.debug( `Reading from local ${contractName}, chainId is ${chainId}` ) const addressDict = addressEvent[chainId] @@ -816,7 +821,7 @@ export class L1IngestionService extends BaseService { const addr = arr[i] if (blockNumber >= addr.Start) { findAddress = addr.Address - this.logger.info( + this.logger.debug( `Read cached contract address for ${contractName} from ${addr.Start} to ${blockNumber}, get ${findAddress}` ) break @@ -824,7 +829,7 @@ export class L1IngestionService extends BaseService { } return findAddress } - this.logger.info(`Searching from RPC ${contractName}`) + this.logger.debug(`Searching from RPC ${contractName}`) const events = await this.state.contracts.Lib_AddressManager.queryFilter( this.state.contracts.Lib_AddressManager.filters.AddressSet(contractName), this.state.startingL1BlockNumber, diff --git a/packages/data-transport-layer/src/services/l2-ingestion/service.ts b/packages/data-transport-layer/src/services/l2-ingestion/service.ts index f2c530f3fafe4..a88e92b6660f7 100755 --- a/packages/data-transport-layer/src/services/l2-ingestion/service.ts +++ b/packages/data-transport-layer/src/services/l2-ingestion/service.ts @@ -1,17 +1,17 @@ /* Imports: External */ import { BaseService, Metrics } from '@eth-optimism/common-ts' -import { LevelUp } from 'levelup' import axios from 'axios' import bfj from 'bfj' -import { Gauge } from 'prom-client' +import { JsonRpcProvider, toNumber } from 'ethersv6' +import { LevelUp } from 'levelup' import path from 'path' -import { toNumber, JsonRpcProvider } from 'ethersv6' +import { Gauge } from 'prom-client' /* Imports: Internal */ import { TransportDB, - TransportDBMapHolder, TransportDBMap, + TransportDBMapHolder, } from '../../db/transport-db' import { sleep, toRpcHexString, validators } from '../../utils' import { L1DataTransportServiceOptions } from '../main/service' @@ -105,10 +105,7 @@ export class L2IngestionService extends BaseService { `${this.options.l2ChainId}` ) } else { - this.state.l2RpcProvider = - typeof this.options.l2RpcProvider === 'string' - ? new JsonRpcProvider(this.options.l2RpcProvider) - : this.options.l2RpcProvider + this.state.l2RpcProvider = new JsonRpcProvider(this.options.l2RpcEndpoint) } } diff --git a/packages/data-transport-layer/src/services/main/service.ts b/packages/data-transport-layer/src/services/main/service.ts index 44d93c6908078..0c59882dce431 100755 --- a/packages/data-transport-layer/src/services/main/service.ts +++ b/packages/data-transport-layer/src/services/main/service.ts @@ -1,15 +1,15 @@ /* Imports: External */ import { BaseService, Metrics } from '@eth-optimism/common-ts' -import { LevelUp } from 'levelup' import level from 'level' +import { LevelUp } from 'levelup' import { Counter } from 'prom-client' /* Imports: Internal */ -import { L1IngestionService } from '../l1-ingestion/service' -import { L1TransportServer } from '../server/service' +import { TransportDBMapHolder } from '../../db/transport-db' import { validators } from '../../utils' +import { L1IngestionService } from '../l1-ingestion/service' import { L2IngestionService } from '../l2-ingestion/service' -import { TransportDBMapHolder } from '../../db/transport-db' +import { L1TransportServer } from '../server/service' export interface L1DataTransportServiceOptions { nodeEnv: string @@ -19,10 +19,10 @@ export interface L1DataTransportServiceOptions { confirmations: number dangerouslyCatchAllErrors?: boolean hostname: string - l1RpcProvider: string - l1BeaconProvider: string + l1RpcEndpoint: string + l1BeaconEndpoint: string l2ChainId: number - l2RpcProvider: string + l2RpcEndpoint: string metrics?: Metrics dbPath: string logsPerPollingInterval: number diff --git a/packages/data-transport-layer/src/services/run.ts b/packages/data-transport-layer/src/services/run.ts index 21452dd851870..c4bf1f1789419 100755 --- a/packages/data-transport-layer/src/services/run.ts +++ b/packages/data-transport-layer/src/services/run.ts @@ -1,7 +1,8 @@ /* Imports: External */ -import * as dotenv from 'dotenv' import { Bcfg } from '@metis.io/core-utils' import Config from 'bcfg' +import { loadTrustedSetup } from 'c-kzg' +import * as dotenv from 'dotenv' /* Imports: Internal */ import { L1DataTransportService } from './main/service' @@ -10,6 +11,7 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli' ;(async () => { try { dotenv.config() + loadTrustedSetup(0) const config: Bcfg = new Config('data-transport-layer') config.load({ @@ -25,7 +27,7 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli' port: config.uint('server-port', 7878), hostname: config.str('server-hostname', 'localhost'), confirmations: config.uint('confirmations', 35), - l1RpcProvider: config.str('l1-rpc-endpoint'), + l1RpcEndpoint: config.str('l1-rpc-endpoint'), addressManager: config.str('address-manager'), pollingInterval: config.uint('polling-interval', 5000), logsPerPollingInterval: config.uint('logs-per-polling-interval', 2000), @@ -33,8 +35,8 @@ type ethNetwork = 'mainnet' | 'kovan' | 'goerli' 'dangerously-catch-all-errors', false ), - l1BeaconProvider: config.str('l1-beacon-endpoint'), - l2RpcProvider: config.str('l2-rpc-endpoint'), + l1BeaconEndpoint: config.str('l1-beacon-endpoint'), + l2RpcEndpoint: config.str('l2-rpc-endpoint'), l2ChainId: config.uint('l2-chain-id'), syncFromL1: config.bool('sync-from-l1', true), syncFromL2: config.bool('sync-from-l2', false), diff --git a/packages/data-transport-layer/src/services/server/service.ts b/packages/data-transport-layer/src/services/server/service.ts index 25a6836e82411..869e4a57c9383 100755 --- a/packages/data-transport-layer/src/services/server/service.ts +++ b/packages/data-transport-layer/src/services/server/service.ts @@ -1,34 +1,31 @@ /* Imports: External */ import { BaseService, Logger, Metrics } from '@eth-optimism/common-ts' +import * as Sentry from '@sentry/node' +import * as Tracing from '@sentry/tracing' +import cors from 'cors' +import { JsonRpcProvider, toBigInt, toNumber } from 'ethersv6' import express, { Request, Response } from 'express' import promBundle from 'express-prom-bundle' -import cors from 'cors' import { LevelUp } from 'levelup' -import * as Sentry from '@sentry/node' -import * as Tracing from '@sentry/tracing' -import { toBigInt, toNumber, JsonRpcProvider, Block } from 'ethersv6' /* Imports: Internal */ import { TransportDB, TransportDBMapHolder } from '../../db/transport-db' import { + AppendBatchElementResponse, + BlockBatchResponse, + BlockResponse, ContextResponse, - GasPriceResponse, EnqueueResponse, + GasPriceResponse, + HighestResponse, StateRootBatchResponse, StateRootResponse, SyncingResponse, TransactionBatchResponse, TransactionResponse, - BlockBatchResponse, - BlockResponse, - VerifierResultResponse, VerifierResultEntry, + VerifierResultResponse, VerifierStakeResponse, - AppendBatchElementResponse, - HighestResponse, - SyncStatusResponse, - L1BlockRef, - L2BlockRef, } from '../../types' import { validators } from '../../utils' import { L1DataTransportServiceOptions } from '../main/service' @@ -101,15 +98,8 @@ export class L1TransportServer extends BaseService { this.options.db, this.options.l2ChainId === 1088 ) - this.state.l1RpcProvider = - typeof this.options.l1RpcProvider === 'string' - ? new JsonRpcProvider(this.options.l1RpcProvider) - : this.options.l1RpcProvider - - this.state.l2RpcProvider = - typeof this.options.l2RpcProvider === 'string' - ? new JsonRpcProvider(this.options.l2RpcProvider) - : this.options.l2RpcProvider + this.state.l1RpcProvider = new JsonRpcProvider(this.options.l1RpcEndpoint) + this.state.l2RpcProvider = new JsonRpcProvider(this.options.l2RpcEndpoint) this._initializeApp() } diff --git a/yarn.lock b/yarn.lock index 814debff3354f..b328f2c134204 100644 --- a/yarn.lock +++ b/yarn.lock @@ -415,7 +415,7 @@ __metadata: eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" ethereum-waffle: "npm:^3.3.0" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" ganache-core: "npm:^2.13.2" hardhat: "npm:^2.3.0" lint-staged: "npm:11.0.0" @@ -527,10 +527,11 @@ __metadata: "@types/levelup": "npm:^4.3.0" "@types/mocha": "npm:^8.2.2" "@types/node-fetch": "npm:^2.5.10" + "@types/qs": "npm:^6.14.0" "@types/workerpool": "npm:^6.0.0" "@typescript-eslint/eslint-plugin": "npm:^4.26.0" "@typescript-eslint/parser": "npm:^4.26.0" - axios: "npm:^0.21.1" + axios: "npm:^1.13.2" bcfg: "npm:^0.1.6" bfj: "npm:^7.0.2" browser-or-node: "npm:^1.3.0" @@ -548,7 +549,7 @@ __metadata: eslint-plugin-prettier: "npm:^3.4.0" eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" express: "npm:^4.17.1" express-prom-bundle: "npm:^6.3.6" hardhat: "npm:^2.3.0" @@ -562,6 +563,7 @@ __metadata: pino-pretty: "npm:^4.7.1" prettier: "npm:^2.3.1" prom-client: "npm:^13.1.0" + qs: "npm:^6.14.0" rimraf: "npm:^3.0.2" rlp: "npm:^3.0.0" ts-node: "npm:^10.0.0" @@ -2808,7 +2810,7 @@ __metadata: eslint-plugin-prettier: "npm:^3.4.0" eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" lint-staged: "npm:11.0.0" lodash: "npm:^4.17.21" merkletreejs: "npm:^0.2.32" @@ -4564,7 +4566,7 @@ __metadata: languageName: node linkType: hard -"@types/qs@npm:*, @types/qs@npm:^6.2.31, @types/qs@npm:^6.9.7": +"@types/qs@npm:*, @types/qs@npm:^6.14.0, @types/qs@npm:^6.2.31, @types/qs@npm:^6.9.7": version: 6.14.0 resolution: "@types/qs@npm:6.14.0" checksum: 10c0/5b3036df6e507483869cdb3858201b2e0b64b4793dc4974f188caa5b5732f2333ab9db45c08157975054d3b070788b35088b4bc60257ae263885016ee2131310 @@ -5709,6 +5711,17 @@ __metadata: languageName: node linkType: hard +"axios@npm:^1.13.2": + version: 1.13.2 + resolution: "axios@npm:1.13.2" + dependencies: + follow-redirects: "npm:^1.15.6" + form-data: "npm:^4.0.4" + proxy-from-env: "npm:^1.1.0" + checksum: 10c0/e8a42e37e5568ae9c7a28c348db0e8cf3e43d06fcbef73f0048669edfe4f71219664da7b6cc991b0c0f01c28a48f037c515263cb79be1f1ae8ff034cd813867b + languageName: node + linkType: hard + "axios@npm:^1.4.0, axios@npm:^1.5.1": version: 1.13.0 resolution: "axios@npm:1.13.0" @@ -10342,9 +10355,9 @@ __metadata: languageName: node linkType: hard -"ethersv6@github:ericlee42/ethers#metis": - version: 6.15.0 - resolution: "ethersv6@https://github.com/ericlee42/ethers.git#commit=5f376edc77c815bf8c992c426bafc4faaad7876a" +"ethersv6@npm:ethers@^6.16.0": + version: 6.16.0 + resolution: "ethers@npm:6.16.0" dependencies: "@adraffy/ens-normalize": "npm:1.10.1" "@noble/curves": "npm:1.2.0" @@ -10353,7 +10366,7 @@ __metadata: aes-js: "npm:4.0.0-beta.5" tslib: "npm:2.7.0" ws: "npm:8.17.1" - checksum: 10c0/3505e9c9ae3a9e3a4191c3131327b4adfa2f82bcb7ea5de1365e369d02195abb8efa4c04763407b3f3181309b437875af7a86509d37561360f4d15a9fc437bac + checksum: 10c0/65c0ff7016b1592b1961c3b68508902b89fc75e1ad025bab3a722df1b5699fd077551875a7285e65b2d0cfea9a85b2459bb84010a0fa4e4494d231848aee3a73 languageName: node linkType: hard @@ -15716,7 +15729,7 @@ __metadata: eslint-plugin-prettier: "npm:^3.4.0" eslint-plugin-react: "npm:^7.24.0" eslint-plugin-unicorn: "npm:^32.0.1" - ethersv6: "github:ericlee42/ethers#metis" + ethersv6: "npm:ethers@^6.16.0" lerna: "npm:^4.0.0" patch-package: "npm:^6.4.7" prettier: "npm:^2.3.1" @@ -17817,7 +17830,7 @@ __metadata: languageName: node linkType: hard -"qs@npm:^6.11.0, qs@npm:^6.12.3, qs@npm:^6.4.0, qs@npm:^6.9.4": +"qs@npm:^6.11.0, qs@npm:^6.12.3, qs@npm:^6.14.0, qs@npm:^6.4.0, qs@npm:^6.9.4": version: 6.14.0 resolution: "qs@npm:6.14.0" dependencies: