From 92ad577ef7e7ccfc0afd246787ec9fe368a2433f Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Tue, 16 Dec 2025 18:32:25 +0100 Subject: [PATCH 01/12] feat(ensindexer): ENSDb Writer Worker Store ENSNode metadata (ENSIndexer Public Config, Indexing Status) into ENSDb. --- apps/ensapi/package.json | 2 +- apps/ensindexer/package.json | 5 + .../ponder/src/ensdb-writer-worker.ts | 123 ++++++++++++++++++ apps/ensindexer/src/lib/ensdb/drizzle.ts | 41 ++++++ .../src/lib/ensdb/ensdb-connection.ts | 63 +++++++++ .../src/lib/ensdb/ensdb-writer-client.ts | 101 ++++++++++++++ apps/ensindexer/src/lib/ensdb/index.ts | 2 + apps/ensindexer/src/lib/logger.ts | 24 ++++ packages/ensnode-schema/src/ponder.schema.ts | 1 + .../src/schemas/ensnode-metadata.schema.ts | 19 +++ pnpm-lock.yaml | 56 ++++++-- pnpm-workspace.yaml | 1 + 12 files changed, 423 insertions(+), 15 deletions(-) create mode 100644 apps/ensindexer/ponder/src/ensdb-writer-worker.ts create mode 100644 apps/ensindexer/src/lib/ensdb/drizzle.ts create mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-connection.ts create mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts create mode 100644 apps/ensindexer/src/lib/ensdb/index.ts create mode 100644 apps/ensindexer/src/lib/logger.ts create mode 100644 packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts diff --git a/apps/ensapi/package.json b/apps/ensapi/package.json index 420b7861f..f9df47ac7 100644 --- a/apps/ensapi/package.json +++ b/apps/ensapi/package.json @@ -44,7 +44,7 @@ "drizzle-orm": "catalog:", "hono": "catalog:", "p-memoize": "^8.0.0", - "p-retry": "^7.1.0", + "p-retry": "catalog:", "pg-connection-string": "catalog:", "pino": "catalog:", "ponder-enrich-gql-docs-middleware": "^0.1.3", diff --git a/apps/ensindexer/package.json b/apps/ensindexer/package.json index 8d1daaa24..a23fbcdf0 100644 --- a/apps/ensindexer/package.json +++ b/apps/ensindexer/package.json @@ -33,7 +33,11 @@ "date-fns": "catalog:", "deepmerge-ts": "^7.1.5", "dns-packet": "^5.6.1", + "drizzle-orm": "catalog:", + "p-retry": "catalog:", + "pino": "catalog:", "pg-connection-string": "catalog:", + "pg": "8.16.3", "hono": "catalog:", "ponder": "catalog:", "viem": "catalog:", @@ -43,6 +47,7 @@ "@ensnode/shared-configs": "workspace:*", "@types/dns-packet": "^5.6.5", "@types/node": "catalog:", + "@types/pg": "8.16.0", "typescript": "catalog:", "vitest": "catalog:" } diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts new file mode 100644 index 000000000..a21db4880 --- /dev/null +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -0,0 +1,123 @@ +/** + * This file manages syncing ENSNode metadata: + * - ENSIndexer Public Config + * - Indexing Status + * into the ENSDb. + */ +import config from "@/config"; + +import pRetry from "p-retry"; + +import { + type CrossChainIndexingStatusSnapshot, + type IndexingStatusResponse, + IndexingStatusResponseCodes, + OmnichainIndexingStatusIds, + type SerializedENSIndexerPublicConfig, +} from "@ensnode/ensnode-sdk"; + +import { EnsDbConnection, EnsDbWriterClient } from "@/lib/ensdb"; +import { makeLogger } from "@/lib/logger"; + +const logger = makeLogger("ensdb-writer-worker"); + +/** + * Wait for ENSIndexer to become healthy. + * + * @throws when the Health endpoint didn't return HTTP Status OK. + */ +async function waitForEnsIndexerHealthy(): Promise { + logger.info("Fetching ENSIndexer Health status"); + + try { + await fetch(new URL("/health", config.ensIndexerUrl)); + + logger.info("ENSIndexer is healthy"); + } catch { + const errorMessage = "Health endpoint for ENSIndexer is not available yet."; + logger.error(errorMessage); + + throw new Error(errorMessage); + } +} + +interface FetchEnsNodeMetadataResult { + ensIndexerPublicConfigSerialized: SerializedENSIndexerPublicConfig; + indexingStatusSnapshot: CrossChainIndexingStatusSnapshot; +} + +/** + * Fetch ENSNode metadata. + * + * @returns ENSNode metadata when it's available and with valid Indexing Status. + * @throws error when ENSNode metadata was not available, or was available but with invalid Indexing Status. + */ +async function fetchEnsNodeMetadata(): Promise { + logger.info("Fetching ENSNode metadata"); + + const [ensIndexerPublicConfigSerialized, indexingStatusSerialized] = await Promise.all([ + fetch(new URL("/api/config", config.ensIndexerUrl)) + .then((response) => response.json()) + .then((response) => response as unknown as SerializedENSIndexerPublicConfig), + fetch(new URL("/api/indexing-status", config.ensIndexerUrl)) + .then((response) => response.json()) + .then((response) => response as unknown as IndexingStatusResponse), + ]); + + logger.info("Fetched ENSNode metadata"); + + if (indexingStatusSerialized.responseCode === IndexingStatusResponseCodes.Ok) { + const { snapshot } = indexingStatusSerialized.realtimeProjection; + const { omnichainSnapshot } = snapshot; + + if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { + throw new Error("Indexing Status ID must be different that 'Unstarted'."); + } + + return { + ensIndexerPublicConfigSerialized, + indexingStatusSnapshot: snapshot, + }; + } + + throw new Error("Indexing Status must be available."); +} + +async function upsertEnsNodeMetadataRecords() { + await pRetry(waitForEnsIndexerHealthy, { retries: 5 }); + + const ensDbConnection = new EnsDbConnection(); + const ensDbClient = ensDbConnection.connect({ + schemaName: config.databaseSchemaName, + poolConfig: { + connectionString: config.databaseUrl, + }, + }); + + logger.info("ENSDb Client connected"); + + const ensDbWriterClient = new EnsDbWriterClient(ensDbClient); + + const { ensIndexerPublicConfigSerialized, indexingStatusSnapshot } = await pRetry( + fetchEnsNodeMetadata, + { + retries: 5, + }, + ); + + await pRetry(() => + Promise.all([ + ensDbWriterClient + .upsertEnsIndexerPublicConfig(ensIndexerPublicConfigSerialized) + .then(() => logger.info("ENSIndexer Public Config successfully stored in ENSDb.")), + + ensDbWriterClient + .upsertIndexingStatus(indexingStatusSnapshot) + .then(() => logger.info("Indexing Status successfully stored in ENSDb.")), + ]), + ); +} + +// Upsert ENSNode Metadata Records in a non-blocking way to +// allow database migrations to proceed in the background. +setTimeout(upsertEnsNodeMetadataRecords, 0); diff --git a/apps/ensindexer/src/lib/ensdb/drizzle.ts b/apps/ensindexer/src/lib/ensdb/drizzle.ts new file mode 100644 index 000000000..50700aa88 --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/drizzle.ts @@ -0,0 +1,41 @@ +// This file was copied 1-to-1 from ENSApi. + +import { isTable, Table } from "drizzle-orm"; +import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; +import { isPgEnum } from "drizzle-orm/pg-core"; +import type { Pool } from "pg"; + +type Schema = { [name: string]: unknown }; + +// https://github.com/ponder-sh/ponder/blob/f7f6444ab8d1a870fe6492023941091df7b7cddf/packages/client/src/index.ts#L226C1-L239C3 +const setDatabaseSchema = (schema: T, schemaName: string) => { + for (const table of Object.values(schema)) { + if (isTable(table)) { + // @ts-expect-error + table[Table.Symbol.Schema] = schemaName; + } else if (isPgEnum(table)) { + // @ts-expect-error + table.schema = schemaName; + } + } +}; + +/** + * Makes a Drizzle DB object. + */ +export const makeDrizzle = ({ + schema, + connectionPool, + databaseSchema, +}: { + schema: SCHEMA; + connectionPool: Pool; + databaseSchema: string; +}): NodePgDatabase & { + $client: Pool; +} => { + // monkeypatch schema onto tables + setDatabaseSchema(schema, databaseSchema); + + return drizzle(connectionPool, { schema, casing: "snake_case" }); +}; diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-connection.ts b/apps/ensindexer/src/lib/ensdb/ensdb-connection.ts new file mode 100644 index 000000000..6ab1baed3 --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/ensdb-connection.ts @@ -0,0 +1,63 @@ +import type { NodePgDatabase } from "drizzle-orm/node-postgres"; +import { Pool, type PoolConfig } from "pg"; + +import * as schema from "@ensnode/ensnode-schema"; + +import { makeDrizzle } from "@/lib/ensdb/drizzle"; + +export type EnsDbClient = NodePgDatabase; + +export interface EnsDbConnectionOptions { + schemaName: string; + poolConfig: PoolConfig; +} + +/** + * ENSDb Connection + * + * Enables: + * - Connecting to ENSDb instance. + * - Disconnecting from ENSDb instance. + * + * Uses application connection pool for improved performance. + */ +export class EnsDbConnection { + #connectionPool: Pool | undefined; + #ensDbClient: EnsDbClient | undefined; + + /** + * Connect to ENSDb instance. + * + * @returns ENSDb Client + */ + connect({ schemaName, poolConfig }: EnsDbConnectionOptions): EnsDbClient { + if (this.#connectionPool) { + throw new Error("ENSDb already connected. Call disconnect() first."); + } + + this.#connectionPool = new Pool(poolConfig); + + this.#ensDbClient = makeDrizzle({ + connectionPool: this.#connectionPool, + databaseSchema: schemaName, + schema, + }); + + return this.#ensDbClient; + } + + /** + * Disconnect to ENSDb instance. + * + * Call this function to free up resources. + */ + async disconnect(): Promise { + if (!this.#connectionPool) return; + + // Free up resources + await this.#connectionPool.end(); + + this.#connectionPool = undefined; + this.#ensDbClient = undefined; + } +} diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts b/apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts new file mode 100644 index 000000000..84a121c66 --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts @@ -0,0 +1,101 @@ +import * as schema from "@ensnode/ensnode-schema"; +import type { + CrossChainIndexingStatusSnapshot, + SerializedENSIndexerPublicConfig, +} from "@ensnode/ensnode-sdk"; + +import type { EnsDbClient } from "./ensdb-connection"; + +/** + * Keys used to distinguish records in `ensnode_metadata` table in the ENSDb. + */ +export const EnsNodeMetadataKeys = { + EnsIndexerPublicConfig: "ensindexer-public-config", + IndexingStatus: "indexing-status", +} as const; + +export type EnsNodeMetadataKey = (typeof EnsNodeMetadataKeys)[keyof typeof EnsNodeMetadataKeys]; + +export interface EnsNodeMetadataEnsIndexerPublicConfig { + key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; + value: SerializedENSIndexerPublicConfig; +} + +export interface EnsNodeMetadataIndexingStatus { + key: typeof EnsNodeMetadataKeys.IndexingStatus; + value: CrossChainIndexingStatusSnapshot; +} + +/** + * ENSNode Metadata + * + * Union type gathering all variants of ENSNode Metadata. + */ +export type EnsNodeMetadata = EnsNodeMetadataEnsIndexerPublicConfig | EnsNodeMetadataIndexingStatus; + +/** + * ENSDb Writer Client + * + * The database client performing write operations. + */ +export class EnsDbWriterClient { + constructor(private ensDbClient: EnsDbClient) {} + + /** + * Upsert ENSIndexer Public Config + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async upsertEnsIndexerPublicConfig( + ensIndexerPublicConfig: SerializedENSIndexerPublicConfig, + ): Promise { + return this.upsertEnsNodeMetadata({ + key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, + value: ensIndexerPublicConfig, + }); + } + + /** + * Upsert Indexing Status + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async upsertIndexingStatus( + indexingStatus: CrossChainIndexingStatusSnapshot, + ): Promise { + return this.upsertEnsNodeMetadata({ + key: EnsNodeMetadataKeys.IndexingStatus, + value: indexingStatus, + }); + } + + /** + * Upsert ENSNode metadata + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + private async upsertEnsNodeMetadata< + EnsNodeMetadataType extends EnsNodeMetadata = EnsNodeMetadata, + >(metadata: EnsNodeMetadataType): Promise { + const [result] = await this.ensDbClient + .insert(schema.ensNodeMetadata) + .values({ + key: metadata.key, + value: metadata.value, + }) + .onConflictDoUpdate({ + target: schema.ensNodeMetadata.key, + set: { value: metadata.value }, + }) + .returning({ value: schema.ensNodeMetadata.value }); + + if (!result) { + throw new Error(`Failed to upsert metadata for key: ${metadata.key}`); + } + + return result.value as EnsNodeMetadataType["value"]; + } +} diff --git a/apps/ensindexer/src/lib/ensdb/index.ts b/apps/ensindexer/src/lib/ensdb/index.ts new file mode 100644 index 000000000..6b0ee5449 --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/index.ts @@ -0,0 +1,2 @@ +export * from "./ensdb-connection"; +export * from "./ensdb-writer-client"; diff --git a/apps/ensindexer/src/lib/logger.ts b/apps/ensindexer/src/lib/logger.ts new file mode 100644 index 000000000..eec3ee1c8 --- /dev/null +++ b/apps/ensindexer/src/lib/logger.ts @@ -0,0 +1,24 @@ +// This file was copied 1-to-1 from ENSApi. +import pino from "pino"; + +import { getLogLevelFromEnv, type LogLevel } from "@ensnode/ensnode-sdk/internal"; + +const DEFAULT_LOG_LEVEL: LogLevel = "info"; + +const logger = pino({ + level: getLogLevelFromEnv({ LOG_LEVEL: process.env.LOG_LEVEL }, DEFAULT_LOG_LEVEL), + transport: + process.env.NODE_ENV === "production" + ? undefined + : { + target: "pino-pretty", + options: { + colorize: true, + ignore: "pid,hostname", + }, + }, +}); + +export const makeLogger = (scope: string) => logger.child({ scope }); + +export default logger; diff --git a/packages/ensnode-schema/src/ponder.schema.ts b/packages/ensnode-schema/src/ponder.schema.ts index 8137dd0f7..a5909df25 100644 --- a/packages/ensnode-schema/src/ponder.schema.ts +++ b/packages/ensnode-schema/src/ponder.schema.ts @@ -2,6 +2,7 @@ * Merge the various sub-schemas into a single ponder (drizzle) schema. */ +export * from "./schemas/ensnode-metadata.schema"; export * from "./schemas/protocol-acceleration.schema"; export * from "./schemas/registrars.schema"; export * from "./schemas/subgraph.schema"; diff --git a/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts b/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts new file mode 100644 index 000000000..3018ab7a4 --- /dev/null +++ b/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts @@ -0,0 +1,19 @@ +/** + * Schema Definitions that hold metadata about the ENSNode instance. + */ + +import { onchainTable } from "ponder"; + +export const ensNodeMetadata = onchainTable("ensnode_metadata", (t) => ({ + /** + * Key + */ + key: t.text().primaryKey(), + + /** + * Value + * + * Guaranteed to be a JSON object. + */ + value: t.jsonb().notNull(), +})); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 60fec97a1..55ce1ddc8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,9 @@ catalogs: hono: specifier: ^4.10.2 version: 4.10.3 + p-retry: + specifier: ^7.1.1 + version: 7.1.1 pg-connection-string: specifier: ^2.9.1 version: 2.9.1 @@ -339,7 +342,7 @@ importers: version: 4.1.0 drizzle-orm: specifier: 'catalog:' - version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(kysely@0.26.3)(pg@8.16.3) + version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3) hono: specifier: 'catalog:' version: 4.10.3 @@ -347,8 +350,8 @@ importers: specifier: ^8.0.0 version: 8.0.0 p-retry: - specifier: ^7.1.0 - version: 7.1.0 + specifier: 'catalog:' + version: 7.1.1 pg-connection-string: specifier: 'catalog:' version: 2.9.1 @@ -416,15 +419,27 @@ importers: dns-packet: specifier: ^5.6.1 version: 5.6.1 + drizzle-orm: + specifier: 'catalog:' + version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3) hono: specifier: 'catalog:' version: 4.10.3 + p-retry: + specifier: 'catalog:' + version: 7.1.1 + pg: + specifier: 8.16.3 + version: 8.16.3 pg-connection-string: specifier: 'catalog:' version: 2.9.1 + pino: + specifier: 'catalog:' + version: 10.1.0 ponder: specifier: 'catalog:' - version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) + version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(@types/pg@8.16.0)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) viem: specifier: 'catalog:' version: 2.38.5(typescript@5.9.3)(zod@3.25.76) @@ -441,6 +456,9 @@ importers: '@types/node': specifier: 'catalog:' version: 22.18.13 + '@types/pg': + specifier: 8.16.0 + version: 8.16.0 typescript: specifier: 'catalog:' version: 5.9.3 @@ -733,7 +751,7 @@ importers: dependencies: ponder: specifier: 'catalog:' - version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) + version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(@types/pg@8.16.0)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) viem: specifier: 'catalog:' version: 2.38.5(typescript@5.9.3)(zod@3.25.76) @@ -820,7 +838,7 @@ importers: version: link:../ensrainbow-sdk drizzle-orm: specifier: 'catalog:' - version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(kysely@0.26.3)(pg@8.16.3) + version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3) parse-prometheus-text-format: specifier: ^1.1.1 version: 1.1.1 @@ -839,7 +857,7 @@ importers: version: 4.10.3 ponder: specifier: 'catalog:' - version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) + version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(@types/pg@8.16.0)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) tsup: specifier: 'catalog:' version: 8.5.0(jiti@2.6.1)(postcss@8.5.6)(tsx@4.20.6)(typescript@5.9.3)(yaml@2.8.1) @@ -866,7 +884,7 @@ importers: version: 2.2.3 drizzle-orm: specifier: 'catalog:' - version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(kysely@0.26.3)(pg@8.16.3) + version: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3) graphql: specifier: ^16.10.0 version: 16.11.0 @@ -3404,6 +3422,9 @@ packages: '@types/node@22.18.13': resolution: {integrity: sha512-Bo45YKIjnmFtv6I1TuC8AaHBbqXtIo+Om5fE4QiU1Tj8QR/qt+8O3BAtOimG5IFmwaWiPmB3Mv3jtYzBA4Us2A==} + '@types/pg@8.16.0': + resolution: {integrity: sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==} + '@types/progress@2.0.7': resolution: {integrity: sha512-iadjw02vte8qWx7U0YM++EybBha2CQLPGu9iJ97whVgJUT5Zq9MjAPYUnbfRI2Kpehimf1QjFJYxD0t8nqzu5w==} @@ -6012,8 +6033,8 @@ packages: resolution: {integrity: sha512-aNZ+VfjobsWryoiPnEApGGmf5WmNsCo9xu8dfaYamG5qaLP7ClhLN6NgsFe6SwJ2UbLEBK5dv9x8Mn5+RVhMWQ==} engines: {node: '>=18'} - p-retry@7.1.0: - resolution: {integrity: sha512-xL4PiFRQa/f9L9ZvR4/gUCRNus4N8YX80ku8kv9Jqz+ZokkiZLM0bcvX0gm1F3PDi9SPRsww1BDsTWgE6Y1GLQ==} + p-retry@7.1.1: + resolution: {integrity: sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==} engines: {node: '>=20'} p-timeout@3.2.0: @@ -10758,6 +10779,12 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/pg@8.16.0': + dependencies: + '@types/node': 22.18.13 + pg-protocol: 1.10.3 + pg-types: 2.2.0 + '@types/progress@2.0.7': dependencies: '@types/node': 22.18.13 @@ -11975,10 +12002,11 @@ snapshots: dotenv@8.6.0: {} - drizzle-orm@0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(kysely@0.26.3)(pg@8.16.3): + drizzle-orm@0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3): optionalDependencies: '@electric-sql/pglite': 0.2.13 '@opentelemetry/api': 1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a) + '@types/pg': 8.16.0 kysely: 0.26.3 pg: 8.16.3 @@ -13810,7 +13838,7 @@ snapshots: eventemitter3: 5.0.1 p-timeout: 6.1.4 - p-retry@7.1.0: + p-retry@7.1.1: dependencies: is-network-error: 1.3.0 @@ -14054,7 +14082,7 @@ snapshots: graphql: 16.11.0 hono: 4.10.3 - ponder@0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76): + ponder@0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(@types/pg@8.16.0)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76): dependencies: '@babel/code-frame': 7.27.1 '@commander-js/extra-typings': 12.1.0(commander@12.1.0) @@ -14071,7 +14099,7 @@ snapshots: dataloader: 2.2.3 detect-package-manager: 3.0.2 dotenv: 16.6.1 - drizzle-orm: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(kysely@0.26.3)(pg@8.16.3) + drizzle-orm: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3) glob: 12.0.0 graphql: 16.11.0 graphql-yoga: 5.16.0(graphql@16.11.0) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a1f6fab8c..b2ed6d731 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -18,6 +18,7 @@ catalog: date-fns: 4.1.0 drizzle-orm: "=0.41.0" hono: ^4.10.2 + p-retry: ^7.1.1 pg-connection-string: ^2.9.1 pino: 10.1.0 ponder: 0.13.16 From 43ed0e212bb2d8275fd900f7a79d4b711b96ec56 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Tue, 16 Dec 2025 23:14:48 +0100 Subject: [PATCH 02/12] feat(ensapi): use ENSDb query and mutation clients --- .../ponder/src/ensdb-writer-worker.ts | 157 +++++++++--------- ...sdb-writer-client.ts => ensdb-mutation.ts} | 38 +---- apps/ensindexer/src/lib/ensdb/ensdb-query.ts | 73 ++++++++ .../src/lib/ensdb/ensnode-metadata.ts | 31 ++++ apps/ensindexer/src/lib/ensdb/index.ts | 3 +- apps/ensindexer/src/lib/ensindexer/client.ts | 70 ++++++++ apps/ensindexer/src/lib/ensindexer/index.ts | 1 + 7 files changed, 263 insertions(+), 110 deletions(-) rename apps/ensindexer/src/lib/ensdb/{ensdb-writer-client.ts => ensdb-mutation.ts} (64%) create mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-query.ts create mode 100644 apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts create mode 100644 apps/ensindexer/src/lib/ensindexer/client.ts create mode 100644 apps/ensindexer/src/lib/ensindexer/index.ts diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts index a21db4880..840d8bf34 100644 --- a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -9,82 +9,51 @@ import config from "@/config"; import pRetry from "p-retry"; import { - type CrossChainIndexingStatusSnapshot, - type IndexingStatusResponse, IndexingStatusResponseCodes, OmnichainIndexingStatusIds, type SerializedENSIndexerPublicConfig, } from "@ensnode/ensnode-sdk"; -import { EnsDbConnection, EnsDbWriterClient } from "@/lib/ensdb"; +import { EnsDbConnection, EnsDbMutation, EnsDbQuery } from "@/lib/ensdb"; +import { EnsIndexerClient } from "@/lib/ensindexer"; import { makeLogger } from "@/lib/logger"; const logger = makeLogger("ensdb-writer-worker"); -/** - * Wait for ENSIndexer to become healthy. - * - * @throws when the Health endpoint didn't return HTTP Status OK. - */ -async function waitForEnsIndexerHealthy(): Promise { - logger.info("Fetching ENSIndexer Health status"); - - try { - await fetch(new URL("/health", config.ensIndexerUrl)); - - logger.info("ENSIndexer is healthy"); - } catch { - const errorMessage = "Health endpoint for ENSIndexer is not available yet."; - logger.error(errorMessage); - - throw new Error(errorMessage); +const ensIndexerClient = new EnsIndexerClient(config.ensIndexerUrl); +const waitForEnsIndexerToBecomeHealthy = pRetry(async () => ensIndexerClient.health(), { + retries: 5, +}); + +function isEnsIndexerPublicConfigCompatible( + configA: ConfigType, + configB: ConfigType, +): boolean { + if ( + !configA.indexedChainIds.every((configAChainId) => + configB.indexedChainIds.includes(configAChainId), + ) + ) { + return false; } -} -interface FetchEnsNodeMetadataResult { - ensIndexerPublicConfigSerialized: SerializedENSIndexerPublicConfig; - indexingStatusSnapshot: CrossChainIndexingStatusSnapshot; -} + if (configA.isSubgraphCompatible !== configB.isSubgraphCompatible) { + return false; + } -/** - * Fetch ENSNode metadata. - * - * @returns ENSNode metadata when it's available and with valid Indexing Status. - * @throws error when ENSNode metadata was not available, or was available but with invalid Indexing Status. - */ -async function fetchEnsNodeMetadata(): Promise { - logger.info("Fetching ENSNode metadata"); - - const [ensIndexerPublicConfigSerialized, indexingStatusSerialized] = await Promise.all([ - fetch(new URL("/api/config", config.ensIndexerUrl)) - .then((response) => response.json()) - .then((response) => response as unknown as SerializedENSIndexerPublicConfig), - fetch(new URL("/api/indexing-status", config.ensIndexerUrl)) - .then((response) => response.json()) - .then((response) => response as unknown as IndexingStatusResponse), - ]); - - logger.info("Fetched ENSNode metadata"); - - if (indexingStatusSerialized.responseCode === IndexingStatusResponseCodes.Ok) { - const { snapshot } = indexingStatusSerialized.realtimeProjection; - const { omnichainSnapshot } = snapshot; - - if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { - throw new Error("Indexing Status ID must be different that 'Unstarted'."); - } + if (configA.namespace !== configB.namespace) { + return false; + } - return { - ensIndexerPublicConfigSerialized, - indexingStatusSnapshot: snapshot, - }; + if (!configA.plugins.every((configAPlugin) => configB.plugins.includes(configAPlugin))) { + return false; } - throw new Error("Indexing Status must be available."); + return true; } async function upsertEnsNodeMetadataRecords() { - await pRetry(waitForEnsIndexerHealthy, { retries: 5 }); + await waitForEnsIndexerToBecomeHealthy; const ensDbConnection = new EnsDbConnection(); const ensDbClient = ensDbConnection.connect({ @@ -96,26 +65,60 @@ async function upsertEnsNodeMetadataRecords() { logger.info("ENSDb Client connected"); - const ensDbWriterClient = new EnsDbWriterClient(ensDbClient); + const ensDbQuery = new EnsDbQuery(ensDbClient); + const ensDbMutation = new EnsDbMutation(ensDbClient); + + const handleEnsIndexerPublicConfigRecord = async () => { + const [storedConfig, inMemoryConfig] = await pRetry(() => + Promise.all([ensDbQuery.getEnsIndexerPublicConfig(), ensIndexerClient.config()]), + ); + + if (storedConfig && !isEnsIndexerPublicConfigCompatible(storedConfig, inMemoryConfig)) { + throw new Error( + "In-memory ENSIndexerPublicConfig object is not compatible with its counterpart stored in ENSDb.", + ); + } else { + // upsert ENSIndexerPublicConfig into ENSDb + await ensDbMutation + .upsertEnsIndexerPublicConfig(inMemoryConfig) + .then(() => logger.info("ENSIndexer Public Config successfully stored in ENSDb.")); + } + }; + + // TODO: clear interval on application shutdown + let _indexingStatusRefreshInterval: ReturnType; + + const handleIndexingStatusRecord = async () => { + try { + const inMemoryIndexingStatus = await ensIndexerClient.indexingStatus(); + + if (inMemoryIndexingStatus.responseCode !== IndexingStatusResponseCodes.Ok) { + throw new Error("Indexing Status must be available."); + } + + const { snapshot } = inMemoryIndexingStatus.realtimeProjection; + const { omnichainSnapshot } = snapshot; + + if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { + throw new Error("Omnichain Status must be different that 'Unstarted'."); + } + + // upsert ENSIndexerPublicConfig into ENSDb + await ensDbMutation + .upsertIndexingStatus(snapshot) + .then(() => logger.info("ENSIndexer Public Config successfully stored in ENSDb.")); + } catch (error) { + logger.error(error, "Could not upsert Indexing Status record"); + } finally { + _indexingStatusRefreshInterval = setTimeout(handleIndexingStatusRecord, 1000); + } + }; - const { ensIndexerPublicConfigSerialized, indexingStatusSnapshot } = await pRetry( - fetchEnsNodeMetadata, - { - retries: 5, - }, - ); - - await pRetry(() => - Promise.all([ - ensDbWriterClient - .upsertEnsIndexerPublicConfig(ensIndexerPublicConfigSerialized) - .then(() => logger.info("ENSIndexer Public Config successfully stored in ENSDb.")), - - ensDbWriterClient - .upsertIndexingStatus(indexingStatusSnapshot) - .then(() => logger.info("Indexing Status successfully stored in ENSDb.")), - ]), - ); + // 1. Handle ENSIndexer Public Config just once. + await handleEnsIndexerPublicConfigRecord(); + + // 2. Handle Indexing Status on recurring basis. + await handleIndexingStatusRecord(); } // Upsert ENSNode Metadata Records in a non-blocking way to diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts b/apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts similarity index 64% rename from apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts rename to apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts index 84a121c66..5efcad6cd 100644 --- a/apps/ensindexer/src/lib/ensdb/ensdb-writer-client.ts +++ b/apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts @@ -1,44 +1,18 @@ import * as schema from "@ensnode/ensnode-schema"; import type { - CrossChainIndexingStatusSnapshot, + SerializedCrossChainIndexingStatusSnapshot, SerializedENSIndexerPublicConfig, } from "@ensnode/ensnode-sdk"; import type { EnsDbClient } from "./ensdb-connection"; +import { type EnsNodeMetadata, EnsNodeMetadataKeys } from "./ensnode-metadata"; /** - * Keys used to distinguish records in `ensnode_metadata` table in the ENSDb. - */ -export const EnsNodeMetadataKeys = { - EnsIndexerPublicConfig: "ensindexer-public-config", - IndexingStatus: "indexing-status", -} as const; - -export type EnsNodeMetadataKey = (typeof EnsNodeMetadataKeys)[keyof typeof EnsNodeMetadataKeys]; - -export interface EnsNodeMetadataEnsIndexerPublicConfig { - key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; - value: SerializedENSIndexerPublicConfig; -} - -export interface EnsNodeMetadataIndexingStatus { - key: typeof EnsNodeMetadataKeys.IndexingStatus; - value: CrossChainIndexingStatusSnapshot; -} - -/** - * ENSNode Metadata - * - * Union type gathering all variants of ENSNode Metadata. - */ -export type EnsNodeMetadata = EnsNodeMetadataEnsIndexerPublicConfig | EnsNodeMetadataIndexingStatus; - -/** - * ENSDb Writer Client + * ENSDb Mutation * * The database client performing write operations. */ -export class EnsDbWriterClient { +export class EnsDbMutation { constructor(private ensDbClient: EnsDbClient) {} /** @@ -63,8 +37,8 @@ export class EnsDbWriterClient { * @throws when upsert operation failed. */ async upsertIndexingStatus( - indexingStatus: CrossChainIndexingStatusSnapshot, - ): Promise { + indexingStatus: SerializedCrossChainIndexingStatusSnapshot, + ): Promise { return this.upsertEnsNodeMetadata({ key: EnsNodeMetadataKeys.IndexingStatus, value: indexingStatus, diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-query.ts b/apps/ensindexer/src/lib/ensdb/ensdb-query.ts new file mode 100644 index 000000000..c3890a5fe --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/ensdb-query.ts @@ -0,0 +1,73 @@ +import { eq } from "drizzle-orm"; + +import * as schema from "@ensnode/ensnode-schema"; +import type { + SerializedCrossChainIndexingStatusSnapshot, + SerializedENSIndexerPublicConfig, +} from "@ensnode/ensnode-sdk"; + +import type { EnsDbClient } from "./ensdb-connection"; +import { + type EnsNodeMetadata, + type EnsNodeMetadataEnsIndexerPublicConfig, + type EnsNodeMetadataIndexingStatus, + EnsNodeMetadataKeys, +} from "./ensnode-metadata"; + +/** + * ENSDb Query + * + * The database client performing read operations. + */ +export class EnsDbQuery { + constructor(private ensDbClient: EnsDbClient) {} + + /** + * Upsert ENSIndexer Public Config + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async getEnsIndexerPublicConfig(): Promise { + return this.getEnsNodeMetadata({ + key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, + }); + } + + /** + * Upsert Indexing Status + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async getIndexingStatus(): Promise { + return this.getEnsNodeMetadata({ + key: EnsNodeMetadataKeys.IndexingStatus, + }); + } + + /** + * Get ENSNode metadata record + * + * @returns selected record in ENSDb. + * @throws when exactly one matching metadata record was not found + */ + private async getEnsNodeMetadata( + metadata: Pick, + ): Promise { + const result = await this.ensDbClient + .select() + .from(schema.ensNodeMetadata) + .where(eq(schema.ensNodeMetadata.key, metadata.key)); + + if (result.length === 0) { + return undefined; + } + + if (result.length === 1 && result[0]) { + return result[0].value as EnsNodeMetadataType["value"]; + } + + throw new Error(`There must be exactly one ENSNodeMetadata record for '${metadata.key}' key`); + } +} diff --git a/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts b/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts new file mode 100644 index 000000000..d73147171 --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts @@ -0,0 +1,31 @@ +import type { + SerializedCrossChainIndexingStatusSnapshot, + SerializedENSIndexerPublicConfig, +} from "@ensnode/ensnode-sdk"; + +/** + * Keys used to distinguish records in `ensnode_metadata` table in the ENSDb. + */ +export const EnsNodeMetadataKeys = { + EnsIndexerPublicConfig: "ensindexer-public-config", + IndexingStatus: "indexing-status", +} as const; + +export type EnsNodeMetadataKey = (typeof EnsNodeMetadataKeys)[keyof typeof EnsNodeMetadataKeys]; + +export interface EnsNodeMetadataEnsIndexerPublicConfig { + key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; + value: SerializedENSIndexerPublicConfig; +} + +export interface EnsNodeMetadataIndexingStatus { + key: typeof EnsNodeMetadataKeys.IndexingStatus; + value: SerializedCrossChainIndexingStatusSnapshot; +} + +/** + * ENSNode Metadata + * + * Union type gathering all variants of ENSNode Metadata. + */ +export type EnsNodeMetadata = EnsNodeMetadataEnsIndexerPublicConfig | EnsNodeMetadataIndexingStatus; diff --git a/apps/ensindexer/src/lib/ensdb/index.ts b/apps/ensindexer/src/lib/ensdb/index.ts index 6b0ee5449..ee6a8ff00 100644 --- a/apps/ensindexer/src/lib/ensdb/index.ts +++ b/apps/ensindexer/src/lib/ensdb/index.ts @@ -1,2 +1,3 @@ export * from "./ensdb-connection"; -export * from "./ensdb-writer-client"; +export * from "./ensdb-mutation"; +export * from "./ensdb-query"; diff --git a/apps/ensindexer/src/lib/ensindexer/client.ts b/apps/ensindexer/src/lib/ensindexer/client.ts new file mode 100644 index 000000000..89ff30561 --- /dev/null +++ b/apps/ensindexer/src/lib/ensindexer/client.ts @@ -0,0 +1,70 @@ +import type { + SerializedENSIndexerPublicConfig, + SerializedIndexingStatusResponse, +} from "@ensnode/ensnode-sdk"; + +import { makeLogger } from "@/lib/logger"; + +const logger = makeLogger("ensindexer.client"); + +export class EnsIndexerClient { + constructor(private ensIndexerUrl: URL) {} + + /** + * Wait for ENSIndexer to become healthy. + * + * @throws when the Health endpoint didn't return HTTP Status OK. + */ + public async health(): Promise { + logger.info("Fetching ENSIndexer Health status"); + + try { + await fetch(new URL("/health", this.ensIndexerUrl)); + + logger.info("ENSIndexer is healthy"); + } catch { + const errorMessage = "Health endpoint for ENSIndexer is not available yet."; + logger.error(errorMessage); + + throw new Error(errorMessage); + } + } + + /** + * Fetch ENSIndexer Public Config + * + * @returns ENSIndexer Public Config + * @throws error when fetching ENSIndexer Public Config failed + */ + public async config(): Promise { + logger.info("Fetching ENSIndexer Public Config"); + + const ensIndexerPublicConfigSerialized = await fetch(new URL("/api/config", this.ensIndexerUrl)) + .then((response) => response.json()) + .then((response) => response as unknown as SerializedENSIndexerPublicConfig); + + logger.info("Fetched ENSIndexer Public Config"); + + return ensIndexerPublicConfigSerialized; + } + + /** + * Fetch Indexing Status + * + * @returns Indexing Status when it's available and with valid. + * @throws error when Indexing Status was either not available, or invalid. + */ + public async indexingStatus(): Promise { + logger.info("Fetching Indexing Status"); + + const indexingStatusSerialized = await fetch( + new URL("/api/indexing-status", this.ensIndexerUrl), + ) + .then((response) => response.json()) + .then((response) => response as unknown as SerializedIndexingStatusResponse); + + logger.info("Fetched Indexing Status"); + + return indexingStatusSerialized; + } +} diff --git a/apps/ensindexer/src/lib/ensindexer/index.ts b/apps/ensindexer/src/lib/ensindexer/index.ts new file mode 100644 index 000000000..5ec76921e --- /dev/null +++ b/apps/ensindexer/src/lib/ensindexer/index.ts @@ -0,0 +1 @@ +export * from "./client"; From 1f362f3bfd132f80200036fdce25f6b6c1f9766b Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Wed, 17 Dec 2025 14:23:41 +0100 Subject: [PATCH 03/12] feat(ensapi): improve ENSDb Writer Worker --- .../ponder/src/ensdb-writer-worker.ts | 130 ++++++++++-------- .../src/config/compatibility.test.ts | 80 +++++++++++ apps/ensindexer/src/config/compatibility.ts | 64 +++++++++ apps/ensindexer/src/lib/ensindexer/client.ts | 14 +- apps/ensindexer/src/lib/ensindexer/index.ts | 1 + .../src/lib/ensindexer/local-client.ts | 16 +++ 6 files changed, 240 insertions(+), 65 deletions(-) create mode 100644 apps/ensindexer/src/config/compatibility.test.ts create mode 100644 apps/ensindexer/src/config/compatibility.ts create mode 100644 apps/ensindexer/src/lib/ensindexer/local-client.ts diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts index 840d8bf34..1f4ea46d0 100644 --- a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -6,55 +6,42 @@ */ import config from "@/config"; +import { secondsToMilliseconds } from "date-fns"; import pRetry from "p-retry"; import { + CrossChainIndexingStatusSnapshot, + type Duration, + ENSIndexerPublicConfig, IndexingStatusResponseCodes, OmnichainIndexingStatusIds, - type SerializedENSIndexerPublicConfig, } from "@ensnode/ensnode-sdk"; +import { validateENSIndexerPublicConfigCompatibility } from "@/config/compatibility"; import { EnsDbConnection, EnsDbMutation, EnsDbQuery } from "@/lib/ensdb"; -import { EnsIndexerClient } from "@/lib/ensindexer"; +import { ensIndexerClient, waitForEnsIndexerToBecomeHealthy } from "@/lib/ensindexer"; import { makeLogger } from "@/lib/logger"; -const logger = makeLogger("ensdb-writer-worker"); +const INDEXING_STATUS_RECORD_UPDATE_INTERVAL: Duration = 1; -const ensIndexerClient = new EnsIndexerClient(config.ensIndexerUrl); -const waitForEnsIndexerToBecomeHealthy = pRetry(async () => ensIndexerClient.health(), { - retries: 5, -}); - -function isEnsIndexerPublicConfigCompatible( - configA: ConfigType, - configB: ConfigType, -): boolean { - if ( - !configA.indexedChainIds.every((configAChainId) => - configB.indexedChainIds.includes(configAChainId), - ) - ) { - return false; - } - - if (configA.isSubgraphCompatible !== configB.isSubgraphCompatible) { - return false; - } - - if (configA.namespace !== configB.namespace) { - return false; - } - - if (!configA.plugins.every((configAPlugin) => configB.plugins.includes(configAPlugin))) { - return false; - } - - return true; -} +const logger = makeLogger("ensdb-writer-worker"); -async function upsertEnsNodeMetadataRecords() { +/** + * ENSDb Writer Worker + * + * Runs the following tasks: + * 1) On application startup, attempt to upsert serialized representation of + * {@link ENSIndexerPublicConfig} into ENSDb. + * 2) On application startup, and then on recurring basis, + * following the {@link INDEXING_STATUS_RECORD_UPDATE_INTERVAL}, attempt to + * upsert serialized representation of {@link CrossChainIndexingStatusSnapshot} + * into ENSDb. + */ +async function ensDbWriterWorker() { + // 0. Wait for ENSIndexer to become healthy before running the worker's logic await waitForEnsIndexerToBecomeHealthy; + // 1. Create ENSDb Client const ensDbConnection = new EnsDbConnection(); const ensDbClient = ensDbConnection.connect({ schemaName: config.databaseSchemaName, @@ -65,33 +52,55 @@ async function upsertEnsNodeMetadataRecords() { logger.info("ENSDb Client connected"); + // 2. Create ENSDb Query object for read operations const ensDbQuery = new EnsDbQuery(ensDbClient); + // 3. Create ENSDb Mutation object for write operations const ensDbMutation = new EnsDbMutation(ensDbClient); + /** + * Handle ENSIndexerPublicConfig Record + */ const handleEnsIndexerPublicConfigRecord = async () => { + // Read stored config and in-memory config. + // Note: we wrap read operations in pRetry to ensure all of them are + // completed successfully. const [storedConfig, inMemoryConfig] = await pRetry(() => Promise.all([ensDbQuery.getEnsIndexerPublicConfig(), ensIndexerClient.config()]), ); - if (storedConfig && !isEnsIndexerPublicConfigCompatible(storedConfig, inMemoryConfig)) { - throw new Error( - "In-memory ENSIndexerPublicConfig object is not compatible with its counterpart stored in ENSDb.", - ); + // Validate in-memory config object compatibility with the stored one, + // if the stored one is available + if (storedConfig) { + try { + validateENSIndexerPublicConfigCompatibility(storedConfig, inMemoryConfig); + } catch (error) { + const errorMessage = + "In-memory ENSIndexerPublicConfig object is not compatible with its counterpart stored in ENSDb."; + + logger.error(error, errorMessage); + + // Throw the error to terminate the ENSIndexer process due to + // found config incompatibility + throw new Error(errorMessage); + } } else { - // upsert ENSIndexerPublicConfig into ENSDb - await ensDbMutation - .upsertEnsIndexerPublicConfig(inMemoryConfig) - .then(() => logger.info("ENSIndexer Public Config successfully stored in ENSDb.")); + // Upsert ENSIndexerPublicConfig into ENSDb. + // Note: we wrap write operation in pRetry to ensure it can complete + // successfully, as there will be no other attempt. + await pRetry(() => ensDbMutation.upsertEnsIndexerPublicConfig(inMemoryConfig)); + logger.info("ENSIndexer Public Config successfully stored in ENSDb."); } }; - // TODO: clear interval on application shutdown - let _indexingStatusRefreshInterval: ReturnType; - - const handleIndexingStatusRecord = async () => { + /** + * Handle Indexing Status Record Recursively + */ + const handleIndexingStatusRecordRecursively = async () => { try { + // Read in-memory Indexing Status. const inMemoryIndexingStatus = await ensIndexerClient.indexingStatus(); + // Check if Indexing Status is available. if (inMemoryIndexingStatus.responseCode !== IndexingStatusResponseCodes.Ok) { throw new Error("Indexing Status must be available."); } @@ -99,28 +108,35 @@ async function upsertEnsNodeMetadataRecords() { const { snapshot } = inMemoryIndexingStatus.realtimeProjection; const { omnichainSnapshot } = snapshot; + // Check if Indexing Status is in expected status. if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { throw new Error("Omnichain Status must be different that 'Unstarted'."); } - // upsert ENSIndexerPublicConfig into ENSDb - await ensDbMutation - .upsertIndexingStatus(snapshot) - .then(() => logger.info("ENSIndexer Public Config successfully stored in ENSDb.")); + // Upsert ENSIndexerPublicConfig into ENSDb. + await ensDbMutation.upsertIndexingStatus(snapshot); + + logger.info("ENSIndexer Public Config successfully stored in ENSDb."); } catch (error) { + // Do nothing about this error, but having it logged. logger.error(error, "Could not upsert Indexing Status record"); } finally { - _indexingStatusRefreshInterval = setTimeout(handleIndexingStatusRecord, 1000); + // Regardless of current iteration result, + // schedule the next callback to handle Indexing Status Record. + setTimeout( + handleIndexingStatusRecordRecursively, + secondsToMilliseconds(INDEXING_STATUS_RECORD_UPDATE_INTERVAL), + ); } }; - // 1. Handle ENSIndexer Public Config just once. + // 4. Handle ENSIndexer Public Config just once. await handleEnsIndexerPublicConfigRecord(); - // 2. Handle Indexing Status on recurring basis. - await handleIndexingStatusRecord(); + // 5. Handle Indexing Status on recurring basis. + await handleIndexingStatusRecordRecursively(); } -// Upsert ENSNode Metadata Records in a non-blocking way to +// Run ENSDb Writer Worker in a non-blocking way to // allow database migrations to proceed in the background. -setTimeout(upsertEnsNodeMetadataRecords, 0); +setTimeout(ensDbWriterWorker, 0); diff --git a/apps/ensindexer/src/config/compatibility.test.ts b/apps/ensindexer/src/config/compatibility.test.ts new file mode 100644 index 000000000..8a4ecd811 --- /dev/null +++ b/apps/ensindexer/src/config/compatibility.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; + +import { ENSNamespaceIds } from "@ensnode/datasources"; +import { PluginName } from "@ensnode/ensnode-sdk"; + +import { + type ENSIndexerPublicConfigCompatibilityCheck, + validateENSIndexerPublicConfigCompatibility, +} from "./compatibility"; + +describe("ENSIndexerConfig compatibility", () => { + describe("validateENSIndexerPublicConfigCompatibility()", () => { + const config = { + indexedChainIds: [1, 10, 8453], + isSubgraphCompatible: false, + namespace: ENSNamespaceIds.Mainnet, + plugins: [PluginName.Subgraph, PluginName.Basenames, PluginName.ThreeDNS], + } satisfies ENSIndexerPublicConfigCompatibilityCheck; + + it("does not throw error when 'configB' is compatible with 'configA' ('configA' is subset of 'configB')", () => { + const configA = structuredClone(config); + + const configB = structuredClone(config); + configB.indexedChainIds.push(59144); + configB.plugins.push(PluginName.Lineanames); + + expect(() => + validateENSIndexerPublicConfigCompatibility(configA, configB), + ).not.toThrowError(); + }); + + it("throws error when 'configA.indexedChainIds' are not subset of 'configB.indexedChainIds'", () => { + const configA = structuredClone(config); + + const configB = structuredClone(config); + configB.indexedChainIds.pop(); + + expect(() => validateENSIndexerPublicConfigCompatibility(configA, configB)).toThrowError( + /'indexedChainIds' must be compatible. Stored Config 'indexedChainIds': '1, 10, 8453'. Current Config 'indexedChainIds': '1, 10'/i, + ); + }); + + it("throws error when 'configA.isSubgraphCompatible' is not same as 'configB.isSubgraphCompatible'", () => { + const configA = structuredClone(config); + + const configB = { + ...structuredClone(config), + isSubgraphCompatible: !configA.isSubgraphCompatible, + } satisfies ENSIndexerPublicConfigCompatibilityCheck; + + expect(() => validateENSIndexerPublicConfigCompatibility(configA, configB)).toThrowError( + /'isSubgraphCompatible' flag must be compatible. Stored Config 'isSubgraphCompatible' flag: 'false'. Current Config 'isSubgraphCompatible' flag: 'true'/i, + ); + }); + + it("throws error when 'configA.isSubgraphCompatible' is not same as 'configB.isSubgraphCompatible'", () => { + const configA = structuredClone(config); + + const configB = { + ...structuredClone(config), + namespace: ENSNamespaceIds.Sepolia, + } satisfies ENSIndexerPublicConfigCompatibilityCheck; + + expect(() => validateENSIndexerPublicConfigCompatibility(configA, configB)).toThrowError( + /'namespace' must be compatible. Stored Config 'namespace': 'mainnet'. Current Config 'namespace': 'sepolia'/i, + ); + }); + + it("throws error when 'configA.plugins' are not subset of 'configB.plugins'", () => { + const configA = structuredClone(config); + + const configB = structuredClone(config); + configB.plugins.pop(); + + expect(() => validateENSIndexerPublicConfigCompatibility(configA, configB)).toThrowError( + /'plugins' must be compatible. Stored Config 'plugins': 'subgraph, basenames, threedns'. Current Config 'plugins': 'subgraph, basenames'/i, + ); + }); + }); +}); diff --git a/apps/ensindexer/src/config/compatibility.ts b/apps/ensindexer/src/config/compatibility.ts new file mode 100644 index 000000000..70399a322 --- /dev/null +++ b/apps/ensindexer/src/config/compatibility.ts @@ -0,0 +1,64 @@ +import type { SerializedENSIndexerPublicConfig } from "@ensnode/ensnode-sdk"; + +export type ENSIndexerPublicConfigCompatibilityCheck = Pick< + SerializedENSIndexerPublicConfig, + "indexedChainIds" | "isSubgraphCompatible" | "namespace" | "plugins" +>; + +/** + * Validate if `configB` is compatible with `configA`, such that `configA` is + * a subset of `configB`. + * + * @throws error if 'indexedChainIds' were incompatible. + * @throws error if 'isSubgraphCompatible' flag was incompatible. + * @throws error if 'namespace' was incompatible. + * @throws error if 'plugins' were incompatible. + */ +export function validateENSIndexerPublicConfigCompatibility( + configA: ENSIndexerPublicConfigCompatibilityCheck, + configB: ENSIndexerPublicConfigCompatibilityCheck, +): void { + if ( + !configA.indexedChainIds.every((configAChainId) => + configB.indexedChainIds.includes(configAChainId), + ) + ) { + throw new Error( + [ + `'indexedChainIds' must be compatible.`, + `Stored Config 'indexedChainIds': '${configA.indexedChainIds.join(", ")}'.`, + `Current Config 'indexedChainIds': '${configB.indexedChainIds.join(", ")}'.`, + ].join(" "), + ); + } + + if (configA.isSubgraphCompatible !== configB.isSubgraphCompatible) { + throw new Error( + [ + `'isSubgraphCompatible' flag must be compatible.`, + `Stored Config 'isSubgraphCompatible' flag: '${configA.isSubgraphCompatible}'.`, + `Current Config 'isSubgraphCompatible' flag: '${configB.isSubgraphCompatible}'.`, + ].join(" "), + ); + } + + if (configA.namespace !== configB.namespace) { + throw new Error( + [ + `'namespace' must be compatible.`, + `Stored Config 'namespace': '${configA.namespace}'.`, + `Current Config 'namespace': '${configB.namespace}'.`, + ].join(" "), + ); + } + + if (!configA.plugins.every((configAPlugin) => configB.plugins.includes(configAPlugin))) { + throw new Error( + [ + `'plugins' must be compatible.`, + `Stored Config 'plugins': '${configA.plugins.join(", ")}'.`, + `Current Config 'plugins': '${configB.plugins.join(", ")}'.`, + ].join(" "), + ); + } +} diff --git a/apps/ensindexer/src/lib/ensindexer/client.ts b/apps/ensindexer/src/lib/ensindexer/client.ts index 89ff30561..671e12658 100644 --- a/apps/ensindexer/src/lib/ensindexer/client.ts +++ b/apps/ensindexer/src/lib/ensindexer/client.ts @@ -39,13 +39,13 @@ export class EnsIndexerClient { public async config(): Promise { logger.info("Fetching ENSIndexer Public Config"); - const ensIndexerPublicConfigSerialized = await fetch(new URL("/api/config", this.ensIndexerUrl)) - .then((response) => response.json()) - .then((response) => response as unknown as SerializedENSIndexerPublicConfig); + const ensIndexerPublicConfigSerialized = await fetch( + new URL("/api/config", this.ensIndexerUrl), + ).then((response) => response.json()); logger.info("Fetched ENSIndexer Public Config"); - return ensIndexerPublicConfigSerialized; + return ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig; } /** @@ -59,12 +59,10 @@ export class EnsIndexerClient { const indexingStatusSerialized = await fetch( new URL("/api/indexing-status", this.ensIndexerUrl), - ) - .then((response) => response.json()) - .then((response) => response as unknown as SerializedIndexingStatusResponse); + ).then((response) => response.json()); logger.info("Fetched Indexing Status"); - return indexingStatusSerialized; + return indexingStatusSerialized as SerializedIndexingStatusResponse; } } diff --git a/apps/ensindexer/src/lib/ensindexer/index.ts b/apps/ensindexer/src/lib/ensindexer/index.ts index 5ec76921e..51262e40d 100644 --- a/apps/ensindexer/src/lib/ensindexer/index.ts +++ b/apps/ensindexer/src/lib/ensindexer/index.ts @@ -1 +1,2 @@ export * from "./client"; +export * from "./local-client"; diff --git a/apps/ensindexer/src/lib/ensindexer/local-client.ts b/apps/ensindexer/src/lib/ensindexer/local-client.ts new file mode 100644 index 000000000..b78e8f060 --- /dev/null +++ b/apps/ensindexer/src/lib/ensindexer/local-client.ts @@ -0,0 +1,16 @@ +import config from "@/config"; + +import pRetry from "p-retry"; + +import { EnsIndexerClient } from "./client"; + +export const ensIndexerClient = new EnsIndexerClient(config.ensIndexerUrl); + +/** + * Wait for ENSIndexer to become healthy. + * + * The global promise that will only resolve after the ENSIndexer has become healthy. + */ +export const waitForEnsIndexerToBecomeHealthy = pRetry(async () => ensIndexerClient.health(), { + retries: 5, +}); From 781e85538e91e46c9fcb1c73b4fecbe38c8fbb6c Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Wed, 17 Dec 2025 14:27:44 +0100 Subject: [PATCH 04/12] docs(changeset): Includes schema for `ENSNodeMetadata`. --- .changeset/rich-buttons-shine.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/rich-buttons-shine.md diff --git a/.changeset/rich-buttons-shine.md b/.changeset/rich-buttons-shine.md new file mode 100644 index 000000000..8fd6b1102 --- /dev/null +++ b/.changeset/rich-buttons-shine.md @@ -0,0 +1,5 @@ +--- +"@ensnode/ensnode-schema": minor +--- + +Includes schema for `ENSNodeMetadata`. From d362c246aaf595f2a7c34c0f047e6d3b4788b654 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Wed, 17 Dec 2025 14:28:59 +0100 Subject: [PATCH 05/12] docs(changeset): Introduces a worker that writes serialized representations of ENSIndexer Public Config and Indexing Status to ENSDb. --- .changeset/moody-tips-run.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/moody-tips-run.md diff --git a/.changeset/moody-tips-run.md b/.changeset/moody-tips-run.md new file mode 100644 index 000000000..57a068258 --- /dev/null +++ b/.changeset/moody-tips-run.md @@ -0,0 +1,5 @@ +--- +"ensindexer": minor +--- + +Introduces a worker that writes serialized representations of ENSIndexer Public Config and Indexing Status to ENSDb. From aecaf14a1a643794d07cec0b3aac69df9b11bfae Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Wed, 17 Dec 2025 16:49:14 +0100 Subject: [PATCH 06/12] feat(ensapi): tune logs --- apps/ensindexer/ponder/src/ensdb-writer-worker.ts | 2 +- apps/ensindexer/src/lib/ensindexer/client.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts index 1f4ea46d0..c7f986ac6 100644 --- a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -116,7 +116,7 @@ async function ensDbWriterWorker() { // Upsert ENSIndexerPublicConfig into ENSDb. await ensDbMutation.upsertIndexingStatus(snapshot); - logger.info("ENSIndexer Public Config successfully stored in ENSDb."); + logger.info("Indexing Status successfully stored in ENSDb."); } catch (error) { // Do nothing about this error, but having it logged. logger.error(error, "Could not upsert Indexing Status record"); diff --git a/apps/ensindexer/src/lib/ensindexer/client.ts b/apps/ensindexer/src/lib/ensindexer/client.ts index 671e12658..d759ed165 100644 --- a/apps/ensindexer/src/lib/ensindexer/client.ts +++ b/apps/ensindexer/src/lib/ensindexer/client.ts @@ -16,12 +16,12 @@ export class EnsIndexerClient { * @throws when the Health endpoint didn't return HTTP Status OK. */ public async health(): Promise { - logger.info("Fetching ENSIndexer Health status"); + logger.debug("Fetching ENSIndexer Health status"); try { await fetch(new URL("/health", this.ensIndexerUrl)); - logger.info("ENSIndexer is healthy"); + logger.debug("Fetching ENSIndexer Health status: healthy"); } catch { const errorMessage = "Health endpoint for ENSIndexer is not available yet."; logger.error(errorMessage); @@ -37,13 +37,13 @@ export class EnsIndexerClient { * @throws error when fetching ENSIndexer Public Config failed */ public async config(): Promise { - logger.info("Fetching ENSIndexer Public Config"); + logger.debug("Fetching ENSIndexer Public Config"); const ensIndexerPublicConfigSerialized = await fetch( new URL("/api/config", this.ensIndexerUrl), ).then((response) => response.json()); - logger.info("Fetched ENSIndexer Public Config"); + logger.debug("Fetched ENSIndexer Public Config"); return ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig; } @@ -55,13 +55,13 @@ export class EnsIndexerClient { * @throws error when Indexing Status was either not available, or invalid. */ public async indexingStatus(): Promise { - logger.info("Fetching Indexing Status"); + logger.debug("Fetching Indexing Status"); const indexingStatusSerialized = await fetch( new URL("/api/indexing-status", this.ensIndexerUrl), ).then((response) => response.json()); - logger.info("Fetched Indexing Status"); + logger.debug("Fetched Indexing Status"); return indexingStatusSerialized as SerializedIndexingStatusResponse; } From 623260f8c2f3b21a58824dbd2fec1d865dead0ab Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Wed, 17 Dec 2025 21:09:23 +0100 Subject: [PATCH 07/12] apply pr feedback Remove logs from ENSIndexerClient class --- apps/ensindexer/src/lib/ensindexer/client.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/apps/ensindexer/src/lib/ensindexer/client.ts b/apps/ensindexer/src/lib/ensindexer/client.ts index d759ed165..9d1a662d5 100644 --- a/apps/ensindexer/src/lib/ensindexer/client.ts +++ b/apps/ensindexer/src/lib/ensindexer/client.ts @@ -3,10 +3,6 @@ import type { SerializedIndexingStatusResponse, } from "@ensnode/ensnode-sdk"; -import { makeLogger } from "@/lib/logger"; - -const logger = makeLogger("ensindexer.client"); - export class EnsIndexerClient { constructor(private ensIndexerUrl: URL) {} @@ -16,15 +12,10 @@ export class EnsIndexerClient { * @throws when the Health endpoint didn't return HTTP Status OK. */ public async health(): Promise { - logger.debug("Fetching ENSIndexer Health status"); - try { await fetch(new URL("/health", this.ensIndexerUrl)); - - logger.debug("Fetching ENSIndexer Health status: healthy"); } catch { const errorMessage = "Health endpoint for ENSIndexer is not available yet."; - logger.error(errorMessage); throw new Error(errorMessage); } @@ -37,14 +28,10 @@ export class EnsIndexerClient { * @throws error when fetching ENSIndexer Public Config failed */ public async config(): Promise { - logger.debug("Fetching ENSIndexer Public Config"); - const ensIndexerPublicConfigSerialized = await fetch( new URL("/api/config", this.ensIndexerUrl), ).then((response) => response.json()); - logger.debug("Fetched ENSIndexer Public Config"); - return ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig; } @@ -55,14 +42,10 @@ export class EnsIndexerClient { * @throws error when Indexing Status was either not available, or invalid. */ public async indexingStatus(): Promise { - logger.debug("Fetching Indexing Status"); - const indexingStatusSerialized = await fetch( new URL("/api/indexing-status", this.ensIndexerUrl), ).then((response) => response.json()); - logger.debug("Fetched Indexing Status"); - return indexingStatusSerialized as SerializedIndexingStatusResponse; } } From dd19ce9a9c8afbeee220703c683b61195a197e99 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Wed, 17 Dec 2025 21:56:54 +0100 Subject: [PATCH 08/12] apply pr feedback consolidate ensdb modules; replace pino logger with native console logger; simplify ensdb worker logic --- apps/ensapi/package.json | 2 +- apps/ensindexer/package.json | 3 +- .../ponder/src/ensdb-writer-worker.ts | 71 ++++---- apps/ensindexer/src/config/compatibility.ts | 5 +- apps/ensindexer/src/lib/ensdb/drizzle.ts | 30 +--- apps/ensindexer/src/lib/ensdb/ensdb-client.ts | 161 ++++++++++++++++++ .../src/lib/ensdb/ensdb-connection.ts | 63 ------- .../src/lib/ensdb/ensdb-mutation.ts | 75 -------- apps/ensindexer/src/lib/ensdb/ensdb-query.ts | 73 -------- apps/ensindexer/src/lib/ensdb/index.ts | 4 +- apps/ensindexer/src/lib/logger.ts | 24 --- pnpm-lock.yaml | 76 ++++++++- pnpm-workspace.yaml | 2 + 13 files changed, 273 insertions(+), 316 deletions(-) create mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-client.ts delete mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-connection.ts delete mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts delete mode 100644 apps/ensindexer/src/lib/ensdb/ensdb-query.ts delete mode 100644 apps/ensindexer/src/lib/logger.ts diff --git a/apps/ensapi/package.json b/apps/ensapi/package.json index f9df47ac7..3a36d57fa 100644 --- a/apps/ensapi/package.json +++ b/apps/ensapi/package.json @@ -43,7 +43,7 @@ "date-fns": "catalog:", "drizzle-orm": "catalog:", "hono": "catalog:", - "p-memoize": "^8.0.0", + "p-memoize": "catalog:", "p-retry": "catalog:", "pg-connection-string": "catalog:", "pino": "catalog:", diff --git a/apps/ensindexer/package.json b/apps/ensindexer/package.json index a23fbcdf0..3b31c32a3 100644 --- a/apps/ensindexer/package.json +++ b/apps/ensindexer/package.json @@ -29,15 +29,14 @@ "@ensnode/ensnode-sdk": "workspace:*", "@ensnode/ensrainbow-sdk": "workspace:*", "@ensnode/ponder-metadata": "workspace:*", + "@ponder/client": "catalog:", "caip": "catalog:", "date-fns": "catalog:", "deepmerge-ts": "^7.1.5", "dns-packet": "^5.6.1", "drizzle-orm": "catalog:", "p-retry": "catalog:", - "pino": "catalog:", "pg-connection-string": "catalog:", - "pg": "8.16.3", "hono": "catalog:", "ponder": "catalog:", "viem": "catalog:", diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts index c7f986ac6..80b7c1166 100644 --- a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -4,8 +4,6 @@ * - Indexing Status * into the ENSDb. */ -import config from "@/config"; - import { secondsToMilliseconds } from "date-fns"; import pRetry from "p-retry"; @@ -18,14 +16,11 @@ import { } from "@ensnode/ensnode-sdk"; import { validateENSIndexerPublicConfigCompatibility } from "@/config/compatibility"; -import { EnsDbConnection, EnsDbMutation, EnsDbQuery } from "@/lib/ensdb"; +import { EnsDbClient } from "@/lib/ensdb"; import { ensIndexerClient, waitForEnsIndexerToBecomeHealthy } from "@/lib/ensindexer"; -import { makeLogger } from "@/lib/logger"; const INDEXING_STATUS_RECORD_UPDATE_INTERVAL: Duration = 1; -const logger = makeLogger("ensdb-writer-worker"); - /** * ENSDb Writer Worker * @@ -38,35 +33,27 @@ const logger = makeLogger("ensdb-writer-worker"); * into ENSDb. */ async function ensDbWriterWorker() { + console.log("ENSDb Writer Worker: waiting for ENSIndexer to become healthy."); + // 0. Wait for ENSIndexer to become healthy before running the worker's logic await waitForEnsIndexerToBecomeHealthy; - // 1. Create ENSDb Client - const ensDbConnection = new EnsDbConnection(); - const ensDbClient = ensDbConnection.connect({ - schemaName: config.databaseSchemaName, - poolConfig: { - connectionString: config.databaseUrl, - }, - }); - - logger.info("ENSDb Client connected"); + console.log("ENSDb Writer Worker: ENSIndexer is healthy, starting tasks."); - // 2. Create ENSDb Query object for read operations - const ensDbQuery = new EnsDbQuery(ensDbClient); - // 3. Create ENSDb Mutation object for write operations - const ensDbMutation = new EnsDbMutation(ensDbClient); + // 1. Create ENSDb Client + const ensDbClient = new EnsDbClient(); /** * Handle ENSIndexerPublicConfig Record */ const handleEnsIndexerPublicConfigRecord = async () => { // Read stored config and in-memory config. - // Note: we wrap read operations in pRetry to ensure all of them are + // Note: we wrap each operation in pRetry to ensure all of them can be // completed successfully. - const [storedConfig, inMemoryConfig] = await pRetry(() => - Promise.all([ensDbQuery.getEnsIndexerPublicConfig(), ensIndexerClient.config()]), - ); + const [storedConfig, inMemoryConfig] = await Promise.all([ + pRetry(() => ensDbClient.getEnsIndexerPublicConfig()), + pRetry(() => ensIndexerClient.config()), + ]); // Validate in-memory config object compatibility with the stored one, // if the stored one is available @@ -74,21 +61,17 @@ async function ensDbWriterWorker() { try { validateENSIndexerPublicConfigCompatibility(storedConfig, inMemoryConfig); } catch (error) { - const errorMessage = - "In-memory ENSIndexerPublicConfig object is not compatible with its counterpart stored in ENSDb."; - - logger.error(error, errorMessage); + const errorMessage = `In-memory ENSIndexerPublicConfig object is not compatible with its counterpart stored in ENSDb.`; // Throw the error to terminate the ENSIndexer process due to // found config incompatibility - throw new Error(errorMessage); + throw new Error(errorMessage, { + cause: error, + }); } } else { // Upsert ENSIndexerPublicConfig into ENSDb. - // Note: we wrap write operation in pRetry to ensure it can complete - // successfully, as there will be no other attempt. - await pRetry(() => ensDbMutation.upsertEnsIndexerPublicConfig(inMemoryConfig)); - logger.info("ENSIndexer Public Config successfully stored in ENSDb."); + await ensDbClient.upsertEnsIndexerPublicConfig(inMemoryConfig); } }; @@ -110,16 +93,14 @@ async function ensDbWriterWorker() { // Check if Indexing Status is in expected status. if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { - throw new Error("Omnichain Status must be different that 'Unstarted'."); + throw new Error("Omnichain Status must be different than 'Unstarted'."); } // Upsert ENSIndexerPublicConfig into ENSDb. - await ensDbMutation.upsertIndexingStatus(snapshot); - - logger.info("Indexing Status successfully stored in ENSDb."); + await ensDbClient.upsertIndexingStatus(snapshot); } catch (error) { // Do nothing about this error, but having it logged. - logger.error(error, "Could not upsert Indexing Status record"); + console.error(error, "Could not upsert Indexing Status record"); } finally { // Regardless of current iteration result, // schedule the next callback to handle Indexing Status Record. @@ -131,12 +112,20 @@ async function ensDbWriterWorker() { }; // 4. Handle ENSIndexer Public Config just once. - await handleEnsIndexerPublicConfigRecord(); + console.log("Task: store ENSIndexer Public Config in ENSDb."); + await handleEnsIndexerPublicConfigRecord().then(() => + console.log("ENSIndexer Public Config successfully stored in ENSDb."), + ); // 5. Handle Indexing Status on recurring basis. - await handleIndexingStatusRecordRecursively(); + console.log("Task: store Indexing Status in ENSDb."); + await handleIndexingStatusRecordRecursively().then(() => + console.log("Indexing Status successfully stored in ENSDb."), + ); } // Run ENSDb Writer Worker in a non-blocking way to // allow database migrations to proceed in the background. -setTimeout(ensDbWriterWorker, 0); +ensDbWriterWorker().catch((error) => + console.error("ENSDb Writer Worker failed to perform its tasks", error), +); diff --git a/apps/ensindexer/src/config/compatibility.ts b/apps/ensindexer/src/config/compatibility.ts index 70399a322..20a249097 100644 --- a/apps/ensindexer/src/config/compatibility.ts +++ b/apps/ensindexer/src/config/compatibility.ts @@ -9,10 +9,7 @@ export type ENSIndexerPublicConfigCompatibilityCheck = Pick< * Validate if `configB` is compatible with `configA`, such that `configA` is * a subset of `configB`. * - * @throws error if 'indexedChainIds' were incompatible. - * @throws error if 'isSubgraphCompatible' flag was incompatible. - * @throws error if 'namespace' was incompatible. - * @throws error if 'plugins' were incompatible. + * @throws error if configs are incompatible. */ export function validateENSIndexerPublicConfigCompatibility( configA: ENSIndexerPublicConfigCompatibilityCheck, diff --git a/apps/ensindexer/src/lib/ensdb/drizzle.ts b/apps/ensindexer/src/lib/ensdb/drizzle.ts index 50700aa88..64f3b5db1 100644 --- a/apps/ensindexer/src/lib/ensdb/drizzle.ts +++ b/apps/ensindexer/src/lib/ensdb/drizzle.ts @@ -1,41 +1,25 @@ // This file was copied 1-to-1 from ENSApi. +// TODO: deduplicate with apps/ensapi/src/lib/handlers/drizzle.ts when ensnode nodejs internal package is created -import { isTable, Table } from "drizzle-orm"; -import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; -import { isPgEnum } from "drizzle-orm/pg-core"; -import type { Pool } from "pg"; +import { setDatabaseSchema } from "@ponder/client"; +import { drizzle } from "drizzle-orm/node-postgres"; type Schema = { [name: string]: unknown }; -// https://github.com/ponder-sh/ponder/blob/f7f6444ab8d1a870fe6492023941091df7b7cddf/packages/client/src/index.ts#L226C1-L239C3 -const setDatabaseSchema = (schema: T, schemaName: string) => { - for (const table of Object.values(schema)) { - if (isTable(table)) { - // @ts-expect-error - table[Table.Symbol.Schema] = schemaName; - } else if (isPgEnum(table)) { - // @ts-expect-error - table.schema = schemaName; - } - } -}; - /** * Makes a Drizzle DB object. */ export const makeDrizzle = ({ schema, - connectionPool, + databaseUrl, databaseSchema, }: { schema: SCHEMA; - connectionPool: Pool; + databaseUrl: string; databaseSchema: string; -}): NodePgDatabase & { - $client: Pool; -} => { +}) => { // monkeypatch schema onto tables setDatabaseSchema(schema, databaseSchema); - return drizzle(connectionPool, { schema, casing: "snake_case" }); + return drizzle(databaseUrl, { schema, casing: "snake_case" }); }; diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-client.ts b/apps/ensindexer/src/lib/ensdb/ensdb-client.ts new file mode 100644 index 000000000..a62bd4e71 --- /dev/null +++ b/apps/ensindexer/src/lib/ensdb/ensdb-client.ts @@ -0,0 +1,161 @@ +import config from "@/config"; + +import { eq } from "drizzle-orm/sql"; + +import * as schema from "@ensnode/ensnode-schema"; +import type { + SerializedCrossChainIndexingStatusSnapshot, + SerializedENSIndexerPublicConfig, +} from "@ensnode/ensnode-sdk"; + +import { makeDrizzle } from "./drizzle"; +import { + type EnsNodeMetadata, + type EnsNodeMetadataEnsIndexerPublicConfig, + type EnsNodeMetadataIndexingStatus, + EnsNodeMetadataKeys, +} from "./ensnode-metadata"; + +/** + * ENSDb Client Query + * + Includes methods for reading from ENSDb. + */ +export interface EnsDbClientQuery { + getEnsIndexerPublicConfig(): Promise; + + getIndexingStatus(): Promise; +} + +/** + * ENSDb Client Mutation + * + * Includes methods for writing into ENSDb. + */ +export interface EnsDbClientMutation { + upsertEnsIndexerPublicConfig( + ensIndexerPublicConfig: SerializedENSIndexerPublicConfig, + ): Promise; + + upsertIndexingStatus( + indexingStatus: SerializedCrossChainIndexingStatusSnapshot, + ): Promise; +} + +/** + * ENSDb Client + */ +export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { + #db = makeDrizzle({ + databaseSchema: config.databaseSchemaName, + databaseUrl: config.databaseUrl, + schema, + }); + + /** + * Upsert ENSIndexer Public Config + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async getEnsIndexerPublicConfig(): Promise { + return this.getEnsNodeMetadata({ + key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, + }); + } + + /** + * Upsert Indexing Status + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async getIndexingStatus(): Promise { + return this.getEnsNodeMetadata({ + key: EnsNodeMetadataKeys.IndexingStatus, + }); + } + + /** + * Upsert ENSIndexer Public Config + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async upsertEnsIndexerPublicConfig( + ensIndexerPublicConfig: SerializedENSIndexerPublicConfig, + ): Promise { + return this.upsertEnsNodeMetadata({ + key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, + value: ensIndexerPublicConfig, + }); + } + + /** + * Upsert Indexing Status + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + async upsertIndexingStatus( + indexingStatus: SerializedCrossChainIndexingStatusSnapshot, + ): Promise { + return this.upsertEnsNodeMetadata({ + key: EnsNodeMetadataKeys.IndexingStatus, + value: indexingStatus, + }); + } + + /** + * Get ENSNode metadata record + * + * @returns selected record in ENSDb. + * @throws when exactly one matching metadata record was not found + */ + private async getEnsNodeMetadata( + metadata: Pick, + ): Promise { + const result = await this.#db + .select() + .from(schema.ensNodeMetadata) + .where(eq(schema.ensNodeMetadata.key, metadata.key)); + + if (result.length === 0) { + return undefined; + } + + if (result.length === 1 && result[0]) { + return result[0].value as EnsNodeMetadataType["value"]; + } + + throw new Error(`There must be exactly one ENSNodeMetadata record for '${metadata.key}' key`); + } + + /** + * Upsert ENSNode metadata + * + * @returns updated record in ENSDb. + * @throws when upsert operation failed. + */ + private async upsertEnsNodeMetadata< + EnsNodeMetadataType extends EnsNodeMetadata = EnsNodeMetadata, + >(metadata: EnsNodeMetadataType): Promise { + const [result] = await this.#db + .insert(schema.ensNodeMetadata) + .values({ + key: metadata.key, + value: metadata.value, + }) + .onConflictDoUpdate({ + target: schema.ensNodeMetadata.key, + set: { value: metadata.value }, + }) + .returning({ value: schema.ensNodeMetadata.value }); + + if (!result) { + throw new Error(`Failed to upsert metadata for key: ${metadata.key}`); + } + + return result.value as EnsNodeMetadataType["value"]; + } +} diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-connection.ts b/apps/ensindexer/src/lib/ensdb/ensdb-connection.ts deleted file mode 100644 index 6ab1baed3..000000000 --- a/apps/ensindexer/src/lib/ensdb/ensdb-connection.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { NodePgDatabase } from "drizzle-orm/node-postgres"; -import { Pool, type PoolConfig } from "pg"; - -import * as schema from "@ensnode/ensnode-schema"; - -import { makeDrizzle } from "@/lib/ensdb/drizzle"; - -export type EnsDbClient = NodePgDatabase; - -export interface EnsDbConnectionOptions { - schemaName: string; - poolConfig: PoolConfig; -} - -/** - * ENSDb Connection - * - * Enables: - * - Connecting to ENSDb instance. - * - Disconnecting from ENSDb instance. - * - * Uses application connection pool for improved performance. - */ -export class EnsDbConnection { - #connectionPool: Pool | undefined; - #ensDbClient: EnsDbClient | undefined; - - /** - * Connect to ENSDb instance. - * - * @returns ENSDb Client - */ - connect({ schemaName, poolConfig }: EnsDbConnectionOptions): EnsDbClient { - if (this.#connectionPool) { - throw new Error("ENSDb already connected. Call disconnect() first."); - } - - this.#connectionPool = new Pool(poolConfig); - - this.#ensDbClient = makeDrizzle({ - connectionPool: this.#connectionPool, - databaseSchema: schemaName, - schema, - }); - - return this.#ensDbClient; - } - - /** - * Disconnect to ENSDb instance. - * - * Call this function to free up resources. - */ - async disconnect(): Promise { - if (!this.#connectionPool) return; - - // Free up resources - await this.#connectionPool.end(); - - this.#connectionPool = undefined; - this.#ensDbClient = undefined; - } -} diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts b/apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts deleted file mode 100644 index 5efcad6cd..000000000 --- a/apps/ensindexer/src/lib/ensdb/ensdb-mutation.ts +++ /dev/null @@ -1,75 +0,0 @@ -import * as schema from "@ensnode/ensnode-schema"; -import type { - SerializedCrossChainIndexingStatusSnapshot, - SerializedENSIndexerPublicConfig, -} from "@ensnode/ensnode-sdk"; - -import type { EnsDbClient } from "./ensdb-connection"; -import { type EnsNodeMetadata, EnsNodeMetadataKeys } from "./ensnode-metadata"; - -/** - * ENSDb Mutation - * - * The database client performing write operations. - */ -export class EnsDbMutation { - constructor(private ensDbClient: EnsDbClient) {} - - /** - * Upsert ENSIndexer Public Config - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ - async upsertEnsIndexerPublicConfig( - ensIndexerPublicConfig: SerializedENSIndexerPublicConfig, - ): Promise { - return this.upsertEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, - value: ensIndexerPublicConfig, - }); - } - - /** - * Upsert Indexing Status - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ - async upsertIndexingStatus( - indexingStatus: SerializedCrossChainIndexingStatusSnapshot, - ): Promise { - return this.upsertEnsNodeMetadata({ - key: EnsNodeMetadataKeys.IndexingStatus, - value: indexingStatus, - }); - } - - /** - * Upsert ENSNode metadata - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ - private async upsertEnsNodeMetadata< - EnsNodeMetadataType extends EnsNodeMetadata = EnsNodeMetadata, - >(metadata: EnsNodeMetadataType): Promise { - const [result] = await this.ensDbClient - .insert(schema.ensNodeMetadata) - .values({ - key: metadata.key, - value: metadata.value, - }) - .onConflictDoUpdate({ - target: schema.ensNodeMetadata.key, - set: { value: metadata.value }, - }) - .returning({ value: schema.ensNodeMetadata.value }); - - if (!result) { - throw new Error(`Failed to upsert metadata for key: ${metadata.key}`); - } - - return result.value as EnsNodeMetadataType["value"]; - } -} diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-query.ts b/apps/ensindexer/src/lib/ensdb/ensdb-query.ts deleted file mode 100644 index c3890a5fe..000000000 --- a/apps/ensindexer/src/lib/ensdb/ensdb-query.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { eq } from "drizzle-orm"; - -import * as schema from "@ensnode/ensnode-schema"; -import type { - SerializedCrossChainIndexingStatusSnapshot, - SerializedENSIndexerPublicConfig, -} from "@ensnode/ensnode-sdk"; - -import type { EnsDbClient } from "./ensdb-connection"; -import { - type EnsNodeMetadata, - type EnsNodeMetadataEnsIndexerPublicConfig, - type EnsNodeMetadataIndexingStatus, - EnsNodeMetadataKeys, -} from "./ensnode-metadata"; - -/** - * ENSDb Query - * - * The database client performing read operations. - */ -export class EnsDbQuery { - constructor(private ensDbClient: EnsDbClient) {} - - /** - * Upsert ENSIndexer Public Config - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ - async getEnsIndexerPublicConfig(): Promise { - return this.getEnsNodeMetadata({ - key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, - }); - } - - /** - * Upsert Indexing Status - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ - async getIndexingStatus(): Promise { - return this.getEnsNodeMetadata({ - key: EnsNodeMetadataKeys.IndexingStatus, - }); - } - - /** - * Get ENSNode metadata record - * - * @returns selected record in ENSDb. - * @throws when exactly one matching metadata record was not found - */ - private async getEnsNodeMetadata( - metadata: Pick, - ): Promise { - const result = await this.ensDbClient - .select() - .from(schema.ensNodeMetadata) - .where(eq(schema.ensNodeMetadata.key, metadata.key)); - - if (result.length === 0) { - return undefined; - } - - if (result.length === 1 && result[0]) { - return result[0].value as EnsNodeMetadataType["value"]; - } - - throw new Error(`There must be exactly one ENSNodeMetadata record for '${metadata.key}' key`); - } -} diff --git a/apps/ensindexer/src/lib/ensdb/index.ts b/apps/ensindexer/src/lib/ensdb/index.ts index ee6a8ff00..b499bfc1c 100644 --- a/apps/ensindexer/src/lib/ensdb/index.ts +++ b/apps/ensindexer/src/lib/ensdb/index.ts @@ -1,3 +1 @@ -export * from "./ensdb-connection"; -export * from "./ensdb-mutation"; -export * from "./ensdb-query"; +export * from "./ensdb-client"; diff --git a/apps/ensindexer/src/lib/logger.ts b/apps/ensindexer/src/lib/logger.ts deleted file mode 100644 index eec3ee1c8..000000000 --- a/apps/ensindexer/src/lib/logger.ts +++ /dev/null @@ -1,24 +0,0 @@ -// This file was copied 1-to-1 from ENSApi. -import pino from "pino"; - -import { getLogLevelFromEnv, type LogLevel } from "@ensnode/ensnode-sdk/internal"; - -const DEFAULT_LOG_LEVEL: LogLevel = "info"; - -const logger = pino({ - level: getLogLevelFromEnv({ LOG_LEVEL: process.env.LOG_LEVEL }, DEFAULT_LOG_LEVEL), - transport: - process.env.NODE_ENV === "production" - ? undefined - : { - target: "pino-pretty", - options: { - colorize: true, - ignore: "pid,hostname", - }, - }, -}); - -export const makeLogger = (scope: string) => logger.child({ scope }); - -export default logger; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 55ce1ddc8..cd95d5fab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,9 @@ catalogs: '@namehash/namekit-react': specifier: 0.12.0 version: 0.12.0 + '@ponder/client': + specifier: 0.13.14 + version: 0.13.14 '@ponder/utils': specifier: 0.2.14 version: 0.2.14 @@ -45,6 +48,9 @@ catalogs: hono: specifier: ^4.10.2 version: 4.10.3 + p-memoize: + specifier: ^8.0.0 + version: 8.0.0 p-retry: specifier: ^7.1.1 version: 7.1.1 @@ -347,7 +353,7 @@ importers: specifier: 'catalog:' version: 4.10.3 p-memoize: - specifier: ^8.0.0 + specifier: 'catalog:' version: 8.0.0 p-retry: specifier: 'catalog:' @@ -407,6 +413,9 @@ importers: '@ensnode/ponder-metadata': specifier: workspace:* version: link:../../packages/ponder-metadata + '@ponder/client': + specifier: 'catalog:' + version: 0.13.14(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3)(typescript@5.9.3) caip: specifier: 'catalog:' version: 1.1.1 @@ -428,15 +437,9 @@ importers: p-retry: specifier: 'catalog:' version: 7.1.1 - pg: - specifier: 8.16.3 - version: 8.16.3 pg-connection-string: specifier: 'catalog:' version: 2.9.1 - pino: - specifier: 'catalog:' - version: 10.1.0 ponder: specifier: 'catalog:' version: 0.13.16(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/node@22.18.13)(@types/pg@8.16.0)(hono@4.10.3)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.20.6)(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))(yaml@2.8.1)(zod@3.25.76) @@ -2427,6 +2430,14 @@ packages: '@pinojs/redact@0.4.0': resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@ponder/client@0.13.14': + resolution: {integrity: sha512-T1hC2wP+47nJA+arDfCdaqaJGyj3z7QSHtqMv2ojy+nA0T5mHACs8kweWzsyxrI6H0jnuBgDviNijs9wUlo5Xw==} + peerDependencies: + typescript: '>=5.0.4' + peerDependenciesMeta: + typescript: + optional: true + '@ponder/utils@0.2.14': resolution: {integrity: sha512-O4t14Hb6/tVcD0WoS13ghFnDntP6x33/DDvA+sd0tRjemzS+Cne4YTkXl9TKW3AawBIEwMjGrGbAn82C8gXQWQ==} peerDependencies: @@ -4732,6 +4743,14 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + eventsource-parser@3.0.6: + resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} + engines: {node: '>=18.0.0'} + + eventsource@3.0.7: + resolution: {integrity: sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==} + engines: {node: '>=18.0.0'} + execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} engines: {node: '>=10'} @@ -9732,6 +9751,43 @@ snapshots: '@pinojs/redact@0.4.0': {} + '@ponder/client@0.13.14(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3)(typescript@5.9.3)': + dependencies: + drizzle-orm: 0.41.0(@electric-sql/pglite@0.2.13)(@opentelemetry/api@1.9.0(patch_hash=4b2adeefaf7c22f9987d0a125d69cab900719bec7ed7636648bea6947107033a))(@types/pg@8.16.0)(kysely@0.26.3)(pg@8.16.3) + eventsource: 3.0.7 + superjson: 2.2.5 + optionalDependencies: + typescript: 5.9.3 + transitivePeerDependencies: + - '@aws-sdk/client-rds-data' + - '@cloudflare/workers-types' + - '@electric-sql/pglite' + - '@libsql/client' + - '@libsql/client-wasm' + - '@neondatabase/serverless' + - '@op-engineering/op-sqlite' + - '@opentelemetry/api' + - '@planetscale/database' + - '@prisma/client' + - '@tidbcloud/serverless' + - '@types/better-sqlite3' + - '@types/pg' + - '@types/sql.js' + - '@vercel/postgres' + - '@xata.io/client' + - better-sqlite3 + - bun-types + - expo-sqlite + - gel + - knex + - kysely + - mysql2 + - pg + - postgres + - prisma + - sql.js + - sqlite3 + '@ponder/utils@0.2.14(typescript@5.9.3)(viem@2.38.5(typescript@5.9.3)(zod@3.25.76))': dependencies: viem: 2.38.5(typescript@5.9.3)(zod@3.25.76) @@ -12171,6 +12227,12 @@ snapshots: events@3.3.0: {} + eventsource-parser@3.0.6: {} + + eventsource@3.0.7: + dependencies: + eventsource-parser: 3.0.6 + execa@5.1.1: dependencies: cross-spawn: 7.0.6 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b2ed6d731..119de7253 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -8,6 +8,7 @@ catalog: "@astrojs/react": ^4.4.1 "@astrojs/tailwind": ^6.0.2 "@namehash/namekit-react": 0.12.0 + "@ponder/client": 0.13.14 "@ponder/utils": 0.2.14 "@testing-library/react": ^16.3.0 "@types/node": 22.18.13 @@ -18,6 +19,7 @@ catalog: date-fns: 4.1.0 drizzle-orm: "=0.41.0" hono: ^4.10.2 + p-memoize: ^8.0.0 p-retry: ^7.1.1 pg-connection-string: ^2.9.1 pino: 10.1.0 From e300dd9d0a56e55127aa50beba324feb4647d85b Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Thu, 18 Dec 2025 12:38:01 +0100 Subject: [PATCH 09/12] Update EnsDbClient and EnsIndexerClient with serialization/deserialization logic --- apps/ensindexer/src/config/compatibility.ts | 14 +-- apps/ensindexer/src/lib/ensdb/ensdb-client.ts | 88 +++++++++---------- .../src/lib/ensdb/ensnode-metadata.ts | 25 ++++++ apps/ensindexer/src/lib/ensindexer/client.ts | 22 +++-- 4 files changed, 92 insertions(+), 57 deletions(-) diff --git a/apps/ensindexer/src/config/compatibility.ts b/apps/ensindexer/src/config/compatibility.ts index 20a249097..be6ccb031 100644 --- a/apps/ensindexer/src/config/compatibility.ts +++ b/apps/ensindexer/src/config/compatibility.ts @@ -1,7 +1,7 @@ -import type { SerializedENSIndexerPublicConfig } from "@ensnode/ensnode-sdk"; +import type { ENSIndexerPublicConfig } from "@ensnode/ensnode-sdk"; export type ENSIndexerPublicConfigCompatibilityCheck = Pick< - SerializedENSIndexerPublicConfig, + ENSIndexerPublicConfig, "indexedChainIds" | "isSubgraphCompatible" | "namespace" | "plugins" >; @@ -15,16 +15,18 @@ export function validateENSIndexerPublicConfigCompatibility( configA: ENSIndexerPublicConfigCompatibilityCheck, configB: ENSIndexerPublicConfigCompatibilityCheck, ): void { + const configAIndexedChainIds = Array.from(configA.indexedChainIds); + const configBIndexedChainIds = Array.from(configB.indexedChainIds); if ( - !configA.indexedChainIds.every((configAChainId) => - configB.indexedChainIds.includes(configAChainId), + !configAIndexedChainIds.every((configAChainId) => + configBIndexedChainIds.includes(configAChainId), ) ) { throw new Error( [ `'indexedChainIds' must be compatible.`, - `Stored Config 'indexedChainIds': '${configA.indexedChainIds.join(", ")}'.`, - `Current Config 'indexedChainIds': '${configB.indexedChainIds.join(", ")}'.`, + `Stored Config 'indexedChainIds': '${configAIndexedChainIds.join(", ")}'.`, + `Current Config 'indexedChainIds': '${configBIndexedChainIds.join(", ")}'.`, ].join(" "), ); } diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-client.ts b/apps/ensindexer/src/lib/ensdb/ensdb-client.ts index a62bd4e71..f4fcb234c 100644 --- a/apps/ensindexer/src/lib/ensdb/ensdb-client.ts +++ b/apps/ensindexer/src/lib/ensdb/ensdb-client.ts @@ -3,17 +3,21 @@ import config from "@/config"; import { eq } from "drizzle-orm/sql"; import * as schema from "@ensnode/ensnode-schema"; -import type { - SerializedCrossChainIndexingStatusSnapshot, - SerializedENSIndexerPublicConfig, +import { + type CrossChainIndexingStatusSnapshot, + deserializeCrossChainIndexingStatusSnapshot, + deserializeENSIndexerPublicConfig, + type ENSIndexerPublicConfig, + serializeCrossChainIndexingStatusSnapshotOmnichain, + serializeENSIndexerPublicConfig, } from "@ensnode/ensnode-sdk"; import { makeDrizzle } from "./drizzle"; import { - type EnsNodeMetadata, - type EnsNodeMetadataEnsIndexerPublicConfig, - type EnsNodeMetadataIndexingStatus, EnsNodeMetadataKeys, + type SerializedEnsNodeMetadata, + type SerializedEnsNodeMetadataEnsIndexerPublicConfig, + type SerializedEnsNodeMetadataIndexingStatus, } from "./ensnode-metadata"; /** @@ -22,9 +26,9 @@ import { Includes methods for reading from ENSDb. */ export interface EnsDbClientQuery { - getEnsIndexerPublicConfig(): Promise; + getEnsIndexerPublicConfig(): Promise; - getIndexingStatus(): Promise; + getIndexingStatus(): Promise; } /** @@ -33,13 +37,9 @@ export interface EnsDbClientQuery { * Includes methods for writing into ENSDb. */ export interface EnsDbClientMutation { - upsertEnsIndexerPublicConfig( - ensIndexerPublicConfig: SerializedENSIndexerPublicConfig, - ): Promise; + upsertEnsIndexerPublicConfig(ensIndexerPublicConfig: ENSIndexerPublicConfig): Promise; - upsertIndexingStatus( - indexingStatus: SerializedCrossChainIndexingStatusSnapshot, - ): Promise; + upsertIndexingStatus(indexingStatus: CrossChainIndexingStatusSnapshot): Promise; } /** @@ -58,10 +58,16 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { * @returns updated record in ENSDb. * @throws when upsert operation failed. */ - async getEnsIndexerPublicConfig(): Promise { - return this.getEnsNodeMetadata({ + async getEnsIndexerPublicConfig(): Promise { + const record = await this.getEnsNodeMetadata({ key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, }); + + if (!record) { + return undefined; + } + + return deserializeENSIndexerPublicConfig(record); } /** @@ -70,39 +76,41 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { * @returns updated record in ENSDb. * @throws when upsert operation failed. */ - async getIndexingStatus(): Promise { - return this.getEnsNodeMetadata({ + async getIndexingStatus(): Promise { + const record = await this.getEnsNodeMetadata({ key: EnsNodeMetadataKeys.IndexingStatus, }); + + if (!record) { + return undefined; + } + + return deserializeCrossChainIndexingStatusSnapshot(record); } /** * Upsert ENSIndexer Public Config * - * @returns updated record in ENSDb. * @throws when upsert operation failed. */ async upsertEnsIndexerPublicConfig( - ensIndexerPublicConfig: SerializedENSIndexerPublicConfig, - ): Promise { - return this.upsertEnsNodeMetadata({ + ensIndexerPublicConfig: ENSIndexerPublicConfig, + ): Promise { + await this.upsertEnsNodeMetadata({ key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, - value: ensIndexerPublicConfig, + value: serializeENSIndexerPublicConfig(ensIndexerPublicConfig), }); } /** * Upsert Indexing Status * - * @returns updated record in ENSDb. * @throws when upsert operation failed. */ - async upsertIndexingStatus( - indexingStatus: SerializedCrossChainIndexingStatusSnapshot, - ): Promise { - return this.upsertEnsNodeMetadata({ + async upsertIndexingStatus(indexingStatus: CrossChainIndexingStatusSnapshot): Promise { + await this.upsertEnsNodeMetadata({ key: EnsNodeMetadataKeys.IndexingStatus, - value: indexingStatus, + value: serializeCrossChainIndexingStatusSnapshotOmnichain(indexingStatus), }); } @@ -112,9 +120,9 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { * @returns selected record in ENSDb. * @throws when exactly one matching metadata record was not found */ - private async getEnsNodeMetadata( - metadata: Pick, - ): Promise { + private async getEnsNodeMetadata< + EnsNodeMetadataType extends SerializedEnsNodeMetadata = SerializedEnsNodeMetadata, + >(metadata: Pick): Promise { const result = await this.#db .select() .from(schema.ensNodeMetadata) @@ -134,13 +142,12 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { /** * Upsert ENSNode metadata * - * @returns updated record in ENSDb. * @throws when upsert operation failed. */ private async upsertEnsNodeMetadata< - EnsNodeMetadataType extends EnsNodeMetadata = EnsNodeMetadata, - >(metadata: EnsNodeMetadataType): Promise { - const [result] = await this.#db + EnsNodeMetadataType extends SerializedEnsNodeMetadata = SerializedEnsNodeMetadata, + >(metadata: EnsNodeMetadataType): Promise { + await this.#db .insert(schema.ensNodeMetadata) .values({ key: metadata.key, @@ -149,13 +156,6 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { .onConflictDoUpdate({ target: schema.ensNodeMetadata.key, set: { value: metadata.value }, - }) - .returning({ value: schema.ensNodeMetadata.value }); - - if (!result) { - throw new Error(`Failed to upsert metadata for key: ${metadata.key}`); - } - - return result.value as EnsNodeMetadataType["value"]; + }); } } diff --git a/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts b/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts index d73147171..a6dfdd719 100644 --- a/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts +++ b/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts @@ -1,4 +1,6 @@ import type { + CrossChainIndexingStatusSnapshot, + ENSIndexerPublicConfig, SerializedCrossChainIndexingStatusSnapshot, SerializedENSIndexerPublicConfig, } from "@ensnode/ensnode-sdk"; @@ -14,11 +16,27 @@ export const EnsNodeMetadataKeys = { export type EnsNodeMetadataKey = (typeof EnsNodeMetadataKeys)[keyof typeof EnsNodeMetadataKeys]; export interface EnsNodeMetadataEnsIndexerPublicConfig { + key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; + value: ENSIndexerPublicConfig; +} + +/** + * Serialized representation of {@link EnsNodeMetadataEnsIndexerPublicConfig}. + */ +export interface SerializedEnsNodeMetadataEnsIndexerPublicConfig { key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; value: SerializedENSIndexerPublicConfig; } export interface EnsNodeMetadataIndexingStatus { + key: typeof EnsNodeMetadataKeys.IndexingStatus; + value: CrossChainIndexingStatusSnapshot; +} + +/** + * Serialized representation of {@link EnsNodeMetadataIndexingStatus}. + */ +export interface SerializedEnsNodeMetadataIndexingStatus { key: typeof EnsNodeMetadataKeys.IndexingStatus; value: SerializedCrossChainIndexingStatusSnapshot; } @@ -29,3 +47,10 @@ export interface EnsNodeMetadataIndexingStatus { * Union type gathering all variants of ENSNode Metadata. */ export type EnsNodeMetadata = EnsNodeMetadataEnsIndexerPublicConfig | EnsNodeMetadataIndexingStatus; + +/** + * Serialized representation of {@link EnsNodeMetadata} + */ +export type SerializedEnsNodeMetadata = + | SerializedEnsNodeMetadataEnsIndexerPublicConfig + | SerializedEnsNodeMetadataIndexingStatus; diff --git a/apps/ensindexer/src/lib/ensindexer/client.ts b/apps/ensindexer/src/lib/ensindexer/client.ts index 9d1a662d5..5d57c12ab 100644 --- a/apps/ensindexer/src/lib/ensindexer/client.ts +++ b/apps/ensindexer/src/lib/ensindexer/client.ts @@ -1,6 +1,10 @@ -import type { - SerializedENSIndexerPublicConfig, - SerializedIndexingStatusResponse, +import { + deserializeENSIndexerPublicConfig, + deserializeIndexingStatusResponse, + type ENSIndexerPublicConfig, + type IndexingStatusResponse, + type SerializedENSIndexerPublicConfig, + type SerializedIndexingStatusResponse, } from "@ensnode/ensnode-sdk"; export class EnsIndexerClient { @@ -27,12 +31,14 @@ export class EnsIndexerClient { * @returns ENSIndexer Public Config * @throws error when fetching ENSIndexer Public Config failed */ - public async config(): Promise { + public async config(): Promise { const ensIndexerPublicConfigSerialized = await fetch( new URL("/api/config", this.ensIndexerUrl), ).then((response) => response.json()); - return ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig; + return deserializeENSIndexerPublicConfig( + ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig, + ); } /** @@ -41,11 +47,13 @@ export class EnsIndexerClient { * @returns Indexing Status when it's available and with valid. * @throws error when Indexing Status was either not available, or invalid. */ - public async indexingStatus(): Promise { + public async indexingStatus(): Promise { const indexingStatusSerialized = await fetch( new URL("/api/indexing-status", this.ensIndexerUrl), ).then((response) => response.json()); - return indexingStatusSerialized as SerializedIndexingStatusResponse; + return deserializeIndexingStatusResponse( + indexingStatusSerialized as SerializedIndexingStatusResponse, + ); } } From 58d6115f1b4501b7a7bf40fcfc7ffa5a2701a8d7 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Thu, 18 Dec 2025 15:35:53 +0100 Subject: [PATCH 10/12] Move `EnsIndexerClient` into ENSNode SDK --- .../ponder/src/ensdb-writer-worker.ts | 4 +- apps/ensindexer/src/lib/ensindexer/client.ts | 59 -------- apps/ensindexer/src/lib/ensindexer/index.ts | 1 - .../src/lib/ensindexer/local-client.ts | 23 ++- .../ensnode-sdk/src/ensindexer/client.test.ts | 136 ++++++++++++++++++ packages/ensnode-sdk/src/ensindexer/client.ts | 117 +++++++++++++++ .../ensindexer}/config/compatibility.test.ts | 6 +- .../src/ensindexer}/config/compatibility.ts | 2 +- .../src/ensindexer/config/conversions.test.ts | 50 +------ .../src/ensindexer/config/index.ts | 1 + .../src/ensindexer/config/mocks.ts | 44 ++++++ packages/ensnode-sdk/src/ensindexer/index.ts | 1 + 12 files changed, 331 insertions(+), 113 deletions(-) delete mode 100644 apps/ensindexer/src/lib/ensindexer/client.ts create mode 100644 packages/ensnode-sdk/src/ensindexer/client.test.ts create mode 100644 packages/ensnode-sdk/src/ensindexer/client.ts rename {apps/ensindexer/src => packages/ensnode-sdk/src/ensindexer}/config/compatibility.test.ts (95%) rename {apps/ensindexer/src => packages/ensnode-sdk/src/ensindexer}/config/compatibility.ts (96%) create mode 100644 packages/ensnode-sdk/src/ensindexer/config/mocks.ts diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts index 80b7c1166..8cd454f87 100644 --- a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -13,9 +13,9 @@ import { ENSIndexerPublicConfig, IndexingStatusResponseCodes, OmnichainIndexingStatusIds, + validateENSIndexerPublicConfigCompatibility, } from "@ensnode/ensnode-sdk"; -import { validateENSIndexerPublicConfigCompatibility } from "@/config/compatibility"; import { EnsDbClient } from "@/lib/ensdb"; import { ensIndexerClient, waitForEnsIndexerToBecomeHealthy } from "@/lib/ensindexer"; @@ -92,6 +92,8 @@ async function ensDbWriterWorker() { const { omnichainSnapshot } = snapshot; // Check if Indexing Status is in expected status. + // The Omnichain Status must indicate that indexing has started already. + // Throw an error if Omnichain Status is "Unstarted". if (omnichainSnapshot.omnichainStatus === OmnichainIndexingStatusIds.Unstarted) { throw new Error("Omnichain Status must be different than 'Unstarted'."); } diff --git a/apps/ensindexer/src/lib/ensindexer/client.ts b/apps/ensindexer/src/lib/ensindexer/client.ts deleted file mode 100644 index 5d57c12ab..000000000 --- a/apps/ensindexer/src/lib/ensindexer/client.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { - deserializeENSIndexerPublicConfig, - deserializeIndexingStatusResponse, - type ENSIndexerPublicConfig, - type IndexingStatusResponse, - type SerializedENSIndexerPublicConfig, - type SerializedIndexingStatusResponse, -} from "@ensnode/ensnode-sdk"; - -export class EnsIndexerClient { - constructor(private ensIndexerUrl: URL) {} - - /** - * Wait for ENSIndexer to become healthy. - * - * @throws when the Health endpoint didn't return HTTP Status OK. - */ - public async health(): Promise { - try { - await fetch(new URL("/health", this.ensIndexerUrl)); - } catch { - const errorMessage = "Health endpoint for ENSIndexer is not available yet."; - - throw new Error(errorMessage); - } - } - - /** - * Fetch ENSIndexer Public Config - * - * @returns ENSIndexer Public Config - * @throws error when fetching ENSIndexer Public Config failed - */ - public async config(): Promise { - const ensIndexerPublicConfigSerialized = await fetch( - new URL("/api/config", this.ensIndexerUrl), - ).then((response) => response.json()); - - return deserializeENSIndexerPublicConfig( - ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig, - ); - } - - /** - * Fetch Indexing Status - * - * @returns Indexing Status when it's available and with valid. - * @throws error when Indexing Status was either not available, or invalid. - */ - public async indexingStatus(): Promise { - const indexingStatusSerialized = await fetch( - new URL("/api/indexing-status", this.ensIndexerUrl), - ).then((response) => response.json()); - - return deserializeIndexingStatusResponse( - indexingStatusSerialized as SerializedIndexingStatusResponse, - ); - } -} diff --git a/apps/ensindexer/src/lib/ensindexer/index.ts b/apps/ensindexer/src/lib/ensindexer/index.ts index 51262e40d..bee6566ef 100644 --- a/apps/ensindexer/src/lib/ensindexer/index.ts +++ b/apps/ensindexer/src/lib/ensindexer/index.ts @@ -1,2 +1 @@ -export * from "./client"; export * from "./local-client"; diff --git a/apps/ensindexer/src/lib/ensindexer/local-client.ts b/apps/ensindexer/src/lib/ensindexer/local-client.ts index b78e8f060..5ef7f8165 100644 --- a/apps/ensindexer/src/lib/ensindexer/local-client.ts +++ b/apps/ensindexer/src/lib/ensindexer/local-client.ts @@ -2,7 +2,13 @@ import config from "@/config"; import pRetry from "p-retry"; -import { EnsIndexerClient } from "./client"; +import { EnsIndexerClient, EnsIndexerHealthCheckResults } from "@ensnode/ensnode-sdk"; + +/** + * How many times retries should be attempted before + * {@link waitForEnsIndexerToBecomeHealthy} becomes a rejected promise. + */ +export const MAX_ENSINDEXER_HEALTHCHECK_ATTEMPTS = 5; export const ensIndexerClient = new EnsIndexerClient(config.ensIndexerUrl); @@ -11,6 +17,15 @@ export const ensIndexerClient = new EnsIndexerClient(config.ensIndexerUrl); * * The global promise that will only resolve after the ENSIndexer has become healthy. */ -export const waitForEnsIndexerToBecomeHealthy = pRetry(async () => ensIndexerClient.health(), { - retries: 5, -}); +export const waitForEnsIndexerToBecomeHealthy = pRetry( + async () => { + const response = await ensIndexerClient.health(); + + if (response !== EnsIndexerHealthCheckResults.Ok) { + throw new Error("ENSIndexer is not healthy yet"); + } + }, + { + retries: MAX_ENSINDEXER_HEALTHCHECK_ATTEMPTS, + }, +); diff --git a/packages/ensnode-sdk/src/ensindexer/client.test.ts b/packages/ensnode-sdk/src/ensindexer/client.test.ts new file mode 100644 index 000000000..11a498fff --- /dev/null +++ b/packages/ensnode-sdk/src/ensindexer/client.test.ts @@ -0,0 +1,136 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { EnsIndexerClient, EnsIndexerHealthCheckResults } from "./client"; +import { mockedConfig, mockedSerializedConfig } from "./config/mocks"; + +describe("EnsIndexerClient", () => { + const mockFetch = vi.fn(); + const baseUrl = new URL("http://exmple.com"); + + beforeEach(() => { + vi.stubGlobal("fetch", mockFetch); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + describe("health()", () => { + it("should return Ok when fetch returns 2xx status", async () => { + mockFetch + .mockResolvedValueOnce( + new Response("ok", { + status: 200, + }), + ) + .mockResolvedValueOnce( + new Response("ok", { + status: 299, + }), + ); + + const client = new EnsIndexerClient(baseUrl); + + const result1 = await client.health(); + expect(result1).toBe(EnsIndexerHealthCheckResults.Ok); + + const result2 = await client.health(); + expect(result2).toBe(EnsIndexerHealthCheckResults.Ok); + }); + + it("should return NotOk when fetch returns non-2xx status", async () => { + mockFetch + .mockResolvedValueOnce( + new Response("bad request", { + status: 400, + }), + ) + .mockResolvedValueOnce( + new Response("internal server error", { + status: 500, + }), + ); + + const client = new EnsIndexerClient(baseUrl); + + const result1 = await client.health(); + expect(result1).toBe(EnsIndexerHealthCheckResults.NotOk); + + const result2 = await client.health(); + expect(result2).toBe(EnsIndexerHealthCheckResults.NotOk); + }); + + it("should return Unknown when fetch throws error", async () => { + mockFetch.mockRejectedValueOnce(new Error("Network error")); + + const client = new EnsIndexerClient(baseUrl); + const result = await client.health(); + + expect(result).toBe(EnsIndexerHealthCheckResults.Unknown); + }); + + it("should update internal health state on successful health check", async () => { + // arrange + mockFetch + .mockResolvedValueOnce( + // `/health` mock + new Response("ok", { + status: 200, + }), + ) + .mockResolvedValueOnce( + // `/api/config` mock + new Response(JSON.stringify(mockedSerializedConfig), { + status: 200, + }), + ); + + const client = new EnsIndexerClient(baseUrl); + + // act + await client.health(); + const config = await client.config(); + + // assert + expect(config).toStrictEqual(mockedConfig); + }); + }); + + describe("other methods", () => { + it("should throw when a method called and health check result is Unknown", async () => { + mockFetch.mockRejectedValue(new Error("Network error")); + + const client = new EnsIndexerClient(baseUrl); + + await expect(client.config()).rejects.toThrowError(/Call the 'health\(\)' method first/i); + await expect(client.indexingStatus()).rejects.toThrowError( + /Call the 'health\(\)' method first/i, + ); + + await expect(client.health()).resolves.toBe(EnsIndexerHealthCheckResults.Unknown); + + await expect(client.config()).rejects.toThrowError(/ENSIndexer must be healthy/); + await expect(client.indexingStatus()).rejects.toThrowError(/ENSIndexer must be healthy/); + }); + + it("should throw when a method called and health check result is NotOk", async () => { + mockFetch.mockResolvedValue( + new Response("internal server error", { + status: 500, + }), + ); + + const client = new EnsIndexerClient(baseUrl); + + await expect(client.config()).rejects.toThrowError(/Call the 'health\(\)' method first/i); + await expect(client.indexingStatus()).rejects.toThrowError( + /Call the 'health\(\)' method first/i, + ); + + await expect(client.health()).resolves.toBe(EnsIndexerHealthCheckResults.NotOk); + + await expect(client.config()).rejects.toThrowError(/ENSIndexer must be healthy/); + await expect(client.indexingStatus()).rejects.toThrowError(/ENSIndexer must be healthy/); + }); + }); +}); diff --git a/packages/ensnode-sdk/src/ensindexer/client.ts b/packages/ensnode-sdk/src/ensindexer/client.ts new file mode 100644 index 000000000..847f4cbb4 --- /dev/null +++ b/packages/ensnode-sdk/src/ensindexer/client.ts @@ -0,0 +1,117 @@ +import { + deserializeENSIndexerPublicConfig, + deserializeIndexingStatusResponse, + type ENSIndexerPublicConfig, + type IndexingStatusResponse, + type SerializedENSIndexerPublicConfig, + type SerializedIndexingStatusResponse, +} from "@ensnode/ensnode-sdk"; + +export const EnsIndexerHealthCheckResults = { + /** + * ENSIndexer Health is unknown if the health check endpoint is unavailable. + */ + Unknown: "unknown", + + /** + * ENSIndexer Health is not OK if the health check endpoint returned + * HTTP status other than `2xx`. + */ + NotOk: "not-ok", + + /** + * ENSIndexer Health is OK if the health check endpoint returned + * `2xx` HTTP status. + */ + Ok: "ok", +} as const; + +export type EnsIndexerHealthCheckResult = + (typeof EnsIndexerHealthCheckResults)[keyof typeof EnsIndexerHealthCheckResults]; + +/** + * ENSIndexer Client + * + * Using this client methods requires first calling `health()` method and + * waiting for it to return {@link EnsIndexerHealthCheckResults.Ok} result. + */ +export class EnsIndexerClient { + #healthCheckResult: EnsIndexerHealthCheckResult | undefined; + + constructor(private ensIndexerUrl: URL) {} + + /** + * ENSIndexer health check endpoint. + */ + public async health(): Promise { + let response: Response; + + try { + response = await fetch(new URL("/health", this.ensIndexerUrl)); + + if (!response.ok) { + this.#healthCheckResult = EnsIndexerHealthCheckResults.NotOk; + } else { + this.#healthCheckResult = EnsIndexerHealthCheckResults.Ok; + } + } catch { + this.#healthCheckResult = EnsIndexerHealthCheckResults.Unknown; + } + + return this.#healthCheckResult; + } + + /** + * Fetch ENSIndexer Public Config + * + * @returns ENSIndexer Public Config + * @throws error when fetching ENSIndexer Public Config failed + */ + public async config(): Promise { + this.validateEnsIndexerHealthCheckResult(); + + const ensIndexerPublicConfigSerialized = await fetch( + new URL("/api/config", this.ensIndexerUrl), + ).then((response) => response.json()); + + return deserializeENSIndexerPublicConfig( + ensIndexerPublicConfigSerialized as SerializedENSIndexerPublicConfig, + ); + } + + /** + * Fetch Indexing Status + * + * @returns Indexing Status when it's available and with valid. + * @throws error when Indexing Status was either not available, or invalid. + */ + public async indexingStatus(): Promise { + this.validateEnsIndexerHealthCheckResult(); + + const indexingStatusSerialized = await fetch( + new URL("/api/indexing-status", this.ensIndexerUrl), + ).then((response) => response.json()); + + return deserializeIndexingStatusResponse( + indexingStatusSerialized as SerializedIndexingStatusResponse, + ); + } + + /** + * Validate ENSIndexer health check result. + * + * @throws if the health check result is other than + * {@link EnsIndexerHealthCheckResults.Ok}. + */ + private validateEnsIndexerHealthCheckResult(): void { + if (typeof this.#healthCheckResult === "undefined") { + throw new Error("Call the 'health()' method first."); + } + + if (this.#healthCheckResult !== EnsIndexerHealthCheckResults.Ok) { + throw new Error( + `ENSIndexer must be healthy. Current health check result is '${this.#healthCheckResult}'. You can keep calling the 'health()' method until it returns the 'ok' result.`, + ); + } + } +} diff --git a/apps/ensindexer/src/config/compatibility.test.ts b/packages/ensnode-sdk/src/ensindexer/config/compatibility.test.ts similarity index 95% rename from apps/ensindexer/src/config/compatibility.test.ts rename to packages/ensnode-sdk/src/ensindexer/config/compatibility.test.ts index 8a4ecd811..9ee6f2074 100644 --- a/apps/ensindexer/src/config/compatibility.test.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/compatibility.test.ts @@ -11,7 +11,7 @@ import { describe("ENSIndexerConfig compatibility", () => { describe("validateENSIndexerPublicConfigCompatibility()", () => { const config = { - indexedChainIds: [1, 10, 8453], + indexedChainIds: new Set([1, 10, 8453]), isSubgraphCompatible: false, namespace: ENSNamespaceIds.Mainnet, plugins: [PluginName.Subgraph, PluginName.Basenames, PluginName.ThreeDNS], @@ -21,7 +21,7 @@ describe("ENSIndexerConfig compatibility", () => { const configA = structuredClone(config); const configB = structuredClone(config); - configB.indexedChainIds.push(59144); + configB.indexedChainIds.add(59144); configB.plugins.push(PluginName.Lineanames); expect(() => @@ -33,7 +33,7 @@ describe("ENSIndexerConfig compatibility", () => { const configA = structuredClone(config); const configB = structuredClone(config); - configB.indexedChainIds.pop(); + configB.indexedChainIds.delete(8453); expect(() => validateENSIndexerPublicConfigCompatibility(configA, configB)).toThrowError( /'indexedChainIds' must be compatible. Stored Config 'indexedChainIds': '1, 10, 8453'. Current Config 'indexedChainIds': '1, 10'/i, diff --git a/apps/ensindexer/src/config/compatibility.ts b/packages/ensnode-sdk/src/ensindexer/config/compatibility.ts similarity index 96% rename from apps/ensindexer/src/config/compatibility.ts rename to packages/ensnode-sdk/src/ensindexer/config/compatibility.ts index be6ccb031..84ad58d47 100644 --- a/apps/ensindexer/src/config/compatibility.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/compatibility.ts @@ -1,4 +1,4 @@ -import type { ENSIndexerPublicConfig } from "@ensnode/ensnode-sdk"; +import type { ENSIndexerPublicConfig } from "./types"; export type ENSIndexerPublicConfigCompatibilityCheck = Pick< ENSIndexerPublicConfig, diff --git a/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts b/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts index e6350f587..4b86f1230 100644 --- a/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts @@ -1,36 +1,18 @@ import { describe, expect, it } from "vitest"; import { deserializeENSIndexerPublicConfig } from "./deserialize"; +import { mockedConfig, mockedSerializedConfig } from "./mocks"; import { serializeENSIndexerPublicConfig } from "./serialize"; import type { SerializedENSIndexerPublicConfig } from "./serialized-types"; import { type ENSIndexerPublicConfig, PluginName } from "./types"; describe("ENSIndexer: Config", () => { + const config = mockedConfig; + const serializedConfig = mockedSerializedConfig; + describe("serialization", () => { it("can serialize ENSIndexerPublicConfig", () => { - // arrange - const config = { - databaseSchemaName: "public", - labelSet: { - labelSetId: "subgraph", - labelSetVersion: 0, - }, - indexedChainIds: new Set([1]), - isSubgraphCompatible: true, - namespace: "mainnet", - plugins: [PluginName.Subgraph], - versionInfo: { - nodejs: "v22.10.12", - ponder: "0.11.25", - ensDb: "0.32.0", - ensIndexer: "0.32.0", - ensNormalize: "1.11.1", - ensRainbow: "0.32.0", - ensRainbowSchema: 2, - }, - } satisfies ENSIndexerPublicConfig; - - // act + // arrange & act const result = serializeENSIndexerPublicConfig(config); // assert @@ -49,27 +31,7 @@ describe("ENSIndexer: Config", () => { }); describe("deserialization", () => { - const correctSerializedConfig = { - databaseSchemaName: "public", - labelSet: { - labelSetId: "subgraph", - labelSetVersion: 0, - }, - indexedChainIds: [1, 10, 8453], - isSubgraphCompatible: true, - namespace: "mainnet", - plugins: [PluginName.Subgraph], - versionInfo: { - nodejs: "v22.10.12", - ponder: "0.11.25", - ensDb: "0.32.0", - ensIndexer: "0.32.0", - ensNormalize: "1.11.1", - ensRainbow: "0.32.0", - ensRainbowSchema: 2, - }, - } satisfies SerializedENSIndexerPublicConfig; - + const correctSerializedConfig = serializedConfig; it("can deserialize SerializedENSIndexerPublicConfig", () => { // arrange const serializedConfig = structuredClone(correctSerializedConfig); diff --git a/packages/ensnode-sdk/src/ensindexer/config/index.ts b/packages/ensnode-sdk/src/ensindexer/config/index.ts index 617d962e0..77df62706 100644 --- a/packages/ensnode-sdk/src/ensindexer/config/index.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/index.ts @@ -1,3 +1,4 @@ +export * from "./compatibility"; export * from "./deserialize"; export * from "./is-subgraph-compatible"; export * from "./label-utils"; diff --git a/packages/ensnode-sdk/src/ensindexer/config/mocks.ts b/packages/ensnode-sdk/src/ensindexer/config/mocks.ts new file mode 100644 index 000000000..5892e3d6c --- /dev/null +++ b/packages/ensnode-sdk/src/ensindexer/config/mocks.ts @@ -0,0 +1,44 @@ +import type { SerializedENSIndexerPublicConfig } from "./serialized-types"; +import { type ENSIndexerPublicConfig, PluginName } from "./types"; + +export const mockedConfig = { + databaseSchemaName: "public", + labelSet: { + labelSetId: "subgraph", + labelSetVersion: 0, + }, + indexedChainIds: new Set([1]), + isSubgraphCompatible: true, + namespace: "mainnet", + plugins: [PluginName.Subgraph], + versionInfo: { + nodejs: "v22.10.12", + ponder: "0.11.25", + ensDb: "0.32.0", + ensIndexer: "0.32.0", + ensNormalize: "1.11.1", + ensRainbow: "0.32.0", + ensRainbowSchema: 2, + }, +} satisfies ENSIndexerPublicConfig; + +export const mockedSerializedConfig = { + databaseSchemaName: "public", + labelSet: { + labelSetId: "subgraph", + labelSetVersion: 0, + }, + indexedChainIds: [1], + isSubgraphCompatible: true, + namespace: "mainnet", + plugins: [PluginName.Subgraph], + versionInfo: { + nodejs: "v22.10.12", + ponder: "0.11.25", + ensDb: "0.32.0", + ensIndexer: "0.32.0", + ensNormalize: "1.11.1", + ensRainbow: "0.32.0", + ensRainbowSchema: 2, + }, +} satisfies SerializedENSIndexerPublicConfig; diff --git a/packages/ensnode-sdk/src/ensindexer/index.ts b/packages/ensnode-sdk/src/ensindexer/index.ts index c896748f2..fcd36988f 100644 --- a/packages/ensnode-sdk/src/ensindexer/index.ts +++ b/packages/ensnode-sdk/src/ensindexer/index.ts @@ -1,2 +1,3 @@ +export * from "./client"; export * from "./config"; export * from "./indexing-status"; From b21c67d87c470fd27b4ddfcd3a905a0e1b895fe3 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Thu, 18 Dec 2025 15:36:28 +0100 Subject: [PATCH 11/12] Fix import paths to avoid circular dependencies --- packages/ensnode-sdk/src/api/name-tokens/prerequisites.ts | 5 ++--- .../ensnode-sdk/src/api/registrar-actions/prerequisites.ts | 5 ++--- packages/ensnode-sdk/src/ensapi/config/conversions.test.ts | 5 +++-- packages/ensnode-sdk/src/ensapi/config/serialize.ts | 2 +- packages/ensnode-sdk/src/ensapi/config/serialized-types.ts | 2 +- packages/ensnode-sdk/src/ensapi/config/types.ts | 2 +- .../ensnode-sdk/src/ensindexer/config/conversions.test.ts | 2 +- .../ensnode-sdk/src/ensindexer/config/serialized-types.ts | 2 +- packages/ensnode-sdk/src/ensindexer/config/types.ts | 4 ++-- 9 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/ensnode-sdk/src/api/name-tokens/prerequisites.ts b/packages/ensnode-sdk/src/api/name-tokens/prerequisites.ts index 6690f93c8..264b2d35b 100644 --- a/packages/ensnode-sdk/src/api/name-tokens/prerequisites.ts +++ b/packages/ensnode-sdk/src/api/name-tokens/prerequisites.ts @@ -1,9 +1,8 @@ +import { type ENSIndexerPublicConfig, PluginName } from "../../ensindexer/config/types"; import { - type ENSIndexerPublicConfig, type OmnichainIndexingStatusId, OmnichainIndexingStatusIds, - PluginName, -} from "../../ensindexer"; +} from "../../ensindexer/indexing-status/types"; export const nameTokensPrerequisites = Object.freeze({ /** diff --git a/packages/ensnode-sdk/src/api/registrar-actions/prerequisites.ts b/packages/ensnode-sdk/src/api/registrar-actions/prerequisites.ts index afdc12f11..dac980d22 100644 --- a/packages/ensnode-sdk/src/api/registrar-actions/prerequisites.ts +++ b/packages/ensnode-sdk/src/api/registrar-actions/prerequisites.ts @@ -1,9 +1,8 @@ +import { type ENSIndexerPublicConfig, PluginName } from "../../ensindexer/config/types"; import { - type ENSIndexerPublicConfig, type OmnichainIndexingStatusId, OmnichainIndexingStatusIds, - PluginName, -} from "../../ensindexer"; +} from "../../ensindexer/indexing-status/types"; export const registrarActionsPrerequisites = Object.freeze({ /** diff --git a/packages/ensnode-sdk/src/ensapi/config/conversions.test.ts b/packages/ensnode-sdk/src/ensapi/config/conversions.test.ts index c394b339e..2e13a9f33 100644 --- a/packages/ensnode-sdk/src/ensapi/config/conversions.test.ts +++ b/packages/ensnode-sdk/src/ensapi/config/conversions.test.ts @@ -2,8 +2,9 @@ import { describe, expect, it } from "vitest"; import { ENSNamespaceIds } from "@ensnode/datasources"; -import { PluginName } from "../../ensindexer"; -import { deserializeENSApiPublicConfig, serializeENSApiPublicConfig } from "."; +import { PluginName } from "../../ensindexer/config/types"; +import { deserializeENSApiPublicConfig } from "./deserialize"; +import { serializeENSApiPublicConfig } from "./serialize"; import type { ENSApiPublicConfig } from "./types"; const MOCK_ENSAPI_PUBLIC_CONFIG = { diff --git a/packages/ensnode-sdk/src/ensapi/config/serialize.ts b/packages/ensnode-sdk/src/ensapi/config/serialize.ts index 44a8d6009..27f5114a7 100644 --- a/packages/ensnode-sdk/src/ensapi/config/serialize.ts +++ b/packages/ensnode-sdk/src/ensapi/config/serialize.ts @@ -1,4 +1,4 @@ -import { serializeENSIndexerPublicConfig } from "../../ensindexer"; +import { serializeENSIndexerPublicConfig } from "../../ensindexer/config/serialize"; import type { SerializedENSApiPublicConfig } from "./serialized-types"; import type { ENSApiPublicConfig } from "./types"; diff --git a/packages/ensnode-sdk/src/ensapi/config/serialized-types.ts b/packages/ensnode-sdk/src/ensapi/config/serialized-types.ts index f437da884..41199141c 100644 --- a/packages/ensnode-sdk/src/ensapi/config/serialized-types.ts +++ b/packages/ensnode-sdk/src/ensapi/config/serialized-types.ts @@ -1,4 +1,4 @@ -import type { SerializedENSIndexerPublicConfig } from "../../ensindexer"; +import type { SerializedENSIndexerPublicConfig } from "../../ensindexer/config/serialized-types"; import type { ENSApiPublicConfig } from "./types"; /** diff --git a/packages/ensnode-sdk/src/ensapi/config/types.ts b/packages/ensnode-sdk/src/ensapi/config/types.ts index b627b98da..16ba34e59 100644 --- a/packages/ensnode-sdk/src/ensapi/config/types.ts +++ b/packages/ensnode-sdk/src/ensapi/config/types.ts @@ -1,6 +1,6 @@ import type { z } from "zod/v4"; -import type { ENSIndexerPublicConfig } from "../../ensindexer"; +import type { ENSIndexerPublicConfig } from "../../ensindexer/config/types"; import type { TheGraphCannotFallbackReasonSchema, TheGraphFallbackSchema } from "./zod-schemas"; export type TheGraphCannotFallbackReason = z.infer; diff --git a/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts b/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts index 4b86f1230..0bcc0065d 100644 --- a/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/conversions.test.ts @@ -42,7 +42,7 @@ describe("ENSIndexer: Config", () => { // assert expect(result).toStrictEqual({ ...serializedConfig, - indexedChainIds: new Set([1, 10, 8453]), + indexedChainIds: new Set([1]), } satisfies ENSIndexerPublicConfig); }); diff --git a/packages/ensnode-sdk/src/ensindexer/config/serialized-types.ts b/packages/ensnode-sdk/src/ensindexer/config/serialized-types.ts index 97a786916..9ddc2181e 100644 --- a/packages/ensnode-sdk/src/ensindexer/config/serialized-types.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/serialized-types.ts @@ -1,4 +1,4 @@ -import type { ChainId } from "../../shared"; +import type { ChainId } from "../../shared/types"; import type { ENSIndexerPublicConfig, ENSIndexerVersionInfo } from "./types"; export type SerializedIndexedChainIds = Array; diff --git a/packages/ensnode-sdk/src/ensindexer/config/types.ts b/packages/ensnode-sdk/src/ensindexer/config/types.ts index 14417376a..aa1514e56 100644 --- a/packages/ensnode-sdk/src/ensindexer/config/types.ts +++ b/packages/ensnode-sdk/src/ensindexer/config/types.ts @@ -1,7 +1,7 @@ import type { ENSNamespaceId } from "@ensnode/datasources"; -import type { EnsRainbowClientLabelSet } from "../../ensrainbow"; -import type { ChainId } from "../../shared"; +import type { EnsRainbowClientLabelSet } from "../../ensrainbow/types"; +import type { ChainId } from "../../shared/types"; /** * A PluginName is a unique id for a 'plugin': we use the notion of From 57fcd2acfbddbe4f3a5a30fa34b038758266a936 Mon Sep 17 00:00:00 2001 From: Tomasz Kopacki Date: Thu, 18 Dec 2025 16:11:16 +0100 Subject: [PATCH 12/12] Include ENSDb Version in ENSNodeMetadata table --- .../ponder/src/ensdb-writer-worker.ts | 33 ++++---- apps/ensindexer/src/lib/ensdb/ensdb-client.ts | 75 +++++++++++++------ .../src/lib/ensdb/ensnode-metadata.ts | 21 +++++- .../src/schemas/ensnode-metadata.schema.ts | 18 +++++ packages/ensnode-sdk/src/ensindexer/client.ts | 20 +++-- 5 files changed, 121 insertions(+), 46 deletions(-) diff --git a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts index 8cd454f87..cc294113c 100644 --- a/apps/ensindexer/ponder/src/ensdb-writer-worker.ts +++ b/apps/ensindexer/ponder/src/ensdb-writer-worker.ts @@ -44,9 +44,9 @@ async function ensDbWriterWorker() { const ensDbClient = new EnsDbClient(); /** - * Handle ENSIndexerPublicConfig Record + * Handle ENSIndexerPublicConfig and ENSDb Version Records */ - const handleEnsIndexerPublicConfigRecord = async () => { + const handleEnsIndexerPublicConfigAndEnsDbVersionRecords = async () => { // Read stored config and in-memory config. // Note: we wrap each operation in pRetry to ensure all of them can be // completed successfully. @@ -70,6 +70,8 @@ async function ensDbWriterWorker() { }); } } else { + // Upsert ENSDb Version into ENSDb. + await ensDbClient.upsertEnsDbVersion(inMemoryConfig.versionInfo.ensDb); // Upsert ENSIndexerPublicConfig into ENSDb. await ensDbClient.upsertEnsIndexerPublicConfig(inMemoryConfig); } @@ -102,7 +104,8 @@ async function ensDbWriterWorker() { await ensDbClient.upsertIndexingStatus(snapshot); } catch (error) { // Do nothing about this error, but having it logged. - console.error(error, "Could not upsert Indexing Status record"); + const errorMessage = error instanceof Error ? error.message : "Unknown error"; + console.error(`Could not upsert Indexing Status record due to: ${errorMessage}`); } finally { // Regardless of current iteration result, // schedule the next callback to handle Indexing Status Record. @@ -113,21 +116,19 @@ async function ensDbWriterWorker() { } }; - // 4. Handle ENSIndexer Public Config just once. - console.log("Task: store ENSIndexer Public Config in ENSDb."); - await handleEnsIndexerPublicConfigRecord().then(() => - console.log("ENSIndexer Public Config successfully stored in ENSDb."), - ); + // 4. Handle ENSIndexer Public Config and ENSDb Version just once. + console.log("Task: store ENSIndexer Public Config and ENSDb Version in ENSDb."); + await handleEnsIndexerPublicConfigAndEnsDbVersionRecords(); + console.log("ENSIndexer Public Config and ENSDb Version successfully stored in ENSDb."); // 5. Handle Indexing Status on recurring basis. console.log("Task: store Indexing Status in ENSDb."); - await handleIndexingStatusRecordRecursively().then(() => - console.log("Indexing Status successfully stored in ENSDb."), - ); + await handleIndexingStatusRecordRecursively(); + console.log("Indexing Status successfully stored in ENSDb."); } -// Run ENSDb Writer Worker in a non-blocking way to -// allow database migrations to proceed in the background. -ensDbWriterWorker().catch((error) => - console.error("ENSDb Writer Worker failed to perform its tasks", error), -); +// Run ENSDb Writer Worker in the background. +ensDbWriterWorker().catch((error) => { + console.error("ENSDb Writer Worker failed to perform its tasks", error); + process.exit(1); +}); diff --git a/apps/ensindexer/src/lib/ensdb/ensdb-client.ts b/apps/ensindexer/src/lib/ensdb/ensdb-client.ts index f4fcb234c..d1e98e452 100644 --- a/apps/ensindexer/src/lib/ensdb/ensdb-client.ts +++ b/apps/ensindexer/src/lib/ensdb/ensdb-client.ts @@ -16,6 +16,7 @@ import { makeDrizzle } from "./drizzle"; import { EnsNodeMetadataKeys, type SerializedEnsNodeMetadata, + type SerializedEnsNodeMetadataEnsDbVersion, type SerializedEnsNodeMetadataEnsIndexerPublicConfig, type SerializedEnsNodeMetadataIndexingStatus, } from "./ensnode-metadata"; @@ -26,8 +27,28 @@ import { Includes methods for reading from ENSDb. */ export interface EnsDbClientQuery { + /** + * Get ENSDb Version + * + * @returns the existing record, or `undefined`. + * @throws if not exactly one record was found. + */ + getEnsDbVersion(): Promise; + + /** + * Get ENSIndexer Public Config + * + * @returns the existing record, or `undefined`. + * @throws if not exactly one record was found. + */ getEnsIndexerPublicConfig(): Promise; + /** + * Get Indexing Status + * + * @returns the existing record, or `undefined`. + * @throws if not exactly one record was found. + */ getIndexingStatus(): Promise; } @@ -37,8 +58,25 @@ export interface EnsDbClientQuery { * Includes methods for writing into ENSDb. */ export interface EnsDbClientMutation { + /** + * Upsert ENSDb Version + * + * @throws when upsert operation failed. + */ + upsertEnsDbVersion(ensDbVersion: string): Promise; + + /** + * Upsert ENSIndexer Public Config + * + * @throws when upsert operation failed. + */ upsertEnsIndexerPublicConfig(ensIndexerPublicConfig: ENSIndexerPublicConfig): Promise; + /** + * Upsert Indexing Status + * + * @throws when upsert operation failed. + */ upsertIndexingStatus(indexingStatus: CrossChainIndexingStatusSnapshot): Promise; } @@ -52,12 +90,14 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { schema, }); - /** - * Upsert ENSIndexer Public Config - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ + async getEnsDbVersion(): Promise { + const record = await this.getEnsNodeMetadata({ + key: EnsNodeMetadataKeys.EnsDbVersion, + }); + + return record; + } + async getEnsIndexerPublicConfig(): Promise { const record = await this.getEnsNodeMetadata({ key: EnsNodeMetadataKeys.EnsIndexerPublicConfig, @@ -70,12 +110,6 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { return deserializeENSIndexerPublicConfig(record); } - /** - * Upsert Indexing Status - * - * @returns updated record in ENSDb. - * @throws when upsert operation failed. - */ async getIndexingStatus(): Promise { const record = await this.getEnsNodeMetadata({ key: EnsNodeMetadataKeys.IndexingStatus, @@ -88,11 +122,13 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { return deserializeCrossChainIndexingStatusSnapshot(record); } - /** - * Upsert ENSIndexer Public Config - * - * @throws when upsert operation failed. - */ + async upsertEnsDbVersion(ensDbVersion: string): Promise { + await this.upsertEnsNodeMetadata({ + key: EnsNodeMetadataKeys.EnsDbVersion, + value: ensDbVersion, + }); + } + async upsertEnsIndexerPublicConfig( ensIndexerPublicConfig: ENSIndexerPublicConfig, ): Promise { @@ -102,11 +138,6 @@ export class EnsDbClient implements EnsDbClientQuery, EnsDbClientMutation { }); } - /** - * Upsert Indexing Status - * - * @throws when upsert operation failed. - */ async upsertIndexingStatus(indexingStatus: CrossChainIndexingStatusSnapshot): Promise { await this.upsertEnsNodeMetadata({ key: EnsNodeMetadataKeys.IndexingStatus, diff --git a/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts b/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts index a6dfdd719..4914a7a4d 100644 --- a/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts +++ b/apps/ensindexer/src/lib/ensdb/ensnode-metadata.ts @@ -9,12 +9,23 @@ import type { * Keys used to distinguish records in `ensnode_metadata` table in the ENSDb. */ export const EnsNodeMetadataKeys = { - EnsIndexerPublicConfig: "ensindexer-public-config", - IndexingStatus: "indexing-status", + EnsDbVersion: "ensdb_version", + EnsIndexerPublicConfig: "ensindexer_public_config", + IndexingStatus: "indexing_status", } as const; export type EnsNodeMetadataKey = (typeof EnsNodeMetadataKeys)[keyof typeof EnsNodeMetadataKeys]; +export interface EnsNodeMetadataEnsDbVersion { + key: typeof EnsNodeMetadataKeys.EnsDbVersion; + value: string; +} + +/** + * Serialized representation of {@link EnsNodeMetadataEnsDbVersion}. + */ +export type SerializedEnsNodeMetadataEnsDbVersion = EnsNodeMetadataEnsDbVersion; + export interface EnsNodeMetadataEnsIndexerPublicConfig { key: typeof EnsNodeMetadataKeys.EnsIndexerPublicConfig; value: ENSIndexerPublicConfig; @@ -46,11 +57,15 @@ export interface SerializedEnsNodeMetadataIndexingStatus { * * Union type gathering all variants of ENSNode Metadata. */ -export type EnsNodeMetadata = EnsNodeMetadataEnsIndexerPublicConfig | EnsNodeMetadataIndexingStatus; +export type EnsNodeMetadata = + | EnsNodeMetadataEnsDbVersion + | EnsNodeMetadataEnsIndexerPublicConfig + | EnsNodeMetadataIndexingStatus; /** * Serialized representation of {@link EnsNodeMetadata} */ export type SerializedEnsNodeMetadata = + | SerializedEnsNodeMetadataEnsDbVersion | SerializedEnsNodeMetadataEnsIndexerPublicConfig | SerializedEnsNodeMetadataIndexingStatus; diff --git a/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts b/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts index 3018ab7a4..1750d032b 100644 --- a/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts +++ b/packages/ensnode-schema/src/schemas/ensnode-metadata.schema.ts @@ -4,15 +4,33 @@ import { onchainTable } from "ponder"; +/** + * ENSNode Metadata + * + * Possible key value pairs are defined by 'EnsNodeMetadata' type: + * - `EnsNodeMetadataEnsDbVersion` + * - `EnsNodeMetadataEnsIndexerPublicConfig` + * - `EnsNodeMetadataIndexingStatus` + */ export const ensNodeMetadata = onchainTable("ensnode_metadata", (t) => ({ /** * Key + * + * Allowed keys: + * - `EnsNodeMetadataEnsDbVersion['key']` + * - `EnsNodeMetadataEnsIndexerPublicConfig['key']` + * - `EnsNodeMetadataIndexingStatus['key']` */ key: t.text().primaryKey(), /** * Value * + * Allowed values: + * - `EnsNodeMetadataEnsDbVersion['value']` + * - `EnsNodeMetadataEnsIndexerPublicConfig['value']` + * - `EnsNodeMetadataIndexingStatus['value']` + * * Guaranteed to be a JSON object. */ value: t.jsonb().notNull(), diff --git a/packages/ensnode-sdk/src/ensindexer/client.ts b/packages/ensnode-sdk/src/ensindexer/client.ts index 847f4cbb4..e09ccbb64 100644 --- a/packages/ensnode-sdk/src/ensindexer/client.ts +++ b/packages/ensnode-sdk/src/ensindexer/client.ts @@ -42,6 +42,8 @@ export class EnsIndexerClient { /** * ENSIndexer health check endpoint. + * + * @returns ENSIndexer health check result. */ public async health(): Promise { let response: Response; @@ -65,7 +67,10 @@ export class EnsIndexerClient { * Fetch ENSIndexer Public Config * * @returns ENSIndexer Public Config - * @throws error when fetching ENSIndexer Public Config failed + * + * @throws if the ENSIndexer request fails + * @throws if the ENSIndexer returns an error response + * @throws if the ENSIndexer response breaks required invariants */ public async config(): Promise { this.validateEnsIndexerHealthCheckResult(); @@ -80,10 +85,13 @@ export class EnsIndexerClient { } /** - * Fetch Indexing Status + * Fetch ENSIndexer Indexing Status + * + * @returns ENSIndexer Indexing Status * - * @returns Indexing Status when it's available and with valid. - * @throws error when Indexing Status was either not available, or invalid. + * @throws if the ENSIndexer request fails + * @throws if the ENSIndexer returns an error response + * @throws if the ENSIndexer response breaks required invariants */ public async indexingStatus(): Promise { this.validateEnsIndexerHealthCheckResult(); @@ -105,7 +113,9 @@ export class EnsIndexerClient { */ private validateEnsIndexerHealthCheckResult(): void { if (typeof this.#healthCheckResult === "undefined") { - throw new Error("Call the 'health()' method first."); + throw new Error( + "Running health check for ENSIndexer is required. Call the 'health()' method first.", + ); } if (this.#healthCheckResult !== EnsIndexerHealthCheckResults.Ok) {