chore: Subgraph monitoring action

.github/workflows/called-subgraph-build.yml

@@ -24,7 +24,7 @@ jobs:
             - name: Setup Node.js environment
               uses: actions/setup-node@v3
               with:
-                  node-version: "18"
+                  node-version: "22"
                   cache: "yarn"
             - run: yarn install --frozen-lockfile
             - name: Code format check

.github/workflows/monitoring.yml

@@ -0,0 +1,46 @@
+name: Subgraph Monitoring
+on:
+  schedule:
+    - cron: "0 */3 * * *"
+  workflow_dispatch:
+jobs:
+  monitor:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v3
+        with:
+          node-version: "22"
+          cache: "yarn"
+
+      - run: yarn install --frozen-lockfile
+
+      - name: Run Monitoring
+        env:
+          MONITOR_ALERT_THRESHOLD: ${{vars.MONITOR_ALERT_THRESHOLD}}
+          RPC_URL_11155111: ${{ secrets.RPC_URL_11155111 }}
+          RPC_URL_1: ${{ secrets.RPC_URL_1 }}
+          ALCHEMY_MAINNET_URL: ${{ secrets.ALCHEMY_MAINNET_URL }}
+          CHAINSTACK_MAINNET_URL: ${{ secrets.CHAINSTACK_MAINNET_URL }}
+          CHAINSTACK_SEPOLIA_URL: ${{ secrets.CHAINSTACK_SEPOLIA_URL }}
+        run: yarn run:monitoring
+  alert-on-failure:
+    needs: [monitor]
+    if: ${{ failure() }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - uses: actions/create-github-app-token@v2
+        id: app-token
+        with:
+          app-id: ${{ vars.APP_ID }}
+          private-key: ${{ secrets.PRIVATE_KEY }}
+      - name: Create Issue and mention team
+        run: |
+          gh issue create --title "Subgraph Indexing unbalanced" --body "${{ vars.TEAM_NAME }}: **Workflow failure** [View workflow run for details](https://github.com/${{ github.repository}}/actions/runs/${{ github.run_id}})"
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}

additional.d.ts

@@ -0,0 +1,28 @@
+declare namespace NodeJS {
+    export interface ProcessEnv {
+        /**
+         * Limit for acceptable block number distance.
+         */
+        MONITOR_ALERT_THRESHOLD: string
+        /**
+         * Ethereum sepolia node rpc to connect.
+         */
+        RPC_URL_11155111: string
+        /**
+         * Ethereum mainnet node rpc to connect.
+         */
+        RPC_URL_1: string
+        /**
+         * alchemy Mainnet subgraph graphql endpoint.
+         */
+        ALCHEMY_MAINNET_URL: string
+        /**
+         *  chainstack Mainnet subgraph graphql endpoint.
+         */
+        CHAINSTACK_MAINNET_URL: string
+        /**
+         * chainstack sepolia subgraph graphql endpoint.
+         */
+        CHAINSTACK_SEPOLIA_URL: string
+    }
+}

package.json

@@ -16,9 +16,18 @@
         ]
     },
     "scripts": {
-        "prettier": "prettier --check **/*.ts"
+        "prettier": "prettier --check **/*.ts",
+        "run:monitoring": "node --experimental-strip-types scripts/subgraph-monitoring.ts"
     },
     "devDependencies": {
-        "prettier": "2.7.1"
-    }
+        "@types/node": "22.18.10",
+        "graphql": "^16.11.0",
+        "graphql-request": "^7.2.0",
+        "prettier": "2.7.1",
+        "viem": "^2.38.2"
+    },
+    "engines": {
+        "node": "22.x"
+    },
+    "packageManager": "yarn@1.22.19"
 }

scripts/subgraph-monitoring.ts

@@ -0,0 +1,113 @@
+import { gql, request } from "graphql-request"
+import type { Hex } from "viem"
+import { createPublicClient, http } from "viem"
+import { mainnet, sepolia } from "viem/chains"
+
+const threshold = process.env.MONITOR_ALERT_THRESHOLD ?? "50"
+const sepoliaRpcUrl = process.env.RPC_URL_11155111
+const mainnetRpcUrl = process.env.RPC_URL_1
+const alchemyUrl = process.env.ALCHEMY_MAINNET_URL ?? ""
+const chainstackUrl = process.env.CHAINSTACK_MAINNET_URL ?? ""
+const sepoliaChainstackUrl = process.env.CHAINSTACK_SEPOLIA_URL ?? ""
+const blockDiffLimit = BigInt(threshold)
+
+const mainnetClient = createPublicClient({
+    chain: mainnet,
+    transport: http(mainnetRpcUrl),
+})
+const sepoliaClient = createPublicClient({
+    transport: http(sepoliaRpcUrl),
+    chain: sepolia,
+})
+
+interface Meta {
+    block: {
+        hash: Hex
+        number: number
+    }
+    deployment: string
+    hasIndexingErrors: boolean
+}
+
+interface MetaQuery {
+    _meta: Meta
+}
+
+const query = gql`
+    query _meta {
+        _meta {
+            block {
+                hash
+                number
+            }
+
+            deployment
+            hasIndexingErrors
+        }
+    }
+`
+
+const absBigInt = (n: bigint) => {
+    if (typeof n !== "bigint") {
+        throw new TypeError("Input must be a BigInt.")
+    }
+    return n < 0n ? -n : n
+}
+
+async function main() {
+    const mainnetPromises = Promise.all([
+        request<MetaQuery>(alchemyUrl, query),
+        request<MetaQuery>(chainstackUrl, query),
+        mainnetClient.getBlockNumber(),
+    ])
+
+    const sepoliaPromises = Promise.all([
+        request<MetaQuery>(sepoliaChainstackUrl, query),
+        sepoliaClient.getBlockNumber(),
+    ])
+
+    const [mainnetResults, sepoliaResults] = await Promise.all([
+        mainnetPromises,
+        sepoliaPromises,
+    ])
+
+    const [alchemyQuery, chainstackQuery, blockNumber] = mainnetResults
+    const [sepoliaChainstackQuery, sepoliaBlockNumber] = sepoliaResults
+
+    const alchemyBlockNumber = alchemyQuery?._meta?.block.number ?? 0
+    const chainstackBlockNumber = chainstackQuery?._meta?.block.number ?? 0
+    const sepoliaChainstackBlockNumber =
+        sepoliaChainstackQuery?._meta?.block.number ?? 0
+
+    const alchemyDiff = BigInt(alchemyBlockNumber) - blockNumber
+    const chainstackDiff = BigInt(chainstackBlockNumber) - blockNumber
+    const sepoliaChainstackDiff =
+        BigInt(sepoliaChainstackBlockNumber) - sepoliaBlockNumber
+
+    console.log(`MAINNET`)
+    console.log(`\tblockNumber: ${blockNumber}`)
+    console.log(`\talchemy: ${alchemyBlockNumber} diff: ${alchemyDiff}`)
+    console.log(
+        `\tchainstack: ${chainstackBlockNumber} diff: ${chainstackDiff}`
+    )
+
+    console.log(`SEPOLIA`)
+    console.log(`\tblockNumber: ${sepoliaBlockNumber}`)
+    console.log(
+        `\tchainstack: ${sepoliaChainstackBlockNumber} diff: ${sepoliaChainstackDiff}`
+    )
+
+    const isAboveThreshold = [
+        alchemyDiff,
+        chainstackDiff,
+        sepoliaChainstackDiff,
+    ].some((diff) => absBigInt(diff) > blockDiffLimit)
+
+    if (isAboveThreshold) {
+        process.exit(1)
+    } else {
+        process.exit(0)
+    }
+}
+
+main()

tsconfig.json

@@ -11,5 +11,6 @@
         "paths": {
             "~/*": ["packages/*"]
         }
-    }
+    },
+    "include": ["additional.d.ts"]
 }