Build Dapptool with local geth repo

.gitignore

@@ -1 +1,2 @@
 result
+geth-linux-amd64

docker-compose.localsetup.yml

@@ -0,0 +1,40 @@
+version: "3.2"
+
+services:
+  dapptools:
+    restart: unless-stopped
+    depends_on:
+      - ipld-eth-db
+    #image: vulcanize/dapptools:v0.30.0-v1.10.14-statediff-0.0.29
+    build:
+      context: ./
+      dockerfile: docker/Dockerfile
+    environment:
+      DB_USER: vdbm
+      DB_NAME: vulcanize_testing
+      DB_HOST: ipld-eth-db
+      DB_PORT: 5432
+      DB_PASSWORD: password
+      DB_WRITE: "true"
+      DB_TYPE: postgres
+      DB_DRIVER: sqlx
+      DB_WAIT_FOR_SYNC: "true"
+    ports:
+      - "127.0.0.1:8545:8545"
+      - "127.0.0.1:8546:8546"
+
+  ipld-eth-db:
+    restart: always
+    image: vulcanize/ipld-eth-db:v3.0.6
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_testing"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+      - vdb_db_eth_server:/var/lib/postgresql/data
+    ports:
+      - "127.0.0.1:8077:5432"
+    command: ["postgres", "-c", "log_statement=all"]
+
+volumes:
+  vdb_db_eth_server:

docker/install.sh

@@ -1,3 +1,4 @@
+set -e
 GREEN='\033[0;32m'
 RED='\033[0;31m'
 NC='\033[0m'
@@ -14,6 +15,7 @@ oops() {
 #
 #[[ $RELEASE == null ]] && oops "No release found in ${API_OUTPUT}"
 
+echo -e "${GREEN}STARTING!!${NC}"
 cachix use dapp
 nix-env -iA dapp hevm seth solc go-ethereum-unlimited -f .
 

docker/startup_script.sh

@@ -8,7 +8,8 @@ trap "exit 1" SIGINT SIGTERM
 
 TMPDIR=$(mktemp -d)
 dapp testnet --rpc-addr 0.0.0.0 --chain-id 4 --db-user $DB_USER --db-password $DB_PASSWORD --db-name $DB_NAME \
-  --db-host $DB_HOST --db-port $DB_PORT --db-write $DB_WRITE --dir "$TMPDIR" --address=$ADDRESS &
+  --db-host $DB_HOST --db-port $DB_PORT --db-write $DB_WRITE --dir "$TMPDIR" --address=$ADDRESS \
+  --db-type $DB_TYPE --db-driver $DB_DRIVER --db-waitforsync $DB_WAIT_FOR_SYNC &
 echo "sleeping 90 sec"
 # give it a few secs to start up
 sleep 90

docker/startup_script_wrapper.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-
+set -e
 nix-shell -p cacert cachix curl jq nix --run "sh docker/startup_script.sh"

local-test.md

@@ -0,0 +1,66 @@
+# How To Test Using Dapptools Locally
+
+# Overview
+
+When testing [vulcanize-geth](https://github.com/vulcanize/go-ethereum), there might be the need to use [dapptools](https://github.com/vulcanize/dapptools). Dapptools provides the following functionality:
+
+- Creates a private network that can be easier to test than the mainnet.
+- Allows you to make changes to the private network for testing purposes.
+
+# Set-Up Steps
+
+The following steps highlight how you can test geth locally using dapptools.
+
+1.  Make sure you’ve cloned both [vulcanize-geth](https://github.com/vulcanize/go-ethereum) and [dapptools](https://github.com/vulcanize/dapptools).
+
+2.  Go to the dapptools repository
+
+3.  Update the following file `setup_local_test.sh`
+
+    1.  Update `local_path_to_geth` to match the relative path to the local [vulcanize-geth](https://github.com/vulcanize/go-ethereum) repository: `local_path_to_geth="../go-ethereum-cerc/"`
+
+4.  Update the following file to use the a local version of geth: `src/go-ethereum-statediff/default.nix`
+
+    ```nix
+    #src = fetchurl {
+      #  url = "<https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64>";
+      #  sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
+      #};
+    src = ../../geth-linux-amd64;
+
+    ```
+
+5.  Update Docker resource allocations (Optional).
+
+    1.  Increase memory allocation to 4GB and storage to 100GB
+    2.  This might be necessary depending on system settings.
+
+6.  Go back to the dapptools repository, and run: `./setup_local_test.sh`
+
+    1.  This will do the following:
+        1.  Build the geth binary
+        2.  Move it to the local repository
+        3.  Build a local container of the `ipld-eth-db`
+        4.  Build a local container of `dapptools`.
+
+7.  If you make a change to your geth code base, you will have to run `./setup_local_test.sh` (Optional)
+
+8.  If you simply want to re-build the docker images, you can run: `docker-compose up --build -f docker-compose.localsetup.yml`
+
+    1.  If you don’t want to rebuild the container: `docker-compose up -f docker-compose.localsetup.yml`
+        1.  If you make any changes to the geth node and don't include the `--build` flag, the new geth binary will not be included in your container!
+
+# Useful Information
+
+## Using The Geth Console
+
+To use the geth console do the following:
+
+1.  Access the container’s command line
+2.  cd `/tmp/tmp.yYWwRfjlka/8545` - The `tmp` directory will vary.
+3.  `geth attach --datadir .`
+
+## Adding New Parameters to Start Geth
+
+1.  The following [PR](https://github.com/vulcanize/dapptools/commit/bdfd6253ea3b310471ab4b95724cbf70a35e56e6) captures all the places where changes need to be made.
+2.  Add any necessary values to the `docker-compose.localsetup.yml` file.

setup_local_test.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This bash script will setup a local dapptools environment.
+# 1. Create a geth binary based on a local repository
+# 2. Use docker compose to build ipld-eth-db and dapptools container.
+# 3. Start your private blockchain
+
+set -e
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+####################################################################
+# Update the variable below to your local vulcanize-geth repo!     #
+####################################################################
+local_path_to_geth="../go-ethereum-cerc/"
+start_path=$(pwd)
+
+if grep "src = ../../geth-linux-amd64"  src/go-ethereum-statediff/default.nix | grep -vq "#"; then
+    echo -e "${GREEN}You've properly Updated: src/go-ethereum-statediff/default.nix${NC}"
+else
+    echo -e "${RED}You need to updated src/go-ethereum-statediff/default.nix based on the documentation!!!${NC}"
+    echo -e "${RED}Comment the fetchurl block and reference local geth${NC}"
+    exit 1
+fi
+
+cd $local_path_to_geth
+echo -e "${GREEN}Building geth!${NC}"
+docker build -t vulcanize/go-ethereum -f Dockerfile .
+docker run --rm --entrypoint cat vulcanize/go-ethereum /usr/local/bin/geth > geth-linux-amd64
+
+echo -e "${GREEN}geth build complete!${NC}"
+
+cd $start_path
+
+mv ${local_path_to_geth}/geth-linux-amd64 .
+
+echo -e "${GREEN}Running docker-compose up${NC}"
+docker-compose -f docker-compose.localsetup.yml up --build

src/dapp/libexec/dapp/dapp---testnet-launch

@@ -10,6 +10,9 @@ db-name=name          database name
 db-host=address       database host
 db-port=port          database port
 db-write=bool         turn on database write mode
+db-type=name          the type of database
+db-driver=name        the driver used for the database
+db-waitforsync=bool   Should the statediff service start once geth has synced to head (default: false)
 rpc-port=port         change RPC port (default: 8545)
 rpc-addr=address      change RPC address (default: 127.0.0.1)
 chain-id=number       change chain ID (default: 99)
@@ -30,6 +33,9 @@ DB_PASSWORD=password
 DB_NAME=vulcanize_public
 DB_HOST=127.0.0.1
 DB_PORT=5432
+DB_TYPE=postgres
+DB_DRIVER=sqlx
+DB_WAIT_FOR_SYNC=false
 RPC_PORT=8545
 RPC_ADDRESS=127.0.0.1
 PERIOD=0
@@ -47,6 +53,9 @@ while [[ $1 ]]; do
     --db-host)       shift; DB_HOST=$1;;
     --db-port)       shift; DB_PORT=$1;;
     --db-write)      shift; DB_WRITE=$1;;
+    --db-type)       shift; DB_TYPE=$1;;
+    --db-driver)     shift; DB_DRIVER=$1;;
+    --db-waitforsync) shift; DB_WAIT_FOR_SYNC=$1;;
     --rpc-port)      shift; RPC_PORT=$1;;
     --rpc-addr)      shift; RPC_ADDRESS=$1;;
     --chain-id)      shift; CHAINID=$1;;
@@ -133,8 +142,10 @@ geth \
   --mine --miner.threads=1 --allow-insecure-unlock \
   --http --http.api "web3,eth,net,debug,personal,statediff" --http.corsdomain '*' --http.vhosts '*' --nodiscover \
   --http.addr="$RPC_ADDRESS" --http.port="$RPC_PORT" --syncmode=full --gcmode=archive \
-  --statediff --statediff.db="host=$DB_HOST port=$DB_PORT user=$DB_USER password=$DB_PASSWORD dbname=$DB_NAME sslmode=disable" \
-  --statediff.dbnodeid 1 --statediff.dbclientname test1 --statediff.writing="$DB_WRITE" \
+  --statediff --statediff.db.host="$DB_HOST" --statediff.db.port="$DB_PORT" --statediff.db.user="$DB_USER" \
+  --statediff.db.password="$DB_PASSWORD" --statediff.db.name="$DB_NAME" \
+  --statediff.db.nodeid 1 --statediff.db.clientname test1 --statediff.writing="$DB_WRITE" \
+  --statediff.db.type="$DB_TYPE" --statediff.db.driver="$DB_DRIVER" --statediff.waitforsync="$DB_WAIT_FOR_SYNC" \
   --ws --ws.addr="0.0.0.0" --unlock="$(IFS=,; echo "${address[*]}")" --password=<(exit) &
 
 gethpid=$!

src/go-ethereum-statediff/default.nix

@@ -1,12 +1,14 @@
-{ stdenv, fetchurl }:
+{ stdenv, fetchurl}:
 
 stdenv.mkDerivation rec {
   name = "go-ethereum-statediff";
   version = "v1.10.14-statediff-0.0.29";
+  # If you want to test locally, comment out the fetchurl src and use the local copy below.
   src = fetchurl {
     url = "https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64";
     sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
   };
+  #src = ../../geth-linux-amd64;
 
   phases = ["installPhase" "patchPhase"];
   installPhase = ''