From 776eec43349d62b0e72db7b5c3337edcc305c467 Mon Sep 17 00:00:00 2001
From: Abdul Rabbani <abdulrabbani00@gmail.com>
Date: Wed, 9 Mar 2022 12:44:15 -0500
Subject: [PATCH 1/4] Build Dapptool with local geth repo

This commit will allow users to reference their local vulcanize-geth repository to build dapptools. This alleviates developers from having to create pre-mature releases on GitHub so the CI/CD pipeline can create a working dapptools container.
---
 .gitignore                                  |  1 +
 docker-compose.localsetup.yml               | 40 +++++++++++++++++++++
 docker/install.sh                           |  2 ++
 docker/startup_script.sh                    |  3 +-
 docker/startup_script_wrapper.sh            |  2 +-
 setup_local_test.sh                         | 39 ++++++++++++++++++++
 src/dapp/libexec/dapp/dapp---testnet-launch | 15 ++++++--
 src/go-ethereum-statediff/default.nix       | 12 ++++---
 8 files changed, 105 insertions(+), 9 deletions(-)
 create mode 100644 docker-compose.localsetup.yml
 create mode 100755 setup_local_test.sh

diff --git a/.gitignore b/.gitignore
index b2be92b7d..18d74854e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 result
+geth-linux-amd64
diff --git a/docker-compose.localsetup.yml b/docker-compose.localsetup.yml
new file mode 100644
index 000000000..b899b4f75
--- /dev/null
+++ b/docker-compose.localsetup.yml
@@ -0,0 +1,40 @@
+version: "3.2"
+
+services:
+  dapptools:
+    restart: unless-stopped
+    depends_on:
+      - ipld-eth-db
+    #image: vulcanize/dapptools:v0.30.0-v1.10.14-statediff-0.0.29
+    build:
+      context: ./
+      dockerfile: docker/Dockerfile
+    environment:
+      DB_USER: vdbm
+      DB_NAME: vulcanize_testing
+      DB_HOST: ipld-eth-db
+      DB_PORT: 5432
+      DB_PASSWORD: password
+      DB_WRITE: "true"
+      DB_TYPE: postgres
+      DB_DRIVER: sqlx
+      DB_WAIT_FOR_SYNC: "true"
+    ports:
+      - "127.0.0.1:8545:8545"
+      - "127.0.0.1:8546:8546"
+
+  ipld-eth-db:
+    restart: always
+    image: vulcanize/ipld-eth-db:v3.0.6
+    environment:
+      POSTGRES_USER: "vdbm"
+      POSTGRES_DB: "vulcanize_testing"
+      POSTGRES_PASSWORD: "password"
+    volumes:
+      - vdb_db_eth_server:/var/lib/postgresql/data
+    ports:
+      - "127.0.0.1:8077:5432"
+    command: ["postgres", "-c", "log_statement=all"]
+
+volumes:
+  vdb_db_eth_server:
diff --git a/docker/install.sh b/docker/install.sh
index 187e90a5c..753232e81 100644
--- a/docker/install.sh
+++ b/docker/install.sh
@@ -1,3 +1,4 @@
+set -e
 GREEN='\033[0;32m'
 RED='\033[0;31m'
 NC='\033[0m'
@@ -14,6 +15,7 @@ oops() {
 #
 #[[ $RELEASE == null ]] && oops "No release found in ${API_OUTPUT}"
 
+echo -e "${GREEN}STARTING!!${NC}"
 cachix use dapp
 nix-env -iA dapp hevm seth solc go-ethereum-unlimited -f .
 
diff --git a/docker/startup_script.sh b/docker/startup_script.sh
index 2b05e6a5d..338012aae 100644
--- a/docker/startup_script.sh
+++ b/docker/startup_script.sh
@@ -8,7 +8,8 @@ trap "exit 1" SIGINT SIGTERM
 
 TMPDIR=$(mktemp -d)
 dapp testnet --rpc-addr 0.0.0.0 --chain-id 4 --db-user $DB_USER --db-password $DB_PASSWORD --db-name $DB_NAME \
-  --db-host $DB_HOST --db-port $DB_PORT --db-write $DB_WRITE --dir "$TMPDIR" --address=$ADDRESS &
+  --db-host $DB_HOST --db-port $DB_PORT --db-write $DB_WRITE --dir "$TMPDIR" --address=$ADDRESS \
+  --db-type $DB_TYPE --db-driver $DB_DRIVER -db-waitforsync $DB_WAIT_FOR_SYNC &
 echo "sleeping 90 sec"
 # give it a few secs to start up
 sleep 90
diff --git a/docker/startup_script_wrapper.sh b/docker/startup_script_wrapper.sh
index 41e77c64f..7b7fe5b38 100755
--- a/docker/startup_script_wrapper.sh
+++ b/docker/startup_script_wrapper.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
-
+set -e
 nix-shell -p cacert cachix curl jq nix --run "sh docker/startup_script.sh"
\ No newline at end of file
diff --git a/setup_local_test.sh b/setup_local_test.sh
new file mode 100755
index 000000000..57739b44e
--- /dev/null
+++ b/setup_local_test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This bash script will setup a local dapptools environment.
+# 1. Create a geth binary based on a local repository
+# 2. Use docker compose to build ipld-eth-db and dapptools container.
+# 3. Start your private blockchain
+
+set -e
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+####################################################################
+# Update the variable below to your local vulcanize-geth repo!     #
+####################################################################
+local_path_to_geth="../go-ethereum-cerc/"
+start_path=$(pwd)
+
+if grep "src = ../../geth-linux-amd64"  src/go-ethereum-statediff/default.nix | grep -vq "#"; then
+    echo -e "${GREEN}You've properly Updated: src/go-ethereum-statediff/default.nix${NC}"
+else
+    echo -e "${RED}You need to updated src/go-ethereum-statediff/default.nix based on the documentation!!!${NC}"
+    echo -e "${RED}Comment the fetchurl block and reference local geth${NC}"
+    exit 1
+fi
+
+cd $local_path_to_geth
+echo -e "${GREEN}Building geth!${NC}"
+docker build -t vulcanize/go-ethereum -f Dockerfile .
+docker run --rm --entrypoint cat vulcanize/go-ethereum /usr/local/bin/geth > geth-linux-amd64
+
+echo -e "${GREEN}geth build complete!${NC}"
+
+cd $start_path
+
+mv ${local_path_to_geth}/geth-linux-amd64 .
+
+echo -e "${GREEN}Running docker-compose up${NC}"
+docker-compose -f docker-compose.localsetup.yml up --build
\ No newline at end of file
diff --git a/src/dapp/libexec/dapp/dapp---testnet-launch b/src/dapp/libexec/dapp/dapp---testnet-launch
index 6c2e08abf..77e97bf39 100755
--- a/src/dapp/libexec/dapp/dapp---testnet-launch
+++ b/src/dapp/libexec/dapp/dapp---testnet-launch
@@ -10,6 +10,9 @@ db-name=name          database name
 db-host=address       database host
 db-port=port          database port
 db-write=bool         turn on database write mode
+db-type=name          the type of database
+db-driver=name        the driver used for the database
+db-waitforsync=bool   Should the statediff service start once geth has synced to head (default: false)
 rpc-port=port         change RPC port (default: 8545)
 rpc-addr=address      change RPC address (default: 127.0.0.1)
 chain-id=number       change chain ID (default: 99)
@@ -30,6 +33,9 @@ DB_PASSWORD=password
 DB_NAME=vulcanize_public
 DB_HOST=127.0.0.1
 DB_PORT=5432
+DB_TYPE=postgres
+DB_DRIVER=sqlx
+DB_WAIT_FOR_SYNC=false
 RPC_PORT=8545
 RPC_ADDRESS=127.0.0.1
 PERIOD=0
@@ -47,6 +53,9 @@ while [[ $1 ]]; do
     --db-host)       shift; DB_HOST=$1;;
     --db-port)       shift; DB_PORT=$1;;
     --db-write)      shift; DB_WRITE=$1;;
+    --db-type)       shift; DB_TYPE=$1;;
+    --db-driver)     shift; DB_DRIVER=$1;;
+    --db-waitforsync) shift; DB_WAIT_FOR_SYNC=$1;;
     --rpc-port)      shift; RPC_PORT=$1;;
     --rpc-addr)      shift; RPC_ADDRESS=$1;;
     --chain-id)      shift; CHAINID=$1;;
@@ -133,8 +142,10 @@ geth \
   --mine --miner.threads=1 --allow-insecure-unlock \
   --http --http.api "web3,eth,net,debug,personal,statediff" --http.corsdomain '*' --http.vhosts '*' --nodiscover \
   --http.addr="$RPC_ADDRESS" --http.port="$RPC_PORT" --syncmode=full --gcmode=archive \
-  --statediff --statediff.db="host=$DB_HOST port=$DB_PORT user=$DB_USER password=$DB_PASSWORD dbname=$DB_NAME sslmode=disable" \
-  --statediff.dbnodeid 1 --statediff.dbclientname test1 --statediff.writing="$DB_WRITE" \
+  --statediff --statediff.db.host="$DB_HOST" --statediff.db.port="$DB_PORT" --statediff.db.user="$DB_USER" \
+  --statediff.db.password="$DB_PASSWORD" --statediff.db.name="$DB_NAME" \
+  --statediff.db.nodeid 1 --statediff.db.clientname test1 --statediff.writing="$DB_WRITE" \
+  --statediff.db.type="$DB_TYPE" --statediff.db.driver="$DB_DRIVER" --statediff.waitforsync="$DB_WAIT_FOR_SYNC" \
   --ws --ws.addr="0.0.0.0" --unlock="$(IFS=,; echo "${address[*]}")" --password=<(exit) &
 
 gethpid=$!
diff --git a/src/go-ethereum-statediff/default.nix b/src/go-ethereum-statediff/default.nix
index 2b1f5cb69..cf4ae33a9 100644
--- a/src/go-ethereum-statediff/default.nix
+++ b/src/go-ethereum-statediff/default.nix
@@ -1,12 +1,14 @@
-{ stdenv, fetchurl }:
+{ stdenv, fetchurl}:
 
 stdenv.mkDerivation rec {
   name = "go-ethereum-statediff";
   version = "v1.10.14-statediff-0.0.29";
-  src = fetchurl {
-    url = "https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64";
-    sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
-  };
+  # If you want to test locally, comment out the fetchurl src and use the local copy below.
+  #src = fetchurl {
+  #  url = "https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64";
+  #  sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
+  #};
+  src = ../../geth-linux-amd64;
 
   phases = ["installPhase" "patchPhase"];
   installPhase = ''

From 7d74631ecb5f00bd23cd80a0a39b9436382c09c9 Mon Sep 17 00:00:00 2001
From: Abdul Rabbani <abdulrabbani00@gmail.com>
Date: Thu, 10 Mar 2022 13:01:06 -0500
Subject: [PATCH 2/4] Fix small error

---
 docker/startup_script.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/startup_script.sh b/docker/startup_script.sh
index 338012aae..c02e18b65 100644
--- a/docker/startup_script.sh
+++ b/docker/startup_script.sh
@@ -9,7 +9,7 @@ trap "exit 1" SIGINT SIGTERM
 TMPDIR=$(mktemp -d)
 dapp testnet --rpc-addr 0.0.0.0 --chain-id 4 --db-user $DB_USER --db-password $DB_PASSWORD --db-name $DB_NAME \
   --db-host $DB_HOST --db-port $DB_PORT --db-write $DB_WRITE --dir "$TMPDIR" --address=$ADDRESS \
-  --db-type $DB_TYPE --db-driver $DB_DRIVER -db-waitforsync $DB_WAIT_FOR_SYNC &
+  --db-type $DB_TYPE --db-driver $DB_DRIVER --db-waitforsync $DB_WAIT_FOR_SYNC &
 echo "sleeping 90 sec"
 # give it a few secs to start up
 sleep 90

From 1a7f81a6d738d6eab8568355e1428feaa2aa766f Mon Sep 17 00:00:00 2001
From: Abdul Rabbani <abdulrabbani00@gmail.com>
Date: Thu, 10 Mar 2022 13:16:52 -0500
Subject: [PATCH 3/4] Revert to default behavior

---
 src/go-ethereum-statediff/default.nix | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/go-ethereum-statediff/default.nix b/src/go-ethereum-statediff/default.nix
index cf4ae33a9..6a866bffe 100644
--- a/src/go-ethereum-statediff/default.nix
+++ b/src/go-ethereum-statediff/default.nix
@@ -4,11 +4,11 @@ stdenv.mkDerivation rec {
   name = "go-ethereum-statediff";
   version = "v1.10.14-statediff-0.0.29";
   # If you want to test locally, comment out the fetchurl src and use the local copy below.
-  #src = fetchurl {
-  #  url = "https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64";
-  #  sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
-  #};
-  src = ../../geth-linux-amd64;
+  src = fetchurl {
+    url = "https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64";
+    sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
+  };
+  #src = ../../geth-linux-amd64;
 
   phases = ["installPhase" "patchPhase"];
   installPhase = ''

From 01b3d04864f6ef893184b7d66e6677adec86e47b Mon Sep 17 00:00:00 2001
From: Abdul Rabbani <abdulrabbani00@gmail.com>
Date: Thu, 10 Mar 2022 13:20:55 -0500
Subject: [PATCH 4/4] Add a doc for local testing

---
 local-test.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 local-test.md

diff --git a/local-test.md b/local-test.md
new file mode 100644
index 000000000..045dc2de8
--- /dev/null
+++ b/local-test.md
@@ -0,0 +1,66 @@
+# How To Test Using Dapptools Locally
+
+# Overview
+
+When testing [vulcanize-geth](https://github.com/vulcanize/go-ethereum), there might be the need to use [dapptools](https://github.com/vulcanize/dapptools). Dapptools provides the following functionality:
+
+- Creates a private network that can be easier to test than the mainnet.
+- Allows you to make changes to the private network for testing purposes.
+
+# Set-Up Steps
+
+The following steps highlight how you can test geth locally using dapptools.
+
+1.  Make sure you’ve cloned both [vulcanize-geth](https://github.com/vulcanize/go-ethereum) and [dapptools](https://github.com/vulcanize/dapptools).
+
+2.  Go to the dapptools repository
+
+3.  Update the following file `setup_local_test.sh`
+
+    1.  Update `local_path_to_geth` to match the relative path to the local [vulcanize-geth](https://github.com/vulcanize/go-ethereum) repository: `local_path_to_geth="../go-ethereum-cerc/"`
+
+4.  Update the following file to use the a local version of geth: `src/go-ethereum-statediff/default.nix`
+
+    ```nix
+    #src = fetchurl {
+      #  url = "<https://github.com/vulcanize/go-ethereum/releases/download/${version}/geth-linux-amd64>";
+      #  sha256 = "0s1brrr7c36y0pap1f4vll07n2lvjc90yxa6rhx5g6jpsivmfci5";
+      #};
+    src = ../../geth-linux-amd64;
+
+    ```
+
+5.  Update Docker resource allocations (Optional).
+
+    1.  Increase memory allocation to 4GB and storage to 100GB
+    2.  This might be necessary depending on system settings.
+
+6.  Go back to the dapptools repository, and run: `./setup_local_test.sh`
+
+    1.  This will do the following:
+        1.  Build the geth binary
+        2.  Move it to the local repository
+        3.  Build a local container of the `ipld-eth-db`
+        4.  Build a local container of `dapptools`.
+
+7.  If you make a change to your geth code base, you will have to run `./setup_local_test.sh` (Optional)
+
+8.  If you simply want to re-build the docker images, you can run: `docker-compose up --build -f docker-compose.localsetup.yml`
+
+    1.  If you don’t want to rebuild the container: `docker-compose up -f docker-compose.localsetup.yml`
+        1.  If you make any changes to the geth node and don't include the `--build` flag, the new geth binary will not be included in your container!
+
+# Useful Information
+
+## Using The Geth Console
+
+To use the geth console do the following:
+
+1.  Access the container’s command line
+2.  cd `/tmp/tmp.yYWwRfjlka/8545` - The `tmp` directory will vary.
+3.  `geth attach --datadir .`
+
+## Adding New Parameters to Start Geth
+
+1.  The following [PR](https://github.com/vulcanize/dapptools/commit/bdfd6253ea3b310471ab4b95724cbf70a35e56e6) captures all the places where changes need to be made.
+2.  Add any necessary values to the `docker-compose.localsetup.yml` file.