Docs etc

DESIGN.md

@@ -0,0 +1,18 @@
+Design
+======
+
+## Methodology.
+
+ 1. Make it work.
+ 2. Simplify / delete.
+ 3. Optimize. Make it efficient/fast.
+
+## Philosophy.
+
+1. Minimum lines of code possible. This is the ultimate test of intelligence. Intelligence is compression and prediction in one. Can you produce a world model which is the smallest most accurate thing? This is what good code looks like - the smallest most functional thing.
+
+This has many advantages. Fewest lines of code for same functionality. Purer more functional code. More auditable. More extensible more easily without additional abstraction loading.
+
+2. Discover the core primitives through distillation.
+
+3. > "Show me your [code] and conceal your [data structures], and I shall continue to be mystified. Show me your [data structures], and I won't usually need your [code]; it'll be obvious."

PROGRESS

@@ -0,0 +1,82 @@
+
+COMPONENTS
+==========
+
+block
+db
+netpeer_server
+netpeer
+pow
+tokenomics
+tx
+wallet
+merkle-tree
+    100%
+
+genesis
+    90%
+
+    need genesis block serialised as json
+    need multiple genesis blocks / network configs potentially
+
+blockdag
+    80%
+
+    needs refactoring some methods
+    needs timestamp calculation (subjective) in there
+    needs parallel sig verification
+    needs(?) to verify tx validity
+
+state-machine
+    50%
+
+    needs either UXTO model or account nonces to prevent duplicate txs
+    needs proper testing
+
+node
+    50%
+
+    needs to sync properly on divergent branches
+    needs to restart miner on new full tip
+    needs to revalidate mempool on new full tip
+    need to manage miner so it doesnt run in headers mode?
+    needs to discover other nodes via dns seed or something
+
+cli
+    10%
+
+    needs light mode to just follow chain
+    needs wallet to send/receive txs
+
+sync
+    50%
+
+    needs simplification pass
+    needs proper testing
+
+sync-downloader
+    50%
+
+    needs fixing since it fails weirdly?
+
+explorer
+    90%
+
+    entire state recomputed after each block. inefficient
+    coins showed in full integer, no decimalisation yet
+
+
+PROJECT
+=======
+
+testnet 1
+    100%
+
+    done
+
+post-testnet 1 pass
+    50%
+
+    need to clean up comments and docs
+    need to clean up repo - delete old files
+

README.md

@@ -46,7 +46,7 @@ Taking inspiration from projects like SQLite, TempleOS, Cosmos/Tendermint, Linux
 
 tinychain is in-development. Currently we can run a node, mine blocks, ingest them into a DAG, create and sign transactions, run a state machine, build the UXTO set from processing transactions, connect to peers and gossip.
 
-In progress: state synchronisation, user wallet API's.
+We ran our first testnet back in Aug 2024 in sprint 1. Now we are refining in preparation for testnet 2. See [PROGRESS](./PROGRESS) for the open state of different subcomponents.
 
 ## Install.
 

TODO

@@ -1,17 +1,4 @@
 
-DAG
-
-    [x] test IngestHeader
-    [x] test IngestBody
-    test updates full tip / headers tip
-
-Parallel download
-
-    [x] implement a really basic version which downloads in parallel. basically a dumb bittorrent
-    [x] make sure it can work with peers joining/leaving
-    [x] probably try to do it without channels just to start with (simpler mental model)
-
-
 Currently:
 - rename Sign(msg) to accept a sighash, and then add a sighash method to tx.go
 - why is block body not being matched with block header?
@@ -21,6 +8,25 @@ Currently:
     sync_test.go:361: Error ingesting body: Block header not found for txs merkle root.
 
 
+mempool
+    listen for new txs
+    put them in mempool
+    inside node/miner?
+        get a bundle from mempool
+        mine on it
+
+adjust miner
+    restart mechanism to mine on new tip
+    how does it do it now? 
+    documentation on the functionality
+
+
+state machine
+    remove sanity check for blockreward/iscoinbase
+    add proper tests
+
+
+
 
 
 Sync / search

cli/cmd/explorer.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/liamzebedee/tinychain-go/core/nakamoto"
 	"github.com/liamzebedee/tinychain-go/explorer"
 	"github.com/urfave/cli/v2"
 
@@ -15,8 +16,8 @@ func RunExplorer(cmdCtx *cli.Context) error {
 	dbPath := cmdCtx.String("db")
 
 	// DAG.
-	networks := getNetworks()
-	dag, _, _ := newBlockdag(dbPath, networks["testnet1"])
+	networks := nakamoto.GetNetworks()
+	dag, _, _ := newBlockdag(dbPath, networks["testnet1"].ConsensusConfig)
 
 	// Handle process signals.
 	c := make(chan os.Signal, 1)

cli/cmd/node.go

@@ -7,9 +7,7 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"database/sql"
-	"encoding/hex"
 	"fmt"
-	"math/big"
 	"net/url"
 	"os"
 	"os/signal"
@@ -26,36 +24,6 @@ func (m *MockStateMachine) VerifyTx(tx nakamoto.RawTransaction) error {
 	return nil
 }
 
-func getNetworks() map[string]nakamoto.ConsensusConfig {
-	genesis_difficulty := new(big.Int)
-	genesis_difficulty.SetString("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16)
-
-	// https://serhack.me/articles/story-behind-alternative-genesis-block-bitcoin/ ;)
-	genesisBlockHash_, err := hex.DecodeString("000006b15d1327d67e971d1de9116bd60a3a01556c91b6ebaa416ebc0cfaa646")
-	if err != nil {
-		panic(err)
-	}
-	genesisBlockHash_[0] += 1
-
-	genesisBlockHash := [32]byte{}
-	copy(genesisBlockHash[:], genesisBlockHash_)
-
-	network_testnet1 := nakamoto.ConsensusConfig{
-		EpochLengthBlocks:       10,
-		TargetEpochLengthMillis: 1000 * 60, // 1min, 1 block every 10s
-		GenesisDifficulty:       *genesis_difficulty,
-		GenesisParentBlockHash:  genesisBlockHash,
-		MaxBlockSizeBytes:       2 * 1024 * 1024, // 2MB
-	}
-
-	networks := map[string]nakamoto.ConsensusConfig{
-		"testnet1":   network_testnet1,
-		"terrydavis": network_testnet1,
-	}
-
-	return networks
-}
-
 func newBlockdag(dbPath string, conf nakamoto.ConsensusConfig) (nakamoto.BlockDAG, nakamoto.ConsensusConfig, *sql.DB) {
 	// TODO validate connection string.
 	fmt.Println("database path: ", dbPath)
@@ -120,8 +88,8 @@ func RunNode(cmdCtx *cli.Context) error {
 	}
 
 	// DAG.
-	networks := getNetworks()
-	conf, ok := networks[network]
+	networks := nakamoto.GetNetworks()
+	net, ok := networks[network]
 	if !ok {
 		availableNetworks := []string{}
 		for k := range networks {
@@ -130,7 +98,7 @@ func RunNode(cmdCtx *cli.Context) error {
 		fmt.Printf("Available networks: %s\n", strings.Join(availableNetworks, ", "))
 		return fmt.Errorf("Unknown network: %s", network)
 	}
-	dag, _, db := newBlockdag(dbPath, conf)
+	dag, _, db := newBlockdag(dbPath, net.ConsensusConfig)
 
 	// Miner.
 	minerWallet, err := getMinerWallet(db)

core/bittorrent_trackers.go

@@ -11,7 +11,7 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"net/url"
@@ -81,7 +81,7 @@ func addPeerToSwarm(peerID string, infoHash string, port int) error {
 	}
 	defer resp.Body.Close()
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}

core/nakamoto/block.go

@@ -185,6 +185,7 @@ func (b *RawBlock) Envelope() []byte {
 	if err != nil {
 		panic(err)
 	}
+	// TODO: sanity check NumTransactions/TransactionsMerkleRoot matches against what's in Transactions.
 	err = binary.Write(buf, binary.BigEndian, b.Nonce)
 	if err != nil {
 		panic(err)

core/nakamoto/blockdag.go

@@ -8,6 +8,7 @@ import (
 	"log"
 	"math/big"
 	"sync"
+	"time"
 
 	"github.com/liamzebedee/tinychain-go/core"
 	_ "github.com/mattn/go-sqlite3"
@@ -43,9 +44,14 @@ type BlockDAG struct {
 	// The "full node" tip. This is the tip of the heaviest chain of full blocks.
 	FullTip Block
 
-	// OnNewTip handler.
+	// Triggered on a new headers tip (light sync).
 	OnNewHeadersTip func(tip Block, prevTip Block)
-	OnNewFullTip    func(tip Block, prevTip Block)
+
+	// Triggered on a new full tip (fully-synced).
+	OnNewFullTip func(tip Block, prevTip Block)
+
+	// Get the current time according to the system clock.
+	ClockTime func() uint64
 
 	log *log.Logger
 }
@@ -72,6 +78,29 @@ func NewBlockDAGFromDB(db *sql.DB, stateMachine StateMachineInterface, consensus
 	return dag, nil
 }
 
+func (dag *BlockDAG) getClockTime() uint64 {
+	if dag.ClockTime != nil {
+		return dag.ClockTime()
+	}
+
+	now := time.Now()
+	milliseconds := now.UnixMilli()
+	return uint64(milliseconds)
+}
+
+func (dag *BlockDAG) IsTipFresh(tip Block) bool {
+	age := dag.getClockTime() - tip.Timestamp
+
+	// sanity-check.
+	if age < 0 {
+		panic("tip.Timestamp is from future")
+	}
+
+	isFresh := age < 6
+	// TODO
+	return isFresh
+}
+
 // Initalises the block DAG with the genesis block.
 func (dag *BlockDAG) initialiseBlockDAG() error {
 	genesisBlock := GetRawGenesisBlockFromConfig(dag.consensus)
@@ -215,7 +244,6 @@ func (dag *BlockDAG) UpdateTip() error {
 // Validation rules for blocks:
 // 1. Verify parent is known.
 // 2. Verify timestamp is within bounds.
-// TODO: subjectivity.
 // 3. Verify num transactions is the same as the length of the transactions list.
 // 4a. Verify coinbase transcation is present.
 // 4b. Verify transactions are valid.
@@ -394,6 +422,14 @@ func (dag *BlockDAG) IngestBlockBody(body []RawTransaction) error {
 	raw.Transactions = body
 
 	// 2. Verify timestamp is within bounds.
+	// 2a. Verify monotonic - parent.timestamp < block.timestamp.
+	// 2b. Verify not in future - block.timestamp < (now + now*1.05)
+	// 2c. Verify not in past (if we are in live mode) -
+	// - if tip is fresh, that is, the tip's age is within statistical bounds for expecting a new block from the network
+	// - ie. 0 < tip.age < avg_block_time(6 blocks)
+	// - we can expect a new block soon, then we verify the tip is within the real clock time
+	// - if we are uncertain when we will be reconnected to the network, due to a split, then we must disable the safety check.
+	//
 	// TODO: subjectivity.
 
 	// 3. Verify num transactions is the same as the length of the transactions list.
@@ -406,6 +442,7 @@ func (dag *BlockDAG) IngestBlockBody(body []RawTransaction) error {
 	if len(raw.Transactions) < 1 {
 		return fmt.Errorf("Missing coinbase tx.")
 	}
+
 	// 4b. Verify transactions.
 	// TODO: We can parallelise this.
 	// This is one of the most expensive operations of the blockchain node.

core/nakamoto/blockdag_client.go

@@ -295,6 +295,7 @@ func (dag *BlockDAG) GetLatestFullTip() (Block, error) {
 
 			-- Case 2: Blocks without transactions.
 			-- If a block has no transactions, then it is fully downloaded and is considered for the "full tip".
+			-- TODO: A block NEVER has 0 transactions! There is always the coinbase.
 			SELECT b.hash, b.acc_work
 			FROM blocks b
 			WHERE b.num_transactions = 0