From 1f629e0087212dd915c2e089f92e64a2f4e63bd9 Mon Sep 17 00:00:00 2001 From: Marcin Gorzynski Date: Wed, 1 Oct 2025 14:24:06 +0200 Subject: [PATCH 1/2] fix: first block num --- common.go | 3 + filter.go | 2 - filter_test.go | 6 +- index.go | 14 ++--- indexer.go | 11 +++- reader.go | 19 +++++-- writer.go | 17 +++++- writer_no_gap.go | 5 +- writer_no_gap_test.go | 64 ++++++++++++++++++++-- writer_test.go | 71 +++++++++++++++++++++++- writer_with_verify_hash_test.go | 97 +++++++++++++++++++++++++++++++++ 11 files changed, 278 insertions(+), 31 deletions(-) diff --git a/common.go b/common.go index 353f34d..02b7f82 100644 --- a/common.go +++ b/common.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math" "os" "path" "sort" @@ -22,6 +23,8 @@ import ( "github.com/c2h5oh/datasize" ) +const NoBlockNum = uint64(math.MaxUint64) + type Dataset struct { Name string Version string diff --git a/filter.go b/filter.go index 0c5c60b..a75ba5d 100644 --- a/filter.go +++ b/filter.go @@ -43,8 +43,6 @@ func (o FilterBuilderOptions[T]) WithDefaults() FilterBuilderOptions[T] { } type filterBuilder[T any] struct { - ctx context.Context - indexes map[IndexName]Index[T] fs storage.FS } diff --git a/filter_test.go b/filter_test.go index c2074b1..54145bd 100644 --- a/filter_test.go +++ b/filter_test.go @@ -58,7 +58,7 @@ func generateMixedIntBlocks() []Block[[]int] { // 45-49 generate 5 blocks with no data // 50-69 generate 20 blocks with random but repeating huge numbers - for i := 1; i <= 20; i++ { + for i := 0; i <= 20; i++ { blocks = append(blocks, Block[[]int]{ Hash: common.BytesToHash([]byte{byte(i)}), Number: uint64(i), @@ -288,7 +288,7 @@ func TestIntMixFiltering(t *testing.T) { } onlyEvenResults := onlyEvenFilter.IndexIterator(context.Background()) - assert.Len(t, onlyEvenResults.Bitmap().ToArray(), 20) + assert.Len(t, onlyEvenResults.Bitmap().ToArray(), 21) for _, block := range onlyEvenResults.Bitmap().ToArray() { assert.True(t, block <= 20) } @@ -373,7 +373,7 @@ func TestLowestIndexedBlockNum(t *testing.T) { }) assert.NoError(t, err) lowestBlockIndexed = indexer.BlockNum() - assert.Equal(t, uint64(0), lowestBlockIndexed) + assert.Equal(t, NoBlockNum, lowestBlockIndexed) blocks := generateIntBlocks() for _, block := range blocks[:50] { err = indexer.Index(context.Background(), block) diff --git a/index.go b/index.go index d3d76c6..9c0ecd9 100644 --- a/index.go +++ b/index.go @@ -50,7 +50,7 @@ func (u *IndexUpdate) Merge(update *IndexUpdate) { u.BlockBitmap[indexValue].Or(bm) } - if u.LastBlockNum < update.LastBlockNum { + if u.LastBlockNum == NoBlockNum || u.LastBlockNum < update.LastBlockNum { u.LastBlockNum = update.LastBlockNum } } @@ -97,7 +97,7 @@ func (i *Index[T]) IndexBlock(ctx context.Context, fs storage.FS, block Block[T] return nil, fmt.Errorf("unexpected: failed to get number of blocks indexed: %w", err) } - if block.Number <= numBlocksIndexed { + if numBlocksIndexed != NoBlockNum && block.Number <= numBlocksIndexed { return nil, nil } } @@ -148,7 +148,7 @@ func (i *Index[T]) Store(ctx context.Context, fs storage.FS, indexUpdate *IndexU if err != nil { return fmt.Errorf("failed to get number of blocks indexed: %w", err) } - if lastBlockNumIndexed >= indexUpdate.LastBlockNum { + if lastBlockNumIndexed != NoBlockNum && lastBlockNumIndexed >= indexUpdate.LastBlockNum { return nil } @@ -191,19 +191,19 @@ func (i *Index[T]) LastBlockNumIndexed(ctx context.Context, fs storage.FS) (uint file, err := fs.Open(ctx, indexedBlockNumFilePath(string(i.name)), nil) if err != nil { // file doesn't exist - return 0, nil + return NoBlockNum, nil } defer file.Close() buf, err := io.ReadAll(file) if err != nil { - return 0, fmt.Errorf("failed to read IndexBlock file: %w", err) + return NoBlockNum, fmt.Errorf("failed to read IndexBlock file: %w", err) } var numBlocksIndexed uint64 err = binary.Read(bytes.NewReader(buf), binary.BigEndian, &numBlocksIndexed) if err != nil { - return 0, fmt.Errorf("failed to unmarshal bitmap: %w", err) + return NoBlockNum, fmt.Errorf("failed to unmarshal bitmap: %w", err) } i.numBlocksIndexed = &atomic.Uint64{} @@ -219,7 +219,7 @@ func (i *Index[T]) storeLastBlockNumIndexed(ctx context.Context, fs storage.FS, prevBlockIndexed = blocksIndexed } - if prevBlockIndexed >= numBlocksIndexed { + if prevBlockIndexed != NoBlockNum && prevBlockIndexed >= numBlocksIndexed { return nil } diff --git a/indexer.go b/indexer.go index e385e0f..5ccb504 100644 --- a/indexer.go +++ b/indexer.go @@ -134,16 +134,21 @@ func (i *Indexer[T]) BlockNum() uint64 { i.mu.Lock() defer i.mu.Unlock() + // initialize lowestBlockNum to math.MaxUint64 var lowestBlockNum uint64 = math.MaxUint64 for _, indexUpdate := range i.indexUpdates { + // if no blocks have been indexed, return NoBlockNum + if indexUpdate.LastBlockNum == NoBlockNum { + return NoBlockNum + } + + // update lowestBlockNum if the current indexUpdate has a lower last block number if indexUpdate.LastBlockNum < lowestBlockNum { lowestBlockNum = indexUpdate.LastBlockNum } } - if lowestBlockNum == math.MaxUint64 { - return 0 - } + // return the lowest block number indexed by all indexes return lowestBlockNum } diff --git a/reader.go b/reader.go index 0dc4afd..21fc271 100644 --- a/reader.go +++ b/reader.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "math" "os" "sync" "time" @@ -29,10 +30,9 @@ type Reader[T any] interface { } type reader[T any] struct { - options Options - path string - fs storage.FS - useCompression bool + options Options + path string + fs storage.FS closer io.Closer @@ -140,8 +140,11 @@ func (r *reader[T]) Read(ctx context.Context) (Block[T], error) { } } - var block Block[T] - for structs.IsZero(block) || block.Number <= r.lastBlockNum { + var ( + block Block[T] + hasSuccessfullyReadBlock bool + ) + for !hasSuccessfullyReadBlock || block.Number <= r.lastBlockNum && r.lastBlockNum != math.MaxUint64 { select { case <-ctx.Done(): return Block[T]{}, ctx.Err() @@ -168,6 +171,8 @@ func (r *reader[T]) Read(ctx context.Context) (Block[T], error) { r.lastBlockNum = block.Number } + hasSuccessfullyReadBlock = true + if !r.isBlockWithin(block) { currentFile := r.fileIndex.At(r.currFileIndex) return Block[T]{}, fmt.Errorf("block number %d is out of file block %d-%d range", @@ -181,6 +186,8 @@ func (r *reader[T]) Read(ctx context.Context) (Block[T], error) { return Block[T]{}, fmt.Errorf("failed to decode file data: %w", err) } + hasSuccessfullyReadBlock = true + if !r.isBlockWithin(block) { currentFile := r.fileIndex.At(r.currFileIndex) return Block[T]{}, fmt.Errorf("block number %d is out of file block %d-%d range", diff --git a/writer.go b/writer.go index 6507c17..23949e6 100644 --- a/writer.go +++ b/writer.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "math" "os" "sync" @@ -77,18 +78,26 @@ func NewWriter[T any](opt Options) (Writer[T], error) { return nil, fmt.Errorf("failed to load file index: %w", err) } + var firstBlockNum uint64 var lastBlockNum uint64 var fileIndexFileList = fileIndex.Files() if len(fileIndexFileList) > 0 { lastBlockNum = fileIndexFileList[len(fileIndexFileList)-1].LastBlockNum } + if lastBlockNum == 0 { + firstBlockNum = NoBlockNum + lastBlockNum = NoBlockNum + } else { + firstBlockNum = lastBlockNum + 1 + } + // create new writer return &writer[T]{ options: opt, path: datasetPath, fs: fs, - firstBlockNum: lastBlockNum + 1, + firstBlockNum: firstBlockNum, lastBlockNum: lastBlockNum, fileIndex: fileIndex, buffer: bytes.NewBuffer(make([]byte, 0, defaultFileSize)), @@ -103,7 +112,7 @@ func (w *writer[T]) Write(ctx context.Context, b Block[T]) error { w.mu.Lock() defer w.mu.Unlock() - if w.lastBlockNum >= b.Number { + if w.lastBlockNum != math.MaxUint64 && w.lastBlockNum >= b.Number { return nil } @@ -118,6 +127,10 @@ func (w *writer[T]) Write(ctx context.Context, b Block[T]) error { return fmt.Errorf("failed to encode file data: %w", err) } + if w.firstBlockNum == NoBlockNum { + w.firstBlockNum = b.Number + } + w.lastBlockNum = b.Number w.options.FileRollPolicy.onBlockProcessed(w.lastBlockNum) return nil diff --git a/writer_no_gap.go b/writer_no_gap.go index 2a1df1b..e665af0 100644 --- a/writer_no_gap.go +++ b/writer_no_gap.go @@ -13,7 +13,7 @@ type noGapWriter[T any] struct { } func NewWriterNoGap[T any](w Writer[T]) Writer[T] { - return &noGapWriter[T]{w: w} + return &noGapWriter[T]{w: w, lastBlockNum: w.BlockNum()} } func (n *noGapWriter[T]) FileSystem() storage.FS { @@ -24,7 +24,7 @@ func (n *noGapWriter[T]) Write(ctx context.Context, b Block[T]) error { defer func() { n.lastBlockNum = b.Number }() // skip if block number is less than or equal to last block number - if b.Number <= n.lastBlockNum { + if n.lastBlockNum != NoBlockNum && b.Number <= n.lastBlockNum { return nil } @@ -40,6 +40,7 @@ func (n *noGapWriter[T]) Write(ctx context.Context, b Block[T]) error { return err } } + return n.w.Write(ctx, b) } diff --git a/writer_no_gap_test.go b/writer_no_gap_test.go index c9ea154..b8f2411 100644 --- a/writer_no_gap_test.go +++ b/writer_no_gap_test.go @@ -30,6 +30,9 @@ func TestWriterNoGap(t *testing.T) { ngw := NewWriterNoGap[int](w) require.NotNil(t, w) + err = ngw.Write(context.Background(), Block[int]{Number: 0}) + require.NoError(t, err) + err = ngw.Write(context.Background(), Block[int]{Number: 1}) require.NoError(t, err) @@ -46,7 +49,7 @@ func TestWriterNoGap(t *testing.T) { require.NoError(t, err) walData, err := os.ReadFile( - path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 1, LastBlockNum: 3}).Path()), + path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 0, LastBlockNum: 3}).Path()), ) require.NoError(t, err) @@ -59,7 +62,57 @@ func TestWriterNoGap(t *testing.T) { blockCount++ } - require.Equal(t, 3, blockCount) + require.Equal(t, 4, blockCount) + }) + + t.Run("gap_first_block", func(t *testing.T) { + defer testTeardown(t) + + opt := Options{ + Dataset: Dataset{ + Name: "int-wal", + Path: testPath, + Version: defaultDatasetVersion, + }, + NewEncoder: NewJSONEncoder, + }.WithDefaults() + + w, err := NewWriter[int](opt) + require.NoError(t, err) + + ngw := NewWriterNoGap[int](w) + require.NotNil(t, w) + + err = ngw.Write(context.Background(), Block[int]{Number: 1}) + require.NoError(t, err) + + err = ngw.Write(context.Background(), Block[int]{Number: 2}) + require.NoError(t, err) + + err = ngw.Write(context.Background(), Block[int]{Number: 3}) + require.NoError(t, err) + + err = (w.(*writer[int])).rollFile(context.Background()) + require.NoError(t, err) + + err = ngw.Close(context.Background()) + require.NoError(t, err) + + walData, err := os.ReadFile( + path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 0, LastBlockNum: 3}).Path()), + ) + require.NoError(t, err) + + d := NewJSONDecoder(bytes.NewBuffer(walData)) + + var b Block[int] + var blockCount int + for d.Decode(&b) != io.EOF { + require.NoError(t, err) + blockCount++ + } + + require.Equal(t, 4, blockCount) }) t.Run("gap_3_10", func(t *testing.T) { @@ -80,6 +133,9 @@ func TestWriterNoGap(t *testing.T) { ngw := NewWriterNoGap[int](w) require.NotNil(t, w) + err = ngw.Write(context.Background(), Block[int]{Number: 0}) + require.NoError(t, err) + err = ngw.Write(context.Background(), Block[int]{Number: 1}) require.NoError(t, err) @@ -98,7 +154,7 @@ func TestWriterNoGap(t *testing.T) { require.NoError(t, err) walData, err := os.ReadFile( - path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 1, LastBlockNum: 10}).Path()), + path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 0, LastBlockNum: 10}).Path()), ) require.NoError(t, err) @@ -111,6 +167,6 @@ func TestWriterNoGap(t *testing.T) { blockCount++ } - require.Equal(t, 10, blockCount) + require.Equal(t, 11, blockCount) }) } diff --git a/writer_test.go b/writer_test.go index 85977c7..07210d5 100644 --- a/writer_test.go +++ b/writer_test.go @@ -15,6 +15,12 @@ import ( func TestWriter_Write(t *testing.T) { blocksFile := Blocks[int]{ + { + Hash: common.BytesToHash([]byte{0x00}), + Number: 0, + TS: 0, + Data: 0, + }, { Hash: common.BytesToHash([]byte{0x01}), Number: 1, @@ -124,7 +130,7 @@ func TestWriter_Write(t *testing.T) { require.NoError(t, err) // check WAL files - filePath := path.Join(buildETHWALPath(tc.options.Dataset.Name, tc.options.Dataset.Version, tc.options.Dataset.Path), (&File{FirstBlockNum: 1, LastBlockNum: 4}).Path()) + filePath := path.Join(buildETHWALPath(tc.options.Dataset.Name, tc.options.Dataset.Version, tc.options.Dataset.Path), (&File{FirstBlockNum: 0, LastBlockNum: 4}).Path()) _, err = os.Stat(filePath) require.NoError(t, err) @@ -153,6 +159,64 @@ func TestWriter_Write(t *testing.T) { } } +func TestWriter_Write_ZeroBlockNum(t *testing.T) { + defer testTeardown(t) + + w, err := NewWriter[int](Options{ + Dataset: Dataset{ + Name: "int-wal", + Path: testPath, + Version: defaultDatasetVersion, + }, + }) + require.NoError(t, err) + + err = w.Write(context.Background(), Block[int]{Number: 0, Hash: common.BytesToHash([]byte{0x01}), Parent: common.Hash{0x00}}) + require.NoError(t, err) + + err = w.Write(context.Background(), Block[int]{Number: 1, Hash: common.BytesToHash([]byte{0x02}), Parent: common.BytesToHash([]byte{0x01})}) + require.NoError(t, err) + + err = w.Write(context.Background(), Block[int]{Number: 2, Hash: common.BytesToHash([]byte{0x03}), Parent: common.BytesToHash([]byte{0x02})}) + require.NoError(t, err) + + err = w.RollFile(context.Background()) + require.NoError(t, err) + + err = w.Write(context.Background(), Block[int]{Number: 3, Hash: common.BytesToHash([]byte{0x04}), Parent: common.BytesToHash([]byte{0x03})}) + require.NoError(t, err) + + err = w.Write(context.Background(), Block[int]{Number: 4, Hash: common.BytesToHash([]byte{0x05}), Parent: common.BytesToHash([]byte{0x04})}) + require.NoError(t, err) + + err = w.Write(context.Background(), Block[int]{Number: 5, Hash: common.BytesToHash([]byte{0x06}), Parent: common.BytesToHash([]byte{0x05})}) + require.NoError(t, err) + + err = w.RollFile(context.Background()) + require.NoError(t, err) + + err = w.Close(context.Background()) + require.NoError(t, err) + + // check WAL files + r, err := NewReader[int](Options{ + Dataset: Dataset{ + Name: "int-wal", + Path: testPath, + Version: defaultDatasetVersion, + }, + }) + require.NoError(t, err) + + err = r.Seek(context.Background(), 0) + require.NoError(t, err) + + for i := 0; i < 6; i++ { + blk, err := r.Read(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(i), blk.Number) + } +} func TestWriter_Continue(t *testing.T) { defer testTeardown(t) @@ -250,6 +314,9 @@ func TestNoGapWriter_FileRollOnClose(t *testing.T) { ngw := NewWriterNoGap[int](w) require.NotNil(t, w) + err = ngw.Write(context.Background(), Block[int]{Number: 0}) + require.NoError(t, err) + err = ngw.Write(context.Background(), Block[int]{Number: 1}) require.NoError(t, err) @@ -265,7 +332,7 @@ func TestNoGapWriter_FileRollOnClose(t *testing.T) { require.Equal(t, uint64(3), w.BlockNum()) // check WAL files - filePath := path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 1, LastBlockNum: 3}).Path()) + filePath := path.Join(buildETHWALPath(opt.Dataset.Name, opt.Dataset.Version, opt.Dataset.Path), (&File{FirstBlockNum: 0, LastBlockNum: 3}).Path()) _, err = os.Stat(filePath) require.NoError(t, err) } diff --git a/writer_with_verify_hash_test.go b/writer_with_verify_hash_test.go index 4aae1e2..dbe8cfb 100644 --- a/writer_with_verify_hash_test.go +++ b/writer_with_verify_hash_test.go @@ -552,3 +552,100 @@ func TestBlockHashGetterFromReader_WithCompression(t *testing.T) { tc.blockNum, tc.hash.String(), hash.String()) } } + +func TestWriterWithVerifyHash_Write_BlockZero(t *testing.T) { + defer testTeardown(t) + + options := Options{ + Dataset: Dataset{ + Name: "test-block-zero", + Path: testPath, + Version: defaultDatasetVersion, + }, + NewEncoder: NewJSONEncoder, + NewDecoder: NewJSONDecoder, + FileRollOnClose: true, + } + + ctx := context.Background() + + // Create a writer + w, err := NewWriter[int](options) + require.NoError(t, err) + + // Create a writerWithVerifyHash + mockGetter := &mockBlockHashGetter{} + verifyWriter := NewWriterWithVerifyHash[int](w, mockGetter.GetHash) + + // Write block 0 + block0 := Block[int]{ + Hash: common.BytesToHash([]byte{0x00}), + Parent: common.Hash{}, // Block 0 has empty parent + Number: 0, + Data: 100, + } + + err = verifyWriter.Write(ctx, block0) + require.NoError(t, err, "Block 0 should be written successfully") + assert.Equal(t, uint64(0), w.BlockNum(), "BlockNum should return 0") + + // Write block 1 to ensure file gets flushed (writer needs at least one block >= firstBlockNum) + block1 := Block[int]{ + Hash: common.BytesToHash([]byte{0x01}), + Parent: block0.Hash, + Number: 1, + Data: 101, + } + + err = verifyWriter.Write(ctx, block1) + require.NoError(t, err, "Block 1 should be written successfully") + + // Debug: Check writer state before close + writerImpl := w.(*writer[int]) + t.Logf("Before close: firstBlockNum=%d, lastBlockNum=%d", writerImpl.firstBlockNum, writerImpl.lastBlockNum) + + // Close writer to flush to disk + err = w.Close(ctx) + require.NoError(t, err) + + // Debug: Check file index + reader, err := NewReader[int](options) + require.NoError(t, err) + defer reader.Close() + + fileIndex := reader.FileIndex() + files := fileIndex.Files() + t.Logf("Number of files: %d", len(files)) + for i, f := range files { + t.Logf("File %d: blocks %d-%d, path=%s", i, f.FirstBlockNum, f.LastBlockNum, f.Path()) + } + + // Verify we can read block 0 directly with the reader (already created above for debugging) + err = reader.Seek(ctx, 0) + require.NoError(t, err, "Should be able to seek to block 0") + + readBlock0, err := reader.Read(ctx) + require.NoError(t, err, "Should be able to read block 0") + assert.Equal(t, uint64(0), readBlock0.Number, "Block number should be 0") + assert.Equal(t, block0.Hash, readBlock0.Hash, "Block 0 hash should match") + assert.Equal(t, block0.Data, readBlock0.Data, "Block 0 data should match") + + // Read block 1 + readBlock1, err := reader.Read(ctx) + require.NoError(t, err, "Should be able to read block 1") + assert.Equal(t, uint64(1), readBlock1.Number, "Block number should be 1") + assert.Equal(t, block1.Hash, readBlock1.Hash, "Block 1 hash should match") + assert.Equal(t, block1.Data, readBlock1.Data, "Block 1 data should match") + + // Also verify using BlockHashGetterFromReader + getter := BlockHashGetterFromReader[int](options) + hash0, err := getter(ctx, 0) + require.NoError(t, err, "Should be able to get block 0 hash via getter") + assert.Equal(t, block0.Hash, hash0, "Block 0 hash from getter should match") + + hash1, err := getter(ctx, 1) + require.NoError(t, err, "Should be able to get block 1 hash via getter") + assert.Equal(t, block1.Hash, hash1, "Block 1 hash from getter should match") + + mockGetter.AssertNotCalled(t, "GetHash") +} From 96075fc76304fc06cd68ffcfca4c0d66fb71b83d Mon Sep 17 00:00:00 2001 From: Marcin Gorzynski Date: Wed, 1 Oct 2025 15:46:23 +0200 Subject: [PATCH 2/2] fix: block 0 can not be read --- reader.go | 9 +-- writer_with_verify_hash.go | 2 +- writer_with_verify_hash_test.go | 104 ++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/reader.go b/reader.go index 21fc271..045fca7 100644 --- a/reader.go +++ b/reader.go @@ -97,10 +97,11 @@ func NewReader[T any](opt Options) (Reader[T], error) { } return &reader[T]{ - options: opt, - path: datasetPath, - fs: fs, - fileIndex: fileIndex, + options: opt, + path: datasetPath, + fs: fs, + fileIndex: fileIndex, + lastBlockNum: uint64(math.MaxUint64), }, nil } diff --git a/writer_with_verify_hash.go b/writer_with_verify_hash.go index 4f93493..1869db4 100644 --- a/writer_with_verify_hash.go +++ b/writer_with_verify_hash.go @@ -46,7 +46,7 @@ func NewWriterWithVerifyHash[T any](writer Writer[T], blockHashGetter BlockHashG func (w *writerWithVerifyHash[T]) Write(ctx context.Context, b Block[T]) error { var err error - if w.prevHash == (common.Hash{}) && b.Number > 1 { + if w.prevHash == (common.Hash{}) && b.Number > 0 { w.prevHash, err = w.blockHashGetter(ctx, b.Number-1) if err != nil { return fmt.Errorf("failed to get block hash: %w", err) diff --git a/writer_with_verify_hash_test.go b/writer_with_verify_hash_test.go index dbe8cfb..f8ee767 100644 --- a/writer_with_verify_hash_test.go +++ b/writer_with_verify_hash_test.go @@ -649,3 +649,107 @@ func TestWriterWithVerifyHash_Write_BlockZero(t *testing.T) { mockGetter.AssertNotCalled(t, "GetHash") } + +func TestBlockHashGetterFromReader(t *testing.T) { + defer testTeardown(t) + + options := Options{ + Dataset: Dataset{ + Name: "test-block-zero", + Path: testPath, + Version: defaultDatasetVersion, + }, + NewEncoder: NewJSONEncoder, + NewDecoder: NewJSONDecoder, + FileRollOnClose: true, + } + + w, err := NewWriter[int](options) + require.NoError(t, err) + + verifyWriter := NewWriterWithVerifyHash[int](w, BlockHashGetterFromReader[int](options)) + + block0 := Block[int]{ + Hash: common.BytesToHash([]byte{0x01}), + Parent: common.Hash{0x00}, + Number: 0, + Data: 100, + } + + err = verifyWriter.Write(context.Background(), block0) + require.NoError(t, err) + + block1 := Block[int]{ + Hash: common.BytesToHash([]byte{0x02}), + Parent: block0.Hash, + Number: 1, + Data: 101, + } + + err = verifyWriter.Write(context.Background(), block1) + require.NoError(t, err) + + err = w.RollFile(context.Background()) + require.NoError(t, err) + + err = w.Close(context.Background()) + require.NoError(t, err) + + r, err := NewReader[int](options) + require.NoError(t, err) + + readBlock0, err := r.Read(context.Background()) + require.NoError(t, err) + assert.Equal(t, block0.Hash, readBlock0.Hash) + assert.Equal(t, block0.Data, readBlock0.Data) + + readBlock1, err := r.Read(context.Background()) + require.NoError(t, err) + assert.Equal(t, block1.Hash, readBlock1.Hash) + assert.Equal(t, block1.Data, readBlock1.Data) + + // continue writing with verify writer + w, err = NewWriter[int](options) + require.NoError(t, err) + + verifyWriter = NewWriterWithVerifyHash[int](w, BlockHashGetterFromReader[int](options)) + + block3 := Block[int]{ + Hash: common.BytesToHash([]byte{0x03}), + Parent: block1.Hash, + Number: 2, + Data: 103, + } + + err = verifyWriter.Write(context.Background(), block3) + require.NoError(t, err) + + block4 := Block[int]{ + Hash: common.BytesToHash([]byte{0x04}), + Parent: block3.Hash, + Number: 3, + Data: 104, + } + + err = verifyWriter.Write(context.Background(), block4) + require.NoError(t, err) + + err = w.RollFile(context.Background()) + require.NoError(t, err) + + err = w.Close(context.Background()) + require.NoError(t, err) + + r, err = NewReader[int](options) + require.NoError(t, err) + + var blocks = []Block[int]{block0, block1, block3, block4} + for _, block := range blocks { + readBlock, err := r.Read(context.Background()) + require.NoError(t, err) + assert.Equal(t, block.Hash, readBlock.Hash) + assert.Equal(t, block.Data, readBlock.Data) + } + + require.NoError(t, r.Close()) +}