Add an implementation of the cp32 rolling hash algorithm

.gitignore

@@ -1 +1,2 @@
 *~
+/cover.out

Readme.md

@@ -1,8 +1,8 @@
 # Hashsplit - content-based splitting of byte streams
 
-[![Go Reference](https://pkg.go.dev/badge/github.com/bobg/hashsplit/v2.svg)](https://pkg.go.dev/github.com/bobg/hashsplit/v2)
-[![Go Report Card](https://goreportcard.com/badge/github.com/bobg/hashsplit/v2)](https://goreportcard.com/report/github.com/bobg/hashsplit/v2)
-![Tests](https://github.com/bobg/hashsplit/actions/workflows/go.yml/badge.svg)
+[![Go Reference](https://pkg.go.dev/badge/github.com/bobg/hashsplit/v3.svg)](https://pkg.go.dev/github.com/bobg/hashsplit/v3)
+[![Go Report Card](https://goreportcard.com/badge/github.com/bobg/hashsplit/v3)](https://goreportcard.com/report/github.com/bobg/hashsplit/v3)
+[![Tests](https://github.com/bobg/hashsplit/actions/workflows/go.yml/badge.svg)](https://github.com/bobg/hashsplit/actions/workflows/go.yml)
 [![Coverage Status](https://coveralls.io/repos/github/bobg/hashsplit/badge.svg?branch=master)](https://coveralls.io/github/bobg/hashsplit?branch=master)
 
 Hashsplitting is a way of dividing a byte stream into pieces

cp32/cp32.go

@@ -0,0 +1,123 @@
+package cp32
+
+import "hash"
+
+// See https://hashsplit.github.io/spec.html#appendix
+var g = [...]uint32{
+	0x6b326ac4, 0x13f8e1bd, 0x1d61066f, 0x87733fc7, 0x37145391, 0x1c115e40,
+	0xd2ea17a3, 0x8650e4b1, 0xe892bb09, 0x408a0c3a, 0x3c40b72c, 0x2a988fb0,
+	0xf691d0f8, 0xb22072d9, 0x6fa8b705, 0x72bd6386, 0xdd905ac3, 0x7fcba0ba,
+	0x4f84a51c, 0x1dd8477e, 0x6f972f2c, 0xaccd018e, 0xe2964f13, 0x7a7d2388,
+	0xebf42ca7, 0xa8e2a0a2, 0x8eb726d3, 0xccd169b6, 0x5444f61e, 0xe178ad7a,
+	0xd556a18d, 0xbac80ef4, 0x34cb8a87, 0x7740a1a9, 0x62640fe1, 0xb1e64472,
+	0xdee2d6c8, 0x27849114, 0xb6333f4b, 0xbb0b5c1d, 0x57e53652, 0xfde51999,
+	0xef773313, 0x1bbaf941, 0x2e9aa084, 0x37587ab8, 0xa61e7c54, 0xb779be61,
+	0xd8795bfd, 0x1707c1f6, 0x50fe9c54, 0x32ff3685, 0x94f55c22, 0x2a32ce1a,
+	0x0b9076ab, 0x14363079, 0xae994b2c, 0x4a8da881, 0x4770b9c4, 0xf4d143dd,
+	0x70a90c0b, 0xa094582a, 0x4b254d10, 0x2454325e, 0x1725a589, 0x9a3380da,
+	0x948eeade, 0x79f88224, 0x7b8dc378, 0xc2090db6, 0x41f7a7ac, 0xd4d9528c,
+	0x7f0bace7, 0xd3157814, 0xd7757bc4, 0xb428db06, 0x2e2b1d02, 0x0499bcf5,
+	0x310f963e, 0xe5f31a83, 0xe0cd600f, 0x8b48af14, 0x568eb23a, 0x01d1150b,
+	0x33f54023, 0xa0e59fdf, 0x8d17c2dd, 0xfb7bd347, 0x4d8cd432, 0x664db8de,
+	0xd48f2a6c, 0x16c3412d, 0x873a32fc, 0x10796a21, 0xed40f0f8, 0x5ca8e9b2,
+	0x0f70d259, 0x0df532c2, 0x016d73aa, 0x45761aa5, 0x189b45a7, 0x4accd733,
+	0x641f90e3, 0x592ed9ee, 0x4b1d72ad, 0x42ff2cd4, 0x0654b609, 0x799012c0,
+	0x595f36a4, 0x082bdbd6, 0x0375ddd3, 0xc16c1fb5, 0x57492df8, 0xa2d56a98,
+	0xdfb2aa28, 0x3728f35f, 0xdc49ea71, 0x9aee8377, 0xd62de2ab, 0x2c3aa155,
+	0x407d9eed, 0xbc5b3832, 0x42961924, 0x1498172a, 0xc7126716, 0x95494b56,
+	0xd40442fb, 0xb22a3ed1, 0x0ad3e0ae, 0x77a6136a, 0xfb1bc3f0, 0x1a715c38,
+	0xccbbd21d, 0x061ff037, 0x85d700cb, 0x8a8fb396, 0x956bbe48, 0xf2556ed8,
+	0x3319c88b, 0xe0d6d3e9, 0x4783b316, 0x03a73543, 0x253be5ed, 0x41322aea,
+	0xdfc00c7a, 0x972b9413, 0xccca42f5, 0x0a1cdf35, 0xa2dc31b8, 0xf48397eb,
+	0xbe3f2b3e, 0xd2950b9f, 0xccd269cf, 0x51a64ca9, 0xea46d96e, 0xcaec892e,
+	0x3fae3a62, 0xf12e53db, 0x3753464c, 0x214fbd91, 0x609ce2f7, 0x6158b44c,
+	0xa74b8027, 0x79f36912, 0x16cac162, 0x5e76df4f, 0xbc4184fb, 0x912cac7d,
+	0xf97e5704, 0x664dd25f, 0x7d837805, 0x5386cfe0, 0x4e585d77, 0xa0fa527e,
+	0xeb5c8401, 0xa186cc51, 0x05ef3f1f, 0xc1efc774, 0x38730c2c, 0xad9c5539,
+	0x27cd4938, 0x7317b4f2, 0x852c186f, 0xa4c9b0f4, 0xf592f010, 0xf6fe86f3,
+	0xb14ba86c, 0x07109a27, 0x0d00568d, 0xd92ee49f, 0xdc643eb3, 0x8d81c333,
+	0xcd1d7bbd, 0x87ff9cda, 0x80fa4285, 0x25258d5b, 0xd9e4065a, 0x78955c18,
+	0x84874c2a, 0xfdae136b, 0x48eeb3d3, 0xc2623958, 0x5a74f96d, 0x0bcb49f5,
+	0x3041cefc, 0xa5b0a1a8, 0x2d29bae6, 0x916ace93, 0x0e70564d, 0xa24894ae,
+	0x9897044d, 0xcba97c2a, 0x52a313b1, 0x318ec481, 0xc4729ec1, 0xd90ad78a,
+	0x55eb9f90, 0x4f159fda, 0xa90fbd44, 0xd0ca6208, 0x5c597269, 0xe05a471e,
+	0x26a5e224, 0x97144944, 0xece2c486, 0xf65c9a9e, 0x82a3fbbb, 0x925d1a62,
+	0xd6c4c29b, 0x61b9292d, 0x161529c9, 0x37713240, 0x68ec933b, 0xed80a4e5,
+	0x02b2db41, 0x47cfd676, 0xbfe26b41, 0x5e8468bb, 0x6e0d15a4, 0x40383ef4,
+	0x81e622fb, 0x194b378c, 0x0c503af5, 0x8e0033a7, 0x003aaa5e, 0x9d7b6723,
+	0x0702e877, 0x34b75166, 0xd1ba98d8, 0x9b9f1794, 0xe8961c84, 0x9d773b17,
+	0xf9783ee9, 0xdff11758, 0x49bea2cf, 0xa0e0887f,
+}
+
+// Hash is a [hash.Hash] implementing the cp32 rolling hash algorithm described in the spec at https://github.com/hashsplit/hashsplit-spec.
+type Hash struct {
+	val    uint32 // current hash value
+	window []byte // a circular buffer of the last bytes written
+	next   int    // pos in window where the next byte will go
+}
+
+var _ hash.Hash32 = &Hash{}
+
+const defaultWindowSize = 64
+
+// New produces a new [Hash] with the given window size,
+// which should be a multiple of 32,
+// and is rounded up to the next multiple of 32 if it isn't.
+func New(windowSize int) *Hash {
+	if windowSize <= 0 {
+		windowSize = defaultWindowSize
+	}
+	if r := windowSize % 32; r != 0 {
+		// round windowSize up to the next multiple of 32
+		windowSize += 32 - r
+	}
+
+	h := &Hash{window: make([]byte, 0, windowSize)}
+	return h
+}
+
+func (h *Hash) Write(inp []byte) (int, error) {
+	for _, b := range inp {
+		h.Roll(b)
+	}
+	return len(inp), nil
+}
+
+func (h *Hash) WriteByte(b byte) error {
+	h.Roll(b)
+	return nil
+}
+
+func (h *Hash) Sum(b []byte) []byte {
+	return append(b, byte(h.val>>24), byte(h.val>>16), byte(h.val>>8), byte(h.val))
+}
+
+// Reset zeroes out the "window" of recently written bytes
+// and sets the hash value to the cp32 back to zero.
+func (h *Hash) Reset() {
+	h.window = h.window[:0]
+	h.val = 0
+	h.next = 0
+}
+
+func (*Hash) Size() int       { return 4 }
+func (*Hash) BlockSize() int  { return 32 }
+func (h *Hash) Sum32() uint32 { return h.val }
+
+func (h *Hash) Roll(b byte) {
+	h.val = rotl(h.val, 1) ^ g[b]
+	if l := len(h.window); l < cap(h.window) {
+		h.window = append(h.window, b)
+		return
+	}
+	x0 := h.window[h.next]
+	h.val ^= g[x0]
+	h.window[h.next] = b
+	h.next++
+	h.next %= cap(h.window)
+}
+
+func rotl(x uint32, n int) uint32 {
+	n %= 32
+	return x<<n | x>>(32-n)
+}

cp32/cp32_test.go

@@ -0,0 +1,78 @@
+package cp32
+
+import (
+	"encoding/binary"
+	"io"
+	"os"
+	"testing"
+)
+
+func BenchmarkHash(b *testing.B) {
+	h := New(64)
+	b.ResetTimer()
+	for range b.N {
+		_, _ = h.Write([]byte("hello"))
+	}
+}
+
+func TestHash(t *testing.T) {
+	cases := []struct {
+		name string
+		want uint32
+	}{{
+		name: "commonsense", want: 0xfd835584,
+	}, {
+		name: "empty", want: 0,
+	}, {
+		name: "one", want: 0x407d9eed,
+	}, {
+		name: "short", want: 0xb88e926a,
+	}}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			h := New(64)
+			f, err := os.Open("../testdata/" + tc.name)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer f.Close()
+
+			_, err = io.Copy(h, f)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			got := h.Sum32()
+			if got != tc.want {
+				t.Errorf("got 0x%x, want 0x%x", got, tc.want)
+			}
+		})
+	}
+}
+
+// Tests that hashing string xS and then rolling on y (and rolling off x) is the same as hashing Sy.
+func FuzzHash(f *testing.F) {
+	f.Fuzz(func(t *testing.T, a, b, c, d uint64, pre byte) {
+		var inp [32]byte
+		binary.Encode(inp[:], binary.LittleEndian, a)
+		binary.Encode(inp[8:], binary.LittleEndian, b)
+		binary.Encode(inp[16:], binary.LittleEndian, c)
+		binary.Encode(inp[24:], binary.LittleEndian, d)
+
+		h1 := New(32)
+		h2 := New(32)
+
+		h1.Write(inp[:])
+		h2.WriteByte(pre)
+		h2.Write(inp[:len(inp)-1])
+		h2.WriteByte(inp[len(inp)-1])
+
+		got1 := h1.Sum32()
+		got2 := h2.Sum32()
+
+		if got1 != got2 {
+			t.Errorf("h1: 0x%x, h2: 0x%x (on input %x/%d)", got1, got2, inp[:], pre)
+		}
+	})
+}

go.mod

@@ -1,10 +1,16 @@
-module github.com/bobg/hashsplit/v2
+module github.com/bobg/hashsplit/v3
 
 go 1.23
 
-require github.com/chmduquesne/rollinghash v4.0.0+incompatible
+require (
+	github.com/bobg/go-generics/v4 v4.1.1
+	github.com/bobg/seqs v1.2.0
+	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
+	github.com/google/go-cmp v0.6.0
+)
 
 require (
 	github.com/bobg/go-generics/v3 v3.7.0 // indirect
-	github.com/bobg/seqs v1.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 )

go.sum

@@ -1,6 +1,22 @@
 github.com/bobg/go-generics/v3 v3.7.0 h1:4SJHDWqONTRcA8al6491VW/ys6061bPCcTcI7YnIHPc=
 github.com/bobg/go-generics/v3 v3.7.0/go.mod h1:wGlMLQER92clsh3cJoQjbUtUEJ03FoxnGhZjaWhf4fM=
+github.com/bobg/go-generics/v4 v4.1.1 h1:At0uu6D/VksvjPjxvYbWs3k4X1/aAFiTapr3kuiuuiM=
+github.com/bobg/go-generics/v4 v4.1.1/go.mod h1:Oj7bxNHiEkq1PaCYmVA/dduJOsFnPnGV93NTkSrASI0=
 github.com/bobg/seqs v1.2.0 h1:y9SS1zisfkNQepd+xia/3ELDYR4T4vuLTyCA4UVOiKA=
 github.com/bobg/seqs v1.2.0/go.mod h1:icgB+vXIoU6s675tLYVAgcUYry1PkYwgEKvzOuFemOk=
-github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
-github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
+github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
+github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=