From be101873685b5287cc88731fd4648fc69d9fae9e Mon Sep 17 00:00:00 2001 From: Queyrouzec Date: Thu, 30 Oct 2025 14:15:47 -0500 Subject: [PATCH 1/6] increased eval branch quota on query parse --- query.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/query.zig b/query.zig index 3f33e08..1a62c11 100644 --- a/query.zig +++ b/query.zig @@ -47,6 +47,7 @@ pub fn ParsedQuery(comptime tmp_query: []const u8) type { }; fn parse() ParsedQueryResult { + @setEvalBranchQuota(100000); // This contains the final SQL query after parsing with our // own typed bind markers removed. var buf: [tmp_query.len]u8 = undefined; @@ -69,6 +70,7 @@ pub fn ParsedQuery(comptime tmp_query: []const u8) type { var hold_pos = 0; for (tmp_query) |c| { + @setEvalBranchQuota(100000); switch (state) { .start => switch (c) { '?', ':', '@', '$' => { From 0ec2f4afbf6234ed318e8e078f5bbb0d0e0b489b Mon Sep 17 00:00:00 2001 From: Queyrouzec Date: Wed, 31 Dec 2025 17:05:28 -0600 Subject: [PATCH 2/6] all tests working but optional binding --- .gitignore | 1 + build.zig | 15 ++++++++--- build/Preprocessor.zig | 17 ++++++------ query.zig | 15 +++++------ sqlite.zig | 60 ++++++++++++++---------------------------- vtab.zig | 4 +-- 6 files changed, 48 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index 1874bb6..5b7b369 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ deps.zig core.* /qemu*.core /fuzz/outputs +.DS_Store diff --git a/build.zig b/build.zig index b0938b9..07fd913 100644 --- a/build.zig +++ b/build.zig @@ -5,6 +5,7 @@ const mem = std.mem; const ResolvedTarget = std.Build.ResolvedTarget; const Query = std.Target.Query; const builtin = @import("builtin"); +const Io = std.Io; const Preprocessor = @import("build/Preprocessor.zig"); @@ -145,6 +146,8 @@ pub fn build(b: *std.Build) !void { const query = b.standardTargetOptionsQueryOnly(.{}); const target = b.resolveTargetQuery(query); const optimize = b.standardOptimizeOption(.{}); + var threaded = Io.Threaded.init_single_threaded; + const io = threaded.io(); // Upstream dependency const sqlite_dep = b.dependency("sqlite", .{ @@ -268,16 +271,17 @@ pub fn build(b: *std.Build) !void { // Tools // - addPreprocessStep(b, sqlite_dep); + addPreprocessStep(b, io, sqlite_dep); } -fn addPreprocessStep(b: *std.Build, sqlite_dep: *std.Build.Dependency) void { +fn addPreprocessStep(b: *std.Build, io: Io, sqlite_dep: *std.Build.Dependency) void { var wf = b.addWriteFiles(); // Preprocessing step const preprocess = PreprocessStep.create(b, .{ .source = sqlite_dep.path("."), .target = wf.getDirectory(), + .io = io, }); preprocess.step.dependOn(&wf.step); @@ -341,12 +345,14 @@ const PreprocessStep = struct { const Config = struct { source: std.Build.LazyPath, target: std.Build.LazyPath, + io: Io, }; step: std.Build.Step, source: std.Build.LazyPath, target: std.Build.LazyPath, + io: Io, fn create(owner: *std.Build, config: Config) *PreprocessStep { const step = owner.allocator.create(PreprocessStep) catch @panic("OOM"); @@ -359,6 +365,7 @@ const PreprocessStep = struct { }), .source = config.source, .target = config.target, + .io = config.io, }; return step; @@ -374,7 +381,7 @@ const PreprocessStep = struct { const loadable_sqlite3_h = try ps.target.path(owner, "loadable-ext-sqlite3.h").getPath3(owner, step).toString(owner.allocator); const loadable_sqlite3ext_h = try ps.target.path(owner, "loadable-ext-sqlite3ext.h").getPath3(owner, step).toString(owner.allocator); - try Preprocessor.sqlite3(owner.allocator, sqlite3_h, loadable_sqlite3_h); - try Preprocessor.sqlite3ext(owner.allocator, sqlite3ext_h, loadable_sqlite3ext_h); + try Preprocessor.sqlite3(owner.allocator, ps.io, sqlite3_h, loadable_sqlite3_h); + try Preprocessor.sqlite3ext(owner.allocator, ps.io, sqlite3ext_h, loadable_sqlite3ext_h); } }; diff --git a/build/Preprocessor.zig b/build/Preprocessor.zig index f3d0dc6..929f28c 100644 --- a/build/Preprocessor.zig +++ b/build/Preprocessor.zig @@ -1,6 +1,7 @@ const std = @import("std"); const debug = std.debug; const mem = std.mem; +const Io = std.Io; // This tool is used to preprocess the sqlite3 headers to make them usable to build loadable extensions. // @@ -24,11 +25,11 @@ const mem = std.mem; // This works but it requires fairly extensive modifications of both sqlite3.h and sqlite3ext.h which is time consuming to do manually; // this tool is intended to automate all these modifications. -fn readOriginalData(allocator: mem.Allocator, path: []const u8) ![]const u8 { - var file = try std.fs.cwd().openFile(path, .{}); - defer file.close(); +fn readOriginalData(allocator: mem.Allocator, io: Io, path: []const u8) ![]const u8 { + var file = try Io.Dir.cwd().openFile(io, path, .{}); + defer file.close(io); var buf: [1024]u8 = undefined; - var reader = file.reader(&buf); + var reader = file.reader(io, &buf); const data = reader.interface.readAlloc(allocator, 1024 * 1024); return data; @@ -153,8 +154,8 @@ const Processor = struct { } }; -pub fn sqlite3(allocator: mem.Allocator, input_path: []const u8, output_path: []const u8) !void { - const data = try readOriginalData(allocator, input_path); +pub fn sqlite3(allocator: mem.Allocator, io: Io, input_path: []const u8, output_path: []const u8) !void { + const data = try readOriginalData(allocator, io, input_path); var processor = try Processor.init(allocator, data); @@ -201,8 +202,8 @@ pub fn sqlite3(allocator: mem.Allocator, input_path: []const u8, output_path: [] try processor.dump(&out_writer); } -pub fn sqlite3ext(allocator: mem.Allocator, input_path: []const u8, output_path: []const u8) !void { - const data = try readOriginalData(allocator, input_path); +pub fn sqlite3ext(allocator: mem.Allocator, io: Io, input_path: []const u8, output_path: []const u8) !void { + const data = try readOriginalData(allocator, io, input_path); var processor = try Processor.init(allocator, data); diff --git a/query.zig b/query.zig index 1a62c11..42435eb 100644 --- a/query.zig +++ b/query.zig @@ -168,7 +168,7 @@ pub fn ParsedQuery(comptime tmp_query: []const u8) type { // Handles optional types const typ = if (type_info_string[0] == '?') blk: { const child_type = ParseType(type_info_string[1..]); - break :blk @Type(std.builtin.Type{ + break :blk @TypeOf(std.builtin.Type{ .optional = .{ .child = child_type, }, @@ -229,14 +229,11 @@ fn ParseType(comptime type_info: []const u8) type { if (mem.eql(u8, "isize", type_info)) return isize; if (type_info[0] == 'u' or type_info[0] == 'i') { - return @Type(std.builtin.Type{ - .int = std.builtin.Type.Int{ - .signedness = if (type_info[0] == 'i') .signed else .unsigned, - .bits = std.fmt.parseInt(usize, type_info[1..type_info.len], 10) catch { - @compileError("invalid type info " ++ type_info); - }, - }, - }); + return @Int( + if (type_info[0] == 'i') .signed else .unsigned, + std.fmt.parseInt(u16, type_info[1..], 10) catch + @compileError("invalid type info " ++ type_info), + ); } // Float diff --git a/sqlite.zig b/sqlite.zig index 8a000aa..3868fb8 100644 --- a/sqlite.zig +++ b/sqlite.zig @@ -3,7 +3,7 @@ const builtin = @import("builtin"); const build_options = @import("build_options"); const debug = std.debug; const heap = std.heap; -const io = std.io; +const Io = std.Io; const mem = std.mem; const testing = std.testing; @@ -116,7 +116,7 @@ pub const Blob = struct { } }; - // Used when reading or binding data. + // TODO remove data: []const u8, // Used for incremental i/o. @@ -132,16 +132,9 @@ pub const Blob = struct { } } - pub const Reader = io.GenericReader(*Self, errors.Error, read); - - /// reader returns a io.Reader. - pub fn reader(self: *Self) Reader { - return .{ .context = self }; - } - - fn read(self: *Self, buffer: []u8) Error!usize { + pub fn read_from_db(self: *Self, buffer: []u8) Error![]u8 { if (self.offset >= self.size) { - return 0; + return ""; } const tmp_buffer = blk: { @@ -161,17 +154,10 @@ pub const Blob = struct { self.offset += @intCast(tmp_buffer.len); - return tmp_buffer.len; + return tmp_buffer; } - pub const Writer = io.GenericWriter(*Self, Error, write); - - /// writer returns a io.Writer. - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - - fn write(self: *Self, data: []const u8) Error!usize { + pub fn write_data_to_db(self: *Self, data: []const u8) Error!void { const result = c.sqlite3_blob_write( self.handle, data.ptr, @@ -183,8 +169,6 @@ pub const Blob = struct { } self.offset += @intCast(data.len); - - return data.len; } /// Reset the offset used for reading and writing. @@ -1470,7 +1454,7 @@ pub fn Iterator(comptime Type: type) type { }, inline .@"struct", .@"union" => |TI| { if (TI.layout == .@"packed" and !@hasField(FieldType, "readField")) { - const Backing = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(FieldType) } }); + const Backing = @TypeOf(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(FieldType) } }); return @bitCast(self.readInt(Backing, i)); } @@ -1701,7 +1685,7 @@ pub const DynamicStatement = struct { }, .@"union" => |info| { if (info.layout == .@"packed") { - const Backing = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(FieldType) } }); + const Backing = @TypeOf(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(FieldType) } }); try self.bindField(Backing, options, field_name, i, @as(Backing, @bitCast(field))); return; } @@ -3091,10 +3075,6 @@ test "sqlite: statement iterator" { } test "sqlite: blob open, reopen" { - var arena = std.heap.ArenaAllocator.init(testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - var db = try getTestDb(); defer db.deinit(); @@ -3120,14 +3100,14 @@ test "sqlite: blob open, reopen" { { // Write the first blob data - var blob_writer = blob.writer(); - try blob_writer.writeAll(blob_data1); - try blob_writer.writeAll(blob_data1); + try blob.write_data_to_db(blob_data1); + try blob.write_data_to_db(blob_data1); blob.reset(); - var blob_reader = blob.reader(); - const data = try blob_reader.readAllAlloc(allocator, 8192); + var read_buff: [8192]u8 = undefined; + + const data = try blob.read_from_db(&read_buff); try testing.expectEqualSlices(u8, blob_data1 ** 2, data); } @@ -3137,14 +3117,14 @@ test "sqlite: blob open, reopen" { { // Write the second blob data - var blob_writer = blob.writer(); - try blob_writer.writeAll(blob_data2); - try blob_writer.writeAll(blob_data2); + try blob.write_data_to_db(blob_data2); + try blob.write_data_to_db(blob_data2); blob.reset(); - var blob_reader = blob.reader(); - const data = try blob_reader.readAllAlloc(allocator, 8192); + var read_buff: [8192]u8 = undefined; + + const data = try blob.read_from_db(&read_buff); try testing.expectEqualSlices(u8, blob_data2 ** 2, data); } @@ -3749,7 +3729,7 @@ test "sqlite: create aggregate function with no aggregate context" { var db = try getTestDb(); defer db.deinit(); - var rand = std.Random.DefaultPrng.init(@intCast(std.time.milliTimestamp())); + var rand = std.Random.DefaultPrng.init(@intCast((try std.time.Instant.now()).timestamp.nsec)); // Create an aggregate function working with a MyContext @@ -3810,7 +3790,7 @@ test "sqlite: create aggregate function with an aggregate context" { var db = try getTestDb(); defer db.deinit(); - var rand = std.Random.DefaultPrng.init(@intCast(std.time.milliTimestamp())); + var rand = std.Random.DefaultPrng.init(@intCast((try std.time.Instant.now()).timestamp.nsec)); try db.createAggregateFunction( "mySum", diff --git a/vtab.zig b/vtab.zig index 19826d0..a7fd770 100644 --- a/vtab.zig +++ b/vtab.zig @@ -1070,15 +1070,13 @@ const TestVirtualTable = struct { _ = self; _ = diags; - var id_str_writer = builder.id_str_buffer.writer(builder.allocator); - var argv_index: i32 = 0; for (builder.constraints) |*constraint| { if (constraint.op == .eq) { argv_index += 1; constraint.usage.argv_index = argv_index; - try id_str_writer.print("={d:<6}", .{constraint.column}); + try builder.id_str_buffer.print(builder.allocator, "={d:<6}", .{constraint.column}); } } From d085ea7ea072d11e2eed47ac979ae41b92e3132a Mon Sep 17 00:00:00 2001 From: Queyrouzec Date: Wed, 31 Dec 2025 17:53:51 -0600 Subject: [PATCH 3/6] finished build system errors --- build.zig | 14 +++++++------- build/Preprocessor.zig | 24 ++++++++++++------------ query.zig | 10 +++++----- sqlite.zig | 35 ++++++++++++++++++----------------- vtab.zig | 2 +- 5 files changed, 43 insertions(+), 42 deletions(-) diff --git a/build.zig b/build.zig index 07fd913..2cc3c95 100644 --- a/build.zig +++ b/build.zig @@ -122,15 +122,15 @@ fn makeSQLiteLib(b: *std.Build, dep: *std.Build.Dependency, c_flags: []const []c .root_module = mod, }); - lib.addIncludePath(dep.path(".")); - lib.addIncludePath(b.path("c")); + lib.root_module.addIncludePath(dep.path(".")); + lib.root_module.addIncludePath(b.path("c")); if (sqlite_c == .with) { - lib.addCSourceFile(.{ + lib.root_module.addCSourceFile(.{ .file = dep.path("sqlite3.c"), .flags = c_flags, }); } - lib.addCSourceFile(.{ + lib.root_module.addCSourceFile(.{ .file = b.path("c/workaround.c"), .flags = c_flags, }); @@ -245,9 +245,9 @@ pub fn build(b: *std.Build) !void { .name = test_name, .root_module = mod, }); - tests.addIncludePath(b.path("c")); - tests.addIncludePath(sqlite_dep.path(".")); - tests.linkLibrary(test_sqlite_lib); + tests.root_module.addIncludePath(b.path("c")); + tests.root_module.addIncludePath(sqlite_dep.path(".")); + tests.root_module.linkLibrary(test_sqlite_lib); const tests_options = b.addOptions(); tests.root_module.addImport("build_options", tests_options.createModule()); diff --git a/build/Preprocessor.zig b/build/Preprocessor.zig index 929f28c..47cb354 100644 --- a/build/Preprocessor.zig +++ b/build/Preprocessor.zig @@ -193,13 +193,14 @@ pub fn sqlite3(allocator: mem.Allocator, io: Io, input_path: []const u8, output_ // Write the result - var output_file = try std.fs.cwd().createFile(output_path, .{ .mode = 0o0644 }); - defer output_file.close(); + var output_file = try Io.Dir.cwd().createFile(io, output_path, .{}); + defer output_file.close(io); - try output_file.writeAll("/* sqlite3.h edited by the zig-sqlite build script */\n"); - var buf: [1024]u8 = undefined; - var out_writer = output_file.writer(&buf); - try processor.dump(&out_writer); + var write_buff: [1028]u8 = undefined; + + var w = output_file.writer(io, &write_buff); + + try w.interface.writeAll("/* sqlite3.h edited by the zig-sqlite build script */\n"); } pub fn sqlite3ext(allocator: mem.Allocator, io: Io, input_path: []const u8, output_path: []const u8) !void { @@ -231,11 +232,10 @@ pub fn sqlite3ext(allocator: mem.Allocator, io: Io, input_path: []const u8, outp // Write the result - var output_file = try std.fs.cwd().createFile(output_path, .{ .mode = 0o0644 }); - defer output_file.close(); + var output_file = try Io.Dir.cwd().createFile(io, output_path, .{}); + defer output_file.close(io); - try output_file.writeAll("/* sqlite3ext.h edited by the zig-sqlite build script */\n"); - var buf: [1024]u8 = undefined; - var out_writer = output_file.writer(&buf); - try processor.dump(&out_writer); + var write_buff: [1028]u8 = undefined; + var w = output_file.writer(io, &write_buff); + try w.interface.writeAll("/* sqlite3ext.h edited by the zig-sqlite build script */\n"); } diff --git a/query.zig b/query.zig index 42435eb..506842a 100644 --- a/query.zig +++ b/query.zig @@ -168,11 +168,7 @@ pub fn ParsedQuery(comptime tmp_query: []const u8) type { // Handles optional types const typ = if (type_info_string[0] == '?') blk: { const child_type = ParseType(type_info_string[1..]); - break :blk @TypeOf(std.builtin.Type{ - .optional = .{ - .child = child_type, - }, - }); + break :blk ?child_type; } else blk: { break :blk ParseType(type_info_string); }; @@ -322,6 +318,10 @@ test "parsed query: bind markers types" { .query = "foobar " ++ prefix ++ "{?[]const u8}", .expected_marker = .{ .typed = ?[]const u8 }, }, + .{ + .query = "foobar " ++ prefix ++ "{[]const u8}", + .expected_marker = .{ .typed = []const u8 }, + }, }; inline for (testCases) |tc| { diff --git a/sqlite.zig b/sqlite.zig index e99f0e7..89efd5e 100644 --- a/sqlite.zig +++ b/sqlite.zig @@ -2944,23 +2944,24 @@ test "sqlite: optional" { try testing.expectEqual(true, row.?.is_published.?); } - { - const data: ?[]const u8 = "hello"; - try db.exec("INSERT INTO article(author_id, data) VALUES(?, :data{?[]const u8})", .{}, .{ - .author_id = 20, - .dhe = data, - }); - - const row = try db.oneAlloc( - []const u8, - arena.allocator(), - "SELECT data FROM article WHERE author_id = ?", - .{}, - .{ .author_id = 20 }, - ); - try testing.expect(row != null); - try testing.expectEqualStrings(data.?, row.?); - } + // TODO enable + // { + // const data: ?[]const u8 = "hello"; + // try db.exec("INSERT INTO article(author_id, data) VALUES(?, :data{?[]const u8})", .{}, .{ + // .author_id = 20, + // .dhe = data, + // }); + + // const row = try db.oneAlloc( + // []const u8, + // arena.allocator(), + // "SELECT data FROM article WHERE author_id = ?", + // .{}, + // .{ .author_id = 20 }, + // ); + // try testing.expect(row != null); + // try testing.expectEqualStrings(data.?, row.?); + // } } test "sqlite: statement reset" { diff --git a/vtab.zig b/vtab.zig index a7fd770..e657b41 100644 --- a/vtab.zig +++ b/vtab.zig @@ -1172,7 +1172,7 @@ const TestVirtualTableCursor = struct { // 3 chars for the '=' marker // 6 chars because we format all columns in a 6 char wide string const col_str = id[pos + 1 .. pos + 1 + 6]; - const col = try fmt.parseInt(i32, mem.trimRight(u8, col_str, " "), 10); + const col = try fmt.parseInt(i32, mem.trimEnd(u8, col_str, " "), 10); id = id[pos + 1 + 6 ..]; From 2dbc5ff62e51c21473cb8fb8bbf5391638c1b5f3 Mon Sep 17 00:00:00 2001 From: Queyrouzec Date: Wed, 31 Dec 2025 17:59:20 -0600 Subject: [PATCH 4/6] added back in test code --- sqlite.zig | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/sqlite.zig b/sqlite.zig index 89efd5e..e99f0e7 100644 --- a/sqlite.zig +++ b/sqlite.zig @@ -2944,24 +2944,23 @@ test "sqlite: optional" { try testing.expectEqual(true, row.?.is_published.?); } - // TODO enable - // { - // const data: ?[]const u8 = "hello"; - // try db.exec("INSERT INTO article(author_id, data) VALUES(?, :data{?[]const u8})", .{}, .{ - // .author_id = 20, - // .dhe = data, - // }); - - // const row = try db.oneAlloc( - // []const u8, - // arena.allocator(), - // "SELECT data FROM article WHERE author_id = ?", - // .{}, - // .{ .author_id = 20 }, - // ); - // try testing.expect(row != null); - // try testing.expectEqualStrings(data.?, row.?); - // } + { + const data: ?[]const u8 = "hello"; + try db.exec("INSERT INTO article(author_id, data) VALUES(?, :data{?[]const u8})", .{}, .{ + .author_id = 20, + .dhe = data, + }); + + const row = try db.oneAlloc( + []const u8, + arena.allocator(), + "SELECT data FROM article WHERE author_id = ?", + .{}, + .{ .author_id = 20 }, + ); + try testing.expect(row != null); + try testing.expectEqualStrings(data.?, row.?); + } } test "sqlite: statement reset" { From 8af0adf25e1738df6b1a949e5de477e56beea0f6 Mon Sep 17 00:00:00 2001 From: Queyrouzec Date: Wed, 31 Dec 2025 18:00:52 -0600 Subject: [PATCH 5/6] updated TODO to better reflect intent --- sqlite.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite.zig b/sqlite.zig index e99f0e7..c79b511 100644 --- a/sqlite.zig +++ b/sqlite.zig @@ -116,7 +116,7 @@ pub const Blob = struct { } }; - // TODO remove + // TODO may be worth while to remove with the new reader and writer updates data: []const u8, // Used for incremental i/o. From 3f72b448e11ac55873dde4711e74ca02e7f97ed3 Mon Sep 17 00:00:00 2001 From: Queyrouzec Date: Thu, 1 Jan 2026 11:03:41 -0600 Subject: [PATCH 6/6] Updated tests to work across platforms --- sqlite.zig | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sqlite.zig b/sqlite.zig index c79b511..7c36860 100644 --- a/sqlite.zig +++ b/sqlite.zig @@ -3729,7 +3729,8 @@ test "sqlite: create aggregate function with no aggregate context" { var db = try getTestDb(); defer db.deinit(); - var rand = std.Random.DefaultPrng.init(@intCast((try std.time.Instant.now()).timestamp.nsec)); + const instant = std.time.Instant.now() catch .{ .timestamp = 2048 }; + var rand = std.Random.DefaultPrng.init(@intCast(instant.timestamp)); // Create an aggregate function working with a MyContext @@ -3790,7 +3791,8 @@ test "sqlite: create aggregate function with an aggregate context" { var db = try getTestDb(); defer db.deinit(); - var rand = std.Random.DefaultPrng.init(@intCast((try std.time.Instant.now()).timestamp.nsec)); + const instant = std.time.Instant.now() catch .{ .timestamp = 2048 }; + var rand = std.Random.DefaultPrng.init(@intCast(instant.timestamp)); try db.createAggregateFunction( "mySum",