From 6d2321dbd81f3ed66da98ac48238e498e3a51cc4 Mon Sep 17 00:00:00 2001 From: Matthew Nibecker Date: Tue, 27 Jan 2026 13:11:07 -0800 Subject: [PATCH] Add quiet support for cut and put This commit adds quiet support for the cut and put operators in vector runtime and adds back the support for sequential runtime. --- book/src/super-sql/functions/errors/quiet.md | 2 +- book/src/super-sql/operators/cut.md | 2 +- book/src/tutorials/jq.md | 2 +- compiler/rungen/op.go | 2 + compiler/rungen/vop.go | 2 + runtime/sam/expr/dequiet.go | 72 ++++++++++++ runtime/vam/expr/quiet.go | 117 +++++++++++++++++++ runtime/vam/op/values.go | 31 +---- runtime/ztests/expr/function/quiet.yaml | 2 +- runtime/ztests/op/cut-quiet.yaml | 10 +- runtime/ztests/op/put-quiet.yaml | 4 +- 11 files changed, 203 insertions(+), 43 deletions(-) create mode 100644 runtime/sam/expr/dequiet.go create mode 100644 runtime/vam/expr/quiet.go diff --git a/book/src/super-sql/functions/errors/quiet.md b/book/src/super-sql/functions/errors/quiet.md index e34222147e..8b06482cd4 100644 --- a/book/src/super-sql/functions/errors/quiet.md +++ b/book/src/super-sql/functions/errors/quiet.md @@ -53,5 +53,5 @@ cut b:=x+1,c:=quiet(x+1),d:=quiet(a+1) # input {a:1} # expected output -{b:error("missing"),c:error("quiet"),d:2} +{b:error("missing"),d:2} ``` diff --git a/book/src/super-sql/operators/cut.md b/book/src/super-sql/operators/cut.md index fec80eb213..4d549594a2 100644 --- a/book/src/super-sql/operators/cut.md +++ b/book/src/super-sql/operators/cut.md @@ -81,7 +81,7 @@ cut a:=quiet(a),d:=quiet(d) # input {a:1,b:2,c:3} # expected output -{a:1,d:error("quiet")} +{a:1} ``` --- diff --git a/book/src/tutorials/jq.md b/book/src/tutorials/jq.md index dbbc34ca58..d00aa69152 100644 --- a/book/src/tutorials/jq.md +++ b/book/src/tutorials/jq.md @@ -387,7 +387,7 @@ echo '{s:"foo", val:1}{s:"bar"}' | super -s -c 'cut quiet(val)' - produces ```mdtest-output {val:1} -{val:error("quiet")} +{} ``` ### Union Types diff --git a/compiler/rungen/op.go b/compiler/rungen/op.go index 8018e8fbbc..a7a1d5cb4e 100644 --- a/compiler/rungen/op.go +++ b/compiler/rungen/op.go @@ -278,6 +278,7 @@ func (b *Builder) compileLeaf(o dag.Op, parent sbuf.Puller) (sbuf.Puller, error) if err != nil { return nil, err } + e = expr.NewDequiet(b.sctx(), e) return values.New(parent, []expr.Evaluator{e}), nil case *dag.DropOp: fields := make(field.List, 0, len(v.Args)) @@ -323,6 +324,7 @@ func (b *Builder) compileLeaf(o dag.Op, parent sbuf.Puller) (sbuf.Puller, error) if err != nil { return nil, err } + e = expr.NewDequiet(b.sctx(), e) putter := expr.NewPutter(b.sctx(), e) return values.New(parent, []expr.Evaluator{putter}), nil case *dag.RenameOp: diff --git a/compiler/rungen/vop.go b/compiler/rungen/vop.go index eb61b21bea..d218eb2ff2 100644 --- a/compiler/rungen/vop.go +++ b/compiler/rungen/vop.go @@ -224,6 +224,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vector.Puller) (vector.Puller, if err != nil { return nil, err } + e = vamexpr.NewDequiet(b.sctx(), e) return vamop.NewValues(b.sctx(), parent, []vamexpr.Evaluator{e}), nil case *dag.DefaultScan: sbufPuller, err := b.compileLeaf(o, nil) @@ -280,6 +281,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vector.Puller) (vector.Puller, if err != nil { return nil, err } + e = vamexpr.NewDequiet(b.sctx(), e) putter := vamexpr.NewPutter(b.sctx(), e) return vamop.NewValues(b.sctx(), parent, []vamexpr.Evaluator{putter}), nil case *dag.RenameOp: diff --git a/runtime/sam/expr/dequiet.go b/runtime/sam/expr/dequiet.go new file mode 100644 index 0000000000..70c8d051ff --- /dev/null +++ b/runtime/sam/expr/dequiet.go @@ -0,0 +1,72 @@ +package expr + +import ( + "github.com/brimdata/super" + "github.com/brimdata/super/scode" +) + +type Dequiet struct { + sctx *super.Context + expr Evaluator + rmtyp *super.TypeError + builder scode.Builder +} + +func NewDequiet(sctx *super.Context, expr Evaluator) Evaluator { + return &Dequiet{ + sctx: sctx, + expr: expr, + rmtyp: sctx.LookupTypeError(sctx.MustLookupTypeRecord(nil)), + } +} + +func (d *Dequiet) Eval(this super.Value) super.Value { + val := d.expr.Eval(this) + if val.Type().Kind() == super.RecordKind { + d.builder.Reset() + typ := d.rec(&d.builder, val.Type(), val.Bytes()) + return super.NewValue(typ, d.builder.Bytes().Body()) + } + return val +} + +func (d *Dequiet) rec(builder *scode.Builder, typ super.Type, b scode.Bytes) super.Type { + if b == nil { + builder.Append(nil) + return typ + } + rtyp := super.TypeRecordOf(typ) + if rtyp == nil { + builder.Append(nil) + return typ + } + var changed bool + builder.BeginContainer() + var fields []super.Field + it := b.Iter() + for _, f := range rtyp.Fields { + fbytes := it.Next() + ftyp := d.dequiet(builder, f.Type, fbytes) + if ftyp == nil { + changed = true + continue + } + fields = append(fields, super.NewField(f.Name, ftyp)) + } + builder.EndContainer() + if !changed { + return typ + } + return d.sctx.MustLookupTypeRecord(fields) +} + +func (d *Dequiet) dequiet(builder *scode.Builder, typ super.Type, b scode.Bytes) super.Type { + if typ.Kind() == super.RecordKind { + return d.rec(builder, typ, b) + } + if errtyp, ok := typ.(*super.TypeError); ok && errtyp.IsQuiet(b) { + return nil + } + builder.Append(b) + return typ +} diff --git a/runtime/vam/expr/quiet.go b/runtime/vam/expr/quiet.go new file mode 100644 index 0000000000..8e2af9c86b --- /dev/null +++ b/runtime/vam/expr/quiet.go @@ -0,0 +1,117 @@ +package expr + +import ( + "github.com/brimdata/super" + "github.com/brimdata/super/vector" + "github.com/brimdata/super/vector/bitvec" +) + +func QuietMask(vec vector.Any) (vector.Any, bool) { + errvec, ok := vector.Under(vec).(*vector.Error) + if !ok || vector.KindOf(errvec.Vals) != vector.KindString { + return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false + } + lhs := vector.NewConst(super.NewString("quiet"), vec.Len(), bitvec.Zero) + out := NewCompare(nil, "!=", nil, nil).Compare(lhs, errvec.Vals) + if nulls := vector.NullsOf(out); !nulls.IsZero() { + // Flip nulls to true since a null result is not error("quiet"). + b := FlattenBool(out) + return vector.NewBool(bitvec.Or(b.Bits, nulls), bitvec.Zero), true + } + return out, true +} + +type Dequiet struct { + sctx *super.Context + expr Evaluator + rmtyp *super.TypeError +} + +func NewDequiet(sctx *super.Context, expr Evaluator) Evaluator { + return &Dequiet{ + sctx: sctx, + expr: expr, + rmtyp: sctx.LookupTypeError(sctx.MustLookupTypeRecord(nil)), + } +} + +func (d *Dequiet) Eval(this vector.Any) vector.Any { + return vector.Apply(true, func(vecs ...vector.Any) vector.Any { + vec := vector.Under(vecs[0]) + if vector.KindOf(vec) == vector.KindRecord { + vec = d.rec(vec) + } + return vec + }, d.expr.Eval(this)) +} + +func (d *Dequiet) rec(vec vector.Any) vector.Any { + var index []uint32 + if view, ok := vec.(*vector.View); ok { + index = view.Index + vec = view.Any + } + var vecs []vector.Any + rec := vec.(*vector.Record) + if len(rec.Fields) == 0 { + return vec + } + for _, field := range rec.Fields { + vec := field + if index != nil { + vec = vector.Pick(field, index) + } + vecs = append(vecs, d.dequiet(vec)) + } + if !rec.Nulls.IsZero() { + // Keep track of incoming nulls + vecs = append(vecs, vector.NewBool(rec.Nulls, bitvec.Zero)) + } + return vector.Apply(true, func(vecs ...vector.Any) vector.Any { + var nulls bitvec.Bits + if !rec.Nulls.IsZero() { + nulls = vecs[len(vecs)-1].(*vector.Bool).Bits + vecs = vecs[:len(vecs)-1] + } + var fields []super.Field + var vals []vector.Any + for i, vec := range vecs { + typ := vec.Type() + if typ == d.rmtyp { + continue + } + fields = append(fields, rec.Typ.Fields[i]) + vals = append(vals, vec) + } + rtyp := d.sctx.MustLookupTypeRecord(fields) + return vector.NewRecord(rtyp, vals, vecs[0].Len(), nulls) + }, vecs...) +} + +func (d *Dequiet) dequiet(vec vector.Any) vector.Any { + if vector.KindOf(vec) == vector.KindRecord { + return d.rec(vec) + } + mask, ok := QuietMask(vec) + if !ok { + return vec + } + b, _ := BoolMask(new(Not).eval(mask)) + if b.IsEmpty() { + return vec + } + n := uint32(b.GetCardinality()) + quiet := d.quietTmp(n) + if n == vec.Len() { + return quiet + } + index := b.ToArray() + vec = vector.ReversePick(vec, index) + out := vector.Combine(vec, index, quiet).(*vector.Dynamic) + utyp := d.sctx.LookupTypeUnion([]super.Type{vec.Type(), quiet.Type()}) + return vector.NewUnion(utyp, out.Tags, out.Values, bitvec.Zero) +} + +func (d *Dequiet) quietTmp(n uint32) vector.Any { + return vector.NewError(d.rmtyp, vector.NewConst(super.Null, n, bitvec.Zero), bitvec.Zero) +} diff --git a/runtime/vam/op/values.go b/runtime/vam/op/values.go index 4161321e4f..7fd937a367 100644 --- a/runtime/vam/op/values.go +++ b/runtime/vam/op/values.go @@ -4,7 +4,6 @@ import ( "github.com/brimdata/super" "github.com/brimdata/super/runtime/vam/expr" "github.com/brimdata/super/vector" - "github.com/brimdata/super/vector/bitvec" ) type Values struct { @@ -63,7 +62,7 @@ func interleave(vals []vector.Any) vector.Any { func filterQuiet(vec vector.Any) vector.Any { var filtered bool mask := vector.Apply(true, func(vecs ...vector.Any) vector.Any { - mask, hasfiltered := quietMask(vecs[0]) + mask, hasfiltered := expr.QuietMask(vecs[0]) filtered = filtered || hasfiltered return mask }, vec) @@ -73,31 +72,3 @@ func filterQuiet(vec vector.Any) vector.Any { masked, _ := applyMask(vec, mask) return masked } - -func quietMask(vec vector.Any) (vector.Any, bool) { - errvec, ok := vec.(*vector.Error) - if !ok { - return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false - } - if _, ok := errvec.Vals.Type().(*super.TypeOfString); !ok { - return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false - } - if c, ok := errvec.Vals.(*vector.Const); ok { - if s, _ := c.AsString(); s == "quiet" { - return vector.NewConst(super.False, vec.Len(), bitvec.Zero), true - } - return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false - } - n := vec.Len() - mask := vector.NewFalse(n) - switch vec := vec.(type) { - case *vector.Error: - for i := uint32(0); i < n; i++ { - if s, _ := vector.StringValue(vec.Vals, i); s == "quiet" { - continue - } - mask.Set(i) - } - } - return mask, true -} diff --git a/runtime/ztests/expr/function/quiet.yaml b/runtime/ztests/expr/function/quiet.yaml index b5f0dd7bf0..3ef1f59bb7 100644 --- a/runtime/ztests/expr/function/quiet.yaml +++ b/runtime/ztests/expr/function/quiet.yaml @@ -1,4 +1,4 @@ -spq: quiet(this) +spq: values quiet(this) vector: true diff --git a/runtime/ztests/op/cut-quiet.yaml b/runtime/ztests/op/cut-quiet.yaml index a27cadf627..32458ff3c0 100644 --- a/runtime/ztests/op/cut-quiet.yaml +++ b/runtime/ztests/op/cut-quiet.yaml @@ -1,5 +1,3 @@ -skip: needs dequiet - spq: cut foo:=quiet(foo), bar:=quiet(bar) vector: true @@ -18,14 +16,15 @@ output: | {bar:"bar2"} {bar:"bar3"} {foo:"foo4",bar:"bar4"} + {} --- -skip: needs dequiet - # Rename fields. spq: cut f:=quiet(foo), b:=quiet(bar) +vector: true + input: | {foo:"foo0"} {foo:"foo1",goo:"goo1"} @@ -40,11 +39,10 @@ output: | {b:"bar2"} {b:"bar3"} {f:"foo4",b:"bar4"} + {} --- -skip: needs dequiet - # Rename nested fields. spq: cut ports:=id, resp_p:=quiet(id.resp_p) diff --git a/runtime/ztests/op/put-quiet.yaml b/runtime/ztests/op/put-quiet.yaml index d90af14416..9214f84f4f 100644 --- a/runtime/ztests/op/put-quiet.yaml +++ b/runtime/ztests/op/put-quiet.yaml @@ -1,5 +1,3 @@ -skip: needs dequiet - spq: x := quiet(y) vector: true @@ -10,4 +8,4 @@ input: | output: | {} - {x:1} + {}