Add quiet support for cut and put

book/src/super-sql/functions/errors/quiet.md

@@ -53,5 +53,5 @@ cut b:=x+1,c:=quiet(x+1),d:=quiet(a+1)
 # input
 {a:1}
 # expected output
-{b:error("missing"),c:error("quiet"),d:2}
+{b:error("missing"),d:2}
 ```

book/src/super-sql/operators/cut.md

@@ -81,7 +81,7 @@ cut a:=quiet(a),d:=quiet(d)
 # input
 {a:1,b:2,c:3}
 # expected output
-{a:1,d:error("quiet")}
+{a:1}
 ```
 
 ---

book/src/tutorials/jq.md

@@ -387,7 +387,7 @@ echo '{s:"foo", val:1}{s:"bar"}' | super -s -c 'cut quiet(val)' -
 produces
 ```mdtest-output
 {val:1}
-{val:error("quiet")}
+{}
 ```
 
 ### Union Types

compiler/rungen/op.go

@@ -278,6 +278,7 @@ func (b *Builder) compileLeaf(o dag.Op, parent sbuf.Puller) (sbuf.Puller, error)
 		if err != nil {
 			return nil, err
 		}
+		e = expr.NewDequiet(b.sctx(), e)
 		return values.New(parent, []expr.Evaluator{e}), nil
 	case *dag.DropOp:
 		fields := make(field.List, 0, len(v.Args))
@@ -323,6 +324,7 @@ func (b *Builder) compileLeaf(o dag.Op, parent sbuf.Puller) (sbuf.Puller, error)
 		if err != nil {
 			return nil, err
 		}
+		e = expr.NewDequiet(b.sctx(), e)
 		putter := expr.NewPutter(b.sctx(), e)
 		return values.New(parent, []expr.Evaluator{putter}), nil
 	case *dag.RenameOp:

compiler/rungen/vop.go

@@ -224,6 +224,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vector.Puller) (vector.Puller,
 		if err != nil {
 			return nil, err
 		}
+		e = vamexpr.NewDequiet(b.sctx(), e)
 		return vamop.NewValues(b.sctx(), parent, []vamexpr.Evaluator{e}), nil
 	case *dag.DefaultScan:
 		sbufPuller, err := b.compileLeaf(o, nil)
@@ -280,6 +281,7 @@ func (b *Builder) compileVamLeaf(o dag.Op, parent vector.Puller) (vector.Puller,
 		if err != nil {
 			return nil, err
 		}
+		e = vamexpr.NewDequiet(b.sctx(), e)
 		putter := vamexpr.NewPutter(b.sctx(), e)
 		return vamop.NewValues(b.sctx(), parent, []vamexpr.Evaluator{putter}), nil
 	case *dag.RenameOp:

runtime/sam/expr/dequiet.go

@@ -0,0 +1,72 @@
+package expr
+
+import (
+	"github.com/brimdata/super"
+	"github.com/brimdata/super/scode"
+)
+
+type Dequiet struct {
+	sctx    *super.Context
+	expr    Evaluator
+	rmtyp   *super.TypeError
+	builder scode.Builder
+}
+
+func NewDequiet(sctx *super.Context, expr Evaluator) Evaluator {
+	return &Dequiet{
+		sctx:  sctx,
+		expr:  expr,
+		rmtyp: sctx.LookupTypeError(sctx.MustLookupTypeRecord(nil)),
+	}
+}
+
+func (d *Dequiet) Eval(this super.Value) super.Value {
+	val := d.expr.Eval(this)
+	if val.Type().Kind() == super.RecordKind {
+		d.builder.Reset()
+		typ := d.rec(&d.builder, val.Type(), val.Bytes())
+		return super.NewValue(typ, d.builder.Bytes().Body())
+	}
+	return val
+}
+
+func (d *Dequiet) rec(builder *scode.Builder, typ super.Type, b scode.Bytes) super.Type {
+	if b == nil {
+		builder.Append(nil)
+		return typ
+	}
+	rtyp := super.TypeRecordOf(typ)
+	if rtyp == nil {
+		builder.Append(nil)
+		return typ
+	}
+	var changed bool
+	builder.BeginContainer()
+	var fields []super.Field
+	it := b.Iter()
+	for _, f := range rtyp.Fields {
+		fbytes := it.Next()
+		ftyp := d.dequiet(builder, f.Type, fbytes)
+		if ftyp == nil {
+			changed = true
+			continue
+		}
+		fields = append(fields, super.NewField(f.Name, ftyp))
+	}
+	builder.EndContainer()
+	if !changed {
+		return typ
+	}
+	return d.sctx.MustLookupTypeRecord(fields)
+}
+
+func (d *Dequiet) dequiet(builder *scode.Builder, typ super.Type, b scode.Bytes) super.Type {
+	if typ.Kind() == super.RecordKind {
+		return d.rec(builder, typ, b)
+	}
+	if errtyp, ok := typ.(*super.TypeError); ok && errtyp.IsQuiet(b) {
+		return nil
+	}
+	builder.Append(b)
+	return typ
+}

runtime/vam/expr/quiet.go

@@ -0,0 +1,117 @@
+package expr
+
+import (
+	"github.com/brimdata/super"
+	"github.com/brimdata/super/vector"
+	"github.com/brimdata/super/vector/bitvec"
+)
+
+func QuietMask(vec vector.Any) (vector.Any, bool) {
+	errvec, ok := vector.Under(vec).(*vector.Error)
+	if !ok || vector.KindOf(errvec.Vals) != vector.KindString {
+		return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false
+	}
+	lhs := vector.NewConst(super.NewString("quiet"), vec.Len(), bitvec.Zero)
+	out := NewCompare(nil, "!=", nil, nil).Compare(lhs, errvec.Vals)
+	if nulls := vector.NullsOf(out); !nulls.IsZero() {
+		// Flip nulls to true since a null result is not error("quiet").
+		b := FlattenBool(out)
+		return vector.NewBool(bitvec.Or(b.Bits, nulls), bitvec.Zero), true
+	}
+	return out, true
+}
+
+type Dequiet struct {
+	sctx  *super.Context
+	expr  Evaluator
+	rmtyp *super.TypeError
+}
+
+func NewDequiet(sctx *super.Context, expr Evaluator) Evaluator {
+	return &Dequiet{
+		sctx:  sctx,
+		expr:  expr,
+		rmtyp: sctx.LookupTypeError(sctx.MustLookupTypeRecord(nil)),
+	}
+}
+
+func (d *Dequiet) Eval(this vector.Any) vector.Any {
+	return vector.Apply(true, func(vecs ...vector.Any) vector.Any {
+		vec := vector.Under(vecs[0])
+		if vector.KindOf(vec) == vector.KindRecord {
+			vec = d.rec(vec)
+		}
+		return vec
+	}, d.expr.Eval(this))
+}
+
+func (d *Dequiet) rec(vec vector.Any) vector.Any {
+	var index []uint32
+	if view, ok := vec.(*vector.View); ok {
+		index = view.Index
+		vec = view.Any
+	}
+	var vecs []vector.Any
+	rec := vec.(*vector.Record)
+	if len(rec.Fields) == 0 {
+		return vec
+	}
+	for _, field := range rec.Fields {
+		vec := field
+		if index != nil {
+			vec = vector.Pick(field, index)
+		}
+		vecs = append(vecs, d.dequiet(vec))
+	}
+	if !rec.Nulls.IsZero() {
+		// Keep track of incoming nulls
+		vecs = append(vecs, vector.NewBool(rec.Nulls, bitvec.Zero))
+	}
+	return vector.Apply(true, func(vecs ...vector.Any) vector.Any {
+		var nulls bitvec.Bits
+		if !rec.Nulls.IsZero() {
+			nulls = vecs[len(vecs)-1].(*vector.Bool).Bits
+			vecs = vecs[:len(vecs)-1]
+		}
+		var fields []super.Field
+		var vals []vector.Any
+		for i, vec := range vecs {
+			typ := vec.Type()
+			if typ == d.rmtyp {
+				continue
+			}
+			fields = append(fields, rec.Typ.Fields[i])
+			vals = append(vals, vec)
+		}
+		rtyp := d.sctx.MustLookupTypeRecord(fields)
+		return vector.NewRecord(rtyp, vals, vecs[0].Len(), nulls)
+	}, vecs...)
+}
+
+func (d *Dequiet) dequiet(vec vector.Any) vector.Any {
+	if vector.KindOf(vec) == vector.KindRecord {
+		return d.rec(vec)
+	}
+	mask, ok := QuietMask(vec)
+	if !ok {
+		return vec
+	}
+	b, _ := BoolMask(new(Not).eval(mask))
+	if b.IsEmpty() {
+		return vec
+	}
+	n := uint32(b.GetCardinality())
+	quiet := d.quietTmp(n)
+	if n == vec.Len() {
+		return quiet
+	}
+	index := b.ToArray()
+	vec = vector.ReversePick(vec, index)
+	out := vector.Combine(vec, index, quiet).(*vector.Dynamic)
+	utyp := d.sctx.LookupTypeUnion([]super.Type{vec.Type(), quiet.Type()})
+	return vector.NewUnion(utyp, out.Tags, out.Values, bitvec.Zero)
+}
+
+func (d *Dequiet) quietTmp(n uint32) vector.Any {
+	return vector.NewError(d.rmtyp, vector.NewConst(super.Null, n, bitvec.Zero), bitvec.Zero)
+}

runtime/vam/op/values.go

@@ -4,7 +4,6 @@ import (
 	"github.com/brimdata/super"
 	"github.com/brimdata/super/runtime/vam/expr"
 	"github.com/brimdata/super/vector"
-	"github.com/brimdata/super/vector/bitvec"
 )
 
 type Values struct {
@@ -63,7 +62,7 @@ func interleave(vals []vector.Any) vector.Any {
 func filterQuiet(vec vector.Any) vector.Any {
 	var filtered bool
 	mask := vector.Apply(true, func(vecs ...vector.Any) vector.Any {
-		mask, hasfiltered := quietMask(vecs[0])
+		mask, hasfiltered := expr.QuietMask(vecs[0])
 		filtered = filtered || hasfiltered
 		return mask
 	}, vec)
@@ -73,31 +72,3 @@ func filterQuiet(vec vector.Any) vector.Any {
 	masked, _ := applyMask(vec, mask)
 	return masked
 }
-
-func quietMask(vec vector.Any) (vector.Any, bool) {
-	errvec, ok := vec.(*vector.Error)
-	if !ok {
-		return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false
-	}
-	if _, ok := errvec.Vals.Type().(*super.TypeOfString); !ok {
-		return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false
-	}
-	if c, ok := errvec.Vals.(*vector.Const); ok {
-		if s, _ := c.AsString(); s == "quiet" {
-			return vector.NewConst(super.False, vec.Len(), bitvec.Zero), true
-		}
-		return vector.NewConst(super.True, vec.Len(), bitvec.Zero), false
-	}
-	n := vec.Len()
-	mask := vector.NewFalse(n)
-	switch vec := vec.(type) {
-	case *vector.Error:
-		for i := uint32(0); i < n; i++ {
-			if s, _ := vector.StringValue(vec.Vals, i); s == "quiet" {
-				continue
-			}
-			mask.Set(i)
-		}
-	}
-	return mask, true
-}

runtime/ztests/expr/function/quiet.yaml

@@ -1,4 +1,4 @@
-spq: quiet(this)
+spq: values quiet(this)
 
 vector: true
 

runtime/ztests/op/cut-quiet.yaml

@@ -1,5 +1,3 @@
-skip: needs dequiet
-
 spq: cut foo:=quiet(foo), bar:=quiet(bar)
 
 vector: true
@@ -18,14 +16,15 @@ output: |
   {bar:"bar2"}
   {bar:"bar3"}
   {foo:"foo4",bar:"bar4"}
+  {}
 
 ---
 
-skip: needs dequiet
-
 # Rename fields.
 spq: cut f:=quiet(foo), b:=quiet(bar)
 
+vector: true
+
 input: |
   {foo:"foo0"}
   {foo:"foo1",goo:"goo1"}
@@ -40,11 +39,10 @@ output: |
   {b:"bar2"}
   {b:"bar3"}
   {f:"foo4",b:"bar4"}
+  {}
 
 ---
 
-skip: needs dequiet
-
 # Rename nested fields.
 spq: cut ports:=id, resp_p:=quiet(id.resp_p)
 

runtime/ztests/op/put-quiet.yaml

@@ -1,5 +1,3 @@
-skip: needs dequiet
-
 spq: x := quiet(y)
 
 vector: true
@@ -10,4 +8,4 @@ input: |
 
 output: |
   {}
-  {x:1}
+  {}