load cold segments

segment/cold/indes_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/apache/arrow/go/v15/arrow/array"
 	"github.com/apache/arrow/go/v15/arrow/memory"
+	"github.com/docker/go-units"
 	"github.com/vinceanalytics/vince/staples"
 )
 
@@ -20,5 +21,11 @@ func TestIndex(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	t.Error(r.NumRows())
+	defer r.Release()
+
+	seg, err := New(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Error(units.BytesSize(float64(seg.Size())))
 }

segment/cold/index.go

@@ -1 +1,26 @@
 package cold
+
+import (
+	"github.com/apache/arrow/go/v15/arrow"
+	"github.com/vinceanalytics/vince/camel"
+	v1 "github.com/vinceanalytics/vince/gen/go/staples/v1"
+	"github.com/vinceanalytics/vince/segment"
+	"github.com/vinceanalytics/vince/segment/ice"
+)
+
+var skipFields = map[string]struct{}{
+	camel.Case(v1.Filters_ID.String()):       {},
+	camel.Case(v1.Filters_Session.String()):  {},
+	camel.Case(v1.Filters_Bounce.String()):   {},
+	camel.Case(v1.Filters_Duration.String()): {},
+}
+
+func New(r arrow.Record) (segment.Segment, error) {
+	document := NewRecord(r, func(r arrow.Record, i int) bool {
+		_, ok := skipFields[r.ColumnName(i)]
+		return !ok
+	})
+	defer document.Release()
+	seg, _, err := ice.New([]segment.Document{document}, func(s string, i int) float32 { return 0 })
+	return seg, err
+}

segment/cold/record.go

@@ -32,7 +32,9 @@ func (r *Record) Release() {
 
 func (r *Record) EachField(vf segment.VisitField) {
 	for i := 0; i < int(r.record.NumCols()); i++ {
-		visitArray(r.record.Column(i), r.record.ColumnName(i), vf)
+		if r.accept(r.record, i) {
+			visitArray(r.record.Column(i), r.record.ColumnName(i), vf)
+		}
 	}
 }