Skip to content

Commit

Permalink
Revert "use *arrow.Binary to reduce allocations"
Browse files Browse the repository at this point in the history
This reverts commit b160132.
  • Loading branch information
gernest committed Feb 10, 2024
1 parent 6c070d1 commit 600ada1
Show file tree
Hide file tree
Showing 5 changed files with 20 additions and 9 deletions.
6 changes: 3 additions & 3 deletions ref/ref.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
var arrowBytes []byte

var name *array.Dictionary
var nameData *array.Binary
var nameData *array.String

//go:embed refs.fst
var fstBytes []byte
Expand All @@ -41,7 +41,7 @@ func init() {
rd.Next()
r := rd.Record()
name = r.Column(0).(*array.Dictionary)
nameData = name.Dictionary().(*array.Binary)
nameData = name.Dictionary().(*array.String)
fst, err = vellum.Load(fstBytes)
if err != nil {
logger.Fail(err.Error())
Expand Down Expand Up @@ -83,7 +83,7 @@ func Search(uri string) string {
_, val := it.Current()
r := nameData.Value(name.GetValueIndex(int(val)))
cache.Set(u.Host, r, int64(len(r)))
return string(r)
return r
}
return ""
}
Expand Down
Binary file modified ref/refs.arrow
Binary file not shown.
17 changes: 14 additions & 3 deletions staples/arrow.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ var baseTypes = map[reflect.Kind]arrow.DataType{
reflect.Bool: arrow.FixedWidthTypes.Boolean,
reflect.String: &arrow.DictionaryType{
IndexType: arrow.PrimitiveTypes.Uint32,
ValueType: arrow.BinaryTypes.Binary,
ValueType: arrow.BinaryTypes.String,
},
reflect.Int64: arrow.PrimitiveTypes.Int64,
reflect.Float64: arrow.PrimitiveTypes.Float64,
Expand Down Expand Up @@ -124,6 +124,17 @@ func write(b array.Builder) func(reflect.Value) {
}
case *array.BinaryDictionaryBuilder:
return func(v reflect.Value) {
if v.Kind() == reflect.Slice {
if v.IsNil() {
e.AppendNull()
return
}
err := e.Append(v.Bytes())
if err != nil {
panic(err)
}
return
}
s := v.String()
if s == "" {
e.AppendNull()
Expand Down Expand Up @@ -185,13 +196,13 @@ func merge(b array.Builder) func(arrow.Array) {
case *array.BinaryDictionaryBuilder:
return func(v arrow.Array) {
a := v.(*array.Dictionary)
x := a.Dictionary().(*array.Binary)
x := a.Dictionary().(*array.String)
for i := 0; i < a.Len(); i++ {
if a.IsNull(i) {
e.AppendNull()
continue
}
e.Append(x.Value(a.GetValueIndex(i)))
e.AppendString(x.Value(a.GetValueIndex(i)))
}
}
default:
Expand Down
Binary file modified ua/ua.arrow
Binary file not shown.
6 changes: 3 additions & 3 deletions ua/ua.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,18 +104,18 @@ func (m *Model) Size() int {

type Field struct {
Dict *array.Dictionary
Str *array.Binary
Str *array.String
}

func (f *Field) Get(i int) string {
return string(f.Str.Value(f.Dict.GetValueIndex(i)))
return f.Str.Value(f.Dict.GetValueIndex(i))
}

func newStr(a arrow.Array) *Field {
d := a.(*array.Dictionary)
return &Field{
Dict: d,
Str: d.Dictionary().(*array.Binary),
Str: d.Dictionary().(*array.String),
}
}

Expand Down

0 comments on commit 600ada1

Please sign in to comment.