start adding time quantum support to batch import

adds QuantizedTime type to support efficiently tracking a time quantum with each Row which will be applied to all the values of time-capable fields in that row.
FeatureBaseDB · Sep 9, 2019 · b811c3b · b811c3b
1 parent c2b019c
commit b811c3b
Show file tree

Hide file tree

Showing 2 changed files with 178 additions and 0 deletions.
diff --git a/gpexp/importbatch.go b/gpexp/importbatch.go
@@ -1,6 +1,8 @@
 package gpexp
 
 import (
+	"time"
+
 	"github.com/pilosa/go-pilosa"
 	"github.com/pilosa/pilosa/roaring"
 	"github.com/pkg/errors"
@@ -153,6 +155,72 @@ func NewBatch(client *pilosa.Client, size int, index *pilosa.Index, fields []*pi
 type Row struct {
 	ID     interface{}
 	Values []interface{}
+	Time   *QuantizedTime
+}
+
+// QuantizedTime represents a moment in time down to some granularity
+// (year, month, day, or hour).
+type QuantizedTime struct {
+	ymdh [10]byte
+}
+
+// Set sets the Quantized time to the given timestamp (down to hour
+// granularity).
+func (qt *QuantizedTime) Set(t time.Time) {
+	copy(qt.ymdh[:], t.Format("2006010215"))
+}
+
+// SetYear sets the quantized time's year, but leaves month, day, and
+// hour untouched.
+func (qt *QuantizedTime) SetYear(year string) {
+	copy(qt.ymdh[:4], year)
+}
+
+// SetMonth sets the QuantizedTime's month, but leaves year, day, and
+// hour untouched.
+func (qt *QuantizedTime) SetMonth(month string) {
+	copy(qt.ymdh[4:6], month)
+}
+
+// SetDay sets the QuantizedTime's day, but leaves year, month, and
+// hour untouched.
+func (qt *QuantizedTime) SetDay(day string) {
+	copy(qt.ymdh[6:8], day)
+}
+
+// SetHour sets the QuantizedTime's hour, but leaves year, month, and
+// day untouched.
+func (qt *QuantizedTime) SetHour(hour string) {
+	copy(qt.ymdh[8:10], hour)
+}
+
+func (qt *QuantizedTime) views(q pilosa.TimeQuantum) ([]string, error) {
+	views := make([]string, 0, len(q))
+	for _, unit := range q {
+		switch unit {
+		case 'Y':
+			if qt.ymdh[0] == 0 {
+				return nil, errors.New("no data set for year")
+			}
+			views = append(views, string(qt.ymdh[:4]))
+		case 'M':
+			if qt.ymdh[4] == 0 {
+				return nil, errors.New("no data set for month")
+			}
+			views = append(views, string(qt.ymdh[:6]))
+		case 'D':
+			if qt.ymdh[6] == 0 {
+				return nil, errors.New("no data set for day")
+			}
+			views = append(views, string(qt.ymdh[:8]))
+		case 'H':
+			if qt.ymdh[8] == 0 {
+				return nil, errors.New("no data set for hour")
+			}
+			views = append(views, string(qt.ymdh[:10]))
+		}
+	}
+	return views, nil
 }
 
 // Add adds a record to the batch. Performance will be best if record

diff --git a/gpexp/importbatch_test.go b/gpexp/importbatch_test.go
@@ -4,6 +4,7 @@ import (
 	"reflect"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/pilosa/go-pilosa"
 	"github.com/pkg/errors"
@@ -483,3 +484,112 @@ outer:
 	}
 	return nil
 }
+
+func TestQuantizedTime(t *testing.T) {
+	cases := []struct {
+		name    string
+		time    time.Time
+		year    string
+		month   string
+		day     string
+		hour    string
+		quantum pilosa.TimeQuantum
+		exp     []string
+		expErr  string
+	}{
+		{
+			name:   "no time quantum",
+			exp:    []string{},
+			expErr: "",
+		},
+		{
+			name:    "no data",
+			quantum: pilosa.TimeQuantumYear,
+			expErr:  "no data set for year",
+		},
+		{
+			name:    "timestamp",
+			time:    time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
+			quantum: "YMDH",
+			exp:     []string{"2013", "201310", "20131016", "2013101617"},
+		},
+		{
+			name:    "timestamp-less-granular",
+			time:    time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
+			quantum: "YM",
+			exp:     []string{"2013", "201310"},
+		},
+		{
+			name:    "timestamp-mid-granular",
+			time:    time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
+			quantum: "MD",
+			exp:     []string{"201310", "20131016"},
+		},
+		{
+			name:    "justyear",
+			year:    "2013",
+			quantum: "Y",
+			exp:     []string{"2013"},
+		},
+		{
+			name:    "justyear-wantmonth",
+			year:    "2013",
+			quantum: "YM",
+			expErr:  "no data set for month",
+		},
+		{
+			name:    "timestamp-changeyear",
+			time:    time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
+			year:    "2019",
+			quantum: "YMDH",
+			exp:     []string{"2019", "201910", "20191016", "2019101617"},
+		},
+		{
+			name:    "yearmonthdayhour",
+			year:    "2013",
+			month:   "10",
+			day:     "16",
+			hour:    "17",
+			quantum: "YMDH",
+			exp:     []string{"2013", "201310", "20131016", "2013101617"},
+		},
+		{
+			name:    "timestamp-changehour",
+			time:    time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
+			hour:    "05",
+			quantum: "MDH",
+			exp:     []string{"201310", "20131016", "2013101605"},
+		},
+	}
+
+	for i, test := range cases {
+		t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
+			tq := &QuantizedTime{}
+			var zt time.Time
+			if zt != test.time {
+				tq.Set(test.time)
+			}
+			if test.year != "" {
+				tq.SetYear(test.year)
+			}
+			if test.month != "" {
+				tq.SetMonth(test.month)
+			}
+			if test.day != "" {
+				tq.SetDay(test.day)
+			}
+			if test.hour != "" {
+				tq.SetHour(test.hour)
+			}
+
+			views, err := tq.views(test.quantum)
+			if !reflect.DeepEqual(views, test.exp) {
+				t.Errorf("unexpected views, got/want:\n%v\n%v\n", views, test.exp)
+			}
+			if (err != nil && err.Error() != test.expErr) || (err == nil && test.expErr != "") {
+				t.Errorf("unexpected error, got/want:\n%v\n%s\n", err, test.expErr)
+			}
+		})
+	}
+
+}