diff --git a/libbeat/common/dtfmt/builder.go b/libbeat/common/dtfmt/builder.go new file mode 100644 index 000000000000..c88f9cf024ef --- /dev/null +++ b/libbeat/common/dtfmt/builder.go @@ -0,0 +1,221 @@ +package dtfmt + +type builder struct { + elements []element +} + +func newBuilder() *builder { + return &builder{} +} + +func (b *builder) estimateSize() int { + sz := 0 + for _, e := range b.elements { + sz += e.estimateSize() + } + return sz +} + +func (b *builder) createConfig() (ctxConfig, error) { + cfg := ctxConfig{} + for _, e := range b.elements { + if err := e.requires(&cfg); err != nil { + return ctxConfig{}, err + } + } + return cfg, nil +} + +func (b *builder) compile() (prog, error) { + p := prog{} + + for _, e := range b.elements { + tmp, err := e.compile() + if err != nil { + return prog{}, err + } + + p.p = append(p.p, tmp.p...) + } + return p, nil +} + +func (b *builder) optimize() { + if len(b.elements) == 0 { + return + } + + // combine rune/string literals + el := b.elements[:1] + for _, e := range b.elements[1:] { + last := el[len(el)-1] + if r, ok := e.(runeLiteral); ok { + if l, ok := last.(runeLiteral); ok { + el[len(el)-1] = stringLiteral{ + append(append([]byte{}, string(l.r)...), string(r.r)...), + } + } else if l, ok := last.(stringLiteral); ok { + el[len(el)-1] = stringLiteral{append(l.s, string(r.r)...)} + } else { + el = append(el, e) + } + } else if s, ok := e.(stringLiteral); ok { + if l, ok := last.(runeLiteral); ok { + el[len(el)-1] = stringLiteral{ + append(append([]byte{}, string(l.r)...), s.s...), + } + } else if l, ok := last.(stringLiteral); ok { + el[len(el)-1] = stringLiteral{append(l.s, s.s...)} + } else { + el = append(el, e) + } + } else { + el = append(el, e) + } + } + b.elements = el +} + +func (b *builder) add(e element) { + b.elements = append(b.elements, e) +} + +func (b *builder) millisOfSecond(digits int) { + b.appendDecimal(ftMillisOfSecond, digits, 3) +} + +func (b *builder) millisOfDay(digits int) { + b.appendDecimal(ftMillisOfDay, digits, 8) +} + +func (b *builder) secondOfMinute(digits int) { + b.appendDecimal(ftSecondOfMinute, digits, 2) +} + +func (b *builder) secondOfDay(digits int) { + b.appendDecimal(ftSecondOfDay, digits, 5) +} + +func (b *builder) minuteOfHour(digits int) { + b.appendDecimal(ftMinuteOfHour, digits, 2) +} + +func (b *builder) minuteOfDay(digits int) { + b.appendDecimal(ftMinuteOfDay, digits, 4) +} + +func (b *builder) hourOfDay(digits int) { + b.appendDecimal(ftHourOfDay, digits, 2) +} + +func (b *builder) clockhourOfDay(digits int) { + b.appendDecimal(ftClockhourOfDay, digits, 2) +} + +func (b *builder) hourOfHalfday(digits int) { + b.appendDecimal(ftHourOfHalfday, digits, 2) +} + +func (b *builder) clockhourOfHalfday(digits int) { + b.appendDecimal(ftClockhourOfHalfday, digits, 2) +} + +func (b *builder) dayOfWeek(digits int) { + b.appendDecimal(ftDayOfWeek, digits, 1) +} + +func (b *builder) dayOfMonth(digits int) { + b.appendDecimal(ftDayOfMonth, digits, 2) +} + +func (b *builder) dayOfYear(digits int) { + b.appendDecimal(ftDayOfYear, digits, 3) +} + +func (b *builder) weekOfWeekyear(digits int) { + b.appendDecimal(ftWeekOfWeekyear, digits, 2) +} + +func (b *builder) weekyear(minDigits, maxDigits int) { + b.appendDecimal(ftWeekyear, minDigits, maxDigits) +} + +func (b *builder) monthOfYear(digits int) { + b.appendDecimal(ftMonthOfYear, digits, 2) +} + +func (b *builder) year(minDigits, maxDigits int) { + b.appendSigned(ftYear, minDigits, maxDigits) +} + +func (b *builder) twoDigitYear() { + b.add(twoDigitYear{ftYear}) +} + +func (b *builder) twoDigitWeekYear() { + b.add(twoDigitYear{ftWeekyear}) +} + +func (b *builder) halfdayOfDayText() { + b.appendText(ftHalfdayOfDay) +} + +func (b *builder) dayOfWeekText() { + b.appendText(ftDayOfWeek) +} + +func (b *builder) dayOfWeekShortText() { + b.appendShortText(ftDayOfWeek) +} + +func (b *builder) monthOfYearText() { + b.appendText(ftMonthOfYear) +} + +func (b *builder) monthOfYearShortText() { + b.appendShortText(ftMonthOfYear) +} + +// TODO: add timezone support + +func (b *builder) appendRune(r rune) { + b.add(runeLiteral{r}) +} + +func (b *builder) appendLiteral(l string) { + switch len(l) { + case 0: + case 1: + b.add(runeLiteral{rune(l[0])}) + default: + b.add(stringLiteral{[]byte(l)}) + } +} + +func (b *builder) appendDecimalValue(ft fieldType, minDigits, maxDigits int, signed bool) { + if maxDigits < minDigits { + maxDigits = minDigits + } + + if minDigits <= 1 { + b.add(unpaddedNumber{ft, maxDigits, signed}) + } else { + b.add(paddedNumber{ft, minDigits, maxDigits, signed}) + } +} + +func (b *builder) appendDecimal(ft fieldType, minDigits, maxDigits int) { + b.appendDecimalValue(ft, minDigits, maxDigits, false) +} + +func (b *builder) appendSigned(ft fieldType, minDigits, maxDigits int) { + b.appendDecimalValue(ft, minDigits, maxDigits, true) +} + +func (b *builder) appendText(ft fieldType) { + b.add(textField{ft, false}) +} + +func (b *builder) appendShortText(ft fieldType) { + b.add(textField{ft, true}) +} diff --git a/libbeat/common/dtfmt/ctx.go b/libbeat/common/dtfmt/ctx.go new file mode 100644 index 000000000000..7b1f4708eee7 --- /dev/null +++ b/libbeat/common/dtfmt/ctx.go @@ -0,0 +1,75 @@ +package dtfmt + +import "time" + +// ctx stores pre-computed time fields used by the formatter. +type ctx struct { + year int + month time.Month + day int + weekday time.Weekday + yearday int + isoWeek, isoYear int + + hour, min, sec int + millis int + + buf []byte +} + +type ctxConfig struct { + date bool + clock bool + weekday bool + yearday bool + millis bool + iso bool +} + +func (c *ctx) initTime(config *ctxConfig, t time.Time) { + if config.date { + c.year, c.month, c.day = t.Date() + } + if config.clock { + c.hour, c.min, c.sec = t.Clock() + } + if config.iso { + c.isoYear, c.isoWeek = t.ISOWeek() + } + + if config.millis { + c.millis = t.Nanosecond() / 1000000 + } + + if config.yearday { + c.yearday = t.YearDay() + } + + if config.weekday { + c.weekday = t.Weekday() + } +} + +func (c *ctxConfig) enableDate() { + c.date = true +} + +func (c *ctxConfig) enableClock() { + c.clock = true +} + +func (c *ctxConfig) enableWeekday() { + c.weekday = true +} + +func (c *ctxConfig) enableYearday() { + c.yearday = true +} + +func (c *ctxConfig) enableISO() { + c.iso = true +} + +func isLeap(year int) bool { + return year%4 == 0 && (year%100 != 0 || year%400 == 0) +} diff --git a/libbeat/common/dtfmt/doc.go b/libbeat/common/dtfmt/doc.go new file mode 100644 index 000000000000..2ff26e7e1f50 --- /dev/null +++ b/libbeat/common/dtfmt/doc.go @@ -0,0 +1,64 @@ +// dtfmt package provides time formatter support with pattern syntax mostly +// similar to joda DateTimeFormat. The pattern syntax supported is a subset +// (mostly compatible) with joda DateTimeFormat. +// +// +// Symbol Meaning Type Supported Examples +// ------ ------- ------- --------- ------- +// G era text no AD +// C century of era (>=0) number no 20 +// Y year of era (>=0) year yes 1996 +// +// x weekyear year yes 1996 +// w week of weekyear number yes 27 +// e day of week number yes 2 +// E day of week text yes Tuesday; Tue +// +// y year year yes 1996 +// D day of year number yes 189 +// M month of year month yes July; Jul; 07 +// d day of month number yes 10 +// +// a halfday of day text yes PM +// K hour of halfday (0~11) number yes 0 +// h clockhour of halfday (1~12) number yes 12 +// +// H hour of day (0~23) number yes 0 +// k clockhour of day (1~24) number yes 24 +// m minute of hour number yes 30 +// s second of minute number yes 55 +// S fraction of second millis no 978 +// +// z time zone text no Pacific Standard Time; PST +// Z time zone offset/id zone no -0800; -08:00; America/Los_Angeles +// +// ' escape for text delimiter +// '' single quote literal +// +// The format is based on pattern letter count. Any character not in the range +// [a-z][A-Z] is interpreted as literal and copied into final string as is. +// Arbitrary Literals can also be written using single quotes `'` +// +// Types: Notes: +// ------ ------ +// text Use full form if number of letters is >= 4. +// Otherwise a short form is used (if available). +// +// number Minimum number of digits depends on number of letters. +// Shorter numbers are zero-padded. +// +// year mostly like number. If Pattern length is 2, +// the year will be displayed as zero-based year +// of the century (modulo 100) +// +// month If pattern length >= 3, formatting is according to +// text type. Otherwise number type +// formatting rules are applied. +// +// millis Not yet supported +// +// zone Not yet supported +// +// literal Literals are copied as is into formatted string +// +package dtfmt diff --git a/libbeat/common/dtfmt/dtfmt.go b/libbeat/common/dtfmt/dtfmt.go new file mode 100644 index 000000000000..b8e63e15a487 --- /dev/null +++ b/libbeat/common/dtfmt/dtfmt.go @@ -0,0 +1,13 @@ +package dtfmt + +import "time" + +// Format applies the format-pattern to the given timestamp. +// Returns the formatted string or an error if pattern is invalid. +func Format(t time.Time, pattern string) (string, error) { + f, err := NewFormatter(pattern) + if err != nil { + return "", err + } + return f.Format(t) +} diff --git a/libbeat/common/dtfmt/dtfmt_test.go b/libbeat/common/dtfmt/dtfmt_test.go new file mode 100644 index 000000000000..28808f27caed --- /dev/null +++ b/libbeat/common/dtfmt/dtfmt_test.go @@ -0,0 +1,95 @@ +package dtfmt + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestFormat(t *testing.T) { + tests := []struct { + time time.Time + pattern string + expected string + }{ + // year.month.day of month + {mkDate(6, 8, 1), "y.M.d", "6.8.1"}, + {mkDate(2006, 8, 1), "y.M.d", "2006.8.1"}, + {mkDate(2006, 8, 1), "yy.MM.dd", "06.08.01"}, + {mkDate(6, 8, 1), "yy.MM.dd", "06.08.01"}, + {mkDate(2006, 8, 1), "yyy.MMM.dd", "2006.Aug.01"}, + {mkDate(2006, 8, 1), "yyyy.MMMM.d", "2006.August.1"}, + {mkDate(2006, 8, 1), "yyyyyy.MM.ddd", "002006.08.001"}, + + // year of era.month.day + {mkDate(6, 8, 1), "Y.M.d", "6.8.1"}, + {mkDate(2006, 8, 1), "Y.M.d", "2006.8.1"}, + {mkDate(2006, 8, 1), "YY.MM.dd", "06.08.01"}, + {mkDate(6, 8, 1), "YY.MM.dd", "06.08.01"}, + {mkDate(2006, 8, 1), "YYY.MMM.dd", "2006.Aug.01"}, + {mkDate(2006, 8, 1), "YYYY.MMMM.d", "2006.August.1"}, + {mkDate(2006, 8, 1), "YYYYYY.MM.ddd", "002006.08.001"}, + + // week year + week of year + day of week + {mkDate(2015, 1, 1), "xx.ww.e", "15.01.4"}, + {mkDate(2014, 12, 31), "xx.ww.e", "15.01.3"}, + {mkDate(2015, 1, 1), "xx.w.E", "15.1.Thu"}, + {mkDate(2014, 12, 31), "xx.w.E", "15.1.Wed"}, + {mkDate(2015, 1, 1), "xx.w.EEEE", "15.1.Thursday"}, + {mkDate(2014, 12, 31), "xx.w.EEEE", "15.1.Wednesday"}, + {mkDate(2015, 1, 1), "xxxx.ww", "2015.01"}, + {mkDate(2014, 12, 31), "xxxx.ww", "2015.01"}, + {mkDate(2015, 1, 1), "xxxx.ww.e", "2015.01.4"}, + {mkDate(2014, 12, 31), "xxxx.ww.e", "2015.01.3"}, + {mkDate(2015, 1, 1), "xxxx.w.E", "2015.1.Thu"}, + {mkDate(2014, 12, 31), "xxxx.w.E", "2015.1.Wed"}, + {mkDate(2015, 1, 1), "xxxx.w.EEEE", "2015.1.Thursday"}, + {mkDate(2014, 12, 31), "xxxx.w.EEEE", "2015.1.Wednesday"}, + + // time + {mkTime(8, 5, 24), "K:m:s a", "8:5:24 AM"}, + {mkTime(8, 5, 24), "KK:mm:ss aa", "08:05:24 AM"}, + {mkTime(20, 5, 24), "K:m:s a", "8:5:24 PM"}, + {mkTime(20, 5, 24), "KK:mm:ss aa", "08:05:24 PM"}, + {mkTime(8, 5, 24), "h:m:s a", "9:5:24 AM"}, + {mkTime(8, 5, 24), "hh:mm:ss aa", "09:05:24 AM"}, + {mkTime(20, 5, 24), "h:m:s a", "9:5:24 PM"}, + {mkTime(20, 5, 24), "hh:mm:ss aa", "09:05:24 PM"}, + {mkTime(8, 5, 24), "H:m:s a", "8:5:24 AM"}, + {mkTime(8, 5, 24), "HH:mm:ss aa", "08:05:24 AM"}, + {mkTime(20, 5, 24), "H:m:s a", "20:5:24 PM"}, + {mkTime(20, 5, 24), "HH:mm:ss aa", "20:05:24 PM"}, + {mkTime(8, 5, 24), "k:m:s a", "9:5:24 AM"}, + {mkTime(8, 5, 24), "kk:mm:ss aa", "09:05:24 AM"}, + {mkTime(20, 5, 24), "k:m:s a", "21:5:24 PM"}, + {mkTime(20, 5, 24), "kk:mm:ss aa", "21:05:24 PM"}, + + // literals + {time.Now(), "--=++,_!/?\\[]{}@#$%^&*()", "--=++,_!/?\\[]{}@#$%^&*()"}, + {time.Now(), "'plain text'", "plain text"}, + {time.Now(), "'plain' 'text'", "plain text"}, + {time.Now(), "'plain' '' 'text'", "plain ' text"}, + {time.Now(), "'plain '' text'", "plain ' text"}, + } + + for i, test := range tests { + t.Logf("run (%v): %v -> %v", i, test.pattern, test.expected) + + actual, err := Format(test.time, test.pattern) + if err != nil { + t.Error(err) + continue + } + + assert.Equal(t, test.expected, actual) + } +} + +func mkDate(y, m, d int) time.Time { + return time.Date(y, time.Month(m), d, 0, 0, 0, 0, time.Local) +} + +func mkTime(h, m, s int) time.Time { + return time.Date(2000, 1, 1, h, m, s, 0, time.Local) +} diff --git a/libbeat/common/dtfmt/elems.go b/libbeat/common/dtfmt/elems.go new file mode 100644 index 000000000000..73cf882440a0 --- /dev/null +++ b/libbeat/common/dtfmt/elems.go @@ -0,0 +1,175 @@ +package dtfmt + +import ( + "errors" + "fmt" + "unicode/utf8" +) + +type element interface { + requires(c *ctxConfig) error + estimateSize() int + compile() (prog, error) +} + +type runeLiteral struct { + r rune +} + +type stringLiteral struct { + s []byte +} + +type unpaddedNumber struct { + ft fieldType + maxDigits int + signed bool +} + +type paddedNumber struct { + ft fieldType + minDigits, maxDigits int + signed bool +} + +type textField struct { + ft fieldType + short bool +} + +type twoDigitYear struct { + ft fieldType +} + +func (runeLiteral) requires(*ctxConfig) error { return nil } +func (runeLiteral) estimateSize() int { return 1 } + +func (stringLiteral) requires(*ctxConfig) error { return nil } +func (s stringLiteral) estimateSize() int { return len(s.s) } + +func (n unpaddedNumber) requires(c *ctxConfig) error { + return numRequires(c, n.ft) +} + +func (n unpaddedNumber) estimateSize() int { + return numSize(n.maxDigits, n.signed) +} + +func (n paddedNumber) requires(c *ctxConfig) error { + return numRequires(c, n.ft) +} + +func (n paddedNumber) estimateSize() int { + return numSize(n.maxDigits, n.signed) +} + +func (n twoDigitYear) requires(c *ctxConfig) error { + return numRequires(c, n.ft) +} + +func (twoDigitYear) estimateSize() int { return 2 } + +func numSize(digits int, signed bool) int { + if signed { + return digits + 1 + } + return digits +} + +func numRequires(c *ctxConfig, ft fieldType) error { + switch ft { + case ftYear, ftMonthOfYear, ftDayOfMonth: + c.enableDate() + + case ftWeekyear, ftWeekOfWeekyear: + c.enableISO() + + case ftDayOfYear: + c.enableYearday() + + case ftDayOfWeek: + c.enableWeekday() + + case ftHalfdayOfDay, + ftHourOfHalfday, + ftClockhourOfHalfday, + ftClockhourOfDay, + ftHourOfDay, + ftMinuteOfDay, + ftMinuteOfHour, + ftSecondOfDay, + ftSecondOfMinute, + ftMillisOfDay, + ftMillisOfSecond: + c.enableClock() + } + + return nil +} + +func (f textField) requires(c *ctxConfig) error { + switch f.ft { + case ftHalfdayOfDay: + c.enableClock() + case ftMonthOfYear: + c.enableDate() + case ftDayOfWeek: + c.enableWeekday() + default: + return fmt.Errorf("time field %v not supported by text", f.ft) + } + return nil +} + +func (f textField) estimateSize() int { + switch f.ft { + case ftHalfdayOfDay: + return 2 + case ftDayOfWeek: + if f.short { + return 3 + } + return 9 // max(weekday) = len(Wednesday) + case ftMonthOfYear: + if f.short { + return 6 + } + return 9 // max(month) = len(September) + default: + return 0 + } +} + +func (r runeLiteral) compile() (prog, error) { + switch utf8.RuneLen(r.r) { + case -1: + return prog{}, errors.New("invalid rune") + } + + var tmp [8]byte + l := utf8.EncodeRune(tmp[:], r.r) + return makeCopy(tmp[:l]) +} + +func (s stringLiteral) compile() (prog, error) { + return makeCopy([]byte(s.s)) +} + +func (n unpaddedNumber) compile() (prog, error) { + return makeProg(opNum, byte(n.ft)) +} + +func (n paddedNumber) compile() (prog, error) { + return makeProg(opNumPadded, byte(n.ft), byte(n.maxDigits)) +} + +func (n twoDigitYear) compile() (prog, error) { + return makeProg(opTwoDigit, byte(n.ft)) +} + +func (f textField) compile() (prog, error) { + if f.short { + return makeProg(opTextShort, byte(f.ft)) + } + return makeProg(opTextLong, byte(f.ft)) +} diff --git a/libbeat/common/dtfmt/fields.go b/libbeat/common/dtfmt/fields.go new file mode 100644 index 000000000000..41355d61a316 --- /dev/null +++ b/libbeat/common/dtfmt/fields.go @@ -0,0 +1,130 @@ +package dtfmt + +import ( + "errors" + "time" +) + +type fieldType uint8 + +const ( + ftYear fieldType = iota + ftDayOfYear + ftMonthOfYear + ftDayOfMonth + ftWeekyear + ftWeekOfWeekyear + ftDayOfWeek + ftHalfdayOfDay + ftHourOfHalfday + ftClockhourOfHalfday + ftClockhourOfDay + ftHourOfDay + ftMinuteOfDay + ftMinuteOfHour + ftSecondOfDay + ftSecondOfMinute + ftMillisOfDay + ftMillisOfSecond +) + +func getIntField(ft fieldType, ctx *ctx, t time.Time) (int, error) { + switch ft { + case ftYear: + return ctx.year, nil + + case ftDayOfYear: + return ctx.yearday, nil + + case ftMonthOfYear: + return int(ctx.month), nil + + case ftDayOfMonth: + return ctx.day, nil + + case ftWeekyear: + return ctx.isoYear, nil + + case ftWeekOfWeekyear: + return ctx.isoWeek, nil + + case ftDayOfWeek: + return int(ctx.weekday), nil + + case ftHalfdayOfDay: + if ctx.hour < 12 { + return 0, nil // AM + } + return 1, nil // PM + + case ftHourOfHalfday: + if ctx.hour < 12 { + return ctx.hour, nil + } + return ctx.hour - 12, nil + + case ftClockhourOfHalfday: + if ctx.hour < 12 { + return ctx.hour + 1, nil + } + return ctx.hour - 12 + 1, nil + + case ftClockhourOfDay: + return ctx.hour + 1, nil + + case ftHourOfDay: + return ctx.hour, nil + + case ftMinuteOfDay: + return ctx.hour*60 + ctx.min, nil + + case ftMinuteOfHour: + return ctx.min, nil + + case ftSecondOfDay: + return (ctx.hour*60+ctx.min)*60 + ctx.sec, nil + + case ftSecondOfMinute: + return ctx.sec, nil + + case ftMillisOfDay: + return ((ctx.hour*60+ctx.min)*60+ctx.sec)*1000 + ctx.millis, nil + + case ftMillisOfSecond: + return ctx.millis, nil + } + + return 0, nil +} + +func getTextField(ft fieldType, ctx *ctx, t time.Time) (string, error) { + switch ft { + case ftHalfdayOfDay: + if ctx.hour < 12 { + return "AM", nil + } + return "PM", nil + case ftDayOfWeek: + return ctx.weekday.String(), nil + case ftMonthOfYear: + return ctx.month.String(), nil + default: + return "", errors.New("no text field") + } +} + +func getTextFieldShort(ft fieldType, ctx *ctx, t time.Time) (string, error) { + switch ft { + case ftHalfdayOfDay: + if ctx.hour < 12 { + return "AM", nil + } + return "PM", nil + case ftDayOfWeek: + return ctx.weekday.String()[:3], nil + case ftMonthOfYear: + return ctx.month.String()[:3], nil + default: + return "", errors.New("no text field") + } +} diff --git a/libbeat/common/dtfmt/fmt.go b/libbeat/common/dtfmt/fmt.go new file mode 100644 index 000000000000..988e8108f2f8 --- /dev/null +++ b/libbeat/common/dtfmt/fmt.go @@ -0,0 +1,261 @@ +package dtfmt + +import ( + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + "unicode/utf8" +) + +// Formatter will format time values into strings, based on pattern used to +// create the Formatter. +type Formatter struct { + prog prog + sz int + config ctxConfig +} + +var ctxPool = &sync.Pool{ + New: func() interface{} { return &ctx{} }, +} + +func newCtx() *ctx { + return ctxPool.Get().(*ctx) +} + +func newCtxWithSize(sz int) *ctx { + ctx := newCtx() + if ctx.buf == nil || cap(ctx.buf) < sz { + ctx.buf = make([]byte, 0, sz) + } + return ctx +} + +func releaseCtx(c *ctx) { + ctxPool.Put(c) +} + +// NewFormatter creates a new time formatter based on provided pattern. +// If pattern is invalid an error is returned. +func NewFormatter(pattern string) (*Formatter, error) { + b := newBuilder() + + err := parsePatternTo(b, pattern) + if err != nil { + return nil, err + } + + b.optimize() + + cfg, err := b.createConfig() + if err != nil { + return nil, err + } + + prog, err := b.compile() + if err != nil { + return nil, err + } + + sz := b.estimateSize() + f := &Formatter{ + prog: prog, + sz: sz, + config: cfg, + } + return f, nil +} + +// EstimateSize estimates the required buffer size required to hold +// the formatted time string. Estimated size gives no exact guarantees. +// Estimated size might still be too low or too big. +func (f *Formatter) EstimateSize() int { + return f.sz +} + +func (f *Formatter) appendTo(ctx *ctx, b []byte, t time.Time) ([]byte, error) { + ctx.initTime(&f.config, t) + return f.prog.eval(b, ctx, t) +} + +// AppendTo appends the formatted time value to the given byte buffer. +func (f *Formatter) AppendTo(b []byte, t time.Time) ([]byte, error) { + ctx := newCtx() + defer releaseCtx(ctx) + return f.appendTo(ctx, b, t) +} + +// Write writes the formatted time value to the given writer. Returns +// number of bytes written or error if formatter or writer fails. +func (f *Formatter) Write(w io.Writer, t time.Time) (int, error) { + var err error + + ctx := newCtxWithSize(f.sz) + defer releaseCtx(ctx) + + ctx.buf, err = f.appendTo(ctx, ctx.buf[:0], t) + if err != nil { + return 0, err + } + return w.Write(ctx.buf) +} + +// Format formats the given time value into a new string. +func (f *Formatter) Format(t time.Time) (string, error) { + var err error + + ctx := newCtxWithSize(f.sz) + defer releaseCtx(ctx) + + ctx.buf, err = f.appendTo(ctx, ctx.buf[:0], t) + if err != nil { + return "", err + } + return string(ctx.buf), nil +} + +func parsePatternTo(b *builder, pattern string) error { + for i := 0; i < len(pattern); { + tok, tokText, err := parseToken(pattern, &i) + if err != nil { + return err + } + + tokLen := len(tokText) + switch tok { + case 'x': // weekyear (year) + if tokLen == 2 { + b.twoDigitWeekYear() + } else { + b.weekyear(tokLen, 4) + } + + case 'y', 'Y': // year and year of era (year) == 'y' + if tokLen == 2 { + b.twoDigitYear() + } else { + b.year(tokLen, 4) + } + + case 'w': // week of weekyear (num) + b.weekOfWeekyear(tokLen) + + case 'e': // day of week (num) + b.dayOfWeek(tokLen) + + case 'E': // day of week (text) + if tokLen >= 4 { + b.dayOfWeekText() + } else { + b.dayOfWeekShortText() + } + + case 'D': // day of year (number) + b.dayOfYear(tokLen) + + case 'M': // month of year (month) + if tokLen >= 3 { + if tokLen >= 4 { + b.monthOfYearText() + } else { + b.monthOfYearShortText() + } + } else { + b.monthOfYear(tokLen) + } + + case 'd': //day of month (number) + b.dayOfMonth(tokLen) + + case 'a': // half of day (text) 'AM/PM' + b.halfdayOfDayText() + + case 'K': // hour of half day (number) (0 - 11) + b.hourOfHalfday(tokLen) + + case 'h': // clock hour of half day (number) (1 - 12) + b.clockhourOfHalfday(tokLen) + + case 'H': // hour of day (number) (0 - 23) + b.hourOfDay(tokLen) + + case 'k': // clock hour of half day (number) (1 - 24) + b.clockhourOfDay(tokLen) + + case 'm': // minute of hour + b.minuteOfHour(tokLen) + + case 's': // second of minute + b.secondOfMinute(tokLen) + + case 'S': // fraction of second + return errors.New("time formatter 'S' not supported") + + case '\'': // literal + if tokLen == 1 { + b.appendRune(rune(tokText[0])) + } else { + b.appendLiteral(tokText) + } + + default: + return fmt.Errorf("unsupport format '%c'", tok) + + } + } + + return nil +} + +func parseToken(pattern string, i *int) (rune, string, error) { + start := *i + idx := start + length := len(pattern) + + r, w := utf8.DecodeRuneInString(pattern[idx:]) + idx += w + if ('A' <= r && r <= 'Z') || ('a' <= r && r <= 'z') { + // Scan a run of the same character, which indicates a time pattern. + + for idx < length { + peek, w := utf8.DecodeRuneInString(pattern[idx:]) + if peek != r { + break + } + + idx += w + } + + *i = idx + return r, pattern[start:idx], nil + } + + if r != '\'' { // single character, no escaped string + *i = idx + return '\'', pattern[start:idx], nil + } + + start = idx // skip ' character + iEnd := strings.IndexRune(pattern[start:], '\'') + if iEnd < 0 { + return r, "", errors.New("missing closing '") + } + + if iEnd == 0 { + // escape single quote literal + *i = idx + 1 + return r, pattern[start : idx+1], nil + } + + iEnd += start + + *i = iEnd + 1 // point after ' + if len(pattern) > iEnd+1 && pattern[iEnd+1] == '\'' { + return r, pattern[start : iEnd+1], nil + } + + return r, pattern[start:iEnd], nil +} diff --git a/libbeat/common/dtfmt/prog.go b/libbeat/common/dtfmt/prog.go new file mode 100644 index 000000000000..eb96e05c0a5b --- /dev/null +++ b/libbeat/common/dtfmt/prog.go @@ -0,0 +1,133 @@ +package dtfmt + +import ( + "errors" + "time" +) + +type prog struct { + p []byte +} + +const ( + opNone byte = iota + opCopy1 // copy next byte + opCopy2 // copy next 2 bytes + opCopy3 // copy next 3 bytes + opCopy4 // copy next 4 bytes + opCopyShort // [op, len, content[len]] + opCopyLong // [op, len1, len, content[len1<<8 + len]] + opNum // [op, ft] + opNumPadded // [op, ft, digits] + opTwoDigit // [op, ft] + opTextShort // [op, ft] + opTextLong // [op, ft] +) + +func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) { + for i := 0; i < len(p.p); { + op := p.p[i] + i++ + switch op { + case opNone: + + case opCopy1: + bytes = append(bytes, p.p[i]) + i++ + case opCopy2: + bytes = append(bytes, p.p[i], p.p[i+1]) + i += 2 + case opCopy3: + bytes = append(bytes, p.p[i], p.p[i+1], p.p[i+2]) + i += 3 + case opCopy4: + bytes = append(bytes, p.p[i], p.p[i+1], p.p[i+2], p.p[i+3]) + i += 4 + case opCopyShort: + l := int(p.p[i]) + i++ + bytes = append(bytes, p.p[i:i+l]...) + i += l + case opCopyLong: + l := int(p.p[i])<<8 | int(p.p[i+1]) + i += 2 + bytes = append(bytes, p.p[i:i+l]...) + i += l + case opNum: + ft := fieldType(p.p[i]) + i++ + v, err := getIntField(ft, ctx, t) + if err != nil { + return bytes, err + } + bytes = appendUnpadded(bytes, v) + case opNumPadded: + ft, digits := fieldType(p.p[i]), int(p.p[i+1]) + i += 2 + v, err := getIntField(ft, ctx, t) + if err != nil { + return bytes, err + } + bytes = appendPadded(bytes, v, digits) + case opTwoDigit: + ft := fieldType(p.p[i]) + i++ + v, err := getIntField(ft, ctx, t) + if err != nil { + return bytes, err + } + bytes = appendPadded(bytes, v%100, 2) + case opTextShort: + ft := fieldType(p.p[i]) + i++ + s, err := getTextFieldShort(ft, ctx, t) + if err != nil { + return bytes, err + } + bytes = append(bytes, s...) + case opTextLong: + ft := fieldType(p.p[i]) + i++ + s, err := getTextField(ft, ctx, t) + if err != nil { + return bytes, err + } + bytes = append(bytes, s...) + default: + return bytes, errors.New("unknown opcode") + } + } + + return bytes, nil +} + +func makeProg(b ...byte) (prog, error) { + return prog{b}, nil +} + +func makeCopy(b []byte) (prog, error) { + l := len(b) + switch l { + case 0: + return prog{}, nil + case 1: + return makeProg(opCopy1, b[0]) + case 2: + return makeProg(opCopy2, b[0], b[1]) + case 3: + return makeProg(opCopy2, b[0], b[1], b[2]) + case 4: + return makeProg(opCopy2, b[0], b[1], b[2], b[3]) + } + + if l < 256 { + return prog{append([]byte{opCopyShort, byte(l)}, b...)}, nil + } + if l < (1 << 16) { + l1 := byte(l >> 8) + l2 := byte(l) + return prog{append([]byte{opCopyLong, l1, l2}, b...)}, nil + } + + return prog{}, errors.New("literal too long") +} diff --git a/libbeat/common/dtfmt/util.go b/libbeat/common/dtfmt/util.go new file mode 100644 index 000000000000..7e33a63337db --- /dev/null +++ b/libbeat/common/dtfmt/util.go @@ -0,0 +1,43 @@ +package dtfmt + +import ( + "math" + "strconv" +) + +func appendUnpadded(bs []byte, i int) []byte { + return strconv.AppendInt(bs, int64(i), 10) +} + +func appendPadded(bs []byte, i int, sz int) []byte { + if i < 0 { + bs = append(bs, '-') + i = -i + } + + if i < 10 { + for ; sz > 1; sz-- { + bs = append(bs, '0') + } + return append(bs, byte(i)+'0') + } + if i < 100 { + for ; sz > 2; sz-- { + bs = append(bs, '0') + } + return strconv.AppendInt(bs, int64(i), 10) + } + + digits := 0 + if i < 1000 { + digits = 3 + } else if i < 10000 { + digits = 4 + } else { + digits = int(math.Log10(float64(i))) + 1 + } + for ; sz > digits; sz-- { + bs = append(bs, '0') + } + return strconv.AppendInt(bs, int64(i), 10) +} diff --git a/libbeat/common/fmtstr/formatevents.go b/libbeat/common/fmtstr/formatevents.go index ac361246c93f..a3c7932c38d6 100644 --- a/libbeat/common/fmtstr/formatevents.go +++ b/libbeat/common/fmtstr/formatevents.go @@ -7,8 +7,10 @@ import ( "reflect" "strconv" "strings" + "time" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/dtfmt" "github.com/elastic/beats/libbeat/logp" ) @@ -25,6 +27,7 @@ type EventFormatString struct { formatter StringFormatter ctx *eventEvalContext fields []fieldInfo + timestamp bool } type eventFieldEvaler struct { @@ -38,10 +41,16 @@ type defaultEventFieldEvaler struct { defaultValue string } +type eventTimestampEvaler struct { + ctx *eventEvalContext + formatter *dtfmt.Formatter +} + type eventFieldCompiler struct { - ctx *eventEvalContext - keys map[string]keyInfo - index int + ctx *eventEvalContext + keys map[string]keyInfo + timestamp bool + index int } type fieldInfo struct { @@ -56,6 +65,7 @@ type keyInfo struct { type eventEvalContext struct { keys []string + ts time.Time } var ( @@ -78,12 +88,13 @@ func MustCompileEvent(in string) *EventFormatString { func CompileEvent(in string) (*EventFormatString, error) { ctx := &eventEvalContext{} efComp := &eventFieldCompiler{ - ctx: ctx, - keys: map[string]keyInfo{}, - index: 0, + ctx: ctx, + keys: map[string]keyInfo{}, + index: 0, + timestamp: false, } - sf, err := Compile(in, efComp.compileEventField) + sf, err := Compile(in, efComp.compileExpression) if err != nil { return nil, err } @@ -101,6 +112,7 @@ func CompileEvent(in string) (*EventFormatString, error) { formatter: sf, ctx: ctx, fields: keys, + timestamp: efComp.timestamp, } return efs, nil } @@ -174,9 +186,43 @@ func (fs *EventFormatString) collectFields(event common.MapStr) error { fs.ctx.keys[i] = s } + if fs.timestamp { + timestamp, found := event["@timestamp"] + if !found { + return errors.New("missing timestamp") + } + + switch t := timestamp.(type) { + case common.Time: + fs.ctx.ts = time.Time(t) + case time.Time: + fs.ctx.ts = t + default: + return errors.New("unknown timestamp type") + } + } + return nil } +func (e *eventFieldCompiler) compileExpression( + s string, + opts []VariableOp, +) (FormatEvaler, error) { + if len(s) == 0 { + return nil, errors.New("empty expression") + } + + switch s[0] { + case '[': + return e.compileEventField(s, opts) + case '+': + return e.compileTimestamp(s, opts) + default: + return nil, fmt.Errorf(`unsupported format expression "%v"`, s) + } +} + func (e *eventFieldCompiler) compileEventField( field string, ops []VariableOp, @@ -221,6 +267,23 @@ func (e *eventFieldCompiler) compileEventField( return &defaultEventFieldEvaler{e.ctx, idx, defaultValue}, nil } +func (e *eventFieldCompiler) compileTimestamp( + expression string, + ops []VariableOp, +) (FormatEvaler, error) { + if expression[0] != '+' { + return nil, errors.New("No timestamp expression") + } + + formatter, err := dtfmt.NewFormatter(expression[1:]) + if err != nil { + return nil, fmt.Errorf("%v in timestamp expression", err) + } + + e.timestamp = true + return &eventTimestampEvaler{e.ctx, formatter}, nil +} + func (e *eventFieldEvaler) Eval(out *bytes.Buffer) error { type stringer interface { String() string @@ -244,6 +307,11 @@ func (e *defaultEventFieldEvaler) Eval(out *bytes.Buffer) error { return err } +func (e *eventTimestampEvaler) Eval(out *bytes.Buffer) error { + _, err := e.formatter.Write(out, e.ctx.ts) + return err +} + func parseEventPath(field string) (string, error) { field = strings.Trim(field, " \n\r\t") var fields []string diff --git a/libbeat/common/fmtstr/formatevents_test.go b/libbeat/common/fmtstr/formatevents_test.go index a23ccd0c9e82..43e79a0c488a 100644 --- a/libbeat/common/fmtstr/formatevents_test.go +++ b/libbeat/common/fmtstr/formatevents_test.go @@ -2,6 +2,7 @@ package fmtstr import ( "testing" + "time" "github.com/elastic/beats/libbeat/common" "github.com/stretchr/testify/assert" @@ -78,6 +79,18 @@ func TestEventFormatString(t *testing.T) { "value - value", []string{"key"}, }, + { + "test timestamp formatter", + "%{[key]}: %{+YYYY.MM.dd}", + common.MapStr{ + "@timestamp": common.Time( + time.Date(2015, 5, 1, 20, 12, 34, 0, time.Local), + ), + "key": "timestamp", + }, + "timestamp: 2015.05.01", + []string{"key"}, + }, } for i, test := range tests { @@ -129,6 +142,11 @@ func TestEventFormatStringErrors(t *testing.T) { "%{[field]:a:b}", false, nil, }, + { + "invalid timestamp formatter", + "%{+abc}", + false, nil, + }, { "missing required field", "%{[key]}",