diff --git a/probes/udp/udp_test.go b/probes/udp/udp_test.go index 4e11d174..1b1ab3be 100644 --- a/probes/udp/udp_test.go +++ b/probes/udp/udp_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-2019 The Cloudprober Authors. +// Copyright 2017-2021 The Cloudprober Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ // Workaround to skip UDP tests using a tag, until // https://github.com/google/cloudprober/issues/199 is fixed. +//go:build !skip_udp_probe_test // +build !skip_udp_probe_test package udp @@ -96,7 +97,10 @@ func startUDPServer(ctx context.Context, t *testing.T, drop bool, delay time.Dur const numTxPorts = 2 -func runProbe(ctx context.Context, t *testing.T, interval, timeout time.Duration, probesToSend int, scs *serverConnStats, conf configpb.ProbeConf) *Probe { +func runProbe(t *testing.T, interval, timeout time.Duration, probesToSend int, scs *serverConnStats, conf *configpb.ProbeConf) *Probe { + ctx, cancelCtx := context.WithCancel(context.Background()) + var wg sync.WaitGroup + sysvars.Init(&logger.Logger{}, nil) p := &Probe{} ipVersion := 6 @@ -110,7 +114,7 @@ func runProbe(ctx context.Context, t *testing.T, interval, timeout time.Duration Targets: targets.StaticTargets("localhost"), Interval: interval, Timeout: timeout, - ProbeConf: &conf, + ProbeConf: conf, StatsExportInterval: 10 * time.Second, } if err := p.Init("udp", opts); err != nil { @@ -120,11 +124,19 @@ func runProbe(ctx context.Context, t *testing.T, interval, timeout time.Duration p.initProbeRunResults() for _, conn := range p.connList { - go p.recvLoop(ctx, conn) + wg.Add(1) + go func(c *net.UDPConn) { + defer wg.Done() + p.recvLoop(ctx, c) + }(conn) } time.Sleep(time.Second) + + wg.Add(1) go func() { + defer wg.Done() + flushTicker := time.NewTicker(p.flushIntv) for { select { @@ -155,6 +167,9 @@ func runProbe(ctx context.Context, t *testing.T, interval, timeout time.Duration } t.Logf("Echo server stats: %v", scs.msgCt) + cancelCtx() + wg.Wait() + return p } @@ -179,18 +194,18 @@ func TestSuccessMultipleCasesResultPerPort(t *testing.T) { } for _, c := range cases { - ctx, cancelCtx := context.WithCancel(context.Background()) + ctx, cancelServerCtx := context.WithCancel(context.Background()) port, scs := startUDPServer(ctx, t, false, c.delay*time.Millisecond) t.Logf("Case(%s): started server on port %d with delay %v", c.name, port, c.delay) - conf := configpb.ProbeConf{ + conf := &configpb.ProbeConf{ UseAllTxPortsPerProbe: proto.Bool(c.useAllPorts), Port: proto.Int32(int32(port)), ExportMetricsByPort: proto.Bool(true), } - p := runProbe(ctx, t, c.interval*time.Millisecond, c.timeout*time.Millisecond, c.probeCount, scs, conf) - cancelCtx() + p := runProbe(t, c.interval*time.Millisecond, c.timeout*time.Millisecond, c.probeCount, scs, conf) + cancelServerCtx() if len(p.connList) != numTxPorts { t.Errorf("Case(%s): len(p.connList)=%d, want %d", c.name, len(p.connList), numTxPorts) @@ -231,15 +246,17 @@ func TestSuccessMultipleCasesDefaultResult(t *testing.T) { } for _, c := range cases { - ctx, cancelCtx := context.WithCancel(context.Background()) + ctx, cancelServerCtx := context.WithCancel(context.Background()) port, scs := startUDPServer(ctx, t, false, c.delay*time.Millisecond) t.Logf("Case(%s): started server on port %d with delay %v", c.name, port, c.delay) - conf := configpb.ProbeConf{ + conf := &configpb.ProbeConf{ UseAllTxPortsPerProbe: proto.Bool(c.useAllPorts), Port: proto.Int32(int32(port)), - ExportMetricsByPort: proto.Bool(false)} - p := runProbe(ctx, t, c.interval*time.Millisecond, c.timeout*time.Millisecond, c.probeCount, scs, conf) - cancelCtx() + ExportMetricsByPort: proto.Bool(false), + } + + p := runProbe(t, c.interval*time.Millisecond, c.timeout*time.Millisecond, c.probeCount, scs, conf) + cancelServerCtx() if len(p.connList) != numTxPorts { t.Errorf("Case(%s): len(p.connList)=%d, want %d", c.name, len(p.connList), numTxPorts) @@ -323,20 +340,19 @@ func TestLossAndDelayed(t *testing.T) { } for _, c := range cases { - ctx, cancelCtx := context.WithCancel(context.Background()) + ctx, cancelServerCtx := context.WithCancel(context.Background()) port, scs := startUDPServer(ctx, t, c.drop, c.delay*time.Millisecond) t.Logf("Case(%s): started server on port %d with loss %v delay %v", c.name, port, c.drop, c.delay) - conf := configpb.ProbeConf{ + conf := &configpb.ProbeConf{ UseAllTxPortsPerProbe: proto.Bool(true), Port: proto.Int32(int32(port)), ExportMetricsByPort: proto.Bool(true), } - p := runProbe(ctx, t, c.interval*time.Millisecond, c.timeout*time.Millisecond, int(pktCount), scs, conf) - - cancelCtx() + p := runProbe(t, c.interval*time.Millisecond, c.timeout*time.Millisecond, int(pktCount), scs, conf) + cancelServerCtx() if len(p.connList) != numTxPorts { t.Errorf("Case(%s): len(p.connList)=%d, want %d", c.name, len(p.connList), numTxPorts) diff --git a/probes/udplistener/udplistener_test.go b/probes/udplistener/udplistener_test.go index d3ef1cd1..74e2adfb 100644 --- a/probes/udplistener/udplistener_test.go +++ b/probes/udplistener/udplistener_test.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Cloudprober Authors. +// Copyright 2018-2021 The Cloudprober Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -145,8 +145,7 @@ func sendPktsAndCollectReplies(ctx context.Context, t *testing.T, srvPort int, i } func runProbe(ctx context.Context, t *testing.T, inp *inputState) ([]int, chan statskeeper.ProbeResult, *probeRunResult, *probeErr) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() + ctx, cancelCtx := context.WithCancel(ctx) sysvars.Init(&logger.Logger{}, nil) p := &Probe{} @@ -179,11 +178,20 @@ func runProbe(ctx context.Context, t *testing.T, inp *inputState) ([]int, chan s p.updateTargets() resultsChan := make(chan statskeeper.ProbeResult, 10) - go p.probeLoop(ctx, resultsChan) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + p.probeLoop(ctx, resultsChan) + }() + time.Sleep(interval) // Wait for echo loop to be active. rxSeq := sendPktsAndCollectReplies(ctx, t, port, inp) - cancel() + cancelCtx() + wg.Wait() return rxSeq, resultsChan, p.res[localhost], p.errs } diff --git a/rds/file/file.go b/rds/file/file.go new file mode 100644 index 00000000..a8f6ace4 --- /dev/null +++ b/rds/file/file.go @@ -0,0 +1,240 @@ +// Copyright 2021 The Cloudprober Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package file implements a file-based targets provider for cloudprober. +*/ +package file + +import ( + "fmt" + "math/rand" + "path/filepath" + "sync" + "time" + + "github.com/google/cloudprober/common/file" + "github.com/google/cloudprober/logger" + configpb "github.com/google/cloudprober/rds/file/proto" + pb "github.com/google/cloudprober/rds/proto" + "github.com/google/cloudprober/rds/server/filter" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/encoding/prototext" +) + +// DefaultProviderID is the povider id to use for this provider if a provider +// id is not configured explicitly. +const DefaultProviderID = "file" + +/* +SupportedFilters defines filters supported by the file-based resources +type. + Example: + filter { + key: "name" + value: "cloudprober.*" + } + filter { + key: "labels.app" + value: "service-a" + } +*/ +var SupportedFilters = struct { + RegexFilterKeys []string + LabelsFilter bool +}{ + []string{"name"}, + true, +} + +// lister implements file-based targets lister. +type lister struct { + mu sync.RWMutex + filePath string + format configpb.ProviderConfig_Format + resources []*pb.Resource + l *logger.Logger +} + +// ListResources returns the last successfully parsed list of resources. +func (ls *lister) ListResources(req *pb.ListResourcesRequest) (*pb.ListResourcesResponse, error) { + allFilters, err := filter.ParseFilters(req.GetFilter(), SupportedFilters.RegexFilterKeys, "") + if err != nil { + return nil, err + } + nameFilter, labelsFilter := allFilters.RegexFilters["name"], allFilters.LabelsFilter + + ls.mu.RLock() + defer ls.mu.RUnlock() + + // Allocate resources for response early but optimize for large number of + // total resources. + allocSize := len(ls.resources) + if allocSize > 10 && len(req.GetFilter()) != 0 { + allocSize = 10 + } + resources := make([]*pb.Resource, 0, allocSize) + + for _, res := range ls.resources { + if nameFilter != nil && !nameFilter.Match(res.GetName(), ls.l) { + continue + } + if labelsFilter != nil && !labelsFilter.Match(res.GetLabels(), ls.l) { + continue + } + resources = append(resources, res) + } + + ls.l.Infof("file.ListResources: returning %d resources out of %d", len(resources), len(ls.resources)) + return &pb.ListResourcesResponse{Resources: resources}, nil +} + +func (ls *lister) parseFileContent(b []byte) ([]*pb.Resource, error) { + resources := &configpb.FileResources{} + + switch ls.format { + case configpb.ProviderConfig_TEXTPB: + err := prototext.Unmarshal(b, resources) + if err != nil { + return nil, fmt.Errorf("file_provider(%s): error unmarshaling as text proto: %v", ls.filePath, err) + } + return resources.GetResource(), nil + case configpb.ProviderConfig_JSON: + err := protojson.Unmarshal(b, resources) + if err != nil { + return nil, fmt.Errorf("file_provider(%s): error unmarshaling as JSON: %v", ls.filePath, err) + } + return resources.GetResource(), nil + } + + return nil, fmt.Errorf("file_provider(%s): unknown format - %v", ls.filePath, ls.format) +} + +func (ls *lister) refresh() error { + b, err := file.ReadFile(ls.filePath) + if err != nil { + return fmt.Errorf("file(%s): error while reading file: %v", ls.filePath, err) + } + + resources, err := ls.parseFileContent(b) + if err != nil { + return err + } + + ls.mu.Lock() + defer ls.mu.Unlock() + ls.resources = resources + + ls.l.Infof("file_provider(%s): Read %d resources.", ls.filePath, len(ls.resources)) + return nil +} + +func formatFromPath(path string) configpb.ProviderConfig_Format { + switch filepath.Ext(path) { + case ".textpb": + return configpb.ProviderConfig_TEXTPB + case ".json": + return configpb.ProviderConfig_JSON + } + return configpb.ProviderConfig_TEXTPB +} + +// newLister creates a new file-based targets lister. +func newLister(filePath string, format configpb.ProviderConfig_Format, reEvalSec int32, l *logger.Logger) (*lister, error) { + if format == configpb.ProviderConfig_UNSPECIFIED { + format = formatFromPath(filePath) + l.Infof("file_provider: Determined file format from file name: %v", format) + } + + ls := &lister{ + filePath: filePath, + format: format, + l: l, + } + + if reEvalSec == 0 { + return ls, ls.refresh() + } + + reEvalInterval := time.Duration(reEvalSec) * time.Second + go func() { + if err := ls.refresh(); err != nil { + l.Error(err.Error()) + } + // Introduce a random delay between 0-reEvalInterval before + // starting the refresh loop. If there are multiple cloudprober + // instances, this will make sure that each instance refreshes + // at a different point of time. + rand.Seed(time.Now().UnixNano()) + randomDelaySec := rand.Intn(int(reEvalInterval.Seconds())) + time.Sleep(time.Duration(randomDelaySec) * time.Second) + for range time.Tick(reEvalInterval) { + if err := ls.refresh(); err != nil { + l.Error(err.Error()) + } + } + }() + + return ls, nil +} + +// ListResources returns the list of resources based on the given request. +func (p *Provider) ListResources(req *pb.ListResourcesRequest) (*pb.ListResourcesResponse, error) { + fPath := req.GetResourcePath() + if fPath != "" { + ls := p.listers[fPath] + if ls == nil { + return nil, fmt.Errorf("file path %s is not available on this server", fPath) + } + return ls.ListResources(req) + } + + var result []*pb.Resource + for _, fp := range p.filePaths { + res, err := p.listers[fp].ListResources(req) + if err != nil { + return nil, err + } + result = append(result, res.Resources...) + } + + return &pb.ListResourcesResponse{Resources: result}, nil +} + +// Provider provides a file-based targets provider for RDS. It implements the +// RDS server's Provider interface. +type Provider struct { + filePaths []string + listers map[string]*lister +} + +// New creates a File (file) provider for RDS server, based on the +// provided config. +func New(c *configpb.ProviderConfig, l *logger.Logger) (*Provider, error) { + filePaths := c.GetFilePath() + p := &Provider{ + filePaths: filePaths, + listers: make(map[string]*lister), + } + + for _, filePath := range filePaths { + lister, err := newLister(filePath, c.GetFormat(), c.GetReEvalSec(), l) + if err != nil { + return nil, err + } + p.listers[filePath] = lister + } + + return p, nil +} diff --git a/rds/file/file_test.go b/rds/file/file_test.go new file mode 100644 index 00000000..1f5474ab --- /dev/null +++ b/rds/file/file_test.go @@ -0,0 +1,163 @@ +// Copyright 2021 The Cloudprober Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "fmt" + "strconv" + "testing" + "time" + + configpb "github.com/google/cloudprober/rds/file/proto" + pb "github.com/google/cloudprober/rds/proto" + rdspb "github.com/google/cloudprober/rds/proto" + "google.golang.org/protobuf/proto" +) + +var testResourcesFiles = map[string][]string{ + "textpb": []string{"testdata/targets1.textpb", "testdata/targets2.textpb"}, + "json": []string{"testdata/targets.json"}, +} + +var testExpectedResources = []*rdspb.Resource{ + { + Name: proto.String("switch-xx-1"), + Port: proto.Int32(8080), + Ip: proto.String("10.1.1.1"), + Labels: map[string]string{ + "device_type": "switch", + "cluster": "xx", + }, + }, + { + Name: proto.String("switch-xx-2"), + Port: proto.Int32(8081), + Ip: proto.String("10.1.1.2"), + Labels: map[string]string{ + "cluster": "xx", + }, + }, + { + Name: proto.String("switch-yy-1"), + Port: proto.Int32(8080), + Ip: proto.String("10.1.2.1"), + }, + { + Name: proto.String("switch-zz-1"), + Port: proto.Int32(8080), + Ip: proto.String("::aaa:1"), + }, +} + +func compareResourceList(t *testing.T, got []*pb.Resource, want []*pb.Resource) { + t.Helper() + + if len(got) != len(want) { + t.Fatalf("Got resources: %d, expected: %d", len(got), len(want)) + } + for i := range want { + if got[i].String() != want[i].String() { + t.Errorf("ListResources: got[%d]:\n%s\nexpected[%d]:\n%s", i, got[i].String(), i, want[i].String()) + } + } +} + +func TestListResources(t *testing.T) { + for _, filetype := range []string{"textpb", "json"} { + t.Run(filetype, func(t *testing.T) { + p, err := New(&configpb.ProviderConfig{FilePath: testResourcesFiles[filetype]}, nil) + if err != nil { + t.Fatalf("Unexpected error while creating new provider: %v", err) + } + + for _, test := range []struct { + desc string + resourcePath string + f []*pb.Filter + wantResources []*pb.Resource + }{ + { + desc: "no_filter", + wantResources: testExpectedResources, + }, + { + desc: "with_filter", + f: []*pb.Filter{ + { + Key: proto.String("labels.cluster"), + Value: proto.String("xx"), + }, + }, + wantResources: testExpectedResources[:2], + }, + } { + t.Run(test.desc, func(t *testing.T) { + got, err := p.ListResources(&rdspb.ListResourcesRequest{Filter: test.f}) + if err != nil { + t.Fatalf("Unexpected error while listing resources: %v", err) + } + compareResourceList(t, got.Resources, test.wantResources) + }) + } + }) + } +} + +func TestListResourcesWithResourcePath(t *testing.T) { + p, err := New(&configpb.ProviderConfig{FilePath: testResourcesFiles["textpb"]}, nil) + if err != nil { + t.Fatalf("Unexpected error while creating new provider: %v", err) + } + got, err := p.ListResources(&rdspb.ListResourcesRequest{ResourcePath: proto.String(testResourcesFiles["textpb"][1])}) + if err != nil { + t.Fatalf("Unexpected error while listing resources: %v", err) + } + compareResourceList(t, got.Resources, testExpectedResources[2:]) +} + +func BenchmarkListResources(b *testing.B) { + for _, n := range []int{100, 10000, 1000000} { + b.Run(fmt.Sprintf("%d-resources", n), func(b *testing.B) { + b.StopTimer() + ls := &lister{ + resources: make([]*pb.Resource, n), + } + for i := 0; i < n; i++ { + ls.resources[i] = &pb.Resource{ + Name: proto.String(fmt.Sprintf("host-%d", i)), + Ip: proto.String("10.1.1.1"), + Port: proto.Int32(80), + Labels: map[string]string{ + "index": strconv.Itoa(i), + }, + LastUpdated: proto.Int64(time.Now().Unix()), + } + } + b.StartTimer() + + for j := 0; j < b.N; j++ { + res, err := ls.ListResources(nil) + + if err != nil { + b.Errorf("Unexpected error while listing resources: %v", err) + } + + if len(res.GetResources()) != n { + b.Errorf("Got %d resources, wanted: %d", len(res.GetResources()), n) + } + } + }) + } +} diff --git a/rds/file/proto/config.pb.go b/rds/file/proto/config.pb.go new file mode 100644 index 00000000..aa18134b --- /dev/null +++ b/rds/file/proto/config.pb.go @@ -0,0 +1,350 @@ +// Configuration proto for Kubernetes provider. +// +// Example provider config: +// { +// pods {} +// } +// +// In probe config: +// probe { +// targets{ +// rds_targets { +// resource_path: "k8s://pods" +// filter { +// key: "namespace" +// value: "default" +// } +// filter { +// key: "name" +// value: "cloudprober.*" +// } +// } +// } +// } + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.11.2 +// source: github.com/google/cloudprober/rds/file/proto/config.proto + +package proto + +import ( + proto "github.com/google/cloudprober/rds/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ProviderConfig_Format int32 + +const ( + ProviderConfig_UNSPECIFIED ProviderConfig_Format = 0 // Determine format using file extension/ + ProviderConfig_TEXTPB ProviderConfig_Format = 1 // Text proto format (.textpb). + ProviderConfig_JSON ProviderConfig_Format = 2 // JSON proto format (.json). +) + +// Enum value maps for ProviderConfig_Format. +var ( + ProviderConfig_Format_name = map[int32]string{ + 0: "UNSPECIFIED", + 1: "TEXTPB", + 2: "JSON", + } + ProviderConfig_Format_value = map[string]int32{ + "UNSPECIFIED": 0, + "TEXTPB": 1, + "JSON": 2, + } +) + +func (x ProviderConfig_Format) Enum() *ProviderConfig_Format { + p := new(ProviderConfig_Format) + *p = x + return p +} + +func (x ProviderConfig_Format) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ProviderConfig_Format) Descriptor() protoreflect.EnumDescriptor { + return file_github_com_google_cloudprober_rds_file_proto_config_proto_enumTypes[0].Descriptor() +} + +func (ProviderConfig_Format) Type() protoreflect.EnumType { + return &file_github_com_google_cloudprober_rds_file_proto_config_proto_enumTypes[0] +} + +func (x ProviderConfig_Format) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *ProviderConfig_Format) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = ProviderConfig_Format(num) + return nil +} + +// Deprecated: Use ProviderConfig_Format.Descriptor instead. +func (ProviderConfig_Format) EnumDescriptor() ([]byte, []int) { + return file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescGZIP(), []int{0, 0} +} + +// File provider config. +type ProviderConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // File that contains resources in either textproto or json format. + // Example in textproto format: + // + // resource { + // name: "switch-xx-01" + // ip: "10.11.112.3" + // port: 8080 + // labels { + // key: "device_type" + // value: "switch" + // } + // } + // resource { + // name: "switch-yy-01" + // ip: "10.16.110.12" + // port: 8080 + // } + FilePath []string `protobuf:"bytes,1,rep,name=file_path,json=filePath" json:"file_path,omitempty"` + Format *ProviderConfig_Format `protobuf:"varint,2,opt,name=format,enum=cloudprober.rds.file.ProviderConfig_Format" json:"format,omitempty"` + // If specified, file will be re-read at the given interval. + ReEvalSec *int32 `protobuf:"varint,3,opt,name=re_eval_sec,json=reEvalSec" json:"re_eval_sec,omitempty"` +} + +func (x *ProviderConfig) Reset() { + *x = ProviderConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ProviderConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProviderConfig) ProtoMessage() {} + +func (x *ProviderConfig) ProtoReflect() protoreflect.Message { + mi := &file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProviderConfig.ProtoReflect.Descriptor instead. +func (*ProviderConfig) Descriptor() ([]byte, []int) { + return file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescGZIP(), []int{0} +} + +func (x *ProviderConfig) GetFilePath() []string { + if x != nil { + return x.FilePath + } + return nil +} + +func (x *ProviderConfig) GetFormat() ProviderConfig_Format { + if x != nil && x.Format != nil { + return *x.Format + } + return ProviderConfig_UNSPECIFIED +} + +func (x *ProviderConfig) GetReEvalSec() int32 { + if x != nil && x.ReEvalSec != nil { + return *x.ReEvalSec + } + return 0 +} + +type FileResources struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Resource []*proto.Resource `protobuf:"bytes,1,rep,name=resource" json:"resource,omitempty"` +} + +func (x *FileResources) Reset() { + *x = FileResources{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileResources) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileResources) ProtoMessage() {} + +func (x *FileResources) ProtoReflect() protoreflect.Message { + mi := &file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileResources.ProtoReflect.Descriptor instead. +func (*FileResources) Descriptor() ([]byte, []int) { + return file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescGZIP(), []int{1} +} + +func (x *FileResources) GetResource() []*proto.Resource { + if x != nil { + return x.Resource + } + return nil +} + +var File_github_com_google_cloudprober_rds_file_proto_config_proto protoreflect.FileDescriptor + +var file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDesc = []byte{ + 0x0a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, + 0x72, 0x64, 0x73, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x14, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x66, 0x69, 0x6c, + 0x65, 0x1a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, + 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x64, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc3, 0x01, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, + 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, + 0x50, 0x61, 0x74, 0x68, 0x12, 0x43, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, + 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x76, + 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x46, 0x6f, 0x72, 0x6d, 0x61, + 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1e, 0x0a, 0x0b, 0x72, 0x65, 0x5f, + 0x65, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, + 0x72, 0x65, 0x45, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x22, 0x2f, 0x0a, 0x06, 0x46, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x54, 0x45, 0x58, 0x54, 0x50, 0x42, 0x10, 0x01, + 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x02, 0x22, 0x46, 0x0a, 0x0d, 0x46, 0x69, + 0x6c, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x72, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, + 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, + 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, +} + +var ( + file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescOnce sync.Once + file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescData = file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDesc +) + +func file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescGZIP() []byte { + file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescOnce.Do(func() { + file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescData) + }) + return file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDescData +} + +var file_github_com_google_cloudprober_rds_file_proto_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_github_com_google_cloudprober_rds_file_proto_config_proto_goTypes = []interface{}{ + (ProviderConfig_Format)(0), // 0: cloudprober.rds.file.ProviderConfig.Format + (*ProviderConfig)(nil), // 1: cloudprober.rds.file.ProviderConfig + (*FileResources)(nil), // 2: cloudprober.rds.file.FileResources + (*proto.Resource)(nil), // 3: cloudprober.rds.Resource +} +var file_github_com_google_cloudprober_rds_file_proto_config_proto_depIdxs = []int32{ + 0, // 0: cloudprober.rds.file.ProviderConfig.format:type_name -> cloudprober.rds.file.ProviderConfig.Format + 3, // 1: cloudprober.rds.file.FileResources.resource:type_name -> cloudprober.rds.Resource + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_github_com_google_cloudprober_rds_file_proto_config_proto_init() } +func file_github_com_google_cloudprober_rds_file_proto_config_proto_init() { + if File_github_com_google_cloudprober_rds_file_proto_config_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ProviderConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileResources); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDesc, + NumEnums: 1, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_google_cloudprober_rds_file_proto_config_proto_goTypes, + DependencyIndexes: file_github_com_google_cloudprober_rds_file_proto_config_proto_depIdxs, + EnumInfos: file_github_com_google_cloudprober_rds_file_proto_config_proto_enumTypes, + MessageInfos: file_github_com_google_cloudprober_rds_file_proto_config_proto_msgTypes, + }.Build() + File_github_com_google_cloudprober_rds_file_proto_config_proto = out.File + file_github_com_google_cloudprober_rds_file_proto_config_proto_rawDesc = nil + file_github_com_google_cloudprober_rds_file_proto_config_proto_goTypes = nil + file_github_com_google_cloudprober_rds_file_proto_config_proto_depIdxs = nil +} diff --git a/rds/file/proto/config.proto b/rds/file/proto/config.proto new file mode 100644 index 00000000..c95f6bd2 --- /dev/null +++ b/rds/file/proto/config.proto @@ -0,0 +1,66 @@ +// Configuration proto for Kubernetes provider. +// +// Example provider config: +// { +// pods {} +// } +// +// In probe config: +// probe { +// targets{ +// rds_targets { +// resource_path: "k8s://pods" +// filter { +// key: "namespace" +// value: "default" +// } +// filter { +// key: "name" +// value: "cloudprober.*" +// } +// } +// } +// } +syntax = "proto2"; + +package cloudprober.rds.file; + +import "github.com/google/cloudprober/rds/proto/rds.proto"; + +option go_package = "github.com/google/cloudprober/rds/file/proto"; + +// File provider config. +message ProviderConfig { + // File that contains resources in either textproto or json format. + // Example in textproto format: + // + // resource { + // name: "switch-xx-01" + // ip: "10.11.112.3" + // port: 8080 + // labels { + // key: "device_type" + // value: "switch" + // } + // } + // resource { + // name: "switch-yy-01" + // ip: "10.16.110.12" + // port: 8080 + // } + repeated string file_path = 1; + + enum Format { + UNSPECIFIED = 0; // Determine format using file extension/ + TEXTPB = 1; // Text proto format (.textpb). + JSON = 2; // JSON proto format (.json). + } + optional Format format = 2; + + // If specified, file will be re-read at the given interval. + optional int32 re_eval_sec = 3; +} + +message FileResources { + repeated .cloudprober.rds.Resource resource = 1; +} diff --git a/rds/file/testdata/targets.json b/rds/file/testdata/targets.json new file mode 100644 index 00000000..14d42380 --- /dev/null +++ b/rds/file/testdata/targets.json @@ -0,0 +1,31 @@ +{ + "resource": [ + { + "name": "switch-xx-1", + "ip": "10.1.1.1", + "port": 8080, + "labels": { + "device_type": "switch", + "cluster": "xx" + } + }, + { + "name": "switch-xx-2", + "ip": "10.1.1.2", + "port": 8081, + "labels": { + "cluster": "xx" + } + }, + { + "name": "switch-yy-1", + "ip": "10.1.2.1", + "port": 8080 + }, + { + "name": "switch-zz-1", + "ip": "::aaa:1", + "port": 8080 + } + ] +} diff --git a/rds/file/testdata/targets1.textpb b/rds/file/testdata/targets1.textpb new file mode 100644 index 00000000..ecad1f25 --- /dev/null +++ b/rds/file/testdata/targets1.textpb @@ -0,0 +1,23 @@ +resource { + name: "switch-xx-1" + ip: "10.1.1.1" + port: 8080 + labels { + key: "device_type" + value: "switch" + } + labels { + key: "cluster" + value: "xx" + } +} + +resource { + name: "switch-xx-2" + ip: "10.1.1.2" + port: 8081 + labels { + key: "cluster" + value: "xx" + } +} diff --git a/rds/file/testdata/targets2.textpb b/rds/file/testdata/targets2.textpb new file mode 100644 index 00000000..fb80c429 --- /dev/null +++ b/rds/file/testdata/targets2.textpb @@ -0,0 +1,11 @@ +resource { + name: "switch-yy-1" + ip: "10.1.2.1" + port: 8080 +} + +resource { + name: "switch-zz-1" + ip: "::aaa:1" + port: 8080 +} diff --git a/rds/server/proto/config.pb.go b/rds/server/proto/config.pb.go index 1bf4d691..a088239a 100644 --- a/rds/server/proto/config.pb.go +++ b/rds/server/proto/config.pb.go @@ -19,8 +19,9 @@ package proto import ( - proto "github.com/google/cloudprober/rds/gcp/proto" - proto1 "github.com/google/cloudprober/rds/kubernetes/proto" + proto "github.com/google/cloudprober/rds/file/proto" + proto1 "github.com/google/cloudprober/rds/gcp/proto" + proto2 "github.com/google/cloudprober/rds/kubernetes/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -91,6 +92,7 @@ type Provider struct { // providers based on this id. Id *string `protobuf:"bytes,1,opt,name=id" json:"id,omitempty"` // Types that are assignable to Config: + // *Provider_FileConfig // *Provider_GcpConfig // *Provider_KubernetesConfig Config isProvider_Config `protobuf_oneof:"config"` @@ -142,14 +144,21 @@ func (m *Provider) GetConfig() isProvider_Config { return nil } -func (x *Provider) GetGcpConfig() *proto.ProviderConfig { +func (x *Provider) GetFileConfig() *proto.ProviderConfig { + if x, ok := x.GetConfig().(*Provider_FileConfig); ok { + return x.FileConfig + } + return nil +} + +func (x *Provider) GetGcpConfig() *proto1.ProviderConfig { if x, ok := x.GetConfig().(*Provider_GcpConfig); ok { return x.GcpConfig } return nil } -func (x *Provider) GetKubernetesConfig() *proto1.ProviderConfig { +func (x *Provider) GetKubernetesConfig() *proto2.ProviderConfig { if x, ok := x.GetConfig().(*Provider_KubernetesConfig); ok { return x.KubernetesConfig } @@ -160,14 +169,20 @@ type isProvider_Config interface { isProvider_Config() } +type Provider_FileConfig struct { + FileConfig *proto.ProviderConfig `protobuf:"bytes,4,opt,name=file_config,json=fileConfig,oneof"` +} + type Provider_GcpConfig struct { - GcpConfig *proto.ProviderConfig `protobuf:"bytes,2,opt,name=gcp_config,json=gcpConfig,oneof"` + GcpConfig *proto1.ProviderConfig `protobuf:"bytes,2,opt,name=gcp_config,json=gcpConfig,oneof"` } type Provider_KubernetesConfig struct { - KubernetesConfig *proto1.ProviderConfig `protobuf:"bytes,3,opt,name=kubernetes_config,json=kubernetesConfig,oneof"` + KubernetesConfig *proto2.ProviderConfig `protobuf:"bytes,3,opt,name=kubernetes_config,json=kubernetesConfig,oneof"` } +func (*Provider_FileConfig) isProvider_Config() {} + func (*Provider_GcpConfig) isProvider_Config() {} func (*Provider_KubernetesConfig) isProvider_Config() {} @@ -179,35 +194,43 @@ var file_github_com_google_cloudprober_rds_server_proto_config_proto_rawDesc = [ 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x1a, 0x38, + 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x1a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, - 0x73, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x6b, 0x75, 0x62, 0x65, - 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x43, 0x0a, 0x0a, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x35, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, - 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0xc5, - 0x01, 0x0a, 0x08, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x44, 0x0a, 0x0a, 0x67, - 0x63, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x23, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, - 0x73, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x09, 0x67, 0x63, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x12, 0x59, 0x0a, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x5f, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x6b, - 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, - 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x10, 0x6b, 0x75, 0x62, 0x65, - 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x08, 0x0a, 0x06, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x73, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, + 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x67, 0x63, 0x70, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x3f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, + 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, + 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x43, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, + 0x6e, 0x66, 0x12, 0x35, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, + 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, + 0x08, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x22, 0x8e, 0x02, 0x0a, 0x08, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x47, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x6c, + 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x66, 0x69, + 0x6c, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x44, 0x0a, 0x0a, 0x67, 0x63, 0x70, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, + 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, + 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x09, 0x67, 0x63, 0x70, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x59, 0x0a, 0x11, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, + 0x74, 0x65, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x2a, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, + 0x64, 0x73, 0x2e, 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x2e, 0x50, 0x72, + 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x10, + 0x6b, 0x75, 0x62, 0x65, 0x72, 0x6e, 0x65, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -226,18 +249,20 @@ var file_github_com_google_cloudprober_rds_server_proto_config_proto_msgTypes = var file_github_com_google_cloudprober_rds_server_proto_config_proto_goTypes = []interface{}{ (*ServerConf)(nil), // 0: cloudprober.rds.ServerConf (*Provider)(nil), // 1: cloudprober.rds.Provider - (*proto.ProviderConfig)(nil), // 2: cloudprober.rds.gcp.ProviderConfig - (*proto1.ProviderConfig)(nil), // 3: cloudprober.rds.kubernetes.ProviderConfig + (*proto.ProviderConfig)(nil), // 2: cloudprober.rds.file.ProviderConfig + (*proto1.ProviderConfig)(nil), // 3: cloudprober.rds.gcp.ProviderConfig + (*proto2.ProviderConfig)(nil), // 4: cloudprober.rds.kubernetes.ProviderConfig } var file_github_com_google_cloudprober_rds_server_proto_config_proto_depIdxs = []int32{ 1, // 0: cloudprober.rds.ServerConf.provider:type_name -> cloudprober.rds.Provider - 2, // 1: cloudprober.rds.Provider.gcp_config:type_name -> cloudprober.rds.gcp.ProviderConfig - 3, // 2: cloudprober.rds.Provider.kubernetes_config:type_name -> cloudprober.rds.kubernetes.ProviderConfig - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 2, // 1: cloudprober.rds.Provider.file_config:type_name -> cloudprober.rds.file.ProviderConfig + 3, // 2: cloudprober.rds.Provider.gcp_config:type_name -> cloudprober.rds.gcp.ProviderConfig + 4, // 3: cloudprober.rds.Provider.kubernetes_config:type_name -> cloudprober.rds.kubernetes.ProviderConfig + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_github_com_google_cloudprober_rds_server_proto_config_proto_init() } @@ -272,6 +297,7 @@ func file_github_com_google_cloudprober_rds_server_proto_config_proto_init() { } } file_github_com_google_cloudprober_rds_server_proto_config_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*Provider_FileConfig)(nil), (*Provider_GcpConfig)(nil), (*Provider_KubernetesConfig)(nil), } diff --git a/rds/server/proto/config.proto b/rds/server/proto/config.proto index da60612c..78121b0d 100644 --- a/rds/server/proto/config.proto +++ b/rds/server/proto/config.proto @@ -13,6 +13,7 @@ syntax = "proto2"; package cloudprober.rds; +import "github.com/google/cloudprober/rds/file/proto/config.proto"; import "github.com/google/cloudprober/rds/gcp/proto/config.proto"; import "github.com/google/cloudprober/rds/kubernetes/proto/config.proto"; @@ -29,6 +30,7 @@ message Provider { optional string id = 1; oneof config { + file.ProviderConfig file_config = 4; gcp.ProviderConfig gcp_config = 2; kubernetes.ProviderConfig kubernetes_config = 3; } diff --git a/rds/server/server.go b/rds/server/server.go index 9ffe5830..b089a5bb 100644 --- a/rds/server/server.go +++ b/rds/server/server.go @@ -24,6 +24,7 @@ import ( "fmt" "github.com/google/cloudprober/logger" + "github.com/google/cloudprober/rds/file" "github.com/google/cloudprober/rds/gcp" "github.com/google/cloudprober/rds/kubernetes" pb "github.com/google/cloudprober/rds/proto" @@ -59,6 +60,14 @@ func (s *Server) initProviders(c *configpb.ServerConf) error { for _, pc := range c.GetProvider() { id := pc.GetId() switch pc.Config.(type) { + case *configpb.Provider_FileConfig: + if id == "" { + id = file.DefaultProviderID + } + s.l.Infof("rds.server: adding file provider with id: %s", id) + if p, err = file.New(pc.GetFileConfig(), s.l); err != nil { + return err + } case *configpb.Provider_GcpConfig: if id == "" { id = gcp.DefaultProviderID diff --git a/targets/file/file.go b/targets/file/file.go index 9a134df2..6e048b28 100644 --- a/targets/file/file.go +++ b/targets/file/file.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Cloudprober Authors. +// Copyright 2020-2021 The Cloudprober Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,212 +18,33 @@ Package file implements a file-based targets for cloudprober. package file import ( - "fmt" - "math/rand" - "net" - "path/filepath" - "sync" - "time" + "context" - "github.com/google/cloudprober/common/file" - "github.com/google/cloudprober/common/iputils" "github.com/google/cloudprober/logger" + "github.com/google/cloudprober/rds/client" + client_configpb "github.com/google/cloudprober/rds/client/proto" + "github.com/google/cloudprober/rds/file" + file_configpb "github.com/google/cloudprober/rds/file/proto" rdspb "github.com/google/cloudprober/rds/proto" - "github.com/google/cloudprober/rds/server/filter" - "github.com/google/cloudprober/targets/endpoint" configpb "github.com/google/cloudprober/targets/file/proto" dnsRes "github.com/google/cloudprober/targets/resolver" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/encoding/prototext" ) -// Targets encapsulates file based targets. -type Targets struct { - path string - format configpb.TargetsConf_Format - r *dnsRes.Resolver - - names []string - resources map[string]*rdspb.Resource - nameFilter *filter.RegexFilter - labelsFilter *filter.LabelsFilter - mu sync.RWMutex - l *logger.Logger -} - -/* -SupportedFilters defines filters supported by the file-based resources -type. - Example: - filter { - key: "name" - value: "cloudprober.*" - } - filter { - key: "labels.app" - value: "service-a" - } -*/ -var SupportedFilters = struct { - RegexFilterKeys []string - LabelsFilter bool -}{ - []string{"name"}, - true, -} - // New returns new file targets. -func New(opts *configpb.TargetsConf, res *dnsRes.Resolver, l *logger.Logger) (*Targets, error) { - ft := &Targets{ - path: opts.GetFilePath(), - resources: make(map[string]*rdspb.Resource), - r: res, - l: l, - } - - ft.format = opts.GetFormat() - if ft.format == configpb.TargetsConf_UNSPECIFIED { - ft.format = ft.formatFromPath() - ft.l.Infof("file_targets: Determined file format from file name: %v", ft.format) - } - - allFilters, err := filter.ParseFilters(opts.GetFilter(), SupportedFilters.RegexFilterKeys, "") +func New(opts *configpb.TargetsConf, res *dnsRes.Resolver, l *logger.Logger) (*client.Client, error) { + lister, err := file.New(&file_configpb.ProviderConfig{ + FilePath: []string{opts.GetFilePath()}, + }, l) if err != nil { return nil, err } - ft.nameFilter, ft.labelsFilter = allFilters.RegexFilters["name"], allFilters.LabelsFilter - - if opts.GetReEvalSec() == 0 { - return ft, ft.refresh() - } - - reEvalInterval := time.Duration(opts.GetReEvalSec()) * time.Second - go func() { - if err := ft.refresh(); err != nil { - ft.l.Error(err.Error()) - } - // Introduce a random delay between 0-reEvalInterval before - // starting the refresh loop. If there are multiple cloudprober - // instances, this will make sure that each instance refreshes - // at a different point of time. - rand.Seed(time.Now().UnixNano()) - randomDelaySec := rand.Intn(int(reEvalInterval.Seconds())) - time.Sleep(time.Duration(randomDelaySec) * time.Second) - for range time.Tick(reEvalInterval) { - if err := ft.refresh(); err != nil { - ft.l.Error(err.Error()) - } - } - }() - - return ft, nil -} - -func (ft *Targets) formatFromPath() configpb.TargetsConf_Format { - switch filepath.Ext(ft.path) { - case ".textpb": - return configpb.TargetsConf_TEXTPB - case ".json": - return configpb.TargetsConf_JSON - } - return configpb.TargetsConf_TEXTPB -} - -// ListEndpoints returns the list of endpoints in the file based targets. -func (ft *Targets) ListEndpoints() []endpoint.Endpoint { - ft.mu.RLock() - defer ft.mu.RUnlock() - endpoints := make([]endpoint.Endpoint, len(ft.names)) - for i, name := range ft.names { - endpoints[i] = endpoint.Endpoint{ - Name: name, - Labels: ft.resources[name].GetLabels(), - Port: int(ft.resources[name].GetPort()), - } - } - - return endpoints -} - -func (ft *Targets) refresh() error { - b, err := file.ReadFile(ft.path) - if err != nil { - return fmt.Errorf("file_targets(%s): error while reading file: %v", ft.path, err) - } - return ft.parseFileContent(b) -} - -func (ft *Targets) parseFileContent(b []byte) error { - resources := &configpb.FileResources{} - - switch ft.format { - case configpb.TargetsConf_TEXTPB: - err := prototext.Unmarshal(b, resources) - if err != nil { - return fmt.Errorf("file_targets(%s): error unmarshaling as text proto: %v", ft.path, err) - } - case configpb.TargetsConf_JSON: - err := protojson.Unmarshal(b, resources) - if err != nil { - return fmt.Errorf("file_targets(%s): error unmarshaling as JSON: %v", ft.path, err) - } - default: - return fmt.Errorf("file_targets(%s): unknown format - %v", ft.path, ft.format) - } - - // Update state. - ft.mu.Lock() - defer ft.mu.Unlock() - - ft.names = make([]string, len(resources.GetResource())) - - i := 0 - for _, resource := range resources.GetResource() { - name := resource.GetName() - if ft.nameFilter != nil && !ft.nameFilter.Match(name, ft.l) { - continue - } - if ft.labelsFilter != nil && !ft.labelsFilter.Match(resource.GetLabels(), ft.l) { - continue - } - ft.resources[name] = resource - ft.names[i] = name - i++ - } - ft.names = ft.names[:i] - - ft.l.Infof("file_targets(%s): Read %d resources, kept %d after filtering", ft.path, len(resources.GetResource()), len(ft.names)) - return nil -} - -// Resolve returns the IP address for the given resource. If no IP address is -// configured in the file, DNS resolver is used to find the IP address. -func (ft *Targets) Resolve(name string, ipVer int) (net.IP, error) { - ft.mu.RLock() - defer ft.mu.RUnlock() - - res := ft.resources[name] - if res == nil { - return nil, fmt.Errorf("file_targets(%s): Resource %s not found", ft.path, name) - } - - // Use global resolver if file doesn't have IP address. - if res.GetIp() == "" { - if ft.r == nil { - return nil, fmt.Errorf("file_targets(%s): IP not configured for %s", ft.path, name) - } - return ft.r.Resolve(name, ipVer) - } - - ip := net.ParseIP(res.GetIp()) - if ip == nil { - return nil, fmt.Errorf("file_targets(%s): Error while parsing IP %s for %s", ft.path, res.GetIp(), name) - } - - if ipVer == 0 || iputils.IPVersion(ip) == ipVer { - return ip, nil + clientConf := &client_configpb.ClientConf{ + Request: &rdspb.ListResourcesRequest{Filter: opts.GetFilter()}, + ReEvalSec: opts.ReEvalSec, } - return nil, fmt.Errorf("file_targets(%s): No IPv%d address (IP: %s) for %s", ft.path, ipVer, ip.String(), name) + return client.New(clientConf, func(_ context.Context, req *rdspb.ListResourcesRequest) (*rdspb.ListResourcesResponse, error) { + return lister.ListResources(req) + }, l) } diff --git a/targets/file/file_test.go b/targets/file/file_test.go index a3c6997d..300e95f3 100644 --- a/targets/file/file_test.go +++ b/targets/file/file_test.go @@ -1,7 +1,20 @@ +// Copyright 2021 The Cloudprober Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package file import ( - "io/ioutil" "reflect" "testing" @@ -11,79 +24,6 @@ import ( "google.golang.org/protobuf/proto" ) -var testResourcesTextpb = ` -resource { - name: "switch-xx-1" - ip: "10.1.1.1" - port: 8080 - labels { - key: "device_type" - value: "switch" - } - labels { - key: "cluster" - value: "xx" - } -} -resource { - name: "switch-xx-2" - ip: "10.1.1.2" - port: 8081 - labels { - key: "cluster" - value: "xx" - } -} -resource { - name: "switch-yy-1" - ip: "10.1.2.1" - port: 8080 -} -resource { - name: "switch-zz-1" - ip: "::aaa:1" - port: 8080 -} -` - -var testResourcesJSON = `{ - "resource": [ - { - "name": "switch-xx-1", - "ip": "10.1.1.1", - "port": 8080, - "labels": { - "device_type": "switch", - "cluster": "xx" - } - }, - { - "name": "switch-xx-2", - "ip": "10.1.1.2", - "port": 8081, - "labels": { - "cluster": "xx" - } - }, - { - "name": "switch-yy-1", - "ip": "10.1.2.1", - "port": 8080 - }, - { - "name": "switch-zz-1", - "ip": "::aaa:1", - "port": 8080 - } - ] -} -` - -var testResourcesFile = map[string]string{ - "textpb": testResourcesTextpb, - "json": testResourcesJSON, -} - var testExpectedEndpoints = []endpoint.Endpoint{ { Name: "switch-xx-1", @@ -110,24 +50,6 @@ var testExpectedEndpoints = []endpoint.Endpoint{ }, } -var testExpectedEndpointsWithFilter = []endpoint.Endpoint{ - { - Name: "switch-xx-1", - Port: 8080, - Labels: map[string]string{ - "device_type": "switch", - "cluster": "xx", - }, - }, - { - Name: "switch-xx-2", - Port: 8081, - Labels: map[string]string{ - "cluster": "xx", - }, - }, -} - var testExpectedIP = map[string]string{ "switch-xx-1": "10.1.1.1", "switch-xx-2": "10.1.1.2", @@ -135,86 +57,59 @@ var testExpectedIP = map[string]string{ "switch-zz-1": "::aaa:1", } -func TestFileTargets(t *testing.T) { - for _, filetype := range []string{"textpb", "json"} { - t.Run("Test "+filetype, func(t *testing.T) { - testFileTargetsForType(t, filetype) - }) - } -} - -func createTestFile(t *testing.T, fileType string) string { - t.Helper() - - if fileType == "" { - fileType = "textpb" - } - - tempFile, err := ioutil.TempFile("", "file_targets_*."+fileType) - if err != nil { - t.Fatal(err) - } - - if err := ioutil.WriteFile(tempFile.Name(), []byte(testResourcesFile[fileType]), 0644); err != nil { - t.Fatal(err) - } - - return tempFile.Name() -} - -func testFileTargetsForType(t *testing.T, fileType string) { - t.Helper() - testFile := createTestFile(t, fileType) - - ft, err := New(&configpb.TargetsConf{FilePath: proto.String(testFile)}, nil, nil) - if err != nil { - t.Fatalf("Unexpected error while parsing textpb: %v", err) - } - - t.Log(ft.names) - - got := ft.ListEndpoints() - - if !reflect.DeepEqual(got, testExpectedEndpoints) { - t.Errorf("ft.ListEndpoints: got: %v, expected: %v", got, testExpectedEndpoints) - } - - for name, ip := range testExpectedIP { - resolvedIP, err := ft.Resolve(name, 0) - if err != nil { - t.Errorf("unexpected error while resolving %s: %v", name, err) - } - got := resolvedIP.String() - if got != ip { - t.Errorf("ft.Resolve(%s): got=%s, expected=%s", name, got, ip) - } - } -} - func TestListEndpointsWithFilter(t *testing.T) { - t.Helper() - - testFile := createTestFile(t, "") - - ft, err := New(&configpb.TargetsConf{ - FilePath: proto.String(testFile), - Filter: []*rdspb.Filter{ - { + for _, test := range []struct { + desc string + f []*rdspb.Filter + wantEndpoints []endpoint.Endpoint + }{ + { + desc: "no_filter", + wantEndpoints: testExpectedEndpoints, + }, + { + desc: "with_filter", + f: []*rdspb.Filter{{ Key: proto.String("labels.cluster"), Value: proto.String("xx"), - }, + }}, + wantEndpoints: testExpectedEndpoints[:2], }, - }, nil, nil) + } { + t.Run(test.desc, func(t *testing.T) { + ft, err := New(&configpb.TargetsConf{ + FilePath: proto.String("../../rds/file/testdata/targets.json"), + Filter: test.f, + }, nil, nil) + + if err != nil { + t.Fatalf("Unexpected error while parsing textpb: %v", err) + } - if err != nil { - t.Fatalf("Unexpected error while parsing textpb: %v", err) - } + got := ft.ListEndpoints() - t.Log(ft.names) + if len(got) != len(test.wantEndpoints) { + t.Fatalf("Got endpoints: %d, expected: %d", len(got), len(test.wantEndpoints)) + } + for i := range test.wantEndpoints { + want := test.wantEndpoints[i] - got := ft.ListEndpoints() + if got[i].Name != want.Name || got[i].Port != want.Port || !reflect.DeepEqual(got[i].Labels, want.Labels) { + t.Errorf("ListResources: got:\n%v\nexpected:\n%v", got[i], want) + } + } - if !reflect.DeepEqual(got, testExpectedEndpointsWithFilter) { - t.Errorf("ft.ListEndpoints: got: %v, expected: %v", got, testExpectedEndpointsWithFilter) + for _, ep := range got { + resolvedIP, err := ft.Resolve(ep.Name, 0) + if err != nil { + t.Errorf("unexpected error while resolving %s: %v", ep.Name, err) + } + ip := resolvedIP.String() + if ip != testExpectedIP[ep.Name] { + t.Errorf("ft.Resolve(%s): got=%s, expected=%s", ep.Name, ip, testExpectedIP[ep.Name]) + } + } + }) } + } diff --git a/targets/file/proto/config.pb.go b/targets/file/proto/config.pb.go index e5a1eaff..14c174c1 100644 --- a/targets/file/proto/config.pb.go +++ b/targets/file/proto/config.pb.go @@ -9,6 +9,7 @@ package proto import ( + proto1 "github.com/google/cloudprober/rds/file/proto" proto "github.com/google/cloudprober/rds/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -23,65 +24,6 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -type TargetsConf_Format int32 - -const ( - TargetsConf_UNSPECIFIED TargetsConf_Format = 0 // Determine format using file extension/ - TargetsConf_TEXTPB TargetsConf_Format = 1 // Text proto format (.textpb). - TargetsConf_JSON TargetsConf_Format = 2 // JSON proto format (.json). -) - -// Enum value maps for TargetsConf_Format. -var ( - TargetsConf_Format_name = map[int32]string{ - 0: "UNSPECIFIED", - 1: "TEXTPB", - 2: "JSON", - } - TargetsConf_Format_value = map[string]int32{ - "UNSPECIFIED": 0, - "TEXTPB": 1, - "JSON": 2, - } -) - -func (x TargetsConf_Format) Enum() *TargetsConf_Format { - p := new(TargetsConf_Format) - *p = x - return p -} - -func (x TargetsConf_Format) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (TargetsConf_Format) Descriptor() protoreflect.EnumDescriptor { - return file_github_com_google_cloudprober_targets_file_proto_config_proto_enumTypes[0].Descriptor() -} - -func (TargetsConf_Format) Type() protoreflect.EnumType { - return &file_github_com_google_cloudprober_targets_file_proto_config_proto_enumTypes[0] -} - -func (x TargetsConf_Format) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Do not use. -func (x *TargetsConf_Format) UnmarshalJSON(b []byte) error { - num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) - if err != nil { - return err - } - *x = TargetsConf_Format(num) - return nil -} - -// Deprecated: Use TargetsConf_Format.Descriptor instead. -func (TargetsConf_Format) EnumDescriptor() ([]byte, []int) { - return file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDescGZIP(), []int{0, 0} -} - type TargetsConf struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -104,9 +46,9 @@ type TargetsConf struct { // ip: "10.16.110.12" // port: 8080 // } - FilePath *string `protobuf:"bytes,1,opt,name=file_path,json=filePath" json:"file_path,omitempty"` - Filter []*proto.Filter `protobuf:"bytes,2,rep,name=filter" json:"filter,omitempty"` - Format *TargetsConf_Format `protobuf:"varint,3,opt,name=format,enum=cloudprober.targets.file.TargetsConf_Format" json:"format,omitempty"` + FilePath *string `protobuf:"bytes,1,opt,name=file_path,json=filePath" json:"file_path,omitempty"` + Filter []*proto.Filter `protobuf:"bytes,2,rep,name=filter" json:"filter,omitempty"` + Format *proto1.ProviderConfig_Format `protobuf:"varint,3,opt,name=format,enum=cloudprober.rds.file.ProviderConfig_Format" json:"format,omitempty"` // If specified, file will be re-read at the given interval. ReEvalSec *int32 `protobuf:"varint,4,opt,name=re_eval_sec,json=reEvalSec" json:"re_eval_sec,omitempty"` } @@ -157,11 +99,11 @@ func (x *TargetsConf) GetFilter() []*proto.Filter { return nil } -func (x *TargetsConf) GetFormat() TargetsConf_Format { +func (x *TargetsConf) GetFormat() proto1.ProviderConfig_Format { if x != nil && x.Format != nil { return *x.Format } - return TargetsConf_UNSPECIFIED + return proto1.ProviderConfig_UNSPECIFIED } func (x *TargetsConf) GetReEvalSec() int32 { @@ -171,53 +113,6 @@ func (x *TargetsConf) GetReEvalSec() int32 { return 0 } -type FileResources struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Resource []*proto.Resource `protobuf:"bytes,1,rep,name=resource" json:"resource,omitempty"` -} - -func (x *FileResources) Reset() { - *x = FileResources{} - if protoimpl.UnsafeEnabled { - mi := &file_github_com_google_cloudprober_targets_file_proto_config_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *FileResources) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*FileResources) ProtoMessage() {} - -func (x *FileResources) ProtoReflect() protoreflect.Message { - mi := &file_github_com_google_cloudprober_targets_file_proto_config_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use FileResources.ProtoReflect.Descriptor instead. -func (*FileResources) Descriptor() ([]byte, []int) { - return file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDescGZIP(), []int{1} -} - -func (x *FileResources) GetResource() []*proto.Resource { - if x != nil { - return x.Resource - } - return nil -} - var File_github_com_google_cloudprober_targets_file_proto_config_proto protoreflect.FileDescriptor var file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDesc = []byte{ @@ -226,33 +121,29 @@ var file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDesc = 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x18, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x74, 0x61, 0x72, - 0x67, 0x65, 0x74, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x1a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, + 0x67, 0x65, 0x74, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x1a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2f, 0x72, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf2, 0x01, 0x0a, - 0x0b, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x1b, 0x0a, 0x09, - 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2f, 0x0a, 0x06, 0x66, 0x69, 0x6c, - 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6c, 0x6f, 0x75, - 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x74, - 0x65, 0x72, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x06, 0x66, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x63, 0x6c, 0x6f, - 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, - 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, - 0x66, 0x2e, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, - 0x12, 0x1e, 0x0a, 0x0b, 0x72, 0x65, 0x5f, 0x65, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x72, 0x65, 0x45, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, - 0x22, 0x2f, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x4e, - 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x54, - 0x45, 0x58, 0x54, 0x50, 0x42, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x4a, 0x53, 0x4f, 0x4e, 0x10, - 0x02, 0x22, 0x46, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x73, 0x12, 0x35, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, - 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x08, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, - 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x73, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x66, 0x69, 0x6c, + 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x31, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, + 0x62, 0x65, 0x72, 0x2f, 0x72, 0x64, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x64, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc0, 0x01, 0x0a, 0x0b, 0x54, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, + 0x50, 0x61, 0x74, 0x68, 0x12, 0x2f, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, + 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x43, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, + 0x62, 0x65, 0x72, 0x2e, 0x72, 0x64, 0x73, 0x2e, 0x66, 0x69, 0x6c, 0x65, 0x2e, 0x50, 0x72, 0x6f, + 0x76, 0x69, 0x64, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x46, 0x6f, 0x72, 0x6d, + 0x61, 0x74, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x1e, 0x0a, 0x0b, 0x72, 0x65, + 0x5f, 0x65, 0x76, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x09, 0x72, 0x65, 0x45, 0x76, 0x61, 0x6c, 0x53, 0x65, 0x63, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x72, 0x2f, 0x74, 0x61, 0x72, 0x67, + 0x65, 0x74, 0x73, 0x2f, 0x66, 0x69, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, } var ( @@ -267,24 +158,20 @@ func file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDescG return file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDescData } -var file_github_com_google_cloudprober_targets_file_proto_config_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_github_com_google_cloudprober_targets_file_proto_config_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_github_com_google_cloudprober_targets_file_proto_config_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_github_com_google_cloudprober_targets_file_proto_config_proto_goTypes = []interface{}{ - (TargetsConf_Format)(0), // 0: cloudprober.targets.file.TargetsConf.Format - (*TargetsConf)(nil), // 1: cloudprober.targets.file.TargetsConf - (*FileResources)(nil), // 2: cloudprober.targets.file.FileResources - (*proto.Filter)(nil), // 3: cloudprober.rds.Filter - (*proto.Resource)(nil), // 4: cloudprober.rds.Resource + (*TargetsConf)(nil), // 0: cloudprober.targets.file.TargetsConf + (*proto.Filter)(nil), // 1: cloudprober.rds.Filter + (proto1.ProviderConfig_Format)(0), // 2: cloudprober.rds.file.ProviderConfig.Format } var file_github_com_google_cloudprober_targets_file_proto_config_proto_depIdxs = []int32{ - 3, // 0: cloudprober.targets.file.TargetsConf.filter:type_name -> cloudprober.rds.Filter - 0, // 1: cloudprober.targets.file.TargetsConf.format:type_name -> cloudprober.targets.file.TargetsConf.Format - 4, // 2: cloudprober.targets.file.FileResources.resource:type_name -> cloudprober.rds.Resource - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 1, // 0: cloudprober.targets.file.TargetsConf.filter:type_name -> cloudprober.rds.Filter + 2, // 1: cloudprober.targets.file.TargetsConf.format:type_name -> cloudprober.rds.file.ProviderConfig.Format + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_github_com_google_cloudprober_targets_file_proto_config_proto_init() } @@ -305,32 +192,19 @@ func file_github_com_google_cloudprober_targets_file_proto_config_proto_init() { return nil } } - file_github_com_google_cloudprober_targets_file_proto_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*FileResources); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_github_com_google_cloudprober_targets_file_proto_config_proto_rawDesc, - NumEnums: 1, - NumMessages: 2, + NumEnums: 0, + NumMessages: 1, NumExtensions: 0, NumServices: 0, }, GoTypes: file_github_com_google_cloudprober_targets_file_proto_config_proto_goTypes, DependencyIndexes: file_github_com_google_cloudprober_targets_file_proto_config_proto_depIdxs, - EnumInfos: file_github_com_google_cloudprober_targets_file_proto_config_proto_enumTypes, MessageInfos: file_github_com_google_cloudprober_targets_file_proto_config_proto_msgTypes, }.Build() File_github_com_google_cloudprober_targets_file_proto_config_proto = out.File diff --git a/targets/file/proto/config.proto b/targets/file/proto/config.proto index d1e9ef51..f22532c9 100644 --- a/targets/file/proto/config.proto +++ b/targets/file/proto/config.proto @@ -3,6 +3,7 @@ syntax = "proto2"; package cloudprober.targets.file; +import "github.com/google/cloudprober/rds/file/proto/config.proto"; import "github.com/google/cloudprober/rds/proto/rds.proto"; option go_package = "github.com/google/cloudprober/targets/file/proto"; @@ -27,19 +28,10 @@ message TargetsConf { // } optional string file_path = 1; - repeated rds.Filter filter = 2; + repeated .cloudprober.rds.Filter filter = 2; - enum Format { - UNSPECIFIED = 0; // Determine format using file extension/ - TEXTPB = 1; // Text proto format (.textpb). - JSON = 2; // JSON proto format (.json). - } - optional Format format = 3; + optional .cloudprober.rds.file.ProviderConfig.Format format = 3; // If specified, file will be re-read at the given interval. optional int32 re_eval_sec = 4; } - -message FileResources { - repeated rds.Resource resource = 1; -}