From 86251b2bd6731736e63b183b37bc203c2030d4cf Mon Sep 17 00:00:00 2001 From: Manu Garg Date: Fri, 3 Jan 2020 17:07:20 -0800 Subject: [PATCH] Add a common "file" module with support for reading files from GCS. File module provides an interface "ReadFile" that either reads the file from the disk using ioutil, or, if its path starts with "gs://", retrieves it from the GCS over HTTP. Use this new interface to read various files like OAuth token, JSON key, TLS cert, and even the config file. PiperOrigin-RevId: 288072059 --- cmd/cloudprober.go | 4 +-- common/file/file.go | 68 +++++++++++++++++++++++++++++++++++ common/file/file_test.go | 68 +++++++++++++++++++++++++++++++++++ common/oauth/bearer.go | 4 +-- common/oauth/oauth.go | 4 +-- common/tlsconfig/tlsconfig.go | 15 ++++++-- 6 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 common/file/file.go create mode 100644 common/file/file_test.go diff --git a/cmd/cloudprober.go b/cmd/cloudprober.go index d84317f6..e41dc917 100644 --- a/cmd/cloudprober.go +++ b/cmd/cloudprober.go @@ -23,7 +23,6 @@ package main import ( "context" "fmt" - "io/ioutil" _ "net/http/pprof" "os" "os/signal" @@ -33,6 +32,7 @@ import ( "flag" "github.com/golang/glog" "github.com/google/cloudprober" + "github.com/google/cloudprober/common/file" "github.com/google/cloudprober/config" "github.com/google/cloudprober/config/runconfig" "github.com/google/cloudprober/sysvars" @@ -112,7 +112,7 @@ func setupProfiling() { } func configFileToString(fileName string) string { - b, err := ioutil.ReadFile(fileName) + b, err := file.ReadFile(fileName) if err != nil { glog.Exitf("Failed to read the config file: %v", err) } diff --git a/common/file/file.go b/common/file/file.go new file mode 100644 index 00000000..7b727d64 --- /dev/null +++ b/common/file/file.go @@ -0,0 +1,68 @@ +// Copyright 2020 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package file implements utilities to read files from various backends. +*/ +package file + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "golang.org/x/oauth2/google" +) + +type readFunc func(path string) ([]byte, error) + +var prefixToReadfunc = map[string]readFunc{ + "gs://": readFileFromGCS, +} + +func readFileFromGCS(objectPath string) ([]byte, error) { + hc, err := google.DefaultClient(context.Background()) + if err != nil { + return nil, err + } + + objURL := "https://storage.googleapis.com/" + objectPath + res, err := hc.Get(objURL) + + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("got error while retrieving GCS object, http status: %s, status code: %d", res.Status, res.StatusCode) + } + + defer res.Body.Close() + return ioutil.ReadAll(res.Body) +} + +// ReadFile returns file contents as a slice of bytes. It's similar to ioutil's +// ReadFile, but includes support for files on non-disk locations. For example, +// files with paths starting with gs:// are assumed to be on GCS, and are read +// from GCS. +func ReadFile(fname string) ([]byte, error) { + for prefix, f := range prefixToReadfunc { + if strings.HasPrefix(fname, prefix) { + return f(fname[len(prefix):]) + } + } + return ioutil.ReadFile(fname) +} diff --git a/common/file/file_test.go b/common/file/file_test.go new file mode 100644 index 00000000..aceb3f2e --- /dev/null +++ b/common/file/file_test.go @@ -0,0 +1,68 @@ +// Copyright 2020 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package file + +import ( + "io/ioutil" + "testing" +) + +func createTempFile(t *testing.T, b []byte) string { + tmpfile, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + return "" + } + + defer tmpfile.Close() + if _, err := tmpfile.Write(b); err != nil { + t.Fatal(err) + } + + return tmpfile.Name() +} + +func testReadFile(path string) ([]byte, error) { + return []byte("content-for-" + path), nil +} + +func TestReadFile(t *testing.T) { + prefixToReadfunc["test://"] = testReadFile + + // Virtual file + testPath := "test://test-file" + + // Disk file + tempContent := "temp-content" + tempPath := createTempFile(t, []byte(tempContent)) + + testData := map[string]string{ + testPath: "content-for-test-file", + tempPath: tempContent, + } + + for path, expectedContent := range testData { + t.Run("ReadFile("+path+")", func(t *testing.T) { + b, err := ReadFile(path) + if err != nil { + t.Fatalf("Error while reading the file: %s", path) + } + + if string(b) != expectedContent { + t.Errorf("ReadFile(%s) = %s, expected=%s", path, string(b), expectedContent) + } + }) + } +} diff --git a/common/oauth/bearer.go b/common/oauth/bearer.go index 572ffd21..c575d340 100644 --- a/common/oauth/bearer.go +++ b/common/oauth/bearer.go @@ -16,12 +16,12 @@ package oauth import ( "fmt" - "io/ioutil" "os/exec" "strings" "sync" "time" + "github.com/google/cloudprober/common/file" configpb "github.com/google/cloudprober/common/oauth/proto" "github.com/google/cloudprober/logger" "golang.org/x/oauth2" @@ -37,7 +37,7 @@ type bearerTokenSource struct { } var getTokenFromFile = func(c *configpb.BearerToken) (string, error) { - b, err := ioutil.ReadFile(c.GetFile()) + b, err := file.ReadFile(c.GetFile()) if err != nil { return "", err } diff --git a/common/oauth/oauth.go b/common/oauth/oauth.go index 5d9e9747..40886897 100644 --- a/common/oauth/oauth.go +++ b/common/oauth/oauth.go @@ -20,8 +20,8 @@ package oauth import ( "context" "fmt" - "io/ioutil" + "github.com/google/cloudprober/common/file" configpb "github.com/google/cloudprober/common/oauth/proto" "github.com/google/cloudprober/logger" "golang.org/x/oauth2" @@ -47,7 +47,7 @@ func TokenSourceFromConfig(c *configpb.Config, l *logger.Logger) (oauth2.TokenSo return creds.TokenSource, nil } - jsonKey, err := ioutil.ReadFile(f) + jsonKey, err := file.ReadFile(f) if err != nil { return nil, fmt.Errorf("error reading Google Credentials file (%s): %v", f, err) } diff --git a/common/tlsconfig/tlsconfig.go b/common/tlsconfig/tlsconfig.go index b4bd3735..0a37f4fc 100644 --- a/common/tlsconfig/tlsconfig.go +++ b/common/tlsconfig/tlsconfig.go @@ -19,8 +19,8 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" + "github.com/google/cloudprober/common/file" configpb "github.com/google/cloudprober/common/tlsconfig/proto" ) @@ -31,7 +31,7 @@ func UpdateTLSConfig(tlsConfig *tls.Config, c *configpb.TLSConfig, addClientCACe } if c.GetCaCertFile() != "" { - caCert, err := ioutil.ReadFile(c.GetCaCertFile()) + caCert, err := file.ReadFile(c.GetCaCertFile()) if err != nil { return err } @@ -48,7 +48,16 @@ func UpdateTLSConfig(tlsConfig *tls.Config, c *configpb.TLSConfig, addClientCACe } if c.GetTlsCertFile() != "" { - cert, err := tls.LoadX509KeyPair(c.GetTlsCertFile(), c.GetTlsKeyFile()) + certPEMBlock, err := file.ReadFile(c.GetTlsCertFile()) + if err != nil { + return err + } + keyPEMBlock, err := file.ReadFile(c.GetTlsKeyFile()) + if err != nil { + return err + } + + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) if err != nil { return err }