Add a common "file" module with support for reading files from GCS.

cmd/cloudprober.go

@@ -23,7 +23,6 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	_ "net/http/pprof"
 	"os"
 	"os/signal"
@@ -33,6 +32,7 @@ import (
 	"flag"
 	"github.com/golang/glog"
 	"github.com/google/cloudprober"
+	"github.com/google/cloudprober/common/file"
 	"github.com/google/cloudprober/config"
 	"github.com/google/cloudprober/config/runconfig"
 	"github.com/google/cloudprober/sysvars"
@@ -112,7 +112,7 @@ func setupProfiling() {
 }
 
 func configFileToString(fileName string) string {
-	b, err := ioutil.ReadFile(fileName)
+	b, err := file.ReadFile(fileName)
 	if err != nil {
 		glog.Exitf("Failed to read the config file: %v", err)
 	}

common/file/file.go

@@ -0,0 +1,68 @@
+// Copyright 2020 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+Package file implements utilities to read files from various backends.
+*/
+package file
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"golang.org/x/oauth2/google"
+)
+
+type readFunc func(path string) ([]byte, error)
+
+var prefixToReadfunc = map[string]readFunc{
+	"gs://": readFileFromGCS,
+}
+
+func readFileFromGCS(objectPath string) ([]byte, error) {
+	hc, err := google.DefaultClient(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	objURL := "https://storage.googleapis.com/" + objectPath
+	res, err := hc.Get(objURL)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("got error while retrieving GCS object, http status: %s, status code: %d", res.Status, res.StatusCode)
+	}
+
+	defer res.Body.Close()
+	return ioutil.ReadAll(res.Body)
+}
+
+// ReadFile returns file contents as a slice of bytes. It's similar to ioutil's
+// ReadFile, but includes support for files on non-disk locations. For example,
+// files with paths starting with gs:// are assumed to be on GCS, and are read
+// from GCS.
+func ReadFile(fname string) ([]byte, error) {
+	for prefix, f := range prefixToReadfunc {
+		if strings.HasPrefix(fname, prefix) {
+			return f(fname[len(prefix):])
+		}
+	}
+	return ioutil.ReadFile(fname)
+}

common/file/file_test.go

@@ -0,0 +1,68 @@
+// Copyright 2020 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package file
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func createTempFile(t *testing.T, b []byte) string {
+	tmpfile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+		return ""
+	}
+
+	defer tmpfile.Close()
+	if _, err := tmpfile.Write(b); err != nil {
+		t.Fatal(err)
+	}
+
+	return tmpfile.Name()
+}
+
+func testReadFile(path string) ([]byte, error) {
+	return []byte("content-for-" + path), nil
+}
+
+func TestReadFile(t *testing.T) {
+	prefixToReadfunc["test://"] = testReadFile
+
+	// Virtual file
+	testPath := "test://test-file"
+
+	// Disk file
+	tempContent := "temp-content"
+	tempPath := createTempFile(t, []byte(tempContent))
+
+	testData := map[string]string{
+		testPath: "content-for-test-file",
+		tempPath: tempContent,
+	}
+
+	for path, expectedContent := range testData {
+		t.Run("ReadFile("+path+")", func(t *testing.T) {
+			b, err := ReadFile(path)
+			if err != nil {
+				t.Fatalf("Error while reading the file: %s", path)
+			}
+
+			if string(b) != expectedContent {
+				t.Errorf("ReadFile(%s) = %s, expected=%s", path, string(b), expectedContent)
+			}
+		})
+	}
+}

common/oauth/bearer.go

@@ -16,12 +16,12 @@ package oauth
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os/exec"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/google/cloudprober/common/file"
 	configpb "github.com/google/cloudprober/common/oauth/proto"
 	"github.com/google/cloudprober/logger"
 	"golang.org/x/oauth2"
@@ -37,7 +37,7 @@ type bearerTokenSource struct {
 }
 
 var getTokenFromFile = func(c *configpb.BearerToken) (string, error) {
-	b, err := ioutil.ReadFile(c.GetFile())
+	b, err := file.ReadFile(c.GetFile())
 	if err != nil {
 		return "", err
 	}

common/oauth/oauth.go

@@ -20,8 +20,8 @@ package oauth
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 
+	"github.com/google/cloudprober/common/file"
 	configpb "github.com/google/cloudprober/common/oauth/proto"
 	"github.com/google/cloudprober/logger"
 	"golang.org/x/oauth2"
@@ -47,7 +47,7 @@ func TokenSourceFromConfig(c *configpb.Config, l *logger.Logger) (oauth2.TokenSo
 			return creds.TokenSource, nil
 		}
 
-		jsonKey, err := ioutil.ReadFile(f)
+		jsonKey, err := file.ReadFile(f)
 		if err != nil {
 			return nil, fmt.Errorf("error reading Google Credentials file (%s): %v", f, err)
 		}

common/tlsconfig/tlsconfig.go

@@ -19,8 +19,8 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 
+	"github.com/google/cloudprober/common/file"
 	configpb "github.com/google/cloudprober/common/tlsconfig/proto"
 )
 
@@ -31,7 +31,7 @@ func UpdateTLSConfig(tlsConfig *tls.Config, c *configpb.TLSConfig, addClientCACe
 	}
 
 	if c.GetCaCertFile() != "" {
-		caCert, err := ioutil.ReadFile(c.GetCaCertFile())
+		caCert, err := file.ReadFile(c.GetCaCertFile())
 		if err != nil {
 			return err
 		}
@@ -48,7 +48,16 @@ func UpdateTLSConfig(tlsConfig *tls.Config, c *configpb.TLSConfig, addClientCACe
 	}
 
 	if c.GetTlsCertFile() != "" {
-		cert, err := tls.LoadX509KeyPair(c.GetTlsCertFile(), c.GetTlsKeyFile())
+		certPEMBlock, err := file.ReadFile(c.GetTlsCertFile())
+		if err != nil {
+			return err
+		}
+		keyPEMBlock, err := file.ReadFile(c.GetTlsKeyFile())
+		if err != nil {
+			return err
+		}
+
+		cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
 		if err != nil {
 			return err
 		}