Merge branch 'master' into overlord-prevalidation

README.md

@@ -405,7 +405,7 @@ $ pebble run --verbose
 ...
 ```
 
-#### Log forwarding
+### Log forwarding
 
 Pebble supports forwarding its services' logs to a remote Loki server. In the `log-targets` section of the plan, you can specify destinations for log forwarding, for example:
 ```yaml
@@ -422,6 +422,8 @@ log-targets:
         services: [svc1, svc2]
 ```
 
+#### Specifying services
+
 For each log target, use the `services` key to specify a list of services to collect logs from. In the above example, the `production-logs` target will collect logs from `svc1` and `svc2`.
 
 Use the special keyword `all` to match all services, including services that might be added in future layers. In the above example, `staging-logs` will collect logs from all services.
@@ -453,6 +455,38 @@ my-target:
 ```
 would remove all services and then add `svc1`, so `my-target` would receive logs from only `svc1`.
 
+#### Labels
+
+In the `labels` section, you can specify custom labels to be added to any outgoing logs. These labels may contain `$ENVIRONMENT_VARIABLES` - these will be interpreted in the environment of the corresponding service. Pebble may also add its own default labels (depending on the protocol). For example, given the following plan:
+```yaml
+services:
+  svc1:
+    environment:
+      OWNER: 'alice'
+  svc2:
+    environment:
+      OWNER: 'bob'
+
+log-targets:
+  tgt1:
+    type: loki
+    labels:
+      product: 'juju'
+      owner: 'user-$OWNER'
+```
+the logs from `svc1` will be sent with the following labels:
+```yaml
+product: juju
+owner: user-alice     # env var $OWNER substituted
+pebble_service: svc1  # default label for Loki
+```
+and for svc2, the labels will be
+```yaml
+product: juju
+owner: user-bob       # env var $OWNER substituted
+pebble_service: svc2  # default label for Loki
+```
+
 
 ## Container usage
 
@@ -733,8 +767,10 @@ log-targets:
     override: merge | replace
 
     # (Required) The type of log target, which determines the format in
-    # which logs will be sent. Currently, the only supported type is 'loki',
-    # but more protocols may be added in the future.
+    # which logs will be sent. The supported types are:
+    #
+    # - loki: Use the Grafana Loki protocol. A "pebble_service" label is
+    #   added automatically, with the name of the Pebble service as its value.
     type: loki
 
     # (Required) The URL of the remote log target.
@@ -749,6 +785,12 @@ log-targets:
     # service name with a minus (e.g. '-svc1') to remove a previously added
     # service. '-all' will remove all services.
     services: [<service names>]
+
+    # (Optional) A list of key/value pairs defining labels which should be set
+    # on the outgoing logs. The label values may contain $ENV_VARS, which will
+    # be substituted using the environment for the corresponding service.
+    labels:
+      <label name>: <label value>
 ```
 
 ## API and clients

client/files.go

@@ -16,11 +16,16 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
+	"io"
+	"mime/multipart"
+	"net/textproto"
 	"net/url"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -364,3 +369,146 @@ func (client *Client) RemovePath(opts *RemovePathOptions) error {
 
 	return nil
 }
+
+type PushOptions struct {
+	// Source is the source of data to write (required).
+	Source io.Reader
+
+	// Path indicates the absolute path of the file in the destination
+	// machine (required).
+	Path string
+
+	// MakeDirs, if true, will create any non-existing directories in the path
+	// to the remote file. If false (the default) the call to Push will
+	// fail if any of the parent directories of path do not exist.
+	MakeDirs bool
+
+	// Permissions indicates the mode of the file on the destination machine.
+	// If 0 or unset, defaults to 0644. Note that, when used together with MakeDirs,
+	// the directories that are created will not use this mode, but 0755.
+	Permissions os.FileMode
+
+	// UserID indicates the user ID of the owner for the file on the destination
+	// machine. When used together with MakeDirs, the directories that are
+	// created will also be owned by this user.
+	UserID *int
+
+	// User indicates the name of the owner user for the file on the destination
+	// machine. When used together with MakeDirs, the directories that are
+	// created will also be owned by this user.
+	User string
+
+	// GroupID indicates the ID of the owner group for the file on the destination
+	// machine. When used together with MakeDirs, the directories that are
+	// created will also be owned by this user.
+	GroupID *int
+
+	// Group indicates the name of the owner group for the file on the
+	// machine. When used together with MakeDirs, the directories that are
+	// created will also be owned by this user.
+	Group string
+}
+
+type writeFilesPayload struct {
+	Action string           `json:"action"`
+	Files  []writeFilesItem `json:"files"`
+}
+
+type writeFilesItem struct {
+	Path        string `json:"path"`
+	MakeDirs    bool   `json:"make-dirs"`
+	Permissions string `json:"permissions"`
+	UserID      *int   `json:"user-id"`
+	User        string `json:"user"`
+	GroupID     *int   `json:"group-id"`
+	Group       string `json:"group"`
+}
+
+var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
+
+func escapeQuotes(s string) string {
+	return quoteEscaper.Replace(s)
+}
+
+// Push writes content to a path on the remote system.
+func (client *Client) Push(opts *PushOptions) error {
+	var permissions string
+	if opts.Permissions != 0 {
+		permissions = fmt.Sprintf("%03o", opts.Permissions)
+	}
+
+	payload := writeFilesPayload{
+		Action: "write",
+		Files: []writeFilesItem{{
+			Path:        opts.Path,
+			MakeDirs:    opts.MakeDirs,
+			Permissions: permissions,
+			UserID:      opts.UserID,
+			User:        opts.User,
+			GroupID:     opts.GroupID,
+			Group:       opts.Group,
+		}},
+	}
+
+	var body bytes.Buffer
+	mw := multipart.NewWriter(&body)
+
+	// Encode metadata part of the header
+	part, err := mw.CreatePart(textproto.MIMEHeader{
+		"Content-Type":        {"application/json"},
+		"Content-Disposition": {`form-data; name="request"`},
+	})
+	if err != nil {
+		return fmt.Errorf("cannot encode metadata in request payload: %w", err)
+	}
+
+	// Buffer for multipart header/footer
+	if err := json.NewEncoder(part).Encode(&payload); err != nil {
+		return err
+	}
+
+	// Encode file part of the header
+	escapedPath := escapeQuotes(opts.Path)
+	_, err = mw.CreatePart(textproto.MIMEHeader{
+		"Content-Type":        {"application/octet-stream"},
+		"Content-Disposition": {fmt.Sprintf(`form-data; name="files"; filename="%s"`, escapedPath)},
+	})
+	if err != nil {
+		return fmt.Errorf("cannot encode file in request payload: %w", err)
+	}
+
+	header := body.String()
+
+	// Encode multipart footer
+	body.Reset()
+	mw.Close()
+	footer := body.String()
+
+	resp, err := client.Requester().Do(context.Background(), &RequestOptions{
+		Type:    SyncRequest,
+		Method:  "POST",
+		Path:    "/v1/files",
+		Headers: map[string]string{"Content-Type": mw.FormDataContentType()},
+		Body:    io.MultiReader(strings.NewReader(header), opts.Source, strings.NewReader(footer)),
+	})
+	if err != nil {
+		return err
+	}
+
+	var result []fileResult
+	if err = resp.DecodeResult(&result); err != nil {
+		return err
+	}
+	if len(result) != 1 {
+		return fmt.Errorf("expected exactly one result from API, got %d", len(result))
+	}
+	if result[0].Error != nil {
+		return &Error{
+			Kind:    result[0].Error.Kind,
+			Value:   result[0].Error.Value,
+			Message: result[0].Error.Message,
+		}
+	}
+
+	return nil
+}

client/files_test.go

@@ -15,8 +15,12 @@
 package client_test
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
 	"os"
+	"path"
+	"strings"
 	"time"
 
 	. "gopkg.in/check.v1"
@@ -574,3 +578,130 @@ func (cs *clientSuite) TestRemovePathFailsWithMultipleAPIResults(c *C) {
 		}},
 	})
 }
+
+type writeFilesPayload struct {
+	Action string           `json:"action"`
+	Files  []writeFilesItem `json:"files"`
+}
+
+type writeFilesItem struct {
+	Path        string `json:"path"`
+	MakeDirs    bool   `json:"make-dirs"`
+	Permissions string `json:"permissions"`
+	UserID      *int   `json:"user-id"`
+	User        string `json:"user"`
+	GroupID     *int   `json:"group-id"`
+	Group       string `json:"group"`
+}
+
+func (cs *clientSuite) TestPush(c *C) {
+	cs.rsp = `{"type": "sync", "result": [{"path": "/file.dat"}]}`
+
+	err := cs.cli.Push(&client.PushOptions{
+		Path:   "/file.dat",
+		Source: strings.NewReader("Hello, world!"),
+	})
+	c.Assert(err, IsNil)
+	mr, err := cs.req.MultipartReader()
+	c.Assert(err, IsNil)
+
+	c.Assert(cs.req.URL.Path, Equals, "/v1/files")
+	c.Assert(cs.req.Method, Equals, "POST")
+
+	// Check metadata part
+	metadata, err := mr.NextPart()
+	c.Assert(err, IsNil)
+	c.Assert(metadata.Header.Get("Content-Type"), Equals, "application/json")
+	c.Assert(metadata.FormName(), Equals, "request")
+
+	buf := bytes.NewBuffer(make([]byte, 0))
+	_, err = buf.ReadFrom(metadata)
+	c.Assert(err, IsNil)
+
+	// Decode metadata
+	var payload writeFilesPayload
+	err = json.NewDecoder(buf).Decode(&payload)
+	c.Assert(err, IsNil)
+	c.Assert(payload, DeepEquals, writeFilesPayload{
+		Action: "write",
+		Files: []writeFilesItem{{
+			Path: "/file.dat",
+		}},
+	})
+
+	// Check file part
+	file, err := mr.NextPart()
+	c.Assert(err, IsNil)
+	c.Assert(file.Header.Get("Content-Type"), Equals, "application/octet-stream")
+	c.Assert(file.FormName(), Equals, "files")
+	c.Assert(path.Base(file.FileName()), Equals, "file.dat")
+
+	buf.Reset()
+	_, err = buf.ReadFrom(file)
+	c.Assert(err, IsNil)
+	c.Assert(buf.String(), Equals, "Hello, world!")
+
+	// Check end of multipart request
+	_, err = mr.NextPart()
+	c.Assert(err, Equals, io.EOF)
+}
+
+func (cs *clientSuite) TestPushFails(c *C) {
+	cs.rsp = `{"type": "error", "result": {"message": "could not foo"}}`
+
+	err := cs.cli.Push(&client.PushOptions{
+		Path:   "/file.dat",
+		Source: strings.NewReader("Hello, world!"),
+	})
+	c.Assert(err, ErrorMatches, "could not foo")
+}
+
+func (cs *clientSuite) TestPushFailsOnFile(c *C) {
+	cs.rsp = `{
+		"type": "sync",
+		"result": [{
+			"path": "/file.dat",
+			"error": {
+				"message": "could not bar",
+				"kind": "permission-denied",
+				"value": 42
+			}
+		}]
+	}`
+
+	err := cs.cli.Push(&client.PushOptions{
+		Path:   "/file.dat",
+		Source: strings.NewReader("Hello, world!"),
+	})
+	clientErr, ok := err.(*client.Error)
+	c.Assert(ok, Equals, true)
+	c.Assert(clientErr.Message, Equals, "could not bar")
+	c.Assert(clientErr.Kind, Equals, "permission-denied")
+}
+
+func (cs *clientSuite) TestPushFailsWithMultipleAPIResults(c *C) {
+	cs.rsp = `{
+		"type": "sync",
+		"result": [{
+			"path": "/file.dat",
+			"error": {
+				"message": "could not bar",
+				"kind": "permission-denied",
+				"value": 42
+			}
+		}, {
+			"path": "/file.dat",
+			"error": {
+				"message": "could not baz",
+				"kind": "generic-file-error",
+				"value": 41
+			}
+		}]
+	}`
+
+	err := cs.cli.Push(&client.PushOptions{
+		Path:   "/file.dat",
+		Source: strings.NewReader("Hello, world!"),
+	})
+	c.Assert(err, ErrorMatches, "expected exactly one result from API, got 2")
+}

internals/cli/cmd_help.go

@@ -192,7 +192,7 @@ var HelpCategories = []HelpCategory{{
 }, {
 	Label:       "Files",
 	Description: "work with files and execute commands",
-	Commands:    []string{"ls", "mkdir", "rm", "exec"},
+	Commands:    []string{"push", "ls", "mkdir", "rm", "exec"},
 }, {
 	Label:       "Changes",
 	Description: "manage changes and their tasks",