pubsys: add SSM parameter publishing

`cargo make ssm` will use the amis.json from `cargo make ami` to populate
parameter name/value templates from files in `policies/ssm`.  The parameters
are set in SSM using the full build version, e.g. "0.5.0-abcdef", and it won't
overwrite by default.

`cargo make promote-ssm -e SSM_TARGET=VERSION` will promote (copy) those values
from the full-build-version name to a more general one.  It's recommended to
promote from the full version to a short version like "0.5.0", then to a
well-known pointer like "latest".  This allows for easy rollback by promoting
from an older version, using `-e SSM_SOURCE=OLD_VERSION`.

Co-authored-by: Zac Mrowicki <[email protected]>
Co-authored-by: Tom Kirchner <[email protected]>

Makefile.toml

@@ -37,9 +37,17 @@ PUBLISH_KEY = "default"
 # AMIs.  (You can also specify PUBLISH_ROOT_VOLUME_SIZE to override the root
 # volume size; by default it's the image size, rounded up.)
 PUBLISH_DATA_VOLUME_SIZE = "20"
-# You can also set PUBLISH_REGIONS to override the list of regions for AMIs
-# from Infra.toml; it's a comma-separated list like "us-west-2,us-east-1".
-# You can also set NO_PROGRESS to not print progress bars during snapshot upload.
+# This can be overridden with -e to change the path to the directory containing
+# SSM parameter template files.  These files determine the parameter names and
+# values that will be published to SSM when you run `cargo make ssm`.
+PUBLISH_SSM_TEMPLATES_PATH = "${BUILDSYS_ROOT_DIR}/tools/pubsys/policies/ssm"
+
+# You can also set PUBLISH_REGIONS to override the list of regions from
+# Infra.toml for AMI and SSM commands; it's a comma-separated list like
+# "us-west-2,us-east-1".
+# You can set NO_PROGRESS=true to not print progress bars during snapshot upload.
+# You can use ALLOW_CLOBBER=true with the `ssm` task to make it overwrite existing values.
+# (This is not required with `promote-ssm` because the intent of promotion is overwriting.)
 
 # Disallow pulling directly Upstream URLs when lookaside cache results in MISSes as a fallback.
 # To use the upstream source as fallback, override this on the command line and set it to 'true'
@@ -472,6 +480,71 @@ pubsys \
 '''
 ]
 
+[tasks.ssm]
+# Rather than depend on "build", which currently rebuilds images each run, we
+# depend on publish-tools and check for the input file below to save time.
+# This does mean that `cargo make ami` must be run before `cargo make ssm`.
+dependencies = ["publish-tools"]
+script_runner = "bash"
+script = [
+'''
+set -e
+
+export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}"
+
+ami_input="${BUILDSYS_OUTPUT_DIR}/${BUILDSYS_NAME_FULL}-amis.json"
+if [ ! -s "${ami_input}" ]; then
+   echo "AMI input file doesn't exist for the current version/commit - ${BUILDSYS_VERSION_FULL} - please run 'cargo make ami'" >&2
+   exit 1
+fi
+
+pubsys \
+   --infra-config-path "${PUBLISH_INFRA_CONFIG_PATH}" \
+   \
+   ssm \
+   \
+   --ami-input "${ami_input}" \
+   --arch "${BUILDSYS_ARCH}" \
+   --variant "${BUILDSYS_VARIANT}" \
+   --version "${BUILDSYS_VERSION_FULL}" \
+   --template-dir "${PUBLISH_SSM_TEMPLATES_PATH}" \
+   \
+   ${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"} \
+   ${ALLOW_CLOBBER:+--allow-clobber}
+'''
+]
+
+[tasks.promote-ssm]
+dependencies = ["publish-tools"]
+script_runner = "bash"
+script = [
+'''
+set -e
+
+export PATH="${BUILDSYS_TOOLS_DIR}/bin:${PATH}"
+
+source="${SSM_SOURCE:-${BUILDSYS_VERSION_FULL}}"
+target="${SSM_TARGET}"
+if [ -z "${target}" ]; then
+   echo "SSM_TARGET is mandatory for promote-ssm; please give the version (or pointer like "latest") to which you want to promote ${source}" >&2
+   exit 1
+fi
+
+pubsys \
+   --infra-config-path "${PUBLISH_INFRA_CONFIG_PATH}" \
+   \
+   promote-ssm \
+   \
+   --arch "${BUILDSYS_ARCH}" \
+   --variant "${BUILDSYS_VARIANT}" \
+   --source "${source}" \
+   --target "${target}" \
+   --template-dir "${PUBLISH_SSM_TEMPLATES_PATH}" \
+   \
+   ${PUBLISH_REGIONS:+--regions "${PUBLISH_REGIONS}"}
+'''
+]
+
 [tasks.clean]
 script_runner = "bash"
 script = [

tools/pubsys/Cargo.toml

@@ -23,14 +23,16 @@ rusoto_credential = "0.45.0"
 rusoto_ebs = "0.45.0"
 rusoto_ec2 = "0.45.0"
 rusoto_signature = "0.45.0"
+rusoto_ssm = "0.45.0"
 rusoto_sts = "0.45.0"
 simplelog = "0.8"
 snafu = "0.6"
 semver = "0.10.0"
 serde = { version = "1.0", features = ["derive"]  }
 serde_json = "1.0"
 structopt = { version = "0.3", default-features = false  }
-tokio = "0.2.21"
+tinytemplate = "1.1"
+tokio = { version = "0.2.21", features = ["time"] }
 toml = "0.5"
 tough = { version = "0.8", features = ["http"] }
 tough-ssm = "0.3"

tools/pubsys/Infra.toml.example

@@ -38,6 +38,8 @@ regions = ["us-west-2", "us-east-1", "us-east-2"]
 profile = "my-profile"
 # If specified, we assume this role before making any API calls.
 role = "arn:aws:iam::012345678901:role/assume-global"
+# If specified, this string will be prefixed on all parameter names published to SSM.
+ssm_prefix = "/your/prefix/here"
 
 [aws.region.us-west-2]
 # If specified, we assume this role before making any API calls in this region.

tools/pubsys/policies/ssm/README.md

@@ -0,0 +1,30 @@
+# Parameter templates
+
+Files in this directory contain template strings that are used to generate SSM parameter names and values.
+You can pass a different directory to `pubsys` to use a different set of parameters.
+
+The directory is expected to contain a file named `defaults.toml` with a table entry per parameter, like this:
+
+```
+[[parameter]]
+name = "{variant}/{arch}/{image_version}/image_id"
+value = "{image_id}"
+```
+
+The `name` and `value` can contain template variables that will be replaced with information from the current build and from the AMI registered from that build.
+
+The available variables include:
+* `variant`, for example "aws-k8s-1.17"
+* `arch`, for example "x86_64"
+* `image_id`, for example "ami-0123456789abcdef0"
+* `image_name`, for example "bottlerocket-aws-k8s-1.17-x86_64-v0.5.0-e0ddf1b"
+* `image_version`, for example "0.5.0-e0ddf1b"
+* `region`, for example "us-west-2"
+
+# Overrides
+
+You can also add or override parameters that are specific to `variant` or `arch`.
+To do so, create a directory named "variant" or "arch" inside parameters directory, and create a file named after the specific variant or arch for which you want overrides.
+
+For example, to add extra parameters just for the "aarch64" architecture, create `arch/aarch64.toml`.
+Inside you can put the same types of `[[parameter]]` declarations that you see in `defaults.toml`, but they'll only be applied for `aarch64` builds.

tools/pubsys/policies/ssm/defaults.toml

@@ -0,0 +1,7 @@
+[[parameter]]
+name = "{variant}/{arch}/{image_version}/image_id"
+value = "{image_id}"
+
+[[parameter]]
+name = "{variant}/{arch}/{image_version}/image_version"
+value = "{image_version}"

tools/pubsys/src/aws/ami/mod.rs

@@ -15,6 +15,7 @@ use register::{get_ami_id, register_image};
 use rusoto_core::{Region, RusotoError};
 use rusoto_ebs::EbsClient;
 use rusoto_ec2::{CopyImageError, CopyImageRequest, CopyImageResult, Ec2, Ec2Client};
+use serde::{Deserialize, Serialize};
 use snafu::{ensure, OptionExt, ResultExt};
 use std::collections::{HashMap, VecDeque};
 use std::fs::File;
@@ -70,11 +71,11 @@ pub(crate) struct AmiArgs {
 /// Common entrypoint from main()
 pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
     match _run(args, ami_args).await {
-        Ok(ami_ids) => {
+        Ok(amis) => {
             // Write the AMI IDs to file if requested
             if let Some(ref path) = ami_args.ami_output {
                 let file = File::create(path).context(error::FileCreate { path })?;
-                serde_json::to_writer_pretty(file, &ami_ids).context(error::Serialize { path })?;
+                serde_json::to_writer_pretty(file, &amis).context(error::Serialize { path })?;
                 info!("Wrote AMI data to {}", path.display());
             }
             Ok(())
@@ -83,8 +84,8 @@ pub(crate) async fn run(args: &Args, ami_args: &AmiArgs) -> Result<()> {
     }
 }
 
-async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>> {
-    let mut ami_ids = HashMap::new();
+async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, Image>> {
+    let mut amis = HashMap::new();
 
     info!(
         "Using infra config from path: {}",
@@ -160,11 +161,14 @@ async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>
         (new_id, false)
     };
 
-    ami_ids.insert(base_region.name().to_string(), image_id.clone());
+    amis.insert(
+        base_region.name().to_string(),
+        Image::new(&image_id, &ami_args.name),
+    );
 
     // If we don't need to copy AMIs, we're done.
     if regions.is_empty() {
-        return Ok(ami_ids);
+        return Ok(amis);
     }
 
     // Wait for AMI to be available so it can be copied
@@ -212,7 +216,7 @@ async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>
                 region.name(),
                 id
             );
-            ami_ids.insert(region.name().to_string(), id.clone());
+            amis.insert(region.name().to_string(), Image::new(&id, &ami_args.name));
             continue;
         }
         let request = CopyImageRequest {
@@ -240,7 +244,7 @@ async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>
 
     // If all target regions already have the AMI, we're done.
     if copy_requests.is_empty() {
-        return Ok(ami_ids);
+        return Ok(amis);
     }
 
     // Start requests; they return almost immediately and the copying work is done by the service
@@ -268,7 +272,10 @@ async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>
                         region.name(),
                         image_id,
                     );
-                    ami_ids.insert(region.name().to_string(), image_id);
+                    amis.insert(
+                        region.name().to_string(),
+                        Image::new(&image_id, &ami_args.name),
+                    );
                 } else {
                     saw_error = true;
                     error!(
@@ -287,7 +294,25 @@ async fn _run(args: &Args, ami_args: &AmiArgs) -> Result<HashMap<String, String>
 
     ensure!(!saw_error, error::AmiCopy);
 
-    Ok(ami_ids)
+    Ok(amis)
+}
+
+/// If JSON output was requested, we serialize out a mapping of region to AMI information; this
+/// struct holds the information we save about each AMI.  The `ssm` subcommand uses this
+/// information to populate templates representing SSM parameter names and values.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub(crate) struct Image {
+    pub(crate) id: String,
+    pub(crate) name: String,
+}
+
+impl Image {
+    fn new(id: &str, name: &str) -> Self {
+        Self {
+            id: id.to_string(),
+            name: name.to_string(),
+        }
+    }
 }
 
 mod error {

tools/pubsys/src/aws/client.rs

@@ -7,6 +7,7 @@ use rusoto_credential::{
 };
 use rusoto_ebs::EbsClient;
 use rusoto_ec2::Ec2Client;
+use rusoto_ssm::SsmClient;
 use rusoto_sts::{StsAssumeRoleSessionCredentialsProvider, StsClient};
 use snafu::ResultExt;
 
@@ -37,6 +38,16 @@ impl NewWith for Ec2Client {
     }
 }
 
+impl NewWith for SsmClient {
+    fn new_with<P, D>(request_dispatcher: D, credentials_provider: P, region: Region) -> Self
+    where
+        P: ProvideAwsCredentials + Send + Sync + 'static,
+        D: DispatchSignedRequest + Send + Sync + 'static,
+    {
+        Self::new_with(request_dispatcher, credentials_provider, region)
+    }
+}
+
 /// Create a rusoto client of the given type using the given region and configuration.
 pub(crate) fn build_client<T: NewWith>(
     region: &Region,

tools/pubsys/src/aws/mod.rs

@@ -6,7 +6,9 @@ use snafu::ResultExt;
 pub(crate) mod client;
 
 pub(crate) mod ami;
+pub(crate) mod promote_ssm;
 pub(crate) mod publish_ami;
+pub(crate) mod ssm;
 
 /// Builds a Region from the given region name, and uses the custom endpoint from the AWS config,
 /// if specified in aws.region.REGION.endpoint.