diff --git a/packages/kernel-5.10/1003-af_unix-increase-default-max_dgram_qlen-to-512.patch b/packages/kernel-5.10/1003-af_unix-increase-default-max_dgram_qlen-to-512.patch new file mode 100644 index 00000000000..9363ddadcab --- /dev/null +++ b/packages/kernel-5.10/1003-af_unix-increase-default-max_dgram_qlen-to-512.patch @@ -0,0 +1,47 @@ +From b3983ebbfa2dc231a2b61092b0a936bd25294239 Mon Sep 17 00:00:00 2001 +From: Markus Boehme +Date: Tue, 23 May 2023 21:24:38 +0000 +Subject: [PATCH] af_unix: increase default max_dgram_qlen to 512 + +The net.unix.max_dgram_qlen sysctl has been defined with a default value of +10 since before the current Git history started in 2005. Systems have more +resources these days, and while the default values for other sysctls like +net.core.somaxconn have been adapted, max_dgram_qlen never was. + +Increase the default value for max_dgram_qlen to 512. A large number of +hosts effectively already run with this or a larger value, since systemd +has been making sure it is set to at least 512 since 2015. + +Signed-off-by: Markus Boehme +--- + Documentation/networking/ip-sysctl.rst | 2 +- + net/unix/af_unix.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst +index 252212998..164a65667 100644 +--- a/Documentation/networking/ip-sysctl.rst ++++ b/Documentation/networking/ip-sysctl.rst +@@ -2688,5 +2688,5 @@ addr_scope_policy - INTEGER + max_dgram_qlen - INTEGER + The maximum length of dgram socket receive queue + +- Default: 10 ++ Default: 512 + +diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c +index 28721e957..a5f081ad8 100644 +--- a/net/unix/af_unix.c ++++ b/net/unix/af_unix.c +@@ -2948,7 +2948,7 @@ static int __net_init unix_net_init(struct net *net) + { + int error = -ENOMEM; + +- net->unx.sysctl_max_dgram_qlen = 10; ++ net->unx.sysctl_max_dgram_qlen = 512; + if (unix_sysctl_register(net)) + goto out; + +-- +2.39.2 + diff --git a/packages/kernel-5.10/kernel-5.10.spec b/packages/kernel-5.10/kernel-5.10.spec index 1e3299c4bbd..e3dda0c57cd 100644 --- a/packages/kernel-5.10/kernel-5.10.spec +++ b/packages/kernel-5.10/kernel-5.10.spec @@ -17,6 +17,8 @@ Source103: config-bottlerocket-vmware Patch1001: 1001-Makefile-add-prepare-target-for-external-modules.patch # Enable INITRAMFS_FORCE config option for our use case. Patch1002: 1002-initramfs-unlink-INITRAMFS_FORCE-from-CMDLINE_-EXTEN.patch +# Increase default of sysctl net.unix.max_dgram_qlen to 512. +Patch1003: 1003-af_unix-increase-default-max_dgram_qlen-to-512.patch # Add zstd support for compressed kernel modules Patch2000: 2000-kbuild-move-module-strip-compression-code-into-scrip.patch diff --git a/packages/kernel-5.15/1004-af_unix-increase-default-max_dgram_qlen-to-512.patch b/packages/kernel-5.15/1004-af_unix-increase-default-max_dgram_qlen-to-512.patch new file mode 100644 index 00000000000..888ebbd3628 --- /dev/null +++ b/packages/kernel-5.15/1004-af_unix-increase-default-max_dgram_qlen-to-512.patch @@ -0,0 +1,47 @@ +From e36140bfb2795377360bb92c343b10c717567c62 Mon Sep 17 00:00:00 2001 +From: Markus Boehme +Date: Tue, 23 May 2023 17:16:44 +0000 +Subject: [PATCH] af_unix: increase default max_dgram_qlen to 512 + +The net.unix.max_dgram_qlen sysctl has been defined with a default value of +10 since before the current Git history started in 2005. Systems have more +resources these days, and while the default values for other sysctls like +net.core.somaxconn have been adapted, max_dgram_qlen never was. + +Increase the default value for max_dgram_qlen to 512. A large number of +hosts effectively already run with this or a larger value, since systemd +has been making sure it is set to at least 512 since 2015. + +Signed-off-by: Markus Boehme +--- + Documentation/networking/ip-sysctl.rst | 2 +- + net/unix/af_unix.c | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst +index 7890b395e..54a0be396 100644 +--- a/Documentation/networking/ip-sysctl.rst ++++ b/Documentation/networking/ip-sysctl.rst +@@ -2885,5 +2885,5 @@ plpmtud_probe_interval - INTEGER + max_dgram_qlen - INTEGER + The maximum length of dgram socket receive queue + +- Default: 10 ++ Default: 512 + +diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c +index a96026dbd..267ee6d29 100644 +--- a/net/unix/af_unix.c ++++ b/net/unix/af_unix.c +@@ -3343,7 +3343,7 @@ static int __net_init unix_net_init(struct net *net) + { + int error = -ENOMEM; + +- net->unx.sysctl_max_dgram_qlen = 10; ++ net->unx.sysctl_max_dgram_qlen = 512; + if (unix_sysctl_register(net)) + goto out; + +-- +2.39.2 + diff --git a/packages/kernel-5.15/kernel-5.15.spec b/packages/kernel-5.15/kernel-5.15.spec index ab5b6eda9e2..d9d733e1f63 100644 --- a/packages/kernel-5.15/kernel-5.15.spec +++ b/packages/kernel-5.15/kernel-5.15.spec @@ -19,6 +19,8 @@ Patch1001: 1001-Makefile-add-prepare-target-for-external-modules.patch Patch1002: 1002-Revert-kbuild-hide-tools-build-targets-from-external.patch # Enable INITRAMFS_FORCE config option for our use case. Patch1003: 1003-initramfs-unlink-INITRAMFS_FORCE-from-CMDLINE_-EXTEN.patch +# Increase default of sysctl net.unix.max_dgram_qlen to 512. +Patch1004: 1004-af_unix-increase-default-max_dgram_qlen-to-512.patch # Backport from v5.15.111 upstream, drop when Amazon Linux base is v5.15.111 or later Patch5001: 5001-netfilter-nf_tables-deactivate-anonymous-set-from-pr.patch diff --git a/sources/host-ctr/cmd/host-ctr/main.go b/sources/host-ctr/cmd/host-ctr/main.go index 5ab2cafe4fd..7c53a3ce5f4 100644 --- a/sources/host-ctr/cmd/host-ctr/main.go +++ b/sources/host-ctr/cmd/host-ctr/main.go @@ -575,16 +575,84 @@ func cleanUp(containerdSocket string, namespace string, containerID string) erro return nil } +// parseImageURISpecialRegions mimics the parsing in ecr.ParseImageURI but +// constructs the canonical ECR references while skipping certain checks. +// We only do this for special regions that are not yet supported by the aws-go-sdk. +// Referenced source: https://github.com/awslabs/amazon-ecr-containerd-resolver/blob/a5058cf091f4fc573813a032db37a9820952f1f9/ecr/ref.go#L70-L71 +func parseImageURISpecialRegions(input string) (ecr.ECRSpec, error) { + ecrRefPrefixMapping := map[string]string{ + "il-central-1": "ecr.aws/arn:aws:ecr:il-central-1:", + } + // Matching on account, region + matches := ecrRegex.FindStringSubmatch(input) + if len(matches) < 3 { + return ecr.ECRSpec{}, fmt.Errorf("invalid image URI: %s", input) + } + account := matches[1] + region := matches[2] + + // Need to include the full repository path and the imageID (e.g. /eks/image-name:tag) + tokens := strings.SplitN(input, "/", 2) + if len(tokens) != 2 { + return ecr.ECRSpec{}, fmt.Errorf("invalid image URI: %s", input) + } + fullRepoPath := tokens[len(tokens)-1] + // Run simple checks on the provided repository. + switch { + case + // Must not be empty + fullRepoPath == "", + // Must not have a partial/unsupplied label + strings.HasSuffix(fullRepoPath, ":"), + // Must not have a partial/unsupplied digest specifier + strings.HasSuffix(fullRepoPath, "@"): + return ecr.ECRSpec{}, errors.New("incomplete reference provided") + } + + // Get the ECR image reference prefix from the AWS region + ecrRefPrefix, ok := ecrRefPrefixMapping[region] + if !ok { + return ecr.ECRSpec{}, fmt.Errorf("%s: %s", "invalid region in internal mapping", region) + } + + return ecr.ParseRef(fmt.Sprintf("%s%s:repository/%s", ecrRefPrefix, account, fullRepoPath)) +} + +// fetchECRRef attempts to resolve the ECR reference from an input source string +// by first using the aws-sdk-go's ParseImageURI function. This will fail for +// special regions that are not yet supported. If it fails for any reason, +// attempt to parse again using parseImageURISpecialRegions in this package. +// This uses a special region reference to build the ECR image references. +// If both fail, an error is returned. +func fetchECRRef(ctx context.Context, input string) (ecr.ECRSpec, error) { + var spec ecr.ECRSpec + spec, err := ecr.ParseImageURI(input) + if err == nil { + return spec, nil + } + log.G(ctx).WithError(err).WithField("source", input).Warn("failed to parse ECR reference") + + // The parsing might fail if the AWS region is special, parse again with special handling: + spec, err = parseImageURISpecialRegions(input) + if err == nil { + return spec, nil + } + + // Return the error for the parseImageURISpecialRegions from this package + // if a valid ECR ref has not yet been returned + log.G(ctx).WithError(err).WithField("source", input).Error("failed to parse special ECR reference") + return ecr.ECRSpec{}, errors.Wrap(err, "could not parse ECR reference for special regions") + +} + // fetchECRImage does some additional conversions before resolving the image reference and fetches the image. func fetchECRImage(ctx context.Context, source string, client *containerd.Client, registryConfigPath string, fetchCachedImageIfExist bool) (containerd.Image, error) { - ref := source - ecrRef, err := ecr.ParseImageURI(ref) + ecrRef, err := fetchECRRef(ctx, source) if err != nil { - log.G(ctx).WithError(err).WithField("source", source).Error("failed to parse ECR reference") return nil, err } + ref := ecrRef.Canonical() - ref = ecrRef.Canonical() log.G(ctx). WithField("ref", ref). WithField("source", source). diff --git a/sources/host-ctr/cmd/host-ctr/main_test.go b/sources/host-ctr/cmd/host-ctr/main_test.go index c36b4c95eda..7d4ed06d8f1 100644 --- a/sources/host-ctr/cmd/host-ctr/main_test.go +++ b/sources/host-ctr/cmd/host-ctr/main_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "testing" "github.com/containerd/containerd/remotes/docker" @@ -160,3 +161,64 @@ func TestBadRegistryHosts(t *testing.T) { _, err := f("docker.io") assert.Error(t, err) } + +func TestFetchECRRef(t *testing.T) { + tests := []struct { + name string + ecrImgURI string + expectedErr bool + expectedRef string + }{ + { + "Parse typical region for normal use-cases", + "111111111111.dkr.ecr.us-west-2.amazonaws.com/bottlerocket/container:1.2.3", + false, + "ecr.aws/arn:aws:ecr:us-west-2:111111111111:repository/bottlerocket/container:1.2.3", + }, + { + "Parse special region", + "111111111111.dkr.ecr.il-central-1.amazonaws.com/bottlerocket/container:1.2.3", + false, + "ecr.aws/arn:aws:ecr:il-central-1:111111111111:repository/bottlerocket/container:1.2.3", + }, + { + "Parse China regions", + "111111111111.dkr.ecr.cn-north-1.amazonaws.com/bottlerocket/container:1.2.3", + false, + "ecr.aws/arn:aws-cn:ecr:cn-north-1:111111111111:repository/bottlerocket/container:1.2.3", + }, + { + "Parse gov regions", + "111111111111.dkr.ecr.us-gov-west-1.amazonaws.com/bottlerocket/container:1.2.3", + false, + "ecr.aws/arn:aws-us-gov:ecr:us-gov-west-1:111111111111:repository/bottlerocket/container:1.2.3", + }, + { + "Fail for invalid region", + "111111111111.dkr.ecr.outer-space.amazonaws.com/bottlerocket/container:1.2.3", + true, + "", + }, + { + "Empty string fails", + "", + true, + "", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result, err := fetchECRRef(context.TODO(), tc.ecrImgURI) + if tc.expectedErr { + // handle error cases + if err == nil { + t.Fail() + } + } else { + // handle happy paths + assert.Equal(t, tc.expectedRef, result.Canonical()) + } + }) + } +} diff --git a/tools/pubsys/src/aws/promote_ssm/mod.rs b/tools/pubsys/src/aws/promote_ssm/mod.rs index bf54cf98e3e..84c240f4f13 100644 --- a/tools/pubsys/src/aws/promote_ssm/mod.rs +++ b/tools/pubsys/src/aws/promote_ssm/mod.rs @@ -211,8 +211,7 @@ pub(crate) async fn run(args: &Args, promote_args: &PromoteArgs) -> Result<()> { // write the newly promoted parameters to `ssm_parameter_output` along with the original // parameters if let Some(ssm_parameter_output) = &promote_args.ssm_parameter_output { - append_rendered_parameters(ssm_parameter_output, &set_parameters, source_target_map) - .await?; + append_rendered_parameters(ssm_parameter_output, &set_parameters).await?; } // SSM set =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= @@ -236,7 +235,6 @@ pub(crate) async fn run(args: &Args, promote_args: &PromoteArgs) -> Result<()> { async fn append_rendered_parameters( ssm_parameters_output: &PathBuf, set_parameters: &HashMap, - source_target_map: HashMap<&String, &String>, ) -> Result<()> { // If the file doesn't exist, assume that there are no existing parameters let parsed_parameters = parse_parameters(&ssm_parameters_output.to_owned()) @@ -251,10 +249,15 @@ async fn append_rendered_parameters( }) .context(error::ParseExistingSsmParametersSnafu { path: ssm_parameters_output, - })?; + })? + // SsmKey contains region information, so we can lose the top-level region. + .into_values() + .fold(HashMap::new(), |mut acc, params| { + acc.extend(params); + acc + }); - let combined_parameters: HashMap> = - combine_parameters(parsed_parameters, set_parameters, source_target_map); + let combined_parameters = merge_parameters(parsed_parameters, set_parameters); write_rendered_parameters( ssm_parameters_output, @@ -270,37 +273,28 @@ async fn append_rendered_parameters( /// Return a HashMap of Region mapped to a HashMap of SsmKey, String pairs, representing the newly /// promoted parameters as well as the original parameters. In case of a parameter collision, /// the parameter takes the promoted value. -fn combine_parameters( - source_parameters: HashMap>, +fn merge_parameters( + source_parameters: HashMap, set_parameters: &HashMap, - source_target_map: HashMap<&String, &String>, ) -> HashMap> { - let mut combined_parameters: HashMap> = HashMap::new(); + let mut combined_parameters = HashMap::new(); + + // Flatten parameters into tuples to simplify processing elements. + fn flatten(parameter: (SsmKey, String)) -> (Region, SsmKey, String) { + let (key, value) = parameter; + (key.region.clone(), key, value) + } source_parameters - .iter() - .flat_map(|(region, parameters)| { - parameters - .iter() - .map(move |(ssm_key, ssm_value)| (region, ssm_key, ssm_value)) - }) + .into_iter() + .map(flatten) + // Process the `set_parameters` second so that they overwrite existing values. + .chain(set_parameters.clone().into_iter().map(flatten)) .for_each(|(region, ssm_key, ssm_value)| { - let add_parameters = vec![ - (ssm_key.clone(), ssm_value.clone()), - ( - SsmKey::new(region.clone(), source_target_map[&ssm_key.name].to_string()), - set_parameters[&SsmKey::new( - region.clone(), - source_target_map[&ssm_key.name].to_string(), - )] - .clone(), - ), - ]; - combined_parameters - .entry(region.clone()) + .entry(region) .or_insert(HashMap::new()) - .extend(add_parameters); + .insert(ssm_key, ssm_value); }); combined_parameters @@ -388,31 +382,30 @@ type Result = std::result::Result; mod test { use std::collections::HashMap; - use crate::aws::{promote_ssm::combine_parameters, ssm::SsmKey}; + use crate::aws::{promote_ssm::merge_parameters, ssm::SsmKey}; use aws_sdk_ssm::Region; #[test] fn combined_parameters() { let existing_parameters = HashMap::from([ ( - Region::new("us-west-2"), - HashMap::from([ - ( - SsmKey::new(Region::new("us-west-2"), "test1-parameter-name".to_string()), - "test1-parameter-value".to_string(), - ), - ( - SsmKey::new(Region::new("us-west-2"), "test2-parameter-name".to_string()), - "test2-parameter-value".to_string(), - ), - ]), + SsmKey::new(Region::new("us-west-2"), "test1-parameter-name".to_string()), + "test1-parameter-value".to_string(), ), ( - Region::new("us-east-1"), - HashMap::from([( - SsmKey::new(Region::new("us-east-1"), "test3-parameter-name".to_string()), - "test3-parameter-value".to_string(), - )]), + SsmKey::new(Region::new("us-west-2"), "test2-parameter-name".to_string()), + "test2-parameter-value".to_string(), + ), + ( + SsmKey::new(Region::new("us-east-1"), "test3-parameter-name".to_string()), + "test3-parameter-value".to_string(), + ), + ( + SsmKey::new( + Region::new("us-east-1"), + "test4-unpromoted-parameter-name".to_string(), + ), + "test4-unpromoted-parameter-value".to_string(), ), ]); let set_parameters = HashMap::from([ @@ -438,18 +431,7 @@ mod test { "test3-parameter-value".to_string(), ), ]); - let test1_parameter_name = "test1-parameter-name".to_string(); - let test2_parameter_name = "test2-parameter-name".to_string(); - let test3_parameter_name = "test3-parameter-name".to_string(); - let test1_parameter_name_promoted = "test1-parameter-name-promoted".to_string(); - let test2_parameter_name_promoted = "test2-parameter-name-promoted".to_string(); - let test3_parameter_name_promoted = "test3-parameter-name-promoted".to_string(); - let source_target_map = HashMap::from([ - (&test1_parameter_name, &test1_parameter_name_promoted), - (&test2_parameter_name, &test2_parameter_name_promoted), - (&test3_parameter_name, &test3_parameter_name_promoted), - ]); - let map = combine_parameters(existing_parameters, &set_parameters, source_target_map); + let map = merge_parameters(existing_parameters, &set_parameters); let expected_map = HashMap::from([ ( Region::new("us-west-2"), @@ -492,6 +474,13 @@ mod test { ), "test3-parameter-value".to_string(), ), + ( + SsmKey::new( + Region::new("us-east-1"), + "test4-unpromoted-parameter-name".to_string(), + ), + "test4-unpromoted-parameter-value".to_string(), + ), ]), ), ]); @@ -502,24 +491,16 @@ mod test { fn combined_parameters_overwrite() { let existing_parameters = HashMap::from([ ( - Region::new("us-west-2"), - HashMap::from([ - ( - SsmKey::new(Region::new("us-west-2"), "test1-parameter-name".to_string()), - "test1-parameter-value".to_string(), - ), - ( - SsmKey::new(Region::new("us-west-2"), "test2-parameter-name".to_string()), - "test2-parameter-value".to_string(), - ), - ]), + SsmKey::new(Region::new("us-west-2"), "test1-parameter-name".to_string()), + "test1-parameter-value".to_string(), ), ( - Region::new("us-east-1"), - HashMap::from([( - SsmKey::new(Region::new("us-east-1"), "test3-parameter-name".to_string()), - "test3-parameter-value".to_string(), - )]), + SsmKey::new(Region::new("us-west-2"), "test2-parameter-name".to_string()), + "test2-parameter-value".to_string(), + ), + ( + SsmKey::new(Region::new("us-east-1"), "test3-parameter-name".to_string()), + "test3-parameter-value".to_string(), ), ]); let set_parameters = HashMap::from([ @@ -539,18 +520,7 @@ mod test { "test3-parameter-value".to_string(), ), ]); - let test1_parameter_name = "test1-parameter-name".to_string(); - let test2_parameter_name = "test2-parameter-name".to_string(); - let test3_parameter_name = "test3-parameter-name".to_string(); - let test1_parameter_name_promoted = "test1-parameter-name".to_string(); - let test2_parameter_name_promoted = "test2-parameter-name".to_string(); - let test3_parameter_name_promoted = "test3-parameter-name-promoted".to_string(); - let source_target_map = HashMap::from([ - (&test1_parameter_name, &test1_parameter_name_promoted), - (&test2_parameter_name, &test2_parameter_name_promoted), - (&test3_parameter_name, &test3_parameter_name_promoted), - ]); - let map = combine_parameters(existing_parameters, &set_parameters, source_target_map); + let map = merge_parameters(existing_parameters, &set_parameters); let expected_map = HashMap::from([ ( Region::new("us-west-2"),