merge changes from main

CMakeLists.txt

@@ -67,6 +67,7 @@ if (LEGACY_BUILD)
     option(USE_TLS_V1_2 "Set http client to enforce TLS 1.2" ON)
     option(USE_TLS_V1_3 "Set http client to enforce TLS 1.3" OFF)
     option(ENABLE_SMOKE_TESTS "Enable smoke tests" OFF)
+    option(ENABLE_PROTOCOL_TESTS "Enable protocol tests" OFF)
     option(DISABLE_DNS_REQUIRED_TESTS "Disable unit tests that require DNS lookup to succeed, useful when using a http client that does not perform DNS lookup" OFF)
 
 
@@ -267,7 +268,6 @@ if (LEGACY_BUILD)
     set_msvc_warnings()
 
     include(sdks)
-
     include(utilities)
 
     if (ENABLE_OPENSSL_ENCRYPTION)
@@ -338,6 +338,7 @@ if (LEGACY_BUILD)
     add_definitions("-DAWS_TEST_REGION=${AWS_TEST_REGION}")
 
     add_sdks()
+    include(tests)
 
     # for user friendly cmake usage
     include(setup_cmake_find_module)

VERSION

@@ -1 +1 @@
-1.11.491
+1.11.496

cmake/tests.cmake

@@ -0,0 +1,35 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0.
+#
+
+function(add_protocol_tests)
+    set(PROTOCOL_TESTS_LOG "")
+    # Add test clients, which are just like a regular SDK client, but must not be installed.
+    file(GLOB subdirs LIST_DIRECTORIES true "${CMAKE_SOURCE_DIR}/generated/protocol-tests/test-clients/*")
+    foreach(subdir ${subdirs})
+        if(EXISTS "${subdir}/CMakeLists.txt")
+            add_subdirectory(${subdir} EXCLUDE_FROM_ALL)
+        endif()
+    endforeach()
+
+    # Add tests
+    file(GLOB protoTestTypes LIST_DIRECTORIES true "${CMAKE_SOURCE_DIR}/generated/protocol-tests/tests/*")
+    foreach(protoTestType ${protoTestTypes})
+        file(GLOB subdirs LIST_DIRECTORIES true ${protoTestType}/*)
+        foreach(subdir ${subdirs})
+            if(EXISTS "${subdir}/CMakeLists.txt")
+                add_subdirectory(${subdir})
+
+                get_filename_component(testGroup ${protoTestType} NAME)
+                get_filename_component(testName ${subdir} NAME)
+                list(APPEND PROTOCOL_TESTS_LOG "${testGroup}/${testName}")
+            endif()
+        endforeach()
+    endforeach()
+
+    message(STATUS "Protocol tests: ${PROTOCOL_TESTS_LOG}")
+endfunction()
+
+if (ENABLE_PROTOCOL_TESTS)
+    add_protocol_tests()
+endif ()

docs/MD5ChecksumFallback.md

@@ -0,0 +1,198 @@
+# MD5 Checksum Fallback for the AWS C++ SDK
+
+Recently the SDKs shipped a feature in the SDK that [changed default object integrity](https://github.com/aws/aws-sdk-cpp/discussions/3252) in S3. What this more or less boils down to is that [S3 supports several different wire checksums](https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html) and we now default to use `CRC64-NVME` to ensure object integrity. Previously we used `MD5 `checksums to ensure object integrity. Some 3rd party S3 compatible services currently do not support this and need time to catch up, or alternatively will not support this. If you wish to fallback to the old behavior of sending MD5 checksums there are three different scenarios that will have have to cover
+
+## An API that has checksum when supported and you wish to send no checksum at all with the request
+
+Some APIs like [Put Object](https://github.com/aws/aws-sdk-cpp/blob/main/tools/code-generation/api-descriptions/s3-2006-03-01.normal.json#L1286) are marked as `"requestChecksumRequired":false` which means that a checksum is not required for this endpoint but the SDK will send a CRC64-NVME checksum by default as it is supported. On operations like this you can completely disable checksums.
+
+```c++
+#include <aws/core/Aws.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/PutObjectRequest.h>
+
+using namespace Aws;
+using namespace Aws::S3;
+using namespace Aws::S3::Model;
+
+namespace {
+  constexpr const char* LOG_TAG = "TestApplication";
+  constexpr const char* BUCKET_NAME = "BUCKET_NAME";
+  constexpr const char* KEY = "OBJECT_KEY";
+}
+
+auto main() -> int {
+  SDKOptions options;
+  options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
+  InitAPI(options);
+  {
+    S3ClientConfiguration configuration;
+    configuration.checksumConfig.requestChecksumCalculation =
+    Client::RequestChecksumCalculation::WHEN_REQUIRED;
+    S3Client client{configuration};
+    auto request = PutObjectRequest().WithBucket(BUCKET_NAME).WithKey(KEY);
+    request.SetBody(body);
+    std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(LOG_TAG,
+      "sample text stream");
+    const auto response = client.PutObject(request);
+    assert(response.IsSuccess());
+  }
+  ShutdownAPI(options);
+  return 0;
+}
+```
+
+the resulting wire log will look something like
+
+```log
+[DEBUG] 2025-01-29 16:11:11.280 CURL [0x2067ccf80] (HeaderOut) PUT /YOUR_KEY HTTP/1.1
+Host: YOUR_BUCKET.s3.us-east-1.amazonaws.com
+Accept: */*
+amz-sdk-invocation-id: invocation_uuid
+amz-sdk-request: attempt=1
+authorization: your_signature
+content-length: 18
+content-type: binary/octet-stream
+user-agent:your user agent
+x-amz-content-sha256: content_sha
+x-amz-date: data
+x-amz-security-token: security_token
+```
+
+Which will have no headers associated with checksums or any checksumming related information. This is not recommended because there is no object integrity checks, and data could be corrupted on the wire.
+
+## An API that has checksum when supported and you wish to send a MD5 but NOT a CRC64 checksum
+
+This is same scenario as the first case but instead of sending no checksum, we will be sending a content MD5 header alongside the request for object validation. This is the default behavior before the object integrity change.
+
+```c++
+#include <aws/core/Aws.h>
+#include <aws/core/utils/HashingUtils.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/PutObjectRequest.h>
+
+using namespace Aws;
+using namespace Aws::Utils;
+using namespace Aws::S3;
+using namespace Aws::S3::Model;
+
+namespace {
+  constexpr const char* LOG_TAG = "TestApplication";
+  constexpr const char* BUCKET_NAME = "BUCKET_NAME";
+  constexpr const char* KEY = "OBJECT_KEY";
+  constexpr const char* CONTENT_MD5_HEADER = "content-md5";
+}
+
+auto main() -> int {
+  SDKOptions options;
+  options.loggingOptions.logLevel = Logging::LogLevel::Debug;
+  InitAPI(options);
+  {
+    S3ClientConfiguration configuration;
+    configuration.checksumConfig.requestChecksumCalculation =
+      Client::RequestChecksumCalculation::WHEN_REQUIRED;
+    S3Client client{configuration};
+    auto request = PutObjectRequest().WithBucket(BUCKET_NAME).WithKey(KEY);
+    std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(LOG_TAG,
+      "sample text stream");
+    request.SetAdditionalCustomHeaderValue(CONTENT_MD5_HEADER,
+      HashingUtils::Base64Encode(HashingUtils::CalculateMD5(*body)));
+    request.SetBody(body);
+    const auto response = client.PutObject(request);
+    assert(response.IsSuccess());
+  }
+  ShutdownAPI(options);
+  return 0;
+}
+```
+
+this will result in a log that looks like
+
+```log
+DEBUG] 2025-01-29 16:31:01.666 CURL [0x2067ccf80] (HeaderOut) PUT /YOUR_KEY HTTP/1.1
+Host: YOUR_BUCKET.s3.us-east-1.amazonaws.com
+Accept: */*
+amz-sdk-invocation-id: invocation_uuid
+amz-sdk-request: attempt=1
+authorization: your_signature
+content-length: 18
+content-md5: rXaQ1aPgNd9/GVs6Fl3zuA==
+content-type: binary/octet-stream
+user-agent:your_user_agent
+x-amz-content-sha256: content_sha
+x-amz-date: data
+x-amz-security-token: security_token
+```
+
+Which will include a MD5 header alongside your request for object integrity. This preserves backwards compatibility.
+
+## An API that has checksum when required and you wish to send a MD5 but NOT  a CRC64 checksum
+
+Some APIs require checksums on requests, like [DeleteObjects](https://github.com/aws/aws-sdk-cpp/blob/main/tools/code-generation/api-descriptions/s3-2006-03-01.normal.json#L350). These APIs require a checksum to be sent alongside the api request. By default the SDK will send CRC64-NVME by default. To revert to the old behavior of only sending the MD5 header this will require overriding the method on the parent request to opt out of this.
+
+```c++
+#include <aws/core/Aws.h>
+#include <aws/core/utils/HashingUtils.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/DeleteObjectsRequest.h>
+
+using namespace Aws;
+using namespace Aws::Utils;
+using namespace Aws::S3;
+using namespace Aws::S3::Model;
+
+namespace {
+  constexpr const char* BUCKET_NAME = "BUCKET_NAME";
+  constexpr const char* KEY = "OBJECT_KEY";
+  constexpr const char* CONTENT_MD5_HEADER = "content-md5";
+}
+
+struct ChecksumOptOutDeleteObjects : public DeleteObjectsRequest {
+  inline bool RequestChecksumRequired() const override {
+    return false;
+  };
+};
+
+auto main() -> int {
+  SDKOptions options;
+  options.loggingOptions.logLevel = Logging::LogLevel::Debug;
+  InitAPI(options);
+  {
+    S3ClientConfiguration configuration;
+    configuration.checksumConfig.requestChecksumCalculation =
+    Client::RequestChecksumCalculation::WHEN_REQUIRED;
+    S3Client client{configuration};
+    auto request = ChecksumOptOutDeleteObjects();
+    request.SetBucket(BUCKET_NAME);
+    request.SetDelete(S3::Model::Delete().WithObjects({ObjectIdentifier()
+      .WithKey(KEY)}));
+    auto payload = request.SerializePayload();
+    request.SetAdditionalCustomHeaderValue(CONTENT_MD5_HEADER,
+      HashingUtils::Base64Encode(HashingUtils::CalculateMD5(payload)));
+    const auto response = client.DeleteObjects(request);
+    assert(response.IsSuccess());
+  }
+  ShutdownAPI(options);
+  return 0;
+}
+```
+
+This will override the need for the SDK to calculate a required checksum, allowing to skip the required checksum, and you can manually add a MD5 header for the serialized payload.
+
+The log should look something like
+
+```log
+[DEBUG] 2025-01-29 18:07:08.164 CURL [0x2067ccf80] (HeaderOut) POST /?delete HTTP/1.1
+Host: YOUR_BUCKET.s3.us-east-1.amazonaws.com
+Accept: */*
+amz-sdk-invocation-id: invocation_uuid
+amz-sdk-request: attempt=1
+authorization: your_signature
+content-length: 144
+content-md5: kJL3pJJmVThrDq352SNTrw==
+content-type: application/xml
+user-agent: aws-sdk-cpp/1.11.493 ua/2.1 api/S3 os/Darwin#23.6.0 lang/c++#C++11 md/aws-crt#0.19.7 md/arch#arm64 md/Clang#15.0.0 m/Duser-agent:your user agent
+x-amz-content-sha256: content_sha
+x-amz-date: date
+x-amz-security-token: security_token
+```

generated/protocol-tests/test-clients/aws-cpp-sdk-ec2-protocol/CMakeLists.txt

@@ -0,0 +1,76 @@
+add_project(aws-cpp-sdk-ec2-protocol "C++ SDK for the AWS ec2-protocol service" aws-cpp-sdk-core)
+
+file(GLOB AWS_EC2-PROTOCOL_HEADERS
+    "include/aws/ec2-protocol/*.h"
+)
+
+file(GLOB AWS_EC2-PROTOCOL_MODEL_HEADERS
+    "include/aws/ec2-protocol/model/*.h"
+)
+
+file(GLOB AWS_EC2-PROTOCOL_SOURCE
+    "source/*.cpp"
+)
+
+file(GLOB AWS_EC2-PROTOCOL_MODEL_SOURCE
+    "source/model/*.cpp"
+)
+
+file(GLOB EC2-PROTOCOL_UNIFIED_HEADERS
+    ${AWS_EC2-PROTOCOL_HEADERS}
+    ${AWS_EC2-PROTOCOL_MODEL_HEADERS}
+)
+
+file(GLOB EC2-PROTOCOL_UNITY_SRC
+    ${AWS_EC2-PROTOCOL_SOURCE}
+    ${AWS_EC2-PROTOCOL_MODEL_SOURCE}
+)
+
+if(ENABLE_UNITY_BUILD)
+    enable_unity_build("EC2-PROTOCOL" EC2-PROTOCOL_UNITY_SRC)
+endif()
+
+file(GLOB EC2-PROTOCOL_SRC
+    ${EC2-PROTOCOL_UNIFIED_HEADERS}
+    ${EC2-PROTOCOL_UNITY_SRC}
+)
+
+if(WIN32)
+    #if we are compiling for visual studio, create a sane directory tree.
+    if(MSVC)
+        source_group("Header Files\\aws\\ec2-protocol" FILES ${AWS_EC2-PROTOCOL_HEADERS})
+        source_group("Header Files\\aws\\ec2-protocol\\model" FILES ${AWS_EC2-PROTOCOL_MODEL_HEADERS})
+        source_group("Source Files" FILES ${AWS_EC2-PROTOCOL_SOURCE})
+        source_group("Source Files\\model" FILES ${AWS_EC2-PROTOCOL_MODEL_SOURCE})
+    endif(MSVC)
+endif()
+
+set(EC2-PROTOCOL_INCLUDES
+    "${CMAKE_CURRENT_SOURCE_DIR}/include/"
+)
+
+add_library(${PROJECT_NAME} ${EC2-PROTOCOL_SRC})
+add_library(AWS::${PROJECT_NAME} ALIAS ${PROJECT_NAME})
+
+set_compiler_flags(${PROJECT_NAME})
+set_compiler_warnings(${PROJECT_NAME})
+
+if(USE_WINDOWS_DLL_SEMANTICS AND BUILD_SHARED_LIBS)
+    target_compile_definitions(${PROJECT_NAME} PRIVATE "AWS_EC2PROTOCOL_EXPORTS")
+endif()
+
+target_include_directories(${PROJECT_NAME} PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+    $<INSTALL_INTERFACE:include>)
+
+target_link_libraries(${PROJECT_NAME} PRIVATE ${PLATFORM_DEP_LIBS} ${PROJECT_LIBS})
+
+
+setup_install()
+
+install (FILES ${AWS_EC2-PROTOCOL_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/ec2-protocol)
+install (FILES ${AWS_EC2-PROTOCOL_MODEL_HEADERS} DESTINATION ${INCLUDE_DIRECTORY}/aws/ec2-protocol/model)
+
+do_packaging()
+
+