feat(honeycomb sink): Support compression (#21889)

* honeycomb sink: support compression

* add changelog

* explain the default gzip format

* use zstd as default format

* update docs

changelog.d/21889_support_compression_for_honeycomb_sink.enhancement.md

@@ -0,0 +1,3 @@
+Support compression for the Honeycomb sink. The `zstd` format is enabled by default.
+
+authors: hgiasac

src/sinks/honeycomb/config.rs

@@ -61,6 +61,11 @@ pub struct HoneycombConfig {
     #[serde(default, skip_serializing_if = "crate::serde::is_default")]
     encoding: Transformer,
 
+    /// The compression algorithm to use.
+    #[configurable(derived)]
+    #[serde(default = "Compression::zstd_default")]
+    compression: Compression,
+
     #[configurable(derived)]
     #[serde(
         default,
@@ -103,13 +108,15 @@ impl SinkConfig for HoneycombConfig {
             encoder: HoneycombEncoder {
                 transformer: self.encoding.clone(),
             },
+            compression: self.compression,
         };
 
         let uri = self.build_uri()?;
 
         let honeycomb_service_request_builder = HoneycombSvcRequestBuilder {
             uri: uri.clone(),
             api_key: self.api_key.clone(),
+            compression: self.compression,
         };
 
         let client = HttpClient::new(None, cx.proxy())?;

src/sinks/honeycomb/request_builder.rs

@@ -9,6 +9,7 @@ use super::encoder::HoneycombEncoder;
 
 pub(super) struct HoneycombRequestBuilder {
     pub(super) encoder: HoneycombEncoder,
+    pub(super) compression: Compression,
 }
 
 impl RequestBuilder<Vec<Event>> for HoneycombRequestBuilder {
@@ -20,7 +21,7 @@ impl RequestBuilder<Vec<Event>> for HoneycombRequestBuilder {
     type Error = io::Error;
 
     fn compression(&self) -> Compression {
-        Compression::None
+        self.compression
     }
 
     fn encoder(&self) -> &Self::Encoder {

src/sinks/honeycomb/service.rs

@@ -5,6 +5,7 @@ use http::{Request, Uri};
 use vector_lib::sensitive_string::SensitiveString;
 
 use crate::sinks::{
+    util::buffer::compression::Compression,
     util::http::{HttpRequest, HttpServiceRequestBuilder},
     HTTPRequestBuilderSnafu,
 };
@@ -16,11 +17,17 @@ use super::config::HTTP_HEADER_HONEYCOMB;
 pub(super) struct HoneycombSvcRequestBuilder {
     pub(super) uri: Uri,
     pub(super) api_key: SensitiveString,
+    pub(super) compression: Compression,
 }
 
 impl HttpServiceRequestBuilder<()> for HoneycombSvcRequestBuilder {
     fn build(&self, mut request: HttpRequest<()>) -> Result<Request<Bytes>, crate::Error> {
-        let builder = Request::post(&self.uri).header(HTTP_HEADER_HONEYCOMB, self.api_key.inner());
+        let mut builder =
+            Request::post(&self.uri).header(HTTP_HEADER_HONEYCOMB, self.api_key.inner());
+
+        if let Some(ce) = self.compression.content_encoding() {
+            builder = builder.header("Content-Encoding".to_string(), ce.to_string());
+        }
 
         builder
             .body(request.take_payload())

website/cue/reference/components/sinks/base/honeycomb.cue

@@ -64,6 +64,36 @@ base: components: sinks: honeycomb: configuration: {
 			}
 		}
 	}
+	compression: {
+		description: "The compression algorithm to use."
+		required:    false
+		type: string: {
+			default: "zstd"
+			enum: {
+				gzip: """
+					[Gzip][gzip] compression.
+
+					[gzip]: https://www.gzip.org/
+					"""
+				none: "No compression."
+				snappy: """
+					[Snappy][snappy] compression.
+
+					[snappy]: https://github.com/google/snappy/blob/main/docs/README.md
+					"""
+				zlib: """
+					[Zlib][zlib] compression.
+
+					[zlib]: https://zlib.net/
+					"""
+				zstd: """
+					[Zstandard][zstd] compression.
+
+					[zstd]: https://facebook.github.io/zstd/
+					"""
+			}
+		}
+	}
 	dataset: {
 		description: "The dataset to which logs are sent."
 		required:    true

website/cue/reference/components/sinks/honeycomb.cue

@@ -23,7 +23,12 @@ components: sinks: honeycomb: {
 				max_bytes:    100_000
 				timeout_secs: 1.0
 			}
-			compression: enabled: false
+			compression: {
+				enabled: true
+				default: "gzip"
+				algorithms: ["none", "gzip", "zstd"]
+				levels: ["none", "fast", "default", "best", 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+			}
 			encoding: {
 				enabled: true
 				codec: enabled: false