From 4cb8e55292ee3f020838a814a9357560499c39c6 Mon Sep 17 00:00:00 2001 From: Viktor Date: Tue, 3 Oct 2017 22:21:10 +0200 Subject: [PATCH] Compression on service level --- docs/config.md | 2 +- docs/usage.md | 4 +++ integration_tests/integration_swarm_test.go | 22 ++++++++++--- proxy/ha_proxy_test.go | 35 +++++++++++++++++++++ proxy/template.go | 6 ++++ proxy/types.go | 5 +++ server/server_test.go | 18 ++++++++--- 7 files changed, 83 insertions(+), 9 deletions(-) diff --git a/docs/config.md b/docs/config.md index cada0da1..566d88f8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -17,7 +17,7 @@ The following environment variables can be used to configure the *Docker Flow Pr |CFG_TEMPLATE_PATH |Path to the configuration template. The path can be absolute (starting with `/`) or relative to `/cfg/tmpl`.
**Default value:** `/cfg/tmpl/haproxy.tmpl`| |CHECK_RESOLVERS |Enable `docker` as a resolver. Provides higher reliability at the cost of backend initialization time. If enabled, it might take a few seconds until a backend is resolved and operational.
**Default value:** `false`| |CERTS |This parameter is **deprecated** as of February 2017. All the certificates from the `/certs/` directory are now loaded automatically.| -|COMPRESSION_ALGO |Enable HTTP compression. The currently supported algorithms are:
**identity**: this is mostly for debugging.
**gzip**: applies gzip compression. This setting is only available when support for zlib or libslz was built in.
**deflate** same as *gzip*, but with deflate algorithm and zlib format. Note that this algorithm has ambiguous support on many browsers and no support at all from recent ones. It is strongly recommended not to use it for anything else than experimentation. This setting is only available when support for zlib or libslz was built in.
**raw-deflate**: same as *deflate* without the zlib wrapper, and used as an alternative when the browser wants "deflate". All major browsers understand it and despite violating the standards, it is known to work better than *deflate*, at least on MSIE and some versions of Safari. This setting is only available when support for zlib or libslz was built in.
Compression will be activated depending on the Accept-Encoding request header. With identity, it does not take care of that header. If backend servers support HTTP compression, these directives will be no-op: haproxy will see the compressed response and will not compress again. If backend servers do not support HTTP compression and there is Accept-Encoding header in request, haproxy will compress the matching response.
Compression is disabled when:
* the request does not advertise a supported compression algorithm in the "Accept-Encoding" header
* the response message is not HTTP/1.1
* HTTP status code is not 200
* response header "Transfer-Encoding" contains "chunked" (Temporary Workaround)
* response contain neither a "Content-Length" header nor a "Transfer-Encoding" whose last value is "chunked"
* response contains a "Content-Type" header whose first value starts with "multipart"
* the response contains the "no-transform" value in the "Cache-control" header
* User-Agent matches "Mozilla/4" unless it is MSIE 6 with XP SP2, or MSIE 7 and later
* The response contains a "Content-Encoding" header, indicating that the response is already compressed (see compression offload)
**Example:** gzip| +|COMPRESSION_ALGO |Enable HTTP compression. The currently supported algorithms are:
**identity**: this is mostly for debugging.
**gzip**: applies gzip compression. This setting is only available when support for zlib or libslz was built in.
**deflate**: same as *gzip*, but with deflate algorithm and zlib format. Note that this algorithm has ambiguous support on many browsers and no support at all from recent ones. It is strongly recommended not to use it for anything else than experimentation. This setting is only available when support for zlib or libslz was built in.
**raw-deflate**: same as *deflate* without the zlib wrapper, and used as an alternative when the browser wants "deflate". All major browsers understand it and despite violating the standards, it is known to work better than *deflate*, at least on MSIE and some versions of Safari. This setting is only available when support for zlib or libslz was built in.
Compression will be activated depending on the Accept-Encoding request header. With identity, it does not take care of that header. If backend servers support HTTP compression, these directives will be no-op: haproxy will see the compressed response and will not compress again. If backend servers do not support HTTP compression and there is Accept-Encoding header in request, haproxy will compress the matching response.
Compression is disabled when:
* the request does not advertise a supported compression algorithm in the "Accept-Encoding" header
* the response message is not HTTP/1.1
* HTTP status code is not 200
* response header "Transfer-Encoding" contains "chunked" (Temporary Workaround)
* response contain neither a "Content-Length" header nor a "Transfer-Encoding" whose last value is "chunked"
* response contains a "Content-Type" header whose first value starts with "multipart"
* the response contains the "no-transform" value in the "Cache-control" header
* User-Agent matches "Mozilla/4" unless it is MSIE 6 with XP SP2, or MSIE 7 and later
* The response contains a "Content-Encoding" header, indicating that the response is already compressed (see compression offload)
**Example:** gzip| |COMPRESSION_TYPE |The type of files that will be compressed.
**Example:** text/css text/html text/javascript application/javascript text/plain text/xml application/json| |CONNECTION_MODE |HAProxy supports 5 connection modes.

`http-keep-alive`: all requests and responses are processed.
`http-tunnel`: only the first request and response are processed, everything else is forwarded with no analysis.
`httpclose`: tunnel with "Connection: close" added in both directions.
`http-server-close`: the server-facing connection is closed after the response.
`forceclose`: the connection is actively closed after end of response.

In general, it is preferred to use `http-server-close` with application servers, and some static servers might benefit from `http-keep-alive`.
**Example:** `http-server-close`
**Default value:** `http-keep-alive`| |DEBUG |Enables logging of each request sent through the proxy. Please consult [Debug Format](#debug-format) for info about the log entries. This feature should be used with caution. **Do not enable debugging in production unless necessary.**
**Example:** true
**Default value:** `false`| diff --git a/docs/usage.md b/docs/usage.md index 51b599b9..c83bfe2b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -45,6 +45,8 @@ The following query parameters can be used only when `reqMode` is set to `http` |Query |Description | |-------------|--------------------------------------------------------------------------------| |allowedMethods|The list of allowed methods. If specified, a request with a method that is not on the list will be denied. Multiple methods can be separated with comma (`,`). Change the environment variable `SEPARATOR` if comma is to be used for other purposes. The parameter can be prefixed with an index thus allowing definition of multiple destinations for a single service (e.g. `allowedMethods.1`, `allowedMethods.2`, and so on).
**Example:** `GET,DELETE`| +|compressionAlgo|Enable HTTP compression for the given service. The currently supported algorithms are:
**identity**: this is mostly for debugging.
**gzip**: applies gzip compression. This setting is only available when support for zlib or libslz was built in.
**deflate**: same as *gzip*, but with deflate algorithm and zlib format. Note that this algorithm has ambiguous support on many browsers and no support at all from recent ones. It is strongly recommended not to use it for anything else than experimentation. This setting is only available when support for zlib or libslz was built in.
**raw-deflate**: same as *deflate* without the zlib wrapper, and used as an alternative when the browser wants "deflate". All major browsers understand it and despite violating the standards, it is known to work better than *deflate*, at least on MSIE and some versions of Safari. This setting is only available when support for zlib or libslz was built in.
Compression will be activated depending on the Accept-Encoding request header. With identity, it does not take care of that header. If backend servers support HTTP compression, these directives will be no-op: haproxy will see the compressed response and will not compress again. If backend servers do not support HTTP compression and there is Accept-Encoding header in request, haproxy will compress the matching response.
Compression is disabled when:
* the request does not advertise a supported compression algorithm in the "Accept-Encoding" header
* the response message is not HTTP/1.1
* HTTP status code is not 200
* response header "Transfer-Encoding" contains "chunked" (Temporary Workaround)
* response contain neither a "Content-Length" header nor a "Transfer-Encoding" whose last value is "chunked"
* response contains a "Content-Type" header whose first value starts with "multipart"
* the response contains the "no-transform" value in the "Cache-control" header
* User-Agent matches "Mozilla/4" unless it is MSIE 6 with XP SP2, or MSIE 7 and later
* The response contains a "Content-Encoding" header, indicating that the response is already compressed (see compression offload)
**Example:** gzip| +|compressionType|The type of files that will be compressed.
**Example:** text/css text/html text/javascript application/javascript text/plain text/xml application/json| |deniedMethods|The list of denied methods. If specified, a request with a method that is on the list will be denied. Multiple methods can be separated with comma (`,`). Change the environment variable `SEPARATOR` if comma is to be used for other purposes. The parameter can be prefixed with an index thus allowing definition of multiple destinations for a single service (e.g. `deniedMethods.1`, `deniedMethods.2`, and so on).
**Example:** `PUT,POST`| |denyHttp |Whether to deny HTTP requests thus allowing only HTTPS. The parameter can be prefixed with an index thus allowing definition of multiple destinations for a single service (e.g. `denyHttp.1`, `denyHttp.2`, and so on).
**Example:** `true`
**Default Value:** `false`| |httpsRedirectCode|HTTP code for HTTP to HTTPS redirects. This parameter is used only if `httpsOnly` is set to `true`
**Example:** `301`| @@ -114,6 +116,8 @@ The map between the HTTP query parameters and environment variables is as follow |addResHeader |ADD_RES_HEADER | |allowedMethods |ALLOWED_METHODS | |backendExtra |BACKEND_EXTRA | +|compressionAlgo |COMPRESSION_ALGO | +|compressionType |COMPRESSION_TYPE | |deniedMethods |DENIED_METHODS | |distribute |DISTRIBUTE | |httpsOnly |HTTPS_ONLY | diff --git a/integration_tests/integration_swarm_test.go b/integration_tests/integration_swarm_test.go index da6dd276..30a98df2 100644 --- a/integration_tests/integration_swarm_test.go +++ b/integration_tests/integration_swarm_test.go @@ -170,10 +170,9 @@ func (s IntegrationSwarmTestSuite) Test_Metrics() { } func (s IntegrationSwarmTestSuite) Test_Compression() { - defer func() { - exec.Command("/bin/sh", "-c", `docker service update --env-rm "COMPRESSION_ALGO" --env-rm "COMPRESSION_TYPE" proxy`).Output() - s.waitForContainers(1, "proxy") - }() + + // Compression defined for all services + _, err := exec.Command( "/bin/sh", "-c", @@ -194,6 +193,21 @@ func (s IntegrationSwarmTestSuite) Test_Compression() { s.Equal(200, resp.StatusCode, s.getProxyConf("")) s.Contains(resp.Header["Content-Encoding"], "gzip", s.getProxyConf("")) } + + exec.Command("/bin/sh", "-c", `docker service update --env-rm "COMPRESSION_ALGO" --env-rm "COMPRESSION_TYPE" proxy`).Output() + s.waitForContainers(1, "proxy") + + // Compression defined on a service level + + s.reconfigureGoDemo("&compressionAlgo=gzip&compressionType=text/css%20text/html%20text/javascript%20application/javascript%20text/plain%20text/xml%20application/json") + + resp, err = client.Do(req) + + s.NoError(err) + if resp != nil { + s.Equal(200, resp.StatusCode, s.getProxyConf("")) + s.Contains(resp.Header["Content-Encoding"], "gzip", s.getProxyConf("")) + } } func (s IntegrationSwarmTestSuite) Test_ZombieProcesses() { diff --git a/proxy/ha_proxy_test.go b/proxy/ha_proxy_test.go index cac10a55..617455c8 100644 --- a/proxy/ha_proxy_test.go +++ b/proxy/ha_proxy_test.go @@ -1564,6 +1564,41 @@ func (s HaProxyTestSuite) Test_CreateConfigFromTemplates_ForwardsToDomain_WhenRe s.Equal(expectedData, actualData) } +func (s HaProxyTestSuite) Test_CreateConfigFromTemplates_AddsCompressionAlgoToTheFrontent() { + var actualData string + tmpl := s.TemplateContent + expectedData := fmt.Sprintf( + `%s + compression algo my-compression-algo + compression type my-compression-type + acl url_my-service1111_0 path_beg /path + use_backend my-service-be1111_0 if url_my-service1111_0%s`, + tmpl, + s.ServicesContent, + ) + writeFile = func(filename string, data []byte, perm os.FileMode) error { + actualData = string(data) + return nil + } + p := NewHaProxy(s.TemplatesPath, s.ConfigsPath) + dataInstance.Services["my-service"] = Service{ + CompressionAlgo: "my-compression-algo", + CompressionType: "my-compression-type", + PathType: "path_beg", + ServiceName: "my-service", + ServiceDest: []ServiceDest{ + { + Port: "1111", + ServicePath: []string{"/path"}, + }, + }, + } + + p.CreateConfigFromTemplates() + + s.Equal(expectedData, actualData) +} + func (s HaProxyTestSuite) Test_CreateConfigFromTemplates_UsesServiceHeader() { var actualData string tmpl := s.TemplateContent diff --git a/proxy/template.go b/proxy/template.go index d8b7b586..d146c0fa 100644 --- a/proxy/template.go +++ b/proxy/template.go @@ -14,6 +14,12 @@ func getFrontTemplate(s Service) string { tmplString := ` {{- range $sd := .ServiceDest}} {{- if eq .ReqMode "http"}} + {{- if ne $.CompressionAlgo ""}} + compression algo {{$.CompressionAlgo}} + {{- if ne $.CompressionType ""}} + compression type {{$.CompressionType}} + {{- end}} + {{- end}} {{- if ne .Port ""}} acl url_{{$.AclName}}{{.Port}}_{{.Index}}{{range .ServicePath}} {{if eq $.PathType ""}}path_beg{{end}}{{if ne $.PathType ""}}{{$.PathType}}{{end}} {{.}}{{end}}{{.SrcPortAcl}} {{- end}} diff --git a/proxy/types.go b/proxy/types.go index 3c1bd11b..916a7c68 100644 --- a/proxy/types.go +++ b/proxy/types.go @@ -75,6 +75,11 @@ type Service struct { BackendExtra string `split_words:"true"` // Whether to use `docker` as a check resolver. Set through the environment variable CHECK_RESOLVERS CheckResolvers bool `split_words:"true"` + // Enable HTTP compression. + // The currently supported algorithms are: identity, gzip, deflate, raw-deflate. + CompressionAlgo string `split_words:"true"` + // The type of files that will be compressed. + CompressionType string `split_words:"true"` // One of the five connection modes supported by the HAProxy. // `http-keep-alive`: all requests and responses are processed. // `http-tunnel`: only the first request and response are processed, everything else is forwarded with no analysis. diff --git a/server/server_test.go b/server/server_test.go index dcc32f61..c1b19f1f 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -520,6 +520,8 @@ func (s *ServerTestSuite) Test_GetServiceFromUrl_ReturnsProxyService() { AclName: "aclName", AddReqHeader: []string{"add-header-1", "add-header-2"}, AddResHeader: []string{"add-header-1", "add-header-2"}, + CompressionAlgo: "compressionAlgo", + CompressionType: "compressionType", ConnectionMode: "my-connection-mode", DelReqHeader: []string{"add-header-1", "add-header-2"}, DelResHeader: []string{"add-header-1", "add-header-2"}, @@ -556,7 +558,7 @@ func (s *ServerTestSuite) Test_GetServiceFromUrl_ReturnsProxyService() { {Username: "user2", Password: "pass2", PassEncrypted: true}}, } addr := fmt.Sprintf( - "%s?serviceName=%s&users=%s&usersPassEncrypted=%t&aclName=%s&serviceCert=%s&outboundHostname=%s&pathType=%s&reqPathSearch=%s&reqPathReplace=%s&templateFePath=%s&templateBePath=%s&timeoutServer=%s&timeoutTunnel=%s&reqMode=%s&httpsOnly=%t&httpsRedirectCode=%s&isDefaultBackend=%t&redirectWhenHttpProto=%t&httpsPort=%d&serviceDomain=%s&redirectFromDomain=%s&distribute=%t&sslVerifyNone=%t&serviceDomainAlgo=%s&addReqHeader=%s&addResHeader=%s&setReqHeader=%s&setResHeader=%s&delReqHeader=%s&delResHeader=%s&servicePath=/&port=1234&connectionMode=%s&serviceHeader=X-Version:3,name:Viktor&allowedMethods=GET,DELETE&deniedMethods=PUT,POST", + "%s?serviceName=%s&users=%s&usersPassEncrypted=%t&aclName=%s&serviceCert=%s&outboundHostname=%s&pathType=%s&reqPathSearch=%s&reqPathReplace=%s&templateFePath=%s&templateBePath=%s&timeoutServer=%s&timeoutTunnel=%s&reqMode=%s&httpsOnly=%t&httpsRedirectCode=%s&isDefaultBackend=%t&redirectWhenHttpProto=%t&httpsPort=%d&serviceDomain=%s&redirectFromDomain=%s&distribute=%t&sslVerifyNone=%t&serviceDomainAlgo=%s&addReqHeader=%s&addResHeader=%s&setReqHeader=%s&setResHeader=%s&delReqHeader=%s&delResHeader=%s&servicePath=/&port=1234&connectionMode=%s&serviceHeader=X-Version:3,name:Viktor&allowedMethods=GET,DELETE&deniedMethods=PUT,POST&compressionAlgo=%s&compressionType=%s", s.BaseUrl, expected.ServiceName, "user1:pass1,user2:pass2", @@ -589,6 +591,8 @@ func (s *ServerTestSuite) Test_GetServiceFromUrl_ReturnsProxyService() { strings.Join(expected.DelReqHeader, ","), strings.Join(expected.DelResHeader, ","), expected.ConnectionMode, + expected.CompressionAlgo, + expected.CompressionType, ) req, _ := http.NewRequest("GET", addr, nil) srv := serve{} @@ -652,9 +656,11 @@ func (s *ServerTestSuite) Test_GetServiceFromUrl_SetsServicePathToSlash_WhenDoma func (s *ServerTestSuite) Test_GetServicesFromEnvVars_ReturnsServices() { service := proxy.Service{ - AclName: "my-AclName", - AddReqHeader: []string{"add-header-1", "add-header-2"}, - AddResHeader: []string{"add-header-1", "add-header-2"}, + AclName: "my-AclName", + AddReqHeader: []string{"add-header-1", "add-header-2"}, + AddResHeader: []string{"add-header-1", "add-header-2"}, + CompressionAlgo: "compressionAlgo", + // CompressionType: "compressionType", ConnectionMode: "my-connection-mode", DelReqHeader: []string{"del-header-1", "del-header-2"}, DelResHeader: []string{"del-header-1", "del-header-2"}, @@ -691,6 +697,8 @@ func (s *ServerTestSuite) Test_GetServicesFromEnvVars_ReturnsServices() { os.Setenv("DFP_SERVICE_ACL_NAME", service.AclName) os.Setenv("DFP_SERVICE_ADD_REQ_HEADER", strings.Join(service.AddReqHeader, ",")) os.Setenv("DFP_SERVICE_ADD_RES_HEADER", strings.Join(service.AddResHeader, ",")) + os.Setenv("DFP_SERVICE_COMPRESSION_ALGO", service.CompressionAlgo) + os.Setenv("DFP_SERVICE_COMPRESSION_TYPE", service.CompressionType) os.Setenv("DFP_SERVICE_CONNECTION_MODE", service.ConnectionMode) os.Setenv("DFP_SERVICE_DEL_REQ_HEADER", strings.Join(service.DelReqHeader, ",")) os.Setenv("DFP_SERVICE_DEL_RES_HEADER", strings.Join(service.DelResHeader, ",")) @@ -724,6 +732,8 @@ func (s *ServerTestSuite) Test_GetServicesFromEnvVars_ReturnsServices() { os.Unsetenv("DFP_SERVICE_ACL_NAME") os.Unsetenv("DFP_SERVICE_ADD_REQ_HEADER") os.Unsetenv("DFP_SERVICE_ADD_RES_HEADER") + os.Unsetenv("DFP_SERVICE_COMPRESSION_ALGO") + os.Unsetenv("DFP_SERVICE_COMPRESSION_TYPE") os.Unsetenv("DFP_SERVICE_CONNECTION_MODE") os.Unsetenv("DFP_SERVICE_DEL_REQ_HEADER") os.Unsetenv("DFP_SERVICE_DEL_RES_HEADER")