Issue 357: Integrating health checks #569
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -28,19 +28,23 @@ var ( | |
| ) | ||
|
|
||
| const ( | ||
| compareVersion string = "0.10.0" | ||
| MajorMinorVersionRegexp string = `^v?(?P<Version>[0-9]+\.[0-9]+\.[0-9]+)` | ||
| ) | ||
|
|
||
| func init() { | ||
| versionRegexp = regexp.MustCompile(MajorMinorVersionRegexp) | ||
| } | ||
|
|
||
| //function to check if v1 is below v2 or not | ||
| func IsVersionBelow(v1 string, v2 string) bool { | ||
| if v1 == "" { | ||
| return true | ||
| } | ||
| if v2 == "" { | ||
| return false | ||
| } | ||
| result, _ := CompareVersions(v1, v2, "<") | ||
| if result { | ||
| return true | ||
| } | ||
|
|
@@ -69,29 +73,52 @@ func IsOrphan(k8sObjectName string, replicas int32) bool { | |
| return int32(ordinal) >= replicas | ||
| } | ||
|
|
||
| func HealthcheckCommand(version string, port int32, restport int32) []string { | ||
| command := "" | ||
| if IsVersionBelow(version, compareVersion) { | ||
| command = fmt.Sprintf("netstat -ltn 2> /dev/null | grep %d || ss -ltn 2> /dev/null | grep %d", port, port) | ||
| } else { | ||
| command = fmt.Sprintf("(netstat -ltn 2> /dev/null | grep %d || ss -ltn 2> /dev/null | grep %d) && (curl -s -X GET 'http://localhost:%d/v1/health/liveness' || curl -s -k -X GET 'https://localhost:%d/v1/health/liveness')", port, port, restport, restport) | ||
| } | ||
| return []string{"/bin/sh", "-c", command} | ||
| } | ||
|
|
||
| func ControllerReadinessCheck(version string, port int32, authflag bool) []string { | ||
| command := "" | ||
| if IsVersionBelow(version, compareVersion) { | ||
| //This function check for the readiness of the controller in the following cases | ||
| //1) Auth and TLS Enabled- in this case, we check if the controller is properly enabled with authentication or not and we do a get on controller and with dummy credentials(testtls:testtls) and the controller returns 401 error in this case if it's correctly configured | ||
| //2) Auth Enabled and TLS Disabled- in this case, we check if the controller is properly enabled with authentication or not and we do a get on controller and with dummy credentials(testtls:testtls) and the controller returns 401 error in this case if it's correctly configured | ||
| //3) Auth Disabled and TLS Enabled- in this case, we check if the controller can create scopes or not by checking if _system scope is present or not | ||
| //4) Auth and TLS Disabled- in this case, we check if the controller can create scopes or not by checking if _system scope is present or not | ||
| if authflag == true { | ||
| // This is to check the readiness of controller in case auth is Enabled | ||
| // here we are using login credential as testtls:testtls which should | ||
| // not be used as auth credential and we depend on controller giving us | ||
| // 401 error which means controller is properly configured with auth | ||
| // it checks both cases when tls is enabled as well as tls disabled | ||
| // with auth enabled | ||
| command = fmt.Sprintf("echo $JAVA_OPTS | grep 'controller.auth.tlsEnabled=true' && curl -v -k -u testtls:testtls -s -X GET 'https://localhost:%d/v1/scopes/' 2>&1 -H 'accept: application/json' | grep 401 || (echo $JAVA_OPTS | grep 'controller.auth.tlsEnabled=false' && curl -v -k -u testtls:testtls -s -X GET 'http://localhost:%d/v1/scopes/' 2>&1 -H 'accept: application/json' | grep 401 ) || (echo $JAVA_OPTS | grep 'controller.security.tls.enable=true' && echo $JAVA_OPTS | grep -v 'controller.auth.tlsEnabled' && curl -v -k -u testtls:testtls -s -X GET 'https://localhost:%d/v1/scopes/' 2>&1 -H 'accept: application/json' | grep 401 ) || (curl -v -k -u testtls:testtls -s -X GET 'http://localhost:%d/v1/scopes/' 2>&1 -H 'accept: application/json' | grep 401 )", port, port, port, port) | ||
SrishT marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } else { | ||
| // This is to check the readiness in case auth is not enabled | ||
| // and it covers both the cases with tls enabled and tls disabled | ||
| // along with auth disabled | ||
| command = fmt.Sprintf("echo $JAVA_OPTS | grep 'controller.auth.tlsEnabled=true' && curl -s -X GET 'https://localhost:%d/v1/scopes/' -H 'accept: application/json' | grep '_system'|| (echo $JAVA_OPTS | grep 'controller.auth.tlsEnabled=false' && curl -s -X GET 'http://localhost:%d/v1/scopes/' -H 'accept: application/json' | grep '_system' ) || (echo $JAVA_OPTS | grep 'controller.security.tls.enable=true' && echo $JAVA_OPTS | grep -v 'controller.auth.tlsEnabled' && curl -s -X GET 'https://localhost:%d/v1/scopes/' -H 'accept: application/json' | grep '_system' ) || (curl -s -X GET 'http://localhost:%d/v1/scopes/' -H 'accept: application/json' | grep '_system') ", port, port, port, port) | ||
| } | ||
| } else { | ||
| command = fmt.Sprintf("curl -s -X GET 'http://localhost:%d/v1/health/readiness' || curl -s -k -X GET 'https://localhost:%d/v1/health/readiness'", port, port) | ||
|
There was a problem hiding this comment. Is there something in the readiness call payload that indicates whether or not pravega controller is running with auth on (I am not suggesting to send bad credentials to test out a 401 in return :) rather, just a field that literally says "auth: enabled" in the health readiness response). If so, then if auth was requested in the java options, we should find in the readiness a confirmation that auth is on. And the opposite if auth was not set or set to off in the java options. If that's not something that is in the readiness payload today; I think it should be added, but there is nothing for this PR to leverage and line 109 is the best it can be at this time. There was a problem hiding this comment. @sarlaccpit the There was a problem hiding this comment. Got it. I think it's fine the way it is then. It makes sense the auth/no auth (and probably many other details) are protected behind the get details endpoint. |
||
| } | ||
| return []string{"/bin/sh", "-c", command} | ||
| } | ||
|
|
||
| func SegmentStoreReadinessCheck(version string, port int32, restport int32) []string { | ||
| command := "" | ||
| if IsVersionBelow(version, compareVersion) { | ||
| command = fmt.Sprintf("netstat -ltn 2> /dev/null | grep %d || ss -ltn 2> /dev/null | grep %d", port, port) | ||
| } else { | ||
| command = fmt.Sprintf("curl -s -X GET 'http://localhost:%d/v1/health/readiness' || curl -s -k -X GET 'https://localhost:%d/v1/health/readiness'", restport, restport) | ||
| } | ||
| return []string{"/bin/sh", "-c", command} | ||
| } | ||
|
|
||
| // Min returns the smaller of x or y. | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is it really required to check the port using
netstat? Why not to send a REST request right away and check for return code?It might happen that
netstatwill not be present in pravega containers starting from 0.10...