From 6b66c6119f44c10a144ea62bb22e99517e0ef2aa Mon Sep 17 00:00:00 2001 From: Will Morrison Date: Thu, 22 Feb 2024 21:24:30 -0500 Subject: [PATCH] Get cluster health from v1/info --- docs/quickstart-config.yaml | 6 +- gateway-ha/gateway-ha-config-docker.yml | 6 +- gateway-ha/gateway-ha-config.yml | 6 +- gateway-ha/pom.xml | 32 +++++++- .../java/io/trino/client/NodeVersion.java | 67 +++++++++++++++++ .../main/java/io/trino/client/ServerInfo.java | 73 +++++++++++++++++++ .../ClusterStatsInfoApiMonitor.java | 63 ++++++++++++++++ .../ha/config/ClusterStatsConfiguration.java | 10 +-- .../ha/config/ClusterStatsMonitorType.java | 22 ++++++ .../ha/module/ClusterStatsMonitorModule.java | 15 ++-- .../TestClusterStatsMonitor.java | 6 ++ 11 files changed, 278 insertions(+), 28 deletions(-) create mode 100644 gateway-ha/src/main/java/io/trino/client/NodeVersion.java create mode 100644 gateway-ha/src/main/java/io/trino/client/ServerInfo.java create mode 100644 gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsInfoApiMonitor.java create mode 100644 gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsMonitorType.java diff --git a/docs/quickstart-config.yaml b/docs/quickstart-config.yaml index 43182e5ff..d1ebefcff 100644 --- a/docs/quickstart-config.yaml +++ b/docs/quickstart-config.yaml @@ -22,11 +22,7 @@ dataStore: driver: org.postgresql.Driver clusterStatsConfiguration: - useApi: true - -backendState: - username: gateway - ssl: false + monitorType: INFO_API modules: - io.trino.gateway.ha.module.HaGatewayProviderModule diff --git a/gateway-ha/gateway-ha-config-docker.yml b/gateway-ha/gateway-ha-config-docker.yml index 4c7e8f587..ced2ef8f7 100644 --- a/gateway-ha/gateway-ha-config-docker.yml +++ b/gateway-ha/gateway-ha-config-docker.yml @@ -15,12 +15,8 @@ dataStore: driver: org.postgresql.Driver queryHistoryHoursRetention: 24 -backendState: - username: lb_query - ssl: false - clusterStatsConfiguration: - useApi: true + monitorType: INFO_API server: applicationConnectors: diff --git a/gateway-ha/gateway-ha-config.yml b/gateway-ha/gateway-ha-config.yml index d60d4ec4d..4c8a6fa69 100644 --- a/gateway-ha/gateway-ha-config.yml +++ b/gateway-ha/gateway-ha-config.yml @@ -15,12 +15,8 @@ dataStore: driver: org.postgresql.Driver queryHistoryHoursRetention: 24 -backendState: - username: lb_query - ssl: false - clusterStatsConfiguration: - useApi: true + monitorType: INFO_API server: applicationConnectors: diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index 18f11084e..a9820b3e3 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -97,6 +97,16 @@ + + io.airlift + http-client + + + + io.airlift + json + + io.dropwizard dropwizard-assets @@ -239,6 +249,13 @@ jakarta.ws.rs-api + + + joda-time + joda-time + + org.apache.commons commons-pool2 @@ -506,9 +523,22 @@ - org.apache.maven.plugins + maven-dependency-plugin + + + + + joda-time:joda-time + + + + + + + + maven-shade-plugin false diff --git a/gateway-ha/src/main/java/io/trino/client/NodeVersion.java b/gateway-ha/src/main/java/io/trino/client/NodeVersion.java new file mode 100644 index 000000000..dd3aee159 --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/client/NodeVersion.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +// copy of https://github.com/trinodb/trino/blob/f83854687229df1ae3325249e7bcd29f03d0943c/client/trino-client/src/main/java/io/trino/client/NodeVersion.java +public class NodeVersion +{ + public static final NodeVersion UNKNOWN = new NodeVersion(""); + + private final String version; + + @JsonCreator + public NodeVersion(@JsonProperty("version") String version) + { + this.version = requireNonNull(version, "version is null"); + } + + @JsonProperty + public String getVersion() + { + return version; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + NodeVersion that = (NodeVersion) o; + return Objects.equals(version, that.version); + } + + @Override + public int hashCode() + { + return Objects.hash(version); + } + + @Override + public String toString() + { + return version; + } +} diff --git a/gateway-ha/src/main/java/io/trino/client/ServerInfo.java b/gateway-ha/src/main/java/io/trino/client/ServerInfo.java new file mode 100644 index 000000000..94979e02e --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/client/ServerInfo.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.lang.Boolean.parseBoolean; +import static java.util.Objects.requireNonNull; + +// based on https://github.com/trinodb/trino/blob/f83854687229df1ae3325249e7bcd29f03d0943c/client/trino-client/src/main/java/io/trino/client/ServerInfo.java#L28 +public class ServerInfo +{ + private final boolean starting; + + @JsonCreator + public ServerInfo( + // Defining as a String allows using requireNonNull to ensure starting is present + @JsonProperty("starting") String starting) + { + this.starting = parseBoolean(requireNonNull(starting, "starting is null")); + } + + @JsonProperty + public boolean isStarting() + { + return starting; + } + + @JsonProperty + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ServerInfo that = (ServerInfo) o; + return Objects.equals(starting, that.starting); + } + + @Override + public int hashCode() + { + return starting ? 1 : 0; + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("starting", starting) + .toString(); + } +} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsInfoApiMonitor.java b/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsInfoApiMonitor.java new file mode 100644 index 000000000..59d6bb24c --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/clustermonitor/ClusterStatsInfoApiMonitor.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.gateway.ha.clustermonitor; + +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.HttpClientConfig; +import io.airlift.http.client.Request; +import io.airlift.http.client.jetty.JettyHttpClient; +import io.trino.client.ServerInfo; +import io.trino.gateway.ha.config.ProxyBackendConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; + +import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static io.airlift.json.JsonCodec.jsonCodec; + +public class ClusterStatsInfoApiMonitor + implements ClusterStatsMonitor +{ + //TODO: make client options configurable + private static final HttpClient client = new JettyHttpClient(new HttpClientConfig()); + private static final Logger log = LoggerFactory.getLogger(ClusterStatsInfoApiMonitor.class); + + @Override + public ClusterStats monitor(ProxyBackendConfiguration backend) + { + return ClusterStats.builder(backend.getName()).healthy(isReadyStatus(backend.getProxyTo())) + .proxyTo(backend.getProxyTo()) + .externalUrl(backend.getExternalUrl()) + .routingGroup(backend.getRoutingGroup()).build(); + } + + private boolean isReadyStatus(String baseUrl) + { + Request request = prepareGet() + .setUri(uriBuilderFrom(URI.create(baseUrl)).appendPath("/v1/info").build()) + .build(); + + try { + ServerInfo serverInfo = client.execute(request, createJsonResponseHandler(jsonCodec(ServerInfo.class))); + return !serverInfo.isStarting(); + } + catch (Exception e) { + log.error("Exception checking {} for health: {} ", request.getUri(), e.getMessage()); + } + return false; + } +} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsConfiguration.java b/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsConfiguration.java index 3b84e6e5e..0c05978e1 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsConfiguration.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsConfiguration.java @@ -15,17 +15,17 @@ public class ClusterStatsConfiguration { - private boolean useApi; + private ClusterStatsMonitorType monitorType; public ClusterStatsConfiguration() {} - public boolean isUseApi() + public ClusterStatsMonitorType getMonitorType() { - return this.useApi; + return monitorType; } - public void setUseApi(boolean useApi) + public void setMonitorType(ClusterStatsMonitorType monitorType) { - this.useApi = useApi; + this.monitorType = monitorType; } } diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsMonitorType.java b/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsMonitorType.java new file mode 100644 index 000000000..ad7835bd2 --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/config/ClusterStatsMonitorType.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.gateway.ha.config; + +public enum ClusterStatsMonitorType +{ + NOOP, + INFO_API, + UI_API, + JDBC +} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/module/ClusterStatsMonitorModule.java b/gateway-ha/src/main/java/io/trino/gateway/ha/module/ClusterStatsMonitorModule.java index dd11be260..fe7495742 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/module/ClusterStatsMonitorModule.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/module/ClusterStatsMonitorModule.java @@ -18,6 +18,7 @@ import io.dropwizard.core.setup.Environment; import io.trino.gateway.baseapp.AppModule; import io.trino.gateway.ha.clustermonitor.ClusterStatsHttpMonitor; +import io.trino.gateway.ha.clustermonitor.ClusterStatsInfoApiMonitor; import io.trino.gateway.ha.clustermonitor.ClusterStatsJdbcMonitor; import io.trino.gateway.ha.clustermonitor.ClusterStatsMonitor; import io.trino.gateway.ha.clustermonitor.NoopClusterStatsMonitor; @@ -41,13 +42,13 @@ public ClusterStatsMonitor getClusterStatsMonitor() { ClusterStatsConfiguration clusterStatsConfig = config.getClusterStatsConfiguration(); if (config.getBackendState() == null) { - return new NoopClusterStatsMonitor(); - } - if (clusterStatsConfig.isUseApi()) { - return new ClusterStatsHttpMonitor(config.getBackendState()); - } - else { - return new ClusterStatsJdbcMonitor(config.getBackendState()); + return new ClusterStatsInfoApiMonitor(); } + return switch (clusterStatsConfig.getMonitorType()) { + case INFO_API -> new ClusterStatsInfoApiMonitor(); + case UI_API -> new ClusterStatsHttpMonitor(config.getBackendState()); + case JDBC -> new ClusterStatsJdbcMonitor(config.getBackendState()); + case NOOP -> new NoopClusterStatsMonitor(); + }; } } diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/clustermonitor/TestClusterStatsMonitor.java b/gateway-ha/src/test/java/io/trino/gateway/ha/clustermonitor/TestClusterStatsMonitor.java index bfbde8f47..9b89728e3 100644 --- a/gateway-ha/src/test/java/io/trino/gateway/ha/clustermonitor/TestClusterStatsMonitor.java +++ b/gateway-ha/src/test/java/io/trino/gateway/ha/clustermonitor/TestClusterStatsMonitor.java @@ -56,6 +56,12 @@ public void testJdbcMonitor() testClusterStatsMonitor(ClusterStatsJdbcMonitor::new); } + @Test + public void testInfoApiMonitor() + { + testClusterStatsMonitor(ignored -> new ClusterStatsInfoApiMonitor()); + } + private void testClusterStatsMonitor(Function monitorFactory) { BackendStateConfiguration backendStateConfiguration = new BackendStateConfiguration();