Fix and enrich pods limit error message

scramjetorg · Dec 14, 2022 · 1ea13d9 · 1ea13d9
1 parent 79bfbab
commit 1ea13d9
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 32 deletions.
diff --git a/packages/adapters/src/kubernetes-client-adapter.ts b/packages/adapters/src/kubernetes-client-adapter.ts
@@ -137,7 +137,7 @@ class KubernetesClientAdapter {
         return response.body.status?.containerStatuses?.[0].state?.terminated?.reason;
     }
 
-    async isPodsLimitReached(quotaName: string) {
+    async getPodsLimit(quotaName: string) {
         const kubeApi = this.config.makeApiClient(k8s.CoreV1Api);
 
         try {
@@ -152,13 +152,19 @@ class KubernetesClientAdapter {
 
                 this.logger.info("Pods limit quota", used, hard);
 
-                return used >= hard;
+                return { used, hard };
             }
         } catch (e) {
             this.logger.warn("Can't get quota object. ");
         }
 
-        return false;
+        return { used: 0, hard: 0 }; // quota failed - no pods!
+    }
+
+    async isPodsLimitReached(quotaName: string) {
+        const { used, hard } = await this.getPodsLimit(quotaName);
+
+        return used >= hard;
     }
 
     private async runWithRetries(retries: number, name: string, callback: any) {

diff --git a/packages/adapters/src/kubernetes-instance-adapter.ts b/packages/adapters/src/kubernetes-instance-adapter.ts
@@ -88,6 +88,16 @@ IComponent {
         };
     }
 
+    async getPodsLimit() {
+        if (this.adapterConfig.quotaName) {
+            return this.kubeClient.getPodsLimit(this.adapterConfig.quotaName);
+        }
+
+        this.logger.warn("Quota name not provided");
+
+        return undefined;
+    }
+
     async run(config: InstanceConfig, instancesServerPort: number, instanceId: string): Promise<ExitCode> {
         if (config.type !== "kubernetes") {
             throw new Error(`Invalid config type for kubernetes adapter: ${config.type}`);

diff --git a/packages/host/src/lib/csi-controller.ts b/packages/host/src/lib/csi-controller.ts
@@ -40,7 +40,7 @@ import { EventEmitter, once } from "events";
 import { ServerResponse } from "http";
 import { DuplexStream, getRouter } from "@scramjet/api-server";
 
-import { getInstanceAdapter } from "@scramjet/adapters";
+import { getInstanceAdapter, KubernetesInstanceAdapter } from "@scramjet/adapters";
 import { cancellableDefer, CancellablePromise, defer, promiseTimeout, TypedEmitter } from "@scramjet/utility";
 import { ObjLogger } from "@scramjet/obj-logger";
 import { ReasonPhrases } from "http-status-codes";
@@ -324,71 +324,90 @@ export class CSIController extends TypedEmitter<Events> {
         });
     }
 
-    private mapRunnerExitCode(exitcode: number): Promise<
+    async handlePodsLimitError(): Promise<never> {
+        let msg = "Instances limit reached";
+
+        if (this.instanceAdapter instanceof KubernetesInstanceAdapter) {
+            const limit = await this.instanceAdapter.getPodsLimit();
+
+            msg = limit ? `Instances limit reached ${limit.used}/${limit.hard}` : msg;
+
+            return Promise.reject({
+                message: msg,
+                exitcode: RunnerExitCode.PODS_LIMIT_REACHED,
+                status: InstanceStatus.ERRORED
+            });
+        }
+
+        this.logger.warn("Incorrect Adapter exitcode");
+
+        return Promise.reject({
+            message: msg,
+            exitcode: RunnerExitCode.PODS_LIMIT_REACHED,
+            status: InstanceStatus.ERRORED
+        });
+    }
+
+    // eslint-disable-next-line complexity
+    private async mapRunnerExitCode(exitcode: number): Promise<
         { message: string, exitcode: number, reason: TerminateReason}
     > {
-        // eslint-disable-next-line default-case
         switch (exitcode) {
-            case RunnerExitCode.INVALID_ENV_VARS: {
+            case RunnerExitCode.SUCCESS:
+                return Promise.resolve({ message: "Instance completed", exitcode, reason: TerminateReason.COMPLETED, status: InstanceStatus.COMPLETED });
+
+            case RunnerExitCode.INVALID_ENV_VARS:
                 return Promise.reject({
                     message: "Runner was started with invalid configuration. This is probably a bug in STH.",
                     exitcode: RunnerExitCode.INVALID_ENV_VARS,
                     status: InstanceStatus.ERRORED
                 });
-            }
-            case RunnerExitCode.PODS_LIMIT_REACHED: {
-                return Promise.reject({
-                    message: "Pods limit reached",
-                    exitcode: RunnerExitCode.PODS_LIMIT_REACHED,
-                    status: InstanceStatus.ERRORED
-                });
-            }
-            case RunnerExitCode.INVALID_SEQUENCE_PATH: {
+
+            case RunnerExitCode.PODS_LIMIT_REACHED:
+                return this.handlePodsLimitError();
+
+            case RunnerExitCode.INVALID_SEQUENCE_PATH:
                 return Promise.reject({
                     message: `Sequence entrypoint path ${this.sequence.config.entrypointPath} is invalid. ` +
                         "Check `main` field in Sequence package.json",
                     exitcode: RunnerExitCode.INVALID_SEQUENCE_PATH,
                     status: InstanceStatus.ERRORED
                 });
-            }
-            case RunnerExitCode.SEQUENCE_FAILED_ON_START: {
+
+            case RunnerExitCode.SEQUENCE_FAILED_ON_START:
                 return Promise.reject({
                     message: "Sequence failed on start",
                     exitcode: RunnerExitCode.SEQUENCE_FAILED_ON_START,
                     status: InstanceStatus.ERRORED
                 });
-            }
-            case RunnerExitCode.SEQUENCE_FAILED_DURING_EXECUTION: {
+
+            case RunnerExitCode.SEQUENCE_FAILED_DURING_EXECUTION:
                 return Promise.reject({
                     message: "Sequence failed during execution",
                     exitcode: RunnerExitCode.SEQUENCE_FAILED_DURING_EXECUTION,
                     status: InstanceStatus.ERRORED
                 });
-            }
-            case RunnerExitCode.SEQUENCE_UNPACK_FAILED: {
+
+            case RunnerExitCode.SEQUENCE_UNPACK_FAILED:
                 return Promise.reject({
                     message: "Sequence unpack failed",
                     exitcode: RunnerExitCode.SEQUENCE_UNPACK_FAILED,
                     status: InstanceStatus.ERRORED
                 });
-            }
-            case RunnerExitCode.KILLED: {
+
+            case RunnerExitCode.KILLED:
                 return Promise.resolve({
                     message: "Instance killed", exitcode: RunnerExitCode.KILLED, reason: TerminateReason.KILLED, status: InstanceStatus.COMPLETED
                 });
-            }
-            case RunnerExitCode.STOPPED: {
+
+            case RunnerExitCode.STOPPED:
                 return Promise.resolve({
                     message: "Instance stopped", exitcode: RunnerExitCode.STOPPED, reason: TerminateReason.STOPPED, status: InstanceStatus.COMPLETED
                 });
-            }
-        }
 
-        if (exitcode > 0) {
-            return Promise.reject({ message: "Runner failed", exitcode, reason: TerminateReason.ERRORED, status: InstanceStatus.ERRORED });
+            default:
+                return Promise.reject({ message: "Runner failed", exitcode, reason: TerminateReason.ERRORED, status: InstanceStatus.ERRORED });
         }
-
-        return Promise.resolve({ message: "Instance completed", exitcode, reason: TerminateReason.COMPLETED, status: InstanceStatus.COMPLETED });
     }
 
     async cleanup() {