diff --git a/core/api-server/api/rest-api/swagger.json b/core/api-server/api/rest-api/swagger.json index e641e8188..ecb5870ed 100644 --- a/core/api-server/api/rest-api/swagger.json +++ b/core/api-server/api/rest-api/swagger.json @@ -3,7 +3,7 @@ "info": { "title": "HKube API", "description": "HKube RESTful API", - "version": "2.2.5", + "version": "2.2.6", "contact": { "email": "hkube.dev@gmail.com" }, @@ -12385,6 +12385,326 @@ } } }, + "/queue/preferred": { + "get": { + "tags": [ + "Preferred" + ], + "summary": "Get preferred", + "description": "Get preferred", + "responses": { + "200": { + "description": "Get preferred", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Preferred" + ], + "summary": "Add to preferred", + "description": "Add to preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/queue/preferred/deletes": { + "post": { + "tags": [ + "Preferred" + ], + "summary": "Remover from preferred", + "description": "Remover from preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/builds/status/{buildId}": { "get": { "tags": [ @@ -17896,6 +18216,88 @@ } } }, + "addToPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + }, + "removeFromPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + }, + "preferredQuery": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, "buildId": { "type": "object", "properties": { diff --git a/core/api-server/package-lock.json b/core/api-server/package-lock.json index 71b862fff..123c35806 100644 --- a/core/api-server/package-lock.json +++ b/core/api-server/package-lock.json @@ -494,11 +494,11 @@ } }, "@hkube/db": { - "version": "1.0.44", - "resolved": "https://registry.npmjs.org/@hkube/db/-/db-1.0.44.tgz", - "integrity": "sha512-GmEc7YsqqO2LrdfMSS4E8f7Y5yOUBu2MDqWXlopBPLcXh8d4icMhpyMKPcLabFrEcmkdliqPYf22SFsNjS5f+A==", + "version": "1.0.47", + "resolved": "https://registry.npmjs.org/@hkube/db/-/db-1.0.47.tgz", + "integrity": "sha512-Ibnpm+A4XKTQoPgvzZOLZJrM2vz+8VJAhdmlvzsAOzQ3pF04ns79r27SHWYgblC9GN45K2RIMCawfoQ4xEVJ8w==", "requires": { - "@hkube/consts": "^1.0.31", + "@hkube/consts": "^1.0.40", "lodash.clonedeep": "^4.5.0", "lodash.merge": "^4.6.2", "moment": "^2.29.1", @@ -506,6 +506,11 @@ "performance-now": "^2.1.0" }, "dependencies": { + "@hkube/consts": { + "version": "1.0.44", + "resolved": "https://registry.npmjs.org/@hkube/consts/-/consts-1.0.44.tgz", + "integrity": "sha512-/K3pNQzMtCbA31RmqVKURz5RUJwcuATTQoLQlcfqkRXZbm2wWOEUdT7j3dwESIKyH4+8MOnehX8EgUY0I60wZw==" + }, "moment": { "version": "2.29.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", diff --git a/core/api-server/package.json b/core/api-server/package.json index a2f5db9a2..3d620f307 100644 --- a/core/api-server/package.json +++ b/core/api-server/package.json @@ -7,7 +7,7 @@ "@hkube/config": "^2.0.11", "@hkube/consts": "^1.0.39", "@hkube/dag": "^2.1.25", - "@hkube/db": "^1.0.44", + "@hkube/db": "^1.0.47", "@hkube/etcd": "^5.1.6", "@hkube/logger": "^2.0.2", "@hkube/metrics": "^1.0.42", diff --git a/core/datasources-service/api/rest-api/swagger.json b/core/datasources-service/api/rest-api/swagger.json index e641e8188..ecb5870ed 100644 --- a/core/datasources-service/api/rest-api/swagger.json +++ b/core/datasources-service/api/rest-api/swagger.json @@ -3,7 +3,7 @@ "info": { "title": "HKube API", "description": "HKube RESTful API", - "version": "2.2.5", + "version": "2.2.6", "contact": { "email": "hkube.dev@gmail.com" }, @@ -12385,6 +12385,326 @@ } } }, + "/queue/preferred": { + "get": { + "tags": [ + "Preferred" + ], + "summary": "Get preferred", + "description": "Get preferred", + "responses": { + "200": { + "description": "Get preferred", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Preferred" + ], + "summary": "Add to preferred", + "description": "Add to preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/queue/preferred/deletes": { + "post": { + "tags": [ + "Preferred" + ], + "summary": "Remover from preferred", + "description": "Remover from preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/builds/status/{buildId}": { "get": { "tags": [ @@ -17896,6 +18216,88 @@ } } }, + "addToPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + }, + "removeFromPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + }, + "preferredQuery": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, "buildId": { "type": "object", "properties": { diff --git a/core/gc-service/api/rest-api/swagger.json b/core/gc-service/api/rest-api/swagger.json index e641e8188..ecb5870ed 100644 --- a/core/gc-service/api/rest-api/swagger.json +++ b/core/gc-service/api/rest-api/swagger.json @@ -3,7 +3,7 @@ "info": { "title": "HKube API", "description": "HKube RESTful API", - "version": "2.2.5", + "version": "2.2.6", "contact": { "email": "hkube.dev@gmail.com" }, @@ -12385,6 +12385,326 @@ } } }, + "/queue/preferred": { + "get": { + "tags": [ + "Preferred" + ], + "summary": "Get preferred", + "description": "Get preferred", + "responses": { + "200": { + "description": "Get preferred", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Preferred" + ], + "summary": "Add to preferred", + "description": "Add to preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/queue/preferred/deletes": { + "post": { + "tags": [ + "Preferred" + ], + "summary": "Remover from preferred", + "description": "Remover from preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/builds/status/{buildId}": { "get": { "tags": [ @@ -17896,6 +18216,88 @@ } } }, + "addToPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + }, + "removeFromPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + }, + "preferredQuery": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, "buildId": { "type": "object", "properties": { diff --git a/core/openapi-spec/package.json b/core/openapi-spec/package.json index 96b04b2f0..224f4f262 100644 --- a/core/openapi-spec/package.json +++ b/core/openapi-spec/package.json @@ -7,9 +7,9 @@ "@hkube/rest-server": "^1.0.18" }, "scripts": { - "build-swagger": "node ./swagger-builder.js && echo ../api-server/api/rest-api ../gc-service/api/rest-api ../datasources-service/api/rest-api | xargs -n 1 cp swagger.json", + "build-swagger": "node ./swagger-builder.js && echo ../api-server/api/rest-api ../gc-service/api/rest-api ../datasources-service/api/rest-api ../pipeline-driver-queue/api/rest-api | xargs -n 1 cp swagger.json", "build": "echo No build!" }, "author": "Hkube", "license": "MIT" -} +} \ No newline at end of file diff --git a/core/openapi-spec/swagger.json b/core/openapi-spec/swagger.json index e641e8188..ecb5870ed 100644 --- a/core/openapi-spec/swagger.json +++ b/core/openapi-spec/swagger.json @@ -3,7 +3,7 @@ "info": { "title": "HKube API", "description": "HKube RESTful API", - "version": "2.2.5", + "version": "2.2.6", "contact": { "email": "hkube.dev@gmail.com" }, @@ -12385,6 +12385,326 @@ } } }, + "/queue/preferred": { + "get": { + "tags": [ + "Preferred" + ], + "summary": "Get preferred", + "description": "Get preferred", + "responses": { + "200": { + "description": "Get preferred", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Preferred" + ], + "summary": "Add to preferred", + "description": "Add to preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/queue/preferred/deletes": { + "post": { + "tags": [ + "Preferred" + ], + "summary": "Remover from preferred", + "description": "Remover from preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, "/builds/status/{buildId}": { "get": { "tags": [ @@ -17896,6 +18216,88 @@ } } }, + "addToPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + }, + "removeFromPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + }, + "preferredQuery": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, "buildId": { "type": "object", "properties": { diff --git a/core/openapi-spec/swagger/definitions/index.yaml b/core/openapi-spec/swagger/definitions/index.yaml index 44e456e59..a8422c408 100644 --- a/core/openapi-spec/swagger/definitions/index.yaml +++ b/core/openapi-spec/swagger/definitions/index.yaml @@ -46,6 +46,14 @@ listBoardResponse: metrics: $ref: ./boards/metrics.yaml +#preferred +addToPreferredRequest: + $ref: ./preferred/addToPreferredRequest.yaml +removeFromPreferredRequest: + $ref: ./preferred/removeFromPreferredRequest.yaml +preferredQuery: + $ref: ./preferred/preferredQuery.yaml + # builds buildId: $ref: ./builds/buildId.yaml diff --git a/core/openapi-spec/swagger/definitions/preferred/addToPreferredRequest.yaml b/core/openapi-spec/swagger/definitions/preferred/addToPreferredRequest.yaml new file mode 100644 index 000000000..81ef58bfa --- /dev/null +++ b/core/openapi-spec/swagger/definitions/preferred/addToPreferredRequest.yaml @@ -0,0 +1,18 @@ +type: object +properties: + jobs: + type: array + items: + $ref: "#/components/schemas/jobId" + position: + type: string + enum: + - first + - last + - before + - after + query: + $ref: "#/components/schemas/preferredQuery" +required: + - jobs + - position \ No newline at end of file diff --git a/core/openapi-spec/swagger/definitions/preferred/preferredQuery.yaml b/core/openapi-spec/swagger/definitions/preferred/preferredQuery.yaml new file mode 100644 index 000000000..7977bb10b --- /dev/null +++ b/core/openapi-spec/swagger/definitions/preferred/preferredQuery.yaml @@ -0,0 +1,8 @@ +type: object +properties: + tag: + $ref: "#/components/schemas/tag" + jobId: + $ref: "#/components/schemas/jobId" + pipeline: + $ref: "#/components/schemas/pipelineName" diff --git a/core/openapi-spec/swagger/definitions/preferred/removeFromPreferredRequest.yaml b/core/openapi-spec/swagger/definitions/preferred/removeFromPreferredRequest.yaml new file mode 100644 index 000000000..06b9a2a7c --- /dev/null +++ b/core/openapi-spec/swagger/definitions/preferred/removeFromPreferredRequest.yaml @@ -0,0 +1,8 @@ +type: object +properties: + jobs: + type: array + items: + $ref: "#/components/schemas/jobId" +required: + - jobs \ No newline at end of file diff --git a/core/openapi-spec/swagger/paths/index.yaml b/core/openapi-spec/swagger/paths/index.yaml index 516c5d2b7..2bd668ac7 100644 --- a/core/openapi-spec/swagger/paths/index.yaml +++ b/core/openapi-spec/swagger/paths/index.yaml @@ -106,6 +106,12 @@ /boards/optunas/: $ref: ./boards/optunaboard_list.yaml +# preferred +/queue/preferred: + $ref: ./preferred/preferred.yaml +/queue/preferred/deletes: + $ref: ./preferred/deletes.yaml + # builds /builds/status/{buildId}: $ref: ./builds/status.yaml diff --git a/core/openapi-spec/swagger/paths/preferred/deletes.yaml b/core/openapi-spec/swagger/paths/preferred/deletes.yaml new file mode 100644 index 000000000..a02621296 --- /dev/null +++ b/core/openapi-spec/swagger/paths/preferred/deletes.yaml @@ -0,0 +1,30 @@ +post: + tags: + - Preferred + summary: Remover from preferred + description: Remover from preferred + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/removeFromPreferredRequest" + required: true + responses: + "200": + description: start board response + content: + application/json: + schema: + $ref: "#/components/schemas/removeFromPreferredRequest" + "400": + description: bad request + content: + application/json: + schema: + $ref: "#/components/schemas/error" + default: + description: Unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/error" \ No newline at end of file diff --git a/core/openapi-spec/swagger/paths/preferred/preferred.yaml b/core/openapi-spec/swagger/paths/preferred/preferred.yaml new file mode 100644 index 000000000..0f1d7a1ee --- /dev/null +++ b/core/openapi-spec/swagger/paths/preferred/preferred.yaml @@ -0,0 +1,56 @@ +get: + tags: + - Preferred + summary: Get preferred + description: Get preferred + responses: + "200": + description: Get preferred + content: + application/json: + schema: + $ref: "#/components/schemas/searchJobsResponse" + "400": + description: Bad request + content: + application/json: + schema: + $ref: "#/components/schemas/error" + default: + description: Unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/error" +post: + tags: + - Preferred + summary: Add to preferred + description: Add to preferred + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/addToPreferredRequest" + required: true + responses: + "200": + description: start board response + content: + application/json: + schema: + $ref: "#/components/schemas/addToPreferredRequest" + "400": + description: bad request + content: + application/json: + schema: + $ref: "#/components/schemas/error" + default: + description: Unexpected error + content: + application/json: + schema: + $ref: "#/components/schemas/error" + + diff --git a/core/pipeline-driver-queue/api/rest-api/app-server.js b/core/pipeline-driver-queue/api/rest-api/app-server.js new file mode 100644 index 000000000..828ef995f --- /dev/null +++ b/core/pipeline-driver-queue/api/rest-api/app-server.js @@ -0,0 +1,51 @@ +const fse = require('fs-extra'); +const path = require('path'); +const RestServer = require('@hkube/rest-server'); +const log = require('@hkube/logger').GetLogFromContainer(); +const component = require('../../lib/consts/component-name').REST_API; +const rest = new RestServer(); +const routeLogBlacklist = ['/metrics']; +const validator = require('../../lib/validation'); + +class AppServer { + async init(options) { + rest.on('error', (data) => { + const error = data.error || data.message || {}; + const status = data.status || data.code; + log.info(`status=${status}, message=${error}`, { component }); + }); + + const { port, prefix, bodySizeLimit, poweredBy } = options.rest; + + const routes = []; + const routers = await fse.readdir(path.join(__dirname, 'routes')); + routers.forEach((r) => { + const file = path.basename(r, '.js'); + routes.push({ + route: path.join('/', prefix, file), + router: require('./' + path.join('routes', file))() // eslint-disable-line + }); + }); + + const opt = { + routes, + bodySizeLimit, + poweredBy, + port: parseInt(port, 10), + name: options.serviceName, + logger: { + filterRoutes: routeLogBlacklist, + onResponse: (data) => { + const { method, url, status, duration } = data; + log.info(`${method}:${url} ${status} ${duration}ms`, { component, route: url, httpStatus: status }); + } + } + }; + const data = await rest.start(opt); + log.info(`🚀 ${data.message}`, { component }); + const swagger = await fse.readJSON('api/rest-api/swagger.json'); + validator.init(swagger.components.schemas); + } +} + +module.exports = new AppServer(); diff --git a/core/pipeline-driver-queue/api/rest-api/routes/preferred.js b/core/pipeline-driver-queue/api/rest-api/routes/preferred.js new file mode 100644 index 000000000..ee08849b1 --- /dev/null +++ b/core/pipeline-driver-queue/api/rest-api/routes/preferred.js @@ -0,0 +1,28 @@ +const RestServer = require('@hkube/rest-server'); +const preferredService = require('../../../lib/service/preferred-jobs'); + +const routes = () => { + const router = RestServer.router(); + + router.get('/', (req, res) => { + const response = preferredService.getPreferredJobsList(); + res.json(response); + }); + router.post('/deletes/', async (req, res) => { + const { jobs } = req.body; + const deleted = await preferredService.deletePreferredJobs(jobs); + res.json(deleted); + }); + router.post('/', async (req, res) => { + const { jobs, query, position } = req.body; + const added = await preferredService.addPreferredJobs({ + query, + position, + jobs + }); + res.json(added); + }); + return router; +}; + +module.exports = routes; diff --git a/core/pipeline-driver-queue/api/rest-api/swagger.json b/core/pipeline-driver-queue/api/rest-api/swagger.json new file mode 100644 index 000000000..ecb5870ed --- /dev/null +++ b/core/pipeline-driver-queue/api/rest-api/swagger.json @@ -0,0 +1,21051 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "HKube API", + "description": "HKube RESTful API", + "version": "2.2.6", + "contact": { + "email": "hkube.dev@gmail.com" + }, + "license": { + "name": "MIT", + "url": "https://opensource.org/licenses/MIT" + } + }, + "tags": [ + { + "name": "Execution", + "description": "Execution pipelines as raw or stored" + }, + { + "name": "Graph", + "description": "Query the pipeline graph" + }, + { + "name": "Builds", + "description": "status, rerun and stop builds" + }, + { + "name": "Versions", + "description": "Versioning for algorithms" + }, + { + "name": "Cron", + "description": "Get pipelines cron result and status" + }, + { + "name": "DataSource", + "description": "DataSources and their related files" + }, + { + "name": "GC", + "description": "Hkube garbage collector operations" + }, + { + "name": "Pipelines", + "description": "Get pipelines status and result" + }, + { + "name": "Pipeline Readme", + "description": "Store readme for pipelines" + }, + { + "name": "Algorithm Readme", + "description": "Store readme for algorithms" + }, + { + "name": "Storage", + "description": "Query the storage" + }, + { + "name": "StoreAlgorithms", + "description": "Store new algorithms, update, get and delete" + }, + { + "name": "StorePipelines", + "description": "Store new pipelines, update, get and delete" + }, + { + "name": "Webhooks", + "description": "Webhooks status and result" + } + ], + "externalDocs": { + "description": "Find out more about Hkube", + "url": "http://hkube.io" + }, + "servers": [], + "paths": { + "/exec/raw": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Run raw pipeline", + "description": "Start pipeline execution with raw input, returns jobId that can be used as a reference for the pipeline to do actions such as results/status/stop, etc.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "an object representing all information needed for pipeline execution", + "required": true + }, + "responses": { + "200": { + "description": "pipeline execution data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/stored": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Run stored pipeline", + "description": "Start pipeline execution when the name of the pipeline is known, all parameters in this action will be merged with the stored pipeline and then returns jobId that can be used as a reference for the pipeline to do actions such as results/status/stop, etc.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "an object representing all information needed for stored pipeline execution", + "required": true + }, + "responses": { + "200": { + "description": "pipeline execution data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/caching": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Run job from a specific node", + "description": "the api allows you to run job from current node and getting the data of the predecessors from caching of the previous run", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "debug": { + "type": "boolean", + "description": "Run as debug" + } + }, + "additionalProperties": false, + "required": [ + "jobId", + "nodeName" + ] + } + } + }, + "description": "an object representing all information needed for stored pipeline execution", + "required": true + }, + "responses": { + "200": { + "description": "pipeline execution data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "node Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/algorithm": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Run algorithm", + "description": "Start execution for single algorithm.", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "debug": { + "type": "boolean", + "description": "Is debug algorithm", + "default": false + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + } + } + }, + "description": "an object representing all information needed for algorithm execution request", + "required": true + }, + "responses": { + "200": { + "description": "pipeline execution data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/rerun": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Rerun pipeline", + "description": "Rerun pipeline", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "additionalProperties": false, + "required": [ + "jobId" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "pipeline execution data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/stop": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Stop pipeline", + "description": "Call to stop pipeline execution", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "reason": { + "type": "string", + "default": "requested by user", + "description": "reason for stop pipeline" + } + }, + "required": [ + "jobId" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "the OK string", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/pause": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Pause pipeline", + "description": "Call to pause pipeline execution", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "the OK string", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/resume": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Resume pipeline", + "description": "Call to resume pipeline execution", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "the OK string", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/search": { + "post": { + "tags": [ + "Execution" + ], + "summary": "Search jobs", + "description": "Search jobs", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "experimentName": { + "type": "string" + }, + "pipelineName": { + "type": "string" + }, + "pipelineType": { + "type": "string" + }, + "algorithmName": { + "type": "string" + }, + "pipelineStatus": { + "type": "string" + }, + "datesRange": { + "type": "object", + "properties": { + "from": { + "type": "string", + "example": "2021-03-11T14:30:00" + }, + "to": { + "type": "string", + "example": "2021-03-11T14:30:00" + } + } + } + } + }, + "cursor": { + "type": "string", + "description": "get the next results starting from this cursor" + }, + "pageNum": { + "type": "integer", + "description": "get the next results starting from this page number" + }, + "sort": { + "type": "string", + "enum": [ + "asc", + "desc" + ], + "default": "desc" + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 100, + "example": 100, + "description": "limit the result size" + }, + "fields": { + "type": "object", + "description": "select which fields to return" + }, + "exists": { + "type": "object", + "description": "only return records that has fields" + } + }, + "example": { + "query": { + "datesRange": { + "from": "2021-05-22T15:00:00", + "to": "2021-06-22T16:00:00" + }, + "experimentName": "main", + "pipelineName": "simple", + "pipelineType": "stored", + "algorithmName": "green-alg", + "pipelineStatus": "completed" + }, + "sort": "desc", + "pageNum": 1, + "limit": 2, + "fields": { + "jobId": true + } + } + } + } + }, + "description": "jobs search specification", + "required": true + }, + "responses": { + "200": { + "description": "search jobs response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "get": { + "tags": [ + "Execution" + ], + "summary": "Search jobs", + "description": "Search jobs", + "parameters": [ + { + "name": "experimentName", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineName", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineType", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "algorithmName", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineStatus", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "example": "2021-05-22T15:00:00", + "schema": { + "type": "string" + } + }, + { + "name": "to", + "example": "2021-06-22T16:00:00", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "cursor", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "page", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "example": "asc,desc", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "fields", + "in": "query", + "example": "result:false, pipeline:false", + "schema": { + "type": "string" + } + }, + { + "name": "exists", + "in": "query", + "example": "result:true, pipeline:false", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "search jobs response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/pipelines/{jobId}": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Get pipeline data", + "description": "Returns the executed pipeline data", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing pipeline execution", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "pipeline data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/pipeline/list": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Get current running pipelines", + "description": "Returns the current running pipelines data", + "responses": { + "200": { + "description": "pipeline data", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/status/{jobId}": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Get pipeline status", + "description": "Returns a status for the current pipeline.", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing pipeline execution", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "pipeline result", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/results/{jobId}": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Get pipeline result", + "description": "returns result for the execution of a specific pipeline run. if called before result is determined - returns error.", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing workflow execution - is given in response to calling pipeline run method .", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/exec/tree/{jobId}": { + "get": { + "tags": [ + "Execution" + ], + "summary": "Pipeline execution tree", + "description": "Returns a tree of pipelines, usually a triggered pipelines", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing pipeline execution", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "pipeline tree", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Pipeline name" + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/pipelines/results": { + "get": { + "tags": [ + "Pipelines" + ], + "summary": "Get stored pipeline results", + "description": "return results of stored pipelines by given pipeline name", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "pipeline name", + "schema": { + "type": "string" + } + }, + { + "name": "experimentName", + "in": "query", + "description": "experiment name", + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipelines Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/pipelines/status": { + "get": { + "tags": [ + "Pipelines" + ], + "summary": "Get pipeline status", + "description": "return status of pipelines by given pipeline name", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "pipeline name", + "schema": { + "type": "string" + } + }, + { + "name": "experimentName", + "in": "query", + "description": "experiment name", + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipelines Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/pipelines/triggers/tree": { + "get": { + "tags": [ + "Pipelines" + ], + "summary": "Get pipelines triggers", + "description": "Get tree of pipelines triggers by given name or for all pipelines", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "pipeline name", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Pipeline name" + }, + "children": { + "type": "array", + "items": {} + } + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipelines Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/cron/results": { + "get": { + "tags": [ + "Cron" + ], + "summary": "Get pipeline cron result", + "description": "return cron results by given pipeline name", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "pipeline name", + "schema": { + "type": "string" + } + }, + { + "name": "experimentName", + "in": "query", + "description": "experiment name", + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "results Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/cron/status": { + "get": { + "tags": [ + "Cron" + ], + "summary": "Get pipeline cron status", + "description": "return cron status by given pipeline name", + "parameters": [ + { + "name": "name", + "in": "query", + "description": "pipeline name", + "schema": { + "type": "string" + } + }, + { + "name": "experimentName", + "in": "query", + "description": "experiment name", + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "status Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/cron/list": { + "get": { + "tags": [ + "Cron" + ], + "summary": "Get cron list", + "description": "return cron list of all stored pipeline", + "parameters": [ + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "cron Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/cron/start": { + "post": { + "tags": [ + "Cron" + ], + "summary": "Start cron", + "description": "Start cron by given pipeline name", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + } + }, + "required": [ + "name" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/cron/stop": { + "post": { + "tags": [ + "Cron" + ], + "summary": "Stop cron", + "description": "Stop cron by given pipeline name", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + } + }, + "required": [ + "name" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/datasource": { + "get": { + "tags": [ + "DataSource" + ], + "summary": "Fetch all dataSources meta data", + "responses": { + "200": { + "description": "A collection of dataSources", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "filesCount": { + "type": "number" + }, + "avgFileSize": { + "type": "string" + }, + "totalSize": { + "type": "number" + }, + "fileTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "DataSource" + ], + "summary": "Create a new dataSource and upload a file", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "string", + "description": "Data file (file types: *)", + "format": "binary" + } + }, + "storage": { + "type": "object", + "description": "Contains the credentials and storage information of \nan S3 bucket, If kind is set to \"internal\" \nno other parameters are required\n", + "oneOf": [ + { + "properties": { + "kind": { + "type": "string", + "enum": [ + "S3" + ] + }, + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + }, + "endpoint": { + "type": "string", + "minLength": 1, + "format": "url" + }, + "bucketName": { + "type": "string" + } + }, + "required": [ + "kind", + "accessKeyId", + "secretAccessKey", + "endpoint", + "bucketName" + ] + }, + { + "properties": { + "kind": { + "type": "string", + "enum": [ + "internal" + ] + } + }, + "required": [ + "kind" + ] + } + ] + }, + "git": { + "type": "object", + "description": "Contains the credentials and Git information,\nIf kind is set to \"internal\" \nno other parameters are required\n", + "oneOf": [ + { + "properties": { + "repositoryUrl": { + "description": "provide a url to an empty git repository", + "type": "string", + "minLength": 1, + "format": "git-url" + }, + "token": { + "type": "string" + }, + "tokenName": { + "description": "the token name is *required* when using a *gitlab* account", + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ] + } + }, + "required": [ + "repositoryUrl", + "token", + "kind" + ] + }, + { + "properties": { + "kind": { + "type": "string", + "enum": [ + "internal" + ] + } + }, + "required": [ + "kind" + ] + } + ] + } + }, + "required": [ + "name", + "files", + "storage", + "git" + ] + } + } + }, + "description": "DataSource name and file to upload", + "required": true + }, + "responses": { + "201": { + "description": "created dataSource", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "filesCount": { + "type": "number" + }, + "avgFileSize": { + "type": "string" + }, + "totalSize": { + "type": "number" + }, + "fileTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + } + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/datasource/{name}": { + "get": { + "tags": [ + "DataSource" + ], + "summary": "Fetch a single dataSource by name", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "description": "request a specific version, if not stated the latest is returned", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A datasource item", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + }, + { + "type": "object", + "properties": { + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + ] + } + } + } + } + } + }, + "post": { + "tags": [ + "DataSource" + ], + "summary": "upload a new file to a given datasource", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "files": { + "type": "array", + "items": { + "type": "string", + "description": "Data file (file types: *)", + "format": "binary" + } + }, + "droppedFileIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + }, + "anyOf": [ + { + "required": [ + "versionDescription", + "filesDropped" + ] + }, + { + "required": [ + "versionDescription", + "files" + ] + } + ] + } + } + }, + "description": "DataSource name and file to upload", + "required": true + }, + "responses": { + "201": { + "description": "new datasource version", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "filesCount": { + "type": "number" + }, + "avgFileSize": { + "type": "string" + }, + "totalSize": { + "type": "number" + }, + "fileTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + } + ] + } + } + } + } + } + }, + "delete": { + "tags": [ + "DataSource" + ], + "summary": "delete a datasource by name", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "successfully deleted the datasource", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "deleted": { + "type": "string", + "description": "a delete acknowledgement" + } + } + } + } + } + } + } + } + }, + "/datasource/{name}/versions": { + "get": { + "tags": [ + "DataSource" + ], + "summary": "list a all the versions of a given dataSource name", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A datasource item", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "commitHash": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "/datasource/{name}/sync": { + "post": { + "tags": [ + "DataSource" + ], + "summary": "sync a version from git to the database", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "DataSource name", + "required": true + }, + "responses": { + "201": { + "description": "new datasource version", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "filesCount": { + "type": "number" + }, + "avgFileSize": { + "type": "string" + }, + "totalSize": { + "type": "number" + }, + "fileTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + } + ] + } + } + } + } + } + } + }, + "/datasource/id/{id}/snapshot": { + "post": { + "tags": [ + "DataSource" + ], + "summary": "create a new snapshot for a given datasource version", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "A datasource id (indicating a version)", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + } + } + } + }, + "responses": { + "201": { + "description": "A newly created snapshot", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 25 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "filteredFilesList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "droppedFiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + } + } + ] + } + } + } + } + } + } + }, + "/datasource/id/{id}/snapshot/preview": { + "post": { + "tags": [ + "DataSource" + ], + "summary": "post a query for *preview only*,\nthe results are NOT persisted to the database\n", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "A datasource id (indicating a version)", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "query" + ] + } + } + } + }, + "responses": { + "201": { + "description": "a list of files", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + } + } + } + }, + "/datasource/{name}/snapshot": { + "post": { + "tags": [ + "DataSource" + ], + "summary": "create a new snapshot for a given datasource version", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + } + } + } + }, + "responses": { + "201": { + "description": "A newly created snapshot", + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 25 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "filteredFilesList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "droppedFiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + } + } + ] + } + } + } + } + } + }, + "get": { + "tags": [ + "DataSource" + ], + "summary": "fetch all snapshots for a given datasource", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "A collection of snapshots", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "allOf": [ + { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 25 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "filteredFilesList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "droppedFiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + } + } + ] + } + } + } + } + } + } + } + }, + "/datasource/{name}/credentials": { + "patch": { + "tags": [ + "DataSource" + ], + "summary": "update the credentials for the git repository and the storage\nupdates throughout the entire version history of a dataSource by its name\n", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "git": { + "type": "object", + "properties": { + "token": { + "type": "string" + }, + "tokenName": { + "type": "string" + } + }, + "required": [ + "token" + ] + }, + "storage": { + "type": "object", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + }, + "required": [ + "accessKeyId", + "secretAccessKey" + ] + } + } + } + } + }, + "description": "New credentials to update" + }, + "responses": { + "201": { + "description": "number of versions updated with the new credentials", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "updatedCount": { + "type": "number" + } + } + } + } + } + } + } + } + }, + "/datasource/{name}/snapshot/{snapshotName}": { + "get": { + "tags": [ + "DataSource" + ], + "summary": "fetch snapshots for a given datasource", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "snapshotName", + "in": "path", + "description": "A Snapshot name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "resolve", + "in": "query", + "description": "return the entire dataSource with the snapshot", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "A collection of snapshots", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + } + }, + "snapshot": { + "allOf": [ + { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 25 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "filteredFilesList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "droppedFiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + } + } + ] + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + } + } + } + ] + } + } + } + } + } + } + }, + "/datasource/id/{id}/download": { + "post": { + "tags": [ + "DataSource" + ], + "summary": "create download link", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "A datasource id (indicating a version)", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "fileIds": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "description": "a list of file ids to include in the generated zip file", + "required": true + }, + "responses": { + "201": { + "description": "A Zip file", + "content": { + "application/octet-stream": { + "schema": { + "type": "object", + "properties": { + "href": { + "type": "string" + } + } + } + } + } + } + } + }, + "get": { + "tags": [ + "DataSource" + ], + "summary": "download an archive of the selected files", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "A datasource id (indicating a version)", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Unique identifier representing a downloadId", + "minLength": 4, + "maxLength": 4, + "format": "download-id" + } + } + ], + "responses": { + "200": { + "description": "A Zip file", + "content": { + "application/octet-stream": { + "schema": { + "type": "string", + "format": "binary" + } + } + } + } + } + } + }, + "/datasource/validate": { + "get": { + "tags": [ + "DataSource" + ], + "summary": "Fetch all dataSources meta data", + "parameters": [ + { + "name": "datasource_name", + "in": "query", + "description": "a datasource name", + "schema": { + "type": "string" + } + }, + { + "name": "snapshot_name", + "in": "query", + "description": "A Snapshot name", + "schema": { + "type": "string" + } + }, + { + "name": "id", + "in": "query", + "description": "request a specific version, if not stated the latest is returned", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "ensure a datasource exists by name, version or snapshot", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "exists": { + "type": "boolean" + } + } + } + } + } + } + } + } + }, + "/store/algorithms/{name}": { + "get": { + "tags": [ + "StoreAlgorithms" + ], + "summary": "Get algorithm", + "description": "get specific algorithm from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "StoreAlgorithms" + ], + "summary": "Delete algorithm", + "description": "Delete existing algorithm from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name to get from the store", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "force", + "in": "query", + "description": "should also delete all related data, e.g. pipelines, builds, versions.", + "required": false, + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/store/algorithms": { + "get": { + "tags": [ + "StoreAlgorithms" + ], + "summary": "Get all algorithms", + "description": "Get all available algorithms from store", + "parameters": [ + { + "name": "sort", + "in": "query", + "description": "sort by", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "StoreAlgorithms" + ], + "summary": "Insert algorithm", + "description": "Insert new algorithm to store if not exists", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "algorithm descriptor to be added to the store", + "required": true + }, + "responses": { + "201": { + "description": "algorithm created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "pipeline conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "put": { + "tags": [ + "StoreAlgorithms" + ], + "summary": "Update algorithm", + "description": "Update existing algorithm", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "algorithm descriptor to be added to the store", + "required": true + }, + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/store/algorithms/apply": { + "post": { + "tags": [ + "StoreAlgorithms" + ], + "summary": "Apply algorithm", + "description": "create or update algorithm, build id will be returned if build was triggered", + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "payload": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + }, + "options": { + "type": "object", + "properties": { + "forceUpdate": { + "type": "boolean", + "description": "should set the newly created version as current", + "default": false + }, + "forceBuild": { + "type": "boolean", + "description": "should force new build", + "default": false + } + } + }, + "file": { + "type": "string", + "format": "binary", + "description": "the algorithm source code (file types: zip, tar.gz)" + } + } + } + } + }, + "description": "algorithm descriptor to be added to the store", + "required": true + }, + "responses": { + "200": { + "description": "algorithm applied", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "buildId": { + "type": "string", + "description": "Pipeline name" + }, + "algorithm": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + }, + "status": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/store/pipelines/{name}": { + "get": { + "tags": [ + "StorePipelines" + ], + "summary": "Get pipeline", + "description": "Get specific pipeline from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "pipeline name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "StorePipelines" + ], + "summary": "Delete pipeline", + "description": "Delete existing pipeline from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "pipeline name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/store/pipelines": { + "get": { + "tags": [ + "StorePipelines" + ], + "summary": "Get all pipelines", + "description": "Get all pipelines from store", + "parameters": [ + { + "name": "sort", + "in": "query", + "description": "sort by", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "StorePipelines" + ], + "summary": "Insert pipeline", + "description": "Insert new pipeline to store if not exist", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "an object representing all information needed for pipeline execution", + "required": true + }, + "responses": { + "201": { + "description": "pipeline created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "pipeline conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "put": { + "tags": [ + "StorePipelines" + ], + "summary": "Update pipeline", + "description": "Update existing pipeline", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + }, + "description": "an object representing all information needed for pipeline execution", + "required": true + }, + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/webhooks/status/{jobId}": { + "get": { + "tags": [ + "Webhooks" + ], + "summary": "Webhook status", + "description": "Returns a webhook status for specific pipeline", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing pipeline execution", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "pipeline result", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "url": { + "type": "string" + }, + "pipelineStatus": { + "type": "string" + }, + "responseStatus": { + "type": "string" + }, + "httpResponse": { + "type": "object", + "properties": { + "statusCode": { + "type": "string" + }, + "statusMessage": { + "type": "string" + } + } + }, + "status": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/webhooks/results/{jobId}": { + "get": { + "tags": [ + "Webhooks" + ], + "summary": "Webhook result", + "description": "Returns a webhook result for specific pipeline", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing pipeline execution", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "url": { + "type": "string" + }, + "pipelineStatus": { + "type": "string" + }, + "responseStatus": { + "type": "string" + }, + "httpResponse": { + "type": "object", + "properties": { + "statusCode": { + "type": "string" + }, + "statusMessage": { + "type": "string" + } + } + }, + "status": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "webhook Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/webhooks/list/{jobId}": { + "get": { + "tags": [ + "Webhooks" + ], + "summary": "Webhook list", + "description": "Returns all webhooks for specific pipeline", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing pipeline execution", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "pipeline result", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "url": { + "type": "string" + }, + "pipelineStatus": { + "type": "string" + }, + "responseStatus": { + "type": "string" + }, + "httpResponse": { + "type": "object", + "properties": { + "statusCode": { + "type": "string" + }, + "statusMessage": { + "type": "string" + } + } + }, + "status": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "webhook Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/boards/tensors": { + "post": { + "tags": [ + "Boards" + ], + "summary": "Start board", + "description": "start board", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "pipelineName": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "nodeName": { + "type": "string" + }, + "jobId": { + "type": "string" + }, + "taskId": { + "type": "string" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "id" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/boards/tensors/{id}": { + "get": { + "tags": [ + "Boards" + ], + "summary": "Board details", + "description": "Board details", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Unique identifier representing board", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "get board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "relativeUrl": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "board Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "Boards" + ], + "summary": "Stop board", + "description": "Stop board", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Unique identifier representing board", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "delete board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "board Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/boards/tensors/": { + "get": { + "tags": [ + "Boards" + ], + "summary": "Board list", + "description": "Board list", + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "array", + "description": "Array of boards", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "relativeUrl": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/boards/optunas": { + "post": { + "tags": [ + "Boards" + ], + "summary": "Start board", + "description": "start board", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "id" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/boards/optunas/{id}": { + "get": { + "tags": [ + "Boards" + ], + "summary": "Board details", + "description": "Board details", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Unique identifier representing board", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "get board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "relativeUrl": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "board Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "Boards" + ], + "summary": "Stop board", + "description": "Stop board", + "parameters": [ + { + "name": "id", + "in": "path", + "description": "Unique identifier representing board", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "delete board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "board Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/boards/optunas/": { + "get": { + "tags": [ + "Boards" + ], + "summary": "Board list", + "description": "Board list", + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "array", + "description": "Array of boards", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "relativeUrl": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/queue/preferred": { + "get": { + "tags": [ + "Preferred" + ], + "summary": "Get preferred", + "description": "Get preferred", + "responses": { + "200": { + "description": "Get preferred", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Preferred" + ], + "summary": "Add to preferred", + "description": "Add to preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/queue/preferred/deletes": { + "post": { + "tags": [ + "Preferred" + ], + "summary": "Remover from preferred", + "description": "Remover from preferred", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "start board response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/builds/status/{buildId}": { + "get": { + "tags": [ + "Builds" + ], + "summary": "Get build status", + "description": "returns the build status by specific build id", + "parameters": [ + { + "name": "buildId", + "in": "path", + "description": "Unique identifier representing build for algorithm", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "algorithm apply response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "build Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/builds/list/{name}": { + "get": { + "tags": [ + "Builds" + ], + "summary": "Get build list", + "description": "returns all builds that related to algorithm", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "algorithm apply list response", + "content": { + "application/json": { + "schema": { + "items": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "build Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/builds/stop": { + "post": { + "tags": [ + "Builds" + ], + "summary": "Stop build", + "description": "stop build", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "buildId": { + "type": "string", + "description": "Unique identifier representing build id" + } + }, + "required": [ + "buildId" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "stop build response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "build Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/builds/rerun": { + "post": { + "tags": [ + "Builds" + ], + "summary": "Rerun build", + "description": "rerun build", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "buildId": { + "type": "string", + "description": "Unique identifier representing build id" + } + }, + "required": [ + "buildId" + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "stop build response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "build Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/builds/webhook/github": { + "post": { + "tags": [ + "Builds" + ], + "summary": "Insert webhook for automating build after push", + "description": "webhook for automating build algorithm directly from github", + "requestBody": { + "content": { + "application/x-www-from-urlencoded": { + "schema": { + "type": "object", + "properties": { + "github": { + "type": "object", + "description": "github object which received after push for more information https://developer.github.com/webhooks/" + } + } + } + } + }, + "description": "github webhook data", + "required": true + }, + "responses": { + "201": { + "description": "webhook created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/builds/webhook/gitlab": { + "post": { + "tags": [ + "Builds" + ], + "summary": "Insert webhook for automating build after push", + "description": "webhook for automating build algorithm directly from gitlab", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "gitlab": { + "type": "object", + "description": "gitlab object which received after push for more information https://docs.gitlab.com/ee/user/project/integrations/webhooks.html" + } + } + } + } + }, + "description": "gitlab webhook data", + "required": true + }, + "responses": { + "201": { + "description": "webhook created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/readme/algorithms/{name}": { + "get": { + "tags": [ + "Algorithm Readme" + ], + "summary": "Get stored readme by name", + "description": "Get stored readme by name", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Algorithm Readme" + ], + "summary": "Insert new readme to algorithm", + "description": "Insert new algorithm to store if not exists", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "README.md": { + "type": "string", + "format": "binary", + "description": "the readme.md file name of the file must be README.md" + } + } + } + } + }, + "description": "algorithm readme to be added to the store", + "required": true + }, + "responses": { + "201": { + "description": "algorithm created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "algorithm conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "put": { + "tags": [ + "Algorithm Readme" + ], + "summary": "Update readme to algorithm", + "description": "Update algorithm's store if not exists", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "README.md": { + "type": "string", + "format": "binary", + "description": "the readme.md file name of the file must be README.md" + } + } + } + } + }, + "description": "algorithm readme to be added to the store", + "required": true + }, + "responses": { + "201": { + "description": "algorithm readme updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "algorithm conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "Algorithm Readme" + ], + "summary": "Delete algorithm", + "description": "Delete existing algorithm from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/readme/pipelines/{name}": { + "get": { + "tags": [ + "Pipeline Readme" + ], + "summary": "Get stored readme by name", + "description": "Get stored readme by name", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "pipelines name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Pipeline Readme" + ], + "summary": "Insert new readme to pipelines", + "description": "Insert new pipeline to store if not exists", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "pipeline name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "README.md": { + "type": "string", + "format": "binary", + "description": "the readme.md file name of the file must be README.md" + } + } + } + } + }, + "description": "pipeline readme to be added to the store", + "required": true + }, + "responses": { + "201": { + "description": "pipeline created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "pipeline conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "put": { + "tags": [ + "Pipeline Readme" + ], + "summary": "Update readme to pipeline", + "description": "Update pipeline's store if not exists", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "pipeline name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "properties": { + "README.md": { + "type": "string", + "format": "binary", + "description": "the readme.md file name of the file must be README.md" + } + } + } + } + }, + "description": "pipeline readme to be added to the store", + "required": true + }, + "responses": { + "201": { + "description": "pipeline readme updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "pipeline conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "Pipeline Readme" + ], + "summary": "Delete pipeline", + "description": "Delete existing pipeline from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "pipeline name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/versions/algorithms/{name}": { + "get": { + "tags": [ + "Versions" + ], + "summary": "Get algorithm versions", + "description": "returns list of algorithm versions.", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by asc or desc", + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "description": "limit", + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/versions/algorithms/{name}/{version}": { + "get": { + "tags": [ + "Versions" + ], + "summary": "Get version", + "description": "returns algorithm version", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + } + }, + { + "name": "version", + "in": "path", + "description": "algorithm version", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "Versions" + ], + "summary": "Delete version", + "description": "Delete existing algorithm version.", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "algorithm name", + "required": true, + "schema": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + } + }, + { + "name": "version", + "in": "path", + "description": "algorithm version", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "algorithm Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/versions/algorithms/apply": { + "post": { + "tags": [ + "Versions" + ], + "summary": "Apply algorithm version", + "description": "Change the current algorithm version", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "version": { + "type": "string", + "description": "Unique identifier representing version id", + "minLength": 1, + "maxLength": 10 + }, + "force": { + "type": "boolean", + "description": "should replace or not current algorithm version for running algorithm instances", + "default": false + } + }, + "required": [ + "name", + "version" + ] + } + } + }, + "description": "an object representing all information needed for stored pipeline execution", + "required": true + }, + "responses": { + "200": { + "description": "the OK string", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/versions/algorithms/tag": { + "post": { + "tags": [ + "Versions" + ], + "summary": "Tag algorithm version", + "description": "Tag algorithm version with pinned and tags", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "version": { + "type": "string", + "description": "Unique identifier representing version id", + "minLength": 1, + "maxLength": 10 + }, + "pinned": { + "type": "boolean", + "description": "Should set the version as pinned", + "default": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + } + }, + "required": [ + "name", + "version" + ] + } + } + }, + "description": "an object representing all information needed for stored pipeline execution", + "required": true + }, + "responses": { + "200": { + "description": "the OK string", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/experiment/{name}": { + "get": { + "tags": [ + "Experiment" + ], + "summary": "get experiment details", + "description": "get experiment details", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "experiment name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "describe the experiment" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "delete": { + "tags": [ + "Experiment" + ], + "summary": "Delete experiment", + "description": "Delete existing experiment from store", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "experiment name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "experiment Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/experiment": { + "get": { + "tags": [ + "Experiment" + ], + "summary": "Get experiments list", + "description": "Returns experiments list", + "responses": { + "200": { + "description": "list of experiments", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "describe the experiment" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Experiment" + ], + "summary": "Insert experiment", + "description": "Insert new experiment to store if not exists", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "describe the experiment" + } + } + } + } + }, + "description": "experiment to be added to the store", + "required": true + }, + "responses": { + "201": { + "description": "experiment created", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "pipeline conflict", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gateway": { + "get": { + "tags": [ + "Gateway" + ], + "summary": "get gateways", + "description": "get list of gateways", + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gateway/{name}": { + "get": { + "tags": [ + "Gateway" + ], + "summary": "get gateway details", + "description": "get gateway details", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "gateway name to get from the store", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "pipeline Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gc/clean": { + "post": { + "tags": [ + "GC" + ], + "summary": "Clean all", + "description": "Clean all", + "requestBody": { + "description": "message body", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maxAge": { + "type": "integer", + "description": "max age to clean (minutes)" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "success response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gc/clean/{type}": { + "post": { + "tags": [ + "GC" + ], + "summary": "Clean by type", + "description": "Clean by type", + "parameters": [ + { + "name": "type", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "message body", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maxAge": { + "type": "integer", + "description": "max age to clean (minutes)" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "success response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gc/dryrun": { + "post": { + "tags": [ + "GC" + ], + "summary": "DryRun all", + "description": "DryRun all", + "requestBody": { + "description": "message body", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maxAge": { + "type": "integer", + "description": "max age to clean (minutes)" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "success response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gc/dryrun/{type}": { + "post": { + "tags": [ + "GC" + ], + "summary": "DryRun by type", + "description": "DryRun by type", + "parameters": [ + { + "name": "type", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "description": "message body", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "maxAge": { + "type": "integer", + "description": "max age to clean (minutes)" + } + } + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "success response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gc/status": { + "get": { + "tags": [ + "GC" + ], + "summary": "Get all statuses", + "description": "Get all statuses", + "responses": { + "200": { + "description": "success response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/gc/status/{type}": { + "get": { + "tags": [ + "GC" + ], + "summary": "Get status by type", + "description": "Get status by type", + "parameters": [ + { + "name": "type", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "success response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/graph/raw/{jobId}": { + "get": { + "tags": [ + "Graph" + ], + "summary": "Get graph as raw data", + "description": "returns the graph data by jobId", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "algorithm apply response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "number" + }, + "edges": { + "type": "array", + "description": "Array of edges" + }, + "nodes": { + "type": "array", + "description": "Array of nodes" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "build Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/graph/parsed/{jobId}": { + "get": { + "tags": [ + "Graph" + ], + "summary": "Query the pipeline graph", + "description": "returns the graph data by jobId, node and tasks", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "Unique identifier representing jobId", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "node", + "in": "query", + "description": "the node name", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by status, error or any other property", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by asc or desc", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "description": "the start index", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "to", + "in": "query", + "description": "the end index", + "required": false, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "algorithm apply response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "number" + }, + "edges": { + "type": "array", + "description": "Array of edges" + }, + "nodes": { + "type": "array", + "description": "Array of nodes" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "jobId Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/info": { + "get": { + "tags": [ + "Storage" + ], + "summary": "Get storage info", + "description": "returns the storage info", + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/prefix/types": { + "get": { + "tags": [ + "Storage" + ], + "summary": "Get storage types", + "description": "returns the storage types that managed by Hkube", + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/prefixes/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "Get all prefixes by path", + "description": "returns all available prefixes found in storage according to path", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by status, error or any other property", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by asc or desc", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "description": "the start index", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "to", + "in": "query", + "description": "the start index", + "required": false, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/keys/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "Get all keys by path", + "description": "returns all available keys found in storage according to path", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "sort", + "in": "query", + "description": "sort by status, error or any other property", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "order", + "in": "query", + "description": "order by asc or desc", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "from", + "in": "query", + "description": "the start index", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "to", + "in": "query", + "description": "the end index", + "required": false, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/values/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "Get storage data", + "description": "returns the storage data", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/stream/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "stream data", + "description": "stream data according to path", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/stream/custom/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "stream data", + "description": "stream data according to path", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/download/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "stream data to file", + "description": "download data as file according to path", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/download/custom/{path}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "stream data to file", + "description": "download data as file according to path", + "parameters": [ + { + "name": "path", + "in": "path", + "description": "path to storage", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/storage/download/pipeline/result/{jobId}": { + "get": { + "tags": [ + "Storage" + ], + "summary": "stream data to file", + "description": "download data as file according to path", + "parameters": [ + { + "name": "jobId", + "in": "path", + "description": "the pipeline jobId", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "storage response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + }, + "400": { + "description": "bad request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "path Not Found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/status/version": { + "get": { + "tags": [ + "Status" + ], + "summary": "Get system status", + "description": "Get system status such as version and health", + "responses": { + "200": { + "description": "action summery", + "content": { + "application/json": { + "schema": { + "items": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + } + } + } + } + }, + "default": { + "description": "Unexpected error", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "algorithm": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + }, + "algorithmKind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "algorithmNameObject": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + } + }, + "required": [ + "name" + ] + }, + "algorithmDelete": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "force": { + "type": "boolean", + "default": false + } + }, + "required": [ + "name" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "algorithmMount": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "algorithmEnv": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmFileExt": { + "type": "string", + "enum": [ + "zip", + "gz" + ] + }, + "algorithmVersionId": { + "type": "string", + "description": "Unique identifier representing version id", + "minLength": 1, + "maxLength": 10 + }, + "applyAlgorithmVersion": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "version": { + "type": "string", + "description": "Unique identifier representing version id", + "minLength": 1, + "maxLength": 10 + }, + "force": { + "type": "boolean", + "description": "should replace or not current algorithm version for running algorithm instances", + "default": false + } + }, + "required": [ + "name", + "version" + ] + }, + "algorithmVersionTag": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "version": { + "type": "string", + "description": "Unique identifier representing version id", + "minLength": 1, + "maxLength": 10 + }, + "pinned": { + "type": "boolean", + "description": "Should set the version as pinned", + "default": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + } + }, + "required": [ + "name", + "version" + ] + }, + "algorithmApplyOptions": { + "type": "object", + "properties": { + "forceUpdate": { + "type": "boolean", + "description": "should set the newly created version as current", + "default": false + }, + "forceBuild": { + "type": "boolean", + "description": "should force new build", + "default": false + } + } + }, + "algorithmApplyResponse": { + "type": "object", + "properties": { + "buildId": { + "type": "string", + "description": "Pipeline name" + }, + "algorithm": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "kind": { + "type": "string", + "enum": [ + "batch", + "stream" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + }, + "algorithmImage": { + "type": "string", + "description": "image name as in the docker registry", + "format": "algorithm-image" + }, + "cpu": { + "type": "number", + "default": 0.1, + "minimum": 0, + "description": "algorithm cpu" + }, + "gpu": { + "type": "number", + "minimum": 0, + "description": "algorithm gpu" + }, + "mem": { + "type": "string", + "description": "algorithm memory", + "format": "algorithm-memory", + "default": "256Mi", + "example": "512Mi" + }, + "reservedMemory": { + "type": "string", + "nullable": true, + "description": "Reserved memory for HKube's operations such as in-memory cache, higher value means faster data retrieval and less algorithm memory, lower value means slower data retrieval and more algorithm memory", + "format": "memory", + "example": "512Mi" + }, + "options": { + "type": "object", + "properties": { + "devMode": { + "type": "boolean", + "description": "runs algorithm with mounted sources to allow rapid development cycles" + }, + "pending": { + "type": "boolean", + "description": "pending algorithm", + "default": false + }, + "opengl": { + "type": "boolean", + "description": "should algorithm support XGL context creation (mount X socket)" + } + }, + "default": {} + }, + "version": { + "type": "string", + "description": "Hkube's auto increment semantic versioning", + "readOnly": true + }, + "mounts": { + "type": "array", + "nullable": true, + "description": "a list of volumes to mount into the algorithm", + "items": { + "type": "object", + "properties": { + "pvcName": { + "type": "string", + "format": "algorithm-mount-pvc", + "minLength": 1, + "description": "name of an existing kubernetes pvc (persistent volume claim)" + }, + "path": { + "type": "string", + "format": "path", + "minLength": 1, + "description": "the mount path in the algorithm container" + }, + "volumeType": { + "type": "string", + "description": "Special volume types such as emptyDir", + "enum": [ + "emptyDir", + "configMap" + ] + } + }, + "required": [ + "pvcName", + "path" + ] + }, + "uniqueItems": true, + "example": [] + }, + "gitRepository": { + "type": "object", + "nullable": true, + "properties": { + "url": { + "type": "string", + "description": "a url for the git repository", + "format": "url" + }, + "commit": { + "type": "object", + "description": "commit details", + "properties": { + "id": { + "type": "string", + "description": "commit id" + }, + "timestamp": { + "type": "string", + "description": "commit time" + }, + "message": { + "type": "string", + "description": "commit message" + } + }, + "required": [ + "id" + ] + }, + "branchName": { + "type": "string", + "default": "master", + "description": "the branch name you wish to create a build from" + }, + "tag": { + "type": "string", + "description": "a specific tag which will trigger the build" + }, + "token": { + "type": "string", + "description": "a token which allows hkube's build system to access private repositories more information https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line" + }, + "gitKind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ], + "default": "github" + } + }, + "required": [ + "url" + ] + }, + "entryPoint": { + "type": "string", + "nullable": true + }, + "dependencyInstallCmd": { + "type": "string", + "nullable": true, + "description": "Command to run to install algorithm dependencies. CWD is the algorithm root folder\nDefaults to language specific defaults.\ne.g. For python: pip install -r requirements.txt\n" + }, + "baseImage": { + "type": "string", + "nullable": true, + "description": "Custom docker image to be used as base to the newly built algorithm image" + }, + "minHotWorkers": { + "type": "integer", + "description": "how many live algorithm instances will always run", + "minimum": 0, + "default": 0 + }, + "maxWorkers": { + "type": "integer", + "description": "Maximum number of workers for this algorithm. 0 marks no limit", + "minimum": 0, + "nullable": true + }, + "quotaGuarantee": { + "type": "integer", + "nullable": true, + "description": "The amount of algorithms required to be scheduled first in a case of cluster pressure", + "minimum": 0 + }, + "algorithmEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for algorithm" + }, + "workerEnv": { + "type": "object", + "nullable": true, + "description": "key value environment variables for worker" + }, + "nodeSelector": { + "type": "object", + "nullable": true, + "description": "key value labels for nodes constraint", + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for users" + }, + "annotations": { + "type": "object", + "nullable": true, + "description": "attach key value metadata to algorithm pod, usually for tools and libraries" + }, + "streamKind": { + "enum": [ + "stateless", + "stateful" + ] + }, + "type": { + "type": "string", + "description": "type of algorithm code resource", + "enum": [ + "Git", + "Code", + "Image" + ], + "default": "Image" + }, + "downloadFileExt": { + "type": "string", + "nullable": true, + "description": "the extension name that will be attached to a file when downloading algorithm result" + } + }, + "required": [ + "name" + ] + }, + "status": { + "type": "string" + }, + "timestamp": { + "type": "string" + } + } + }, + "algorithmBuild": { + "type": "object", + "properties": { + "fileExt": { + "type": "string", + "enum": [ + "zip", + "gz" + ] + }, + "env": { + "type": "string", + "enum": [ + "nodejs", + "python", + "java" + ] + } + }, + "required": [ + "env" + ] + }, + "kubernetesValueFrom": { + "type": "string", + "enum": [ + "fieldRef", + "configMapKeyRef", + "resourceFieldRef", + "secretKeyRef" + ] + }, + "createBoardRequest": { + "type": "object", + "properties": { + "pipelineName": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "nodeName": { + "type": "string" + }, + "jobId": { + "type": "string" + }, + "taskId": { + "type": "string" + } + } + }, + "createOptunaboardRequest": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + } + } + }, + "createBoardResponse": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + "getBoardResponse": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "relativeUrl": { + "type": "string" + } + } + }, + "listBoardResponse": { + "type": "array", + "description": "Array of boards", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "relativeUrl": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "addToPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "position": { + "type": "string", + "enum": [ + "first", + "last", + "before", + "after" + ] + }, + "query": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + } + }, + "required": [ + "jobs", + "position" + ] + }, + "removeFromPreferredRequest": { + "type": "object", + "properties": { + "jobs": { + "type": "array", + "items": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "required": [ + "jobs" + ] + }, + "preferredQuery": { + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "pipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + "buildId": { + "type": "object", + "properties": { + "buildId": { + "type": "string", + "description": "Unique identifier representing build id" + } + }, + "required": [ + "buildId" + ] + }, + "buildIdObject": { + "type": "object", + "properties": { + "buildId": { + "type": "string" + } + }, + "required": [ + "buildId" + ] + }, + "defaultResponse": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "defaultResponseWithName": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "error": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "jobIdObject": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + }, + "tag": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + }, + "queryList": { + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "" + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "sort": { + "type": "string", + "default": "desc", + "enum": [ + "asc", + "desc" + ] + }, + "limit": { + "type": "integer", + "default": 1, + "minimum": 1, + "maximum": 100000 + } + }, + "required": [ + "name" + ] + }, + "queryRange": { + "type": "object", + "properties": { + "sort": { + "type": "string" + }, + "order": { + "type": "string", + "default": "desc", + "enum": [ + "asc", + "desc" + ] + }, + "from": { + "type": "integer", + "default": 0, + "minimum": 0 + }, + "to": { + "type": "integer", + "default": 10, + "minimum": 0 + } + } + }, + "cronPattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "cronRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + } + }, + "required": [ + "name" + ] + }, + "dataSource-name": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "dataSource-snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + "dataSource-id": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + "Files": { + "type": "array", + "items": { + "type": "string", + "description": "Data file (file types: *)", + "format": "binary" + } + }, + "FileMeta": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + }, + "Error": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + }, + "DataSourceMeta": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "filesCount": { + "type": "number" + }, + "avgFileSize": { + "type": "string" + }, + "totalSize": { + "type": "number" + }, + "fileTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "DataSource": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + }, + "DataSourceId": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "DataSourceWithMeta": { + "allOf": [ + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "filesCount": { + "type": "number" + }, + "avgFileSize": { + "type": "string" + }, + "totalSize": { + "type": "number" + }, + "fileTypes": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + } + ] + }, + "DataSourceName": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "DataSourceVersion": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "commitHash": { + "type": "string" + } + } + }, + "VersionDescription": { + "type": "string", + "minLength": 1 + }, + "dataSourceExists": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + } + } + } + } + }, + "SnapshotName": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "SnapshotQuery": { + "type": "string", + "minLength": 1 + }, + "SnapshotRequest": { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + }, + "SnapshotCreate": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + } + } + } + }, + "required": [ + "name", + "query", + "dataSource" + ] + }, + "Snapshot": { + "allOf": [ + { + "type": "object", + "properties": { + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + } + } + } + }, + "required": [ + "name", + "query" + ] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 25 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "filteredFilesList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "droppedFiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + } + } + } + } + ] + }, + "ResolvedSnapshot": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 25 + }, + "query": { + "type": "string", + "minLength": 1 + }, + "dataSource": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "commitHash": { + "description": "a commit hash", + "type": "string" + }, + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "versionDescription": { + "description": "a commit message supplied by the user", + "type": "string" + }, + "gitRepositoryUrl": { + "type": "string" + } + }, + "required": [ + "name", + "id" + ] + } + } + }, + "SnapshotPreviewRequest": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "query": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "query", + "id" + ] + }, + "createRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "files": { + "type": "array", + "items": { + "type": "string", + "description": "Data file (file types: *)", + "format": "binary" + } + }, + "storage": { + "type": "object", + "description": "Contains the credentials and storage information of \nan S3 bucket, If kind is set to \"internal\" \nno other parameters are required\n", + "oneOf": [ + { + "properties": { + "kind": { + "type": "string", + "enum": [ + "S3" + ] + }, + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + }, + "endpoint": { + "type": "string", + "minLength": 1, + "format": "url" + }, + "bucketName": { + "type": "string" + } + }, + "required": [ + "kind", + "accessKeyId", + "secretAccessKey", + "endpoint", + "bucketName" + ] + }, + { + "properties": { + "kind": { + "type": "string", + "enum": [ + "internal" + ] + } + }, + "required": [ + "kind" + ] + } + ] + }, + "git": { + "type": "object", + "description": "Contains the credentials and Git information,\nIf kind is set to \"internal\" \nno other parameters are required\n", + "oneOf": [ + { + "properties": { + "repositoryUrl": { + "description": "provide a url to an empty git repository", + "type": "string", + "minLength": 1, + "format": "git-url" + }, + "token": { + "type": "string" + }, + "tokenName": { + "description": "the token name is *required* when using a *gitlab* account", + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "github", + "gitlab" + ] + } + }, + "required": [ + "repositoryUrl", + "token", + "kind" + ] + }, + { + "properties": { + "kind": { + "type": "string", + "enum": [ + "internal" + ] + } + }, + "required": [ + "kind" + ] + } + ] + } + }, + "required": [ + "name", + "files", + "storage", + "git" + ] + }, + "updateRequest": { + "type": "object", + "properties": { + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "files": { + "type": "array", + "items": { + "type": "string", + "description": "Data file (file types: *)", + "format": "binary" + } + }, + "droppedFileIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + } + }, + "anyOf": [ + { + "required": [ + "versionDescription", + "filesDropped" + ] + }, + { + "required": [ + "versionDescription", + "files" + ] + } + ] + }, + "deleteRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + } + }, + "required": [ + "name" + ] + }, + "update": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "versionDescription": { + "type": "string", + "minLength": 1 + }, + "files": { + "type": "object", + "properties": { + "mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "description": { + "type": "string" + }, + "uploadedAt": { + "type": "number" + } + }, + "required": [ + "id", + "name", + "path" + ] + } + }, + "added": { + "type": "array", + "items": { + "type": "string", + "description": "Data file (file types: *)", + "format": "binary" + } + }, + "dropped": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + }, + "syncRequest": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + "CreateDownloadLinkRequest": { + "type": "object", + "properties": { + "dataSourceId": { + "type": "string", + "minLength": 24, + "maxLength": 24 + }, + "fileIds": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + }, + "required": [ + "dataSourceId", + "fileIds" + ] + }, + "DownloadId": { + "type": "string", + "description": "Unique identifier representing a downloadId", + "minLength": 4, + "maxLength": 4, + "format": "download-id" + }, + "Credentials": { + "type": "object", + "properties": { + "git": { + "type": "object", + "properties": { + "token": { + "type": "string" + }, + "tokenName": { + "type": "string" + } + }, + "required": [ + "token" + ] + }, + "storage": { + "type": "object", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + }, + "required": [ + "accessKeyId", + "secretAccessKey" + ] + } + } + }, + "updateCredentials": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific dataSource", + "minLength": 1, + "maxLength": 32, + "format": "dataSource-name" + }, + "credentials": { + "type": "object", + "properties": { + "git": { + "type": "object", + "properties": { + "token": { + "type": "string" + }, + "tokenName": { + "type": "string" + } + }, + "required": [ + "token" + ] + }, + "storage": { + "type": "object", + "properties": { + "accessKeyId": { + "type": "string" + }, + "secretAccessKey": { + "type": "string" + } + }, + "required": [ + "accessKeyId", + "secretAccessKey" + ] + } + } + } + }, + "required": [ + "name", + "credentials" + ] + }, + "URL": { + "type": "string", + "minLength": 1, + "format": "url" + }, + "GitURL": { + "type": "string", + "minLength": 1, + "format": "git-url" + }, + "caching": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "debug": { + "type": "boolean", + "description": "Run as debug" + } + }, + "additionalProperties": false, + "required": [ + "jobId", + "nodeName" + ] + }, + "execTreeNode": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Pipeline name" + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + }, + "execTreeResult": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Pipeline name" + }, + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + } + } + }, + "stopRequest": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "reason": { + "type": "string", + "default": "requested by user", + "description": "reason for stop pipeline" + } + }, + "required": [ + "jobId" + ] + }, + "pauseRequest": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + }, + "resumeRequest": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "required": [ + "jobId" + ] + }, + "execAlgorithmRequest": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "debug": { + "type": "boolean", + "description": "Is debug algorithm", + "default": false + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + "searchJobs": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "experimentName": { + "type": "string" + }, + "pipelineName": { + "type": "string" + }, + "pipelineType": { + "type": "string" + }, + "algorithmName": { + "type": "string" + }, + "pipelineStatus": { + "type": "string" + }, + "datesRange": { + "type": "object", + "properties": { + "from": { + "type": "string", + "example": "2021-03-11T14:30:00" + }, + "to": { + "type": "string", + "example": "2021-03-11T14:30:00" + } + } + } + } + }, + "cursor": { + "type": "string", + "description": "get the next results starting from this cursor" + }, + "pageNum": { + "type": "integer", + "description": "get the next results starting from this page number" + }, + "sort": { + "type": "string", + "enum": [ + "asc", + "desc" + ], + "default": "desc" + }, + "limit": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 100, + "example": 100, + "description": "limit the result size" + }, + "fields": { + "type": "object", + "description": "select which fields to return" + }, + "exists": { + "type": "object", + "description": "only return records that has fields" + } + }, + "example": { + "query": { + "datesRange": { + "from": "2021-05-22T15:00:00", + "to": "2021-06-22T16:00:00" + }, + "experimentName": "main", + "pipelineName": "simple", + "pipelineType": "stored", + "algorithmName": "green-alg", + "pipelineStatus": "completed" + }, + "sort": "desc", + "pageNum": 1, + "limit": 2, + "fields": { + "jobId": true + } + } + }, + "searchJobsResponse": { + "type": "object", + "properties": { + "hits": { + "type": "array", + "description": "the actual result records", + "items": {} + }, + "cursor": { + "type": "string", + "description": "this cursor to the next query can be used for pagination" + }, + "timeTook": { + "type": "string", + "description": "the query completion time took in milliseconds" + } + } + }, + "rerun": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + } + }, + "additionalProperties": false, + "required": [ + "jobId" + ] + }, + "experiment": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "describe the experiment" + } + } + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "gatewayName": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "gateway": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + "output": { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + "hyperparamsTuner": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + "hyperparamsTunerName": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "hyperparamsTunerSpec": { + "type": "object", + "properties": { + "numberOfTrials": { + "type": "number", + "description": "number Of Trials" + }, + "hyperParams": { + "type": "array", + "description": "list of hyper params to optimize", + "items": { + "type": "object", + "properties": { + "suggest": { + "type": "string", + "enum": [ + "uniform", + "loguniform", + "int", + "discrete_uniform", + "categorical" + ] + }, + "low": { + "type": "number" + }, + "hight": { + "type": "number" + }, + "choices": { + "type": "array" + }, + "name": { + "type": "string" + } + }, + "required": [ + "name", + "suggest" + ] + } + }, + "objectivePipeline": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "sampler": { + "type": "object", + "properties": { + "name": { + "type": "string", + "enum": [ + "Grid", + "PartialFixed", + "Random", + "TPE", + "CmaEs" + ] + }, + "fixed_values": { + "type": "object" + }, + "search_space": { + "type": "object" + } + }, + "required": [ + "name" + ] + } + }, + "required": [ + "numberOfTrials", + "objectivePipeline" + ] + }, + "hyperParams": { + "type": "object", + "properties": { + "suggest": { + "type": "string", + "enum": [ + "uniform", + "loguniform", + "int", + "discrete_uniform", + "categorical" + ] + }, + "low": { + "type": "number" + }, + "hight": { + "type": "number" + }, + "choices": { + "type": "array" + }, + "name": { + "type": "string" + } + }, + "required": [ + "name", + "suggest" + ] + }, + "sampler": { + "type": "object", + "properties": { + "name": { + "type": "string", + "enum": [ + "Grid", + "PartialFixed", + "Random", + "TPE", + "CmaEs" + ] + }, + "fixed_values": { + "type": "object" + }, + "search_space": { + "type": "object" + } + }, + "required": [ + "name" + ] + }, + "clean": { + "type": "object", + "properties": { + "maxAge": { + "type": "integer", + "description": "max age to clean (minutes)" + } + } + }, + "gcError": { + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + }, + "response": { + "type": "object", + "properties": { + "message": { + "type": "string" + } + } + }, + "graphQuery": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "nodeName": { + "type": "string" + }, + "sort": { + "type": "string" + }, + "order": { + "type": "string" + }, + "from": { + "type": "integer", + "minimum": 0 + }, + "to": { + "type": "integer", + "minimum": 0 + } + }, + "required": [ + "jobId" + ] + }, + "graphResponse": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "number" + }, + "edges": { + "type": "array", + "description": "Array of edges" + }, + "nodes": { + "type": "array", + "description": "Array of nodes" + } + } + }, + "options": { + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "pipeline": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "experimentName": { + "type": "string", + "description": "experiment name", + "default": "main", + "minLength": 1, + "maxLength": 32, + "format": "experiment-name" + }, + "description": { + "type": "string", + "description": "pipeline description" + }, + "nodes": { + "type": "array", + "description": "Array of nodes", + "items": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + } + }, + "flowInput": { + "type": "object", + "nullable": true, + "description": "input object" + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "options": { + "default": {}, + "type": "object", + "properties": { + "ttl": { + "type": "integer", + "description": "pipeline time to live in seconds", + "minimum": 1, + "default": 3600, + "example": 3600 + }, + "batchTolerance": { + "type": "integer", + "minimum": 0, + "maximum": 100, + "default": 80, + "example": 80 + }, + "concurrentPipelines": { + "type": "object", + "properties": { + "amount": { + "type": "integer", + "minimum": 1, + "maximum": 10000, + "example": 1 + }, + "rejectOnFailure": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false + }, + "progressVerbosityLevel": { + "type": "string", + "default": "info", + "example": "info", + "enum": [ + "trace", + "debug", + "info", + "warn", + "error", + "critical" + ] + }, + "debugOverride": { + "type": "array", + "description": "A list of algorithms to debug in this execution", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "description": "optional properties" + }, + "priority": { + "default": 3, + "type": "integer", + "minimum": 1, + "maximum": 5, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "tags": { + "type": "array", + "description": "Array of tags", + "items": { + "type": "string", + "description": "Unique identifier representing a specific tag", + "minLength": 1, + "maxLength": 32 + } + }, + "rootJobId": { + "type": "string", + "description": "the root job id of the current tree" + }, + "streaming": { + "type": "object", + "description": "streaming options", + "properties": { + "defaultFlow": { + "type": "string", + "description": "the name of the default stream flow" + }, + "flows": { + "type": "object", + "description": "define streaming flows in simple syntax\nexample: node A stream to B and C, node B stream to D.\nit will be: A >> B&C | B >> D\n", + "example": { + "main": "A >> B&C | B >> D | C >> E", + "second": "A >> B >> C" + } + } + } + }, + "edges": { + "type": "array", + "description": "edges define relation between nodes", + "items": {}, + "example": [ + { + "source": "A", + "target": "B" + } + ] + } + }, + "required": [ + "name" + ] + }, + "pipelineName": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + }, + "kindPipeline": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + "pipelineKind": { + "type": "string", + "default": "batch", + "example": "batch", + "enum": [ + "batch", + "stream" + ] + }, + "nodeKind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "pipelineNode": { + "type": "object", + "properties": { + "nodeName": { + "type": "string", + "description": "Unique node identifier", + "minLength": 1 + }, + "algorithmName": { + "type": "string", + "description": "Unique identifier representing a specific algorithm", + "minLength": 1, + "maxLength": 32, + "format": "algorithm-name" + }, + "input": { + "type": "array", + "description": "The input for the algorithm", + "items": {}, + "default": [] + }, + "ttl": { + "type": "integer", + "description": "Algorithm execution time to live in seconds. 0 to disable", + "minimum": 0, + "example": 0 + }, + "includeInResult": { + "type": "boolean" + }, + "metrics": { + "type": "object", + "properties": { + "tensorboard": { + "type": "boolean", + "description": "Should tensorboard metrics be collected." + } + } + }, + "retry": { + "type": "object", + "properties": { + "policy": { + "type": "string", + "default": "OnCrash", + "enum": [ + "Never", + "Always", + "OnError", + "OnCrash" + ] + }, + "limit": { + "type": "integer", + "minimum": 1, + "default": 3, + "example": 1 + } + } + }, + "batchOperation": { + "type": "string", + "enum": [ + "indexed", + "cartesian" + ] + }, + "kind": { + "type": "string", + "default": "algorithm", + "enum": [ + "algorithm", + "pipeline", + "debug", + "dataSource", + "gateway", + "output", + "hyperparamsTuner" + ] + }, + "spec": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Unique identifier representing a specific pipeline", + "minLength": 1, + "format": "pipeline-name" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + } + }, + "required": [ + "id" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + } + }, + "required": [ + "name" + ] + }, + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "snapshot": { + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + } + }, + "required": [ + "name", + "snapshot" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string", + "description": "describe the output" + }, + "mem": { + "type": "string", + "description": "Memory allocated for output", + "format": "algorithm-memory", + "default": "512Mi" + } + } + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "gateway name", + "minLength": 1, + "maxLength": 32, + "format": "gateway-name" + }, + "description": { + "type": "string", + "description": "describe the gateway" + }, + "mem": { + "type": "string", + "description": "Memory allocated for streaming gateway", + "format": "algorithm-memory", + "default": "512Mi" + }, + "cpu": { + "type": "number", + "description": "Cpu allocated for streaming gateway", + "default": 0.5 + } + } + } + ] + } + }, + "required": [ + "nodeName" + ] + }, + "pipelineResult": { + "type": "object", + "properties": { + "jobId": { + "type": "string", + "description": "Unique identifier representing pipeline execution" + }, + "timestamp": { + "type": "string" + }, + "pipeline": { + "type": "string" + }, + "data": { + "type": "array", + "items": {} + }, + "status": { + "type": "string" + }, + "timeTook": { + "type": "number" + }, + "storageModule": { + "type": "string" + } + } + }, + "priority": { + "type": "integer", + "minimum": 1, + "maximum": 5, + "default": 3, + "example": 3 + }, + "triggers": { + "type": "object", + "properties": { + "pipelines": { + "type": "array", + "description": "pipelines to activate upon result", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "cron": { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "cron job", + "minLength": 1, + "format": "cron" + }, + "enabled": { + "type": "boolean", + "description": "enable or disable cron job", + "default": false + } + } + } + }, + "additionalProperties": false + }, + "pipelineTriggersNode": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Pipeline name" + }, + "children": { + "type": "array", + "items": {} + } + } + }, + "pipelineTriggersResult": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Pipeline name" + }, + "children": { + "type": "array", + "items": {} + } + } + } + }, + "codeApiPipelineRequest": { + "type": "object", + "properties": { + "jobId": { + "type": "string" + }, + "taskId": { + "type": "string" + } + }, + "required": [ + "jobId", + "taskId" + ] + }, + "webhookResult": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "url": { + "type": "string" + }, + "pipelineStatus": { + "type": "string" + }, + "responseStatus": { + "type": "string" + }, + "httpResponse": { + "type": "object", + "properties": { + "statusCode": { + "type": "string" + }, + "statusMessage": { + "type": "string" + } + } + }, + "status": { + "type": "string" + } + } + }, + "webhooks": { + "type": "object", + "properties": { + "progress": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon progress", + "example": "http://my-url-to-progress" + }, + "result": { + "type": "string", + "format": "url", + "minLength": 1, + "description": "url to activate upon result", + "example": "http://my-url-to-result" + } + }, + "additionalProperties": false + }, + "githubWebhook": { + "type": "object", + "properties": { + "github": { + "type": "object", + "description": "github object which received after push for more information https://developer.github.com/webhooks/" + } + } + }, + "gitlabWebhook": { + "type": "object", + "properties": { + "gitlab": { + "type": "object", + "description": "gitlab object which received after push for more information https://docs.gitlab.com/ee/user/project/integrations/webhooks.html" + } + } + } + }, + "parameters": { + "nameParam": { + "name": "name", + "in": "path", + "description": "A dataSource name", + "required": true, + "schema": { + "type": "string" + } + }, + "idParam": { + "name": "id", + "in": "path", + "description": "A datasource id (indicating a version)", + "required": true, + "schema": { + "type": "string" + } + }, + "snapshotNameParam": { + "name": "snapshotName", + "in": "path", + "description": "A Snapshot name", + "required": true, + "schema": { + "type": "string" + } + }, + "snapshotNameQuery": { + "name": "snapshot_name", + "in": "query", + "description": "A Snapshot name", + "schema": { + "type": "string" + } + }, + "nameQuery": { + "name": "datasource_name", + "in": "query", + "description": "a datasource name", + "schema": { + "type": "string" + } + }, + "idQuery": { + "name": "id", + "in": "query", + "description": "request a specific version, if not stated the latest is returned", + "schema": { + "type": "string" + } + }, + "idQueryRequired": { + "name": "id", + "in": "query", + "description": "request a specific version, if not stated the latest is returned", + "required": true, + "schema": { + "type": "string" + } + }, + "downloadIdQuery": { + "name": "id", + "in": "query", + "required": true, + "schema": { + "type": "string", + "description": "Unique identifier representing a downloadId", + "minLength": 4, + "maxLength": 4, + "format": "download-id" + } + } + } + } +} \ No newline at end of file diff --git a/core/pipeline-driver-queue/bootstrap.js b/core/pipeline-driver-queue/bootstrap.js index 60aea4564..ef4e82dff 100644 --- a/core/pipeline-driver-queue/bootstrap.js +++ b/core/pipeline-driver-queue/bootstrap.js @@ -8,12 +8,14 @@ const component = require('./lib/consts').componentName.MAIN; const { tracer } = require('@hkube/metrics'); const gracefulShutdown = require('./lib/graceful-shutdown'); + const modules = [ require('./lib/persistency/data-store'), require('./lib/metrics/aggregation-metrics-factory'), require('./lib/queue-runner'), require('./lib/jobs/consumer'), - require('./lib/jobs/producer') + require('./lib/jobs/producer'), + require('./api/rest-api/app-server') ]; class Bootstrap { diff --git a/core/pipeline-driver-queue/config/main/config.base.js b/core/pipeline-driver-queue/config/main/config.base.js index 2e91537d7..e414b84dd 100644 --- a/core/pipeline-driver-queue/config/main/config.base.js +++ b/core/pipeline-driver-queue/config/main/config.base.js @@ -9,6 +9,12 @@ const storageEncoding = process.env.STORAGE_ENCODING || 'bson'; config.defaultStorage = process.env.DEFAULT_STORAGE || 's3'; config.clusterName = process.env.CLUSTER_NAME || 'local'; +config.rest = { + port: formatter.parseInt(process.env.REST_PORT, 7100), + prefix: 'api/v1/queue', + poweredBy: 'HKube Pipeline Driver queue', + bodySizeLimit: process.env.BODY_SIZE_LIMIT || '50mb' +}; config.redis = { host: useSentinel ? process.env.REDIS_SENTINEL_SERVICE_HOST : process.env.REDIS_SERVICE_HOST || 'localhost', port: useSentinel ? process.env.REDIS_SENTINEL_SERVICE_PORT : process.env.REDIS_SERVICE_PORT || 6379, diff --git a/core/pipeline-driver-queue/lib/consts/regex.js b/core/pipeline-driver-queue/lib/consts/regex.js new file mode 100644 index 000000000..3af9e7f6a --- /dev/null +++ b/core/pipeline-driver-queue/lib/consts/regex.js @@ -0,0 +1,5 @@ +const Regex = { + PIPELINE_NAME_REGEX: /^[:\-_.A-Za-z0-9]+$/i +}; + +module.exports = Regex; diff --git a/core/pipeline-driver-queue/lib/consts/validationMessages.js b/core/pipeline-driver-queue/lib/consts/validationMessages.js new file mode 100644 index 000000000..75b92be21 --- /dev/null +++ b/core/pipeline-driver-queue/lib/consts/validationMessages.js @@ -0,0 +1,3 @@ +module.exports = { + PIPELINE_NAME_FORMAT: 'pipeline name must contain only alphanumeric, dash, dot or underscore' +}; diff --git a/core/pipeline-driver-queue/lib/errors/InvalidDataError copy.js b/core/pipeline-driver-queue/lib/errors/InvalidDataError copy.js new file mode 100644 index 000000000..e25394d4d --- /dev/null +++ b/core/pipeline-driver-queue/lib/errors/InvalidDataError copy.js @@ -0,0 +1,10 @@ +const HttpStatus = require('http-status-codes'); + +class InvalidDataError extends Error { + constructor(message) { + super(message); + this.status = HttpStatus.BAD_REQUEST; + } +} + +module.exports = InvalidDataError; diff --git a/core/pipeline-driver-queue/lib/errors/InvalidDataError.js b/core/pipeline-driver-queue/lib/errors/InvalidDataError.js new file mode 100644 index 000000000..e25394d4d --- /dev/null +++ b/core/pipeline-driver-queue/lib/errors/InvalidDataError.js @@ -0,0 +1,10 @@ +const HttpStatus = require('http-status-codes'); + +class InvalidDataError extends Error { + constructor(message) { + super(message); + this.status = HttpStatus.BAD_REQUEST; + } +} + +module.exports = InvalidDataError; diff --git a/core/pipeline-driver-queue/lib/errors/index.js b/core/pipeline-driver-queue/lib/errors/index.js new file mode 100644 index 000000000..08310ae13 --- /dev/null +++ b/core/pipeline-driver-queue/lib/errors/index.js @@ -0,0 +1,4 @@ +const InvalidDataError = require('./InvalidDataError'); +module.exports = { + InvalidDataError +}; diff --git a/core/pipeline-driver-queue/lib/graceful-shutdown.js b/core/pipeline-driver-queue/lib/graceful-shutdown.js index 4e3cb18e4..b296dce77 100644 --- a/core/pipeline-driver-queue/lib/graceful-shutdown.js +++ b/core/pipeline-driver-queue/lib/graceful-shutdown.js @@ -11,8 +11,7 @@ class GracefulShutdown { await consumer.shutdown(); await producer.shutdown(); if (queueRunner.queue) { - const queue = queueRunner.queue.getQueue(); - await queueRunner.queue.persistenceStore(queue); + await queueRunner.queue.persistenceStore(); } log.info('finish graceful shutdown', { component }); } diff --git a/core/pipeline-driver-queue/lib/jobs/consumer.js b/core/pipeline-driver-queue/lib/jobs/consumer.js index e84710826..1683bf257 100644 --- a/core/pipeline-driver-queue/lib/jobs/consumer.js +++ b/core/pipeline-driver-queue/lib/jobs/consumer.js @@ -77,6 +77,7 @@ class JobConsumer { _stopJob(jobId, status) { log.info(`job ${status} ${jobId}`, { component }); + queueRunner.preferredQueue.remove(jobId); queueRunner.queue.remove(jobId); } @@ -93,6 +94,7 @@ class JobConsumer { priority: pipeline.priority, maxExceeded: pipeline.maxExceeded, entranceTime: Date.now(), + tags: pipeline.tags || [], calculated: { latestScores: {} } diff --git a/core/pipeline-driver-queue/lib/jobs/producer.js b/core/pipeline-driver-queue/lib/jobs/producer.js index 154c67926..b6e8eb664 100644 --- a/core/pipeline-driver-queue/lib/jobs/producer.js +++ b/core/pipeline-driver-queue/lib/jobs/producer.js @@ -55,7 +55,7 @@ class JobProducer { // create job first time only, then rely on 3 events (active/completed/enqueue) this._firstJobDequeue = true; log.info('firstJobDequeue', { component }); - await this.createJob(queue[0]); + await this.createJob(queue[0], queueRunner.queue); } } } @@ -71,8 +71,8 @@ class JobProducer { async _updateState() { try { - const queue = queueRunner.queue.getQueue(); - await queueRunner.queue.persistenceStore(queue); + await queueRunner.queue.persistenceStore(); + await queueRunner.preferredQueue.persistenceStore(); } catch (error) { log.throttle.error(error.message, { component }, error); @@ -118,9 +118,16 @@ class JobProducer { */ async _dequeueJob() { try { - const queue = queueRunner.queue.getQueue(q => !q.maxExceeded); - if (queue.length > 0) { - await this.createJob(queue[0]); + const preferredQueue = queueRunner.preferredQueue.getQueue(q => !q.maxExceeded); + + if (preferredQueue.length > 0) { + await this.createJob(preferredQueue[0], queueRunner.preferredQueue); + } + else { + const queue = queueRunner.queue.getQueue(q => !q.maxExceeded); + if (queue.length > 0) { + await this.createJob(queue[0], queueRunner.queue); + } } } catch (error) { @@ -129,9 +136,14 @@ class JobProducer { } _checkMaxExceeded({ experiment, pipeline }) { - const job = queueRunner.queue + let job = queueRunner.preferredQueue .getQueue(q => q.maxExceeded) .find(q => q.experimentName === experiment && q.pipelineName === pipeline); + if (!job) { + job = queueRunner.queue + .getQueue(q => q.maxExceeded) + .find(q => q.experimentName === experiment && q.pipelineName === pipeline); + } if (job) { log.info(`found and disable job with experiment ${experiment} and pipeline ${pipeline} that marked as maxExceeded`, { component }); job.maxExceeded = false; @@ -164,8 +176,8 @@ class JobProducer { }; } - async createJob(job) { - queueRunner.queue.dequeue(job); + async createJob(job, queue) { + queue.dequeue(job); log.debug(`creating new job ${job.jobId}, calculated score: ${job.score}`, { component }); const jobData = this._pipelineToJob(job); await this._producer.createJob(jobData); diff --git a/core/pipeline-driver-queue/lib/persistency/persistency.js b/core/pipeline-driver-queue/lib/persistency/persistency.js index ae24b9813..2d8e9293c 100644 --- a/core/pipeline-driver-queue/lib/persistency/persistency.js +++ b/core/pipeline-driver-queue/lib/persistency/persistency.js @@ -19,13 +19,12 @@ const LOG_TOPICS = { class Persistency { init(options) { - this._queueName = options.persistence.type; this._maxScoringSize = options.scoring.maxSize; } - async store(data) { + async store(data, queueName) { await snapshot.store({ - key: this._queueName, + key: queueName, data, onStart: (...args) => this._onStartSnapshot(...args), onEnd: (...args) => this._onEndSnapshot(...args), @@ -33,7 +32,7 @@ class Persistency { }); await scoring.store({ - key: this._queueName, + key: queueName, data, maxSize: this._maxScoringSize, onStart: (...args) => this._onStartScoring(...args), @@ -42,9 +41,9 @@ class Persistency { }); } - async get() { + async get(queueName) { return snapshot.get({ - key: this._queueName, + key: queueName, onStart: (...args) => this._onStartGetSnapshot(...args), onEnd: (...args) => this._onEndGetSnapshot(...args), onError: (...args) => this._onErrorGetSnapshot(...args) diff --git a/core/pipeline-driver-queue/lib/queue-runner.js b/core/pipeline-driver-queue/lib/queue-runner.js index b5b3b8d89..3a8d484f2 100644 --- a/core/pipeline-driver-queue/lib/queue-runner.js +++ b/core/pipeline-driver-queue/lib/queue-runner.js @@ -19,12 +19,19 @@ class QueueRunner { persistency.init(this.config); this.queue = new Queue({ scoreHeuristic: (...args) => this.heuristicRunner.run(...args), - persistency + persistency, + name: 'main' + }); + this.preferredQueue = new Queue({ + scoreHeuristic: job => job, + persistency, + name: 'preferred' }); this.queue.on(queueEvents.INSERT, job => this._jobAdded(job)); this.queue.on(queueEvents.POP, job => this._jobRemoved(job)); this.queue.on(queueEvents.REMOVE, job => this._jobRemoved(job)); await this.queue.persistencyLoad(); + await this.preferredQueue.persistencyLoad(true); } _jobAdded(job) { diff --git a/core/pipeline-driver-queue/lib/queue.js b/core/pipeline-driver-queue/lib/queue.js index 7fa01727c..f2396e4a4 100644 --- a/core/pipeline-driver-queue/lib/queue.js +++ b/core/pipeline-driver-queue/lib/queue.js @@ -6,12 +6,13 @@ const { queueEvents, componentName } = require('./consts'); const component = componentName.QUEUE; class Queue extends Events { - constructor({ scoreHeuristic, persistency } = {}) { + constructor({ scoreHeuristic, persistency, name } = {}) { super(); this.scoreHeuristic = scoreHeuristic; this.queue = []; this._active = true; this._persistency = persistency; + this._name = name; } flush() { @@ -25,33 +26,50 @@ class Queue extends Events { await this.persistencyStore({ data: this.queue, pendingAmount }); } - async persistencyLoad() { + async persistencyLoad(staticOrder = false) { if (!this._persistency) { return; } - const data = await this._persistency.get(); - if (data?.length > 0) { - data.forEach(q => { + const data = await this._persistency.get(this._name); + const orderedData = []; + let previous = 'FirstInLine'; + data?.forEach(() => { + const item = data.find(job => job.next === previous); + previous = item.jobId; + orderedData.push(item); + }); + + if (orderedData.length > 0) { + orderedData.forEach(q => { const item = { ...q, calculated: { latestScores: {} } }; - this.enqueue(item); + if (staticOrder) { + this.queue.push(item); + } + else { + this.enqueue(item); + } }); } } - async persistenceStore(data) { + async persistenceStore() { + const data = this.getQueue(); if (!this._persistency || !data) { return; } + let previous = 'FirstInLine'; const mapData = data.map(q => { const { calculated, ...rest } = q; - return rest; + const result = { ...rest, next: previous }; + previous = result.jobId; + return result; }); - await this._persistency.store(mapData); + await this._persistency.store(mapData, this._name); } updateHeuristic(scoreHeuristic) { @@ -67,9 +85,12 @@ class Queue extends Events { } dequeue(job) { - remove(this.queue, j => j.jobId === job.jobId); - this.emit(queueEvents.POP, job); - log.info(`job pop from queue, queue size: ${this.size}`, { component }); + const removedJob = remove(this.queue, j => j.jobId === job.jobId); + if (removedJob.length > 0) { + this.emit(queueEvents.POP, job); + log.info(`job pop from queue, queue size: ${this.size}`, { component }); + } + return removedJob; } remove(jobId) { diff --git a/core/pipeline-driver-queue/lib/service/preferred-jobs.js b/core/pipeline-driver-queue/lib/service/preferred-jobs.js new file mode 100644 index 000000000..755147603 --- /dev/null +++ b/core/pipeline-driver-queue/lib/service/preferred-jobs.js @@ -0,0 +1,85 @@ +const queuePosition = require('@hkube/consts').queuePositions; +const queueRunner = require('../queue-runner'); +const validator = require('../validation'); +const InvalidDataError = require('../errors/InvalidDataError'); +class PreferredJobs { + getPreferredJobsList() { + return queueRunner.preferredQueue.queue.map(job => { + const { score, calculated, next, ...rest } = job; + return rest; + }); + } + + deletePreferredJobs(jobIds) { + const deletedJobs = jobIds.map(jobId => { + const deletedArr = queueRunner.preferredQueue.dequeue({ jobId }); + if (deletedArr.length > 0) { + queueRunner.queue.enqueue(deletedArr[0]); + } + return deletedArr.length > 0 ? deletedArr[0] : null; + }).filter(job => job !== null); + return deletedJobs; + } + + query(job, tag, pipeline, jobId) { + if (jobId) { + return job.jobId === jobId; + } + if (pipeline) { + return job.pipeline === pipeline; + } + if (tag) { + return job.tags.includes(tag); + } + return false; + } + + getIndex(position, tag, pipeline, jobId) { + let index; + if (position === queuePosition.BEFORE) { + index = queueRunner.preferredQueue.queue.findIndex(job => this.query(job, tag, pipeline, jobId)); + if (index === -1) { + index = 0; + } + } + if (position === queuePosition.AFTER) { + index = queueRunner.preferredQueue.queue.slice(0).reverse().findIndex(job => this.query(job, tag, pipeline, jobId)); + if (index === -1) { + index = 0; + } + else { + index = queueRunner.preferredQueue.queue.length - index; + } + } + if (position === queuePosition.FIRST) { + index = 0; + } + if (position === queuePosition.LAST) { + index = queueRunner.preferredQueue.queue.length; + } + return index; + } + + addPreferredJobs(addedJobs) { + validator.preference.validatePreferenceRequest(addedJobs); + const { jobs, position, query } = addedJobs; + const { tag, pipeline, jobId } = query || {}; + const index = this.getIndex(position, tag, pipeline, jobId); + const allDequeued = []; + jobs.reverse().forEach(id => { + const dequeued = queueRunner.queue.dequeue({ jobId: id }); + if (dequeued.length > 0) { + allDequeued.push(dequeued[0]); + queueRunner.preferredQueue.queue.splice(index, 0, dequeued[0]); + } + }); + if (allDequeued.length === 0) { + throw new InvalidDataError('None of the jobs exist in the general queue'); + } + else { + return allDequeued; + } + } +} + +module.exports = new PreferredJobs(); diff --git a/core/pipeline-driver-queue/lib/validation/custom-formats.js b/core/pipeline-driver-queue/lib/validation/custom-formats.js new file mode 100644 index 000000000..6a2f79dde --- /dev/null +++ b/core/pipeline-driver-queue/lib/validation/custom-formats.js @@ -0,0 +1,39 @@ +const validationMessages = require('../consts/validationMessages'); +const regex = require('../consts/regex'); +const formatMessages = new Map(); + +class ApiValidator { + init(definitions, ...validators) { + validators.forEach(v => this._init(v, definitions)); + this._addFormatMessages(); + } + + _init(validatorInstance, definitions) { + validatorInstance.errorsText = this.wrapErrorMessageFn(validatorInstance.errorsText.bind(validatorInstance)).bind(this);// eslint-disable-line + validatorInstance.addFormat('pipeline-name', this._validatePipelineName); + Object.entries(definitions).forEach(([k, v]) => { + validatorInstance.addSchema(v, `#/components/schemas/${k}`); + }); + } + + _addFormatMessages() { + formatMessages.set('pipeline-name', validationMessages.PIPELINE_NAME_FORMAT); + } + + _validatePipelineName(name) { + return regex.PIPELINE_NAME_REGEX.test(name); + } + + wrapErrorMessageFn(wrappedFn) { + const errorsTextWrapper = (errors, options) => { + let message; + if (errors) { + [{ message }] = errors; + } + return message || wrappedFn(errors, options); + }; + return errorsTextWrapper; + } +} + +module.exports = new ApiValidator(); diff --git a/core/pipeline-driver-queue/lib/validation/index.js b/core/pipeline-driver-queue/lib/validation/index.js new file mode 100644 index 000000000..2845b4997 --- /dev/null +++ b/core/pipeline-driver-queue/lib/validation/index.js @@ -0,0 +1,15 @@ +const innerValidator = require('./inner-validator'); +const Preference = require('./preference'); + +class ApiValidator { + constructor() { + this.preference = null; + } + + init(schemas) { + innerValidator.init(schemas); + this.preference = new Preference(innerValidator); + } +} + +module.exports = new ApiValidator(); diff --git a/core/pipeline-driver-queue/lib/validation/inner-validator.js b/core/pipeline-driver-queue/lib/validation/inner-validator.js new file mode 100644 index 000000000..865f2c4d7 --- /dev/null +++ b/core/pipeline-driver-queue/lib/validation/inner-validator.js @@ -0,0 +1,45 @@ +const Validator = require('ajv'); +const { InvalidDataError } = require('../errors'); +const customFormats = require('./custom-formats'); + +const validator = new Validator({ + useDefaults: false, + coerceTypes: true, + nullable: true, +}); +const defaulter = new Validator({ + useDefaults: true, + coerceTypes: true, + nullable: true, +}); + +class ApiValidator { + init(schemas) { + this.definitions = schemas; + customFormats.init(schemas, validator, defaulter); + } + + validate(schema, object, useDefaults, options) { + this._validateInner(defaulter, schema, object, options); + } + + _validateInner(validatorInstance, schema, obj) { + const object = obj || {}; + const valid = validatorInstance.validate(schema, object); + if (!valid) { + const { errors } = validatorInstance; + let error = validatorInstance.errorsText(errors, { + extraInfo: true, + }); + if (errors[0].params && errors[0].params.allowedValues) { + error += ` (${errors[0].params.allowedValues.join(',')})`; + } + else if (errors[0].params && errors[0].params.additionalProperty) { + error += ` (${errors[0].params.additionalProperty})`; + } + throw new InvalidDataError(error); + } + } +} + +module.exports = new ApiValidator(); diff --git a/core/pipeline-driver-queue/lib/validation/preference.js b/core/pipeline-driver-queue/lib/validation/preference.js new file mode 100644 index 000000000..0455f59e0 --- /dev/null +++ b/core/pipeline-driver-queue/lib/validation/preference.js @@ -0,0 +1,32 @@ +const queuePosition = require('@hkube/consts').queuePositions; +const InvalidDataError = require('../errors/InvalidDataError'); +class Preference { + constructor(validator) { + this._validator = validator; + } + + validatePreferenceRequest(preferenceRequest) { + this._validator.validate( + this._validator.definitions.addToPreferredRequest, + preferenceRequest + ); + const { position, query } = preferenceRequest; + if ((position === queuePosition.FIRST || position === queuePosition.LAST) && query) { + throw new InvalidDataError('No query is needed in case of position first or last'); + } + else if (position === queuePosition.BEFORE || position === queuePosition.AFTER) { + if (!query) { + throw new InvalidDataError('Must supply query in case of position before or after'); + } + const numberOfFields = Object.keys(query).filter(key => query[key] !== undefined).length; + if (numberOfFields > 1) { + throw new InvalidDataError('Query must contain only one of jobId ,tag ,pipelineName'); + } + if (numberOfFields === 0) { + throw new InvalidDataError('Query must contain one of jobId ,tag ,pipelineName'); + } + } + } +} + +module.exports = Preference; diff --git a/core/pipeline-driver-queue/package-lock.json b/core/pipeline-driver-queue/package-lock.json index b6e92ebbe..bbaf3ef98 100644 --- a/core/pipeline-driver-queue/package-lock.json +++ b/core/pipeline-driver-queue/package-lock.json @@ -5,19 +5,35 @@ "requires": true, "dependencies": { "@apidevtools/json-schema-ref-parser": { - "version": "9.0.6", - "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz", - "integrity": "sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==", + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.9.tgz", + "integrity": "sha512-GBD2Le9w2+lVFoc4vswGI/TjkNIZSVp7+9xPf+X3uidBfWnAeUWmquteSyt0+VCrhNMWj/FTABISQrD3Z/YA+w==", "requires": { "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.6", "call-me-maybe": "^1.0.1", - "js-yaml": "^3.13.1" + "js-yaml": "^4.1.0" + }, + "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "requires": { + "argparse": "^2.0.1" + } + } } }, "@apidevtools/openapi-schemas": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.0.4.tgz", - "integrity": "sha512-ob5c4UiaMYkb24pNhvfSABShAwpREvUGCkqjiz/BX9gKZ32y/S22M+ALIHftTAuv9KsFVSpVdIDzi9ZzFh5TCA==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", + "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==" }, "@apidevtools/swagger-methods": { "version": "3.0.2", @@ -25,16 +41,16 @@ "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" }, "@apidevtools/swagger-parser": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.0.2.tgz", - "integrity": "sha512-JFxcEyp8RlNHgBCE98nwuTkZT6eNFPc1aosWV6wPcQph72TSEEu1k3baJD4/x1qznU+JiDdz8F5pTwabZh+Dhg==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.0.3.tgz", + "integrity": "sha512-sNiLY51vZOmSPFZA5TF35KZ2HbgYklQnTSDnkghamzLb3EkNtcQnrBQEj5AOCxHpTtXpqMCRM1CrmV2rG6nw4g==", "requires": { "@apidevtools/json-schema-ref-parser": "^9.0.6", "@apidevtools/openapi-schemas": "^2.0.4", "@apidevtools/swagger-methods": "^3.0.2", "@jsdevtools/ono": "^7.1.3", "call-me-maybe": "^1.0.1", - "z-schema": "^4.2.3" + "z-schema": "^5.0.1" } }, "@babel/code-frame": { @@ -301,16 +317,16 @@ } }, "@hkube/consts": { - "version": "1.0.31", - "resolved": "https://registry.npmjs.org/@hkube/consts/-/consts-1.0.31.tgz", - "integrity": "sha512-4zTJd3bnkg4pdGxBbGH/n+SQkWCTsVcRl34XuGQdKD3BmT9sgFfSZoU2uuBPyWdexHEg7NbsDKHmKZB+jeCKGw==" + "version": "1.0.45", + "resolved": "https://registry.npmjs.org/@hkube/consts/-/consts-1.0.45.tgz", + "integrity": "sha512-vDFolDFWo+lBheKq6FFObVyxPAVX4KViHtOt8cl5Y/iKbv4lMxeGO6IjWPA3PvuUG/1iZ0Uf2fcFX1GWpwxspg==" }, "@hkube/db": { - "version": "1.0.42", - "resolved": "https://registry.npmjs.org/@hkube/db/-/db-1.0.42.tgz", - "integrity": "sha512-qDTVzWPFVDSd+GUvuryzXYZVGotghLD8AwGoLtJjWHEaAU3P2dEv5FN4vDdX1IE0LvIYSXMOYD9PLq8dKFvO+A==", + "version": "1.0.47", + "resolved": "https://registry.npmjs.org/@hkube/db/-/db-1.0.47.tgz", + "integrity": "sha512-Ibnpm+A4XKTQoPgvzZOLZJrM2vz+8VJAhdmlvzsAOzQ3pF04ns79r27SHWYgblC9GN45K2RIMCawfoQ4xEVJ8w==", "requires": { - "@hkube/consts": "^1.0.31", + "@hkube/consts": "^1.0.40", "lodash.clonedeep": "^4.5.0", "lodash.merge": "^4.6.2", "moment": "^2.29.1", @@ -318,6 +334,11 @@ "performance-now": "^2.1.0" }, "dependencies": { + "@hkube/consts": { + "version": "1.0.44", + "resolved": "https://registry.npmjs.org/@hkube/consts/-/consts-1.0.44.tgz", + "integrity": "sha512-/K3pNQzMtCbA31RmqVKURz5RUJwcuATTQoLQlcfqkRXZbm2wWOEUdT7j3dwESIKyH4+8MOnehX8EgUY0I60wZw==" + }, "moment": { "version": "2.29.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", @@ -404,6 +425,19 @@ "requires": { "fs-extra": "^9.0.1", "recursive-readdir": "^2.2.2" + }, + "dependencies": { + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + } } }, "@hkube/logger": { @@ -498,16 +532,18 @@ } }, "@hkube/rest-server": { - "version": "1.0.16", - "resolved": "https://registry.npmjs.org/@hkube/rest-server/-/rest-server-1.0.16.tgz", - "integrity": "sha512-eMHSbO8lQqutAlRO7+Ms2NAN2cJOcF2BfxRgiqMs9GOT25xmu3CqChZDOYdED/mjvqUJqGCd5iFUtxNWkkpnqg==", - "requires": { - "body-parser": "^1.18.3", - "express": "^4.16.3", - "express-rate-limit": "^2.11.0", + "version": "1.0.18", + "resolved": "https://registry.npmjs.org/@hkube/rest-server/-/rest-server-1.0.18.tgz", + "integrity": "sha512-myqp/HwZcJyS1GL0kLg7PjnEBfAiS43Kq+SGkxE0mDGuTDbajqqns50SlAQlFBXKBT5dTE/BzO3ufldReLdmPA==", + "requires": { + "body-parser": "^1.19.0", + "express": "^4.17.1", + "express-async-errors": "^3.1.1", + "express-rate-limit": "^5.2.6", "fs-extra": "^9.0.1", "js-yaml": "^4.0.0", "json-refs": "^3.0.15", + "performance-now": "^2.1.0", "swagger-parser": "^10.0.2" }, "dependencies": { @@ -516,10 +552,21 @@ "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, "js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "requires": { "argparse": "^2.0.1" } @@ -722,6 +769,11 @@ "@types/node": "*" } }, + "@types/json-schema": { + "version": "7.0.9", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.9.tgz", + "integrity": "sha512-qcUXuemtEu+E5wZSJHNxUXeCZhAfXKQ41D+duX+VYPde7xyEVZci+/oXKJL13tnRs9lR2pr4fod59GT6/X1/yQ==" + }, "@types/json5": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", @@ -1308,11 +1360,6 @@ "wrap-ansi": "^2.0.0" } }, - "clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=" - }, "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", @@ -1462,9 +1509,9 @@ "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" }, "cookiejar": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.2.tgz", - "integrity": "sha512-Mw+adcfzPxcPeI+0WlvRrr/3lGVO0bD75SxX6811cxSh1Wbxx7xZBGK1eVtDf6si8rg2lhnUjsVLMFMfbRIuwA==" + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.3.tgz", + "integrity": "sha512-JxbCBUdrfr6AQjOXrxoTvAMJO4HBTUIlBzslcJPAz+/KT8yk53fXun51u+RenNYvad/+Vc2DIz5o9UxlCDymFQ==" }, "core-util-is": { "version": "1.0.2", @@ -1598,14 +1645,6 @@ "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", "dev": true }, - "defaults": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", - "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", - "requires": { - "clone": "^1.0.2" - } - }, "define-properties": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", @@ -2196,13 +2235,15 @@ } } }, + "express-async-errors": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/express-async-errors/-/express-async-errors-3.1.1.tgz", + "integrity": "sha512-h6aK1da4tpqWSbyCa3FxB/V6Ehd4EEB15zyQq9qe75OZBp0krinNKuH4rAY+S/U/2I36vdLAUFSjQJ+TFmODng==" + }, "express-rate-limit": { - "version": "2.14.2", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-2.14.2.tgz", - "integrity": "sha512-aVYzfYU2Bv+v6ry/fBpTrX8MorM0p/TeDnx4CqiLevg9ftG8eW+pyuZ6JEreGdG2t1vXyTGHuSNRKLeYixuqZg==", - "requires": { - "defaults": "^1.0.3" - } + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-5.5.1.tgz", + "integrity": "sha512-MTjE2eIbHv5DyfuFz4zLYWxpqVhEhkTiwFGuB74Q9CSou2WHO52nlE5y3Zlg6SIsiYUIPj6ifFxnkPz6O3sIUg==" }, "extend": { "version": "3.0.2", @@ -2499,9 +2540,9 @@ } }, "formidable": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz", - "integrity": "sha512-V8gLm+41I/8kguQ4/o1D3RIHRmhYFG4pnNyonvua+40rqcEmT4+V71yaZ3B457xbbgCsCfjSPi65u/W6vK1U5Q==" + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.6.tgz", + "integrity": "sha512-KcpbcpuLNOwrEjnbpMC0gS+X8ciDoZE1kkqzat4a8vrprf+s9pKNQ/QIwWfbfs4ltgmFl3MD177SNTkve3BwGQ==" }, "forwarded": { "version": "0.1.2", @@ -2514,14 +2555,13 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-extra": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.0.1.tgz", - "integrity": "sha512-h2iAoN838FqAFJY2/qVpzFXy+EBxfVE220PalAqQLDVsFOHLJrZvut5puAbCdNv6WJk+B8ihI+k0c7JK5erwqQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.0.0.tgz", + "integrity": "sha512-C5owb14u9eJwizKGdchcDUQeFtlSHHthBk8pbX9Vc1PFZrLombudjDnNns88aYslCyF6IY5SUw3Roz6xShcEIQ==", "requires": { - "at-least-node": "^1.0.0", "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", - "universalify": "^1.0.0" + "universalify": "^2.0.0" } }, "fs-minipass": { @@ -2877,6 +2917,11 @@ "sshpk": "^1.7.0" } }, + "http-status-codes": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-2.1.4.tgz", + "integrity": "sha512-MZVIsLKGVOVE1KEnldppe6Ij+vmemMuApDfjhVSLzyYP+td0bREEYyAoIw9yFePoBXManCuBqmiNP5FqJS5Xkg==" + }, "https-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", @@ -3382,13 +3427,6 @@ "requires": { "graceful-fs": "^4.1.6", "universalify": "^2.0.0" - }, - "dependencies": { - "universalify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" - } } }, "jsprim": { @@ -3999,14 +4037,14 @@ } }, "mongodb": { - "version": "3.6.9", - "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.6.9.tgz", - "integrity": "sha512-1nSCKgSunzn/CXwgOWgbPHUWOO5OfERcuOWISmqd610jn0s8BU9K4879iJVabqgpPPbA6hO7rG48eq+fGED3Mg==", + "version": "3.7.3", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.7.3.tgz", + "integrity": "sha512-Psm+g3/wHXhjBEktkxXsFMZvd3nemI0r3IPsE0bU+4//PnvNWKkzhZcEsbPcYiWqe8XqXJJEg4Tgtr7Raw67Yw==", "requires": { "bl": "^2.2.1", "bson": "^1.1.4", "denque": "^1.4.1", - "optional-require": "^1.0.3", + "optional-require": "^1.1.8", "safe-buffer": "^5.1.2", "saslprep": "^1.0.0" } @@ -4665,9 +4703,12 @@ "integrity": "sha512-XLKtEfHxqrWyF1fzxznsv78w3csW41ucHnjiKnfzZLD5FN8UBDZZL1i4q0FR29zjxXhm+2Hop+5Vr/b8tKIvEg==" }, "optional-require": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.0.3.tgz", - "integrity": "sha512-RV2Zp2MY2aeYK5G+B/Sps8lW5NHAzE5QClbFP15j+PWmP+T9PxlJXBOOLoSAdgwFvS4t0aMR4vpedMkbHfh0nA==" + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/optional-require/-/optional-require-1.1.8.tgz", + "integrity": "sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA==", + "requires": { + "require-at": "^1.0.6" + } }, "optionator": { "version": "0.8.3", @@ -5095,6 +5136,11 @@ } } }, + "require-at": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/require-at/-/require-at-1.0.6.tgz", + "integrity": "sha512-7i1auJbMUrXEAZCOQ0VNJgmcT2VOKPRl2YGJwgpHpC9CE91Mv4/4UYIUm4chGJaI381ZDq1JUicFii64Hapd8g==" + }, "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -5526,11 +5572,11 @@ } }, "swagger-parser": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/swagger-parser/-/swagger-parser-10.0.2.tgz", - "integrity": "sha512-9jHkHM+QXyLGFLk1DkXBwV+4HyNm0Za3b8/zk/+mjr8jgOSiqm3FOTHBSDsBjtn9scdL+8eWcHdupp2NLM8tDw==", + "version": "10.0.3", + "resolved": "https://registry.npmjs.org/swagger-parser/-/swagger-parser-10.0.3.tgz", + "integrity": "sha512-nF7oMeL4KypldrQhac8RyHerJeGPD1p2xDh900GPvc+Nk7nWP6jX2FcC7WmkinMoAmoO774+AFXcWsW8gMWEIg==", "requires": { - "@apidevtools/swagger-parser": "10.0.2" + "@apidevtools/swagger-parser": "10.0.3" } }, "table": { @@ -5867,9 +5913,9 @@ } }, "universalify": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-1.0.0.tgz", - "integrity": "sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" }, "unpipe": { "version": "1.0.0", @@ -5999,9 +6045,9 @@ } }, "validator": { - "version": "12.2.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-12.2.0.tgz", - "integrity": "sha512-jJfE/DW6tIK1Ek8nCfNFqt8Wb3nzMoAbocBF6/Icgg1ZFSBpObdnwVY2jQj6qUqzhx5jc71fpvBWyLGO7Xl+nQ==" + "version": "13.7.0", + "resolved": "https://registry.npmjs.org/validator/-/validator-13.7.0.tgz", + "integrity": "sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw==" }, "vary": { "version": "1.1.2", @@ -6343,14 +6389,14 @@ } }, "z-schema": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/z-schema/-/z-schema-4.2.3.tgz", - "integrity": "sha512-zkvK/9TC6p38IwcrbnT3ul9in1UX4cm1y/VZSs4GHKIiDCrlafc+YQBgQBUdDXLAoZHf2qvQ7gJJOo6yT1LH6A==", + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/z-schema/-/z-schema-5.0.2.tgz", + "integrity": "sha512-40TH47ukMHq5HrzkeVE40Ad7eIDKaRV2b+Qpi2prLc9X9eFJFzV7tMe5aH12e6avaSS/u5l653EQOv+J9PirPw==", "requires": { "commander": "^2.7.1", "lodash.get": "^4.4.2", "lodash.isequal": "^4.5.0", - "validator": "^12.0.0" + "validator": "^13.7.0" }, "dependencies": { "commander": { diff --git a/core/pipeline-driver-queue/package.json b/core/pipeline-driver-queue/package.json index 2f3811ded..92ae25be0 100644 --- a/core/pipeline-driver-queue/package.json +++ b/core/pipeline-driver-queue/package.json @@ -18,8 +18,8 @@ "license": "MIT", "dependencies": { "@hkube/config": "^2.0.11", - "@hkube/consts": "^1.0.31", - "@hkube/db": "^1.0.42", + "@hkube/consts": "^1.0.45", + "@hkube/db": "^1.0.47", "@hkube/etcd": "^5.1.2", "@hkube/logger": "^2.0.2", "@hkube/metrics": "^1.0.42", @@ -29,7 +29,11 @@ "lodash.isequal": "^4.5.0", "lodash.orderby": "^4.6.0", "lodash.remove": "^4.7.0", - "performance-now": "^2.1.0" + "performance-now": "^2.1.0", + "fs-extra": "^10.0.0", + "@hkube/rest-server": "^1.0.18", + "http-status-codes": "^2.1.4", + "ajv": "^6.12.0" }, "devDependencies": { "@hkube/uid": "^1.0.4", diff --git a/core/pipeline-driver-queue/tests/stub/stub.js b/core/pipeline-driver-queue/tests/stub/stub.js index a1a8c8e7c..46d3b97d3 100644 --- a/core/pipeline-driver-queue/tests/stub/stub.js +++ b/core/pipeline-driver-queue/tests/stub/stub.js @@ -4,19 +4,21 @@ const stubTemplate = ({ pipelineUuid = `pipeline-${uuidv4()}`, priority = Math.floor((Math.random() * 5)), score = Math.floor((Math.random() * 100)), + preference, entranceTime = Date.now() } = {}) => ( - { - jobId: `${uuid}`, - pipelineName: `${pipelineUuid}`, - priority: priority, - entranceTime: entranceTime, - score, - calculated: { - latestScores: {} - } + { + jobId: `${uuid}`, + pipelineName: `${pipelineUuid}`, + priority: priority, + entranceTime: entranceTime, + score, + preference, + calculated: { + latestScores: {} } - ); + } +); const generateConsumedArray = (number = 100) => ({ jobId: `jobId-${uuidv4()}`, @@ -30,4 +32,4 @@ module.exports = { stubTemplate, generateArr, generateConsumedArray -}; \ No newline at end of file +}; \ No newline at end of file diff --git a/core/pipeline-driver-queue/tests/test.js b/core/pipeline-driver-queue/tests/test.js index 2f016dc51..e010aa9f8 100644 --- a/core/pipeline-driver-queue/tests/test.js +++ b/core/pipeline-driver-queue/tests/test.js @@ -12,7 +12,13 @@ const queueRunner = require('../lib/queue-runner'); const dataStore = require('../lib/persistency/data-store'); const producerLib = require('../lib/jobs/producer'); const setting = { prefix: 'pipeline-driver-queue' } +const preferredService = require('../lib/service/preferred-jobs'); const producer = new Producer({ setting }); +const configIt = require('@hkube/config'); +const { main: config } = configIt.load(); +const baseUrl = `http://localhost:${config.rest.port}`; +const restUrl = `${baseUrl}/${config.rest.prefix}`; +const { request } = require('./utils'); const Queue = require('../lib/queue'); const heuristic = score => job => ({ ...job, entranceTime: Date.now(), score, ...{ calculated: { latestScore: {} } } }) const heuristicStub = score => job => ({ ...job }) @@ -28,6 +34,7 @@ let _semaphore = null; describe('Test', () => { before(async () => { + require('../lib/jobs/producer')._updateState = function () { }; await bootstrap.init(); consumer = require('../lib/jobs/consumer'); }); @@ -40,6 +47,7 @@ describe('Test', () => { afterEach(() => { producerLib._isConsumerActive = true; }); + describe('queue-tests', () => { describe('add', () => { it('should added to queue', async () => { @@ -174,13 +182,101 @@ describe('Test', () => { }); describe('persistency tests', () => { it('persistent load', async () => { - queueRunner.queue.queue = [] + queueRunner.preferredQueue.queue = []; + queueRunner.queue.queue = []; const jobs = generateArr(100); - await queueRunner.queue.persistenceStore(jobs); + queueRunner.queue.queue = jobs; + await queueRunner.queue.persistenceStore(); await queueRunner.queue.persistencyLoad(); const q = queueRunner.queue.getQueue(); expect(q.length).to.be.greaterThan(98); + queueRunner.preferredQueue.queue = jobs; + await queueRunner.preferredQueue.persistenceStore(); + await queueRunner.preferredQueue.persistencyLoad(true); + const pq = queueRunner.preferredQueue.getQueue(); + expect(jobs[0].jobId == pq[0].jobId && jobs[99].jobId == pq[99].jobId) + }); + }); + describe('preferred tests', () => { + it('preferred order', async () => { + const jobs = []; + jobs.push({ jobId: 'a', pipeline: 'p_a', entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'b', pipeline: 'p_a', entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'c', pipeline: 'p_a', entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'b_a', pipeline: 'p_b', entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'b_b', pipeline: 'p_b', entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'b_c', pipeline: 'p_b', entranceTime: 10, calculated: { latestScores: [] } }); + await Promise.all(jobs.map(job => queueRunner.queue.enqueue(job))); + preferredService.addPreferredJobs({ 'jobs': ['b'], position: 'first' }); + preferredService.addPreferredJobs({ 'jobs': ['a'], position: 'first' }); + preferredService.addPreferredJobs({ 'jobs': ['c'], position: 'last' }); + preferredService.addPreferredJobs({ 'jobs': ['b_c'], position: 'last' }); + preferredService.addPreferredJobs({ 'jobs': ['b_b'], position: 'after', query: { pipeline: 'p_a' } }); + preferredService.addPreferredJobs({ 'jobs': ['b_a'], position: 'before', query: { pipeline: 'p_b' } }); + expect(queueRunner.preferredQueue.queue.every((val, index) => val.jobId === jobs[index].jobId)); }); + + describe('preferred api', () => { + + it('preferred api', async () => { + const jobs = []; + jobs.push({ jobId: 'a', pipeline: 'p_a', entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'b', pipeline: 'p_a', tags: ['tag1'], entranceTime: 10, calculated: { latestScores: [] } }); + jobs.push({ jobId: 'c', pipeline: 'p_a', entranceTime: 10, calculated: { latestScores: [] } }); + queueRunner.queue.queue = []; + queueRunner.preferredQueue.queue = []; + await Promise.all(jobs.map(job => queueRunner.queue.enqueue(job))); + let result = await request({ + url: `${restUrl}/preferred`, method: 'POST', body: { + "jobs": ["b"], + "position": "first" + } + }); + expect(result.body[0].jobId === 'b'); + result = await request({ + url: `${restUrl}/preferred`, method: 'POST', body: { + "jobs": ["b"], + "position": "first" + } + }); + expect(result.body.error.message === 'None of the jobs exist in the general queue'); + result = await request({ + url: `${restUrl}/preferred`, method: 'POST', body: { + "jobs": ["a"], + "position": "before", + "query": { jobId: 'b' } + } + }); + result = await request({ + url: `${restUrl}/preferred`, method: 'POST', body: { + "jobs": ["c"], + "position": "after", + "query": { tag: 'tag1' } + } + }); + result = await request({ + url: `${restUrl}/preferred`, method: 'POST', body: { + "jobs": ["c"], + "position": "after", + "query": { tag: 'tag1', jobId: 'd' } + } + }); + expect(result.body.error.message === 'Query must contain only one of jobId ,tag ,pipelineName'); + result = await request({ + url: `${restUrl}/preferred`, method: 'GET' + }); + expect(result.body[0].jobId === 'a' && result.body[1].jobId === 'b' && result.body[2].jobId === 'c') + result = await request({ + url: `${restUrl}/preferred/deletes`, method: 'POST', body: { + "jobs": ['c'] + } + }); + expect(result.body[0].jobId === 'c') + + }); + }); + + }); describe('job-consume', () => { it('should consume job with params', async () => { diff --git a/core/pipeline-driver-queue/tests/utils.js b/core/pipeline-driver-queue/tests/utils.js new file mode 100644 index 000000000..6bcd57cf2 --- /dev/null +++ b/core/pipeline-driver-queue/tests/utils.js @@ -0,0 +1,29 @@ +const requestClient = require('request'); + + + +const delay = (d) => new Promise((r) => setTimeout(r, d)); + +const request = (options) => { + return new Promise((resolve, reject) => { + const method = options.method || 'POST'; + requestClient( + { + ...options, + method, + json: true + }, + (error, response, body) => { + if (error) { + return reject(error); + } + return resolve({ body, response }); + } + ); + }); +}; + +module.exports = { + delay, + request +};