From e392c1379f548d1463f13b3ba4d200f18d661641 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sun, 3 Mar 2024 23:17:49 -0500 Subject: [PATCH] new plugins --- bin/docker-plugin.js | 104 ++++++++++-------- bin/sshx-plugin.js | 233 ++++++++++++++++++++++------------------- bin/url-plugin.js | 11 +- sample_conf/setup.json | 1 - 4 files changed, 198 insertions(+), 151 deletions(-) diff --git a/bin/docker-plugin.js b/bin/docker-plugin.js index 8217f46..7d395e0 100644 --- a/bin/docker-plugin.js +++ b/bin/docker-plugin.js @@ -11,26 +11,7 @@ const fs = require('fs') let job = {} try { job = JSON.parse(fs.readFileSync(process.stdin.fd)) } catch { } - -const docker = new Docker(); -let stderr_msg - -// PARAMETERS -const ENTRYPOINT_PATH = process.env['ENTRYPOINT_PATH'] || '/cronicle.sh' -const cname = 'cronicle-' + (process.env['JOB_ID'] || process.pid) -let imageName = process.env['IMAGE'] || 'alpine' -let script = process.env['SCRIPT'] ?? "#!/bin/sh\necho 'No script specified'" -const autoPull = !!parseInt(process.env['PULL_IMAGE']) -const autoRemove = !parseInt(process.env['KEEP_CONTAINER']) -const keepEntrypoint = !!parseInt(process.env['KEEP_ENTRYPOINT']) -const json = !!parseInt(process.env['JSON']) - -let command = [] -if((process.env['COMMAND'] || '').trim()) { - command = process.env['COMMAND'].trim().match(/(?:[^\s"]+|"[^"]*")+/g).map(e=> e.replace(/["]+/g, '')) -} - -// helpers +// helpers functions const print = (text) => process.stdout.write(text + EOL) const printInfo = (text) => process.stdout.write(`[INFO] \x1b[32m${text}\x1b[0m` + EOL) const printWarning = (text) => process.stdout.write(`[INFO] \x1b[33m${text}\x1b[0m` + EOL) @@ -46,6 +27,53 @@ const exit = (message) => { process.exit(1) } +let dockerOpts = {} + +let registryAuth = { + username: process.env['DOCKER_USER'], + password: process.env['DOCKER_PASSWORD'] +} + +// check if user specified DOCKER_HOST. If not just user socket default connection +let dh = process.env['DOCKER_HOST'] + +if (dh) { + try { // resolve password/user from uri + let uri = new URL(process.env[dh] || dh) // uri could be passed as a reference to env var + if(uri.password) dockerOpts.password = decodeURIComponent(uri.password) + if(uri.username) dockerOpts.username = uri.username + + // for ssh:// also check env variables for auth + if(process.env['SSH_PASSWORD'] && uri.protocol.startsWith('ssh')) dockerOpts.password = process.env['SSH_PASSWORD'] + if(process.env['SSH_KEY'] && uri.protocol.startsWith('ssh')) dockerOpts.sshOptions = { privateKey: process.env['SSH_KEY'] } + + } catch (e) { + printError('Invalid DOCKER HOST format, use ssh://user:password@host:port or http://host:2375') + exit(e.message) + } +} + + +// DOCKER CLIENT + +const docker = new Docker(dockerOpts) + +// CONTAINER PARAMETERS +const ENTRYPOINT_PATH = process.env['ENTRYPOINT_PATH'] || '/cronicle.sh' +const cname = 'cronicle-' + (process.env['JOB_ID'] || process.pid) +let imageName = process.env['IMAGE'] || 'alpine' +let script = process.env['SCRIPT'] ?? "#!/bin/sh\necho 'No script specified'" +const autoPull = !!parseInt(process.env['PULL_IMAGE']) +const autoRemove = !parseInt(process.env['KEEP_CONTAINER']) +const keepEntrypoint = !!parseInt(process.env['KEEP_ENTRYPOINT']) +const json = !!parseInt(process.env['JSON']) +let stderr_msg + +let command = [] +if ((process.env['COMMAND'] || '').trim()) { + command = process.env['COMMAND'].trim().match(/(?:[^\s"]+|"[^"]*")+/g).map(e => e.replace(/["]+/g, '')) +} + sig = process.connected ? 'disconnect' : 'SIGTERM' process.on(sig, async (message) => { printInfo('Caught SIGTERM') @@ -62,13 +90,13 @@ const stdout = new Writable({ if (line.match(/^\s*(\d+)\%\s*$/)) { // handle progress let progress = Math.max(0, Math.min(100, parseInt(RegExp.$1))) / 100; - print(JSON.stringify({progress: progress})) + print(JSON.stringify({ progress: progress })) } else if (line.match(/^\s*\#(.{1,60})\#\s*$/)) { // handle memo let memoText = RegExp.$1 - print(JSON.stringify({memo: memoText})) + print(JSON.stringify({ memo: memoText })) } - else { + else { // hack: wrap line with ANSI color to prevent JSON interpretation (default Cronicle behavior) print(json ? line : `\x1b[109m${line}\x1b[0m`) } @@ -88,7 +116,7 @@ const stderr = new Writable({ }) // env variables -let exclude = ['JOB_SECRET', 'SSH_HOST', 'SSH_KEY', 'PLUGIN_SECRET'] +let exclude = ['SSH_HOST', 'SSH_KEY', 'SSH_PASSWORD', 'DOCKER_PASSWORD'] let include = ['BASE_URL', 'BASE_APP_URL', 'DOCKER_HOST', 'PULL_IMAGE', 'KEEP_CONTAINER', 'IMAGE', 'ENTRYPOINT_PATH'] let vars = Object.entries(process.env) .filter(([k, v]) => ((k.startsWith('JOB_') || k.startsWith('DOCKER_') || k.startsWith('ARG') || include.indexOf(k) > -1) && exclude.indexOf(k) === -1)) @@ -108,7 +136,7 @@ const createOptions = { }, }; -if(!keepEntrypoint) { +if (!keepEntrypoint) { createOptions.Entrypoint = [ENTRYPOINT_PATH] createOptions.WorkingDir = path.dirname(ENTRYPOINT_PATH) } @@ -121,29 +149,23 @@ const dockerRun = async () => { // create tar archive for entrypoint script const pack = tar.pack() pack.entry({ name: ENTRYPOINT_PATH, mode: 0o755 }, script) - if(job.chain_data) { - pack.entry({name: path.join(path.dirname(ENTRYPOINT_PATH), 'chain_data')}, JSON.stringify(job.chain_data)) + if (job.chain_data) { + pack.entry({ name: path.join(path.dirname(ENTRYPOINT_PATH), 'chain_data') }, JSON.stringify(job.chain_data)) } pack.finalize() let chunks = [] for await (const data of pack) chunks.push(data) let arch = Buffer.concat(chunks) - - try { + + try { container = await docker.createContainer(createOptions) // copy entrypoint file to root directory container.putArchive(arch, { path: '/' }) + if(docker.modem.host) printInfo('docker host: ' + docker.modem.protocol + '://' + docker.modem.host) printInfo(`Container ready: name: [${createOptions.name}], image: [${imageName}], keep: ${!autoRemove}`) - // setTimeout(()=>{ - // docker.getContainer(cname).stop() - // }, 8000) - - // docker.getContainer(cname). - let stream = await container.attach({ stream: true, stdout: true, stderr: true }) container.modem.demuxStream(stream, stdout, stderr); - await container.start() let exit = await container.wait() @@ -185,12 +207,10 @@ async function main(image, onFinish) { if (autoPull) { // printWarning(`Image not found, pulling from registry`) try { - let pullStream = await docker.pull(image) + let pullStream = await docker.pull(image, {'authconfig': registryAuth}) docker.modem.followProgress(pullStream, onFinish, onProgress) } - catch (e) { - exit(e.message) - } + catch (e) { exit(e.message) } } else { printError(`No such image [${image}], pull it manually or check "Pull Image" option`) @@ -199,4 +219,6 @@ async function main(image, onFinish) { } } -main(imageName, dockerRun) \ No newline at end of file +// ------ MAIN ----- + +main(imageName, dockerRun) diff --git a/bin/sshx-plugin.js b/bin/sshx-plugin.js index ac24462..f2e8a9b 100644 --- a/bin/sshx-plugin.js +++ b/bin/sshx-plugin.js @@ -3,12 +3,12 @@ const { readFileSync } = require('fs'); const { Client } = require('ssh2'); const conn = new Client(); -const {EOL} = require('os') +const { EOL } = require('os') const JSONStream = require('pixl-json-stream'); const fs = require('fs') // read job info from stdin (sent by Cronicle engine) -const job = JSON.parse(fs.readFileSync(process.stdin.fd)) +const job = JSON.parse(fs.readFileSync(process.stdin.fd)) const print = (text) => process.stdout.write(text + EOL) const printInfo = (text) => process.stdout.write(`[INFO] \x1b[32m${text}\x1b[0m` + EOL) @@ -18,7 +18,7 @@ const printError = (text) => process.stdout.write(`\x1b[31m${text}\x1b[0m` + EOL let shuttingDown = false let hostInfo = process.env['SSH_HOST'] || process.env['JOB_ARG'] -if(! hostInfo) throw new Error("Host info is not provided. Specify SSH_HOST parameter or pass it via Workflow argument") +if (!hostInfo) throw new Error("Host info is not provided. Specify SSH_HOST parameter or pass it via Workflow argument") let killCmd = (process.env['KILL_CMD'] || '').trim() || 'pkill -s $$' // $$ will be resolved in bootstrap script @@ -26,7 +26,7 @@ hostInfo = process.env[hostInfo] || hostInfo // host info might be passed as nam let json = parseInt(process.env['JSON'] || '') -let ext = { powershell: 'ps1', csharp: 'cs', java: 'java', python: 'py', javascript: 'js'} // might be useful in Windows (TODO) +let ext = { powershell: 'ps1', csharp: 'cs', java: 'java', python: 'py', javascript: 'js' } // might be useful in Windows (TODO) let tmpFile = '/tmp/cronicle-' + process.env['JOB_ID'] + '.' + (ext[process.env['LANG']] || 'sh') @@ -38,14 +38,14 @@ let SCRIPT_BASE64 = Buffer.from(process.env['SCRIPT'] ?? '#!/usr/bin/env sh\nech let prefix = process.env['PREFIX'] || '' // generate stdin script to pass variables and user script in base64 format -let exclude = ['JOB_SECRET', 'SSH_HOST', 'SSH_KEY', 'PLUGIN_SECRET'] +let exclude = ['SSH_HOST', 'SSH_KEY', 'SSH_PASSWORD'] let include = ['BASE_URL', 'BASE_APP_URL'] process.env['JOB_CHAIN_DATA'] = JSON.stringify(job.chain_data) || 'has no data' let vars = Object.entries(process.env) -.filter(([k,v]) => ((k.startsWith('JOB_') || k.startsWith('SSH_') || k.startsWith('ARG') || include.indexOf(k) > -1) && exclude.indexOf(k) === -1)) -.map(([key, value]) => `export ${key}=$(printf "${Buffer.from(value).toString('base64')}" | base64 -di)`) -.join('\n') + .filter(([k, v]) => ((k.startsWith('JOB_') || k.startsWith('SSH_') || k.startsWith('ARG') || include.indexOf(k) > -1) && exclude.indexOf(k) === -1)) + .map(([key, value]) => `export ${key}=$(printf "${Buffer.from(value).toString('base64')}" | base64 -di)`) + .join('\n') let script = ` temp_file="${tmpFile}" @@ -83,128 +83,145 @@ function printJSONmessage(complete, code, desc) { stream.write({ complete: complete, code: code || 0, description: desc || "" }) } - if (!hostInfo.startsWith('sftp://')) hostInfo = 'sftp://' + hostInfo +if (!hostInfo.startsWith('ssh://')) hostInfo = 'ssh://' + hostInfo - let uri = new URL(hostInfo) +let uri = new URL(hostInfo) - let conf = { - host: uri.hostname, - port: parseInt(uri.port) || 22, - username: uri.username, - pty: true - } +let conf = { + host: uri.hostname, + port: parseInt(uri.port) || 22, + username: uri.username, + pty: true +} - if (uri.password || process.env['PLUGIN_SECRET']) conf.password = decodeURIComponent(uri.password) || process.env['PLUGIN_SECRET'] - if (process.env['SSH_KEY']) conf.privateKey = Buffer.from(process.env['SSH_KEY']) - if (uri.searchParams.get('privateKey')) conf.privateKey = readFileSync(String(uri.searchParams.get('privateKey'))) - if (uri.searchParams.get('passphrase')) conf.passphrase = uri.searchParams.get('passphrase') +// Resolve credential +// can be passed via secret +if(process.env['SSH_PASSWORD']) conf.password = process.env['SSH_PASSWORD'] +if(process.env['SSH_KEY']) conf.privateKey = Buffer.from(process.env['SSH_KEY']) +if(process.env['SSH_PASSPHRASE']) conf.passphrase = process.env['SSH_PASSPHRASE'] +// from URI +if(uri.password) conf.password = decodeURIComponent(uri.password) +if (uri.searchParams.get('privateKey')) conf.privateKey = readFileSync(String(uri.searchParams.get('privateKey'))) +if (uri.searchParams.get('passphrase')) conf.passphrase = uri.searchParams.get('passphrase') - conn.on('error', (err) => { // handle configuration errors - stream.write({ - complete: 1, - code: 1, - description: err.message - }); - shuttingDown = true - if(process.connected) process.disconnect() - }) +try { + conn.on('error', (err) => { // handle configuration errors + stream.write({ + complete: 1, + code: 1, + description: err.message + }); + shuttingDown = true + if (process.connected) process.disconnect() + }) - let streamRef = null + let streamRef = null - conn.on('ready', () => { - printInfo(`Connected to ${conf.host}`) + conn.on('ready', () => { + printInfo(`Connected to ${conf.host}`) - conn.exec(command, (err, stream) => { + conn.exec(command, (err, stream) => { - if (err) printJSONmessage(1, 1, err.message) + if (err) printJSONmessage(1, 1, err.message) - streamRef = stream + streamRef = stream - stream.on('close', (code, signal) => { + stream.on('close', (code, signal) => { - shuttingDown = true + shuttingDown = true - if (kill_timer) clearTimeout(kill_timer); + if (kill_timer) clearTimeout(kill_timer); - code = (code || signal || 0); + code = (code || signal || 0); - conn.end() - if(process.connected) process.disconnect() - - printJSONmessage(1, code, code ? `Script exited with code: ${code}; ${stderr_msg}` : "") + conn.end() + if (process.connected) process.disconnect() - }).on('data', (data) => { + printJSONmessage(1, code, code ? `Script exited with code: ${code}; ${stderr_msg}` : "") - String(data).trim().split('\n').forEach(line => { + }).on('data', (data) => { - if (line.trim().startsWith('trap:')) { - trapCmd = line.trim().substring(5) - printInfo(`Kill command set to: ${trapCmd}\x1b[0m`) - } + String(data).trim().split('\n').forEach(line => { - else if (line.match(/^\s*(\d+)\%\s*$/)) { // handle progress - let progress = Math.max(0, Math.min(100, parseInt(RegExp.$1))) / 100; - stream.write({ - progress: progress - }) - } - else if (line.match(/^\s*\#(.{1,60})\#\s*$/)) { // handle memo - let memoText = RegExp.$1 - stream.write({ - memo: memoText - }) - } - else { - // adding ANSI sequence (grey-ish color) to prevent JSON interpretation - print(json ? line : `\x1b[109m${line}\x1b[0m`) - } - }) // foreach + if (line.trim().startsWith('trap:')) { + trapCmd = line.trim().substring(5) + printInfo(`Kill command set to: ${trapCmd}\x1b[0m`) + } - }).stderr.on('data', (data) => { - let d = String(data).trim() - if (d) { - printError(d); // red - stderr_msg = d.split("\n")[0].substring(0, 128) + else if (line.match(/^\s*(\d+)\%\s*$/)) { // handle progress + let progress = Math.max(0, Math.min(100, parseInt(RegExp.$1))) / 100; + stream.write({ + progress: progress + }) } - }); - - stream.stdin.write(script + "\n") - stream.stdin.end() - - }) // ------- exec - }).connect(conf) - - // process should be connected for Windows compat - let sig = process.connected ? 'disconnect' : 'SIGTERM' - - process.on(sig, (signal) => { - - if(shuttingDown) return // if normal shutdown in progress - ignore - - printWarning(`Caugth ${sig}`) - if (trapCmd) { - printWarning(`Executing KILL command: ${trapCmd}`) - conn.exec(trapCmd, (err, trapStream) => { - if (err) { - printError("Failed to abort: ", err.message) - conn.end() - if(process.connected) process.disconnect() + else if (line.match(/^\s*\#(.{1,60})\#\s*$/)) { // handle memo + let memoText = RegExp.$1 + stream.write({ + memo: memoText + }) } - trapStream.on('data', (d) => { print(String(d)) }) - trapStream.stderr.on('data', (d) => { print(String(d)) }) - trapStream.on('exit', (cd) => { - if(cd) printError("! Kill command failed, you script may still run on remote host") - else printWarning("Kill command completed sucessfully") - conn.end() - if(process.connected) process.disconnect() - }) - }) - } - else { - printError("! Kill command is not detected.") - printError("You process may still run on remote host") + else { + // adding ANSI sequence (grey-ish color) to prevent JSON interpretation + print(json ? line : `\x1b[109m${line}\x1b[0m`) + } + }) // foreach + + }).stderr.on('data', (data) => { + let d = String(data).trim() + if (d) { + printError(d); // red + stderr_msg = d.split("\n")[0].substring(0, 128) + } + }); + + stream.stdin.write(script + "\n") + stream.stdin.end() + + }) // ------- exec + }).connect(conf) +} +catch (err) { + stream.write({ + complete: 1, + code: 1, + description: err.message + }); + if (process.connected) process.disconnect() + process.exit(1) +} + + +// process should be connected for Windows compat +let sig = process.connected ? 'disconnect' : 'SIGTERM' + +process.on(sig, (signal) => { + + if (shuttingDown) return // if normal shutdown in progress - ignore + + printWarning(`Caugth ${sig}`) + if (trapCmd) { + printWarning(`Executing KILL command: ${trapCmd}`) + conn.exec(trapCmd, (err, trapStream) => { + if (err) { + printError("Failed to abort: ", err.message) conn.end() - if(process.connected) process.disconnect() + if (process.connected) process.disconnect() } + trapStream.on('data', (d) => { print(String(d)) }) + trapStream.stderr.on('data', (d) => { print(String(d)) }) + trapStream.on('exit', (cd) => { + if (cd) printError("! Kill command failed, you script may still run on remote host") + else printWarning("Kill command completed sucessfully") + conn.end() + if (process.connected) process.disconnect() + }) }) + } + else { + printError("! Kill command is not detected.") + printError("You process may still run on remote host") + conn.end() + if (process.connected) process.disconnect() + } +}) diff --git a/bin/url-plugin.js b/bin/url-plugin.js index 48a3d51..207c0e8 100644 --- a/bin/url-plugin.js +++ b/bin/url-plugin.js @@ -47,7 +47,7 @@ stream.on('json', function(job) { if (params.headers) { // allow headers to be substituted using [placeholders] params.headers = Tools.sub( params.headers, job ); - + print("\nRequest Headers:\n" + params.headers.trim() + "\n"); params.headers.replace(/\r\n/g, "\n").trim().split(/\n/).forEach( function(pair) { if (pair.match(/^([^\:]+)\:\s*(.+)$/)) { @@ -55,6 +55,15 @@ stream.on('json', function(job) { } } ); } + + // set athentication header if set via secrets + if(process.env['AUTH']) { + process.env['AUTH'].replace(/\r\n/g, "\n").trim().split(/\n/).forEach( function(pair) { + if (pair.match(/^([^\:]+)\:\s*(.+)$/)) { + request.setHeader( RegExp.$1, RegExp.$2 ); + } + }) + } // follow redirects if (params.follow) request.setFollow( 32 ); diff --git a/sample_conf/setup.json b/sample_conf/setup.json index a7962cd..84d3de3 100644 --- a/sample_conf/setup.json +++ b/sample_conf/setup.json @@ -89,7 +89,6 @@ { "id":"script", "type":"textarea", "rows":10, "title":"Script", "value": "#!/bin/sh\n\necho \"Running SSHX job on $HOSTNAME\"\n\n# Specify conneciton info using URI format ([sftp://]user[:Password]@host[:port])\n# Can use directly or via ENV variable\n# URI/variable can also be passed via WF argument (if not specified in event)\n\n# If password in URI contains special character use escape characters (e.g. @ => %40)\n# You can also store password in PLUGIN_SECRET variable (instead of URI)\n# It can be set in plugin patameters on admin tab \n\n# If using ssh key\n# sftp://user@host:port?privateKey=/path/to/file&passphrase=Password)\n# You can also keep ssh key in [SSH Key] parameter\n\n# Kill Command parameter is used to properly handle job abortion\n# $$ is refering to PID of your script parent process (bootstrap script)\n" }, {"type":"select","id":"lang","title":"syntax","items":["shell","powershell","javascript","python","perl","groovy","java","csharp","scala","sql","yaml","dockerfile","json","props"],"value":"shell"}, {"type":"select","id":"theme","title":"theme","items":["default","gruvbox-dark","solarized light","solarized dark","darcula"],"value":"default"}, - { "id":"ssh_key", "type":"textarea", "rows":5, "title":"SSH Key", "value": "" }, { "id":"json", "type":"checkbox", "title":"Interpret JSON in Output", "value": 0 } ] } ],