From 93629ef267c41e5f71b233e6f4951181469c89b9 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Tue, 19 Mar 2024 21:21:53 -0400 Subject: [PATCH 01/31] sqlite bundle --- bundle | 23 ++++++++++++++++++++--- sample_conf/examples/storage.sqlite.json | 2 +- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/bundle b/bundle index 46085ea..c191a5c 100755 --- a/bundle +++ b/bundle @@ -41,6 +41,7 @@ while (( "$#" )); do --tools | -t ) tools=1 ;; # bundle up repair and migration tools --all | -a ) all=1 ;; # install all engines --restart | -r ) restart=1 ;; # start cronicle upon completion + --engine ) shift; engine=$1 ;; -*) echo "invalid parameter: $1"; usage ;; * ) dist=$1; x=$(($x + 1)) esac @@ -311,7 +312,7 @@ fi sqlDrivers=() sqlArgs=("--bundle" "--minify" "--platform=node" "--outdir=$dist/bin/engines") # exclude unused drivers -sqlArgs+=("--external:better-sqlite3" "--external:mysql" "--external:sqlite3") +sqlArgs+=("--external:better-sqlite3" "--external:mysql") if [ "$sql" = 1 ]; then oracle=1 @@ -347,7 +348,13 @@ else sqlArgs+=("--external:tedious") fi -if [[ ${#sqlDrivers[@]} -gt 0 || "$sqlite" = 1 ]]; then +if [ "$sqlite" = 1 ]; then + sqlDrivers+=("sqlite3") +else + sqlArgs+=("--external:sqlite3") +fi + +if [[ ${#sqlDrivers[@]} -gt 0 ]]; then sqlInstall=("install" "--no-save" "--loglevel" "silent" "knex") sqlInstall+=("${sqlDrivers[@]}") @@ -356,6 +363,9 @@ if [[ ${#sqlDrivers[@]} -gt 0 || "$sqlite" = 1 ]]; then echo " - bundling SQL Engine [${sqlDrivers[@]}]" npm "${sqlInstall[@]}" esbuild "${sqlArgs[@]}" + if [ "$sqlite" = 1 ]; then + cp -r node_modules/sqlite3/build $dist/bin/ + fi fi if [ "$redis" = 1 ]; then @@ -382,6 +392,12 @@ if [ ! -d $dist/conf ]; then chmod 400 $dist/conf/secret_key fi +# -- override storage engine if specified +if [ -f "sample_conf/examples/storage.$engine.json" ]; then + writehead "Overriding stoarge.json for $engine" + cp "sample_conf/examples/storage.$engine.json" $dist/conf/storage.json +fi + # ---- set up npm pac in dist if not yet, add some deps if needed cd $dist @@ -476,4 +492,5 @@ if [ "$test" = 1 ]; then node node_modules/pixl-unit/unit.js $dist/bin/test.js # remove test files rm $dist/bin/test.js -fi \ No newline at end of file +fi + diff --git a/sample_conf/examples/storage.sqlite.json b/sample_conf/examples/storage.sqlite.json index c9280a2..32cb8e8 100644 --- a/sample_conf/examples/storage.sqlite.json +++ b/sample_conf/examples/storage.sqlite.json @@ -8,7 +8,7 @@ "table": "cronicle", "useNullAsDefault": true, "connection": { - "filename": "/tmp/cronicle.db" + "filename": "cronicle.db" } } } \ No newline at end of file From 82ffe304c718af44109acc3c02e8f6cf8378f21b Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Thu, 21 Mar 2024 21:55:22 -0400 Subject: [PATCH 02/31] lmdb and sql fixes --- bin/storage-cli.js | 8 +++++- bin/storage-migrate.js | 19 ++++++++++-- bundle | 2 +- engines/Lmdb.js | 49 +++++++++++++++++-------------- engines/SQL.js | 65 +++++++++++++++++++++--------------------- 5 files changed, 85 insertions(+), 58 deletions(-) diff --git a/bin/storage-cli.js b/bin/storage-cli.js index 0888cb0..864feaf 100755 --- a/bin/storage-cli.js +++ b/bin/storage-cli.js @@ -47,7 +47,13 @@ args = args.get(); // simple hash // copy debug flag into config (for standalone) config.Storage.debug = args.debug; -// disable storage transactions for CLI +// indicate that you want to enable engine level transaction +// to pack multiple crud operation in one transaction +// this is default for setup/migration, to avoid use --notrx argument +if(cmd == 'install' || cmd == 'setup') config.Storage.trans = true +if(args.notrx) config.Storage.trans = false + +// disable storage transactions for CLI (this is storage level transaction) config.Storage.transactions = false; var print = function (msg) { diff --git a/bin/storage-migrate.js b/bin/storage-migrate.js index 1cd4dc2..c44bf3b 100644 --- a/bin/storage-migrate.js +++ b/bin/storage-migrate.js @@ -76,10 +76,13 @@ var StorageMigrator = { // massage config, override logger config.Storage.logger = self.logger; config.Storage.log_event_types = { all: 1 }; - + config.NewStorage.logger = self.logger; config.NewStorage.log_event_types = { all: 1 }; - + // if engine supports (sql and lmdb) begin transaction + config.NewStorage.trans = true + if(args.notrx) config.NewStorage.trans = false + // start both standalone storage instances async.series( [ @@ -100,8 +103,12 @@ var StorageMigrator = { self.logPrint( 3, "Switching to user: " + config.uid ); process.setuid( config.uid ); } + + self.logPrint(2, "before test"); self.testStorage(); + // self.startMigration(); + } ); // series }, @@ -110,18 +117,26 @@ var StorageMigrator = { // test both old and new storage var self = this; this.logDebug(3, "Testing storage engines"); + self.logPrint(2, "test begin test"); async.series( [ function(callback) { self.oldStorage.get('global/users', callback); + self.logPrint(2, "test old users"); }, function(callback) { self.newStorage.put('test/test1', { "foo1": "bar1" }, function(err) { if (err) return callback(err); + self.logPrint(2, "test new foo test"); + + // throw new Error('before del') self.newStorage.delete('test/test1', function(err) { + + self.logPrint(2, "test new before delete", err); if (err) return callback(err); + self.logPrint(2, "test new delete"); callback(); }); diff --git a/bundle b/bundle index c191a5c..1868e55 100755 --- a/bundle +++ b/bundle @@ -411,7 +411,7 @@ if [ ! -e 'package.json' ]; then fi if [ "$lmdb" = 1 ]; then - npm i lmdb --loglevel silent + npm i lmdb@2.9.4 --loglevel silent fi cd - &>/dev/null diff --git a/engines/Lmdb.js b/engines/Lmdb.js index f993b89..5818652 100644 --- a/engines/Lmdb.js +++ b/engines/Lmdb.js @@ -11,7 +11,6 @@ // }, const Component = require("pixl-server/component"); -// const Component = require("pixl-component"); const { Readable } = require('stream'); const lmdb = require("lmdb"); @@ -44,12 +43,19 @@ module.exports = class LmdbEngine extends Component { async setup(callback) { - // this.db = new Level(this.dbpath, { valueEncoding: 'json' }) + const self = this; + + let db = lmdb.open(this.dbpath, { valueEncoding: 'json' }) + this.db = db // // if db is in use by other process we'll get an error here // await this.db.open() - - this.db = lmdb.open(this.dbpath, this.config) + + if(this.storage.config.get('trans')) { + // beginTransaction + db.transactionSync(() => new Promise(resolve => self.commit = resolve)); + + } callback(); } @@ -77,7 +83,7 @@ module.exports = class LmdbEngine extends Component { err.message = "Failed to store object: " + key + ": " + err; self.logError('error', '' + err); if (callback) callback(err); - } + } } putStream(key, inp, callback) { @@ -109,8 +115,6 @@ module.exports = class LmdbEngine extends Component { else { callback(new NoSuchKeyError(key), null) } - // this.db.doesExist(key) ? - // callback(null, { mod: 1, len: 0 }) : callback(new NoSuchKeyError(key, 'head'), null) } catch (err) { err.message = "Failed to head key: " + key + ": " + err.message; @@ -122,7 +126,7 @@ module.exports = class LmdbEngine extends Component { get(key, callback) { // fetch LevelDB value by given key const self = this; - + key = this.prepKey(key); let isBinary = self.storage.isBinaryKey(key) @@ -148,7 +152,7 @@ module.exports = class LmdbEngine extends Component { finally { callback(getError, val) } - + } getStream(key, callback) { @@ -171,25 +175,25 @@ module.exports = class LmdbEngine extends Component { } - delete(key, callback) { + async delete(key, callback) { // delete LevelDb key given key var self = this; key = this.prepKey(key); this.logDebug(9, "Deleting LevelDb Object: " + key); - let delError = null + let delError = null; - this.db.del(key, function (err, deleted) { - if (!err && !deleted) delError = new NoSuchKeyError(key) - if (err) { - self.logError('lmdb', "Failed to delete object: " + key + ": " + err); - delError = err - } - else self.logDebug(9, "Delete complete: " + key); + try { + delError = await this.db.del(key) ? self.logDebug(9, "Delete complete: " + key) : new NoSuchKeyError(key) + } + catch (err) { + self.logDebug(9, "Delete complete: " + key); + delError = err + } + + callback(delError) - callback(delError); - }); } runMaintenance(callback) { @@ -197,10 +201,11 @@ module.exports = class LmdbEngine extends Component { callback(); } - shutdown(callback) { + async shutdown(callback) { // shutdown storage this.logDebug(2, "Closing Lmdb"); - if (this.db) this.db.close(); + if(this.commit) await this.commit(); + if (this.db) await this.db.close(); callback(); } diff --git a/engines/SQL.js b/engines/SQL.js index 3a96c75..6e30dbd 100644 --- a/engines/SQL.js +++ b/engines/SQL.js @@ -63,8 +63,9 @@ module.exports = class SQLEngine extends Component { // setup SQL connection const self = this; const sql_config = this.config.get(); - - this.db = knex(sql_config) + + let db = knex(sql_config); + this.db = db; this.db.client.pool.on('createSuccess', () => { self.logDebug(3, "SQL connected successfully") @@ -124,6 +125,8 @@ module.exports = class SQLEngine extends Component { }) } + if(self.storage.config.get('trans')) this.trx = await db.transaction() + if (!self.storage.started) return callback(); } @@ -139,9 +142,11 @@ module.exports = class SQLEngine extends Component { */ async put(key, value, callback) { // store key+value in SQL - var self = this; + const self = this; + const db = this.trx || this.db + key = this.prepKey(key); - + if (this.storage.isBinaryKey(key)) { this.logDebug(9, "Storing SQL Binary Object: " + key, '' + value.length + ' bytes'); } @@ -153,10 +158,10 @@ module.exports = class SQLEngine extends Component { // For oracle/mssql use MERGE statement, for other drivers use "INSEERT/ON CONFLICT" mechanism try { if (this.mergeStmt) { // this.client === 'mssql' || this.client === 'oracledb' - await this.db.raw(this.mergeStmt, [ key, Buffer.from(value)]) + await db.raw(this.mergeStmt, [ key, Buffer.from(value)]) } else { - await this.db(this.tableName) + await db(this.tableName) .insert({ K: key, V: Buffer.from(value), updated: this.db.fn.now() }) .onConflict('K') .merge() @@ -176,8 +181,8 @@ module.exports = class SQLEngine extends Component { putStream(key, inp, callback) { // store key+value in SQL using read stream - var self = this; - + const self = this; + // There is no common way to stream BLOBs from SQL // So, we have to do this the RAM-hard way... @@ -193,13 +198,15 @@ module.exports = class SQLEngine extends Component { async head(key, callback) { // head value by given key. Just return blob size - var self = this; + const self = this; + const db = this.trx || this.db + key = this.prepKey(key); try { - let rows = await this.db(this.tableName).where('K', key).select([ - this.db.raw(`${this.getBlobSizeFn} as len`), - this.db.raw('1 as mod') + let rows = await db(this.tableName).where('K', key).select([ + db.raw(`${this.getBlobSizeFn} as len`), + db.raw('1 as mod') ]) if (rows.length > 0) { callback(null, rows[0]); @@ -221,13 +228,14 @@ module.exports = class SQLEngine extends Component { async get(key, callback) { // fetch SQL value given key - var self = this; + const self = this; + const db = this.trx || this.db key = this.prepKey(key); - + this.logDebug(9, "Fetching SQL Object: " + key); try { - let data = (await this.db(this.tableName).where('K', key).select(["K as key", 'V as value']))[0] // expected {key: key, value: value} + let data = (await db(this.tableName).where('K', key).select(["K as key", 'V as value']))[0] // expected {key: key, value: value} let result = (data || {}).value if (result) { if (self.storage.isBinaryKey(key)) { @@ -328,32 +336,24 @@ module.exports = class SQLEngine extends Component { async delete(key, callback) { // delete SQL key given key - var self = this; + const self = this; + const db = this.trx || this.db key = this.prepKey(key); this.logDebug(9, "Deleting SQL Object: " + key); - let ERR + let delError try { - let d = await this.db(this.tableName).where('K', key).del() - - if (d > 0) { - self.logDebug(9, "Delete complete: " + key); - if (callback) callback(null) - } else { - ERR = new Error("Failed to fetch key: " + key + ": Not found"); - ERR.code = "NoSuchKey"; - - } - + let d = await db(this.tableName).where('K', key).del() + delError = d > 0 ? self.logDebug(9, "Delete complete: " + key) : new Error("Failed to fetch key: " + key + ": Not found"); } catch (err) { self.logError('sql', "Failed to delete object: " + key + ": " + err); - ERR = err + delError = err } - if (callback) callback(ERR) + callback(delError) } @@ -362,10 +362,11 @@ module.exports = class SQLEngine extends Component { callback(); } - shutdown(callback) { + async shutdown(callback) { // shutdown storage this.logDebug(2, "Shutting down SQL"); - this.db.destroy() + if(this.trx) await this.trx.commit() + await this.db.destroy() callback(); } From 9672ea75be2456d241b413d503970859ce79117e Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Fri, 22 Mar 2024 11:39:15 -0400 Subject: [PATCH 03/31] bundle bash : added --help --- bundle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bundle b/bundle index 1868e55..b7abd16 100755 --- a/bundle +++ b/bundle @@ -3,7 +3,14 @@ cd "$(dirname "$0")" usage() { - echo "Usage: $0 /path/to/dist [--s3 | --lmdb | --level | --sftp | --sql | --dev | --verbose]" + echo "Usage: $0 /path/to/dist" + echo " [ --s3 | --lmdb | --level | --sftp | --sqlite | --mysql | --oracle | --mysql | --pgsql ] # bundle 1 or more storage engine" + echo " [ --dev ] # avoid minification, add verbosity to esbuild/npm" + echo " [ --versbose ] # add verbosity" + echo " [ --tool ] # bundle migration and detahced job tools" + echo " [ --restart ] # restart cronicle upon bundle completion (in dev mode)" + echo " [ --engine engine] # copy sample_conf/example/storage.engine.json to dist/conf/storage.json (engine: fs/sqlite/s3/...)" + echo " [ --test ] # run unit test upon bundle completion" exit 1 } @@ -42,6 +49,7 @@ while (( "$#" )); do --all | -a ) all=1 ;; # install all engines --restart | -r ) restart=1 ;; # start cronicle upon completion --engine ) shift; engine=$1 ;; + --help ) usage ;; -*) echo "invalid parameter: $1"; usage ;; * ) dist=$1; x=$(($x + 1)) esac From 19acd9d22822c7f782dcebc662e38035281c3ab6 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Fri, 22 Mar 2024 11:39:41 -0400 Subject: [PATCH 04/31] adjust sample conf to work with docker --- sample_conf/examples/storage.mssql.json | 6 +++--- sample_conf/examples/storage.mysql.json | 6 +++--- sample_conf/examples/storage.oracle.json | 5 ++--- sample_conf/examples/storage.postgres.json | 6 +++--- sample_conf/examples/storage.s3.json | 6 +++--- sample_conf/examples/storage.sftp.json | 10 +++++----- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/sample_conf/examples/storage.mssql.json b/sample_conf/examples/storage.mssql.json index 215e838..47b47ab 100644 --- a/sample_conf/examples/storage.mssql.json +++ b/sample_conf/examples/storage.mssql.json @@ -15,9 +15,9 @@ "connection": { "host": "localhost", "port": 1433, - "user": "cronicle", - "password": "P@ssword", - "database": "cronicle" + "user": "SA", + "password": "StrongPassword123!", + "database": "master" } } } \ No newline at end of file diff --git a/sample_conf/examples/storage.mysql.json b/sample_conf/examples/storage.mysql.json index 41a13c3..da61172 100644 --- a/sample_conf/examples/storage.mysql.json +++ b/sample_conf/examples/storage.mysql.json @@ -15,9 +15,9 @@ "connection": { "host": "localhost", "port": 3306, - "user": "root", - "password": "P@ssword", - "database": "cronicle" + "user": "cronicle", + "password": "cronicle", + "database": "cron" } } } \ No newline at end of file diff --git a/sample_conf/examples/storage.oracle.json b/sample_conf/examples/storage.oracle.json index 618244a..9076bec 100644 --- a/sample_conf/examples/storage.oracle.json +++ b/sample_conf/examples/storage.oracle.json @@ -13,10 +13,9 @@ "client": "oracledb", "table": "cronicle", "connection": { - "connectString": "(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=localhost...)))", + "connectString": "localhost:1521/CRON", "user": "cronicle", - "password": "P@ssword", - "database": "CRONICLE_SCHEMA" + "password": "cronicle" } } } \ No newline at end of file diff --git a/sample_conf/examples/storage.postgres.json b/sample_conf/examples/storage.postgres.json index 3635e33..b5cccab 100644 --- a/sample_conf/examples/storage.postgres.json +++ b/sample_conf/examples/storage.postgres.json @@ -15,9 +15,9 @@ "connection": { "host": "localhost", "port": 5432, - "user": "postgres", - "password": "P@ssword", - "database": "postgres" + "user": "cronicle", + "password": "cronicle", + "database": "cron" } } } \ No newline at end of file diff --git a/sample_conf/examples/storage.s3.json b/sample_conf/examples/storage.s3.json index bd2a2c1..9ad01b8 100644 --- a/sample_conf/examples/storage.s3.json +++ b/sample_conf/examples/storage.s3.json @@ -1,14 +1,14 @@ { "engine": "S3", "AWS": { - "endpoint": "http://minio:9000", + "endpoint": "http://localhost:9000", "endpointPrefix": false, "forcePathStyle": true, "region": "us-east-1", "hostPrefixEnabled": false, "credentials": { - "secretAccessKey": "minioadmin", - "accessKeyId": "minioadmin" + "accessKeyId": "minioadmin", + "secretAccessKey": "minioadmin" }, "correctClockSkew": true, "maxRetries": 5, diff --git a/sample_conf/examples/storage.sftp.json b/sample_conf/examples/storage.sftp.json index 1537ed2..03ea754 100644 --- a/sample_conf/examples/storage.sftp.json +++ b/sample_conf/examples/storage.sftp.json @@ -10,13 +10,13 @@ "expire_set": 1 }, "Sftp": { - "base_dir": "data", + "base_dir": "cronicle", "key_namespaces": 1, "connection": { - "host": "192.168.0.1", - "port": 22, - "username": "cronicle", - "password": "P@ssword" + "host": "localhost", + "port": 2022, + "username": "cron", + "password": "cronpass" } } } \ No newline at end of file From 7912cbdaf6409821cc4e809805196c6e09ff510e Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Fri, 22 Mar 2024 11:40:05 -0400 Subject: [PATCH 05/31] fix delete on lmdb/sql --- engines/Lmdb.js | 7 ++++--- engines/SQL.js | 33 ++++++++++++++++++++------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/engines/Lmdb.js b/engines/Lmdb.js index 5818652..a50b2c9 100644 --- a/engines/Lmdb.js +++ b/engines/Lmdb.js @@ -175,9 +175,10 @@ module.exports = class LmdbEngine extends Component { } - async delete(key, callback) { - // delete LevelDb key given key - var self = this; + async delete(key, callback) { + + const self = this; + key = this.prepKey(key); this.logDebug(9, "Deleting LevelDb Object: " + key); diff --git a/engines/SQL.js b/engines/SQL.js index 6e30dbd..74d5e2e 100644 --- a/engines/SQL.js +++ b/engines/SQL.js @@ -15,6 +15,13 @@ const Component = require("pixl-server/component"); const { knex, Knex } = require('knex') const { Readable } = require('stream'); +class NoSuchKeyError extends Error { + constructor(key = 'key', crudOp = 'Fetch') { + super(`Failed to ${crudOp} key: ${key}: Not found`) + } + code = "NoSuchKey" +} + module.exports = class SQLEngine extends Component { __name = 'SQLEngine' @@ -212,9 +219,9 @@ module.exports = class SQLEngine extends Component { callback(null, rows[0]); } else { - let err = new Error("Failed to head key: " + key + ": Not found"); - err.code = "NoSuchKey"; - callback(err, null); + // let err = new Error("Failed to head key: " + key + ": Not found"); + // err.code = "NoSuchKey"; + callback(new NoSuchKeyError(key, 'head') , null); } } @@ -254,9 +261,9 @@ module.exports = class SQLEngine extends Component { } } else { - let err = new Error("Failed to fetch key: " + key + ": Not found"); - err.code = "NoSuchKey"; - callback(err, null); + // let err = new Error("Failed to fetch key: " + key + ": Not found"); + // err.code = "NoSuchKey"; + callback(new NoSuchKeyError(key, 'fetch'), null); } } @@ -284,9 +291,9 @@ module.exports = class SQLEngine extends Component { } else if (!buf) { // record not found - let ERR = new Error("Failed to fetch key: " + key + ": Not found"); - ERR.code = "NoSuchKey"; - return callback(ERR, null); + // let ERR = new Error("Failed to fetch key: " + key + ": Not found"); + // ERR.code = "NoSuchKey"; + return callback(new NoSuchKeyError(key, 'fetch'), null); } let stream = Readable.from(buf) // new BufferStream(buf); @@ -310,9 +317,9 @@ module.exports = class SQLEngine extends Component { } else if (!buf) { // record not found - let ERR = new Error("Failed to fetch key: " + key + ": Not found"); - ERR.code = "NoSuchKey"; - return callback(ERR, null); + // let ERR = new Error("Failed to fetch key: " + key + ": Not found"); + // ERR.code = "NoSuchKey"; + return callback(new NoSuchKeyError(key, 'fetch'), null); } // validate byte range, now that we have the head info @@ -346,7 +353,7 @@ module.exports = class SQLEngine extends Component { try { let d = await db(this.tableName).where('K', key).del() - delError = d > 0 ? self.logDebug(9, "Delete complete: " + key) : new Error("Failed to fetch key: " + key + ": Not found"); + delError = d > 0 ? self.logDebug(9, "Delete complete: " + key) : new NoSuchKeyError(key, 'fetch') } catch (err) { self.logError('sql', "Failed to delete object: " + key + ": " + err); From 983211fcd7baf683e3c97f8070cd33820ea13f21 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Fri, 22 Mar 2024 11:40:29 -0400 Subject: [PATCH 06/31] migrate.js add to/from arg --- bin/storage-migrate.js | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/bin/storage-migrate.js b/bin/storage-migrate.js index c44bf3b..89364ab 100644 --- a/bin/storage-migrate.js +++ b/bin/storage-migrate.js @@ -24,6 +24,20 @@ var cli = require('pixl-cli'); var args = cli.args; cli.global(); + +// check if --from/--to args + +let oldConfPath; +let newConfPath; +if(args.from) { + if(fs.existsSync(Path.resolve(args.from))) oldConfPath = Path.resolve(args.from) + else { console.log("config file does not exist:", Path.resolve(args.from)); process.exit(1)} +} +if(args.to) { + if(fs.existsSync(Path.resolve(args.to))) newConfPath = Path.resolve(args.to) + else { console.log("config file does not exist:", Path.resolve(args.to)); process.exit(1)} +} + var StandaloneStorage = require('pixl-server-storage/standalone'); // chdir to the proper server root dir @@ -59,7 +73,9 @@ var StorageMigrator = { this.logPrint(1, "Cronicle Storage Migration Script v" + this.version + " starting up"); this.logPrint(2, "Starting storage engines"); + if(oldConfPath) config.Storage = require(oldConfPath) // custom storage config file if (!config.Storage) this.fatal("Your Cronicle configuration lacks a 'Storage' property"); + if (newConfPath) config.NewStorage = require(newConfPath) if (!config.NewStorage) this.fatal("Your Cronicle configuration lacks a 'NewStorage' property."); if (config.uid && (process.getuid() != 0)) { @@ -157,7 +173,7 @@ var StorageMigrator = { var self = this; this.logPrint(3, "Starting migration"); - this.timeStart = Tools.timeNow(true); + this.timeStart = Date.now() this.numRecords = 0; this.copyKey( 'global/state', { ignore: true } ); @@ -276,13 +292,14 @@ var StorageMigrator = { finish: function() { // all done var self = this; - var elapsed = Tools.timeNow(true) - this.timeStart; + var elapsed = Date.now() - this.timeStart; + var dur = elapsed < 2000 ? `${elapsed} ms` : Tools.getTextFromSeconds(elapsed/1000, false, true) cli.progress.end(); print("\n"); this.logPrint(1, "Storage migration complete!"); - this.logPrint(2, Tools.commify(this.numRecords) + " total records copied in " + Tools.getTextFromSeconds(elapsed, false, true) + "."); + this.logPrint(2, Tools.commify(this.numRecords) + " total records copied in " + dur + "."); this.logPrint(4, "You should now overwrite 'Storage' with 'NewStorage' in your config.json."); this.logPrint(3, "Shutting down"); From 7448ae8aa841e92a67d0624478f91e9e926b372d Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Fri, 22 Mar 2024 11:40:50 -0400 Subject: [PATCH 07/31] random event - respect category --- htdocs/js/pages/Schedule.class.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/htdocs/js/pages/Schedule.class.js b/htdocs/js/pages/Schedule.class.js index 25857db..232f0ea 100644 --- a/htdocs/js/pages/Schedule.class.js +++ b/htdocs/js/pages/Schedule.class.js @@ -2757,7 +2757,7 @@ Class.subclass(Page.Base, "Page.Schedule", { "max_children": 1, "timeout": 3600, "catch_up": 0, "queue_max": 1000, "timezone": "America/New_York", "plugin": "testplug", "title": event_title, - "category": "general", + "category": $("#fe_sch_cat").val() || "general", "target": "allgrp", "algo": "random", "multiplex": 0, "stagger": 0, "retries": 0, "retry_delay": 0, "detached": 0, "queue": 0, "chain": "", "chain_error": "", "notify_success": "", "notify_fail": "", "web_hook": "", "cpu_limit": 0, "cpu_sustain": 0, "memory_limit": 0, "memory_sustain": 0, "log_max_size": 0, "notes": "Randomly Generated Job", From 96f0c64b2eddfdcb636df883705d0a899fae7a1d Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Fri, 22 Mar 2024 16:59:18 -0400 Subject: [PATCH 08/31] migration cleanup --- bin/storage-migrate.js | 12 +++--------- bundle | 3 ++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/bin/storage-migrate.js b/bin/storage-migrate.js index 89364ab..8b0cd97 100644 --- a/bin/storage-migrate.js +++ b/bin/storage-migrate.js @@ -118,12 +118,8 @@ var StorageMigrator = { if (config.uid && (process.getuid() == 0)) { self.logPrint( 3, "Switching to user: " + config.uid ); process.setuid( config.uid ); - } - - self.logPrint(2, "before test"); - - self.testStorage(); - // self.startMigration(); + } + self.testStorage(); } ); // series @@ -133,13 +129,11 @@ var StorageMigrator = { // test both old and new storage var self = this; this.logDebug(3, "Testing storage engines"); - self.logPrint(2, "test begin test"); - + async.series( [ function(callback) { self.oldStorage.get('global/users', callback); - self.logPrint(2, "test old users"); }, function(callback) { self.newStorage.put('test/test1', { "foo1": "bar1" }, function(err) { diff --git a/bundle b/bundle index b7abd16..84eefd2 100755 --- a/bundle +++ b/bundle @@ -142,6 +142,7 @@ mkdir -p $dist cp -r htdocs $dist/ cp -r bin $dist/ cp package.json $dist/bin/ +rm $dist/bin/storage-migrate.js $dist/bin/run-detached.js # ------------------------------- @@ -327,9 +328,9 @@ if [ "$sql" = 1 ]; then mssql=1 mysql=1 pgsql=1 + sqlite=1 fi -# for sqlite - just bundle plain SQL engine with no driver # driver need to be installed separetly (cannot bundle native libs) if [ "$mysql" = 1 ]; then From 205396f7babf05b81865f99df124e6b79091be1d Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Fri, 22 Mar 2024 16:59:54 -0400 Subject: [PATCH 09/31] bunlde for windows help + sqlite --- bundle.ps1 | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/bundle.ps1 b/bundle.ps1 index 8a381b8..26a46d9 100644 --- a/bundle.ps1 +++ b/bundle.ps1 @@ -14,17 +14,34 @@ param( [switch]$Level, # bundle level db engine [switch]$Lmdb, # bundle lmdb engine * [switch]$Sftp, # bundle sftp engine + [string][ValidateSet("s3","lmdb","level","sqlite","oracle","mysql","mssql","pgsql","sftp","redis")]$Engine, # copy storage.engine.json from sample_conf to dist [switch]$Force, # force reinstall if something is broken in node_modules [switch]$Dev, # prevent minificaiton and add verbosity [Switch]$Restart, # for dev purposes only: it will force kill cronicle if running, and start it over again once bundling is complete [Switch]$Tools, # bundle repair/migrate tools [Switch]$All, # bundle all storage engines and tools [switch]$V, # verbose - [switch]$Test, # run unit test at the end - [ValidateSet("warning", "debug", "info", "warning", "error","silent")][string]$ESBuildLogLevel = "warning" + [switch]$Test, # run unit test at the end + [ValidateSet("warning", "debug", "info", "warning", "error","silent")][string]$ESBuildLogLevel = "warning", + [switch]$Help ) +if($Help) { + Write-Host "Usage: ./bundle.ps1 [path\to\dist] # default bundle location is dist" + Write-Host " [ -S3 | -Redis | -Lmdb | -Level | -Redis | -Sftp ] # 1 or more storage engines (FS always added)" + Write-Host " [ -Mysql | -Pgsql | -Sqlite | -Oracle | -MSSQL ] # SQL storage engines" + Write-Host " [ -Engine engine ] # (for dev) copy storage engine file from sample_conf to dist/conf/storage.json, engine is s3,lmdb,sqlite,..." + Write-Host " [ -SQL | -All] # bundle all sql or just all engines " + Write-Host " [ -Force ] # force reinstall if something is broken in node_modules " + Write-Host " [ -Dev ] # prevent minificaiton and add verbosity" + Write-Host " [ -Restart ] for dev purposes only: it will force kill cronicle if running, and start it over again once bundling is complete" + Write-Host " [ -V ] # add verbosity" + Write-Host " [ -Tools ] # bundle repair/migrate tools" + Write-Host " [ -Help ] # see this message again" + exit +} + $ErrorActionPreference = 'Stop' Write-Host "-----------------------------------------" @@ -299,7 +316,7 @@ if ($Level.IsPresent) { $sqlDrivers = [System.Collections.ArrayList]::new() $sqlArgs = [System.Collections.ArrayList]::new(@("--bundle", "--minify", "--platform=node", "--outdir=$Path/bin/engines")) # exclude unused drivers -$sqlArgs.AddRange(@("--external:better-sqlite3", "--external:mysql", "--external:sqlite3")) +$sqlArgs.AddRange(@("--external:better-sqlite3", "--external:mysql")) if($SQL) { $Oracle = $MSSQL = $Mysql = $Pgsql = $true } @@ -307,9 +324,10 @@ $Mysql ? $sqlDrivers.Add("mysql2"): $sqlArgs.Add("--external:mysql2") | Out-Null $Pgsql ? $sqlDrivers.AddRange(@("pg", "pg-query-stream")) : $sqlArgs.AddRange(@("--external:pg", "--external:pg-query-stream")) | Out-Null $Oracle ? $sqlDrivers.Add("oracledb") : $sqlArgs.Add("--external:oracledb") | Out-Null $MSSQL ? $sqlDrivers.Add("tedious") : $sqlArgs.Add("--external:tedious") | Out-Null +$Sqlite ? $sqlDrivers.Add("sqlite3") : $sqlArgs.Add("--external:sqlite3") | Out-Null # bundle SQL engine if at least 1 SQL driver selected -if($sqlDrivers.Count -gt 0 -OR $Sqlite) { +if($sqlDrivers.Count -gt 0) { $sqlInstall = [System.Collections.ArrayList]::new(@("install", "--no-save", "--loglevel", "silent", "knex")) $sqlInstall.AddRange($sqlDrivers) | Out-Null $sqlArgs.Add("engines/SQL.js") | Out-Null @@ -317,6 +335,9 @@ if($sqlDrivers.Count -gt 0 -OR $Sqlite) { Write-Host " - bundling SQL Engine [$($sqlDrivers -join ",")]" & npm $sqlInstall & esbuild $sqlArgs + if($Sqlite.IsPresent) { + Copy-Item -Recurse -Force node_modules/sqlite3/build $Path/bin/ + } } # Lmdb, need to install lmdb separetly (npm i lmdb) @@ -340,6 +361,10 @@ if (!(Test-Path $Path/conf)) { } +if(Test-Path "sample_conf\examples\storage.$Engine.json") { + Copy-Item -Force "sample_conf\examples\storage.$Engine.json" $Path\conf\storage.json +} + # --- CRONICLE.JS Write-Host "`n ---- Bundling cronicle.js`n" esbuild --bundle $minify --keep-names --platform=node --outfile=$Path/bin/cronicle.js lib/main.js @@ -372,7 +397,7 @@ if(!(Test-Path "package.json")) { npm pkg set main="bin/cronicle.js" npm pkg set scripts.start="node bin/cronicle.js --foreground --echo --manager --color" } -if($Lmdb.IsPresent) { npm i lmdb --loglevel silent} +if($Lmdb.IsPresent) { npm i "lmdb@2.9.4" --loglevel silent} Pop-Location From 05c0211fcdfd9d21b6f508a3acefb61761b564e0 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Sat, 23 Mar 2024 14:52:45 -0400 Subject: [PATCH 10/31] add storage info on server tab --- lib/comm.js | 6 ++++-- lib/engine.js | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/comm.js b/lib/comm.js index dcd83c6..4c1fd33 100644 --- a/lib/comm.js +++ b/lib/comm.js @@ -481,7 +481,8 @@ module.exports = Class.create({ data: this.multi.data || {}, disabled: 0, pid: process.pid, - nodev: process.version + nodev: process.version, + engine: this.storage.config.get('engine') || 'unknown' }; // then add all workers @@ -496,7 +497,8 @@ module.exports = Class.create({ data: worker.data || {}, disabled: worker.disabled || 0, pid: worker.pid || -1, - nodev: worker.nodev || 'NA' + // nodev: worker.nodev || 'NA', + engine: worker.engine || 'unknown' }; } // unique hostname } // foreach worker diff --git a/lib/engine.js b/lib/engine.js index 526c298..7c5c267 100644 --- a/lib/engine.js +++ b/lib/engine.js @@ -416,6 +416,7 @@ module.exports = Class.create({ status.data = this.multi.data; status.uptime = now - (this.server.started || now); status.nodev = process.version; + status.engine = this.storage.config.get('engine') || 'unknown' status.pid = process.pid; } From bc38558d654ce6235459a46a1b0b2739fc309c62 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Sat, 23 Mar 2024 14:53:29 -0400 Subject: [PATCH 11/31] add plugin import --- htdocs/js/pages/admin/Plugins.js | 60 +++++++++++++++++++++++++++++++- htdocs/js/pages/admin/Servers.js | 3 +- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/htdocs/js/pages/admin/Plugins.js b/htdocs/js/pages/admin/Plugins.js index 88edf92..1b2cf90 100644 --- a/htdocs/js/pages/admin/Plugins.js +++ b/htdocs/js/pages/admin/Plugins.js @@ -78,6 +78,8 @@ Class.add( Page.Admin, { html += '
'; html += '
'; html += ''; + html += '' + html += ''; html += '
  Add New Plugin...
 
   from JSON
'; html += ''; // padding @@ -97,6 +99,62 @@ Class.add( Page.Admin, { this.plugin = this.plugins[idx]; this.show_delete_plugin_dialog(); }, + + setImportEditor: function() { + + let editor = CodeMirror.fromTextArea(document.getElementById("plugin_import"), { + mode: 'application/json', + styleActiveLine: true, + lineWrapping: false, + scrollbarStyle: "overlay", + lineNumbers: false, + theme: app.getPref('theme') == 'dark' ? 'gruvbox-dark' : 'default', + matchBrackets: true, + gutters: [''], + lint: true + }) + + editor.on('change', function(cm){ + document.getElementById("plugin_import").value = cm.getValue(); + }); + + editor.setSize('52vw', '52vh') + + }, + + import_plugin: function (args) { + + const self = this; + + setTimeout(() => self.setImportEditor(), 30) + app.confirm(`Import Plugin from JSON

+ + `, '', "Import", function (result) { + if (result) { + var importData = document.getElementById('plugin_import').value; + let plugin; + try { plugin = JSON.parse(importData) + } catch (e) { + return app.doError("Invalid JSON: " + e.message) + } + delete plugin.id + app.showProgress(1.0, "Importing..."); + app.api.post('app/create_plugin', plugin, function (resp) { + app.hideProgress(); + + report = `Plugin ${resp.id} created` + + setTimeout(function () { + Nav.go('#Admin?sub=plugins', 'force'); + app.show_info(`
${report}
`, ''); + + }, 50); + + }); + } + }); + }, + show_delete_plugin_dialog: function() { // delete selected plugin @@ -166,7 +224,7 @@ Class.add( Page.Admin, { html += '
Cancel
'; html += ' '; html += '
  Create Plugin
'; - html += ''; + html += ''; html += ''; html += ''; diff --git a/htdocs/js/pages/admin/Servers.js b/htdocs/js/pages/admin/Servers.js index be1a078..7deb2ea 100644 --- a/htdocs/js/pages/admin/Servers.js +++ b/htdocs/js/pages/admin/Servers.js @@ -29,7 +29,7 @@ Class.add( Page.Admin, { // Active Server Cluster - var cols = ['Hostname', 'IP Address', 'PID', 'Node', 'Groups', 'Status', 'Active Jobs', 'Uptime', 'CPU', 'Mem', 'Actions']; + var cols = ['Hostname', 'IP Address', 'PID', 'Node', 'Engine', 'Groups', 'Status', 'Active Jobs', 'Uptime', 'CPU', 'Mem', 'Actions']; html += '
'; html += 'Server Cluster'; @@ -108,6 +108,7 @@ Class.add( Page.Admin, { (server.ip || 'n/a').replace(/^\:\:ffff\:(\d+\.\d+\.\d+\.\d+)$/, '$1'), server.pid, server.nodev, + server.engine || '', group_names.length ? group_names.join(', ') : '(None)', server.manager ? ' Manager' : (eligible ? 'Backup' : 'Worker'), num_jobs ? commify( num_jobs ) : '(None)', From c0999d782c02819a75147c17322784467c92a484 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Sat, 23 Mar 2024 14:54:11 -0400 Subject: [PATCH 12/31] manager cli add args --- bin/manager | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) mode change 100644 => 100755 bin/manager diff --git a/bin/manager b/bin/manager old mode 100644 new mode 100755 index 6f33b60..012ccb2 --- a/bin/manager +++ b/bin/manager @@ -1,6 +1,43 @@ #!/bin/bash HOMEDIR="$(dirname "$(cd -- "$(dirname "$(readlink -f "$0")")" && (pwd -P 2>/dev/null || pwd))")" +usage() { + echo "USAGE: ./manager [ --port 3012 ] [ --storage /path/to/storage.json ] [ --key secreKey ] [ --color ] " + exit 1 +} + +color=0 + +while (( "$#" )); do + case $1 in + --color ) color=1 ;; + --port ) shift; port=$1 ;; + --key ) shift; key=$1 ;; + --storage ) shift; storage=$1 ;; + --help ) usage ;; + -*) echo "invalid parameter: $1"; usage ;; + esac +shift +done + +if [[ $port ]]; then + export CRONICLE_WebServer__http_port="$port" + echo "Custom port set: $port" +fi + +if [[ $storage ]]; then + if [[ -f $storage ]]; then + export CRONICLE_storage_config="$(realpath $storage)" + echo "Custom storage set: $(basename $storage)" + else echo "Error: $storage file doesn't exist"; exit 1 + fi +fi + +if [[ $key ]]; then + export CRONICLE_secret_key=$key + echo "Custom secret key set: *****" +fi + # pull data from git if needed # if [ ! -d data/global ] && [ -v GIT_REPO ]; then # git clone $GIT_REPO $HOMEDIR/data @@ -33,12 +70,8 @@ if [ -f "$HOMEDIR/bin/cronicle.js" ]; then # echo "starting bundle" fi -# check for custom http port (pass as first arg) -if [ -n "$1" ]; then - export CRONICLE_WebServer__http_port="$1" -fi #$HOMEDIR/bin/control.sh start -exec $BINARY --echo --foreground --manager --color 1 +exec $BINARY --echo --foreground --manager --color $color From a1e2dd9b0d6e2082d4e8b094853037542c983185 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Sat, 23 Mar 2024 14:54:22 -0400 Subject: [PATCH 13/31] bundle cleanup --- bundle | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/bundle b/bundle index 84eefd2..162cb94 100755 --- a/bundle +++ b/bundle @@ -3,7 +3,7 @@ cd "$(dirname "$0")" usage() { - echo "Usage: $0 /path/to/dist" + echo "Usage: $0 [/path/to/dist]" echo " [ --s3 | --lmdb | --level | --sftp | --sqlite | --mysql | --oracle | --mysql | --pgsql ] # bundle 1 or more storage engine" echo " [ --dev ] # avoid minification, add verbosity to esbuild/npm" echo " [ --versbose ] # add verbosity" @@ -138,13 +138,12 @@ fi # ---------------- -mkdir -p $dist +mkdir -p $dist/bin cp -r htdocs $dist/ -cp -r bin $dist/ +# cp -r bin $dist/ cp package.json $dist/bin/ -rm $dist/bin/storage-migrate.js $dist/bin/run-detached.js - - +cp bin/manager bin/worker bin/cronicled.init bin/control.sh bin/getnode.sh $dist/bin/ + # ------------------------------- writehead "Building frontend" @@ -425,13 +424,10 @@ fi cd - &>/dev/null -# ------ clean up -writehead "Final cleanup" -rm -rf $dist/bin/cronicled.init $dist/bin/debug.sh $dist/bin/install.js $dist/bin/build.js $dist/bin/build-tools.js \ -$dist/bin/manager.bat $dist/bin/worker.bat $dist/bin/win-* $dist/bin/*.ps1 +# ------ final steps +writehead "Setting up permissions" chmod -R 755 $dist/bin - # -------------------- we are done # if in dev mode, start cronicle on background in manager mode @@ -448,16 +444,12 @@ echo "" echo "---------------------------------------------------------------------------------------------------------------------------------------" echo "" echo "Bundle is ready: $(readlink -f $dist)" -if [ "$sql" = 1 ]; then - echo " - SQL Engine is bundled for mysql and postgress only. SQLite, MSSQL Oracle need to be installed separetly in $dist" -fi + if [ "$lmdb" = 1 ]; then - echo " - Lmdb cannot be fully bundeled. Lmdb package was installed in $dist" -fi -if [ "$level" = 1 ]; then - echo " - If not running this on linux, copy OS proper binary from node_modules/classic-level/prebuilds to $dist/bin/engines/prebuilds/" + echo " - Lmdb package was installed in $dist" fi + if [ "$restart" = 1 ]; then echo "Running in dev mode. Version: $CRONICLE_dev_version" exit 0 From 21791b60e365b413634ab8646e02c05597add304 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Sat, 23 Mar 2024 14:55:00 -0400 Subject: [PATCH 14/31] xss for client name --- htdocs/js/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/htdocs/js/app.js b/htdocs/js/app.js index 2e7a643..6156516 100644 --- a/htdocs/js/app.js +++ b/htdocs/js/app.js @@ -39,7 +39,7 @@ app.extend({ // allow visible app name to be changed in config this.name = config.name; - $('#d_header_title').html( '' + this.name + '' ); + $('#d_header_title').html( '' + filterXSS(this.name) + '' ); // hit the manager server directly from now on this.setmanagerHostname( resp.manager_hostname ); From a9a6e743a3acbfaaa2510ae27044943f739eb90a Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 01:43:01 -0400 Subject: [PATCH 15/31] migrate.js help + sqlite dump --- bin/storage-migrate.js | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/bin/storage-migrate.js b/bin/storage-migrate.js index 8b0cd97..e992b49 100644 --- a/bin/storage-migrate.js +++ b/bin/storage-migrate.js @@ -27,12 +27,29 @@ cli.global(); // check if --from/--to args +if(args.help) { + console.log('Usage: ./storage-migrate ') + console.log(' [ --from /path/to/storage.json] # alter source storage config (default is "Storage" property from config.json)') + console.log(' [ --to /path/to/storage.json] # alter destincation storage config (default is "NewStorage" property from config.json)') + console.log(' [ --notrx # disable transaction on destination storage (only applicable to SQL and Lmdb)') + console.log(' [ --sqlite /path/to/file.db # dump source storage into local sqlite DB (sqlite engine is required') + console.log(' [ --full ] # include job log files') + console.log(' [ --help ] # see this message again') + process.exit(1) +} + let oldConfPath; let newConfPath; +let sqlitePath; + if(args.from) { if(fs.existsSync(Path.resolve(args.from))) oldConfPath = Path.resolve(args.from) else { console.log("config file does not exist:", Path.resolve(args.from)); process.exit(1)} } +if(args.sqlite) { + if(fs.existsSync(Path.dirname(Path.resolve(args.sqlite)))) sqlitePath = Path.resolve(args.sqlite) + else { console.log("sqlite file location does not exist:", Path.resolve(args.sqlite)); process.exit(1)} +} if(args.to) { if(fs.existsSync(Path.resolve(args.to))) newConfPath = Path.resolve(args.to) else { console.log("config file does not exist:", Path.resolve(args.to)); process.exit(1)} @@ -75,6 +92,23 @@ var StorageMigrator = { if(oldConfPath) config.Storage = require(oldConfPath) // custom storage config file if (!config.Storage) this.fatal("Your Cronicle configuration lacks a 'Storage' property"); + if(sqlitePath) { + config.NewStorage = { + "engine": "SQL", + "list_page_size": 50, + "concurrency": 4, + "log_event_types": { "get": 1, "put": 1, "head": 1, "delete": 1, "expire_set": 1 }, + "SQL": { + "client": "sqlite3", + "table": "cronicle", + "useNullAsDefault": true, + "connection": { + "filename": sqlitePath + } + } + } + } + if (newConfPath) config.NewStorage = require(newConfPath) if (!config.NewStorage) this.fatal("Your Cronicle configuration lacks a 'NewStorage' property."); @@ -138,15 +172,12 @@ var StorageMigrator = { function(callback) { self.newStorage.put('test/test1', { "foo1": "bar1" }, function(err) { if (err) return callback(err); - self.logPrint(2, "test new foo test"); // throw new Error('before del') - self.newStorage.delete('test/test1', function(err) { - - self.logPrint(2, "test new before delete", err); + self.newStorage.delete('test/test1', function(err) { + if (err) return callback(err); - self.logPrint(2, "test new delete"); callback(); }); From 1e1d1061d680298f3e6a49486eef1f324b278b2b Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 01:43:59 -0400 Subject: [PATCH 16/31] auto theme change --- htdocs/js/pages/Admin.class.js | 1 + htdocs/js/pages/Home.class.js | 9 ++++++++- htdocs/js/pages/admin/Secrets.js | 10 ++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/htdocs/js/pages/Admin.class.js b/htdocs/js/pages/Admin.class.js index 12de02e..94f88b0 100644 --- a/htdocs/js/pages/Admin.class.js +++ b/htdocs/js/pages/Admin.class.js @@ -119,6 +119,7 @@ Class.subclass( Page.Base, "Page.Admin", { onDeactivate: function() { // called when page is deactivated // this.div.html( '' ); + if(this.observer) this.observer.disconnect() return true; } diff --git a/htdocs/js/pages/Home.class.js b/htdocs/js/pages/Home.class.js index be4146d..8f63825 100644 --- a/htdocs/js/pages/Home.class.js +++ b/htdocs/js/pages/Home.class.js @@ -135,6 +135,12 @@ Class.subclass( Page.Base, "Page.Home", { this.refresh_header_stats(); this.refresh_completed_job_chart(); this.refresh_event_queues(); + + const self = this; + self.observer = new MutationObserver((mutationList, observer)=> { + self.refresh_completed_job_chart(); + }); + self.observer.observe(document.querySelector('body'), {attributes: true}) return true; }, @@ -404,7 +410,7 @@ Class.subclass( Page.Base, "Page.Home", { let scaleType = $('#fe_cmp_job_chart_scale').val() || 'logarithmic'; // if chart is already generated only update data - if(app.jobHistoryChart) { + if(app.jobHistoryChart) { app.jobHistoryChart.data.datasets = datasets; app.jobHistoryChart.data.labels = labels; app.jobHistoryChart.options.scales.yAxes[0].type = scaleType; @@ -824,6 +830,7 @@ Class.subclass( Page.Base, "Page.Home", { onDeactivate: function() { // called when page is deactivated // this.div.html( '' ); + if(this.observer) this.observer.disconnect() return true; } diff --git a/htdocs/js/pages/admin/Secrets.js b/htdocs/js/pages/admin/Secrets.js index 19fd57e..f9404d9 100644 --- a/htdocs/js/pages/admin/Secrets.js +++ b/htdocs/js/pages/admin/Secrets.js @@ -9,8 +9,9 @@ Class.add( Page.Admin, { app.setWindowTitle("Secrets"); self.div.addClass('loading'); self.secret = {}; - self.secretId = args.id - + self.secretId = args.id + if(self.observer) self.observer.disconnect() // kill old observer if set by editor + app.api.post('/api/app/get_secret', { id: args.id || 'globalenv' }, self.receive_secrets.bind(self)); }, @@ -42,6 +43,11 @@ Class.add( Page.Admin, { editor.setValue(secret.data || ''); + self.observer = new MutationObserver((mutationList, observer)=> { + editor.setOption('theme', app.getPref('theme') == 'light' ? 'default' : 'solarized dark') + }); + self.observer.observe(document.querySelector('body'), {attributes: true}) + }, receive_secrets: function(resp) { From c020f371673fa2d04c0d511ca8708108d6191197 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 01:44:22 -0400 Subject: [PATCH 17/31] plugin import/export --- htdocs/js/pages/admin/Plugins.js | 85 +++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 12 deletions(-) diff --git a/htdocs/js/pages/admin/Plugins.js b/htdocs/js/pages/admin/Plugins.js index 1b2cf90..848eb5b 100644 --- a/htdocs/js/pages/admin/Plugins.js +++ b/htdocs/js/pages/admin/Plugins.js @@ -17,6 +17,8 @@ Class.add( Page.Admin, { this.div.removeClass('loading'); app.setWindowTitle( "Plugins" ); + if(this.observer) this.observer.disconnect() // kill old observer if set by editor + var size = get_inner_window_size(); var col_width = Math.floor( ((size.width * 0.9) + 500) / 6 ); @@ -52,7 +54,8 @@ Class.add( Page.Admin, { html += this.getBasicTable( this.plugins, cols, 'plugin', function(plugin, idx) { var actions = [ 'Edit', - 'Delete' + 'Delete', + 'Export' ]; var plugin_events = find_objects( app.schedule, { plugin: plugin.id } ); @@ -79,7 +82,7 @@ Class.add( Page.Admin, { html += '
'; html += ''; html += '' - html += ''; + html += ''; html += '
  Add New Plugin...
 
   from JSON
   From JSON
'; html += '
'; // padding @@ -101,6 +104,8 @@ Class.add( Page.Admin, { }, setImportEditor: function() { + + const self = this; let editor = CodeMirror.fromTextArea(document.getElementById("plugin_import"), { mode: 'application/json', @@ -110,25 +115,47 @@ Class.add( Page.Admin, { lineNumbers: false, theme: app.getPref('theme') == 'dark' ? 'gruvbox-dark' : 'default', matchBrackets: true, - gutters: [''], + // gutters: [''], lint: true }) editor.on('change', function(cm){ - document.getElementById("plugin_import").value = cm.getValue(); + document.getElementById("plugin_import").value = editor.getValue(); }); editor.setSize('52vw', '52vh') }, + export_plugin: function(idx) { + let plug = this.plugins[idx]; + let data; + if(plug) { + plug = deep_copy_object(plug) + delete plug.username + delete plug.created + delete plug.modified + delete plug.id + data = JSON.stringify(plug, null, 2) + } + else { return } + + app.show_info(` + Back Up Scheduler


+
Use this output to import plugin via "From Json" option on some other Cronicle instance (command binary should be exported/installed separetly)
+ `, '', function (result) { + + }); + + }, + import_plugin: function (args) { const self = this; setTimeout(() => self.setImportEditor(), 30) - app.confirm(`Import Plugin from JSON

- + app.confirm(`Import Plugin from JSON

+
`, '', "Import", function (result) { if (result) { var importData = document.getElementById('plugin_import').value; @@ -137,12 +164,39 @@ Class.add( Page.Admin, { } catch (e) { return app.doError("Invalid JSON: " + e.message) } - delete plugin.id + + let newPlugin = {} + + if(!plugin.title) return app.doError("Plugin is missing Title") + if(find_object(self.plugins, {title: plugin.title})) return app.doError(`Plugin with title [${plugin.title}] already exist`) + if(!plugin.command) return app.doError("Plugin is missing Command") + + if(Array.isArray(plugin.params)) { + newPlugin.params = plugin.params + for(let i = 0; i < plugin.params.length; i++){ + let e = plugin.params[i] + if(!e.id) return app.doError("One of the plugin parameters is missing [id] property") + if(!e.type) return app.doError("One of the plugin parameters is missing [type] property") + if(!e.title) return app.doError("One of the plugin parameters is missing [title] property") + } + } + + newPlugin.title = plugin.title + newPlugin.command = plugin.command + newPlugin.enabled = !!plugin.enabled + newPlugin.ipc = !!plugin.ipc + newPlugin.wf = !!plugin.wf + newPlugin.stdin = !!plugin.stdin + if(typeof plugin.uid === 'string' || parseInt(plugin.uid)) newPlugin.uid = plugin.uid + if(typeof plugin.gid === 'string' || parseInt(plugin.gid)) newPlugin.gid = plugin.gid + if(typeof plugin.cwd === 'string') newPlugin.cwd = plugin.cwd + if(typeof plugin.script === 'string') newPlugin.script = plugin.script + app.showProgress(1.0, "Importing..."); - app.api.post('app/create_plugin', plugin, function (resp) { + app.api.post('app/create_plugin', newPlugin, function (resp) { app.hideProgress(); - report = `Plugin ${resp.id} created` + report = `Plugin ${newPlugin.title} [ ${resp.id} ] has been created` setTimeout(function () { Nav.go('#Admin?sub=plugins', 'force'); @@ -433,11 +487,18 @@ Class.add( Page.Admin, { "Esc": (cm) => cm.getOption("fullScreen") ? cm.setOption("fullScreen", false) : null, "Ctrl-/": (cm) => cm.execCommand('toggleComment') } - }) + }) + + self.observer = new MutationObserver((mutationList, observer)=> { + editor.setOption('theme', app.getPref('theme') == 'dark' ? 'ambiance' : 'default') + }); + self.observer.observe(document.querySelector('body'), {attributes: true}) - editor.on('change', (cm) => { plugin.script= cm.getValue() }); + editor.on('change', (cm) => { plugin.script = cm.getValue() }); editor.setValue(plugin.script || ''); - editor.setSize('900px', '25vh') + editor.setSize('900px', '25vh'); + + }, get_plugin_edit_html: function() { From c1aa4c0a232d6d1e2504794f124b6b4b6b2c056d Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 02:11:31 -0400 Subject: [PATCH 18/31] style refactor for wf logs --- htdocs/css/style.css | 25 +++++++++++++++- htdocs/js/pages/JobDetails.class.js | 44 ++--------------------------- 2 files changed, 27 insertions(+), 42 deletions(-) diff --git a/htdocs/css/style.css b/htdocs/css/style.css index c3fb063..c25a957 100644 --- a/htdocs/css/style.css +++ b/htdocs/css/style.css @@ -708,6 +708,7 @@ td.table_label { .CodeMirror { border: 1px solid lightgray; + text-align: left; font-family: consolas, monaco, "Andale Mono", monospace; font-style: normal; font-weight: 400; @@ -749,4 +750,26 @@ td.table_label { span.red2 { background-color: #ac394d; - } \ No newline at end of file + } + + /* WF job details grid*/ + + .wflog.grid-container { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 10px; + padding: 20px; + } + + .wflog.grid-item { + border: 1px solid #ccc; + padding: 10px; + word-wrap: break-word; + overflow: hidden; + white-space: normal; + } + + .wflog.grid-title { + /* font-weight: bold; */ + font-size: 1.6em; + } \ No newline at end of file diff --git a/htdocs/js/pages/JobDetails.class.js b/htdocs/js/pages/JobDetails.class.js index 0774810..b8444fa 100644 --- a/htdocs/js/pages/JobDetails.class.js +++ b/htdocs/js/pages/JobDetails.class.js @@ -383,45 +383,7 @@ Class.subclass(Page.Base, "Page.JobDetails", { // log grid - html += ` - - -
- - ` + html += `
` // job log (IFRAME) html += '
'; @@ -732,8 +694,8 @@ Class.subclass(Page.Base, "Page.JobDetails", { data = new AnsiUp().ansi_to_html(resp.split("\n").slice(-1*size - 4, -4).join("\n")) const newItem = document.createElement('div'); newItem.setAttribute('id', 'log_' + id) - newItem.className = 'grid-item'; // Apply any necessary classes - newItem.innerHTML = `
${title}
${data}
`; + newItem.className = 'wflog grid-item'; // Apply any necessary classes + newItem.innerHTML = `
${title}
${data}
`; const gridContainer = document.getElementById('log_grid'); gridContainer.appendChild(newItem); From fc6cb77a69d2f2a36afffb21e89809c893a1364c Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 02:12:07 -0400 Subject: [PATCH 19/31] minor bundle help tweak --- bundle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle b/bundle index 162cb94..8292bda 100755 --- a/bundle +++ b/bundle @@ -3,13 +3,13 @@ cd "$(dirname "$0")" usage() { - echo "Usage: $0 [/path/to/dist]" + echo "Usage: $0 [ /path/to/dist ] # default location is dist" echo " [ --s3 | --lmdb | --level | --sftp | --sqlite | --mysql | --oracle | --mysql | --pgsql ] # bundle 1 or more storage engine" echo " [ --dev ] # avoid minification, add verbosity to esbuild/npm" echo " [ --versbose ] # add verbosity" - echo " [ --tool ] # bundle migration and detahced job tools" + echo " [ --tool ] # storage migration and repair tools" echo " [ --restart ] # restart cronicle upon bundle completion (in dev mode)" - echo " [ --engine engine] # copy sample_conf/example/storage.engine.json to dist/conf/storage.json (engine: fs/sqlite/s3/...)" + echo " [ --engine engine ] # copy sample_conf/example/storage.engine.json to dist/conf/storage.json (engine: fs/sqlite/s3/...)" echo " [ --test ] # run unit test upon bundle completion" exit 1 } From 07d8ab95a40b50fc72da7d2d0dac0b1991a3a743 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 09:34:39 -0400 Subject: [PATCH 20/31] sync to 0.9.45 (gid feature) --- htdocs/js/pages/admin/Plugins.js | 10 +++++++--- lib/job.js | 20 +++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/htdocs/js/pages/admin/Plugins.js b/htdocs/js/pages/admin/Plugins.js index 848eb5b..22e9589 100644 --- a/htdocs/js/pages/admin/Plugins.js +++ b/htdocs/js/pages/admin/Plugins.js @@ -572,7 +572,9 @@ Class.add( Page.Admin, {
Run as User (UID):
-
+
+
Run as Group (GID):
+
@@ -581,8 +583,8 @@ Class.add( Page.Admin, { `); html += get_form_table_caption( - `Optionally enter a working directory path, and/or a custom UID for the Plugin.
- The UID may be either numerical or a string ('root', 'wheel', etc.).
+ `Optionally enter a working directory path, and/or a custom UID/GID for the Plugin.
+ The UID/GID may be either numerical or strings ('root', 'wheel', etc.).
` ); html += get_form_table_spacer(); @@ -911,8 +913,10 @@ Class.add( Page.Admin, { plugin.cwd = trim( $('#fe_ep_cwd').val() ); plugin.uid = trim( $('#fe_ep_uid').val() ); + plugin.gid = trim( $('#fe_ep_gid').val() ); if (plugin.uid.match(/^\d+$/)) plugin.uid = parseInt( plugin.uid ); + if (plugin.gid.match(/^\d+$/)) plugin.gid = parseInt( plugin.gid ); return plugin; } diff --git a/lib/job.js b/lib/job.js index be78de2..3dc71fb 100644 --- a/lib/job.js +++ b/lib/job.js @@ -653,9 +653,27 @@ module.exports = Class.create({ return; } + + if (job.gid) { + var grp_info = Tools.getgrnam(job.gid, true); + if (grp_info) { + child_opts.gid = grp_info.gid; + } + else { + // group not found + job.pid = 0; + job.code = 1; + job.description = "Plugin Error: Group does not exist: " + job.gid; + this.logError("child", job.description); + this.activeJobs[job.id] = job; + this.finishLocalJob(job); + return; + } + } + child_opts.uid = parseInt(child_opts.uid); child_opts.gid = parseInt(child_opts.gid); - } + } // if unix // add plugin params as env vars if (job.params) { From cc36b7d989289ebec18403ea749e24f3cdbc1c93 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Sat, 30 Mar 2024 09:35:41 -0400 Subject: [PATCH 21/31] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 30e43ff..7d27132 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cronicle-edge", - "version": "1.8.1", + "version": "1.8.2", "description": "A simple, distributed task scheduler and runner with a web based UI.", "author": "Joseph Huckaby ", "homepage": "https://github.com/jhuckaby/Cronicle", From 9a07574163e5dbdf6ce49a867f60937c68138624 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Tue, 2 Apr 2024 20:52:06 -0400 Subject: [PATCH 22/31] fix server view --- htdocs/js/pages/admin/Servers.js | 2 +- lib/comm.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/htdocs/js/pages/admin/Servers.js b/htdocs/js/pages/admin/Servers.js index 7deb2ea..17bf6f9 100644 --- a/htdocs/js/pages/admin/Servers.js +++ b/htdocs/js/pages/admin/Servers.js @@ -63,7 +63,7 @@ Class.add( Page.Admin, { var tds = [ '
 ' + server.hostname.replace(/\.[\w\-]+\.\w+$/, '') + '
', (server.ip || 'n/a').replace(/^\:\:ffff\:(\d+\.\d+\.\d+\.\d+)$/, '$1'), - '-', '(Nearby)', '-', '-', '-', '-', '-', '-', + '-', '(Nearby)', '-', '-', '-', '-', '-', '-', '-', 'Add Server' ]; tds.className = 'blue'; diff --git a/lib/comm.js b/lib/comm.js index 4c1fd33..dce48e0 100644 --- a/lib/comm.js +++ b/lib/comm.js @@ -482,7 +482,7 @@ module.exports = Class.create({ disabled: 0, pid: process.pid, nodev: process.version, - engine: this.storage.config.get('engine') || 'unknown' + engine: this.storage.config.get('engine') }; // then add all workers @@ -497,8 +497,8 @@ module.exports = Class.create({ data: worker.data || {}, disabled: worker.disabled || 0, pid: worker.pid || -1, - // nodev: worker.nodev || 'NA', - engine: worker.engine || 'unknown' + nodev: worker.nodev || 'NA', + // engine: worker.engine || 'unknown' }; } // unique hostname } // foreach worker From 176f5c5ecc29d2cd9ed6aa39a85a524b8b4bb92a Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Tue, 2 Apr 2024 20:58:05 -0400 Subject: [PATCH 23/31] plugin import - allow empty param title --- htdocs/js/pages/admin/Plugins.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/htdocs/js/pages/admin/Plugins.js b/htdocs/js/pages/admin/Plugins.js index 22e9589..aedb284 100644 --- a/htdocs/js/pages/admin/Plugins.js +++ b/htdocs/js/pages/admin/Plugins.js @@ -177,7 +177,7 @@ Class.add( Page.Admin, { let e = plugin.params[i] if(!e.id) return app.doError("One of the plugin parameters is missing [id] property") if(!e.type) return app.doError("One of the plugin parameters is missing [type] property") - if(!e.title) return app.doError("One of the plugin parameters is missing [title] property") + // if(!e.title) return app.doError("One of the plugin parameters is missing [title] property") } } From e1acba4bedfb319e5efd3fff7ee137753c752822 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Wed, 3 Apr 2024 16:29:28 -0400 Subject: [PATCH 24/31] fix pass issue in #95 --- htdocs/js/app.js | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/htdocs/js/app.js b/htdocs/js/app.js index 6156516..1c9f46b 100644 --- a/htdocs/js/app.js +++ b/htdocs/js/app.js @@ -719,17 +719,36 @@ app.extend({ //if (!window.zxcvbn) load_script('js/external/zxcvbn.js'); }, - update_password_strength: function($field, $cont) { + update_password_strength: function ($field, $cont) { // update password strength indicator after keypress + let score = 0 + let crack_time = 'instant' + let password = $field.val() + + if (window.zxcvbn) { - var password = $field.val(); - var result = zxcvbn( password ); + var result = zxcvbn(password); // Debug.trace("Password score: " + password + ": " + result.score); var $bar = $cont.find('div.psi_bar'); $bar.removeClass('str0 str1 str2 str3 str4'); if (password.length) $bar.addClass('str' + result.score); app.last_password_strength = result; } + else { // basic strength checker + if (password.match(/[a-z]+/)) score += 0.5 + if (password.match(/[A-Z]+/)) score += 0.5 + if (password.match(/[0-9]+/)) score += 1 + if (password.match(/[$@#&!]+/)) score += 1 + if (password.length >= 8) { + score += 0.5 + crack_time = 'few hours or days' + } else { score -= 0.5} + + app.last_password_strength = { + score: score, + crack_time_display: crack_time + } + } }, get_password_warning: function() { From f107972aa90cca6c52d3a0680d2cf122cb9ecca7 Mon Sep 17 00:00:00 2001 From: mikeTWC1984 Date: Wed, 3 Apr 2024 23:58:42 -0400 Subject: [PATCH 25/31] ui - micro optimization --- htdocs/js/pages/Home.class.js | 201 +++++++++++++++++------------- htdocs/js/pages/Schedule.class.js | 4 +- 2 files changed, 115 insertions(+), 90 deletions(-) diff --git a/htdocs/js/pages/Home.class.js b/htdocs/js/pages/Home.class.js index 8f63825..73ae5b5 100644 --- a/htdocs/js/pages/Home.class.js +++ b/htdocs/js/pages/Home.class.js @@ -7,24 +7,26 @@ Class.subclass( Page.Base, "Page.Home", { this.worker = new Worker('js/home-worker.js'); this.worker.onmessage = this.render_upcoming_events.bind(this); - var html = ''; - html += '
'; - - // header stats - html += '
'; - html += '
'; - - // active jobs - html += '
'; - html += 'Active Jobs'; - html += '
'; - html += '
'; - html += '
'; - html += '
'; + this.div.html(` +
+ + + +
+
+ + + +
+ Active Jobs +
+
+
+
+
- // Completed job chart + - html += `
Event Flow
 
@@ -49,46 +51,44 @@ Class.subclass( Page.Base, "Page.Home", {
- - - +
- - ` - - // queued jobs - html += ''; - - // upcoming events - html += '
'; - html += '
'; - html += '
'; - html += '
'; // container - - this.div.html( html ); + + + + + + +
+
+
+
+ `) }, - + onActivate: function(args) { // page activation if (!this.requireLogin(args)) return true; if (!args) args = {}; this.args = args; + + // initial event flow rendering + let ui = app.config.ui || {} + let lmt = Number(app.getPref('job_chart_limit') || ui.job_chart_limit || 50) + let scale = app.getPref('job_chart_scale') || ui.job_chart_scale || 'linear' + let lmtActual = [1, 10, 25, 35, 50, 100, 120, 150, 250, 500].includes(lmt) ? lmt : 50 + document.getElementById('fe_cmp_job_chart_scale').value = scale; + document.getElementById('fe_cmp_job_chart_limit').value = lmtActual; + app.setWindowTitle('Home'); app.showTabBar(true); @@ -118,19 +118,21 @@ Class.subclass( Page.Base, "Page.Home", { html += '
'; - $('#d_home_upcoming_header').html( html ); + // $('#d_home_upcoming_header').html( html ); + document.getElementById('d_home_upcoming_header').innerHTML = html setTimeout( function() { - $('#fe_home_keywords').keypress( function(event) { - if (event.keyCode == '13') { // enter key + document.getElementById('fe_home_keywords').addEventListener('keypress', function(event) { + if (event.key === 'Enter') { // Enter key event.preventDefault(); $P().set_search_filters(); } - } ); + }); }, 1 ); // refresh datas - $('#d_home_active_jobs').html( this.get_active_jobs_html() ); + // $('#d_home_active_jobs').html( this.get_active_jobs_html() ); + document.getElementById('d_home_active_jobs').innerHTML = this.get_active_jobs_html() this.refresh_upcoming_events(); this.refresh_header_stats(); this.refresh_completed_job_chart(); @@ -193,7 +195,8 @@ Class.subclass( Page.Base, "Page.Home", { ` - $('#d_home_header_stats').html(html); + // $('#d_home_header_stats').html(html); + document.getElementById('d_home_header_stats').innerHTML = html; }, refresh_upcoming_events: function() { @@ -218,18 +221,18 @@ Class.subclass( Page.Base, "Page.Home", { set_search_filters: function() { // grab values from search filters, and refresh - var args = this.args; - - args.plugin = $('#fe_home_plugin').val(); + var args = this.args; + + args.plugin = document.getElementById('fe_home_plugin').value; if (!args.plugin) delete args.plugin; - - args.target = $('#fe_home_target').val(); + + args.target = document.getElementById('fe_home_target').value; if (!args.target) delete args.target; - - args.category = $('#fe_home_cat').val(); + + args.category = document.getElementById('fe_home_cat').value; if (!args.category) delete args.category; - - args.keywords = $('#fe_home_keywords').val(); + + args.keywords = document.getElementById('fe_home_keywords').value; if (!args.keywords) delete args.keywords; this.nav_upcoming(0); @@ -237,13 +240,13 @@ Class.subclass( Page.Base, "Page.Home", { render_upcoming_events: function(e) { // receive data from worker, render table now - var self = this; + const self = this; var html = ''; var now = app.epoch || hires_time_now(); var args = this.args; this.upcoming_events = e.data; - var viewType = $("#fe_up_eventlimit").val(); // compact or show all + var viewType = document.getElementById('fe_up_eventlimit').value; // apply filters var events = []; @@ -363,40 +366,50 @@ Class.subclass( Page.Base, "Page.Home", { } // row callback }); // table - $('#d_home_upcoming_events').removeClass('loading').html( html ); + // $('#d_home_upcoming_events').removeClass('loading').html( html ); + let upcoming = document.getElementById('d_home_upcoming_events'); + upcoming.classList.remove('loading'); + upcoming.innerHTML = html; }, - refresh_completed_job_chart: function () { + refresh_completed_job_chart: async function () { - if($('#fe_cmp_job_chart_limit').val() < 2) { + if (document.getElementById('fe_cmp_job_chart_limit').value < 2) { if(app.jobHistoryChart) { app.jobHistoryChart.destroy() location.reload(true) // no easy way to kill graph, just reload the page - } - + } return } - let isDark = app.getPref('theme') === 'dark' - let green = isDark ? '#44bb44' : 'lightgreen' // success - let orange = isDark ? '#bbbb44' : 'orange' // warning - let red = isDark ? '#bb4444' : 'pink' // error - - let statusMap = { 0: green, 255: orange } + let jobLimit = document.getElementById('fe_cmp_job_chart_limit').value || 50; - let jobLimit = $('#fe_cmp_job_chart_limit').val() || 50 + let body = { offset: 0, limit: jobLimit, session_id: localStorage['session_id']} - app.api.post('app/get_history', { offset: 0, limit: jobLimit }, function (d) { + fetch('api/app/get_history', {method: 'POST', body: JSON.stringify(body)}) + .then(response => { + if(!response.ok) throw new Error('Failed to fetch job history') + return response.json() + }) + .then(data => { - let jobs = d.rows.reverse().filter(e=>e.event_title); + if(!data.rows) throw new Error('Job history: Response has no rows') + + let jobs = data.rows.reverse().filter(e=>e.event_title); if(jobs.length > 1) { let jFrom = moment.unix(jobs[0].time_start).format('MMM DD, HH:mm:ss'); let jTo = moment.unix(jobs[jobs.length-1].time_start + (jobs[jobs.length-1].elapsed || 0)).format('MMM DD, HH:mm:ss'); - $("#chart_times").text(` from ${jFrom} | to ${jTo}`); + document.getElementById('chart_times').textContent = ` from ${jFrom} | to ${jTo}` } + let isDark = app.getPref('theme') === 'dark' + let green = isDark ? '#44bb44' : 'lightgreen' // success + let orange = isDark ? '#bbbb44' : 'orange' // warning + let red = isDark ? '#bb4444' : 'pink' // error + let statusMap = { 0: green, 255: orange } + let labels = jobs.map(e => '') if(jobLimit < 100) labels = jobs.map((j, i) => i == 0 ? j.event_title.substring(0, 4) : j.event_title); let datasets = [{ @@ -407,7 +420,7 @@ Class.subclass( Page.Base, "Page.Home", { jobs: jobs // borderWidth: 0.3 }]; - let scaleType = $('#fe_cmp_job_chart_scale').val() || 'logarithmic'; + let scaleType = document.getElementById('fe_cmp_job_chart_scale').value || 'logarithmic'; // if chart is already generated only update data if(app.jobHistoryChart) { @@ -478,8 +491,10 @@ Class.subclass( Page.Base, "Page.Home", { let job = app.jobHistoryChart.data.datasets[firstPoint._datasetIndex].jobs[firstPoint._index] window.open("#JobDetails?id=" + job.id, "_blank"); }; + }) // end respose data processing + .catch(e => console.error(e.message)); - }); // callback + }, get_active_jobs_html: function() { @@ -643,7 +658,8 @@ Class.subclass( Page.Base, "Page.Home", { } ); // getBasicTable - $('#d_home_queued_jobs').html( html ); + // $('#d_home_queued_jobs').html( html ); + document.getElementById('d_home_queued_jobs').innerHTML = html $('#d_home_queue_container').show(); }, @@ -735,7 +751,8 @@ Class.subclass( Page.Base, "Page.Home", { // received status update (websocket), update page if needed if (data.jobs_changed) { // refresh tables - $('#d_home_active_jobs').html( this.get_active_jobs_html() ); + // $('#d_home_active_jobs').html( this.get_active_jobs_html() ); + document.getElementById('d_home_active_jobs').innerHTML = this.get_active_jobs_html() } else { // update progress, time remaining, no refresh @@ -744,26 +761,32 @@ Class.subclass( Page.Base, "Page.Home", { if (job.pending) { // update countdown - $('#d_home_jt_progress_' + job.id).html( this.getNiceJobPendingText(job) ); + // $('#d_home_jt_progress_' + job.id).html( this.getNiceJobPendingText(job) ); + document.getElementById('d_home_jt_progress_' + job.id).innerHTML = this.getNiceJobPendingText(job); if (job.log_file) { // retry delay - $('#d_home_jt_elapsed_' + job.id).html( this.getNiceJobElapsedTime(job) ); + // $('#d_home_jt_elapsed_' + job.id).html( this.getNiceJobElapsedTime(job) ); + document.getElementById('d_home_jt_elapsed_' + job.id).innerHTML = this.getNiceJobElapsedTime(job); } } // pending job else { - $('#d_home_jt_elapsed_' + job.id).html( this.getNiceJobElapsedTime(job) ); - $('#d_home_jt_remaining_' + job.id).html( this.getNiceJobRemainingTime(job) ); + // $('#d_home_jt_elapsed_' + job.id).html( this.getNiceJobElapsedTime(job) ); + // $('#d_home_jt_remaining_' + job.id).html( this.getNiceJobRemainingTime(job) ); + document.getElementById('d_home_jt_elapsed_' + job.id).innerHTML = this.getNiceJobElapsedTime(job); + document.getElementById('d_home_jt_remaining_' + job.id).innerHTML = this.getNiceJobRemainingTime(job); if(job.memo) { let memoClass = String(job.memo).startsWith('OK:') ? 'color_label green' : '' if(String(job.memo).startsWith('WARN:')) memoClass = 'color_label yellow' if(String(job.memo).startsWith('ERR:')) memoClass = 'color_label red' - $('#d_home_jt_memo_' + job.id).html(`${encode_entities(job.memo)}`); + // $('#d_home_jt_memo_' + job.id).html(`${encode_entities(job.memo)}`); + document.getElementById('d_home_jt_memo_' + job.id).innerHTML = `${encode_entities(job.memo)}` } if(job.cpu) { - $('#d_home_jt_perf_' + job.id).text(short_float(job.cpu.current) + '% | ' + get_text_from_bytes(job.mem.current) + ' | ' + get_text_from_bytes(job.log_file_size)) + // $('#d_home_jt_perf_' + job.id).text(short_float(job.cpu.current) + '% | ' + get_text_from_bytes(job.mem.current) + ' | ' + get_text_from_bytes(job.log_file_size)) + document.getElementById('d_home_jt_perf_' + job.id).textContent = short_float(job.cpu.current) + '% | ' + get_text_from_bytes(job.mem.current) + ' | ' + get_text_from_bytes(job.log_file_size) } // update progress bar without redrawing it (so animation doesn't jitter) diff --git a/htdocs/js/pages/Schedule.class.js b/htdocs/js/pages/Schedule.class.js index 232f0ea..ba1a8d7 100644 --- a/htdocs/js/pages/Schedule.class.js +++ b/htdocs/js/pages/Schedule.class.js @@ -961,7 +961,9 @@ Class.subclass(Page.Base, "Page.Schedule", { var last_code = app.state.jobCodes[event_id]; var status_html = last_code ? ' Error' : ' Success'; if (last_code == 255) status_html = ' Warning' - this.div.find('#ss_' + event_id).html(status_html); + // this.div.find('#ss_' + event_id).html(status_html); + document.getElementById('ss_' + event_id).innerHTML = status_html + } }, From c1cd615827325c6923a33c785bb056838ea5bf6a Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Fri, 5 Apr 2024 00:39:10 -0400 Subject: [PATCH 26/31] windows tool refactor --- bin/control.sh.ps1 | 6 +- bin/manager.bat | 47 ++++++++++- bundle.ps1 | 200 ++++++++++++++++++++++++--------------------- 3 files changed, 155 insertions(+), 98 deletions(-) diff --git a/bin/control.sh.ps1 b/bin/control.sh.ps1 index e1bb6aa..5736c2d 100644 --- a/bin/control.sh.ps1 +++ b/bin/control.sh.ps1 @@ -13,12 +13,14 @@ param( # list collection items [Int32]$Limit = 50, [Int32]$Offset = 0, - [switch]$Full + [switch]$Full, + + [switch]$Help ) $ErrorActionPreference = "stop" -if (!$Command) { +if (!$Command -OR $Help) { Write-Host "Usage: .\control.ps1 start [-Manager] [-Echo] [-Force] [-Color] [-WinMon] [-Hide] diff --git a/bin/manager.bat b/bin/manager.bat index d7e0fba..1cc849a 100644 --- a/bin/manager.bat +++ b/bin/manager.bat @@ -1,6 +1,49 @@ @echo off -cd /D "%~dp0" + +set SCRIPT_LOC=%~dp0 + +:parseArgs + +if "%1"=="" goto endArgs + +if /I "%1"=="--port" ( + if "%2"=="" ( + echo Specify http port for cronicle + exit + ) + set CRONICLE_WebServer__http_port=%2 + echo Custom port set: %2 + shift + shift +) else if /I "%1"=="--storage" ( + if "%2"=="" ( + echo Specify path to storage.json config file + exit + ) + set CRONICLE_storage_config=%~f2 + echo Custom storage set: %~f2 + shift + shift +) else if /I "%1"=="--key" ( + if "%2"=="" ( + echo Secret key not specified + exit + ) + set CRONICLE_secret_key=%2 + echo Using custom secret key: ***** + shift + shift +) else if /I "%1"=="--help" ( + echo Usage: .\manager [--port port] [ --storage /path/to/storage.json] + shift +) else (exit) + +goto parseArgs + +:endArgs + +cd /D %SCRIPT_LOC% REM check for custom node version IF EXIST "%~dp0..\nodejs\node.exe" ( @@ -14,4 +57,4 @@ if not "%~1"=="" ( echo CRONICLE_http_port is set to %1 ) -node .\cronicle.js --manager --echo --foreground --color \ No newline at end of file +node .\cronicle.js --manager --echo --foreground --color diff --git a/bundle.ps1 b/bundle.ps1 index 26a46d9..f090982 100644 --- a/bundle.ps1 +++ b/bundle.ps1 @@ -1,6 +1,5 @@ -# $Path = "dist" -# if($args[0]) { $Path = $args[0] } + param( [Parameter(Position = 0)][String]$Path = "dist", # install directory [switch]$S3, # bundle s3 engine @@ -14,7 +13,6 @@ param( [switch]$Level, # bundle level db engine [switch]$Lmdb, # bundle lmdb engine * [switch]$Sftp, # bundle sftp engine - [string][ValidateSet("s3","lmdb","level","sqlite","oracle","mysql","mssql","pgsql","sftp","redis")]$Engine, # copy storage.engine.json from sample_conf to dist [switch]$Force, # force reinstall if something is broken in node_modules [switch]$Dev, # prevent minificaiton and add verbosity [Switch]$Restart, # for dev purposes only: it will force kill cronicle if running, and start it over again once bundling is complete @@ -24,9 +22,42 @@ param( [switch]$Test, # run unit test at the end [ValidateSet("warning", "debug", "info", "warning", "error","silent")][string]$ESBuildLogLevel = "warning", [switch]$Help + # dummy parameters to capture "--" args + ,[Parameter(Position = 1)][String]$arg1 + ,[Parameter(Position = 2)][String]$arg2 + ,[Parameter(Position = 3)][String]$arg3 + ,[Parameter(Position = 4)][String]$arg4 + ,[Parameter(Position = 5)][String]$arg5 ) +foreach ($arg in $PSBoundParameters.Values) { + if ($arg -eq "--s3") { $S3 = $true } + if ($arg -eq "--sql") { $SQL = $true } + if ($arg -eq "--oracle") { $Oracle = $true } + if ($arg -eq "--mssql") { $MSSQL = $true } + if ($arg -eq "--mysql") { $Mysql = $true } + if ($arg -eq "--pgsql") { $Pgsql = $true } + if ($arg -eq "--sqlite") { $Sqlite = $true } + if ($arg -eq "--redis") { $Redis = $true } + if ($arg -eq "--level") { $Level = $true } + if ($arg -eq "--lmdb") { $Lmdb = $true } + if ($arg -eq "--sftp") { $Sftp = $true } + if ($arg -eq "--force") { $Force = $true } + if ($arg -eq "--dev") { $Dev = $true } + if ($arg -eq "--restart") { $Restart = $true } + if ($arg -eq "--tools") { $Tools = $true } + if ($arg -eq "--all") { $All = $true } + if ($arg -eq "--test") { $Test = $true } + if ($arg -eq "--help") { $Help = $true } + if ($arg -eq "--verbose") { $V = $true } +} + +if($Path -like "--*") { + # perhaps user meant some flag, fallback Path to default + $Path = "dist" +} + if($Help) { Write-Host "Usage: ./bundle.ps1 [path\to\dist] # default bundle location is dist" Write-Host " [ -S3 | -Redis | -Lmdb | -Level | -Redis | -Sftp ] # 1 or more storage engines (FS always added)" @@ -44,9 +75,17 @@ if($Help) { $ErrorActionPreference = 'Stop' -Write-Host "-----------------------------------------" -Write-Host " Installing cronicle bundle into $Path" -Write-Host "-----------------------------------------" +function Write-Bold { param($Text, [switch]$U) + $b = "[1m"; if($U) {$b = "[1;4m" }; Write-Output "$( [char]27 )$b$Text$( [char]27 )[0m" +} + +$FullPath = mkdir -Force $Path + +Write-Bold "`nInstalling cronicle bundle into $FullPath`n" -U | Write-Host -ForegroundColor Green + +# Write-Host "-----------------------------------------" +# Write-Host " Installing cronicle bundle into $($Path)" +# Write-Host "-----------------------------------------" $proc = $null $pidFile = "$Path\logs\cronicled.pid" @@ -56,7 +95,7 @@ if(Test-Path $pidFile ) { } if($proc) { - if($Restart.IsPresent) { + if($Restart) { Write-Host "Shutting down cronicle..." if(!$proc.CloseMainWindow()) {Stop-Process -id $proc.Id } } @@ -71,20 +110,20 @@ $minify = "--minify=true" $ESBuildLogLevel = "warning" $npmLogLevel = "warn" -if($Restart.IsPresent) { $minify = "--minify=false" } +if($Restart) { $minify = "--minify=false" } -if ($Dev.IsPresent) { +if ($Dev) { $minify = "--minify=false" $ESBuildLogLevel = "info" $npmLogLevel = "info" } -if($V.IsPresent) { +if($V) { $ESBuildLogLevel = "info" $npmLogLevel = "info" } -if($All.IsPresent) { +if($All) { $S3 = $true $Sftp = $true $Lmdb = $true @@ -95,15 +134,13 @@ if($All.IsPresent) { # ------------------------- -# if (!(Get-Command esbuild -ErrorAction SilentlyContinue)) { npm i esbuild -g --loglevel warn } - if(Test-Path $PSScriptRoot\nodejs\node.exe) { $env:Path = "$PSScriptRoot\nodejs\;$env:Path;" Write-Warning "Using custom node version $(node -v)" # exit 0 } -if (!(Test-Path .\node_modules) -or $Force.IsPresent) { +if (!(Test-Path .\node_modules) -or $Force) { Write-Host "`n---- Installing npm packages`n" npm install --loglevel $npmLogLevel Write-Host "`n-----------------------------------------" @@ -115,7 +152,7 @@ if (!(Get-Command esbuild -ErrorAction SilentlyContinue)) { } # ---- set up bin and htdocs folders on dist (will overwrite) -mkdir -Force $Path +# mkdir -Force $Path | Out-Null Copy-Item -Force -r htdocs $Path/ mkdir -EA SilentlyContinue $Path/htdocs/js/external, $Path/htdocs/css, $Path/htdocs/fonts | Out-Null @@ -198,7 +235,7 @@ Get-Content ` , node_modules/codemirror/addon/display/placeholder.js ` , node_modules/codemirror/mode/xml/xml.js ` , node_modules/codemirror/mode/sql/sql.js ` - , node_modules/js-yaml/dist/js-yaml.js ` + , node_modules/js-yaml/dist/js-yaml.js ` , node_modules/codemirror/addon/lint/lint.js ` , node_modules/codemirror/addon/lint/json-lint.js ` , node_modules/codemirror/addon/lint/yaml-lint.js ` @@ -249,16 +286,16 @@ Copy-Item -Force htdocs/index-bundle.html $Path/htdocs/index.html # -------- Bundle storage-cli and event plugins ----------------------------------------------- # -Write-Host "`n ---- Building storage-cli and plugins`n" +Write-Bold "Building storage-cli and plugins" esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/ ` --external:../conf/config.json --external:../conf/storage.json --external:../conf/setup.json ` bin/storage-cli.js -if($Tools.IsPresent) { - Write-Host " - storage-repair.js`n" +if($Tools) { + Write-Host " - storage-repair.js" esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/ --external:../conf/config.json bin/storage-repair.js - Write-Host " - storage-migrate.js`n" + Write-Host " - storage-migrate.js" esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/ --external:../conf/config.json bin/storage-migrate.js } @@ -275,35 +312,35 @@ esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$ # ------------ Bundle Storage Engines ------------------------------ # -Write-Host "`n ---- Building Storage Engines:`n" +Write-Bold "Building Storage Engines:" $engines = "FS" -Write-Host " - bundling FS Engine`n" +Write-Host " - bundling FS Engine" esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/engines engines/Filesystem.js -if ($S3.IsPresent ) { - Write-Host " - bundling S3 Engine`n" +if ($S3 ) { + Write-Host " - bundling S3 Engine" $engines += ", S3" npm i @aws-sdk/client-s3 @aws-sdk/lib-storage --no-save --loglevel silent esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/engines engines/S3.js } -if ($Redis.IsPresent) { - Write-Host " - bundling Redis Engine`n" +if ($Redis) { + Write-Host " - bundling Redis Engine" $engines += ", Redis" npm i redis@3.1.2 --no-save --loglevel silent esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/engines engines/Redis.js } -if ($Sftp.IsPresent) { - Write-Host " - bundling Sftp Engine`n" +if ($Sftp) { + Write-Host " - bundling Sftp Engine" $engines += ", Sftp" npm i ssh2-sftp-client --no-save --loglevel silent esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/engines --loader:.node=file engines/Sftp.js } -if ($Level.IsPresent) { - Write-Host " - bundling Level Engine`n" +if ($Level) { + Write-Host " - bundling Level Engine" $engines += ", Level" npm i level --no-save --loglevel silent esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/engines engines/Level.js @@ -335,14 +372,14 @@ if($sqlDrivers.Count -gt 0) { Write-Host " - bundling SQL Engine [$($sqlDrivers -join ",")]" & npm $sqlInstall & esbuild $sqlArgs - if($Sqlite.IsPresent) { + if($Sqlite) { Copy-Item -Recurse -Force node_modules/sqlite3/build $Path/bin/ } } # Lmdb, need to install lmdb separetly (npm i lmdb) -if($Lmdb.IsPresent) { - Write-Host " - bundling Lmdb Engine*`n" +if($Lmdb) { + Write-Host " - bundling Lmdb Engine*" $engines += ", Lmdb" esbuild --bundle --log-level=$ESBuildLogLevel $minify --platform=node --outdir=$Path/bin/engines --external:lmdb engines/Lmdb.js } @@ -350,7 +387,7 @@ if($Lmdb.IsPresent) { #### ------- SET UP CONFIGS ----- only do it if config folder doesnt exist # if (!(Test-Path $Path/conf)) { - Write-Host "`n ---- Setting up initial config files`n" + Write-Bold "Setting up initial config files" Copy-Item -Force -r sample_conf/ $Path/conf @@ -365,27 +402,13 @@ if(Test-Path "sample_conf\examples\storage.$Engine.json") { Copy-Item -Force "sample_conf\examples\storage.$Engine.json" $Path\conf\storage.json } -# --- CRONICLE.JS -Write-Host "`n ---- Bundling cronicle.js`n" -esbuild --bundle $minify --keep-names --platform=node --outfile=$Path/bin/cronicle.js lib/main.js - # --- fix formidable -Write-Host "`n ---- Applyig some patches" +Write-Bold "Applyig some patches" esbuild --bundle --log-level=$ESBuildLogLevel $minify --keep-names --platform=node --outdir=$Path/bin/plugins node_modules/formidable/src/plugins/*.js -# --------------- Final Clean up --------------------------- - -# Remove-Item -Recurse -Force ` -# $Path/bin/jars ` -# , $Path/bin/cms ` -# , $Path/bin/cronicled.init ` -# , $Path/bin/importkey.sh ` -# , $Path/bin/debug.sh ` -# , $Path/bin/java-plugin.js ` -# , $Path/bin/install.js ` -# , $Path/bin/build.js ` -# , $Path/bin/build-tools.js ` - +# --- CRONICLE.JS +Write-Bold "Bundling cronicle.js" +esbuild --bundle $minify --keep-names --platform=node --outfile=$Path/bin/cronicle.js lib/main.js # --- if needed set up npm package in dist folder to install some deps that cannot be bundled Push-Location $Path @@ -397,70 +420,59 @@ if(!(Test-Path "package.json")) { npm pkg set main="bin/cronicle.js" npm pkg set scripts.start="node bin/cronicle.js --foreground --echo --manager --color" } -if($Lmdb.IsPresent) { npm i "lmdb@2.9.4" --loglevel silent} +if($Lmdb) { npm i "lmdb@2.9.4" --loglevel silent} Pop-Location - -if($Restart.IsPresent) { - Write-Host "`n---- Restarting cronicle`n" +if($Restart) { + Write-Bold "`Restarting cronicle" # --- $env:CRONICLE_dev_version="1.x.dev-$([datetime]::Now.ToString("yyyy-MM-dd HH:mm:ss"))" - Start-Process node -WindowStyle Minimized -ArgumentList @("$Path\bin\cronicle.js", "--foreground", "--echo", "--manager", "--color") + Start-Process "$Path\bin\manager" -WindowStyle Minimized + #Start-Process node -WindowStyle Minimized -ArgumentList @("$Path\bin\cronicle.js", "--foreground", "--echo", "--manager", "--color") } # --- Print setup info / stats -Write-Host "`n-------------------------------------------------------------------------------------------------------------------------------------------`n" -Write-Host "Bundle is ready: $FullPath `n" -ForegroundColor Green -Write-Host "Minified: $($minify -like '*true*' )" -Write-Host "Engines bundled: $engines" -if($SQL.IsPresent) { - Write-Host " * SQL bundle includes mysql and postgres drivers. You can additionally install sqlite3, oracledb, tedious (for mssql)" -} -if($Lmdb.IsPresent) { - Write-Host " * Lmdb cannot be fully bundled. lmdb package is installed in the dist folder using npm" -} +Write-Host "`n [ Bundle is ready: $FullPath ] `n" -ForegroundColor Green +Write-Bold "Info:" +Write-Host " - minified: $($minify -like '*true*' )" +Write-Host " - engines bundled: $engines" -if($Restart.IsPresent) { - Write-Host "Running in dev mode. Version: $env:CRONICLE_dev_version `n" +if($Restart) { + Write-Bold "Running in dev mode. Version: $env:CRONICLE_dev_version `n" -U exit 0 } if($env:Path.indexOf("$FullPath\bin") -lt 0) { $env:Path = $env:Path + ";$FullPath\bin"; Write-Host "$Path\bin is added to path variable"} -Write-Host " -Before you begin: - - Configure you storage engine setting in conf/config.json || conf/storage.json || CRONICLE_storage_config=/path/to/custom/storage.json - - Set you Secret key in conf/congig.json || conf/secret_key file || CRONICLE_secret_key_file || CRONICLE_secret_key env variables - -To setup cronicle storage (on the first run): - node .\$Path\bin\storage-cli.js setup +if(!$Dev -and !$Restart) { # do not print below info during dev or debug -Start as manager in foreground: - node .\$Path\bin\cronicle.js --echo --foreground --manager --color +Write-Bold "`nBefore you begin:" +Write-Host " - Configure you storage engine setting in conf/config.json OR conf/storage.json OR CRONICLE_storage_config=/path/to/custom/storage.json" +Write-Host " - Set you Secret key in conf/congig.json OR via conf/secret_key file OR via CRONICLE_secret_key_file / CRONICLE_secret_key env variables" +Write-Host " - Init cronicle storage (on the first run): node .\$Path\bin\storage-cli.js setup" +Write-Host " - Start cronicle as manage: node .\$Path\bin\cronicle.js --echo --foreground --manager --color`n" -Or both together: .\$Path\bin\manager +Write-Bold "You can also setup/start cronicle using [manager] entrypoint:" +Write-Host ".\$Path\bin\manager [ --port 3012 ] [ --storage Path\to\storage.json ] [ --key someSecretKey ]`n" -------------------------------------------------------------------------------------------------------------------------------------------- +Write-Bold "To Reinstall/upgrade run (please back up $FullPath first):" +Write-Host ".\bundle.ps1 $Path -Force`n" -to reinstall/upgrade run (please back up $FullPath first): - .\bundle.ps1 $Path -Force - -to install as windows service: - cd $Path +Write-Bold "To install as Windows Service:" -U +Write-Host " cd $Path npm i node-windows -g npm link node-windows node bin\install.js - -test: Get-Service cronicle - -to remove service: - node bin\uninstall.js - + ### Make sure it's running: Get-Service cronicle + ### Remove service: node bin\uninstall.js " -if($Test.IsPresent) { +} + +# perform unit test if needed +if($Test) { - Write-Host "Running unit test" - Write-Host "build test script" + Write-Bold "Running unit test" + Write-Host " - building test script" esbuild --bundle --outdir=$Path/bin/ --platform=node lib/test.js node .\node_modules\pixl-unit\unit.js $Path\bin\test.js Remove-Item $Path\bin\test.js From 063e784ac66f0e05f7817a7e8971185a4dddd056 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Fri, 5 Apr 2024 00:39:26 -0400 Subject: [PATCH 27/31] sample cleanup --- sample_conf/examples/backup | 20 -------------------- sample_conf/examples/docker.sh | 16 ---------------- 2 files changed, 36 deletions(-) delete mode 100644 sample_conf/examples/backup delete mode 100644 sample_conf/examples/docker.sh diff --git a/sample_conf/examples/backup b/sample_conf/examples/backup deleted file mode 100644 index b5faab4..0000000 --- a/sample_conf/examples/backup +++ /dev/null @@ -1,20 +0,0 @@ -# Cronicle Data Export v1.0 -# Hostname: local -# Date/Time: Mon Dec 27 2021 06:20:45 GMT+0000 (Coordinated Universal Time) -# Format: KEY - JSON - -users/admin - {"username":"admin","password":"$2a$10$VAF.FNvz1JqhCAB5rCh9GOa965eYWH3fcgWIuQFAmsZnnVS/.ye1y","full_name":"Administrator","email":"admin@cronicle.com","active":1,"modified":1612277068,"created":1612277068,"salt":"salty","privileges":{"admin":1}} -global/users - {"page_size":100,"first_page":0,"last_page":0,"length":1,"type":"list"} -global/users/0 - {"type":"list_page","items":[{"username":"admin"}]} -global/categories - {"page_size":50,"first_page":0,"last_page":0,"length":4,"type":"list"} -global/categories/0 - {"type":"list_page","items":[{"title":"Admin","description":"","max_children":0,"notify_success":"","notify_fail":"","web_hook":"","enabled":1,"gcolor":"#f4d03f","id":"ckksife2k01","modified":1612542932,"created":1612542932,"username":"admin"},{"title":"Chain","description":"","max_children":0,"notify_success":"","notify_fail":"","web_hook":"","enabled":1,"gcolor":"#58d68d","id":"ckkqcfxkg26","modified":1612411947,"created":1612411947,"username":"admin"},{"title":"Demo","description":"","max_children":0,"notify_success":"","notify_fail":"","web_hook":"","enabled":1,"gcolor":"#ec7063 ","id":"ckkog7wj41t","modified":1612297359,"created":1612297359,"username":"admin"},{"id":"general","title":"General","enabled":1,"username":"admin","modified":1612277068,"created":1612277068,"description":"For events that don't fit anywhere else.","gcolor":"#3498DB","max_children":0}]} -global/schedule - {"page_size":50,"first_page":0,"last_page":0,"length":14,"type":"list"} -global/schedule/0 - {"type":"list_page","items":[{"enabled":1,"params":{"wf_concur":"(sync)","wf_maxerr":"(None)","wf_strict":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"workflow","title":"workflow_demo","silent":0,"graph_icon":"f111","args":"","ticks":"","category":"general","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"workflow":[{"id":"ekkog584p1r","title":"00_wf_event","arg":"7","wait":false},{"id":"ekko472v803","title":"01_wf_step_one","arg":"","wait":false},{"id":"ekko473os04","title":"02_wf_step_two","arg":"LA","wait":false},{"id":"ekko474ip05","title":"03_wf_step_three","arg":"NY","wait":false}],"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","id":"ekxoae0xp0j","modified":1640585930,"created":1640585737,"username":"admin","salt":""},{"enabled":1,"params":{"script":"#!/usr/bin/env bash\n\n# set up data folder for git sync\n# once completed set git.enabled=true (will sync on pressing backup button, restart and shutdown)\n# for auto update set git.auto=true (to invoke git sync on each metadata change)\n\nUSER=\"cronicle\"\nPASSWORD=\"P@ssw0rd\"\nREPO=\"http://$USER:$PASSWORD@github.com/yourUser/yourRepo.git\"\n# or use ssh keys for auth if possible\n\nexport GIT_AUTHOR_NAME=\"cronicle\"\nexport GIT_COMMITER_NAME=\"cronicle\"\nexport EMAIL=\"admin@cronicle.com\"\n\ncd data && \\\ngit init && \\\ngit remote add origin $REPO && \\\ngit add global users && \\\ngit commit -m \"Initial commit\" && \\\ngit branch --set-upstream-to=origin/master master && \\\ngit push -u origin master\n\n# later on you can configure user/email/branch and remote names in config file/keys\n# git.add option let you control what folders/files to sync:\n# global,users - (default) just metadata (schedules/events/users/etc)\n# global,users,logs - metadata + job history + activity log (10-20MB per 5K jobs)\n# global,users,logs,jobs - to cover everything (~100-150MB per 5K jobs)\n\n# to debug git sync try to \"git push\" manually or check logs for \"Git Sync Error\"\n\n# next time while setting up cronicle on other machine just do \"git clone $REPO data\" instead of setup\n# if using cronicle-edge docker image just set GIT_REPO env variable to take care of it, e.g.\n# docker run -d -e GIT_REPO=$REPO -p 3012:3012 cronicle:edge manager","annotate":0,"json":0,"lang":"shell","theme":"default","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"gitInit","silent":0,"graph_icon":"f09b","args":"","ticks":"","category":"ckksife2k01","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","id":"ekkrmrzb2h6","modified":1612543627,"created":1612489772,"username":"admin","salt":""},{"enabled":1,"params":{"script":"#!/usr/bin/env node\n\nlet stdin = JSON.parse(require(\"fs\").readFileSync(0));\n// stdin will contain some info about the job that is chaining this notification\n\nconst PixlMail = require('/opt/cronicle/node_modules/pixl-mail');\nconst mail = new PixlMail( 'mailrelay.cronicle.com', 25 );\n// mail.setOption( 'secure', true ); // use ssl\n// mail.setOption( 'auth', { user: 'fsmith', pass: '12345' } );\n\nlet body = `\n Exit code: [chain_code]\n Description: [chain_description]\n Data: [chain_data]\n`\nlet message = \n \"To: notify@cronicle.com\\n\" + \n \"From: admin@cronicle.com\\n\" + \n \"Subject: [source] failed\\n\" +\n \"\\n\" + body + \"\\n\" ;\n \n\nmail.send( message, stdin, function(err) {\n if (err) return console.log( \"Mail Error: \" + err );\n console.log(\"message sent\");\n} );","annotate":0,"json":0,"lang":"javascript","theme":"default","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"notify","category":"ckksife2k01","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"Randomly Generated Job","id":"ekkqcopde2a","modified":1612543044,"created":1612412357,"username":"admin","salt":"","silent":0,"graph_icon":"f0e0","args":"","ticks":"","web_hook_start":"","web_hook_error":0},{"enabled":1,"params":{"script":"#!/usr/bin/env bash\n\necho \"Chain 3\"\n\nsleep 2","annotate":0,"json":0,"lang":"shell","theme":"default","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","category":"ckkqcfxkg26","title":"chain3","silent":0,"graph_icon":"f111","args":"","ticks":"","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"ekkqcgqiw27","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","salt":"","id":"ekkqchhlc29","modified":1612543706,"created":1612412020,"username":"admin"},{"enabled":1,"params":{"script":"#!/usr/bin/env bash\n\necho \"Chain 2\"\n\nsleep 2","annotate":0,"json":0,"lang":"shell","theme":"default","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","category":"ckkqcfxkg26","title":"chain2","silent":0,"graph_icon":"f111","args":"","ticks":"","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"ekkqchhlc29","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","salt":"","id":"ekkqch38n28","modified":1612412112,"created":1612412001,"username":"admin"},{"enabled":1,"params":{"script":"#!/usr/bin/env bash\n\necho \"Chain 1\"\n\nsleep 2","annotate":0,"json":0,"lang":"shell","theme":"default","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","category":"ckkqcfxkg26","title":"chain1","silent":0,"graph_icon":"f111","args":"","ticks":"","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"ekkqch38n28","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","id":"ekkqcgqiw27","modified":1612412033,"created":1612411985,"username":"admin","salt":""},{"enabled":1,"params":{"script":"#!/usr/bin/env python3\n\nimport os\nfrom pyspark import SparkContext, SparkConf\nfrom pyspark.sql import SparkSession\n\nconf = SparkConf()\n# place DB drivers into jars folder (e.g. /opt/cronicle/jars)\nconf.set(\"spark.driver.extraClassPath\", \"jars/*\");\nconf.set(\"spark.executor.extraClassPath\", \"jars/*\"); \n\nsc = SparkContext(conf = conf)\nspark = SparkSession(sc)\n\nurl = \"jdbc:mysql://127.0.0.1:3306/demo?user=root&password=root\"\n\ndf = spark \\\n .read \\\n .format(\"jdbc\") \\\n .option(\"url\", url) \\\n .option(\"query\", \"select * from mytable LIMIT 100\") \\\n .load()\n\n# Looks the schema of this DataFrame.\ndf.show(10)\n\n# ! on alpine install gcompat: apk add gcompat\ndf.write.format(\"parquet\").mode(\"overwrite\").save(\"/tmp/data.parquet\")","annotate":0,"json":0,"lang":"python","theme":"solarized light","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"PySpark_Parquet","silent":0,"graph_icon":"f111","args":"","ticks":"","category":"ckkog7wj41t","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"ekkqcopde2a","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","salt":"","id":"ekkqcf9m725","modified":1612501484,"created":1612411916,"username":"admin"},{"enabled":1,"params":{"script":"#!/usr/bin/env python3\n\n# to install python/pyspark on alpine:\n# apk add python3 gcompat\n# pip3 install pyspark\n# make sure Java (open-jdk) is installed too\n\nimport sys\nfrom pyspark import SparkContext\nfrom pyspark.sql import SparkSession\n\nsc = SparkContext(\"local\", \"word count\")\nspark = SparkSession(sc)\n\ntext_file = sc.textFile(\"/etc/os-release\")\ncounts = text_file.flatMap(lambda line: line.split(\" \")) \\\n .map(lambda word: (word, 1)) \\\n .reduceByKey(lambda a, b: a + b)\ncounts.toDF([\"word\", \"count\"]).show(50)","annotate":0,"json":0,"lang":"python","theme":"solarized light","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"PySpark_Hello","silent":0,"graph_icon":"f111","args":"","ticks":"","category":"ckkog7wj41t","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"ekkqcopde2a","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","id":"ekkqcd1gb23","modified":1612543081,"created":1612411813,"username":"admin","salt":""},{"enabled":1,"params":{"script":"#!/usr/bin/env -S java --source 11 -cp \"jars/*\"\n\n// use Java 11 or higher to run code as script\n// if using modern syntax (e.g. var or textblocks) you might switch to Scala syntax\n\nimport java.io.*;\nimport java.net.*;\n\npublic class HttpRequestDemo {\n \n public static void main(String[] args) throws Exception\n {\n System.out.println(getHTML(\"http://www.example.com\"));\n }\n\n public static String getHTML(String urlToRead) throws Exception {\n StringBuilder result = new StringBuilder();\n URL url = new URL(urlToRead);\n HttpURLConnection conn = (HttpURLConnection) url.openConnection();\n conn.setRequestMethod(\"GET\");\n BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));\n String line;\n while ((line = rd.readLine()) != null) {\n result.append(line);\n }\n rd.close();\n return result.toString();\n }\n}\n","annotate":0,"json":0,"lang":"java","theme":"darcula","sub_params":0,"tty":0},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"java_demo","category":"ckkog7wj41t","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"Randomly Generated Job","id":"ekkqa4w6s1a","modified":1612543529,"created":1612408073,"username":"admin","salt":"","silent":0,"graph_icon":"f111","args":"","ticks":"","web_hook_start":"","web_hook_error":0},{"enabled":1,"params":{"script":"#!/usr/bin/env bash\n\n# this event will be invoked by workflow event\n# you can get argument value using JOB_ARG variable\n\necho argument value: $JOB_ARG\nsleep $JOB_ARG","annotate":0,"json":0,"lang":"shell","theme":"default","sub_params":0},"timing":false,"max_children":3,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"00_wf_event","silent":0,"graph_icon":"f111","args":"","ticks":"","category":"general","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","web_hook_start":"","web_hook_error":0,"cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"","id":"ekkog584p1r","modified":1640585906,"created":1612297234,"username":"admin","salt":""},{"enabled":1,"params":{"duration":"7","progress":1,"burn":0,"action":"Success","secret":"Will not be shown in Event UI"},"timing":{"minutes":[29],"hours":[16]},"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"testplug","title":"03_wf_step_three","category":"general","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"Randomly Generated Job","id":"ekko474ip05","modified":1612296599,"created":1612277167,"username":"admin","salt":"","silent":0,"graph_icon":"f111","args":"","ticks":"","web_hook_start":"","web_hook_error":0},{"enabled":1,"params":{"duration":"10","progress":1,"burn":0,"action":"Success","secret":"Will not be shown in Event UI"},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"testplug","title":"02_wf_step_two","category":"general","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"wf step 2","id":"ekko473os04","modified":1612296610,"created":1612277166,"username":"admin","salt":"","silent":0,"graph_icon":"f111","args":"","ticks":"","web_hook_start":"","web_hook_error":0},{"enabled":1,"params":{"duration":"14","progress":1,"burn":0,"action":"Success","secret":"Will not be shown in Event UI"},"timing":false,"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"testplug","title":"01_wf_step_one","category":"general","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"Randomly Generated Job","id":"ekko472v803","modified":1612296628,"created":1612277165,"username":"admin","salt":"","silent":0,"graph_icon":"f111","args":"","ticks":"","web_hook_start":"","web_hook_error":0},{"enabled":1,"params":{"script":"#!/usr/bin/env bash\n\n# Cronicle injects some metadata to stdin \n# please note - stdin will be supressed if using tty option\n# also, printing JSON strings might cause issues (cronicle tries to interpret them)\ninput=\"$(cat - )\"\necho \"JSON data from stdin:\"\necho \"--> $input\"\n# most of the items from stdin JSON are available as env variables (e.g. JOB_ID)\n\necho \"------------------------------------------------\"\n\necho \"reporting progress:\"\n sleep 2 && echo \" -- step1 completed (10%)\" && echo 10%\n sleep 2 && echo \" -- step2 completed (50%)\" && echo 50%\n sleep 2 && echo \" -- step3 completed (90%)\" && echo 90%\n sleep 1\n\n# reporting performance \"Interpret JSON\" should be checked\necho '{\"perf\":{\"step3\":55,\"step2\":30,\"step1\":20}}'\n\n# alter chain reaction. Chain data can be sent to stdin of the new job\necho '{ \"chain\": \"ekko473os04\", \"chain_data\": { \"custom_key\": \"foobar\", \"value\": 42 } }' # on success\necho '{ \"chain_error\": \"ekko473os04\" }' # chain event on failure\n\n# alter email notification\necho '{ \"notify_success\": \"admin@cronicle.com\" }'\necho '{ \"notify_fail\": \"\" }'\n\n# paramter placeholders. Requires \"Resolve parameters\" to be checked\n# below example populates params.sql.demo value (from config)\necho sample parameter from config: \"[/sql/demo]\"\n\n# arguments (let non-editor users to parametrize script)\necho argument1: $ARG1 # as env variable\necho argument2: [/ARG2] # as parameter. Requires \"Resolve parameters\" to be checked\n\n# PLUGIN_SECRET a string you can set in plugin options.\n# It's injected as env variable to the running job, and never gets back to UI\necho plugin secret: $PLUGIN_SECRET","annotate":0,"json":1,"lang":"shell","theme":"gruvbox-dark","sub_params":1},"timing":{"minutes":[50],"hours":[23]},"max_children":1,"timeout":3600,"catch_up":0,"queue_max":1000,"timezone":"America/New_York","plugin":"shellplug","title":"shell_demo","category":"ckkog7wj41t","target":"allgrp","algo":"random","multiplex":0,"stagger":0,"retries":0,"retry_delay":0,"detached":0,"queue":0,"chain":"","chain_error":"","notify_success":"","notify_fail":"","web_hook":"","cpu_limit":0,"cpu_sustain":0,"memory_limit":0,"memory_sustain":0,"log_max_size":0,"notes":"Randomly Generated Job","id":"ekko4719i01","modified":1613680392,"created":1612277163,"username":"admin","salt":"","silent":0,"graph_icon":"f111","args":"New_York, 33","ticks":"2021-02-05 11:58","web_hook_start":"","web_hook_error":0}]} -global/servers - {"page_size":50,"first_page":0,"last_page":0,"length":1,"type":"list"} -global/servers/0 - {"type":"list_page","items":[{"hostname":"acad6ba3015a","ip":"172.17.0.3"}]} -global/api_keys - {"page_size":50,"first_page":0,"last_page":0,"length":0,"type":"list"} -global/api_keys/0 - {"type":"list_page","items":[]} -global/conf_keys - {"page_size":50,"first_page":0,"last_page":0,"length":25,"type":"list"} -global/conf_keys/0 - {"type":"list_page","items":[{"id":"base_app_url","title":"base_app_url","key":"http://localhost:3012","description":"overrides app url displayed in notifications"},{"id":"ad_domain","title":"ad_domain","key":"corp.cronicle.com","description":"default AD domain for external auth. You can also prepend domain to the username (e.g. user@domain.com)"},{"id":"smtp_hostname","title":"smtp_hostname","key":"mailrelay.cronicle.com","description":"SMTP server (port 25 is used default)"},{"id":"email_from","title":"email_from","key":"admin@cronicle.com","description":"Notification sender"},{"id":"admin_web_hook","title":"admin_web_hook","key":"","description":"Webhook for activity log notifications. Uses slack markdown.\nTip: use cronicle run api to handle notification with custom event"},{"id":"custom_live_log_socket_url","title":"custom_live_log_socket_url","key":"http://localhost:3012","description":"!this requires browser page refresh\noverrides the host for live log connection. On multinode cluster this can be assigned to each node, e.g. \ncustom_live_log_socket_url.manager\ncustom_live_log_socket_url.worker1\nCan specify custom port too. This is useful if using reverse proxy or docker/swarm"},{"id":"web_hook_text_templates_job_complete","title":"web_hook_text_templates.job_complete","key":"✔️ *[event_title]* completed successfully on [hostname] <[job_details_url] | More details>","description":"Success notification (slack markdown by default)"},{"id":"web_hook_text_templates_job_failure","title":"web_hook_text_templates.job_failure","key":"❌ *[event_title]* failed on [hostname]: Error: _*[description]*_ <[job_details_url] | More details>","description":"Error notification (slack markdown by default)"},{"id":"web_hook_text_templates_job_start","title":"web_hook_text_templates.job_start","key":"🚀 *[event_title]* started on [hostname] <[job_details_url] | More details>","description":"Start notification (slack markdown by default)"},{"id":"web_hook_text_templates_job_warning","title":"web_hook_text_templates.job_warning","key":"⚠️ *[event_title]* completed with warning on [hostname]: Warning: _*[description]*_ <[job_details_url] | More details>","description":"Warning notification. Warning is exit code 255 (-1) and it's treaded as success"},{"id":"web_hooks_slack_general","title":"web_hooks.slack_general","key":"https://hooks.slack.com/services/yourIncomingWebHook","description":"You can add webhook info under web_hooks object and then use property name (e.g. slack_general) to specify that webhook in notification options, instead of using full url. Use either url string (like this example) or object to specify custom data/options/headers and some other items (see example below)"},{"id":"web_hooks_slack_info_data_channel","title":"web_hooks.slack_info.data.channel","key":"cronicle","description":"Add custom key to request body (e.g. to specify channel)"},{"id":"web_hooks_slack_info_textkey","title":"web_hooks.slack_info.textkey","key":"markdown","description":"By default cronicle message is added as text key on webhook request body. Use this config if you need to use something else (e.g. markdown, html, etc). You can specify nested key too using dot notation e.g. 'data.mytextkey'"},{"id":"web_hooks_slack_info_compact","title":"web_hooks.slack_info.compact","type":"bool","key":false,"description":"(Notification webhooks only) Include only basic info in payload (id, title, action) and your custom data. Useful in case of key conflicts"},{"id":"web_hooks_slack_info_token","title":"web_hooks.slack_info.token","key":"xoxp-xxxxxxxxx-xxxx","description":"This is a shortcut for web_hooks.slack_info.headers.Authorization = Bearer xoxp-xxxxxxxxx-xxxx"},{"id":"web_hooks_slack_info_url","title":"web_hooks.slack_info.url","key":"https://slack.com/api/chat.postMessage","description":"Specify webhook url (for object). If using incoming webhooks then just specify it as string (see slack_general example above)"},{"id":"oninfo_web_hook","title":"oninfo_web_hook","key":"","description":"Special webhook - will fire on info message, e.g. server startup/restart/error. Those messages appear on activity log"},{"id":"universal_web_hook","title":"universal_web_hook","key":"","description":"Special webhook - will fire on each job start/completion"},{"id":"onupdate_web_hook","title":"onupdate_web_hook","key":"","description":"Special webhook - will fire on metadata update (e.g. on event update)"},{"id":"ui_live_log_ws","title":"ui.live_log_ws","type":"bool","key":false,"description":"Turns on classic websocket api for live log"},{"id":"params_demo","title":"params.sql.demo","key":"SELECT * FROM\nSOMETABLE s \nWHERE s.col = 30","type":"text/x-sql","description":" params config (object) can be used to set placeholders in shell scripts. You need to check 'resolve parameters' box in event parameters. To set placeholder use square braket syntax, e.g. for this parameter you should use [/sql/demo]"},{"id":"git_enabled","title":"git.enabled","type":"bool","key":false,"description":"Sync up data folder with remote git repository. To push data press backup button on scheduled event page.\n Should init git repo in data folder first. You can also set remote, branch and user via config (default is origin/master/cronicle)"},{"id":"git_auto","title":"git.auto","type":"bool","key":false,"description":"If git is enabled this option will trigger git sync on each metadata update (e.g. on event change)"},{"id":"git_add","title":"git.add","key":"global,users","description":"List of folders/files to sync via git sync. Global and user folders are default (covers metadata).\n Add 'logs' folder to keep trends/activity logs and 'jobs' for entire job logs"},{"id":"_read_me_","title":"_read_me_","key":"please read","description":"Those keys are applied right after storage and webserver init, and then can be updated at runtime (no need to restart cronicle). Please note that you cannot override storage/webserver parameters.\nTo add nested config (object) use dot syntax, e.g. servers.host1. If you convert some nested key into string it would erase related subkeys from config object. In this case just remove that string key and click reload button . To check actual config state use Config Viewer link"}]} -global/secrets - {"page_size":50,"first_page":0,"last_page":0,"length":1,"type":"list"} -global/secrets/0 - {"type":"list_page","items":[{"id":"globalenv","encrypted":false,"target":null,"form":"props","data":"# dotenv style key/value pairs representing env variables\nmyvar = some_value"}]} diff --git a/sample_conf/examples/docker.sh b/sample_conf/examples/docker.sh deleted file mode 100644 index 2741721..0000000 --- a/sample_conf/examples/docker.sh +++ /dev/null @@ -1,16 +0,0 @@ - -# some docker commands to set up various storage systems -# should work out of the box with sample storage configs (if using localhost) - -# Postgres: -docker run -d --name postgres -e POSTGRES_PASSWORD='P@ssword' -p 5432:5432 -d postgres - -# Mysql -docker run -d --name mysql -p 3306:3306 -e MYSQL_DATABASE=cronicle -e MYSQL_ROOT_PASSWORD='P@ssword' mysql - -# S3 / minio - create "cronicle" bucket in UI (http://localhost:9001), use "minioadmin" as user and password -docker run -d -p 9000:9000 -p 9001:9001 quay.io/minio/minio server /data --console-address ":9001" - -# Redis -docker run -d --name redis -p 6379:6379 redis - From 0df1ce29eaa04726968a00867b1f20e848f7f8b8 Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Fri, 5 Apr 2024 11:07:20 -0400 Subject: [PATCH 28/31] added --sqlite option for manager (win) /storage-cli --- bin/manager.bat | 9 +++++++++ bin/storage-cli.js | 18 ++++++++++++++++++ bundle.ps1 | 7 ++++--- lib/main.js | 18 ++++++++++++++---- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/bin/manager.bat b/bin/manager.bat index 1cc849a..6e209a1 100644 --- a/bin/manager.bat +++ b/bin/manager.bat @@ -34,6 +34,15 @@ if /I "%1"=="--port" ( echo Using custom secret key: ***** shift shift +) else if /I "%1"=="--sqlite" ( + if "%2"=="" ( + echo Sqlite db path is not specified + exit + ) + set CRONICLE_sqlite=%~f2 + echo Using sqlite as storage: %~f2 + shift + shift ) else if /I "%1"=="--help" ( echo Usage: .\manager [--port port] [ --storage /path/to/storage.json] shift diff --git a/bin/storage-cli.js b/bin/storage-cli.js index 864feaf..6e7ed85 100755 --- a/bin/storage-cli.js +++ b/bin/storage-cli.js @@ -27,6 +27,24 @@ if(fs.existsSync(storage_config)) { config.Storage = require(storage_config) } +// overwrite storage if sqlite option is specified +if(process.env['CRONICLE_sqlite']) { + config.Storage = { + "engine": "SQL", + "list_page_size": 50, + "concurrency": 4, + "log_event_types": { "get": 1, "put": 1, "head": 1, "delete": 1, "expire_set": 1 }, + "SQL": { + "client": "sqlite3", + "table": "cronicle", + "useNullAsDefault": true, + "connection": { + "filename": process.env['CRONICLE_sqlite'] + } + } + } +} + // shift commands off beginning of arg array var argv = JSON.parse(JSON.stringify(process.argv.slice(2))); var commands = []; diff --git a/bundle.ps1 b/bundle.ps1 index f090982..df78f5d 100644 --- a/bundle.ps1 +++ b/bundle.ps1 @@ -54,7 +54,7 @@ foreach ($arg in $PSBoundParameters.Values) { } if($Path -like "--*") { - # perhaps user meant some flag, fallback Path to default + # User likely meant some flag, fallback Path to default $Path = "dist" } @@ -81,7 +81,7 @@ function Write-Bold { param($Text, [switch]$U) $FullPath = mkdir -Force $Path -Write-Bold "`nInstalling cronicle bundle into $FullPath`n" -U | Write-Host -ForegroundColor Green +Write-Bold "`nInstalling cronicle bundle into $FullPath`n" -U # Write-Host "-----------------------------------------" # Write-Host " Installing cronicle bundle into $($Path)" @@ -370,6 +370,7 @@ if($sqlDrivers.Count -gt 0) { $sqlArgs.Add("engines/SQL.js") | Out-Null Write-Host " - bundling SQL Engine [$($sqlDrivers -join ",")]" + $engines += ", SQL [$($sqlDrivers -join ",")]" & npm $sqlInstall & esbuild $sqlArgs if($Sqlite) { @@ -453,7 +454,7 @@ Write-Host " - Init cronicle storage (on the first run): node .\$Path\bin\stora Write-Host " - Start cronicle as manage: node .\$Path\bin\cronicle.js --echo --foreground --manager --color`n" Write-Bold "You can also setup/start cronicle using [manager] entrypoint:" -Write-Host ".\$Path\bin\manager [ --port 3012 ] [ --storage Path\to\storage.json ] [ --key someSecretKey ]`n" +Write-Host ".\$Path\bin\manager [ --port 3012 ] [ --storage Path\to\storage.json ] [ --sqlite Path\to\sqlite.db ] [ --key someSecretKey ]`n" Write-Bold "To Reinstall/upgrade run (please back up $FullPath first):" Write-Host ".\bundle.ps1 $Path -Force`n" diff --git a/lib/main.js b/lib/main.js index 6ce3b60..a8b3f9a 100644 --- a/lib/main.js +++ b/lib/main.js @@ -29,10 +29,20 @@ if(fs.existsSync(config_file)) configFiles.push( { file: config_file }) -let storage_config = process.env['CRONICLE_storage_config'] || "conf/storage.json" -if(fs.existsSync(storage_config)) configFiles.push( { - file: storage_config , key: "Storage" -}) +// override storage config if needed +if (process.env['CRONICLE_sqlite']) { // use sqlite + process.env["CRONICLE_Storage__engine"] = "SQL" + process.env["CRONICLE_Storage__SQL__connection__filename"] = process.env['CRONICLE_sqlite'] + process.env["CRONICLE_Storage__SQL__client"] = "sqlite3" + process.env["CRONICLE_Storage__SQL__table"] = "cronicle" + process.env["CRONICLE_Storage__SQL__useNullAsDefault"] = 1 +} +else { // or resolve storage config from files + let storage_config = process.env['CRONICLE_storage_config'] || "conf/storage.json" + if (fs.existsSync(storage_config)) configFiles.push({ + file: storage_config, key: "Storage" + }) +} const server = new PixlServer({ From 9441a4ab398ae412037ea621e74d9e9f634a87fe Mon Sep 17 00:00:00 2001 From: miketwc1984 Date: Fri, 5 Apr 2024 13:36:51 -0400 Subject: [PATCH 29/31] event flow to the top --- htdocs/js/pages/Home.class.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/htdocs/js/pages/Home.class.js b/htdocs/js/pages/Home.class.js index 73ae5b5..e1cc214 100644 --- a/htdocs/js/pages/Home.class.js +++ b/htdocs/js/pages/Home.class.js @@ -13,17 +13,8 @@ Class.subclass( Page.Base, "Page.Home", {
-
- - +
-
- Active Jobs -
-
-
-
- @@ -55,6 +46,16 @@ Class.subclass( Page.Base, "Page.Home", {
+ + +
+ Active Jobs +
+
+
+
+ +