Merge pull request cronicle-edge#98 from cronicle-edge/182

v182

Dockerfile

@@ -11,7 +11,7 @@ FROM cronicle/base-alpine as build
 RUN apk add --no-cache npm 
 COPY . /build
 WORKDIR /build
-RUN ./bundle /dist --mysql --pgsql --s3 --lmdb
+RUN ./bundle /dist --mysql --pgsql --s3 --sqlite --tools
 
 FROM cronicle/base-alpine 
 

Readme.md

@@ -9,7 +9,7 @@ You can quickly try it using Docker using image from Docker Hub:
 ```
 Or build the image locally - clone or download the latest release and run command below from project root
 ```bash
-docker build -t cronicle:edge -f DockerfileDev --build-arg echo=1  .
+docker build -t cronicle/edge -f Dockerfile .
 ```
 
 To test multi-node set up (in swarm mode)

bin/control.sh.ps1

@@ -13,12 +13,14 @@ param(
     # list collection items
     [Int32]$Limit = 50,
     [Int32]$Offset = 0,
-    [switch]$Full
+    [switch]$Full,
+
+    [switch]$Help
 )
 
 $ErrorActionPreference = "stop"
 
-if (!$Command) {
+if (!$Command -OR $Help) {
     Write-Host "Usage:
 
     .\control.ps1 start [-Manager] [-Echo] [-Force] [-Color] [-WinMon] [-Hide]

bin/manager

@@ -1,6 +1,49 @@
 #!/bin/bash
 HOMEDIR="$(dirname "$(cd -- "$(dirname "$(readlink -f "$0")")" && (pwd -P 2>/dev/null || pwd))")"
 
+usage() {
+  echo "USAGE: ./manager [ --port 3012 ] [ --storage /path/to/storage.json ] [ --key secreKey ] [ --color ] "
+  exit 1
+}
+
+color=0
+
+while (( "$#" )); do
+    case $1 in
+        --color ) color=1 ;;
+        --port ) shift; port=$1 ;;
+        --key ) shift; key=$1 ;;
+        --storage ) shift; storage=$1 ;;
+        --sqlite ) shift; sqlite=$1 ;;
+        --help ) usage ;;
+        -*) echo "invalid parameter: $1"; usage ;;
+    esac
+shift
+done
+
+if [[ $port ]]; then
+  export CRONICLE_WebServer__http_port="$port"
+  echo "Custom port set: $port"
+fi
+
+if [[ $sqlite ]]; then
+  export CRONICLE_sqlite="$(realpath $sqlite)"
+  echo "Using custom sqlite db as storage: $sqlite"
+fi
+
+if [[ $storage ]]; then
+  if [[ -f $storage ]]; then
+    export CRONICLE_storage_config="$(realpath $storage)"
+    echo "Custom storage set: $(basename $storage)"
+  else echo "Error: $storage file doesn't exist"; exit 1
+  fi
+fi
+
+if [[ $key ]]; then
+  export CRONICLE_secret_key=$key
+  echo "Custom secret key set: *****"
+fi
+
 # pull data from git if needed
 # if [ ! -d data/global ] && [ -v GIT_REPO ]; then
 #         git clone $GIT_REPO $HOMEDIR/data
@@ -33,12 +76,8 @@ if [ -f "$HOMEDIR/bin/cronicle.js" ]; then
     # echo "starting bundle"
 fi
 
-# check for custom http port (pass as first arg)
-if [ -n "$1" ]; then
-  export CRONICLE_WebServer__http_port="$1"
-fi
 #$HOMEDIR/bin/control.sh start
-exec $BINARY --echo --foreground --manager --color 1
+exec $BINARY --echo --foreground --manager --color $color
 
 
 

bin/manager.bat

@@ -1,6 +1,58 @@
 
 @echo off
-cd /D "%~dp0"
+
+set SCRIPT_LOC=%~dp0
+
+:parseArgs
+
+if "%1"=="" goto endArgs
+
+if /I "%1"=="--port" (
+    if "%2"=="" (
+      echo Specify http port for cronicle
+      exit
+    )    
+    set CRONICLE_WebServer__http_port=%2
+    echo Custom port set: %2
+    shift
+    shift
+) else if /I "%1"=="--storage" (
+    if "%2"=="" (
+      echo Specify path to storage.json config file
+      exit
+    )
+    set CRONICLE_storage_config=%~f2
+    echo Custom storage set: %~f2
+    shift
+    shift
+) else if /I "%1"=="--key" (
+    if "%2"=="" (
+      echo Secret key not specified
+      exit
+    )    
+    set CRONICLE_secret_key=%2
+    echo Using custom secret key: *****
+    shift
+    shift
+) else if /I "%1"=="--sqlite" (
+    if "%2"=="" (
+      echo Sqlite db path is not specified
+      exit
+    )    
+    set CRONICLE_sqlite=%~f2
+    echo Using sqlite as storage: %~f2
+    shift
+    shift
+) else if /I "%1"=="--help" (
+    echo Usage:  .\manager [--port  port] [ --storage /path/to/storage.json]
+    shift    
+) else (exit)
+
+goto parseArgs
+
+:endArgs
+
+cd /D %SCRIPT_LOC%
 
 REM check for custom node version
 IF EXIST "%~dp0..\nodejs\node.exe" (
@@ -14,4 +66,4 @@ if not "%~1"=="" (
     echo CRONICLE_http_port is set to %1
 )
 
-node .\cronicle.js --manager --echo --foreground --color
+node .\cronicle.js --manager --echo --foreground --color

bin/storage-cli.js

@@ -27,6 +27,24 @@ if(fs.existsSync(storage_config)) {
         config.Storage = require(storage_config)                                                    
 }
 
+// overwrite storage if sqlite option is specified
+if(process.env['CRONICLE_sqlite']) {
+	config.Storage = {
+		"engine": "SQL",
+		"list_page_size": 50,
+		"concurrency": 4,
+		"log_event_types": { "get": 1, "put": 1, "head": 1,	"delete": 1, "expire_set": 1 },
+		"SQL": {
+			"client": "sqlite3",
+			"table": "cronicle",
+			"useNullAsDefault": true,
+			"connection": {
+				"filename": process.env['CRONICLE_sqlite']
+			}
+		}
+	}
+}
+
 // shift commands off beginning of arg array
 var argv = JSON.parse(JSON.stringify(process.argv.slice(2)));
 var commands = [];
@@ -47,7 +65,13 @@ args = args.get(); // simple hash
 // copy debug flag into config (for standalone)
 config.Storage.debug = args.debug;
 
-// disable storage transactions for CLI
+// indicate that you want to enable engine level transaction
+// to pack multiple crud operation in one transaction
+// this is default for setup/migration, to avoid use --notrx argument
+if(cmd == 'install' || cmd == 'setup') config.Storage.trans = true
+if(args.notrx) config.Storage.trans = false
+
+// disable storage transactions for CLI (this is storage level transaction)
 config.Storage.transactions = false;
 
 var print = function (msg) {

bin/storage-migrate.js

@@ -24,6 +24,37 @@ var cli = require('pixl-cli');
 var args = cli.args;
 cli.global();
 
+
+// check if --from/--to args
+
+if(args.help) {
+	console.log('Usage: ./storage-migrate ')
+	console.log(' [ --from /path/to/storage.json] # alter source storage config (default is "Storage" property from config.json)')
+	console.log(' [ --to /path/to/storage.json] # alter destincation storage config (default is "NewStorage" property from config.json)')
+	console.log(' [ --notrx # disable transaction on destination storage (only applicable to SQL and Lmdb)')
+	console.log(' [ --sqlite /path/to/file.db  # dump source storage into local sqlite DB (sqlite engine is required')
+	console.log(' [ --full ] # include job log files')
+	console.log(' [ --help ] # see this message again')
+	process.exit(1)
+}
+
+let oldConfPath;
+let newConfPath;
+let sqlitePath;
+
+if(args.from) {
+	if(fs.existsSync(Path.resolve(args.from))) oldConfPath = Path.resolve(args.from)
+	else { console.log("config file does not exist:", Path.resolve(args.from)); process.exit(1)}
+}
+if(args.sqlite) {
+	if(fs.existsSync(Path.dirname(Path.resolve(args.sqlite)))) sqlitePath = Path.resolve(args.sqlite)
+	else { console.log("sqlite file location does not exist:", Path.resolve(args.sqlite)); process.exit(1)}	
+}
+if(args.to) {
+	if(fs.existsSync(Path.resolve(args.to))) newConfPath = Path.resolve(args.to)
+	else { console.log("config file does not exist:", Path.resolve(args.to)); process.exit(1)}
+}
+
 var StandaloneStorage = require('pixl-server-storage/standalone');
 
 // chdir to the proper server root dir
@@ -59,7 +90,26 @@ var StorageMigrator = {
 		this.logPrint(1, "Cronicle Storage Migration Script v" + this.version + " starting up");
 		this.logPrint(2, "Starting storage engines");
 
+		if(oldConfPath) config.Storage = require(oldConfPath) // custom storage config file
 		if (!config.Storage) this.fatal("Your Cronicle configuration lacks a 'Storage' property");
+		if(sqlitePath) {
+			config.NewStorage = {
+				"engine": "SQL",
+				"list_page_size": 50,
+				"concurrency": 4,
+				"log_event_types": { "get": 1, "put": 1, "head": 1,	"delete": 1, "expire_set": 1 },
+				"SQL": {
+					"client": "sqlite3",
+					"table": "cronicle",
+					"useNullAsDefault": true,
+					"connection": {
+						"filename": sqlitePath
+					}
+				}
+			}
+		}
+
+		if (newConfPath) config.NewStorage = require(newConfPath)
 		if (!config.NewStorage) this.fatal("Your Cronicle configuration lacks a 'NewStorage' property.");
 
 		if (config.uid && (process.getuid() != 0)) {
@@ -76,10 +126,13 @@ var StorageMigrator = {
 		// massage config, override logger
 		config.Storage.logger = self.logger;
 		config.Storage.log_event_types = { all: 1 };
-
+				
 		config.NewStorage.logger = self.logger;
 		config.NewStorage.log_event_types = { all: 1 };
-
+		// if engine supports (sql and lmdb) begin transaction
+		config.NewStorage.trans = true
+		if(args.notrx) config.NewStorage.trans = false
+
 		// start both standalone storage instances
 		async.series(
 			[
@@ -99,9 +152,9 @@ var StorageMigrator = {
 				if (config.uid && (process.getuid() == 0)) {
 					self.logPrint( 3, "Switching to user: " + config.uid );
 					process.setuid( config.uid );
-				}
-
-				self.testStorage();
+				}				
+				self.testStorage();		
+
 			}
 		); // series
 	},
@@ -110,7 +163,7 @@ var StorageMigrator = {
 		// test both old and new storage
 		var self = this;
 		this.logDebug(3, "Testing storage engines");
-		
+
 		async.series(
 			[
 				function(callback) {
@@ -119,8 +172,11 @@ var StorageMigrator = {
 				function(callback) {
 					self.newStorage.put('test/test1', { "foo1": "bar1" }, function(err) {
 						if (err) return callback(err);
+
+						// throw new Error('before del')
 
-						self.newStorage.delete('test/test1', function(err) {
+						self.newStorage.delete('test/test1', function(err) {						
+
 							if (err) return callback(err);
 
 							callback();
@@ -142,7 +198,7 @@ var StorageMigrator = {
 		var self = this;
 		this.logPrint(3, "Starting migration");
 
-		this.timeStart = Tools.timeNow(true);
+		this.timeStart = Date.now()
 		this.numRecords = 0;
 
 		this.copyKey( 'global/state', { ignore: true } );
@@ -261,13 +317,14 @@ var StorageMigrator = {
 	finish: function() {
 		// all done
 		var self = this;
-		var elapsed = Tools.timeNow(true) - this.timeStart;
+		var elapsed = Date.now() - this.timeStart;
+		var dur = elapsed < 2000 ? `${elapsed} ms` : Tools.getTextFromSeconds(elapsed/1000, false, true) 
 
 		cli.progress.end();
 
 		print("\n");
 		this.logPrint(1, "Storage migration complete!");
-		this.logPrint(2, Tools.commify(this.numRecords) + " total records copied in " + Tools.getTextFromSeconds(elapsed, false, true) + ".");
+		this.logPrint(2, Tools.commify(this.numRecords) + " total records copied in " + dur + ".");
 		this.logPrint(4, "You should now overwrite 'Storage' with 'NewStorage' in your config.json.");
 		this.logPrint(3, "Shutting down");