3.0.3

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 3.0.3 - 2024-04-08
+### Update
+* Refine templates handling
+* `Backtesting`: extract trial parameters into `/home/ubuntu/.MartinBinance/trial_params.json`, [detail here.](https://github.com/DogsTailFarmer/martin-binance/wiki/Back-testing-and-parameters-optimization#some-details)
+* `Backtesting`: add logging importance parameters rating for each optimisation cycle
+* Up requirements for exchanges-wrapper==2.1.8
+
 ## 3.0.2 - 2024-04-03
 ### Fix
 * `Backtesting`: updating the balances at slippage
@@ -102,9 +109,9 @@
 * ModuleNotFoundError: No module named 'optimizer' #73
 
 ### Update
-* Do not send orders if was getting 'FAILED_PRECONDITION' response
-* Do not started Telegram control process for Simulate mode or if bot id is not setting
-* Do not started DB control process for Simulate mode
+* Do not send orders if it was getting 'FAILED_PRECONDITION' response
+* Do not start Telegram control process for Simulate mode or if bot id is not setting
+* Do not start DB control process for Simulate mode
 * Changed initialization sequence during recovery from saved state
 
 ## 2.1.4 - 2024-02-22
@@ -419,6 +426,8 @@ Ticker history includes fulfillment events that may not be in the original strea
 at price P, and there was no such price in the stream at a given time period. This led to a divergence of the test
 session against the original (reference).
 
+<!--suppress ALL -->
+
 <img height="350" src="https://user-images.githubusercontent.com/77513676/253804148-ed92bf13-ca04-46bf-8714-28380d0c2e52.png" width="350"/>
 
 * `README.md`: some design correction

martin_binance/__init__.py

@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.2"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 
@@ -26,24 +26,36 @@
 LOG_PATH = Path(WORK_PATH, "log")
 LAST_STATE_PATH = Path(WORK_PATH, "last_state")
 BACKTEST_PATH = Path(WORK_PATH, "back_test")
+TRIAL_PARAMS = Path(WORK_PATH, "trial_params.json")
 EQUAL_STR = "================================================================"
 
+# TODO remove after update to 3.0.3
+if CONFIG_FILE.exists() and not TRIAL_PARAMS.exists():
+    copy(Path(Path(__file__).parent.absolute(), "templates/trial_params.json"), TRIAL_PARAMS)
+
 
 def init():
     if CONFIG_FILE.exists():
         print(f"Client config found at {CONFIG_FILE}")
     else:
         print("Can't find client config file! Creating it...")
-        CONFIG_PATH.mkdir(parents=True, exist_ok=True)
-        LOG_PATH.mkdir(parents=True, exist_ok=True)
-        LAST_STATE_PATH.mkdir(parents=True, exist_ok=True)
-        copy(Path(Path(__file__).parent.absolute(), "ms_cfg.toml.template"), CONFIG_FILE)
-        copy(Path(Path(__file__).parent.absolute(), "funds_rate.db.template"), DB_FILE)
-        copy(Path(Path(__file__).parent.absolute(), "cli_0_BTCUSDT.py.template"), Path(WORK_PATH, "cli_0_BTCUSDT.py"))
-        copy(Path(Path(__file__).parent.absolute(), "cli_1_BTCUSDT.py.template"), Path(WORK_PATH, "cli_1_BTCUSDT.py"))
-        copy(Path(Path(__file__).parent.absolute(), "cli_2_TESTBTCTESTUSDT.py.template"),
-             Path(WORK_PATH, "cli_2_TESTBTCTESTUSDT.py"))
-        copy(Path(Path(__file__).parent.absolute(), "cli_3_BTCUSDT.py.template"), Path(WORK_PATH, "cli_3_BTCUSDT.py"))
+        for path in [CONFIG_PATH, LOG_PATH, LAST_STATE_PATH]:
+            path.mkdir(parents=True, exist_ok=True)
+
+        templates = Path(Path(__file__).parent.absolute(), "templates")
+
+        copy(Path(templates, "ms_cfg.toml"), CONFIG_FILE)
+
+        files_to_copy = [
+            "funds_rate.db",
+            "trial_params.json",
+            "cli_0_BTCUSDT.py",
+            "cli_1_BTCUSDT.py",
+            "cli_2_TESTBTCTESTUSDT.py",
+            "cli_3_BTCUSDT.py"
+        ]
+        [copy(Path(templates, file_name), Path(WORK_PATH, file_name)) for file_name in files_to_copy]
+
         print(f"Before the first run, set the parameters in {CONFIG_FILE}")
         raise SystemExit(1)
 

martin_binance/backtest/optimizer.py

@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2024 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.1"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 
@@ -22,7 +22,7 @@
 import optuna
 import ujson as json
 
-from martin_binance import LOG_PATH
+from martin_binance import LOG_PATH, TRIAL_PARAMS
 
 OPTIMIZER = Path(__file__).absolute()
 OPTIMIZER.chmod(OPTIMIZER.stat().st_mode | stat.S_IEXEC)
@@ -54,23 +54,25 @@ def optimize(study_name, cli, n_trials, storage_name=None, _prm_best=None, skip_
     sys.excepthook = notify_exception
     optuna.logging.set_verbosity(optuna.logging.WARNING)
 
+    # Load parameter definitions from JSON file
+    with open(TRIAL_PARAMS) as f:
+        param_defs = json.load(f)
+
     spec = iu.spec_from_file_location("strategy", cli)
     mbs = iu.module_from_spec(spec)
     spec.loader.exec_module(mbs)
 
     def objective(_trial):
-        params = {
-            'GRID_MAX_COUNT': _trial.suggest_int('GRID_MAX_COUNT', 3, 5),
-            'PRICE_SHIFT': _trial.suggest_float('PRICE_SHIFT', 0, 0.05, step=0.01),
-            'PROFIT': _trial.suggest_float('PROFIT', 0.05, 0.2, step=0.05),
-            'PROFIT_MAX': _trial.suggest_float('PROFIT_MAX', 0.4, 1.0, step=0.05),
-            'OVER_PRICE': _trial.suggest_float('OVER_PRICE', 0.1, 1, step=0.1),
-            'ORDER_Q': _trial.suggest_int('ORDER_Q', 6, 12),
-            'MARTIN': _trial.suggest_float('MARTIN', 5, 15, step=1),
-            'SHIFT_GRID_DELAY': _trial.suggest_int('SHIFT_GRID_DELAY', 10, 150, step=10),
-            'KBB': _trial.suggest_float('KBB', 0.5, 4, step=0.5),
-            'LINEAR_GRID_K': _trial.suggest_int('LINEAR_GRID_K', 0, 500, step=50),
-        }
+        params = {}
+        for param_name, param_props in param_defs.items():
+            if param_props['type'] == 'int':
+                params[param_name] = _trial.suggest_int(
+                    param_name, *param_props['range'], step=param_props.get('step', 1)
+                )
+            elif param_props['type'] == 'float':
+                params[param_name] = _trial.suggest_float(
+                    param_name, *param_props['range'], step=param_props.get('step', 0.1)
+                )
         return try_trade(mbs, skip_log, **params)
 
     # noinspection PyArgumentList
@@ -117,8 +119,13 @@ async def run_optimize(*args):
     else:
         new_value = round(study.best_value, ndigits=6)
         bp = {k: int(any2str(v)) if isinstance(v, int) else float(any2str(v)) for k, v in study.best_params.items()}
+
         logger.info(f"Optimal parameters: {bp} for get {new_value}")
+        if new_value:
+            logger.info(f"Importance parameters: {optuna.importance.get_param_importances(study)}")
+
         _value = round(study.get_trials()[0].value, ndigits=6)
+
         if not prm_best or new_value > _value:
             bp |= {'new_value': any2str(new_value), '_value': any2str(_value)}
             print(json.dumps(bp))

martin_binance/db_utils.py

@@ -4,13 +4,13 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.0"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 
 import contextlib
 import sqlite3
-from datetime import datetime
+from datetime import datetime, timezone
 import logging
 
 from martin_binance import DB_FILE
@@ -92,7 +92,7 @@ def save_to_db(queue_to_db) -> None:
                                              float(data.get('s_depo')),
                                              float(data.get('f_profit')),
                                              float(data.get('s_profit')),
-                                             datetime.utcnow(),
+                                             datetime.now(timezone.utc).replace(tzinfo=None),
                                              float(data.get('PRICE_SHIFT')),
                                              float(data.get('PROFIT')),
                                              float(data.get('over_price')),

martin_binance/executor.py

@@ -4,7 +4,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.2"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = 'https://github.com/DogsTailFarmer'
 ##################################################################
@@ -324,7 +324,8 @@ def event_exec_command(self):
     def event_report(self):
         is_time_for_report_update = STATUS_DELAY and (self.get_time() - self.status_time) / 60 > STATUS_DELAY
         if self.command == 'status' or is_time_for_report_update:
-            self.command = None
+            if self.command == 'status':
+                self.command = None
             last_price = self.get_buffered_ticker().last_price
             ticker_update = int(self.get_time()) - self.last_ticker_update
             if self.cycle_time:
@@ -392,7 +393,8 @@ def event_report(self):
                 self.message_log(f"{header}\n"
                                  f"{'*** Shift grid mode ***' if self.shift_grid_threshold else '* **  **  ** *'}\n"
                                  f"{'Buy' if self.cycle_buy else 'Sell'}{' Reverse' if self.reverse else ''}"
-                                 f"{' Hold reverse' if self.reverse_hold else ''} {MODE}-cycle with"
+                                 f"{' Hold reverse' if self.reverse_hold else ''} "
+                                 f"{MODE}{'-SO' if MODE == 'TC' and SELF_OPTIMIZATION else ''}-cycle with"
                                  f" {order_buy} buy and {order_sell} sell active orders.\n"
                                  f"{order_hold or 'No'} hold grid orders\n"
                                  f"Over price: {self.over_price:.2f}%\n"

martin_binance/cli_0_BTCUSDT.py.template → martin_binance/templates/cli_0_BTCUSDT.py

@@ -7,7 +7,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.1"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 """

martin_binance/cli_1_BTCUSDT.py.template → martin_binance/templates/cli_1_BTCUSDT.py

@@ -7,7 +7,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.1"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 """

martin_binance/cli_2_TESTBTCTESTUSDT.py.template → martin_binance/templates/cli_2_TESTBTCTESTUSDT.py

@@ -7,7 +7,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.1"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 """

martin_binance/cli_3_BTCUSDT.py.template → martin_binance/templates/cli_3_BTCUSDT.py

@@ -7,7 +7,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "3.0.1"
+__version__ = "3.0.3"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 """

martin_binance/templates/trial_params.json

@@ -0,0 +1,12 @@
+{
+  "GRID_MAX_COUNT": {"type": "int", "range": [3, 5]},
+  "PRICE_SHIFT": {"type": "float", "range": [0, 0.05], "step": 0.01},
+  "PROFIT": {"type": "float", "range": [0.05, 0.2], "step": 0.05},
+  "PROFIT_MAX": {"type": "float", "range": [0.4, 1.0], "step": 0.05},
+  "OVER_PRICE": {"type": "float", "range": [0.1, 1], "step": 0.1},
+  "ORDER_Q": {"type": "int", "range": [6, 12]},
+  "MARTIN": {"type": "float", "range": [5, 15], "step": 0.5},
+  "SHIFT_GRID_DELAY": {"type": "int", "range": [10, 150], "step": 10},
+  "KBB": {"type": "float", "range": [0.5, 4], "step": 0.5},
+  "LINEAR_GRID_K": {"type": "int", "range": [0, 500], "step": 50}
+}

pyproject.toml

@@ -17,7 +17,7 @@ dynamic = ["version", "description"]
 requires-python = ">=3.9"
 
 dependencies = [
-    "exchanges-wrapper==2.1.7",
+    "exchanges-wrapper==2.1.8",
     "jsonpickle==3.0.2",
     "psutil==5.9.6",
     "requests==2.31.0",

requirements.txt

@@ -1,4 +1,4 @@
-exchanges-wrapper==2.1.7
+exchanges-wrapper==2.1.8
 jsonpickle==3.0.2
 psutil==5.9.6
 requests==2.31.0