2.1.0

DogsTailFarmer · Feb 16, 2024 · 2b1553d · 2b1553d
1 parent e037af2
commit 2b1553d
Show file tree

Hide file tree

Showing 12 changed files with 67 additions and 96 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,11 +1,13 @@
+## 2.1.0 - 2024-02-16
+### Added for new features
+:rocket: Complete cyclic optimization of strategy parameters based on [`optuna` framework](https://optuna.readthedocs.io/en/stable/index.html)
+For reference see [Back testing and parameters optimization](https://github.com/DogsTailFarmer/martin-binance/wiki/Back-testing-and-parameters-optimization)
+
 ### Update
 * `cancel_order_call`: added timeout handler
 * Utilised partially filled TP amount during grid update
 * `create_limit_order`: Supress call strategy handler for `grpc.StatusCode.FAILED_PRECONDITION` exception,
-fire tlg message only. As a result, there are no attempts to re-place the order.
-
-### Added for new features
-* Complete cyclic optimization of strategy parameters
+fire tlg message only. As a result, there are no attempts to re-place the order with incorrect condition.
 
 ## 2.1.0rc36 - 2024-02-11
 ### Fix

diff --git a/README.md b/README.md
@@ -26,10 +26,8 @@ All risks and possible losses associated with use of this strategy lie with you.
 Strongly recommended that you test the strategy in the demo mode before using real bidding.
 
 ## Important notices
-* For [exchanges-wrapper](https://github.com/DogsTailFarmer/exchanges-wrapper) `v>= v1.4.0` must be updated `exch_srv_cfg.toml`
-* After updating the version to `v2.0.0`, the configuration files `cli_XX_AAABBB.py` for all running trading pairs
-should be updated
-* Lost compatibility with `margin.de` terminal.
+* After update to `2.1.0`, the configuration files `cli_XX_AAABBB.py` for all running trading pairs
+should be updated, also update the configuration files `exch_srv_cfg.toml` and `ms_cfg.toml`. Use templates for reference.
 
 * You cannot run multiple pairs with overlapping currencies on the same account!
 

diff --git a/martin_binance/__init__.py b/martin_binance/__init__.py
@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "2.1.0rc40"
+__version__ = "2.1.0"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 

diff --git a/martin_binance/backtest/OoTSP.py b/martin_binance/backtest/OoTSP.py
@@ -18,7 +18,7 @@
 from martin_binance import BACKTEST_PATH
 from optimizer import optimize
 
-SKIP_LOG = False
+SKIP_LOG = True
 
 vis = optuna.visualization
 ii_params = []

diff --git a/martin_binance/backtest/exchange_simulator.py b/martin_binance/backtest/exchange_simulator.py
@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "1.3.4"
+__version__ = "2.1.0"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 
@@ -137,6 +137,7 @@ class Account:
         "grid_buy",
         "grid_sell",
         "ticker_last",
+        "market_ids",
     )
 
     def __init__(self, save_ds: bool):
@@ -152,6 +153,7 @@ def __init__(self, save_ds: bool):
         self.grid_buy = {}
         self.grid_sell = {}
         self.ticker_last = Decimal('0')
+        self.market_ids = []
 
     def create_order(
             self,
@@ -172,38 +174,23 @@ def create_order(
                       price=price,
                       lt=lt)
 
-        if self.ticker_last and ((buy and Decimal(price) >= self.ticker_last) or
-                                 (not buy and Decimal(price) <= self.ticker_last)):
-            # Market event
-            order.transact_time = lt
-            order.executed_qty = order.orig_qty
-            order.cummulative_quote_qty = str(Decimal(order.orig_qty) * Decimal(order.price))
-            order.status = 'FILLED'
-            order.event_time = order.transact_time
-            order.last_executed_quantity = order.orig_qty
-            order.cumulative_filled_quantity = order.orig_qty
-            order.last_executed_price = order.price
-            order.trade_id = self.trade_id = self.trade_id + 1
-            order.quote_asset_transacted = order.cummulative_quote_qty
-            order.last_quote_asset_transacted = order.cummulative_quote_qty
-            order.quote_order_quantity = order.cummulative_quote_qty
-            #
-            self.funds.on_order_created(buy=buy, amount=amount, price=price)
-            self.funds.on_order_filled(order.side, order.orig_qty, order.last_executed_price, self.fee_taker)
+        if buy:
+            self.orders_buy.at[order_id] = Decimal(price)
+            if self.save_ds:
+                self.grid_buy[lt] = self.orders_buy
         else:
-            if buy:
-                self.orders_buy.at[order_id] = Decimal(price)
-                if self.save_ds:
-                    self.grid_buy[lt] = self.orders_buy
-            else:
-                self.orders_sell.at[order_id] = Decimal(price)
-                if self.save_ds:
-                    self.grid_sell[lt] = self.orders_sell
+            self.orders_sell.at[order_id] = Decimal(price)
+            if self.save_ds:
+                self.grid_sell[lt] = self.orders_sell
             #
-            self.funds.on_order_created(buy=buy, amount=amount, price=price)
-
+        self.funds.on_order_created(buy=buy, amount=amount, price=price)
         self.orders[order_id] = order
 
+        if self.ticker_last and ((buy and Decimal(price) >= self.ticker_last) or
+                                 (not buy and Decimal(price) <= self.ticker_last)):
+            # Market event
+            self.market_ids.append(order_id)
+
         # print(f"create_order.order: {vars(order)}")
         return {'symbol': order.symbol,
                 'orderId': order.order_id,
@@ -263,19 +250,16 @@ def on_ticker_update(self, ticker: {}, ts: int) -> [dict]:
         self.ticker_last = Decimal(ticker['lastPrice'])
 
         orders_id = []
+        if self.market_ids:
+            orders_id.extend(self.market_ids)
+            self.market_ids.clear()
 
         _i = self.orders_buy[self.orders_buy >= self.ticker_last].index
-        try:
-            self.orders_buy = self.orders_buy.drop(_i.values)
-        except Exception as ex:
-            print(ex)
+        self.orders_buy = self.orders_buy.drop(_i.values)
         orders_id.extend(_i.values)
 
         _i = self.orders_sell[self.orders_sell <= self.ticker_last].index
-        try:
-            self.orders_sell = self.orders_sell.drop(_i.values)
-        except Exception as ex:
-            print(ex)
+        self.orders_sell = self.orders_sell.drop(_i.values)
         orders_id.extend(_i.values)
 
         if self.save_ds:

diff --git a/martin_binance/backtest/optimizer.py b/martin_binance/backtest/optimizer.py
@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2024 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "2.1.0rc37"
+__version__ = "2.1.0"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 
@@ -40,8 +40,8 @@ def objective(_trial):
         params = {
             'GRID_MAX_COUNT': _trial.suggest_int('GRID_MAX_COUNT', 3, 5),
             'PRICE_SHIFT': _trial.suggest_float('PRICE_SHIFT', 0, 0.05, step=0.01),
-            'PROFIT': _trial.suggest_float('PROFIT', 0.05, 0.15, step=0.05),
-            'PROFIT_MAX': _trial.suggest_float('PROFIT_MAX', 0.25, 1.0, step=0.05),
+            'PROFIT': _trial.suggest_float('PROFIT', 0.05, 0.2, step=0.05),
+            'PROFIT_MAX': _trial.suggest_float('PROFIT_MAX', 0.35, 1.0, step=0.05),
             'OVER_PRICE': _trial.suggest_float('OVER_PRICE', 0.1, 1, step=0.1),
             'ORDER_Q': _trial.suggest_int('ORDER_Q', 6, 12),
             'MARTIN': _trial.suggest_float('MARTIN', 5, 15, step=1),
@@ -70,13 +70,7 @@ async def run_optimize(*args):
 
 
 if __name__ == "__main__":
-    # study = optimize(sys.argv[1], sys.argv[2], int(sys.argv[3]), storage_name=sys.argv[4])
-    study = optimize(
-        'binance_BTCUSDT',
-        '/home/ubuntu/.MartinBinance/back_test/binance_BTCUSDT/cli_7_BTCUSDT.py',
-        1,
-        skip_log=False
-    )
+    study = optimize(sys.argv[1], sys.argv[2], int(sys.argv[3]), storage_name=sys.argv[4])
     new_value = study.best_value
     _value = study.get_trials()[0].value
     if new_value > _value:

diff --git a/martin_binance/cli_0_BTCUSDT.py.template b/martin_binance/cli_0_BTCUSDT.py.template
@@ -53,8 +53,8 @@ ex.PRICE_LIMIT_RULES = Decimal('0')  # +-% from last ticker price. Use on Bybit
 # Round pattern, set pattern 1.0123456789 or if not set used exchange settings
 ex.ROUND_BASE = str()
 ex.ROUND_QUOTE = str()
-ex.PROFIT = Decimal('0.1')  # 0.1 - 0.85
-ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit
+ex.PROFIT = Decimal('0.15')  # recommended FEE_MAKER*2<PROFIT<=0.85
+ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit, PROFIT<PROFIT_MAX<=1.0
 ex.OVER_PRICE = Decimal('0.6')  # Overlap price in one direction
 ex.ORDER_Q = 12  # Target grid orders quantity in moment
 ex.MARTIN = Decimal('10')  # 5-20, % increments volume of orders in the grid
@@ -90,6 +90,7 @@ ex.MODE = 'T'  # 'T' - Trade, 'TC' - Trade and Collect, 'S' - Simulate
 ex.SAVE_DS = False  # Save session result data (ticker, orders) for compare
 ex.SAVE_PERIOD = 24 * 60 * 60  # sec, timetable for save data portion
 ex.SELF_OPTIMIZATION = True  # Cyclic self-optimization of parameters, together with MODE == 'TC'
+ex.N_TRIALS = 500  # Number of optimization cycles for optuna study in self optimization mode
 ################################################################
 #                 DO NOT EDIT UNDER THIS LINE                ###
 ################################################################
@@ -115,7 +116,7 @@ def trade():
     # For autoload last state
     ex.LOAD_LAST_STATE = 1 if len(sys.argv) > 1 else 0
     #
-    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.log")
+    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}{'_S' if ex.MODE == 'S' else ''}.log")
     ex.LAST_STATE_FILE = Path(LAST_STATE_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.json")
     #
     _logger = logging.getLogger('logger')

diff --git a/martin_binance/cli_1_BTCUSDT.py.template b/martin_binance/cli_1_BTCUSDT.py.template
@@ -53,8 +53,8 @@ ex.PRICE_LIMIT_RULES = Decimal('0')  # +-% from last ticker price. Use on Bybit
 # Round pattern, set pattern 1.0123456789 or if not set used exchange settings
 ex.ROUND_BASE = str()
 ex.ROUND_QUOTE = str()
-ex.PROFIT = Decimal('0.1')  # 0.1 - 0.85
-ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit
+ex.PROFIT = Decimal('0.15')  # recommended FEE_MAKER*2<PROFIT<=0.85
+ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit, PROFIT<PROFIT_MAX<=1.0
 ex.OVER_PRICE = Decimal('0.6')  # Overlap price in one direction
 ex.ORDER_Q = 12  # Target grid orders quantity in moment
 ex.MARTIN = Decimal('10')  # 5-20, % increments volume of orders in the grid
@@ -90,6 +90,7 @@ ex.MODE = 'T'  # 'T' - Trade, 'TC' - Trade and Collect, 'S' - Simulate
 ex.SAVE_DS = False  # Save session result data (ticker, orders) for compare
 ex.SAVE_PERIOD = 24 * 60 * 60  # sec, timetable for save data portion
 ex.SELF_OPTIMIZATION = True  # Cyclic self-optimization of parameters, together with MODE == 'TC'
+ex.N_TRIALS = 500  # Number of optimization cycles for optuna study in self optimization mode
 ################################################################
 #                 DO NOT EDIT UNDER THIS LINE                ###
 ################################################################
@@ -115,7 +116,7 @@ def trade():
     # For autoload last state
     ex.LOAD_LAST_STATE = 1 if len(sys.argv) > 1 else 0
     #
-    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.log")
+    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}{'_S' if ex.MODE == 'S' else ''}.log")
     ex.LAST_STATE_FILE = Path(LAST_STATE_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.json")
     #
     _logger = logging.getLogger('logger')

diff --git a/martin_binance/cli_2_TESTBTCTESTUSDT.py.template b/martin_binance/cli_2_TESTBTCTESTUSDT.py.template
@@ -53,10 +53,10 @@ ex.PRICE_LIMIT_RULES = Decimal('0')  # +-% from last ticker price. Use on Bybit
 # Round pattern, set pattern 1.0123456789 or if not set used exchange settings
 ex.ROUND_BASE = str()
 ex.ROUND_QUOTE = str()
-ex.PROFIT = Decimal('0.1')  # 0.1 - 0.85
-ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit
-ex.OVER_PRICE = Decimal('0.9')  # Overlap price in one direction
-ex.ORDER_Q = 8  # Target grid orders quantity in moment
+ex.PROFIT = Decimal('0.15')  # recommended FEE_MAKER*2<PROFIT<=0.85
+ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit, PROFIT<PROFIT_MAX<=1.0
+ex.OVER_PRICE = Decimal('0.6')  # Overlap price in one direction
+ex.ORDER_Q = 12  # Target grid orders quantity in moment
 ex.MARTIN = Decimal('10')  # 5-20, % increments volume of orders in the grid
 ex.SHIFT_GRID_DELAY = 15  # sec delay for shift grid action
 # Other
@@ -90,6 +90,7 @@ ex.MODE = 'T'  # 'T' - Trade, 'TC' - Trade and Collect, 'S' - Simulate
 ex.SAVE_DS = False  # Save session result data (ticker, orders) for compare
 ex.SAVE_PERIOD = 24 * 60 * 60  # sec, timetable for save data portion
 ex.SELF_OPTIMIZATION = True  # Cyclic self-optimization of parameters, together with MODE == 'TC'
+ex.N_TRIALS = 500  # Number of optimization cycles for optuna study in self optimization mode
 ################################################################
 #                 DO NOT EDIT UNDER THIS LINE                ###
 ################################################################
@@ -115,7 +116,7 @@ def trade():
     # For autoload last state
     ex.LOAD_LAST_STATE = 1 if len(sys.argv) > 1 else 0
     #
-    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.log")
+    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}{'_S' if ex.MODE == 'S' else ''}.log")
     ex.LAST_STATE_FILE = Path(LAST_STATE_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.json")
     #
     _logger = logging.getLogger('logger')

diff --git a/martin_binance/cli_3_BTCUSDT.py.template b/martin_binance/cli_3_BTCUSDT.py.template
@@ -53,8 +53,8 @@ ex.PRICE_LIMIT_RULES = Decimal('3')  # +-% from last ticker price. Use on Bybit
 # Round pattern, set pattern 1.0123456789 or if not set used exchange settings
 ex.ROUND_BASE = str()
 ex.ROUND_QUOTE = str()
-ex.PROFIT = Decimal('0.1')  # 0.1 - 0.85
-ex.PROFIT_MAX = Decimal('0.5')  # If set it is maximum adapted cycle profit
+ex.PROFIT = Decimal('0.15')  # recommended FEE_MAKER*2<PROFIT<=0.85
+ex.PROFIT_MAX = Decimal('0.85')  # If set it is maximum adapted cycle profit, PROFIT<PROFIT_MAX<=1.0
 ex.OVER_PRICE = Decimal('0.6')  # Overlap price in one direction
 ex.ORDER_Q = 11  # Target grid orders quantity in moment
 ex.MARTIN = Decimal('10')  # 5-20, % increments volume of orders in the grid
@@ -90,6 +90,7 @@ ex.MODE = 'T'  # 'T' - Trade, 'TC' - Trade and Collect, 'S' - Simulate
 ex.SAVE_DS = False  # Save session result data (ticker, orders) for compare
 ex.SAVE_PERIOD = 24 * 60 * 60  # sec, timetable for save data portion
 ex.SELF_OPTIMIZATION = True  # Cyclic self-optimization of parameters, together with MODE == 'TC'
+ex.N_TRIALS = 500  # Number of optimization cycles for optuna study in self optimization mode
 ################################################################
 #                 DO NOT EDIT UNDER THIS LINE                ###
 ################################################################
@@ -115,7 +116,7 @@ def trade():
     # For autoload last state
     ex.LOAD_LAST_STATE = 1 if len(sys.argv) > 1 else 0
     #
-    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.log")
+    log_file = Path(LOG_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}{'_S' if ex.MODE == 'S' else ''}.log")
     ex.LAST_STATE_FILE = Path(LAST_STATE_PATH, f"{ex.ID_EXCHANGE}_{ex.SYMBOL}.json")
     #
     _logger = logging.getLogger('logger')

diff --git a/martin_binance/executor.py b/martin_binance/executor.py
@@ -4,7 +4,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "2.1.0rc40"
+__version__ = "2.1.0"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = 'https://github.com/DogsTailFarmer'
 ##################################################################
@@ -1684,8 +1684,7 @@ def place_profit_order(self, by_market=False, after_error=False) -> None:
                                      f" vlm: {amount}, price: {price}, profit: {profit}%")
                     self.tp_target = target
                     self.tp_order = (buy_side, amount, price, self.get_time())
-                    check = after_error or (len(self.orders_grid) + len(self.orders_hold)) > 2
-                    self.tp_wait_id = self.place_limit_order_check(buy_side, amount, price, check=check)
+                    self.tp_wait_id = self.place_limit_order_check(buy_side, amount, price, check=after_error)
         elif self.tp_order_id and self.tp_cancel:
             self.cancel_order_id = self.tp_order_id
             self.cancel_order_exp(self.tp_order_id)
@@ -2247,18 +2246,7 @@ def grid_handler(
             self.restore_orders_fire()
             if after_full_fill and self.orders_hold and self.order_q_placed and not self.grid_remove:
                 # PLace one hold grid order and remove it from hold list
-                _, _buy, _amount, _price = self.orders_hold.get_first()
-                check = (len(self.orders_grid) + len(self.orders_hold)) <= 2
-                waiting_order_id = self.place_limit_order_check(
-                    _buy,
-                    _amount,
-                    _price,
-                    check=check,
-                    price_limit_rules=True
-                )
-                if waiting_order_id:
-                    self.orders_init.append_order(waiting_order_id, _buy, _amount, _price)
-                    del self.orders_hold.orders_list[0]
+                self.place_grid_part()
             if self.tp_was_filled:
                 # Exist filled but non processing TP
                 self.after_filled_tp(one_else_grid=True)