2.1.0rc20

CHANGELOG.md

@@ -1,10 +1,14 @@
+## 2.1.0rc20 - 2024-01-27
+### Fix
+* An order that was `PARTIALLY FILLED` at the time of creation was counted as `FILLED`
+
 ### Update
 * margin_wrapper.buffered_funds(): some minor improvements
 * executor.get_free_assets(): for "free" mode the volume of the utilized deposit is taken into account when not TP
 * rollback saving/restore trades history, but used for it specified file in `/last_state/X_AAABBB.csv`
 * executor: remove parameters `SAVE_TRADE_HISTORY` and `SAVED_STATE`, already use for saving/restore trade events
 * comprehensive improvement of the internal accounting and recovery system
-* Up requirements for exchanges-wrapper==1.4.6
+* Up requirements for exchanges-wrapper==1.4.7
 
 ### Added for new features
 * Per 10 mins update trade rules for pair

martin_binance/__init__.py

@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "2.1.0rc19"
+__version__ = "2.1.0rc20"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 

martin_binance/backtest/OoTSP.py

@@ -18,6 +18,8 @@
 from martin_binance import BACKTEST_PATH
 from optimizer import optimize
 
+SKIP_LOG = True
+
 vis = optuna.visualization
 ii_params = []
 
@@ -55,7 +57,14 @@ def main():
         except StopIteration:
             raise UserWarning(f"Can't find cli_*.py in {Path(BACKTEST_PATH, study_name)}")
 
-        study = optimize(study_name, strategy, int(answers.get('n_trials', '0')), storage_name)
+        study = optimize(
+            study_name,
+            strategy,
+            int(answers.get('n_trials', '0')),
+            storage_name,
+            skip_log=SKIP_LOG,
+            show_progress_bar=SKIP_LOG
+        )
         print_study_result(study)
         print(f"Study instance saved to {storage_name} for later use")
     elif answers.get('mode') == 'Analise saved study session':

martin_binance/backtest/optimizer.py

@@ -6,7 +6,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2024 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "2.1.0rc16"
+__version__ = "2.1.0rc20"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = "https://github.com/DogsTailFarmer"
 
@@ -24,21 +24,20 @@
 OPTIMIZER = Path(__file__).absolute()
 OPTIMIZER.chmod(OPTIMIZER.stat().st_mode | stat.S_IEXEC)
 PARAMS_FLOAT = ['KBB']
-N_TRIALS = 1000
 logger = logging.getLogger('logger')
 
 
-def try_trade(mbs, **kwargs):
+def try_trade(mbs, skip_log, **kwargs):
     for key, value in kwargs.items():
         setattr(mbs.ex, key, value if isinstance(value, int) or key in PARAMS_FLOAT else Decimal(f"{value}"))
     mbs.ex.MODE = 'S'
     mbs.ex.SAVE_DS = False
-    mbs.ex.LOGGING = False
+    mbs.ex.LOGGING = False if skip_log else True
     mbs.trade()
     return float(mbs.session_result.get('profit', 0)) + float(mbs.session_result.get('free', 0))
 
 
-def optimize(study_name, strategy, n_trials, storage_name=None):
+def optimize(study_name, strategy, n_trials, storage_name=None, skip_log=True, show_progress_bar=False):
     def objective(_trial):
         params = {
             'GRID_MAX_COUNT': _trial.suggest_int('GRID_MAX_COUNT', 3, 5),
@@ -52,15 +51,15 @@ def objective(_trial):
             'KBB': _trial.suggest_float('KBB', 1, 5, step=0.5),
             'LINEAR_GRID_K': _trial.suggest_int('LINEAR_GRID_K', 0, 100, step=20),
         }
-        return try_trade(mbs, **params)
+        return try_trade(mbs, skip_log, **params)
 
     spec = iu.spec_from_file_location("strategy", strategy)
     mbs = iu.module_from_spec(spec)
     spec.loader.exec_module(mbs)
-    _study = optuna.create_study(study_name=study_name, storage=storage_name, direction="maximize")
     optuna.logging.set_verbosity(optuna.logging.WARNING)
+    _study = optuna.create_study(study_name=study_name, storage=storage_name, direction="maximize")
     try:
-        _study.optimize(objective, n_trials=n_trials)
+        _study.optimize(objective, n_trials=n_trials, gc_after_trial=True, show_progress_bar=show_progress_bar)
     except KeyboardInterrupt:
         pass
     return _study
@@ -79,12 +78,14 @@ async def run_optimize(*args):
 
 if __name__ == "__main__":
     try:
-        study = optimize(sys.argv[1], sys.argv[2], N_TRIALS)
+        study = optimize(sys.argv[1], sys.argv[2], int(sys.argv[3]))
+        # study = optimize("test", "/home/ubuntu/.MartinBinance/back_test/binance_ETHTUSD", 15)
     except KeyboardInterrupt:
         pass
     else:
         new_value = study.best_value
         _value = study.get_trials()[0].value
+        logger.info(f"Best trial: {study.best_trial}, new_value: {new_value}, _value: {_value}")
         if new_value > _value:
             res = study.best_params
             print(json.dumps(res.update({'new_value': new_value, '_value': _value})))

martin_binance/executor.py

@@ -4,7 +4,7 @@
 __author__ = "Jerry Fedorenko"
 __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM"
 __license__ = "MIT"
-__version__ = "2.1.0rc16"
+__version__ = "2.1.0rc20"
 __maintainer__ = "Jerry Fedorenko"
 __contact__ = 'https://github.com/DogsTailFarmer'
 ##################################################################
@@ -106,6 +106,7 @@
 SAVE_PERIOD = 1 * 60 * 60  # sec, timetable for save data portion, but memory limitation consider also matter
 LOGGING = True
 SELF_OPTIMIZATION = True  # Cyclic self-optimization of parameters, together with MODE == 'TC'
+N_TRIALS = 250  # Number of optimization cycles for optuna study
 # endregion
 
 
@@ -1715,7 +1716,7 @@ def set_trade_conditions(self,
             amount_first_grid = self.round_truncate(amount_first_grid, base=True, _rounding=ROUND_CEILING)
             if amount_first_grid >= depo_c:
                 raise UserWarning(f"Amount first grid order: {amount_first_grid} is greater than depo:"
-                                  f" {float(depo_c):f}. Increase depo amount.")
+                                  f" {float(depo_c):f}. Increase depo amount or PROFIT parameter.")
         else:
             amount_first_grid = amount_min
         if self.reverse:
@@ -1896,21 +1897,21 @@ def fee_for_grid(self,
             if FEE_BNB_IN_PAIR:
                 if self.cycle_buy:
                     amount_first -= self.round_fee(fee, amount_first, base=True)
-                    message = f"For grid order First - fee: {float(amount_first):f}"
+                    message = f"For grid order First - fee: {any2str(amount_first)}"
                 else:
                     amount_first += self.round_fee(fee, amount_first, base=True)
-                    message = f"For grid order First + fee: {float(amount_first):f}"
+                    message = f"For grid order First + fee: {any2str(amount_first)}"
             else:
                 if self.cycle_buy:
                     if FEE_SECOND:
                         amount_second += self.round_fee(fee, amount_second, base=False)
-                        message = f"For grid order Second + fee: {float(amount_second):f}"
+                        message = f"For grid order Second + fee: {any2str(amount_second)}"
                     else:
                         amount_first -= self.round_fee(fee, amount_first, base=True)
-                        message = f"For grid order First - fee: {float(amount_first):f}"
+                        message = f"For grid order First - fee: {any2str(amount_first)}"
                 else:
                     amount_second -= self.round_fee(fee, amount_second, base=False)
-                    message = f"For grid order Second - fee: {float(amount_second):f}"
+                    message = f"For grid order Second - fee: {any2str(amount_second)}"
         if print_info and message:
             self.message_log(message, log_level=LogLevel.DEBUG)
         return self.round_truncate(amount_first, fee=True), self.round_truncate(amount_second, fee=True)
@@ -2609,23 +2610,29 @@ def on_order_update(self, update: OrderUpdate) -> None:
         self.message_log(f"Order {update.original_order.id}: {update.status}", color=Style.B_WHITE)
         result_trades = update.resulting_trades
         amount_first = amount_second = O_DEC
+        by_market = False
         if update.status == OrderUpdate.PARTIALLY_FILLED:
             # Get last trade row
             if result_trades:
                 i = result_trades[-1]
                 amount_first = i.amount
                 amount_second = i.amount * i.price
-                self.message_log(f"trade id={i.id}, first: {i.amount}, price: {i.price}", log_level=LogLevel.DEBUG)
+                by_market = not bool(i.is_maker)
+                self.message_log(f"trade id={i.id}, first: {any2str(i.amount)}, price: {any2str(i.price)},"
+                                 f" by_market: {by_market}", log_level=LogLevel.DEBUG)
             else:
                 self.message_log(f"No records for {update.original_order.id}", log_level=LogLevel.WARNING)
         else:
             for i in result_trades:
                 # Calculate sum trade amount for both currency
                 amount_first += i.amount
                 amount_second += i.amount * i.price
-                self.message_log(f"trade id={i.id}, first: {i.amount}, price: {i.price}", log_level=LogLevel.DEBUG)
+                by_market = by_market or not bool(i.is_maker)
+                self.message_log(f"trade id={i.id}, first: {any2str(i.amount)}, price: {any2str(i.price)},"
+                                 f" by_market: {by_market}", log_level=LogLevel.DEBUG)
         self.avg_rate = amount_second / amount_first
-        self.message_log(f"Executed amount: First: {amount_first}, Second: {amount_second}, price: {self.avg_rate}")
+        self.message_log(f"Executed amount: First: {any2str(amount_first)}, Second: {any2str(amount_second)},"
+                         f" price: {any2str(self.avg_rate)}")
         if update.status in (OrderUpdate.FILLED, OrderUpdate.ADAPTED_AND_FILLED):
             if not GRID_ONLY:
                 self.shift_grid_threshold = None
@@ -2638,6 +2645,7 @@ def on_order_update(self, update: OrderUpdate) -> None:
                         self.restore_orders = False
                 self.grid_handler(_amount_first=amount_first,
                                   _amount_second=amount_second,
+                                  by_market=by_market,
                                   after_full_fill=True,
                                   order_id=update.original_order.id)
             elif self.tp_order_id == update.original_order.id:
@@ -2651,15 +2659,15 @@ def on_order_update(self, update: OrderUpdate) -> None:
                     self.update_sum_amount(- self.tp_part_amount_first, - self.tp_part_amount_second)
                     self.tp_part_amount_first = self.tp_part_amount_second = O_DEC
                     self.tp_part_free = False
-                self.tp_was_filled = (amount_first, amount_second, False)
+                self.tp_was_filled = (amount_first, amount_second, by_market)
                 # print(f"on_order_update.was_filled_tp: {self.tp_was_filled}")
                 if self.tp_hold:
                     # After place but before execute TP was filled some grid
                     self.tp_hold = False
                     self.after_filled_tp(one_else_grid=True)
                 elif self.tp_cancel_from_grid_handler:
                     self.tp_cancel_from_grid_handler = False
-                    self.grid_handler()
+                    self.grid_handler(by_market=by_market)
                 else:
                     self.restore_orders = False
                     self.grid_remove = None
@@ -2669,7 +2677,7 @@ def on_order_update(self, update: OrderUpdate) -> None:
         elif update.status == OrderUpdate.PARTIALLY_FILLED:
             if self.tp_order_id == update.original_order.id:
                 self.message_log("Take profit partially filled", color=Style.B_WHITE)
-                amount_first_fee, amount_second_fee = self.fee_for_tp(amount_first, amount_second)
+                amount_first_fee, amount_second_fee = self.fee_for_tp(amount_first, amount_second, by_market=by_market)
                 # Calculate profit for filled part TP
                 _profit_first = _profit_second = O_DEC
                 if self.cycle_buy:
@@ -2732,7 +2740,7 @@ def on_order_update(self, update: OrderUpdate) -> None:
                     # Get min trade amount
                     if self.check_min_amount():
                         self.shift_grid_threshold = None
-                        self.grid_handler(after_full_fill=False)
+                        self.grid_handler(by_market=by_market, after_full_fill=False)
                     else:
                         self.last_shift_time = self.get_time() + 2 * SHIFT_GRID_DELAY
                         self.message_log("Partially trade too small, ignore", color=Style.B_WHITE)
@@ -2747,93 +2755,44 @@ def cancel_reverse_hold(self):
 
     def on_place_order_success(self, place_order_id: int, order: Order) -> None:
         # print(f"on_place_order_success.place_order_id: {place_order_id}")
-        if order.amount > order.received_amount > 0:
-            self.message_log(f"Order {place_order_id} was partially filled", color=Style.B_WHITE)
-            self.shift_grid_threshold = None
-        if order.remaining_amount == 0:
-            self.shift_grid_threshold = None
-            # Get actual parameter of last trade order
-            market_order = self.get_buffered_completed_trades()
-            amount_first = amount_second = O_DEC
-            self.message_log(f"Order {place_order_id} executed by market", color=Style.B_WHITE)
-            for o in market_order:
-                if o.order_id == order.id:
-                    amount_first += o.amount
-                    amount_second += o.amount * o.price
-            if not amount_first:
-                amount_first += order.amount
-                amount_second += order.amount * order.price
-            self.avg_rate = amount_second / amount_first
-            self.message_log(f"For {order.id} first: {float(amount_first):f},"
-                             f" second: {float(amount_second):f}, price: {float(self.avg_rate):f}")
-            if self.orders_init.exist(place_order_id):
-                self.ts_grid_update = self.get_time()
-                self.message_log(f"Grid order {order.id} execute by market")
-                self.orders_init.remove(place_order_id)
-                self.message_log(f"Waiting order count is: {len(self.orders_init)}, hold: {len(self.orders_hold)}")
-                # Place take profit order
-                self.grid_handler(
-                    _amount_first=amount_first,
-                    _amount_second=amount_second,
-                    by_market=True,
-                    after_full_fill=True
-                )
-            elif place_order_id == self.tp_wait_id:
-                # Take profit order execute by market, restart
-                self.tp_wait_id = None
-                if self.reverse_hold:
-                    self.cancel_reverse_hold()
-                self.message_log(f"Take profit order {order.id} execute by market")
-                self.tp_was_filled = (amount_first, amount_second, True)
-                if self.tp_hold or self.tp_cancel_from_grid_handler:
-                    self.tp_cancel_from_grid_handler = False
-                    self.tp_hold = False
-                    # After place but before accept TP was filled some grid
-                    self.after_filled_tp(one_else_grid=True)
-                else:
-                    self.restore_orders = False
-                    self.grid_remove = None
-                    self.cancel_grid(cancel_all=True)
-            else:
-                self.message_log(f"Did not have waiting order id for {place_order_id}", LogLevel.ERROR,
-                                 color=Style.B_RED)
-        else:
-            if self.orders_init.exist(place_order_id):
-                self.ts_grid_update = self.get_time()
-                self.orders_grid.append_order(order.id, order.buy, order.amount, order.price)
-                self.orders_grid.sort(self.cycle_buy)
-                self.orders_init.remove(place_order_id)
-                if not self.orders_init:
-                    self.last_shift_time = self.get_time()
-                    if GRID_ONLY and self.orders_hold:
-                        # Place next part of grid orders
-                        self.place_grid_part()
-                    else:
-                        if self.grid_place_flag or not self.orders_hold:
-                            if self.orders_hold:
-                                self.message_log(f"Part of grid orders are placed. Left: {len(self.orders_hold)}",
-                                                 color=Style.B_WHITE)
-                            else:
-                                self.order_q_placed = True
-                                self.message_log('All grid orders place successfully', color=Style.B_WHITE)
-                        elif not self.order_q_placed and not self.shift_grid_threshold:
-                            self.place_grid_part()
-                        if self.cancel_grid_hold:
-                            self.message_log('Continue remove grid orders', color=Style.B_WHITE)
-                            self.cancel_grid_hold = False
-                            self.cancel_grid()
-            elif place_order_id == self.tp_wait_id:
-                self.tp_wait_id = None
-                self.tp_order_id = order.id
-                if self.tp_hold or self.tp_cancel or self.tp_cancel_from_grid_handler:
-                    self.cancel_order_id = self.tp_order_id
-                    self.cancel_order_exp(self.tp_order_id)
-                else:
+        if self.orders_init.exist(place_order_id):
+            if order.remaining_amount == 0 or order.amount > order.received_amount > 0:
+                self.shift_grid_threshold = None
+            self.ts_grid_update = self.get_time()
+            self.orders_grid.append_order(order.id, order.buy, order.amount, order.price)
+            self.orders_grid.sort(self.cycle_buy)
+            self.orders_init.remove(place_order_id)
+            if not self.orders_init:
+                self.last_shift_time = self.get_time()
+                if GRID_ONLY and self.orders_hold:
                     # Place next part of grid orders
-                    if self.orders_hold and not self.order_q_placed and not self.orders_init:
+                    self.place_grid_part()
+                else:
+                    if self.grid_place_flag or not self.orders_hold:
+                        if self.orders_hold:
+                            self.message_log(f"Part of grid orders are placed. Left: {len(self.orders_hold)}",
+                                             color=Style.B_WHITE)
+                        else:
+                            self.order_q_placed = True
+                            self.message_log('All grid orders place successfully', color=Style.B_WHITE)
+                    elif not self.order_q_placed and not self.shift_grid_threshold:
                         self.place_grid_part()
+                    if self.cancel_grid_hold:
+                        self.message_log('Continue remove grid orders', color=Style.B_WHITE)
+                        self.cancel_grid_hold = False
+                        self.cancel_grid()
+        elif place_order_id == self.tp_wait_id:
+            self.tp_wait_id = None
+            self.tp_order_id = order.id
+            if self.tp_hold or self.tp_cancel or self.tp_cancel_from_grid_handler:
+                self.cancel_order_id = self.tp_order_id
+                self.cancel_order_exp(self.tp_order_id)
             else:
-                self.message_log(f"Did not have waiting order id for {place_order_id}", LogLevel.ERROR)
+                # Place next part of grid orders
+                if self.orders_hold and not self.order_q_placed and not self.orders_init:
+                    self.place_grid_part()
+        else:
+            self.message_log(f"Did not have waiting order id for {place_order_id}", LogLevel.ERROR)
 
     def on_place_order_error_string(self, place_order_id: int, error: str) -> None:
         # Check all orders on exchange if exists required