From 10aceac4cb7f0f574b09ff1dee1bf3011b7a1d1c Mon Sep 17 00:00:00 2001 From: DogsTailFarmer Date: Thu, 21 Mar 2024 22:37:00 +0300 Subject: [PATCH] 3.0.1rc3 --- CHANGELOG.md | 7 + martin_binance/__init__.py | 2 +- martin_binance/backtest/OoTSP.py | 110 +++++----- martin_binance/backtest/exchange_simulator.py | 203 ++++++++++-------- martin_binance/backtest/optimizer.py | 14 +- martin_binance/client.py | 6 +- martin_binance/executor.py | 28 +-- martin_binance/lib.py | 12 +- martin_binance/strategy_base.py | 63 +++--- pyproject.toml | 2 +- requirements.txt | 2 +- 11 files changed, 245 insertions(+), 204 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77efe4c..71964a9 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 3.0.1rc3 - 2024-03-21 +### Added for new features +:rocket: `Backtesting`: handling of partially filling events + +### Update +* Up requirements for exchanges-wrapper==2.1.3 + ## 3.0.1rc1 - 2024-03-19 ### Fix * Cyclic Backtesting workflow diff --git a/martin_binance/__init__.py b/martin_binance/__init__.py index b2b36ca..e5d11a7 100755 --- a/martin_binance/__init__.py +++ b/martin_binance/__init__.py @@ -6,7 +6,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.1rc1" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" diff --git a/martin_binance/backtest/OoTSP.py b/martin_binance/backtest/OoTSP.py index 36d13fc..6fd75ee 100644 --- a/martin_binance/backtest/OoTSP.py +++ b/martin_binance/backtest/OoTSP.py @@ -6,7 +6,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.1rc1" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" @@ -24,56 +24,56 @@ def main(): - questions = [ - inquirer.List( - "path", - message="Select from saved: exchange_PAIR with the strategy you want to optimize", - choices=[f.name for f in BACKTEST_PATH.iterdir() if f.is_dir() and f.name.count('_') == 1], - ), - inquirer.List( - "mode", - message="New study session or analise from saved one", - choices=["New", "Analise saved study session"], - ), - inquirer.Text( - "n_trials", - message="Enter number of cycles, from 10 to 1000", - ignore=lambda x: x["mode"] == "Analise saved study session", - default='150', - validate=lambda _, c: 10 <= int(c) <= 1000, - ), - ] - - answers = inquirer.prompt(questions, theme=GreenPassion()) - - study_name = answers.get('path') # Unique identifier of the study - storage_name = f"sqlite:///{Path(BACKTEST_PATH, study_name, 'study.db')}" - - if answers.get('mode') == 'New': - Path(BACKTEST_PATH, study_name, 'study.db').unlink(missing_ok=True) - try: - strategy = next(Path(BACKTEST_PATH, study_name).glob("cli_*.py")) - except StopIteration: - raise UserWarning(f"Can't find cli_*.py in {Path(BACKTEST_PATH, study_name)}") - - study = optimize( - study_name, - strategy, - int(answers.get('n_trials', '0')), - storage_name, - skip_log=SKIP_LOG, - show_progress_bar=SKIP_LOG - ) - print_study_result(study) - print(f"Study instance saved to {storage_name} for later use") - elif answers.get('mode') == 'Analise saved study session': - # noinspection PyArgumentList - study = optuna.load_study(study_name=study_name, storage=storage_name) - - print(f"Best value: {study.best_value}") - print(f"Original value: {study.get_trials()[0].value}") - - while 1: + while 1: + questions = [ + inquirer.List( + "path", + message="Select from saved: exchange_PAIR with the strategy you want to optimize", + choices=[f.name for f in BACKTEST_PATH.iterdir() if f.is_dir() and f.name.count('_') == 1], + ), + inquirer.List( + "mode", + message="New study session or analise from saved one", + choices=["New", "Analise saved study session", "Exit"], + ), + inquirer.Text( + "n_trials", + message="Enter number of cycles, from 2 to 1000", + ignore=lambda x: x["mode"] in ("Analise saved study session", "Exit"), + default='150', + validate=lambda _, c: 15 <= int(c) <= 1000, + ), + ] + + answers = inquirer.prompt(questions, theme=GreenPassion()) + + study_name = answers.get('path') # Unique identifier of the study + storage_name = f"sqlite:///{Path(BACKTEST_PATH, study_name, 'study.db')}" + + if answers.get('mode') == 'New': + Path(BACKTEST_PATH, study_name, 'study.db').unlink(missing_ok=True) + try: + strategy = next(Path(BACKTEST_PATH, study_name).glob("cli_*.py")) + except StopIteration: + raise UserWarning(f"Can't find cli_*.py in {Path(BACKTEST_PATH, study_name)}") + + study = optimize( + study_name, + strategy, + int(answers.get('n_trials', '0')), + storage_name, + skip_log=SKIP_LOG, + show_progress_bar=SKIP_LOG + ) + print_study_result(study) + print(f"Study instance saved to {storage_name} for later use") + elif answers.get('mode') == 'Analise saved study session': + # noinspection PyArgumentList + study = optuna.load_study(study_name=study_name, storage=storage_name) + + print(f"Best value: {study.best_value}") + print(f"Original value: {study.get_trials()[0].value}") + questions = [ inquirer.List( "mode", @@ -106,12 +106,14 @@ def main(): print("Can't find GUI, you can copy study instance to another environment for analyze it") elif answers.get('mode') == 'Get parameters for specific trial': trial = study.get_trials()[int(answers.get('n_trial', '0'))] - print(trial.number) + print(f"number: {trial.number}") print(trial.state) - print(trial.value) - print(trial.params) + print(f"value: {trial.value}") + print(f"params: {trial.params}") else: break + else: + break def print_study_result(study): diff --git a/martin_binance/backtest/exchange_simulator.py b/martin_binance/backtest/exchange_simulator.py index 9b927ea..160ede6 100644 --- a/martin_binance/backtest/exchange_simulator.py +++ b/martin_binance/backtest/exchange_simulator.py @@ -6,12 +6,10 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "2.1.4" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" -# TODO Add part filling event - from decimal import Decimal import pandas as pd @@ -94,10 +92,10 @@ def __init__(self, symbol: str, order_id: int, client_order_id: str, buy: bool, self.order_list_id = -1 self.client_order_id = client_order_id self.transact_time = lt # local time - self.price = price - self.orig_qty = amount - self.executed_qty = '0.00000000' - self.cummulative_quote_qty = '0.00000000' + self.price = Decimal(price) + self.orig_qty = Decimal(amount) + self.executed_qty = Decimal('0') + self.cummulative_quote_qty = Decimal('0') self.status = 'NEW' self.time_in_force = 'GTC' self.type = 'LIMIT' @@ -106,14 +104,14 @@ def __init__(self, symbol: str, order_id: int, client_order_id: str, buy: bool, self.self_trade_prevention_mode = 'NONE' # self.event_time: int - self.last_executed_quantity: str - self.cumulative_filled_quantity: str - self.last_executed_price: str + self.last_executed_quantity = Decimal('0') + self.cumulative_filled_quantity = Decimal('0') + self.last_executed_price = Decimal('0') self.trade_id: int self.order_creation_time = lt - self.quote_asset_transacted: str - self.last_quote_asset_transacted: str - self.quote_order_quantity: str + self.quote_asset_transacted = Decimal('0') + self.last_quote_asset_transacted = Decimal('0') + self.quote_order_quantity = self.orig_qty * self.price class Account: @@ -159,13 +157,15 @@ def create_order( order_id=None) -> {}: order_id = order_id or ((max(self.orders.keys()) + 1) if self.orders else 1) - order = Order(symbol=symbol, - order_id=order_id, - client_order_id=client_order_id, - buy=buy, - amount=amount, - price=price, - lt=lt) + order = Order( + symbol=symbol, + order_id=order_id, + client_order_id=client_order_id, + buy=buy, + amount=amount, + price=price, + lt=lt + ) if buy: self.orders_buy.at[order_id] = Decimal(price) @@ -205,6 +205,7 @@ def cancel_order(self, order_id: int, ts: int): order = self.orders.get(order_id) if order is None: raise UserWarning(f"Error on Cancel order, can't find {order_id} anymore") + order.status = 'CANCELED' try: if order.side == 'BUY': @@ -219,16 +220,16 @@ def cancel_order(self, order_id: int, ts: int): raise UserWarning(f"Order {order_id} not active: {ex}") from ex else: self.orders[order_id] = order - self.funds.on_order_canceled(order.side, Decimal(order.orig_qty), Decimal(order.price)) + self.funds.on_order_canceled(order.side, order.orig_qty, order.price) return {'symbol': order.symbol, 'origClientOrderId': order.client_order_id, 'orderId': order.order_id, 'orderListId': order.order_list_id, 'clientOrderId': 'qwert', - 'price': order.price, - 'origQty': order.orig_qty, - 'executedQty': order.executed_qty, - 'cummulativeQuoteQty': order.cummulative_quote_qty, + 'price': str(order.price), + 'origQty': str(order.orig_qty), + 'executedQty': str(order.executed_qty), + 'cummulativeQuoteQty': str(order.cummulative_quote_qty), 'status': order.status, 'timeInForce': order.time_in_force, 'type': order.type, @@ -236,24 +237,24 @@ def cancel_order(self, order_id: int, ts: int): 'selfTradePreventionMode': order.self_trade_prevention_mode} def on_ticker_update(self, ticker: {}, ts: int) -> [dict]: - # print(f"on_ticker_update.ticker: {ts}: {ticker['lastPrice']}") + # print(f"on_ticker_update.ticker: {ts}: {ticker}") # print(f"BUY: {self.orders_buy}") # print(f"SELL: {self.orders_sell}") + filled_buy_id = [] + filled_sell_id = [] + orders_id = [] + orders_filled = [] + self.ticker_last = Decimal(ticker['lastPrice']) + qty = Decimal(ticker['Qty']) + part = bool(qty) - orders_id = [] if self.market_ids: orders_id.extend(self.market_ids) - self.market_ids.clear() - _i = self.orders_buy[self.orders_buy >= self.ticker_last].index - self.orders_buy = self.orders_buy.drop(_i.values) - orders_id.extend(_i.values) - - _i = self.orders_sell[self.orders_sell <= self.ticker_last].index - self.orders_sell = self.orders_sell.drop(_i.values) - orders_id.extend(_i.values) + orders_id.extend(self.orders_buy[self.orders_buy >= self.ticker_last].index.values) + orders_id.extend(self.orders_sell[self.orders_sell <= self.ticker_last].index.values) if self.save_ds: # Save data for analytics @@ -263,65 +264,87 @@ def on_ticker_update(self, ticker: {}, ts: int) -> [dict]: if self.orders_buy.values.size: self.grid_buy[ts] = self.orders_buy # - orders_filled = [] for order_id in orders_id: + if part and not qty: + break + order = self.orders.get(order_id) - if order and order.status == 'NEW': - order.transact_time = int(ticker['closeTime']) - order.executed_qty = order.orig_qty - order.cummulative_quote_qty = str(Decimal(order.orig_qty) * Decimal(order.price)) + + order.transact_time = int(ticker['closeTime']) + order.event_time = order.transact_time + order.trade_id = self.trade_id = self.trade_id + 1 + + order.last_executed_price = self.ticker_last + + delta = order.orig_qty - order.executed_qty + order.last_executed_quantity = last_executed_qty = min(delta, qty) if part else delta + order.executed_qty += last_executed_qty + order.last_quote_asset_transacted = order.last_executed_price * last_executed_qty + order.quote_asset_transacted += order.last_quote_asset_transacted + + if part: + qty -= last_executed_qty + + order.cumulative_filled_quantity = order.executed_qty + order.cummulative_quote_qty = order.quote_asset_transacted + + if order.executed_qty >= order.orig_qty: order.status = 'FILLED' - order.event_time = order.transact_time - order.last_executed_quantity = order.orig_qty - order.cumulative_filled_quantity = order.orig_qty - order.last_executed_price = order.price - order.trade_id = self.trade_id = self.trade_id + 1 - order.quote_asset_transacted = order.cummulative_quote_qty - order.last_quote_asset_transacted = order.cummulative_quote_qty - order.quote_order_quantity = order.cummulative_quote_qty - # - self.orders[order_id] = order - # - res = {'event_time': order.event_time, - 'symbol': order.symbol, - 'client_order_id': order.client_order_id, - 'side': order.side, - 'order_type': order.type, - 'time_in_force': order.time_in_force, - 'order_quantity': order.orig_qty, - 'order_price': order.price, - 'stop_price': '0.00000000', - 'iceberg_quantity': '0.00000000', - 'order_list_id': -1, - 'original_client_id': order.client_order_id, - 'execution_type': 'TRADE', - 'order_status': order.status, - 'order_reject_reason': 'NONE', - 'order_id': order_id, - 'last_executed_quantity': order.last_executed_quantity, - 'cumulative_filled_quantity': order.cumulative_filled_quantity, - 'last_executed_price': order.last_executed_price, - 'commission_amount': '0.00000000', - 'commission_asset': '', - 'transaction_time': order.transact_time, - 'trade_id': order.trade_id, - 'ignore_a': 12345678, - 'in_order_book': False, - 'is_maker_side': True, - 'ignore_b': True, - 'order_creation_time': order.order_creation_time, - 'quote_asset_transacted': order.quote_asset_transacted, - 'last_quote_asset_transacted': order.last_quote_asset_transacted, - 'quote_order_quantity': order.quote_order_quantity} - # - orders_filled.append(res) - self.funds.on_order_filled( - order.side, - Decimal(order.orig_qty), - Decimal(order.last_executed_price), - self.fee_maker - ) + if order.side == 'BUY': + filled_buy_id.append(order_id) + else: + filled_sell_id.append(order_id) + elif 0 < order.executed_qty < order.orig_qty: + order.status = 'PARTIALLY_FILLED' + # + self.orders[order_id] = order # + res = { + 'event_time': order.event_time, + 'symbol': order.symbol, + 'client_order_id': order.client_order_id, + 'side': order.side, + 'order_type': order.type, + 'time_in_force': order.time_in_force, + 'order_quantity': str(order.orig_qty), + 'order_price': str(order.price), + 'stop_price': '0', + 'iceberg_quantity': '0', + 'order_list_id': -1, + 'original_client_id': order.client_order_id, + 'execution_type': 'TRADE', + 'order_status': order.status, + 'order_reject_reason': 'NONE', + 'order_id': order_id, + 'last_executed_quantity': str(order.last_executed_quantity), + 'cumulative_filled_quantity': str(order.cumulative_filled_quantity), + 'last_executed_price': str(order.last_executed_price), + 'commission_amount': '0', + 'commission_asset': '', + 'transaction_time': order.transact_time, + 'trade_id': order.trade_id, + 'ignore_a': 12345678, + 'in_order_book': False, + 'is_maker_side': False if order_id in self.market_ids else True, + 'ignore_b': True, + 'order_creation_time': order.order_creation_time, + 'quote_asset_transacted': str(order.quote_asset_transacted), + 'last_quote_asset_transacted': str(order.last_quote_asset_transacted), + 'quote_order_quantity': str(order.quote_order_quantity) + } + # + orders_filled.append(res) + self.funds.on_order_filled( + order.side, + order.last_executed_quantity, + order.last_executed_price, + self.fee_taker if order_id in self.market_ids else self.fee_maker + ) + # + self.orders_buy = self.orders_buy.drop(filled_buy_id) + self.orders_sell = self.orders_sell.drop(filled_sell_id) + self.market_ids.clear() + return orders_filled def restore_state(self, symbol: str, lt: int, orders: [], sum_amount: ()): diff --git a/martin_binance/backtest/optimizer.py b/martin_binance/backtest/optimizer.py index 8044d57..c3dd98b 100755 --- a/martin_binance/backtest/optimizer.py +++ b/martin_binance/backtest/optimizer.py @@ -6,7 +6,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2024 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.1rc1" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" @@ -30,6 +30,7 @@ STRATEGY = None +# noinspection PyUnusedLocal def notify_exception(*args): pass # Supress message from sys.excepthook @@ -49,7 +50,7 @@ def try_trade(mbs, skip_log, **kwargs): return float(mbs.ex.SESSION_RESULT.get('profit', 0)) + float(mbs.ex.SESSION_RESULT.get('free', 0)) -def optimize(study_name, cli, n_trials, storage_name=None, prm_best=None, skip_log=True, show_progress_bar=False): +def optimize(study_name, cli, n_trials, storage_name=None, _prm_best=None, skip_log=True, show_progress_bar=False): sys.excepthook = notify_exception optuna.logging.set_verbosity(optuna.logging.WARNING) @@ -71,11 +72,12 @@ def objective(_trial): spec = iu.spec_from_file_location("strategy", cli) mbs = iu.module_from_spec(spec) spec.loader.exec_module(mbs) + # noinspection PyArgumentList _study = optuna.create_study(study_name=study_name, storage=storage_name, direction="maximize") - if prm_best: - logger.info(f"Previous best params: {prm_best}") - _study.enqueue_trial(prm_best) + if _prm_best: + logger.info(f"Previous best params: {_prm_best}") + _study.enqueue_trial(_prm_best) _study.optimize(objective, n_trials=n_trials, gc_after_trial=True, show_progress_bar=show_progress_bar) return _study @@ -105,7 +107,7 @@ async def run_optimize(*args): sys.argv[2], int(sys.argv[3]), storage_name=sys.argv[4], - prm_best=prm_best + _prm_best=prm_best ) except KeyboardInterrupt: pass # ignore diff --git a/martin_binance/client.py b/martin_binance/client.py index b4890a4..cf65494 100644 --- a/martin_binance/client.py +++ b/martin_binance/client.py @@ -4,7 +4,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.0rc22" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" @@ -12,9 +12,9 @@ import random import logging +# noinspection PyPackageRequirements import grpclib.exceptions import shortuuid -import traceback from exchanges_wrapper import martin as mr, Channel, Status, GRPCError @@ -92,7 +92,7 @@ async def send_request(self, _request, _request_type, **kwargs): pass # Task cancellation should not be logged as an error except grpclib.exceptions.StreamTerminatedError: raise UserWarning("Have not connection to gRPC server") - except ConnectionRefusedError as ex: + except ConnectionRefusedError: raise UserWarning("Connection to gRPC server broken") except GRPCError as ex: status_code = ex.status diff --git a/martin_binance/executor.py b/martin_binance/executor.py index cee6eb6..8f10c95 100755 --- a/martin_binance/executor.py +++ b/martin_binance/executor.py @@ -4,7 +4,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.0rc19" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = 'https://github.com/DogsTailFarmer' ################################################################## @@ -271,7 +271,7 @@ def save_strategy_state(self, return_only=False) -> Dict[str, str]: last_price = self.get_buffered_ticker().last_price ticker_update = int(self.get_time()) - self.last_ticker_update if self.cycle_time: - ct = str(datetime.now(timezone.utc) - self.cycle_time).rsplit('.')[0] + ct = str(datetime.now(timezone.utc).replace(tzinfo=None) - self.cycle_time).rsplit('.')[0] else: self.message_log("save_strategy_state: cycle_time is None!", log_level=logging.DEBUG) ct = str(datetime.now(timezone.utc)).rsplit('.')[0] @@ -579,7 +579,7 @@ def start(self, profit_f: Decimal = O_DEC, profit_s: Decimal = O_DEC) -> None: else: df = self.deposit_first - self.profit_first ds = O_DEC - ct = datetime.now(timezone.utc) - self.cycle_time + ct = datetime.now(timezone.utc).replace(tzinfo=None) - self.cycle_time ct = ct.total_seconds() # noinspection PyUnboundLocalVariable data_to_db = { @@ -641,7 +641,7 @@ def start(self, profit_f: Decimal = O_DEC, profit_s: Decimal = O_DEC) -> None: f"Second: {self.sum_profit_second}\n" f"Summary: {self.sum_profit_first * self.avg_rate + self.sum_profit_second:f}\n") if self.first_run or MODE in ('T', 'TC'): - self.cycle_time = datetime.now(timezone.utc) + self.cycle_time = datetime.now(timezone.utc).replace(tzinfo=None) # memory = psutil.virtual_memory() swap = psutil.swap_memory() @@ -688,6 +688,7 @@ def start(self, profit_f: Decimal = O_DEC, profit_s: Decimal = O_DEC) -> None: color=Style.B_WHITE) self.debug_output() if MODE in ('TC', 'S') and self.start_collect is None: + self.start_collect = True self.first_run = False self.place_grid(self.cycle_buy, amount, self.reverse_target_amount) @@ -1167,7 +1168,7 @@ def place_grid(self, self.message_log(f"Hold grid for {'Buy' if buy_side else 'Sell'} cycle with {depo} {currency} depo." f" Available funds is {fund} {currency}", tlg=False) if self.tp_hold_additional: - self.message_log("Replace take profit order after place additional grid orders", tlg=True) + self.message_log("Replace take profit order after place additional grid orders") self.tp_hold = False self.tp_hold_additional = False self.place_profit_order() @@ -1625,7 +1626,7 @@ def after_filled_tp(self, one_else_grid: bool = False): self.message_log(f"Cycle profit first {self.profit_first} + {profit_reverse}") transfer_sum_amount_first = transfer_sum_amount_second = O_DEC if one_else_grid: - self.message_log("Some grid orders was execute after TP was filled", tlg=True) + self.message_log("Some grid orders was execute after TP was filled") self.tp_was_filled = () if self.convert_tp(amount_first_fee - profit_first, amount_second_fee - profit_second): return @@ -1696,7 +1697,7 @@ def reverse_after_grid_ending(self): self.reverse = False self.restart = True # Calculate profit and time for Reverse cycle - self.cycle_time = self.cycle_time_reverse or datetime.now(timezone.utc) + self.cycle_time = self.cycle_time_reverse or datetime.now(timezone.utc).replace(tzinfo=None) if self.cycle_buy: self.profit_first += self.round_truncate(self.sum_amount_first - self.reverse_init_amount + self.tp_part_amount_first, base=True) @@ -1733,7 +1734,7 @@ def reverse_after_grid_ending(self): trend_up = adx.get('adx') > ADX_THRESHOLD and adx.get('+DI') > adx.get('-DI') trend_down = adx.get('adx') > ADX_THRESHOLD and adx.get('-DI') > adx.get('+DI') # print('adx: {}, +DI: {}, -DI: {}'.format(adx.get('adx'), adx.get('+DI'), adx.get('-DI'))) - self.cycle_time_reverse = self.cycle_time or datetime.now(timezone.utc) + self.cycle_time_reverse = self.cycle_time or datetime.now(timezone.utc).replace(tzinfo=None) self.start_reverse_time = self.get_time() # Calculate target return amount tp = self.calc_profit_order(not self.cycle_buy) @@ -1877,10 +1878,10 @@ def grid_handler( self.tp_part_amount_first, self.tp_part_amount_second, _update_sum_amount=False): - self.message_log("No grid orders after part filled TP, converted TP to grid", tlg=True) + self.message_log("No grid orders after part filled TP, converted TP to grid") self.tp_part_amount_first = self.tp_part_amount_second = O_DEC elif self.tp_was_filled: - self.message_log("Was filled TP and all grid orders, converse TP to grid", tlg=True) + self.message_log("Was filled TP and all grid orders, converse TP to grid") self.after_filled_tp(one_else_grid=True) else: # Ended grid order, calculate depo and Reverse @@ -1944,7 +1945,7 @@ def convert_tp( self.restore_orders_fire() if replace_tp: self.tp_hold_additional = True - self.message_log("Replace TP", tlg=True) + self.message_log("Replace TP") self.place_grid(self.cycle_buy, amount, reverse_target_amount, @@ -2162,7 +2163,6 @@ def on_new_order_book(self, order_book: OrderBook) -> None: ############################################################## def on_balance_update_ex(self, balance: Dict) -> None: - # TODO Check amount and correct appropriate order volume asset = balance['asset'] delta = Decimal(balance['balance_delta']) restart = False @@ -2406,7 +2406,7 @@ def on_order_update_ex(self, update: OrderUpdate) -> None: replace_tp=False): self.tp_part_free = False self.cancel_reverse_hold() - self.message_log("Part filled TP was converted to grid", tlg=True) + self.message_log("Part filled TP was converted to grid") else: self.message_log(f"Wild order, do not know it: {update.original_order.id}", tlg=True) @@ -2416,7 +2416,7 @@ def cancel_reverse_hold(self): self.reverse_target_amount = O_DEC self.reverse_init_amount = O_DEC self.initial_reverse_first = self.initial_reverse_second = O_DEC - self.message_log("Cancel hold reverse cycle", color=Style.B_WHITE, tlg=True) + self.message_log("Cancel hold reverse cycle", color=Style.B_WHITE) def on_place_order_success(self, place_order_id: int, order: Order) -> None: # print(f"on_place_order_success.place_order_id: {place_order_id}") diff --git a/martin_binance/lib.py b/martin_binance/lib.py index 7940e4b..fb5c4f9 100644 --- a/martin_binance/lib.py +++ b/martin_binance/lib.py @@ -4,7 +4,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.0rc3" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" @@ -421,9 +421,9 @@ class Ticker: __slots__ = ("last_day_price", "last_price", "timestamp") def __init__(self, _ticker): - self.last_day_price = Decimal(_ticker.get('openPrice', '0')) - self.last_price = Decimal(_ticker.get('lastPrice', '0')) - self.timestamp = int(_ticker.get('closeTime', 0)) + self.last_day_price = Decimal(_ticker['openPrice']) + self.last_price = Decimal(_ticker['lastPrice']) + self.timestamp = int(_ticker['closeTime']) def __call__(self): return self @@ -433,8 +433,8 @@ class FundsEntry: __slots__ = ("available", "reserved", "total_for_currency") def __init__(self, _funds): - self.available = Decimal(_funds.get('free')) - self.reserved = Decimal(_funds.get('locked')) + self.available = Decimal(_funds['free']) + self.reserved = Decimal(_funds['locked']) self.total_for_currency = self.available + self.reserved def __call__(self): diff --git a/martin_binance/strategy_base.py b/martin_binance/strategy_base.py index d31ca44..e2b1bff 100644 --- a/martin_binance/strategy_base.py +++ b/martin_binance/strategy_base.py @@ -4,7 +4,7 @@ __author__ = "Jerry Fedorenko" __copyright__ = "Copyright © 2021 Jerry Fedorenko aka VM" __license__ = "MIT" -__version__ = "3.0.0rc24" +__version__ = "3.0.1rc3" __maintainer__ = "Jerry Fedorenko" __contact__ = "https://github.com/DogsTailFarmer" @@ -327,7 +327,7 @@ async def backtest_control(self): self.session_data_handler() self.reset_backtest_vars() if prm.SELF_OPTIMIZATION and self.command != 'stopped': - _ts = datetime.now(timezone.utc) + _ts = datetime.now(timezone.utc).replace(tzinfo=None) storage_name = Path(self.session_root, "_study.db") try: _res = await run_optimize( @@ -360,7 +360,9 @@ async def backtest_control(self): prm, key, value if isinstance(value, int) or key in PARAMS_FLOAT else Decimal(f"{value}") ) - l_m = str(datetime.now(timezone.utc) - _ts + timedelta(seconds=prm.SAVE_PERIOD)).rsplit('.')[0] + l_m = str( + datetime.now(timezone.utc).replace(tzinfo=None) - _ts + timedelta(seconds=prm.SAVE_PERIOD) + ).rsplit('.')[0] self.message_log( f"Strategy parameters are optimal now. Optimization cycle duration {l_m}", color=Style.B_WHITE, @@ -467,7 +469,7 @@ def back_test_handler(self): s_free = prm.SESSION_RESULT['free'] = f"{self.get_free_assets(mode='free', backtest=True)[2]}" if prm.LOGGING: print(f"Session profit: {s_profit}, free: {s_free}, total: {float(s_profit) + float(s_free)}") - test_time = datetime.now(timezone.utc) - self.cycle_time + test_time = datetime.now(timezone.utc).replace(tzinfo=None) - self.cycle_time original_time = (self.backtest['ticker_index_last'] - self.backtest['ticker_index_first']) / 1000 original_time = timedelta(seconds=original_time) print(f"Original time: {original_time}, test time: {test_time}, x = {original_time / test_time:.2f}") @@ -539,7 +541,7 @@ async def heartbeat(self, _session): update_max_queue_size = True self.wss_fire_up = True # - if self.client_id and self.wss_fire_up: + if self.wss_fire_up: try: if await self.session.get_client(): self.update_vars(self.session) @@ -660,7 +662,8 @@ async def ask_exit(self): self.start_collect = False self.session_data_handler() - self.channel.close() + if self.channel: + self.channel.close() tasks = [t for t in asyncio.all_tasks() if t is not asyncio.current_task()] [task.cancel() for task in tasks] @@ -730,7 +733,7 @@ def on_funds_update_handler(self, funds): async def loop_ds(self, ds, ticker=False): while not self.start_collect: - await asyncio.sleep(0.001) + await asyncio.sleep(0.010) batches = ds.iter_batches(PYARROW_BATCH_BUFFER_SIZE) index_prev = 0 @@ -748,6 +751,7 @@ async def loop_ds(self, ds, ticker=False): delay /= prm.XTIME await asyncio.sleep(delay) yield orjson.loads(row['row']) + if ticker: self.backtest['ticker_index_last'] = index_prev * 1000 @@ -1150,13 +1154,6 @@ async def on_order_update_handler(self, ed): if ed['order_status'] == 'FILLED': # Remove from orders dict self.remove_from_orders_lists([ed['order_id']]) - if prm.MODE == 'TC' and self.start_collect and self.s_ticker['pylist']: - s_tic = self.s_ticker['pylist'].pop() - s_tic_row = orjson.loads(s_tic['row']) - s_tic_row['lastPrice'] = ed['last_executed_price'] - s_tic['row'] = orjson.dumps(s_tic_row) - if prm.SAVE_DS: - self.open_orders_snapshot() elif ed['order_status'] == 'PARTIALLY_FILLED': # Update order in orders dict _order = { @@ -1170,6 +1167,19 @@ async def on_order_update_handler(self, ed): } self.orders |= {ed['order_id']: Order(_order)} + if prm.MODE == 'TC' and self.start_collect and self.s_ticker['pylist']: + # print(f"1 s_ticker: {self.s_ticker['pylist'][-1]}") + s_tic = self.s_ticker['pylist'].pop() + s_tic_row = orjson.loads(s_tic['row']) + s_tic_row['lastPrice'] = ed['last_executed_price'] + if ed['order_status'] == 'PARTIALLY_FILLED': + s_tic_row['Qty'] = ed['last_executed_quantity'] + s_tic['row'] = orjson.dumps(s_tic_row) + self.s_ticker['pylist'].append(s_tic) + if prm.SAVE_DS: + self.open_orders_snapshot() + # print(f"2 s_ticker: {self.s_ticker['pylist'][-1]}") + def _on_order_update_handler_ext(self, ed): trade = { "qty": ed['last_executed_quantity'], @@ -1207,14 +1217,14 @@ async def on_ticker_update(self): # if prm.MODE == 'TC' and self.start_collect: ts = int(time.time() * 1000) + self.ticker |= {'delay': self.delay_ordering_s, 'Qty': "0"} if len(self.s_ticker['pylist']) > PYARROW_BATCH_BUFFER_SIZE: # noinspection PyArgumentList self.s_ticker['writer'].write_batch( pa.RecordBatch.from_pylist(mapping=self.s_ticker['pylist']) ) self.s_ticker['pylist'].clear() - self.ticker['delay'] = self.delay_ordering_s - # print(f"on_ticker_update.ticker_24h: {ticker_24h}") + # print(f"on_ticker_update.ticker: {self.ticker}") self.s_ticker['pylist'].append({"key": ts, "row": orjson.dumps(self.ticker)}) if prm.SAVE_DS: self.open_orders_snapshot(ts=ts) @@ -1386,8 +1396,8 @@ async def wss_declare(self): self.tasks_list.append(asyncio.ensure_future(self.backtest_control())) async def wss_init(self, update_max_queue_size=False): - self.message_log(f"Init WSS, client_id: {self.client_id}") if self.client_id: + self.message_log(f"Init WSS, client_id: {self.client_id}") self.task_cancel() await self.wss_declare() # WSS start @@ -1513,15 +1523,12 @@ async def main(self, _symbol): self.order_book['bids'] = self.order_book['bids'] or [[price['price'], amount]] self.order_book['asks'] = self.order_book['asks'] or [[price['price'], amount]] # endregion - _ticker = await self.send_request(self.stub.fetch_ticker_price_change_statistics, - mr.MarketRequest, - symbol=_symbol) + _ticker = await self.send_request( + self.stub.fetch_ticker_price_change_statistics, + mr.MarketRequest, + symbol=_symbol + ) self.ticker = _ticker.to_pydict() - if prm.MODE == 'TC': - # Save first order_book and ticker raw's - ts = int(time.time() * 1000) - self.s_order_book['pylist'].append({"key": ts, "row": orjson.dumps(self.order_book)}) - self.s_ticker['pylist'].append({"key": ts, "row": orjson.dumps(self.ticker)}) # if prm.MODE in ('TC', 'S'): self.session_root = Path(BACKTEST_PATH, f"{self.exchange}_{self.symbol}") @@ -1591,7 +1598,7 @@ async def main(self, _symbol): last_state.pop('ms_start_time_ms', str(int(time.time() * 1000))) ) - # TODO Replace after update + # TODO Replace after update to 3.0.0 # self.orders = jsonpickle.decode(last_state.pop(MS_ORDERS, '{}'), keys=True) _orders = last_state.pop(MS_ORDERS, '{}') @@ -1626,16 +1633,16 @@ async def main(self, _symbol): self.time_operational['new'] = self.backtest['ticker_index_first'] / 1000 self.get_buffered_funds_last_time = self.get_time() self.start_time_ms = int(self.get_time() * 1000) - self.cycle_time = datetime.now(timezone.utc) + self.cycle_time = datetime.now(timezone.utc).replace(tzinfo=None) # await self.wss_declare() if self.state_file.exists(): self.restore_state_before_backtesting() self.init(check_funds=False) + self.start_collect = True else: self.init() self.start() - self.start_collect = True if prm.MODE in ('T', 'TC'): await self.wss_init() diff --git a/pyproject.toml b/pyproject.toml index 50345b8..2038c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dynamic = ["version", "description"] requires-python = ">=3.9" dependencies = [ - "exchanges-wrapper==2.1.2", + "exchanges-wrapper==2.1.3", "jsonpickle==3.0.2", "psutil==5.9.6", "requests==2.31.0", diff --git a/requirements.txt b/requirements.txt index 16c0a1c..03c4f18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -exchanges-wrapper==2.1.2 +exchanges-wrapper==2.1.3 jsonpickle==3.0.2 psutil==5.9.6 requests==2.31.0