Merge pull request #399 from enarjord/v5.9.7_slim_analysis

V5.9.7 slim analysis

README.md

@@ -4,7 +4,7 @@
 
 :warning: **Use at own risk** :warning:
 
-v5.9.6
+v5.9.7
 
 
 ## Overview

configs/backtest/default.hjson

@@ -13,7 +13,7 @@
 
   # format YYYY-MM-DD
   start_date: 2020-01-01
-  end_date: 2023-03-17
+  end_date: 2023-05-05
 
   # non default historical data path
   # defaults to local dir if unspecified

configs/forager/example_config.hjson

@@ -1,8 +1,8 @@
 {
   // supported exchanges: [kucoin, okx, bybit, binance]
   user: binance_01
-  twe_long: 3.2
-  twe_short: 1.6
+  twe_long: 1.6
+  twe_short: 0.8
   n_longs: 6
   n_shorts: 3
   max_min_cost: 5.0

downloader.py

@@ -107,7 +107,7 @@ def __init__(self, config: dict):
                 )
             )
 
-    def validate_dataframe(self, df: pd.DataFrame) -> tuple[bool, pd.DataFrame, pd.DataFrame]:
+    def validate_dataframe(self, df: pd.DataFrame) -> Tuple[bool, pd.DataFrame, pd.DataFrame]:
         """
         Validates a dataframe and detects gaps in it. Also detects missing trades in the beginning and end.
         @param df: Dataframe to check for gaps.

exchanges/kucoin.py

@@ -82,6 +82,7 @@ def init_market_type(self):
                 "public_token_ws": "/api/v1/bullet-public",
                 "private_token_ws": "/api/v1/bullet-private",
                 "income": "/api/v1/recentFills",
+                "server_time": "/api/v1/timestamp",
                 "recent_orders": "/api/v1/recentDoneOrders",
             }
             self.hedge_mode = self.config["hedge_mode"] = False
@@ -253,7 +254,8 @@ async def transfer_from_derivatives_to_spot(self, coin: str, amount: float):
         raise "Not implemented"
 
     async def get_server_time(self):
-        raise "Not implemented"
+        server_time = await self.public_get(self.endpoints["server_time"])
+        return server_time["data"]
 
     async def fetch_position(self) -> dict:
         positions, balance = None, None

inspect_opt_results.py

@@ -21,6 +21,19 @@
 from njit_funcs import round_dynamic
 
 
+def shorten(key):
+    key_ = key
+    for src, dst in [
+        ("weighted", "w"),
+        ("exposure", "exp"),
+        ("distance", "dist"),
+        ("ratio", "rt"),
+        ("mean_of_10_worst", "10_worst_mean"),
+    ]:
+        key_ = key_.replace(src, dst)
+    return key_
+
+
 def main():
 
     parser = argparse.ArgumentParser(prog="view conf", description="inspect conf")
@@ -96,6 +109,7 @@ def main():
             scores_res["individual_scores"],
             scores_res["keys"],
         )
+        keys = keys[:1] + [("adg_per_exposure", True)] + keys[1:]
         for side in sides:
             all_scores[-1][side] = {
                 "config": cfg[side],
@@ -121,7 +135,7 @@ def main():
     if os.path.exists(table_filepath):
         os.remove(table_filepath)
     for side in sides:
-        row_headers = ["symbol"] + [k[0] for k in keys] + ["n_days", "score"]
+        row_headers = ["symbol"] + [shorten(k[0]) for k in keys] + ["n_days", "score"]
         table = PrettyTable(row_headers)
         for rh in row_headers:
             table.align[rh] = "l"
@@ -139,7 +153,7 @@ def main():
                 [("-> " if sym in best_candidate[side]["symbols_to_include"] else "") + sym]
                 + [round_dynamic(x, 4) if np.isfinite(x) else x for x in xs]
                 + [round(best_candidate[side]["n_days"][sym], 2)]
-                + [best_candidate[side]["individual_scores"][sym]]
+                + [round_dynamic(best_candidate[side]["individual_scores"][sym], 12)]
             )
         means = [
             np.mean(
@@ -160,7 +174,7 @@ def main():
             ["mean"]
             + [round_dynamic(m, 4) if np.isfinite(m) else m for m in means]
             + [round(np.mean(list(best_candidate[side]["n_days"].values())), 2)]
-            + [ind_scores_mean]
+            + [round_dynamic(ind_scores_mean, 12)]
         )
         with open(make_get_filepath(table_filepath), "a") as f:
             output = table.get_string(border=True, padding_width=1)

njit_clock.py

@@ -91,7 +91,7 @@ def calc_clock_entry_long(
     psize_long: float,
     pprice_long: float,
     highest_bid: float,
-    ema_band_lower: float,
+    emas: float,
     utc_now_ms: float,
     prev_clock_fill_ts_entry: float,
     inverse: bool,
@@ -116,9 +116,7 @@ def calc_clock_entry_long(
         wallet_exposure_long = qty_to_cost(psize_long, pprice_long, inverse, c_mult) / balance
         if wallet_exposure_long < wallet_exposure_limit * 0.99:
             # entry long
-            bid_price_long = calc_clock_price_bid(
-                ema_band_lower, highest_bid, ema_dist_entry, price_step
-            )
+            bid_price_long = calc_clock_price_bid(emas.min(), highest_bid, ema_dist_entry, price_step)
             qty_long = calc_clock_qty(
                 balance,
                 wallet_exposure_long,
@@ -563,7 +561,7 @@ def backtest_clock(
                     psize_long,
                     pprice_long,
                     closes[k - 1],
-                    emas_long.min(),
+                    emas_long,
                     timestamps[k - 1],
                     prev_clock_fill_ts_entry_long,
                     inverse,

optimize.py

@@ -13,7 +13,7 @@
 from multiprocessing import Pool, shared_memory
 from njit_funcs import round_dynamic
 from pure_funcs import (
-    analyze_fills,
+    analyze_fills_slim,
     denumpyize,
     get_template_live_config,
     ts_to_date,
@@ -68,13 +68,7 @@ def backtest_wrap(config_: dict, ticks_caches: dict):
     try:
         assert "adg_n_subdivisions" in config
         fills_long, fills_short, stats = backtest(config, ticks)
-        longs, shorts, sdf, analysis = analyze_fills(fills_long, fills_short, stats, config)
-        logging.debug(
-            f"backtested {config['symbol']: <12} pa distance long {analysis['pa_distance_mean_long']:.6f} "
-            + f"pa distance short {analysis['pa_distance_mean_short']:.6f} adg long {analysis['adg_long']:.6f} "
-            + f"adg short {analysis['adg_short']:.6f} std long {analysis['pa_distance_std_long']:.5f} "
-            + f"std short {analysis['pa_distance_std_short']:.5f}"
-        )
+        analysis = analyze_fills_slim(fills_long, fills_short, stats, config)
     except Exception as e:
         analysis = get_empty_analysis()
         logging.error(f'error with {config["symbol"]} {e}')
@@ -270,7 +264,7 @@ async def run_opt(args, config):
                 if config["ohlcv"]:
                     data = load_hlc_cache(
                         symbol,
-                        config['inverse'],
+                        config["inverse"],
                         config["start_date"],
                         config["end_date"],
                         base_dir=config["base_dir"],
@@ -300,7 +294,8 @@ async def run_opt(args, config):
                     for fname in os.listdir(args.starting_configs):
                         try:
                             """
-                            if config["symbols"][0] not in os.path.join(args.starting_configs, fname):
+                            #  uncomment to skip single starting configs with wrong symbol
+                            if not any(x in os.path.join(args.starting_configs, fname) for x in [config["symbols"][0], "symbols"]):
                                 print("skipping", os.path.join(args.starting_configs, fname))
                                 continue
                             """

passivbot.py

@@ -79,6 +79,7 @@ def __init__(self, config: dict):
 
         self.hedge_mode = self.config["hedge_mode"] = True
         self.set_config(self.config)
+        self.server_time = utc_ms()
 
         self.ts_locked = {
             k: 0.0
@@ -215,6 +216,7 @@ async def _init(self):
             self.init_exchange_config(),
             self.init_order_book(),
             self.init_emas(),
+            self.update_server_time(),
         )
         print("done")
         if "price_step_custom" in self.config and self.config["price_step_custom"] is not None:
@@ -634,8 +636,8 @@ def calc_orders(self):
                             psize_long,
                             pprice_long,
                             self.ob[0],
-                            min(self.emas_long),
-                            utc_ms(),
+                            self.emas_long,
+                            self.server_time,
                             0
                             if psize_long == 0.0
                             else self.last_fills_timestamps["clock_entry_long"],
@@ -695,8 +697,8 @@ def calc_orders(self):
                         psize_long,
                         pprice_long,
                         self.ob[1],
-                        max(self.emas_long),
-                        utc_ms(),
+                        self.emas_long,
+                        self.server_time,
                         self.last_fills_timestamps["clock_close_long"],
                         self.xk["inverse"],
                         self.xk["qty_step"],
@@ -842,8 +844,8 @@ def calc_orders(self):
                             psize_short,
                             pprice_short,
                             self.ob[1],
-                            max(self.emas_short),
-                            utc_ms(),
+                            self.emas_short,
+                            self.server_time,
                             0
                             if psize_short == 0.0
                             else self.last_fills_timestamps["clock_entry_short"],
@@ -903,8 +905,8 @@ def calc_orders(self):
                         psize_short,
                         pprice_short,
                         self.ob[0],
-                        min(self.emas_short),
-                        utc_ms(),
+                        self.emas_short,
+                        self.server_time,
                         self.last_fills_timestamps["clock_close_short"],
                         self.xk["inverse"],
                         self.xk["qty_step"],
@@ -1358,6 +1360,18 @@ async def update_last_fills_timestamps(self):
         finally:
             self.ts_released["update_last_fills_timestamps"] = time.time()
 
+    async def update_server_time(self):
+        server_time = None
+        try:
+            server_time = await self.get_server_time()
+            self.server_time = server_time
+            return True
+        except Exception as e:
+            self.server_time = utc_ms()
+            traceback.print_exc()
+            print_async_exception(server_time)
+            return False
+
     async def start_ohlcv_mode(self):
         logging.info("starting bot...")
         while True:
@@ -1391,9 +1405,14 @@ async def on_minute_mark(self):
                 self.heartbeat_ts = time.time()
             self.prev_price = self.ob[0]
             prev_pos = self.position.copy()
-            to_update = [self.update_position(), self.update_open_orders(), self.init_order_book()]
+            to_update = [
+                self.update_position(),
+                self.update_open_orders(),
+                self.init_order_book(),
+            ]
             if self.passivbot_mode == "clock":
                 to_update.append(self.update_last_fills_timestamps())
+                to_update.append(self.get_server_time())
             res = await asyncio.gather(*to_update)
             self.update_emas(self.ob[0], self.prev_price)
             """
@@ -1405,7 +1424,7 @@ async def on_minute_mark(self):
             print(res)
             """
             if not all(res):
-                reskeys = ["pos", "open orders", "order book", "last fills"]
+                reskeys = ["pos", "open orders", "order book", "last fills", "server_time"]
                 line = "error with "
                 for i in range(len(to_update)):
                     if not to_update[i]:

pure_funcs.py

@@ -654,11 +654,15 @@ def analyze_fills_slim(fills_long: list, fills_short: list, stats: list, config:
     #  eqbal_ratio_mean_of_10_worst,
     #  eqbal_ratio_std,
 
+    #  plus n_days, starting_balance and adg_per_exposure
+
     if "adg_n_subdivisions" not in config:
         config["adg_n_subdivisions"] = 1
 
     ms_per_day = 1000 * 60 * 60 * 24
 
+    n_days = (stats[-1][0] - stats[0][0]) / ms_per_day
+
     if stats[-1][10] <= 0.0:
         adg_long = adg_weighted_long = stats[-1][10]
     else:
@@ -723,6 +727,10 @@ def analyze_fills_slim(fills_long: list, fills_short: list, stats: list, config:
         "adg_weighted_per_exposure_long": adg_weighted_long / config["long"]["wallet_exposure_limit"],
         "adg_weighted_per_exposure_short": adg_weighted_short
         / config["short"]["wallet_exposure_limit"],
+        "adg_per_exposure_long": adg_long / config["long"]["wallet_exposure_limit"],
+        "adg_per_exposure_short": adg_short / config["short"]["wallet_exposure_limit"],
+        "n_days": n_days,
+        "starting_balance": stats[0][10],
         "pa_distance_std_long": pa_dists_long.std(),
         "pa_distance_std_short": pa_dists_short.std(),
         "pa_distance_mean_long": pa_dists_long.mean(),