hand_tuner_recursive_grid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d04cc5b-ff14-4d8a-8b01-36d4935897ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38c240d7-c65b-457b-a10f-7a8f24dccf96",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from njit_funcs_recursive_grid import *\n",
    "from backtest import backtest\n",
    "from plotting import plot_fills, make_table\n",
    "from downloader import Downloader, load_hlc_cache\n",
    "from pure_funcs import (\n",
    "    denumpyize,\n",
    "    numpyize,\n",
    "    candidate_to_live_config,\n",
    "    calc_spans,\n",
    "    analyze_fills,\n",
    "    create_xk,\n",
    "    round_dynamic,\n",
    "    round_values,\n",
    "    sort_dict_keys,\n",
    ")\n",
    "from procedures import (\n",
    "    dump_live_config,\n",
    "    load_live_config,\n",
    "    add_argparse_args,\n",
    "    prepare_backtest_config,\n",
    "    make_get_filepath,\n",
    ")\n",
    "from time import time\n",
    "import sys\n",
    "import argparse\n",
    "import pprint\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c81212aa-6395-46b5-9b57-f6a8d7faa245",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = [24, 13.5]\n",
    "plt.rcParams[\"figure.facecolor\"] = \"w\"\n",
    "pd.set_option(\"display.precision\", 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "457aed4f-fa83-4cd9-bd74-d6c89b4e269d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Args:\n",
    "    def __init__(self):\n",
    "        self.backtest_config_path = \"configs/backtest/default.hjson\"\n",
    "        self.exchange = \"binance\"\n",
    "        self.symbols = \"XMRUSDT\"\n",
    "        self.market_type = \"futures\"\n",
    "        self.user = \"binance_01\"\n",
    "        self.start_date = \"2021-05-01\"\n",
    "        self.end_date = \"now\"\n",
    "        self.starting_balance = 10000.0\n",
    "        self.starting_configs = \"\"\n",
    "        self.base_dir = \"backtests\"\n",
    "        self.ohlcv = \"y\"  # set to \"n\" to use 1s candles\n",
    "\n",
    "\n",
    "config = prepare_backtest_config(Args())\n",
    "\n",
    "config[\"adg_n_subdivisions\"] = 10  # see configs/backtest/default.hjson for details\n",
    "config[\"inverse\"] = False\n",
    "sts = time()\n",
    "if config[\"ohlcv\"]:\n",
    "    data = await load_hlc_cache(\n",
    "        config[\"symbol\"],\n",
    "        config[\"inverse\"],\n",
    "        config[\"start_date\"],\n",
    "        config[\"end_date\"],\n",
    "        base_dir=config[\"base_dir\"],\n",
    "        spot=config[\"spot\"],\n",
    "        exchange=config[\"exchange\"],\n",
    "    )\n",
    "    prices = data[:, 3]\n",
    "    highs = data[:, 1]\n",
    "    lows = data[:, 2]\n",
    "    closes = data[:, 3]\n",
    "else:\n",
    "    downloader = Downloader(config)\n",
    "    data = await downloader.get_sampled_ticks()\n",
    "    prices = data[:, 2]\n",
    "    highs = data[:, 2]\n",
    "    lows = data[:, 2]\n",
    "    closes = data[:, 2]\n",
    "timestamps = data[:, 0]\n",
    "config[\"n_days\"] = (timestamps[-1] - timestamps[0]) / (1000 * 60 * 60 * 24)\n",
    "\n",
    "print(f\"millis to load {len(prices)} ticks {(time() - sts) * 1000:.0f}ms\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f38d8fd3-ccf4-4465-837b-24abaa121cee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\"timestamp\": timestamps, \"price\": prices}).set_index(\"timestamp\")\n",
    "df.price.iloc[::100].plot(title=\"Coin Price\", xlabel=\"Time\", ylabel=\"Price\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7239f8ed-1096-41ba-809b-00bb188ceb97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose a slice on which to test\n",
    "wsize_days = 3650\n",
    "ts = int(data[-1][0] - 60 * 60 * 24 * 1000 * wsize_days)\n",
    "idx = np.argmax(data[:, 0] >= ts)\n",
    "dataslice = data[idx:]\n",
    "len(dataslice)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a44cb695-002e-4cd4-b652-36d40f8008c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "hand_tuned = {\n",
    "    \"long\": {\n",
    "        \"enabled\": True,\n",
    "        \"ema_span_0\": 700,\n",
    "        \"ema_span_1\": 1440,\n",
    "        \"initial_qty_pct\": 0.01,\n",
    "        \"initial_eprice_ema_dist\": -0.02,\n",
    "        \"wallet_exposure_limit\": 1.0,\n",
    "        \"ddown_factor\": 0.6,\n",
    "        \"rentry_pprice_dist\": 0.015,\n",
    "        \"rentry_pprice_dist_wallet_exposure_weighting\": 15,\n",
    "        \"min_markup\": 0.002,\n",
    "        \"markup_range\": 0.018,\n",
    "        \"n_close_orders\": 16,\n",
    "        \"auto_unstuck_wallet_exposure_threshold\": 0.15,\n",
    "        \"auto_unstuck_ema_dist\": 0.001,\n",
    "        \"auto_unstuck_qty_pct\": 0.04,\n",
    "        \"auto_unstuck_delay_minutes\": 1000.0,\n",
    "        \"backwards_tp\": True,\n",
    "    },\n",
    "    \"short\": {\n",
    "        \"enabled\": True,\n",
    "        \"ema_span_0\": 1440,\n",
    "        \"ema_span_1\": 2880,\n",
    "        \"initial_qty_pct\": 0.01,\n",
    "        \"initial_eprice_ema_dist\": 0.02,\n",
    "        \"wallet_exposure_limit\": 1.0,\n",
    "        \"ddown_factor\": 0.19,\n",
    "        \"rentry_pprice_dist\": 0.015,\n",
    "        \"rentry_pprice_dist_wallet_exposure_weighting\": 23,\n",
    "        \"min_markup\": 0.002,\n",
    "        \"markup_range\": 0.018,\n",
    "        \"n_close_orders\": 16,\n",
    "        \"auto_unstuck_wallet_exposure_threshold\": 0.05,\n",
    "        \"auto_unstuck_ema_dist\": 0.001,\n",
    "        \"auto_unstuck_qty_pct\": 0.04,\n",
    "        \"auto_unstuck_delay_minutes\": 1000.0,\n",
    "        \"backwards_tp\": True,\n",
    "    },\n",
    "}\n",
    "config_to_test = {**config, **hand_tuned}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74c7f771-3b72-49b2-8210-8e92ca667fd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_long = calc_recursive_entries_long(\n",
    "    Args().starting_balance,\n",
    "    0.0,\n",
    "    0.0,\n",
    "    df.price.iloc[-1],\n",
    "    df.price.iloc[-1],\n",
    "    config_to_test[\"inverse\"],\n",
    "    config_to_test[\"qty_step\"],\n",
    "    config_to_test[\"price_step\"],\n",
    "    config_to_test[\"min_qty\"],\n",
    "    config_to_test[\"min_cost\"],\n",
    "    config_to_test[\"c_mult\"],\n",
    "    config_to_test[\"long\"][\"initial_qty_pct\"],\n",
    "    config_to_test[\"long\"][\"initial_eprice_ema_dist\"],\n",
    "    config_to_test[\"long\"][\"ddown_factor\"],\n",
    "    config_to_test[\"long\"][\"rentry_pprice_dist\"],\n",
    "    config_to_test[\"long\"][\"rentry_pprice_dist_wallet_exposure_weighting\"],\n",
    "    config_to_test[\"long\"][\"wallet_exposure_limit\"],\n",
    "    config_to_test[\"long\"][\"auto_unstuck_ema_dist\"],\n",
    "    config_to_test[\"long\"][\"auto_unstuck_wallet_exposure_threshold\"],\n",
    "    config_to_test[\"long\"][\"auto_unstuck_delay_minutes\"]\n",
    "    or config_to_test[\"long\"][\"auto_unstuck_qty_pct\"],\n",
    "    whole_grid=True,\n",
    ")\n",
    "\n",
    "grid_short = calc_recursive_entries_short(\n",
    "    Args().starting_balance,\n",
    "    0.0,\n",
    "    0.0,\n",
    "    df.price.iloc[-1],\n",
    "    df.price.iloc[-1],\n",
    "    config_to_test[\"inverse\"],\n",
    "    config_to_test[\"qty_step\"],\n",
    "    config_to_test[\"price_step\"],\n",
    "    config_to_test[\"min_qty\"],\n",
    "    config_to_test[\"min_cost\"],\n",
    "    config_to_test[\"c_mult\"],\n",
    "    config_to_test[\"short\"][\"initial_qty_pct\"],\n",
    "    config_to_test[\"short\"][\"initial_eprice_ema_dist\"],\n",
    "    config_to_test[\"short\"][\"ddown_factor\"],\n",
    "    config_to_test[\"short\"][\"rentry_pprice_dist\"],\n",
    "    config_to_test[\"short\"][\"rentry_pprice_dist_wallet_exposure_weighting\"],\n",
    "    config_to_test[\"short\"][\"wallet_exposure_limit\"],\n",
    "    config_to_test[\"short\"][\"auto_unstuck_ema_dist\"],\n",
    "    config_to_test[\"short\"][\"auto_unstuck_wallet_exposure_threshold\"],\n",
    "    config_to_test[\"short\"][\"auto_unstuck_delay_minutes\"]\n",
    "    or config_to_test[\"short\"][\"auto_unstuck_qty_pct\"],\n",
    "    whole_grid=True,\n",
    ")\n",
    "\n",
    "gdf_long = pd.DataFrame(\n",
    "    grid_long, columns=[\"qty\", \"price\", \"type\", \"psize\", \"pprice\", \"wallet_exposure\"]\n",
    ")\n",
    "gdf_long.loc[:, \"eprice_pprice_diff\"] = abs(gdf_long.price - gdf_long.pprice) / gdf_long.price\n",
    "gdf_long.loc[:, \"ddown_factor\"] = gdf_long.qty / gdf_long.psize.shift()\n",
    "gdf_long.loc[:, \"bkr_price\"] = gdf_long.apply(\n",
    "    lambda x: calc_bankruptcy_price(\n",
    "        Args().starting_balance,\n",
    "        x.psize,\n",
    "        x.pprice,\n",
    "        0.0,\n",
    "        0.0,\n",
    "        config_to_test[\"inverse\"],\n",
    "        config_to_test[\"c_mult\"],\n",
    "    ),\n",
    "    axis=1,\n",
    ")\n",
    "gdf_short = pd.DataFrame(\n",
    "    grid_short, columns=[\"qty\", \"price\", \"type\", \"psize\", \"pprice\", \"wallet_exposure\"]\n",
    ")\n",
    "gdf_short.loc[:, \"eprice_pprice_diff\"] = abs(gdf_short.price - gdf_short.pprice) / gdf_short.price\n",
    "gdf_short.loc[:, \"ddown_factor\"] = gdf_short.qty / gdf_short.psize.shift()\n",
    "gdf_short.loc[:, \"bkr_price\"] = gdf_short.apply(\n",
    "    lambda x: calc_bankruptcy_price(\n",
    "        Args().starting_balance,\n",
    "        x.psize,\n",
    "        x.pprice,\n",
    "        0.0,\n",
    "        0.0,\n",
    "        config_to_test[\"inverse\"],\n",
    "        config_to_test[\"c_mult\"],\n",
    "    ),\n",
    "    axis=1,\n",
    ")\n",
    "\n",
    "# Display grid on graph\n",
    "n_ticks = 60 * 60 * 24 * 14  # 14 days\n",
    "dfx = pd.DataFrame(\n",
    "    {\n",
    "        \"timestamp\": pd.to_datetime(timestamps[-n_ticks:], unit=\"ms\"),\n",
    "        \"price\": prices[-n_ticks:],\n",
    "    }\n",
    ").set_index(\"timestamp\")\n",
    "lastdayfrom = pd.to_datetime(Args().end_date)\n",
    "# lastdayfrom = pd.Timestamp('2021-05-21')\n",
    "fig = (\n",
    "    dfx.price.loc[lastdayfrom - pd.Timedelta(days=180) : lastdayfrom]\n",
    "    .iloc[::100]\n",
    "    .plot(title=\"Grid position\", xlabel=\"Time\", ylabel=\"Price\")\n",
    ")\n",
    "if config_to_test[\"long\"][\"enabled\"]:\n",
    "    for i, e in enumerate(gdf_long.itertuples()):\n",
    "        fig.axhline(y=e.price, color=\"b\", linestyle=\"-\")\n",
    "    print(f\"\\nGrid Span Long = {-(gdf_long.price.iloc[-1] / gdf_long.price.iloc[0] - 1) * 100:.2f}%\")\n",
    "\n",
    "if config_to_test[\"short\"][\"enabled\"]:\n",
    "    for i, e in enumerate(gdf_short.itertuples()):\n",
    "        fig.axhline(y=e.price, color=\"r\", linestyle=\"-\")\n",
    "    print(f\"Grid Span Short = {(gdf_short.price.iloc[-1] / gdf_short.price.iloc[0] - 1) * 100:.2f}%\")\n",
    "\n",
    "pd.concat([gdf_long, gdf_short]).sort_values(\"price\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "063edb35-27ff-4dd0-a03e-7996f2bfa9e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "sts = time()\n",
    "fills_long, fills_short, stats = backtest(config_to_test, dataslice)\n",
    "elapsed = time() - sts\n",
    "print(f\"seconds elapsed {elapsed:.4f}\")\n",
    "longs, shorts, sdf, analysis = analyze_fills(fills_long, fills_short, stats, config_to_test)\n",
    "table = make_table({**config_to_test, **analysis})\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c2d6679-1ee7-4016-8c25-9c1163870bf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9d89e7e-8d14-4722-b5ad-4346dc120b6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf.balance_long.plot()\n",
    "sdf.equity_long.plot(title=\"Balance and equity long\", xlabel=\"Time\", ylabel=\"Balance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53c3110a-f9d4-4df7-9335-67900e47529f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf.balance_short.plot()\n",
    "sdf.equity_short.plot(title=\"Balance and equity short\", xlabel=\"Time\", ylabel=\"Balance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32b2dc74-24e4-487b-bdcb-d30281e3cf12",
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(longs) > 0:\n",
    "    plot_fills(df, longs, plot_whole_df=True, title=\"Overview Fills Long\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7df6ee86-4e63-4e14-bb9f-fb9abf9a06e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "if len(shorts) > 0:\n",
    "    plot_fills(df, shorts, plot_whole_df=True, title=\"Overview Fills Short\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa9d609-ccd2-4ae5-862e-0d8758ee4953",
   "metadata": {},
   "outputs": [],
   "source": [
    "wes = pd.DataFrame(\n",
    "    {\"WE_short\": (sdf.wallet_exposure_short.abs() * -1), \"WE_long\": sdf.wallet_exposure_long}\n",
    ")\n",
    "wes.plot(title=\"Wallet exposures\", xlabel=\"Time\", ylabel=\"Wallet Exposure\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9cad96ab-9268-4b08-9d08-e77d8b46e140",
   "metadata": {},
   "outputs": [],
   "source": [
    "sdf.price.plot(title=\"Average entry price\", xlabel=\"Time\", ylabel=\"Price\")\n",
    "sdf[sdf.psize_long != 0].pprice_long.replace(0.0, np.nan).plot()\n",
    "sdf[sdf.psize_short != 0].pprice_short.replace(0.0, np.nan).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ff902c7-a17a-4a48-8069-55f9c1c6e05d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lpprices = sdf[sdf.psize_long != 0.0]\n",
    "spprices = sdf[sdf.psize_short != 0.0]\n",
    "\n",
    "PADistance_long = (lpprices.pprice_long - lpprices.price).abs() / lpprices.price\n",
    "PADistance_short = ((spprices.pprice_short - spprices.price).abs() / spprices.price) * -1\n",
    "print(f\"Mean price action distance long {PADistance_long.mean():.6f} std {PADistance_long.std():.6f}\")\n",
    "print(\n",
    "    f\"Mean price action distance short {PADistance_short.mean():.6f} std {PADistance_short.std():.6f}\"\n",
    ")\n",
    "\n",
    "PADistance_long.plot()\n",
    "PADistance_short.plot(title=\"Price action distance\", xlabel=\"Time\", ylabel=\"Price action distance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc9a54ff-d102-4765-9f95-1bb0c399ed72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inspect long EMAs\n",
    "# blue: lower unstucking; red: upper unstucking; green: initial entry\n",
    "spans = [\n",
    "    config_to_test[\"long\"][\"ema_span_0\"],\n",
    "    (config_to_test[\"long\"][\"ema_span_0\"] * config_to_test[\"long\"][\"ema_span_1\"]) ** 0.5,\n",
    "    config_to_test[\"long\"][\"ema_span_1\"],\n",
    "]\n",
    "print(\n",
    "    f\"spans in minutes {spans}\",\n",
    "    f\"n_days {(df.index[-1] - df.index[0]) / 1000 / 60 / 60 / 24:.1f}\",\n",
    ")\n",
    "\n",
    "spans_multiplier = 60 / ((timestamps[1] - timestamps[0]) / 1000)\n",
    "for i in range(3):\n",
    "    # change to seconds\n",
    "    spans[i] = max(1.0, spans[i] * spans_multiplier)\n",
    "emas = pd.DataFrame({str(span): df.price.ewm(span=span, adjust=False).mean() for span in spans})\n",
    "ema_band_lower = emas.min(axis=1)\n",
    "unstucking_band_lower = ema_band_lower * (1 - config_to_test[\"long\"][\"auto_unstuck_ema_dist\"])\n",
    "ema_band_upper = emas.max(axis=1)\n",
    "unstucking_band_upper = ema_band_upper * (1 + config_to_test[\"long\"][\"auto_unstuck_ema_dist\"])\n",
    "long_ientry_band = ema_band_lower * (1 - config_to_test[\"long\"][\"initial_eprice_ema_dist\"])\n",
    "df.iloc[::100].price.plot(style=\"y-\", title=\"Unstucking Bands and Initial Entry Band\")\n",
    "unstucking_band_lower.iloc[::100].plot(\n",
    "    style=\"b--\",\n",
    ")\n",
    "unstucking_band_upper.iloc[::100].plot(style=\"r--\")\n",
    "long_ientry_band.iloc[::100].plot(style=\"g-.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "649a0610-84cb-4012-997c-b1a5c6c21228",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inspect short EMAs\n",
    "# blue: lower unstucking; red: upper unstucking; green: initial entry\n",
    "spans = [\n",
    "    config_to_test[\"short\"][\"ema_span_0\"],\n",
    "    (config_to_test[\"short\"][\"ema_span_0\"] * config_to_test[\"short\"][\"ema_span_1\"]) ** 0.5,\n",
    "    config_to_test[\"short\"][\"ema_span_1\"],\n",
    "]\n",
    "print(\n",
    "    f\"spans in minutes {spans}\",\n",
    "    f\"n_days {(df.index[-1] - df.index[0]) / 1000 / 60 / 60 / 24:.1f}\",\n",
    ")\n",
    "\n",
    "spans_multiplier = 60 / ((timestamps[1] - timestamps[0]) / 1000)\n",
    "for i in range(3):\n",
    "    # change to seconds\n",
    "    spans[i] = max(1.0, spans[i] * spans_multiplier)\n",
    "emas = pd.DataFrame({str(span): df.price.ewm(span=span, adjust=False).mean() for span in spans})\n",
    "ema_band_lower = emas.min(axis=1)\n",
    "unstucking_band_lower = ema_band_lower * (1 - config_to_test[\"short\"][\"auto_unstuck_ema_dist\"])\n",
    "ema_band_upper = emas.max(axis=1)\n",
    "unstucking_band_upper = ema_band_upper * (1 + config_to_test[\"short\"][\"auto_unstuck_ema_dist\"])\n",
    "short_ientry_band = ema_band_upper * (1 + config_to_test[\"short\"][\"initial_eprice_ema_dist\"])\n",
    "df.iloc[::100].price.plot(style=\"y-\", title=\"Unstucking Bands and Initial Entry Band\")\n",
    "unstucking_band_lower.iloc[::100].plot(style=\"b--\")\n",
    "unstucking_band_upper.iloc[::100].plot(style=\"r--\")\n",
    "short_ientry_band.iloc[::100].plot(style=\"g-.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8450bf0-0eeb-4ac7-8ba9-7ed385c8656f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aaf8c0f-0921-4850-8a78-bbbaee24e70c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}