hand_tuner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d04cc5b-ff14-4d8a-8b01-36d4935897ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38c240d7-c65b-457b-a10f-7a8f24dccf96",
   "metadata": {},
   "outputs": [],
   "source": [
    "from backtest import backtest\n",
    "from plotting import plot_fills\n",
    "from downloader import Downloader, prep_config\n",
    "from pure_funcs import denumpyize, numpyize, get_template_live_config, candidate_to_live_config, calc_spans, get_template_live_config, analyze_fills, \\\n",
    "    create_xk\n",
    "from njit_funcs import calc_emas, calc_long_orders, round_, calc_bankruptcy_price, calc_diff\n",
    "from procedures import dump_live_config, load_live_config, add_argparse_args, prep_config, make_get_filepath\n",
    "from time import time\n",
    "import sys\n",
    "import argparse\n",
    "import pprint\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c81212aa-6395-46b5-9b57-f6a8d7faa245",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams['figure.figsize'] = [29, 18]\n",
    "pd.set_option('precision', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "457aed4f-fa83-4cd9-bd74-d6c89b4e269d",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Args:\n",
    "    def __init__(self):\n",
    "        self.backtest_config_path = 'configs/backtest/default.hjson'\n",
    "        self.optimize_config_path = 'configs/optimize/default.hjson'\n",
    "        self.exchange = 'binance'\n",
    "        self.symbol = 'HBARUSDT'\n",
    "        self.user = 'your_user_name'\n",
    "        self.start_date = '2021-01-01'\n",
    "        self.end_date = '2021-06-30'\n",
    "        self.starting_balance = 100.0\n",
    "config = await prep_config(Args())\n",
    "dl = Downloader(config)\n",
    "sts = time()\n",
    "data = await dl.get_data()\n",
    "prices, buyer_maker, timestamps = data\n",
    "config['n_days'] = (timestamps[-1] - timestamps[0]) / (1000 * 60 * 60 * 24)\n",
    "\n",
    "print(f'millis to load {len(prices)} ticks {(time() - sts) * 1000:.0f}ms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f38d8fd3-ccf4-4465-837b-24abaa121cee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'price': prices, 'buyer_maker': buyer_maker, 'timestamp': timestamps})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7239f8ed-1096-41ba-809b-00bb188ceb97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# choose a slice on which to test\n",
    "wsize_days = 95\n",
    "ts = int(data[2][-1] - 60 * 60 * 24 * 1000 * wsize_days)\n",
    "idx = np.argmax(data[2] >= ts)\n",
    "dataslice = tuple(d[idx:] for d in data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e3fb767-6167-461c-b601-8b0f45b0d8ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "template = \\\n",
    "    {\"config_name\": \"hand_tuned\",\n",
    "     \"logging_level\": 0,\n",
    "     \"long\": {\"enabled\": True,\n",
    "              \"iprc_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"iprc_const\": 0.99,\n",
    "              \"iqty_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"iqty_const\": 0.05,\n",
    "              \"pbr_limit\": 2.0,\n",
    "              \"markup_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"markup_const\": 1.005,\n",
    "              \"rprc_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"rprc_PBr_coeffs\": [[-0.1, -0.15]],\n",
    "              \"rprc_const\": 0.99,\n",
    "              \"rqty_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"rqty_const\": 0.5,\n",
    "              \"pbr_stop_loss\": 0.5},\n",
    "     \"max_span\": 60000,\n",
    "     \"min_span\": 6000,\n",
    "     \"n_spans\": 3,\n",
    "     \"shrt\": {\"enabled\": True,\n",
    "              \"iprc_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"iprc_const\": 1.01,\n",
    "              \"iqty_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"iqty_const\": 0.05,\n",
    "              \"pbr_limit\": 2.0,\n",
    "              \"markup_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"markup_const\": 0.995,\n",
    "              \"rprc_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"rprc_PBr_coeffs\": [[0.1, 0.15]],\n",
    "              \"rprc_const\": 1.01,\n",
    "              \"rqty_MAr_coeffs\": [[0, 0], [0, 0], [0, 0]],\n",
    "              \"rqty_const\": 0.5,\n",
    "              \"pbr_stop_loss\": 0.5}}\n",
    "config_to_test = {**config, **numpyize(template)}\n",
    "dump_live_config(config_to_test, make_get_filepath('tmp/hand_tuned.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70102f1a-0324-4920-ac0f-253c4b4ecd5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sts = time()\n",
    "fills, info = backtest(config_to_test, dataslice)\n",
    "elapsed = time() - sts\n",
    "print(f'seconds elapsed {elapsed:.4f}')\n",
    "print(f'did finish {info[0]}, lowest eqbal ratio {info[1]:.4f}, closest bkr {info[2]:.4f}')\n",
    "result = {**config_to_test, **{'lowest_eqbal_ratio': info[1], 'closest_bkr': info[2]}}\n",
    "fdf, analysis = analyze_fills(fills, result, timestamps[0], timestamps[-1])\n",
    "pprint.pprint(analysis)\n",
    "fdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32b2dc74-24e4-487b-bdcb-d30281e3cf12",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_fills(df, fdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7c9c3fd-9722-4c09-abb7-aa2eace8b8b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.balance.plot()\n",
    "fdf.equity.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c4ee6b0-70b4-44bf-b1e3-87c272250472",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.pnl.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa9d609-ccd2-4ae5-862e-0d8758ee4953",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.psize.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c4c7093-5bd2-4079-8fe2-a3308a0ac311",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.head(40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7388c40-42e8-4d9a-8646-da953c9beb13",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.adg.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "982b3c55-eec3-48dc-b641-8420a6253a6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# plot MA bands given spans\n",
    "spans = calc_spans(*[config_to_test[k] for k in ['min_span', 'max_span', 'n_spans']])\n",
    "print('spans', spans)\n",
    "emas = pd.DataFrame({str(span): df.price.ewm(span=span, adjust=False).mean() for span in spans})\n",
    "lband = emas.min(axis=1)\n",
    "uband = emas.max(axis=1)\n",
    "df.price.iloc[::100].plot()\n",
    "uband.iloc[::100].plot()\n",
    "lband.iloc[::100].plot()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "236d30c6-cd5a-4649-8295-abe19cf700a8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6c375c9-cccf-4bb7-95e4-f7e4bba5de5f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "634de0d5-df1f-4e53-bd9e-362b5fc39f04",
   "metadata": {},
   "outputs": [],
   "source": [
    "# manual grid design\n",
    "# modify parameters to see resulting grid\n",
    "# long only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdc01646-5eeb-4c8f-ae55-8d5c9aa23f34",
   "metadata": {},
   "outputs": [],
   "source": [
    "xk = create_xk(config_to_test)\n",
    "inverse = xk['inverse']\n",
    "qty_step = xk['qty_step']\n",
    "price_step = xk['price_step']\n",
    "min_qty = xk['min_qty']\n",
    "min_cost = xk['min_cost']\n",
    "c_mult = xk['c_mult']\n",
    "pbr_stop_loss = xk['pbr_stop_loss'][0]\n",
    "pbr_limit = xk['pbr_limit'][0]\n",
    "iqty_const = xk['iqty_const'][0]\n",
    "iprc_const = xk['iprc_const'][0]\n",
    "rqty_const = xk['rqty_const'][0]\n",
    "rprc_const = xk['rprc_const'][0]\n",
    "markup_const = xk['markup_const'][0]\n",
    "iqty_MAr_coeffs = xk['iqty_MAr_coeffs'][0]\n",
    "iprc_MAr_coeffs = xk['iprc_MAr_coeffs'][0]\n",
    "rprc_PBr_coeffs = xk['rprc_PBr_coeffs'][0]\n",
    "rqty_MAr_coeffs = xk['rqty_MAr_coeffs'][0]\n",
    "rprc_MAr_coeffs = xk['rprc_MAr_coeffs'][0]\n",
    "markup_MAr_coeffs = xk['markup_MAr_coeffs'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4225b47-9856-41ca-85d9-c4ab17526b2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "balance = 100.0\n",
    "long_psize = 0.0\n",
    "long_pprice = 0.0\n",
    "highest_bid = prices[-1]\n",
    "lowest_ask = prices[-1]\n",
    "MA_band_lower = prices[-1]\n",
    "MA_band_upper = prices[-1]\n",
    "MA_ratios = np.array([1.0, 1.0, 1.0])\n",
    "available_margin = 2000.0\n",
    "\n",
    "pbr_limit = 2.0\n",
    "iqty_const = 0.01\n",
    "iprc_const = 0.995\n",
    "rqty_const = 1.3\n",
    "rprc_const = 0.98\n",
    "rprc_PBr_coeffs = np.array([[-0.0, -0.05]])\n",
    "\n",
    "\n",
    "orders = []\n",
    "\n",
    "print(\"qty          price          psize         pprice        pprice / price ratio    pbr        bkr diff\")\n",
    "for k in range(10):\n",
    "    long_entry, long_close = calc_long_orders(\n",
    "        balance,\n",
    "        long_psize,\n",
    "        long_pprice,\n",
    "        highest_bid,\n",
    "        lowest_ask,\n",
    "        MA_band_lower,\n",
    "        MA_band_upper,\n",
    "        MA_ratios,\n",
    "        available_margin,\n",
    "\n",
    "        inverse,\n",
    "        qty_step,\n",
    "        price_step,\n",
    "        min_qty,\n",
    "        min_cost,\n",
    "        c_mult,\n",
    "        pbr_stop_loss,\n",
    "        pbr_limit,\n",
    "        iqty_const,\n",
    "        iprc_const,\n",
    "        rqty_const,\n",
    "        rprc_const,\n",
    "        markup_const,\n",
    "        iqty_MAr_coeffs,\n",
    "        iprc_MAr_coeffs,\n",
    "        rprc_PBr_coeffs,\n",
    "        rqty_MAr_coeffs,\n",
    "        rprc_MAr_coeffs,\n",
    "        markup_MAr_coeffs)\n",
    "    \n",
    "    \n",
    "    new_long_psize = long_psize + long_entry[0]\n",
    "    long_pprice = long_entry[1] * (long_entry[0] / new_long_psize) + long_pprice * (long_psize / new_long_psize)\n",
    "    long_psize = new_long_psize\n",
    "    long_pcost = long_psize * long_pprice\n",
    "    bkr_price = calc_bankruptcy_price(balance,\n",
    "                          long_psize,\n",
    "                          long_pprice,\n",
    "                          0.0,\n",
    "                          0.0,\n",
    "                          inverse, c_mult)\n",
    "    bkr_diff = calc_diff(bkr_price, long_pprice)\n",
    "    pbr = long_pcost / balance\n",
    "    if pbr > pbr_limit:\n",
    "        break\n",
    "    print(f\"{long_entry[0]: <12} {long_entry[1]: <14} {long_psize: <12}  {round_(long_pprice, price_step): <14}\"\n",
    "          f\"{round(long_pprice / long_entry[1], 4): <23} {round(pbr, 4): <10} {round(bkr_diff, 6)}\")\n",
    "    orders.append(long_entry)\n",
    "print()\n",
    "orders = numpyize(orders)\n",
    "entry_prices = orders[:,1].astype(float)\n",
    "print('grid range', entry_prices[0] / entry_prices[-1])\n",
    "print('dist between grid entries', entry_prices[:-1] / entry_prices[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bac82031-0281-49d9-ae54-f039c5c9698d",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_days_ = 14\n",
    "nticks = len(df[df.timestamp > (time() - 60 * 60 * 24 * n_days_) * 1000])\n",
    "print('n ticks', nticks)\n",
    "edf = pd.DataFrame({p: np.repeat(p, nticks) for p in entry_prices}, index=df.index[-nticks:]).join(df.price.iloc[-nticks:])\n",
    "edf.iloc[::100].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cfc5d02-e379-4bee-b01f-8ec6531cdc02",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a59c9de4-2cbb-42f2-9c34-267aca072f92",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b259e553-3f0f-4c91-b65e-6fdfb57da6b0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5292ba1-a381-4dea-b22e-e9b47332785c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17853327-6043-4258-afe1-87e7973be435",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load external fills to inspect\n",
    "\n",
    "fdf = pd.read_csv('backtests/binance/BTSUSDT/plots/2021-06-12T114030/fills.csv').set_index('trade_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfb05d81-3e38-416a-93bd-0cc1c6af1c16",
   "metadata": {},
   "outputs": [],
   "source": [
    "longs = fdf[fdf.type.str.contains('long')]\n",
    "shrts = fdf[fdf.type.str.contains('shrt')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d450b1-cb1c-47c6-8263-5f4770592ac2",
   "metadata": {},
   "outputs": [],
   "source": [
    "longs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6efb0912-5ddf-49d2-b94d-5cdbc7037ab4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5acae26-67ff-46a3-9382-187e22dea8f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "longs.tail(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66b38c53-cc0b-4aef-81af-6a230b775c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_fills(df, fdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95f4974c-3d0e-4043-8b58-61b1d810aefe",
   "metadata": {},
   "outputs": [],
   "source": [
    "fdf.balance.plot()\n",
    "fdf.equity.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e9d8220-34ca-4b96-beb5-0124f75b4de0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64f578fb-4dab-4a1d-a964-3c03e7b501b9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06deac29-0297-4e52-86e8-925768b54863",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a4d4e35-226e-4d20-894c-404e086158cf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}