From 55e4b09979629b1632200ba0d79a0f1ccf7d0541 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 1 Nov 2022 12:31:34 +0100 Subject: [PATCH 01/27] rewrite score logic --- particle_swarm_optimization.py | 203 ++++++++------------------------- 1 file changed, 50 insertions(+), 153 deletions(-) diff --git a/particle_swarm_optimization.py b/particle_swarm_optimization.py index 9efa6b55f..e7fe11be5 100644 --- a/particle_swarm_optimization.py +++ b/particle_swarm_optimization.py @@ -176,167 +176,67 @@ def post_process(self, wi: int): ) if set(results) == set(self.symbols): # completed multisymbol iter - adgs_long = [v["adg_long"] for v in results.values()] - adg_mean_long = np.mean(adgs_long) - pa_distance_std_long_raw = np.mean([v["pa_distance_std_long"] for v in results.values()]) - pa_distance_std_long = np.mean( - [ - max(self.config["maximum_pa_distance_std_long"], v["pa_distance_std_long"]) - for v in results.values() - ] - ) - PAD_mean_long_raw = np.mean([v["pa_distance_mean_long"] for v in results.values()]) - PAD_mean_long = np.mean( - [ - max(self.config["maximum_pa_distance_mean_long"], v["pa_distance_mean_long"]) - for v in results.values() - ] - ) - adg_DGstd_ratios_long = [v["adg_DGstd_ratio_long"] for v in results.values()] - adg_DGstd_ratios_long_mean = np.mean(adg_DGstd_ratios_long) - adgs_short = [v["adg_short"] for v in results.values()] - adg_mean_short = np.mean(adgs_short) - pa_distance_std_short_raw = np.mean( - [v["pa_distance_std_short"] for v in results.values()] - ) - - pa_distance_std_short = np.mean( - [ - max(self.config["maximum_pa_distance_std_short"], v["pa_distance_std_short"]) - for v in results.values() - ] - ) - PAD_mean_short_raw = np.mean([v["pa_distance_mean_short"] for v in results.values()]) - - PAD_mean_short = np.mean( - [ - max(self.config["maximum_pa_distance_mean_short"], v["pa_distance_mean_short"]) - for v in results.values() - ] - ) - adg_DGstd_ratios_short = [v["adg_DGstd_ratio_short"] for v in results.values()] - adg_DGstd_ratios_short_mean = np.mean(adg_DGstd_ratios_short) - eqbal_ratios_std_long = [v["equity_balance_ratio_std_long"] for v in results.values()] - eqbal_ratios_std_short = [v["equity_balance_ratio_std_short"] for v in results.values()] - eqbal_ratio_std_long_mean = np.mean(eqbal_ratios_std_long) - eqbal_ratio_std_short_mean = np.mean(eqbal_ratios_std_short) - adgs_realized_long = [v["adg_realized_per_exposure_long"] for v in results.values()] - adgs_realized_short = [v["adg_realized_per_exposure_short"] for v in results.values()] - adg_realized_long_mean = np.mean(adgs_realized_long) - adg_realized_short_mean = np.mean(adgs_realized_short) - loss_profit_ratio_long_mean = np.mean( - [ - max(self.config["maximum_loss_profit_ratio_long"], v["loss_profit_ratio_long"]) - for v in results.values() - ] - ) - loss_profit_ratio_short_mean = np.mean( - [ - max(self.config["maximum_loss_profit_ratio_short"], v["loss_profit_ratio_short"]) - for v in results.values() - ] - ) - - if self.config["score_formula"] == "adg_PAD_mean": - score_long = -adg_mean_long * min( - 1.0, self.config["maximum_pa_distance_mean_long"] / PAD_mean_long - ) - score_short = -adg_mean_short * min( - 1.0, self.config["maximum_pa_distance_mean_short"] / PAD_mean_short - ) - elif self.config["score_formula"] == "adg_realized_PAD_mean": - score_long = -adg_realized_long_mean / max( - self.config["maximum_pa_distance_mean_long"], PAD_mean_long - ) - score_short = -adg_realized_short_mean / max( - self.config["maximum_pa_distance_mean_short"], PAD_mean_short - ) - elif self.config["score_formula"] == "adg_realized_PAD_std": - score_long = -adg_realized_long_mean / max( - self.config["maximum_pa_distance_std_long"], pa_distance_std_long - ) - score_short = -adg_realized_short_mean / max( - self.config["maximum_pa_distance_std_short"], pa_distance_std_short - ) - - elif self.config["score_formula"] == "adg_PAD_std": - score_long = -adg_mean_long / max( - self.config["maximum_pa_distance_std_long"], pa_distance_std_long - ) - score_short = -adg_mean_short / max( - self.config["maximum_pa_distance_std_short"], pa_distance_std_short - ) - elif self.config["score_formula"] == "adg_DGstd_ratio": - score_long = -adg_DGstd_ratios_long_mean - score_short = -adg_DGstd_ratios_short_mean - elif self.config["score_formula"] == "adg_mean": - score_long = -adg_mean_long - score_short = -adg_mean_short - elif self.config["score_formula"] == "adg_min": - score_long = -min(adgs_long) - score_short = -min(adgs_short) - elif self.config["score_formula"] == "adg_PAD_std_min": - # best worst score - scores_long = [ - v["adg_long"] - / max(v["pa_distance_std_long"], self.config["maximum_pa_distance_std_long"]) - for v in results.values() - ] - score_long = -min(scores_long) - scores_short = [ - v["adg_short"] - / max(v["pa_distance_std_short"], self.config["maximum_pa_distance_std_short"]) - for v in results.values() - ] - score_short = -min(scores_short) - else: - raise Exception(f"unknown score formula {self.config['score_formula']}") - score_long *= self.config["maximum_loss_profit_ratio_long"] / max( - loss_profit_ratio_long_mean, self.config["maximum_loss_profit_ratio_long"] - ) - score_short *= self.config["maximum_loss_profit_ratio_short"] / max( - loss_profit_ratio_short_mean, self.config["maximum_loss_profit_ratio_short"] - ) + sides = ["long", "short"] + keys = [ + ("adg_realized_per_exposure", True), + ("pa_distance_std", False), + ("pa_distance_mean", False), + ("hrs_stuck_max", False), + ("loss_profit_ratio", False), + ] + means = {s: {} for s in sides} # adjusted means + scores = {s: -1.0 for s in sides} + raws = {s: {} for s in sides} # unadjusted means + for side in sides: + for key, mult in keys: + max_key = f"maximum_{key}_{side}" + raws[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) + if max_key in self.config: + if self.config[max_key] >= 0.0: + ms = [ + max(self.config[max_key], v[f"{key}_{side}"]) + for v in results.values() + ] + means[side][key] = max(np.mean(ms), self.config[max_key]) + else: + means[side][key] = 1.0 + else: + means[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) + if mult: + scores[side] *= means[side][key] + else: + scores[side] /= means[side][key] - line = f"completed multisymbol iter {cfg['config_no']} " - if self.do_long: - line += f"- adg long {adg_mean_long:.6f} PAD long {PAD_mean_long:.6f} std long " - line += f"{pa_distance_std_long:.5f} score long {score_long:.7f} " - if self.do_short: - line += f"- adg short {adg_mean_short:.6f} PAD short {PAD_mean_short:.6f} std short " - line += f"{pa_distance_std_short:.5f} score short {score_short:.7f}" - logging.debug(line) - self.swarm[swarm_key]["long"]["score"] = score_long - self.swarm[swarm_key]["short"]["score"] = score_short + self.swarm[swarm_key]["long"]["score"] = scores["long"] + self.swarm[swarm_key]["short"]["score"] = scores["short"] # check if better than lbest long if ( type(self.lbests_long[swarm_key]["score"]) == str - or score_long < self.lbests_long[swarm_key]["score"] + or scores["long"] < self.lbests_long[swarm_key]["score"] ): - self.lbests_long[swarm_key] = deepcopy({"config": cfg["long"], "score": score_long}) + self.lbests_long[swarm_key] = deepcopy( + {"config": cfg["long"], "score": scores["long"]} + ) # check if better than lbest short if ( type(self.lbests_short[swarm_key]["score"]) == str - or score_short < self.lbests_short[swarm_key]["score"] + or scores["short"] < self.lbests_short[swarm_key]["score"] ): self.lbests_short[swarm_key] = deepcopy( - {"config": cfg["short"], "score": score_short} + {"config": cfg["short"], "score": scores["short"]} ) tmp_fname = f"{self.results_fpath}{cfg['config_no']:06}_best_config" is_better = False # check if better than gbest long - if self.gbest_long is None or score_long < self.gbest_long["score"]: - self.gbest_long = deepcopy({"config": cfg["long"], "score": score_long}) + if self.gbest_long is None or scores["long"] < self.gbest_long["score"]: + self.gbest_long = deepcopy({"config": cfg["long"], "score": scores["long"]}) is_better = True - logging.info( - f"i{cfg['config_no']} - new best config long, score {score_long:.7f} " - + f"adg {adg_realized_long_mean:.6f} " - + f"PAD mean {PAD_mean_long_raw:.6f} " - + f"PAD std {pa_distance_std_long_raw:.5f} " - + f"eqbal ratio std {eqbal_ratio_std_long_mean:.6f}" - ) + line = f"i{cfg['config_no']} - new best config long, score {round_dynamic(scores['long'], 4)} " + for key, _ in keys: + line += f"{key} {round_dynamic(raws['long'][key], 4)} " + logging.info(line) tmp_fname += "_long" json.dump( results, @@ -345,16 +245,13 @@ def post_process(self, wi: int): sort_keys=True, ) # check if better than gbest short - if self.gbest_short is None or score_short < self.gbest_short["score"]: - self.gbest_short = deepcopy({"config": cfg["short"], "score": score_short}) + if self.gbest_short is None or scores["short"] < self.gbest_short["score"]: + self.gbest_short = deepcopy({"config": cfg["short"], "score": scores["short"]}) is_better = True - logging.info( - f"i{cfg['config_no']} - new best config short, score {score_short:.7f} " - + f"adg {adg_realized_short_mean:.6f} " - + f"PAD mean {PAD_mean_short_raw:.6f} " - + f"PAD std {pa_distance_std_short_raw:.5f} " - + f"eqbal ratio std {eqbal_ratio_std_short_mean:.6f}" - ) + line = f"i{cfg['config_no']} - new best config short, score {round_dynamic(scores['short'], 4)} " + for key, _ in keys: + line += f"{key} {round_dynamic(raws['short'][key], 4)} " + logging.info(line) tmp_fname += "_short" json.dump( results, From 7cbb1535cb4437fad090c722f85d2a44e6ed9c8c Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 1 Nov 2022 12:31:50 +0100 Subject: [PATCH 02/27] rewrite score logic --- harmony_search.py | 210 ++++++++++------------------------------------ 1 file changed, 46 insertions(+), 164 deletions(-) diff --git a/harmony_search.py b/harmony_search.py index 8bd4dd66d..bf21d4881 100644 --- a/harmony_search.py +++ b/harmony_search.py @@ -162,141 +162,41 @@ def post_process(self, wi: int): results = deepcopy(self.unfinished_evals[id_key]["single_results"]) if set(results) == set(self.symbols): # completed multisymbol iter - adgs_long = [v["adg_long"] for v in results.values()] - adg_mean_long = np.mean(adgs_long) - pa_distance_std_long_raw = np.mean([v["pa_distance_std_long"] for v in results.values()]) - pa_distance_std_long = np.mean( - [ - max(self.config["maximum_pa_distance_std_long"], v["pa_distance_std_long"]) - for v in results.values() - ] - ) - PAD_mean_long_raw = np.mean([v["pa_distance_mean_long"] for v in results.values()]) - PAD_mean_long = np.mean( - [ - max(self.config["maximum_pa_distance_mean_long"], v["pa_distance_mean_long"]) - for v in results.values() - ] - ) - adg_DGstd_ratios_long = [v["adg_DGstd_ratio_long"] for v in results.values()] - adg_DGstd_ratios_long_mean = np.mean(adg_DGstd_ratios_long) - adgs_short = [v["adg_short"] for v in results.values()] - adg_mean_short = np.mean(adgs_short) - pa_distance_std_short_raw = np.mean( - [v["pa_distance_std_short"] for v in results.values()] - ) - - pa_distance_std_short = np.mean( - [ - max(self.config["maximum_pa_distance_std_short"], v["pa_distance_std_short"]) - for v in results.values() - ] - ) - PAD_mean_short_raw = np.mean([v["pa_distance_mean_short"] for v in results.values()]) - - PAD_mean_short = np.mean( - [ - max(self.config["maximum_pa_distance_mean_short"], v["pa_distance_mean_short"]) - for v in results.values() - ] - ) - adg_DGstd_ratios_short = [v["adg_DGstd_ratio_short"] for v in results.values()] - adg_DGstd_ratios_short_mean = np.mean(adg_DGstd_ratios_short) - eqbal_ratios_std_long = [v["equity_balance_ratio_std_long"] for v in results.values()] - eqbal_ratios_std_short = [v["equity_balance_ratio_std_short"] for v in results.values()] - eqbal_ratio_std_long_mean = np.mean(eqbal_ratios_std_long) - eqbal_ratio_std_short_mean = np.mean(eqbal_ratios_std_short) - adgs_realized_long = [v["adg_realized_per_exposure_long"] for v in results.values()] - adgs_realized_short = [v["adg_realized_per_exposure_short"] for v in results.values()] - adg_realized_long_mean = np.mean(adgs_realized_long) - adg_realized_short_mean = np.mean(adgs_realized_short) - loss_profit_ratio_long_mean = np.mean( - [ - max(self.config["maximum_loss_profit_ratio_long"], v["loss_profit_ratio_long"]) - for v in results.values() - ] - ) - loss_profit_ratio_short_mean = np.mean( - [ - max(self.config["maximum_loss_profit_ratio_short"], v["loss_profit_ratio_short"]) - for v in results.values() - ] - ) - - if self.config["score_formula"] == "adg_PAD_mean": - score_long = -adg_mean_long * min( - 1.0, self.config["maximum_pa_distance_mean_long"] / PAD_mean_long - ) - score_short = -adg_mean_short * min( - 1.0, self.config["maximum_pa_distance_mean_short"] / PAD_mean_short - ) - elif self.config["score_formula"] == "adg_realized_PAD_mean": - score_long = -adg_realized_long_mean / max( - self.config["maximum_pa_distance_mean_long"], PAD_mean_long - ) - score_short = -adg_realized_short_mean / max( - self.config["maximum_pa_distance_mean_short"], PAD_mean_short - ) - elif self.config["score_formula"] == "adg_realized_PAD_std": - score_long = -adg_realized_long_mean / max( - self.config["maximum_pa_distance_std_long"], pa_distance_std_long - ) - score_short = -adg_realized_short_mean / max( - self.config["maximum_pa_distance_std_short"], pa_distance_std_short - ) - - elif self.config["score_formula"] == "adg_PAD_std": - score_long = -adg_mean_long / max( - self.config["maximum_pa_distance_std_long"], pa_distance_std_long - ) - score_short = -adg_mean_short / max( - self.config["maximum_pa_distance_std_short"], pa_distance_std_short - ) - elif self.config["score_formula"] == "adg_DGstd_ratio": - score_long = -adg_DGstd_ratios_long_mean - score_short = -adg_DGstd_ratios_short_mean - elif self.config["score_formula"] == "adg_mean": - score_long = -adg_mean_long - score_short = -adg_mean_short - elif self.config["score_formula"] == "adg_min": - score_long = -min(adgs_long) - score_short = -min(adgs_short) - elif self.config["score_formula"] == "adg_PAD_std_min": - # best worst score - scores_long = [ - v["adg_long"] - / max(v["pa_distance_std_long"], self.config["maximum_pa_distance_std_long"]) - for v in results.values() - ] - score_long = -min(scores_long) - scores_short = [ - v["adg_short"] - / max(v["pa_distance_std_short"], self.config["maximum_pa_distance_std_short"]) - for v in results.values() - ] - score_short = -min(scores_short) - else: - raise Exception(f"unknown score formula {self.config['score_formula']}") - - score_long *= self.config["maximum_loss_profit_ratio_long"] / max( - loss_profit_ratio_long_mean, self.config["maximum_loss_profit_ratio_long"] - ) - score_short *= self.config["maximum_loss_profit_ratio_short"] / max( - loss_profit_ratio_short_mean, self.config["maximum_loss_profit_ratio_short"] - ) + sides = ["long", "short"] + keys = [ + ("adg_realized_per_exposure", True), + ("pa_distance_std", False), + ("pa_distance_mean", False), + ("hrs_stuck_max", False), + ("loss_profit_ratio", False), + ] + means = {s: {} for s in sides} # adjusted means + scores = {s: -1.0 for s in sides} + raws = {s: {} for s in sides} # unadjusted means + for side in sides: + for key, mult in keys: + max_key = f"maximum_{key}_{side}" + raws[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) + if max_key in self.config: + if self.config[max_key] >= 0.0: + ms = [ + max(self.config[max_key], v[f"{key}_{side}"]) + for v in results.values() + ] + means[side][key] = max(np.mean(ms), self.config[max_key]) + else: + means[side][key] = 1.0 + else: + means[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) + if mult: + scores[side] *= means[side][key] + else: + scores[side] /= means[side][key] - line = f"completed multisymbol iter {cfg['config_no']} " - if self.do_long: - line += f"- adg long {adg_mean_long:.6f} PAD long {PAD_mean_long:.6f} std long " - line += f"{pa_distance_std_long:.5f} score long {score_long:.7f} " - if self.do_short: - line += f"- adg short {adg_mean_short:.6f} PAD short {PAD_mean_short:.6f} std short " - line += f"{pa_distance_std_short:.5f} score short {score_short:.7f}" - logging.debug(line) # check whether initial eval or new harmony if "initial_eval_key" in cfg: - self.hm[cfg["initial_eval_key"]]["long"]["score"] = score_long - self.hm[cfg["initial_eval_key"]]["short"]["score"] = score_short + self.hm[cfg["initial_eval_key"]]["long"]["score"] = scores["long"] + self.hm[cfg["initial_eval_key"]]["short"]["score"] = scores["short"] else: # check if better than worst in harmony memory worst_key_long = sorted( @@ -305,15 +205,10 @@ def post_process(self, wi: int): if type(self.hm[x]["long"]["score"]) != str else -np.inf, )[-1] - if self.do_long and score_long < self.hm[worst_key_long]["long"]["score"]: - logging.debug( - f"improved long harmony, prev score " - + f"{self.hm[worst_key_long]['long']['score']:.7f} new score {score_long:.7f} - " - + " ".join([str(round_dynamic(e[1], 3)) for e in sorted(cfg["long"].items())]) - ) + if self.do_long and scores["long"] < self.hm[worst_key_long]["long"]["score"]: self.hm[worst_key_long]["long"] = { "config": deepcopy(cfg["long"]), - "score": score_long, + "score": scores["long"], } json.dump( self.hm, @@ -327,17 +222,10 @@ def post_process(self, wi: int): if type(self.hm[x]["short"]["score"]) != str else -np.inf, )[-1] - if self.do_short and score_short < self.hm[worst_key_short]["short"]["score"]: - logging.debug( - f"improved short harmony, prev score " - + f"{self.hm[worst_key_short]['short']['score']:.7f} new score {score_short:.7f} - " - + " ".join( - [str(round_dynamic(e[1], 3)) for e in sorted(cfg["short"].items())] - ), - ) + if self.do_short and scores["short"] < self.hm[worst_key_short]["short"]["score"]: self.hm[worst_key_short]["short"] = { "config": deepcopy(cfg["short"]), - "score": score_short, + "score": scores["short"], } json.dump( self.hm, @@ -369,15 +257,12 @@ def post_process(self, wi: int): } tmp_fname = f"{self.results_fpath}{cfg['config_no']:06}_best_config" is_better = False - if self.do_long and score_long <= self.hm[best_key_long]["long"]["score"]: + if self.do_long and scores["long"] <= self.hm[best_key_long]["long"]["score"]: is_better = True - logging.info( - f"i{cfg['config_no']} - new best config long, score {score_long:.7f} " - + f"adg {adg_realized_long_mean:.6f} " - + f"PAD mean {PAD_mean_long_raw:.6f} " - + f"PAD std {pa_distance_std_long_raw:.5f} " - + f"eqbal ratio std {eqbal_ratio_std_long_mean:.6f}" - ) + line = f"i{cfg['config_no']} - new best config long, score {round_dynamic(scores['long'], 4)} " + for key, _ in keys: + line += f"{key} {round_dynamic(raws['long'][key], 4)} " + logging.info(line) tmp_fname += "_long" json.dump( results, @@ -385,15 +270,12 @@ def post_process(self, wi: int): indent=4, sort_keys=True, ) - if self.do_short and score_short <= self.hm[best_key_short]["short"]["score"]: + if self.do_short and scores["short"] <= self.hm[best_key_short]["short"]["score"]: is_better = True - logging.info( - f"i{cfg['config_no']} - new best config short, score {score_short:.7f} " - + f"adg {adg_realized_short_mean:.6f} " - + f"PAD mean {PAD_mean_short_raw:.6f} " - + f"PAD std {pa_distance_std_short_raw:.5f} " - + f"eqbal ratio std {eqbal_ratio_std_short_mean:.6f}" - ) + line = f"i{cfg['config_no']} - new best config short, score {round_dynamic(scores['short'], 4)} " + for key, _ in keys: + line += f"{key} {round_dynamic(raws['short'][key], 4)} " + logging.info(line) tmp_fname += "_short" json.dump( results, From 0d451c41c539c2d615945d89cbf016efbc6e1db8 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 1 Nov 2022 12:32:20 +0100 Subject: [PATCH 03/27] rewrite score logic --- configs/optimize/harmony_search.hjson | 133 ++++++-------- .../particle_swarm_optimization.hjson | 163 ++++++++---------- 2 files changed, 127 insertions(+), 169 deletions(-) diff --git a/configs/optimize/harmony_search.hjson b/configs/optimize/harmony_search.hjson index 034aab963..c8c4660aa 100644 --- a/configs/optimize/harmony_search.hjson +++ b/configs/optimize/harmony_search.hjson @@ -8,47 +8,26 @@ n_cpus: 4 iters: 8000 - # score formula choices: - # adg_PAD_mean - # adg_PAD_std - # adg_PAD_std_min - # adg_DGstd_ratio - # adg_mean - # adg_min - # adg_realized_PAD_mean - # adg_realized_PAD_std - score_formula: adg_realized_PAD_std + # score = adg_realized_per_exposure - # adg_PAD_std: - # adg / max(max_pa_dist_std, mean([max(max_pa_dist_std, PAD_std) for PAD_std in results])) - maximum_pa_distance_std_long: 0.015 - maximum_pa_distance_std_short: 0.015 + # score weights + # set any to -1 (less than zero) to disable - # adg_PAD_std_min: - # min([adg / max(max_pa_dist_std, mean([max(max_pa_dist_std, PAD_std) for adg, PAD_std in results]))) + # score /= max(maximum_pa_distance_std, pa_distance_std) + maximum_pa_distance_std_long: 0.01 + maximum_pa_distance_std_short: 0.01 - # adg_PAD_mean: - # adg * min(1, max_pa_dist_mean / mean([max(max_pa_dist_mean, pa_dist) for pa_dist in results])) + # score /= max(maximum_pa_distance_mean, pa_distance_mean) + maximum_pa_distance_mean_long: 0.01 + maximum_pa_distance_mean_short: 0.01 - # adg_realized_PAD_mean: - # adg_realized / max(max_pa_dist_mean, mean([max(max_pa_dist_mean, PAD_mean) for PAD_mean in results])) - - maximum_pa_distance_mean_long: 0.015 - maximum_pa_distance_mean_short: 0.015 - - # profit_loss_ratio = abs(sum(loss)) / sum(profit) - # set to high value (e.g. 100.0) to disable + # score /= max(maximum_loss_profit_ratio, loss_profit_ratio) maximum_loss_profit_ratio_long: 0.1 maximum_loss_profit_ratio_short: 0.1 - # adg_DGstd_ratio: - # mean([dg_mean_std_ratio]) / std(dg_mean_std_ratio) - - # adg_mean: - # mean(adgs) - - # adg_min: - # min(adgs) + # score /= max(maximum_hrs_stuck_max, hrs_stuck_max) + maximum_hrs_stuck_max_long: -1 + maximum_hrs_stuck_max_short: -1 # will override starting configs' parameters do_long: true @@ -56,7 +35,7 @@ backwards_tp_long: true backwards_tp_short: true - # choices: [recursive_grid, static_grid, neat_grid] + # choices: [r/recursive_grid, s/static_grid, n/neat_grid] passivbot_mode: neat_grid # sorted highest to lowest vol last 250 days @@ -132,72 +111,72 @@ { long: { - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] - initial_qty_pct: [0.007, 0.05] - initial_eprice_ema_dist: [-0.1, 0.0] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] + initial_qty_pct: [0.015, 0.015] + initial_eprice_ema_dist: [-0.1, 0.01] wallet_exposure_limit: [0.1, 0.1] ddown_factor: [0.05, 3.0] rentry_pprice_dist: [0.005, 0.05] rentry_pprice_dist_wallet_exposure_weighting: [0.0, 90.0] - min_markup: [0.002, 0.02] - markup_range: [0.0, 0.04] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + min_markup: [0.001, 0.004] + markup_range: [0.0, 0.03] + n_close_orders: [4, 20] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.001, 0.1] } short: { - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] - initial_qty_pct: [0.007, 0.05] - initial_eprice_ema_dist: [-0.1, 0.0] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] + initial_qty_pct: [0.015, 0.015] + initial_eprice_ema_dist: [-0.1, 0.01] wallet_exposure_limit: [0.1, 0.1] ddown_factor: [0.05, 3.0] rentry_pprice_dist: [0.005, 0.05] rentry_pprice_dist_wallet_exposure_weighting: [0.0, 90.0] - min_markup: [0.002, 0.02] - markup_range: [0.0, 0.04] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + min_markup: [0.001, 0.004] + markup_range: [0.0, 0.03] + n_close_orders: [4, 20] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.001, 0.1] } } bounds_neat_grid: { long: { - grid_span: [0.1, 0.7] - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] + grid_span: [0.34, 0.7] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] - max_n_entry_orders: [5, 20] + max_n_entry_orders: [7, 30] initial_qty_pct: [0.015, 0.015] - initial_eprice_ema_dist: [-0.1, 0.0] - eqty_exp_base: [0.1, 5.0] - eprice_exp_base: [0.1, 5.0] - min_markup: [0.0015, 0.01] - markup_range: [0.0, 0.06] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + initial_eprice_ema_dist: [-0.1, 0.01] + eqty_exp_base: [0.9, 4.0] + eprice_exp_base: [0.9, 4.0] + min_markup: [0.001, 0.003] + markup_range: [0.0, 0.04] + n_close_orders: [7, 30] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.0, 0.02] } short: { - grid_span: [0.1, 0.7] - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] + grid_span: [0.34, 0.7] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] - max_n_entry_orders: [5, 20] + max_n_entry_orders: [7, 30] initial_qty_pct: [0.015, 0.015] - initial_eprice_ema_dist: [-0.1, 0.0] - eqty_exp_base: [0.1, 5.0] - eprice_exp_base: [0.1, 5.0] - min_markup: [0.0015, 0.01] - markup_range: [0.0, 0.06] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + initial_eprice_ema_dist: [-0.1, 0.01] + eqty_exp_base: [0.9, 4.0] + eprice_exp_base: [0.9, 4.0] + min_markup: [0.001, 0.003] + markup_range: [0.0, 0.04] + n_close_orders: [7, 30] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.0, 0.02] } } } diff --git a/configs/optimize/particle_swarm_optimization.hjson b/configs/optimize/particle_swarm_optimization.hjson index d3356d31d..d6ab803b2 100644 --- a/configs/optimize/particle_swarm_optimization.hjson +++ b/configs/optimize/particle_swarm_optimization.hjson @@ -1,54 +1,33 @@ { # particle swarm optimization parameters n_particles: 24 - w: 0.7298 # weight of previous velocity + w: 0.73 # weight of previous velocity c0: 1.0 # attraction to local best c1: 0.5 # attraction to global best - n_cpus: 7 - iters: 8000 + n_cpus: 4 + iters: 4000 - # score formula choices: - # adg_PAD_mean - # adg_PAD_std - # adg_PAD_std_min - # adg_DGstd_ratio - # adg_mean - # adg_min - # adg_realized_PAD_mean - # adg_realized_PAD_std - score_formula: adg_realized_PAD_std + # score = adg_realized_per_exposure - # adg_PAD_std: - # adg / max(max_pa_dist_std, mean([max(max_pa_dist_std, PAD_std) for PAD_std in results])) + # score weights + # set any to -1 (less than zero) to disable + + # score /= max(maximum_pa_distance_std, pa_distance_std) maximum_pa_distance_std_long: 0.01 maximum_pa_distance_std_short: 0.01 - # adg_PAD_std_min: - # min([adg / max(max_pa_dist_std, mean([max(max_pa_dist_std, PAD_std) for adg, PAD_std in results]))) - - # adg_PAD_mean: - # adg * min(1, max_pa_dist_mean / mean([max(max_pa_dist_mean, pa_dist) for pa_dist in results])) - - # adg_realized_PAD_mean: - # adg_realized / max(max_pa_dist_mean, mean([max(max_pa_dist_mean, PAD_mean) for PAD_mean in results])) - + # score /= max(maximum_pa_distance_mean, pa_distance_mean) maximum_pa_distance_mean_long: 0.01 maximum_pa_distance_mean_short: 0.01 - # profit_loss_ratio = abs(sum(loss)) / sum(profit) - # set to high value (e.g. 100.0) to disable + # score /= max(maximum_loss_profit_ratio, loss_profit_ratio) maximum_loss_profit_ratio_long: 0.1 maximum_loss_profit_ratio_short: 0.1 - # adg_DGstd_ratio: - # mean([dg_mean_std_ratio]) / std(dg_mean_std_ratio) - - # adg_mean: - # mean(adgs) - - # adg_min: - # min(adgs) + # score /= max(maximum_hrs_stuck_max, hrs_stuck_max) + maximum_hrs_stuck_max_long: -1 + maximum_hrs_stuck_max_short: -1 # will override starting configs' parameters do_long: true @@ -91,113 +70,113 @@ { long: { - grid_span: [0.1, 0.7] - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] + grid_span: [0.35, 0.7] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] max_n_entry_orders: [7, 13] - initial_qty_pct: [0.007, 0.05] - initial_eprice_ema_dist: [-0.1, 0.0] + initial_qty_pct: [0.015, 0.015] + initial_eprice_ema_dist: [-0.1, 0.01] eprice_pprice_diff: [0.002, 0.05] secondary_allocation: [0.0, 0.7] secondary_pprice_diff: [0.05, 0.5] eprice_exp_base: [1.0, 2.1] - min_markup: [0.002, 0.02] + min_markup: [0.001, 0.003] markup_range: [0.0, 0.04] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + n_close_orders: [7, 20] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.001, 0.1] } short: { - grid_span: [0.1, 0.7] - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] + grid_span: [0.35, 0.7] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] max_n_entry_orders: [7, 13] - initial_qty_pct: [0.007, 0.05] - initial_eprice_ema_dist: [-0.1, 0.0] + initial_qty_pct: [0.015, 0.015] + initial_eprice_ema_dist: [-0.1, 0.01] eprice_pprice_diff: [0.002, 0.05] secondary_allocation: [0.0, 0.7] secondary_pprice_diff: [0.05, 0.5] eprice_exp_base: [1.0, 2.1] - min_markup: [0.002, 0.02] + min_markup: [0.001, 0.003] markup_range: [0.0, 0.04] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + n_close_orders: [7, 20] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.001, 0.1] } } bounds_recursive_grid: { long: { - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] - initial_qty_pct: [0.007, 0.05] - initial_eprice_ema_dist: [-0.1, 0.0] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] + initial_qty_pct: [0.015, 0.015] + initial_eprice_ema_dist: [-0.1, 0.01] wallet_exposure_limit: [0.1, 0.1] ddown_factor: [0.05, 3.0] rentry_pprice_dist: [0.005, 0.05] rentry_pprice_dist_wallet_exposure_weighting: [0.0, 90.0] - min_markup: [0.002, 0.02] + min_markup: [0.001, 0.004] markup_range: [0.0, 0.04] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + n_close_orders: [4, 20] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.001, 0.1] } short: { - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] - initial_qty_pct: [0.007, 0.05] - initial_eprice_ema_dist: [-0.1, 0.0] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] + initial_qty_pct: [0.015, 0.015] + initial_eprice_ema_dist: [-0.1, 0.01] wallet_exposure_limit: [0.1, 0.1] ddown_factor: [0.05, 3.0] rentry_pprice_dist: [0.005, 0.05] rentry_pprice_dist_wallet_exposure_weighting: [0.0, 90.0] - min_markup: [0.002, 0.02] + min_markup: [0.001, 0.004] markup_range: [0.0, 0.04] - n_close_orders: [5, 20] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.6] - auto_unstuck_ema_dist: [0.0, 0.1] + n_close_orders: [4, 20] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.001, 0.1] } } bounds_neat_grid: { long: { - grid_span: [0.01, 0.7] - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] + grid_span: [0.34, 0.7] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] - max_n_entry_orders: [5, 30] + max_n_entry_orders: [7, 30] initial_qty_pct: [0.015, 0.015] - initial_eprice_ema_dist: [-0.1, 0.0] - eqty_exp_base: [0.1, 5.0] - eprice_exp_base: [0.1, 5.0] - min_markup: [0.0015, 0.01] - markup_range: [0.0, 0.06] - n_close_orders: [5, 30] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.1] - auto_unstuck_ema_dist: [0.0, 0.04] + initial_eprice_ema_dist: [-0.1, 0.01] + eqty_exp_base: [0.9, 4.0] + eprice_exp_base: [0.9, 4.0] + min_markup: [0.001, 0.003] + markup_range: [0.0, 0.04] + n_close_orders: [7, 30] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.0, 0.02] } short: { - grid_span: [0.01, 0.7] - ema_span_0: [120, 4320] - ema_span_1: [120, 10080] + grid_span: [0.34, 0.7] + ema_span_0: [240, 4320] + ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] - max_n_entry_orders: [5, 30] + max_n_entry_orders: [7, 30] initial_qty_pct: [0.015, 0.015] - initial_eprice_ema_dist: [-0.1, 0.0] - eqty_exp_base: [0.1, 5.0] - eprice_exp_base: [0.1, 5.0] - min_markup: [0.0015, 0.01] - markup_range: [0.0, 0.06] - n_close_orders: [5, 30] - auto_unstuck_wallet_exposure_threshold: [0.0, 0.1] - auto_unstuck_ema_dist: [0.0, 0.04] + initial_eprice_ema_dist: [-0.1, 0.01] + eqty_exp_base: [0.9, 4.0] + eprice_exp_base: [0.9, 4.0] + min_markup: [0.001, 0.003] + markup_range: [0.0, 0.04] + n_close_orders: [7, 30] + auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] + auto_unstuck_ema_dist: [0.0, 0.02] } } } From f0c42fc27504d14f87cdb838540e20bb380b7c60 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 2 Nov 2022 15:48:45 +0100 Subject: [PATCH 04/27] update inspect opt results to match new score logic --- inspect_opt_results.py | 282 +++++++++++++++-------------------------- 1 file changed, 100 insertions(+), 182 deletions(-) diff --git a/inspect_opt_results.py b/inspect_opt_results.py index 7e3ec67d3..41b2c8fd7 100755 --- a/inspect_opt_results.py +++ b/inspect_opt_results.py @@ -6,45 +6,46 @@ import json import pprint import numpy as np +from prettytable import PrettyTable import argparse import hjson from procedures import load_live_config, dump_live_config, make_get_filepath from pure_funcs import config_pretty_str, candidate_to_live_config +from njit_funcs import round_dynamic def main(): parser = argparse.ArgumentParser(prog="view conf", description="inspect conf") parser.add_argument("results_fpath", type=str, help="path to results file") + weights_keys = [ + ("psl", "maximum_pa_distance_std_long"), + ("pss", "maximum_pa_distance_std_short"), + ("pml", "maximum_pa_distance_mean_long"), + ("pms", "maximum_pa_distance_mean_short"), + ("pll", "maximum_loss_profit_ratio_long"), + ("pls", "maximum_loss_profit_ratio_short"), + ("hsl", "maximum_hrs_stuck_max_long"), + ("hss", "maximum_hrs_stuck_max_short"), + ] + for k0, k1 in weights_keys: + parser.add_argument( + f"-{k0}", + f"--{k1}", + dest=k1, + type=float, + required=False, + default=None, + help=f"max {k1}", + ) parser.add_argument( - "-p", - "--PAD", - "--pad", - dest="PAD_max", - type=float, - required=False, - default=None, - help="max pa dist", - ) - parser.add_argument( - "-l", - "--profit_loss_ratio", - dest="profit_loss_ratio_max", - type=float, - required=False, - default=None, - help="max profit loss ratio", - ) - parser.add_argument( - "-i", "--index", dest="index", type=int, required=False, default=1, help="best conf index" - ) - parser.add_argument( - "-sf", - dest="score_formula", - type=str, + "-i", + "--index", + dest="index", + type=int, required=False, - default="adgrealizedPADstd", - help="choices: [adgPADstd, adg_mean, adg_min, adgPADmean, adgrealizedPADmean, adgrealizedPADstd]", + default=0, + help="best conf index, default=0", ) parser.add_argument( "-d", @@ -55,178 +56,95 @@ def main(): args = parser.parse_args() + # attempt guessing whether harmony search or particle swarm opt_config_path = ( "configs/optimize/harmony_search.hjson" if "harmony" in args.results_fpath else "configs/optimize/particle_swarm_optimization.hjson" ) - if args.PAD_max is None: - opt_config = hjson.load(open(opt_config_path)) - PAD_max_long = opt_config["maximum_pa_distance_std_long"] - PAD_max_short = opt_config["maximum_pa_distance_std_short"] - else: - PAD_max_long = args.PAD_max - PAD_max_short = args.PAD_max + opt_config = hjson.load(open(opt_config_path)) + maximums = {} + for _, k1 in weights_keys: + maximums[k1] = opt_config[k1] if getattr(args, k1) is None else getattr(args, k1) + klen = max([len(k) for k in maximums]) + for k, v in maximums.items(): + print(f"{k: <{klen}} {v}") - if args.profit_loss_ratio_max is None: - opt_config = hjson.load(open(opt_config_path)) - maximum_loss_profit_ratio_long = opt_config["maximum_loss_profit_ratio_long"] - maximum_loss_profit_ratio_short = opt_config["maximum_loss_profit_ratio_short"] - else: - maximum_loss_profit_ratio_long = args.profit_loss_ratio_max - maximum_loss_profit_ratio_short = args.profit_loss_ratio_max - - keys_ = [ - "adg", - "adg_realized_per_exposure", - "pa_distance_mean", - "pa_distance_std", - "loss_profit_ratio", - "profit_sum", - "loss_sum", - ] - keys = [k + "_long" for k in keys_] + [k + "_short" for k in keys_] with open(args.results_fpath) as f: - """ - results = [] - for x in f.readlines(): - j = json.loads(x) - rsl = { - "results": { - s: {k: j["results"][s][k] for k in keys if k in j["results"][s]} - for s in j["results"] - if s != "config_no" - } - } - rsl["results"]["config_no"] = j["results"]["config_no"] - rsl["config"] = j["config"] - results.append(rsl) - """ results = [json.loads(x) for x in f.readlines()] + print(f"{'n results': <{klen}} {len(results)}") - print( - "n results", - len(results), - f"score formula: {args.score_formula}, PAD max:", - [PAD_max_long, PAD_max_short], - f"loss_profit_ratio max:", - [maximum_loss_profit_ratio_long, maximum_loss_profit_ratio_short], - ) - best_config = {} - adgs_sides = {"long": {}, "short": {}} - PAD_max = {"long": PAD_max_long, "short": PAD_max_short} - maximum_loss_profit_ratio = { - "long": maximum_loss_profit_ratio_long, - "short": maximum_loss_profit_ratio_short, - } sides = ["long", "short"] - for side in sides: - stats = [] - for r in results: - adgs, adgs_realized, PAD_stds, PAD_means, loss_profit_ratios = [], [], [], [], [] - for s in (rs := r["results"]): - try: - adgs.append(rs[s][f"adg_{side}"]) - adgs_realized.append(rs[s][f"adg_realized_per_exposure_{side}"]) - PAD_stds.append(rs[s][f"pa_distance_std_{side}"]) - PAD_means.append(rs[s][f"pa_distance_mean_{side}"]) - if f"loss_profit_ratio_{side}" in rs[s]: - loss_profit_ratios.append(rs[s][f"loss_profit_ratio_{side}"]) - else: - # for backwards compatibility - lpr = ( - (abs(rs[s][f"loss_sum_{side}"]) / rs[s][f"profit_sum_{side}"]) - if rs[s][f"profit_sum_{side}"] - else 1.0 - ) - rs[s][f"loss_profit_ratio_{side}"] = lpr - loss_profit_ratios.append(lpr) - except Exception as e: - pass - adg_mean = np.mean(adgs) - adg_realized_mean = np.mean(adgs_realized) - PAD_std_mean_raw = np.mean(PAD_stds) - PAD_std_mean = np.mean([max(PAD_max[side], x) for x in PAD_stds]) - PAD_mean_mean_raw = np.mean(PAD_means) - PAD_mean_mean = np.mean([max(PAD_max[side], x) for x in PAD_means]) - loss_profit_ratio_mean = np.mean( - [max(maximum_loss_profit_ratio[side], x) for x in loss_profit_ratios] - ) - loss_profit_ratio_mean_raw = np.mean(loss_profit_ratios) - if args.score_formula.lower() == "adgpadstd": - score = adg_mean / max(PAD_max[side], PAD_std_mean) - elif args.score_formula.lower() == "adg_mean": - score = adg_mean - elif args.score_formula.lower() == "adg_min": - score = min(adgs) - elif args.score_formula.lower() == "adgpadmean": - score = adg_mean * min(1, PAD_max[side] / PAD_mean_mean) - elif args.score_formula.lower() == "adgrealizedpadmean": - score = adg_realized_mean / max(PAD_max[side], PAD_mean_mean) - elif args.score_formula.lower() == "adgrealizedpadstd": - score = adg_realized_mean / max(PAD_max[side], PAD_std_mean) - else: - raise Exception("unknown score formula") - - score *= maximum_loss_profit_ratio[side] / max( - loss_profit_ratio_mean, maximum_loss_profit_ratio[side] - ) + keys = [ + ("adg_realized_per_exposure", True), + ("pa_distance_std", False), + ("pa_distance_mean", False), + ("hrs_stuck_max", False), + ("loss_profit_ratio", False), + ] + all_scores = [] + symbols = [s for s in results[0]["results"] if s != "config_no"] + for r in results: + cfg = r["config"] + ress = r["results"] - stats.append( - { - "config": r["config"], - "adg_mean": adg_mean, - "adg_realized_mean": adg_realized_mean, - "PAD_std_mean": PAD_std_mean, - "PAD_std_mean_raw": PAD_std_mean_raw, - "PAD_mean_mean": PAD_mean_mean, - "PAD_mean_mean_raw": PAD_mean_mean_raw, - "loss_profit_ratio_mean_raw": loss_profit_ratio_mean_raw, - "score": score, - "config_no": r["results"]["config_no"], - } - ) - ss = sorted(stats, key=lambda x: x["score"]) - bc = ss[-args.index] - best_config[side] = bc["config"][side] - for r in results: - if r["results"]["config_no"] == bc["config_no"]: - rs = r["results"] - syms = [s for s in rs if "config" not in s] - print(f"results {side} best config no {bc['config_no']}") - print("symbol adg PADmean PADstd lp_ratio score") - for s in sorted(syms, key=lambda x: rs[x][f"adg_realized_per_exposure_{side}"]): - adgs_sides[side][s] = rs[s][f"adg_realized_per_exposure_{side}"] - print( - f"{s: <20} {rs[s][f'adg_realized_per_exposure_{side}']:.6f} " - + f"{rs[s][f'pa_distance_std_{side}']:.6f} {rs[s][f'pa_distance_mean_{side}']:.6f} " - + f"{rs[s][f'loss_profit_ratio_{side}']:.6f} " - + f"{bc['score']:.6f}" - ) - print( - f"{'means': <20} {bc['adg_realized_mean']:.6f} " - + f"{bc['PAD_std_mean_raw']:.6f} " - + f"{bc['PAD_mean_mean_raw']:.6f} " - + f"{bc['loss_profit_ratio_mean_raw']:.6f} " - ) - bcstats = r - other_side = [x for x in sides if x != side][0] - if len(r["results"]) < 3: - for symbol in r["results"]: - if type(r["results"][symbol]) == dict: - pprint.pprint( - {k: v for k, v in r["results"][symbol].items() if other_side not in k} - ) + means = {s: {} for s in sides} # adjusted means + scores = {s: -1.0 for s in sides} + raws = {s: {} for s in sides} # unadjusted means + all_scores.append({}) + for side in sides: + for key, mult in keys: + max_key = f"maximum_{key}_{side}" + raws[side][key] = np.mean([ress[s][f"{key}_{side}"] for s in symbols]) + if max_key in maximums: + if maximums[max_key] >= 0.0: + ms = [max(maximums[max_key], ress[s][f"{key}_{side}"]) for s in symbols] + means[side][key] = max(maximums[max_key], np.mean(ms)) + else: + means[side][key] = 1.0 + else: + means[side][key] = np.mean([ress[s][f"{key}_{side}"] for s in symbols]) + if mult: + scores[side] *= means[side][key] + else: + scores[side] /= means[side][key] + all_scores[-1][side] = { + "config": cfg[side], + "score": scores[side], + "stats": {s: {k: v for k, v in ress[s].items() if side in k} for s in symbols}, + } + best_candidate = {} + for side in sides: + scoress = sorted([sc[side] for sc in all_scores], key=lambda x: x["score"]) + best_candidate[side] = scoress[args.index] + best_config = { + "long": best_candidate["long"]["config"], + "short": best_candidate["short"]["config"], + } + for side in sides: + row_headers = ["symbol"] + [k[0] for k in keys] + table = PrettyTable(row_headers) + for rh in row_headers: + table.align[rh] = "l" + table.title = side + for s in sorted( + symbols, + key=lambda x: best_candidate[side]["stats"][x][f"adg_realized_per_exposure_{side}"], + ): + xs = [best_candidate[side]["stats"][s][f"{k[0]}_{side}"] for k in keys] + table.add_row([s] + [round_dynamic(x, 4) for x in xs]) + means = [ + np.mean([best_candidate[side]["stats"][s_][f"{k[0]}_{side}"] for s_ in symbols]) + for k in keys + ] + table.add_row(["mean"] + [round_dynamic(m, 4) for m in means]) + print(table) live_config = candidate_to_live_config(best_config) if args.dump_live_config: lc_fpath = make_get_filepath(f"{args.results_fpath.replace('.txt', '_best_config.json')}") print(f"dump_live_config {lc_fpath}") dump_live_config(live_config, lc_fpath) - adgs_sums = {s: adgs_sides["long"][s] + adgs_sides["short"][s] for s in adgs_sides["long"]} - print("\nsum adgs") - for k, v in sorted(adgs_sums.items(), key=lambda x: x[1]): - print(f"{k: <12} {v:.6f}") print(config_pretty_str(live_config)) From 6b93de2bd3516b2b1142c711365eb3038cbd6db1 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 2 Nov 2022 15:49:27 +0100 Subject: [PATCH 05/27] fix typo --- bitget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitget.py b/bitget.py index 677912132..0eb099e6c 100644 --- a/bitget.py +++ b/bitget.py @@ -697,7 +697,7 @@ async def subscribe_to_user_stream(self, ws): print(res) async def transfer(self, type_: str, amount: float, asset: str = "USDT"): - return {"code": "-1", "msg": "Transferring funds not supported for Bybit"} + return {"code": "-1", "msg": "Transferring funds not supported for Bitget"} def standardize_user_stream_event( self, event: Union[List[Dict], Dict] From 7c180462c3845edce46c7e3f2682bc093ed32100 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 2 Nov 2022 15:50:03 +0100 Subject: [PATCH 06/27] small aesthetic change --- particle_swarm_optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/particle_swarm_optimization.py b/particle_swarm_optimization.py index e7fe11be5..b3dd0d81e 100644 --- a/particle_swarm_optimization.py +++ b/particle_swarm_optimization.py @@ -198,7 +198,7 @@ def post_process(self, wi: int): max(self.config[max_key], v[f"{key}_{side}"]) for v in results.values() ] - means[side][key] = max(np.mean(ms), self.config[max_key]) + means[side][key] = max(self.config[max_key], np.mean(ms)) else: means[side][key] = 1.0 else: From 4f400b89bf31899edb0ecfa765bfb97e6cf3dcb1 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 4 Nov 2022 12:01:42 +0100 Subject: [PATCH 07/27] updated neat AU enabled default live config --- ...rid_mode_auto_unstuck_enabled.example.json | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/configs/live/neat_grid_mode_auto_unstuck_enabled.example.json b/configs/live/neat_grid_mode_auto_unstuck_enabled.example.json index 84ab020ee..4e1b11376 100644 --- a/configs/live/neat_grid_mode_auto_unstuck_enabled.example.json +++ b/configs/live/neat_grid_mode_auto_unstuck_enabled.example.json @@ -1,35 +1,34 @@ -{"config_name": "neat_grid_118_symbols_501days", +{"config_name": "neat_grid_118_symbols_626days", "logging_level": 0, - "long": {"auto_unstuck_ema_dist": 0.01880226832240887, - "auto_unstuck_wallet_exposure_threshold": 0.09121829413942732, + "long": {"auto_unstuck_ema_dist": 0, + "auto_unstuck_wallet_exposure_threshold": 0.012195009245955374, "backwards_tp": true, - "ema_span_0": 3831.4946575149174, - "ema_span_1": 2480.1173686640755, + "ema_span_0": 240, + "ema_span_1": 307.0566300659701, "enabled": true, - "eprice_exp_base": 0.5904755088660214, - "eqty_exp_base": 0.9508455449906771, - "grid_span": 0.32197240512816766, - "initial_eprice_ema_dist": -0.009924751401749293, + "eprice_exp_base": 0.9814428052289045, + "eqty_exp_base": 1.8910056941476132, + "grid_span": 0.35120092600982644, + "initial_eprice_ema_dist": -0.0724883315452062, "initial_qty_pct": 0.015, - "markup_range": 0.01194525537178334, - "max_n_entry_orders": 6, - "min_markup": 0.00840900691016795, - "n_close_orders": 28, + "markup_range": 0.029915290808625504, + "max_n_entry_orders": 9, + "min_markup": 0.003, + "n_close_orders": 10, "wallet_exposure_limit": 0.1}, - "short": {"auto_unstuck_ema_dist": 0, - "auto_unstuck_wallet_exposure_threshold": 0.019161472858399167, + "short": {"auto_unstuck_ema_dist": 0.02, + "auto_unstuck_wallet_exposure_threshold": 0.010010044896137589, "backwards_tp": true, - "ema_span_0": 243.45482492298726, - "ema_span_1": 472.86240623590015, + "ema_span_0": 3578.5992758249126, + "ema_span_1": 1300.2248624251254, "enabled": true, - "eprice_exp_base": 0.5518227984685108, - "eqty_exp_base": 0.519995842628588, - "grid_span": 0.39431042562399177, - "initial_eprice_ema_dist": -0.056160235600267956, + "eprice_exp_base": 0.9, + "eqty_exp_base": 2.741199913514829, + "grid_span": 0.35422351795434553, + "initial_eprice_ema_dist": 0.005310285956060753, "initial_qty_pct": 0.015, - "markup_range": 0.017177998232144262, - "max_n_entry_orders": 5, - "min_markup": 0.005455977824072189, - "n_close_orders": 16, + "markup_range": 0.011750423363748088, + "max_n_entry_orders": 7, + "min_markup": 0.003, + "n_close_orders": 8, "wallet_exposure_limit": 0.1}} - From f0e082a2f364b81d339124acd9189136cd081f4b Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 7 Nov 2022 19:03:05 +0100 Subject: [PATCH 08/27] bug fix bitget user stream --- passivbot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/passivbot.py b/passivbot.py index d977fc08d..22b0d8a7a 100644 --- a/passivbot.py +++ b/passivbot.py @@ -887,6 +887,9 @@ async def on_user_stream_events(self, events: Union[List[Dict], List]) -> None: async def on_user_stream_event(self, event: dict) -> None: try: + if "logged_in" in event: + # bitget needs to login before sending subscribe requests + await self.subscribe_to_user_stream(self.ws_user) pos_change = False if "wallet_balance" in event: new_wallet_balance = self.adjust_wallet_balance(event["wallet_balance"]) @@ -969,9 +972,9 @@ async def start_websocket(self) -> None: await self.init_exchange_config() await self.init_order_book() await self.init_emas() + logging.info("starting websockets...") self.user_stream_task = asyncio.create_task(self.start_websocket_user_stream()) self.market_stream_task = asyncio.create_task(self.start_websocket_market_stream()) - logging.info("starting websockets...") await asyncio.gather(self.user_stream_task, self.market_stream_task) async def beat_heart_user_stream(self) -> None: From 4fd9e065709ab3f10d3b681000688d208f46374f Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 7 Nov 2022 19:03:41 +0100 Subject: [PATCH 09/27] bug fix bitget user stream --- bitget.py | 55 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/bitget.py b/bitget.py index 0eb099e6c..221503a1b 100644 --- a/bitget.py +++ b/bitget.py @@ -30,6 +30,7 @@ def truncate_float(x: float, d: int) -> float: class BitgetBot(Bot): def __init__(self, config: dict): + self.is_logged_into_user_stream = False self.exchange = "bitget" super().__init__(config) self.base_endpoint = "https://api.bitget.com" @@ -610,7 +611,7 @@ async def beat_heart_market_stream(self) -> None: print_(["error sending heartbeat market", e]) async def subscribe_to_market_stream(self, ws): - await ws.send( + res = await ws.send( json.dumps( { "op": "subscribe", @@ -625,25 +626,16 @@ async def subscribe_to_market_stream(self, ws): ) ) - async def subscribe_to_user_stream(self, ws): - timestamp = int(time()) - signature = base64.b64encode( - hmac.new( - self.secret.encode("utf-8"), - f"{timestamp}GET/user/verify".encode("utf-8"), - digestmod="sha256", - ).digest() - ).decode("utf-8") + async def subscribe_to_user_streams(self, ws): res = await ws.send( json.dumps( { - "op": "login", + "op": "subscribe", "args": [ { - "apiKey": self.key, - "passphrase": self.passphrase, - "timestamp": timestamp, - "sign": signature, + "instType": self.product_type.upper(), + "channel": "account", + "instId": "default", } ], } @@ -657,7 +649,7 @@ async def subscribe_to_user_stream(self, ws): "args": [ { "instType": self.product_type.upper(), - "channel": "account", + "channel": "positions", "instId": "default", } ], @@ -671,8 +663,8 @@ async def subscribe_to_user_stream(self, ws): "op": "subscribe", "args": [ { + "channel": "orders", "instType": self.product_type.upper(), - "channel": "positions", "instId": "default", } ], @@ -680,15 +672,32 @@ async def subscribe_to_user_stream(self, ws): ) ) print(res) + + async def subscribe_to_user_stream(self, ws): + if self.is_logged_into_user_stream: + await self.subscribe_to_user_streams(ws) + else: + await self.login_to_user_stream(ws) + + async def login_to_user_stream(self, ws): + timestamp = int(time()) + signature = base64.b64encode( + hmac.new( + self.secret.encode("utf-8"), + f"{timestamp}GET/user/verify".encode("utf-8"), + digestmod="sha256", + ).digest() + ).decode("utf-8") res = await ws.send( json.dumps( { - "op": "subscribe", + "op": "login", "args": [ { - "channel": "orders", - "instType": self.product_type.upper(), - "instId": "default", + "apiKey": self.key, + "passphrase": self.passphrase, + "timestamp": timestamp, + "sign": signature, } ], } @@ -704,6 +713,10 @@ def standardize_user_stream_event( ) -> Union[List[Dict], Dict]: events = [] + if "event" in event and event["event"] == "login": + self.is_logged_into_user_stream = True + return {'logged_in': True} + # print('debug 0', event) if "arg" in event and "data" in event and "channel" in event["arg"]: if event["arg"]["channel"] == "orders": for elm in event["data"]: From b298dbac5d96bac9e921b0aab9981c9ff5352b92 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 8 Nov 2022 00:00:06 +0100 Subject: [PATCH 10/27] update setting leverage upon startup --- bybit.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/bybit.py b/bybit.py index 460cad266..92a261c01 100644 --- a/bybit.py +++ b/bybit.py @@ -549,7 +549,6 @@ async def fetch_fills( except Exception as e: print("error fetching fills", e) return [] - print("ntufnt") return fetched print("fetch_fills not implemented for Bybit") return [] @@ -563,18 +562,17 @@ async def init_exchange_config(self): "/futures/private/position/leverage/save", { "symbol": self.symbol, - "position_idx": 1, - "buy_leverage": 0, - "sell_leverage": 0, + "buy_leverage": 7, + "sell_leverage": 7, }, ), self.private_post( - "/futures/private/position/leverage/save", + "/futures/private/position/switch-isolated", { "symbol": self.symbol, - "position_idx": 2, - "buy_leverage": 0, - "sell_leverage": 0, + "is_isolated": False, + "buy_leverage": 7, + "sell_leverage": 7, }, ), ) @@ -601,12 +599,17 @@ async def init_exchange_config(self): ) print(res) elif "inverse_perpetual" in self.market_type: + res = await self.private_post( + "/v2/private/position/switch-isolated", + {"symbol": self.symbol, "is_isolated": False, + "buy_leverage": 7, "sell_leverage": 7}, + ) + print('1', res) res = await self.private_post( "/v2/private/position/leverage/save", - {"symbol": self.symbol, "leverage": 0}, + {"symbol": self.symbol, "leverage": 7, "leverage_only": True}, ) - - print(res) + print('2', res) except Exception as e: print(e) From 46dbec4ba7129c4f187122a9c181aeb3a44e1883 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 8 Nov 2022 12:47:58 +0100 Subject: [PATCH 11/27] add eqbal_ratio_min metric to score func --- configs/optimize/harmony_search.hjson | 4 +++ .../particle_swarm_optimization.hjson | 4 +++ harmony_search.py | 13 ++++++++-- inspect_opt_results.py | 26 ++++++++++++------- particle_swarm_optimization.py | 16 +++++++++--- 5 files changed, 48 insertions(+), 15 deletions(-) diff --git a/configs/optimize/harmony_search.hjson b/configs/optimize/harmony_search.hjson index c8c4660aa..448e0d546 100644 --- a/configs/optimize/harmony_search.hjson +++ b/configs/optimize/harmony_search.hjson @@ -29,6 +29,10 @@ maximum_hrs_stuck_max_long: -1 maximum_hrs_stuck_max_short: -1 + # score /= min(minimum_eqbal_ratio_min, eqbal_ratio_min) + minimum_eqbal_ratio_min_long: -1 + minimum_eqbal_ratio_min_short: -1 + # will override starting configs' parameters do_long: true do_short: true diff --git a/configs/optimize/particle_swarm_optimization.hjson b/configs/optimize/particle_swarm_optimization.hjson index d6ab803b2..a1d38c9ef 100644 --- a/configs/optimize/particle_swarm_optimization.hjson +++ b/configs/optimize/particle_swarm_optimization.hjson @@ -29,6 +29,10 @@ maximum_hrs_stuck_max_long: -1 maximum_hrs_stuck_max_short: -1 + # score /= min(minimum_eqbal_ratio_min, eqbal_ratio_min) + minimum_eqbal_ratio_min_long: -1 + minimum_eqbal_ratio_min_short: -1 + # will override starting configs' parameters do_long: true do_short: true diff --git a/harmony_search.py b/harmony_search.py index bf21d4881..5d081a090 100644 --- a/harmony_search.py +++ b/harmony_search.py @@ -169,15 +169,15 @@ def post_process(self, wi: int): ("pa_distance_mean", False), ("hrs_stuck_max", False), ("loss_profit_ratio", False), + ("eqbal_ratio_min", True), ] means = {s: {} for s in sides} # adjusted means scores = {s: -1.0 for s in sides} raws = {s: {} for s in sides} # unadjusted means for side in sides: for key, mult in keys: - max_key = f"maximum_{key}_{side}" raws[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) - if max_key in self.config: + if (max_key := f"maximum_{key}_{side}") in self.config: if self.config[max_key] >= 0.0: ms = [ max(self.config[max_key], v[f"{key}_{side}"]) @@ -186,6 +186,15 @@ def post_process(self, wi: int): means[side][key] = max(np.mean(ms), self.config[max_key]) else: means[side][key] = 1.0 + elif (min_key := f"minimum_{key}_{side}") in self.config: + if self.config[min_key] >= 0.0: + ms = [ + min(self.config[min_key], v[f"{key}_{side}"]) + for v in results.values() + ] + means[side][key] = min(np.mean(ms), self.config[min_key]) + else: + means[side][key] = 1.0 else: means[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) if mult: diff --git a/inspect_opt_results.py b/inspect_opt_results.py index 41b2c8fd7..623e68966 100755 --- a/inspect_opt_results.py +++ b/inspect_opt_results.py @@ -27,6 +27,8 @@ def main(): ("pls", "maximum_loss_profit_ratio_short"), ("hsl", "maximum_hrs_stuck_max_long"), ("hss", "maximum_hrs_stuck_max_short"), + ("erl", "minimum_eqbal_ratio_min_long"), + ("ers", "minimum_eqbal_ratio_min_short"), ] for k0, k1 in weights_keys: parser.add_argument( @@ -64,11 +66,11 @@ def main(): ) opt_config = hjson.load(open(opt_config_path)) - maximums = {} + minsmaxs = {} for _, k1 in weights_keys: - maximums[k1] = opt_config[k1] if getattr(args, k1) is None else getattr(args, k1) - klen = max([len(k) for k in maximums]) - for k, v in maximums.items(): + minsmaxs[k1] = opt_config[k1] if getattr(args, k1) is None else getattr(args, k1) + klen = max([len(k) for k in minsmaxs]) + for k, v in minsmaxs.items(): print(f"{k: <{klen}} {v}") with open(args.results_fpath) as f: @@ -82,6 +84,7 @@ def main(): ("pa_distance_mean", False), ("hrs_stuck_max", False), ("loss_profit_ratio", False), + ("eqbal_ratio_min", True), ] all_scores = [] symbols = [s for s in results[0]["results"] if s != "config_no"] @@ -95,12 +98,17 @@ def main(): all_scores.append({}) for side in sides: for key, mult in keys: - max_key = f"maximum_{key}_{side}" raws[side][key] = np.mean([ress[s][f"{key}_{side}"] for s in symbols]) - if max_key in maximums: - if maximums[max_key] >= 0.0: - ms = [max(maximums[max_key], ress[s][f"{key}_{side}"]) for s in symbols] - means[side][key] = max(maximums[max_key], np.mean(ms)) + if (max_key := f"maximum_{key}_{side}") in minsmaxs: + if minsmaxs[max_key] >= 0.0: + ms = [max(minsmaxs[max_key], ress[s][f"{key}_{side}"]) for s in symbols] + means[side][key] = max(minsmaxs[max_key], np.mean(ms)) + else: + means[side][key] = 1.0 + elif (min_key := f"maximum_{key}_{side}") in minsmaxs: + if minsmaxs[min_key] >= 0.0: + ms = [min(minsmaxs[min_key], ress[s][f"{key}_{side}"]) for s in symbols] + means[side][key] = min(minsmaxs[min_key], np.mean(ms)) else: means[side][key] = 1.0 else: diff --git a/particle_swarm_optimization.py b/particle_swarm_optimization.py index b3dd0d81e..57a69d763 100644 --- a/particle_swarm_optimization.py +++ b/particle_swarm_optimization.py @@ -176,7 +176,6 @@ def post_process(self, wi: int): ) if set(results) == set(self.symbols): # completed multisymbol iter - sides = ["long", "short"] keys = [ ("adg_realized_per_exposure", True), @@ -184,21 +183,30 @@ def post_process(self, wi: int): ("pa_distance_mean", False), ("hrs_stuck_max", False), ("loss_profit_ratio", False), + ("eqbal_ratio_min", True), ] means = {s: {} for s in sides} # adjusted means scores = {s: -1.0 for s in sides} raws = {s: {} for s in sides} # unadjusted means for side in sides: for key, mult in keys: - max_key = f"maximum_{key}_{side}" raws[side][key] = np.mean([v[f"{key}_{side}"] for v in results.values()]) - if max_key in self.config: + if (max_key := f"maximum_{key}_{side}") in self.config: if self.config[max_key] >= 0.0: ms = [ max(self.config[max_key], v[f"{key}_{side}"]) for v in results.values() ] - means[side][key] = max(self.config[max_key], np.mean(ms)) + means[side][key] = max(np.mean(ms), self.config[max_key]) + else: + means[side][key] = 1.0 + elif (min_key := f"minimum_{key}_{side}") in self.config: + if self.config[min_key] >= 0.0: + ms = [ + min(self.config[min_key], v[f"{key}_{side}"]) + for v in results.values() + ] + means[side][key] = min(np.mean(ms), self.config[min_key]) else: means[side][key] = 1.0 else: From 60d3d32f99150283b70e0133881e5a3b56f952a2 Mon Sep 17 00:00:00 2001 From: enarjord Date: Sat, 12 Nov 2022 16:28:42 +0100 Subject: [PATCH 12/27] set leverage by kwarg neater error msg when failing order cancellation minor refactoring --- binance.py | 13 ++++++++----- bybit.py | 30 +++++++++++++++++------------- passivbot.py | 33 +++++++++++++++++++++++---------- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/binance.py b/binance.py index 2641f204c..a16744944 100644 --- a/binance.py +++ b/binance.py @@ -9,7 +9,7 @@ import aiohttp import numpy as np -from passivbot import Bot +from passivbot import Bot, logging from procedures import print_, print_async_exception from pure_funcs import ts_to_date, sort_dict_keys, format_float @@ -209,7 +209,7 @@ async def transfer_from_derivatives_to_spot(self, coin: str, amount: float): ) async def execute_leverage_change(self): - lev = 7 # arbitrary + lev = self.leverage return await self.private_post( self.endpoints["leverage"], {"symbol": self.symbol, "leverage": lev} ) @@ -362,9 +362,12 @@ async def execute_cancellation(self, order: dict) -> dict: "price": float(cancellation["price"]), } except Exception as e: - print(f"error cancelling order {order} {e}") - print_async_exception(cancellation) - traceback.print_exc() + if cancellation is not None and "code" in cancellation and cancellation["code"] == -2011: + logging.error(f"error cancelling order {cancellation} {order}") # neater error message + else: + print(f"error cancelling order {order} {e}") + print_async_exception(cancellation) + traceback.print_exc() self.ts_released["force_update"] = 0.0 return {} diff --git a/bybit.py b/bybit.py index 92a261c01..2e2d539e4 100644 --- a/bybit.py +++ b/bybit.py @@ -12,7 +12,7 @@ import numpy as np from njit_funcs import round_ -from passivbot import Bot +from passivbot import Bot, logging from procedures import print_async_exception, print_ from pure_funcs import ts_to_date, sort_dict_keys, date_to_ts @@ -324,9 +324,13 @@ async def execute_cancellation(self, order: dict) -> dict: "price": order["price"], } except Exception as e: - print(f"error cancelling order {order} {e}") - print_async_exception(cancellation) - traceback.print_exc() + if cancellation is not None and "ret_code" in cancellation and cancellation["ret_code"] == 20001: + error_cropped = {k: v for k, v in cancellation.items() if k in ["ret_msg", "ret_code"]} + logging.error(f"error cancelling order {error_cropped} {order}") # neater error message + else: + print(f"error cancelling order {order} {e}") + print_async_exception(cancellation) + traceback.print_exc() self.ts_released["force_update"] = 0.0 return {} @@ -562,8 +566,8 @@ async def init_exchange_config(self): "/futures/private/position/leverage/save", { "symbol": self.symbol, - "buy_leverage": 7, - "sell_leverage": 7, + "buy_leverage": self.leverage, + "sell_leverage": self.leverage, }, ), self.private_post( @@ -571,8 +575,8 @@ async def init_exchange_config(self): { "symbol": self.symbol, "is_isolated": False, - "buy_leverage": 7, - "sell_leverage": 7, + "buy_leverage": self.leverage, + "sell_leverage": self.leverage, }, ), ) @@ -588,26 +592,26 @@ async def init_exchange_config(self): { "symbol": self.symbol, "is_isolated": False, - "buy_leverage": 7, - "sell_leverage": 7, + "buy_leverage": self.leverage, + "sell_leverage": self.leverage, }, ) print(res) res = await self.private_post( "/private/linear/position/set-leverage", - {"symbol": self.symbol, "buy_leverage": 7, "sell_leverage": 7}, + {"symbol": self.symbol, "buy_leverage": self.leverage, "sell_leverage": self.leverage}, ) print(res) elif "inverse_perpetual" in self.market_type: res = await self.private_post( "/v2/private/position/switch-isolated", {"symbol": self.symbol, "is_isolated": False, - "buy_leverage": 7, "sell_leverage": 7}, + "buy_leverage": self.leverage, "sell_leverage": self.leverage}, ) print('1', res) res = await self.private_post( "/v2/private/position/leverage/save", - {"symbol": self.symbol, "leverage": 7, "leverage_only": True}, + {"symbol": self.symbol, "leverage": self.leverage, "leverage_only": True}, ) print('2', res) except Exception as e: diff --git a/passivbot.py b/passivbot.py index 22b0d8a7a..5a9ea32d7 100644 --- a/passivbot.py +++ b/passivbot.py @@ -16,6 +16,7 @@ make_get_filepath, load_exchange_key_secret_passphrase, numpyize, + print_async_exception, ) from pure_funcs import ( filter_orders, @@ -390,11 +391,9 @@ async def cancel_orders(self, orders_to_cancel: [dict]) -> [dict]: self.open_orders = [ oo for oo in self.open_orders if oo["order_id"] != o["order_id"] ] - - else: - logging.error(f"error cancelling order {o}") except Exception as e: - logging.error(f"error cancelling order {oc} {c.exception()} {e}") + logging.error(f"error cancelling order {oc} {e}") + print_async_exception(c) return cancelled_orders finally: self.ts_released["cancel_orders"] = time() @@ -1127,6 +1126,15 @@ async def main() -> None: default="api-keys.json", help="File containing users/accounts and api-keys for each exchange", ) + parser.add_argument( + "-lev", + "--leverage", + type=int, + required=False, + dest="leverage", + default=7, + help="Leverage set on exchange, if applicable. Default is 7.", + ) float_kwargs = [ ("-lmm", "--long_min_markup", "--long-min-markup", "long_min_markup"), @@ -1169,20 +1177,21 @@ async def main() -> None: except Exception as e: logging.error(f"{e} failed to load config {args.live_config_path}") return - config["user"] = args.user - config["api_keys"] = args.api_keys config["exchange"] = exchange - config["symbol"] = args.symbol + for k in ["user", "api_keys", "symbol", "leverage", "price_distance_threshold"]: + config[k] = getattr(args, k) config["market_type"] = args.market_type if args.market_type is not None else "futures" config["passivbot_mode"] = determine_passivbot_mode(config) - config["price_distance_threshold"] = args.price_distance_threshold if args.assigned_balance is not None: logging.info(f"assigned balance set to {args.assigned_balance}") config["assigned_balance"] = args.assigned_balance if args.long_mode is None: - if not config["long"]["enabled"]: + if config["long"]["enabled"]: + logging.info("long normal mode") + else: config["long_mode"] = "manual" + logging.info("long manual mode enabled; will neither cancel nor create long orders") else: if args.long_mode in ["gs", "graceful_stop", "graceful-stop"]: logging.info( @@ -1202,9 +1211,13 @@ async def main() -> None: elif args.long_mode.lower() in ["t", "tp_only", "tp-only"]: logging.info("long tp only mode enabled") config["long_mode"] = "tp_only" + if args.short_mode is None: - if not config["short"]["enabled"]: + if config["short"]["enabled"]: + logging.info("short normal mode") + else: config["short_mode"] = "manual" + logging.info("short manual mode enabled; will neither cancel nor create short orders") else: if args.short_mode in ["gs", "graceful_stop", "graceful-stop"]: logging.info( From 1b6f200c131ef00142706c547197557453f2a022 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 16 Nov 2022 12:06:09 +0100 Subject: [PATCH 13/27] add 'config no.' to table title --- inspect_opt_results.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inspect_opt_results.py b/inspect_opt_results.py index 623e68966..1a9b8538b 100755 --- a/inspect_opt_results.py +++ b/inspect_opt_results.py @@ -121,6 +121,7 @@ def main(): "config": cfg[side], "score": scores[side], "stats": {s: {k: v for k, v in ress[s].items() if side in k} for s in symbols}, + "config_no": ress["config_no"], } best_candidate = {} for side in sides: @@ -135,7 +136,7 @@ def main(): table = PrettyTable(row_headers) for rh in row_headers: table.align[rh] = "l" - table.title = side + table.title = f"{side} (config no. {best_candidate[side]['config_no']})" for s in sorted( symbols, key=lambda x: best_candidate[side]["stats"][x][f"adg_realized_per_exposure_{side}"], From c986c57adde1c105d636135f87f28c7d82712747 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 16 Nov 2022 12:59:44 +0100 Subject: [PATCH 14/27] bug fix, typo, change 'maximum' to 'minimum' --- inspect_opt_results.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/inspect_opt_results.py b/inspect_opt_results.py index 1a9b8538b..4995722dd 100755 --- a/inspect_opt_results.py +++ b/inspect_opt_results.py @@ -105,7 +105,7 @@ def main(): means[side][key] = max(minsmaxs[max_key], np.mean(ms)) else: means[side][key] = 1.0 - elif (min_key := f"maximum_{key}_{side}") in minsmaxs: + elif (min_key := f"minimum_{key}_{side}") in minsmaxs: if minsmaxs[min_key] >= 0.0: ms = [min(minsmaxs[min_key], ress[s][f"{key}_{side}"]) for s in symbols] means[side][key] = min(minsmaxs[min_key], np.mean(ms)) @@ -136,7 +136,10 @@ def main(): table = PrettyTable(row_headers) for rh in row_headers: table.align[rh] = "l" - table.title = f"{side} (config no. {best_candidate[side]['config_no']})" + table.title = ( + f"{side} (config no. {best_candidate[side]['config_no']}," + + f" score {round_dynamic(best_candidate[side]['score'], 6)})" + ) for s in sorted( symbols, key=lambda x: best_candidate[side]["stats"][x][f"adg_realized_per_exposure_{side}"], From 73d1180f7a5a979814a7215e23a26509400610c9 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 16 Nov 2022 13:07:50 +0100 Subject: [PATCH 15/27] new default recursive example config --- ...rid_mode_auto_unstuck_enabled.example.json | 58 ++++++++++--------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json b/configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json index c75efdba1..058dcabfb 100644 --- a/configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json +++ b/configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json @@ -1,30 +1,32 @@ -{"config_name": "recursive_grid_125_symbols_300days_2022-03-23", +{"config_name": "recursive_grid_118_symbols_683days", "logging_level": 0, - "long": {"auto_unstuck_ema_dist": 0, - "auto_unstuck_wallet_exposure_threshold": 0.07741146038930685, - "ddown_factor": 0.36212132404627734, - "ema_span_0": 1212.4469909346994, - "ema_span_1": 1440, + "long": {"auto_unstuck_ema_dist": 0.0019391572496129435, + "auto_unstuck_wallet_exposure_threshold": 0.044528321840958944, + "backwards_tp": true, + "ddown_factor": 1.224375160419077, + "ema_span_0": 1076.6587443009, + "ema_span_1": 1187.7871298718173, "enabled": true, - "initial_eprice_ema_dist": -0.00048730756130311095, - "initial_qty_pct": 0.011717897308450924, - "markup_range": 0.05658615052935292, - "min_markup": 0.05038435202887167, - "n_close_orders": 11, - "rentry_pprice_dist": 0.03965644731126013, - "rentry_pprice_dist_wallet_exposure_weighting": 5.94156387043507, - "wallet_exposure_limit": 0.05}, - "short": {"auto_unstuck_ema_dist": 0.02360107918060952, - "auto_unstuck_wallet_exposure_threshold": 0.8696017484288251, - "ddown_factor": 2.8578807917477422, - "ema_span_0": 1845.9622772519149, - "ema_span_1": 2220.149357438459, - "enabled": false, - "initial_eprice_ema_dist": -0.08285630558057448, - "initial_qty_pct": 0.030171219081039008, - "markup_range": 0.023949596946105916, - "min_markup": 0.03925744129630296, - "n_close_orders": 5, - "rentry_pprice_dist": 0.024734943307523287, - "rentry_pprice_dist_wallet_exposure_weighting": 54.338880623212155, - "wallet_exposure_limit": 0.05}} + "initial_eprice_ema_dist": 0.007523724212829893, + "initial_qty_pct": 0.015, + "markup_range": 0.007254654150028206, + "min_markup": 0.0030598609160217625, + "n_close_orders": 13, + "rentry_pprice_dist": 0.03863683327619397, + "rentry_pprice_dist_wallet_exposure_weighting": 0.4693269437233695, + "wallet_exposure_limit": 0.1}, + "short": {"auto_unstuck_ema_dist": 0.002009296387350128, + "auto_unstuck_wallet_exposure_threshold": 0.08002480908575456, + "backwards_tp": true, + "ddown_factor": 0.26224255571672533, + "ema_span_0": 3895.5544068711365, + "ema_span_1": 10079.734473634135, + "enabled": true, + "initial_eprice_ema_dist": -0.06579348877665478, + "initial_qty_pct": 0.015, + "markup_range": 0.014646281970597362, + "min_markup": 0.003644775732237631, + "n_close_orders": 14, + "rentry_pprice_dist": 0.04696705916756642, + "rentry_pprice_dist_wallet_exposure_weighting": 89.99476344700297, + "wallet_exposure_limit": 0.1}} From ade2bd4d156bcee08372ebca9366919902ec47ae Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 21 Nov 2022 14:14:05 +0100 Subject: [PATCH 16/27] add new default static config --- configs/live/static_grid_mode.example.json | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 configs/live/static_grid_mode.example.json diff --git a/configs/live/static_grid_mode.example.json b/configs/live/static_grid_mode.example.json new file mode 100644 index 000000000..a603e68f9 --- /dev/null +++ b/configs/live/static_grid_mode.example.json @@ -0,0 +1,39 @@ +{"config_name": "static_grid_118_symbols_683days", + "logging_level": 0, + "long": {"auto_unstuck_ema_dist": 0.01676344226553098, + "auto_unstuck_wallet_exposure_threshold": 0.1, + "backwards_tp": true, + "ema_span_0": 314.301792088284, + "ema_span_1": 240.0, + "enabled": true, + "eprice_exp_base": 1.0, + "eprice_pprice_diff": 0.04973332928054233, + "grid_span": 0.37319616544019824, + "initial_eprice_ema_dist": -0.041490979006885385, + "initial_qty_pct": 0.015, + "markup_range": 0.04, + "max_n_entry_orders": 11, + "min_markup": 0.003, + "n_close_orders": 8, + "secondary_allocation": 0, + "secondary_pprice_diff": 0.0814578629877079, + "wallet_exposure_limit": 0.1}, + "short": {"auto_unstuck_ema_dist": 0.02, + "auto_unstuck_wallet_exposure_threshold": 0.01, + "backwards_tp": true, + "ema_span_0": 240.0, + "ema_span_1": 10069.173228557336, + "enabled": true, + "eprice_exp_base": 1.9035509396568349, + "eprice_pprice_diff": 0.04908051871359724, + "grid_span": 0.5277691820128494, + "initial_eprice_ema_dist": 0.01, + "initial_qty_pct": 0.015, + "markup_range": 5.476933199347456e-05, + "max_n_entry_orders": 9, + "min_markup": 0.002690111798761108, + "n_close_orders": 9, + "secondary_allocation": 0.5876491107026843, + "secondary_pprice_diff": 0.40338712809793054, + "wallet_exposure_limit": 0.1}} + From 6cd1bd4c4e98426ae76e977dff90748a38140c9b Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 21 Nov 2022 14:14:27 +0100 Subject: [PATCH 17/27] remove old static default configs --- ...id_mode_auto_unstuck_disabled.example.json | 36 ------------------- ...rid_mode_auto_unstuck_enabled.example.json | 36 ------------------- 2 files changed, 72 deletions(-) delete mode 100644 configs/live/static_grid_mode_auto_unstuck_disabled.example.json delete mode 100644 configs/live/static_grid_mode_auto_unstuck_enabled.example.json diff --git a/configs/live/static_grid_mode_auto_unstuck_disabled.example.json b/configs/live/static_grid_mode_auto_unstuck_disabled.example.json deleted file mode 100644 index a5a41cf64..000000000 --- a/configs/live/static_grid_mode_auto_unstuck_disabled.example.json +++ /dev/null @@ -1,36 +0,0 @@ -{"config_name": "static_grid_125_symbols_300days", - "logging_level": 0, - "long": {"auto_unstuck_ema_dist": 0, - "auto_unstuck_wallet_exposure_threshold": 0, - "ema_span_0": 2732.0236491811465, - "ema_span_1": 6920.183045159956, - "enabled": true, - "eprice_exp_base": 1.0100329642857278, - "eprice_pprice_diff": 0.031149910203178106, - "grid_span": 0.4227253014717693, - "initial_eprice_ema_dist": -0.028021710932818293, - "initial_qty_pct": 0.016197354956635976, - "markup_range": 0.04970024105463802, - "max_n_entry_orders": 10, - "min_markup": 0.02144406148326509, - "n_close_orders": 4.565132435500947, - "secondary_allocation": 0.09114781453844102, - "secondary_pprice_diff": 0.1989419690803816, - "wallet_exposure_limit": 0.05}, - "short": {"auto_unstuck_ema_dist": 0, - "auto_unstuck_wallet_exposure_threshold": 0, - "ema_span_0": 2278.724652455613, - "ema_span_1": 2467.2565364133497, - "enabled": false, - "eprice_exp_base": 1.5433177133006661, - "eprice_pprice_diff": 0.03646634641369769, - "grid_span": 0.12439608988166556, - "initial_eprice_ema_dist": -0.08773848646779292, - "initial_qty_pct": 0.018000711292423612, - "markup_range": 0.014462448804936088, - "max_n_entry_orders": 10, - "min_markup": 0.014081575671317424, - "n_close_orders": 13.20116842270376, - "secondary_allocation": 0.602317066009605, - "secondary_pprice_diff": 0.22301448926709833, - "wallet_exposure_limit": 0.05}} diff --git a/configs/live/static_grid_mode_auto_unstuck_enabled.example.json b/configs/live/static_grid_mode_auto_unstuck_enabled.example.json deleted file mode 100644 index 34b84d24c..000000000 --- a/configs/live/static_grid_mode_auto_unstuck_enabled.example.json +++ /dev/null @@ -1,36 +0,0 @@ -{"config_name": "static_grid_125_symbols_300days", - "logging_level": 0, - "long": {"auto_unstuck_ema_dist": 0.05070953610814623, - "auto_unstuck_wallet_exposure_threshold": 0.6722077702999611, - "ema_span_0": 1351.0501771821905, - "ema_span_1": 1444.815397030421, - "enabled": true, - "eprice_exp_base": 1.0381543933378048, - "eprice_pprice_diff": 0.039732093769580555, - "grid_span": 0.47831503832736966, - "initial_eprice_ema_dist": -0.07196269003119887, - "initial_qty_pct": 0.010376041465319156, - "markup_range": 0.05598771329140069, - "max_n_entry_orders": 10, - "min_markup": 0.04688353879965964, - "n_close_orders": 5.776377910206165, - "secondary_allocation": 0.14568550324525736, - "secondary_pprice_diff": 0.2998323039442074, - "wallet_exposure_limit": 0.05}, - "short": {"auto_unstuck_ema_dist": 0.029554753810196455, - "auto_unstuck_wallet_exposure_threshold": 0.9266879327378665, - "ema_span_0": 2056.3711531607564, - "ema_span_1": 1488.2746753489837, - "enabled": false, - "eprice_exp_base": 2.0481490641956337, - "eprice_pprice_diff": 0.022063563358067362, - "grid_span": 0.31367331041410446, - "initial_eprice_ema_dist": -0.07621676235543176, - "initial_qty_pct": 0.016978000469196664, - "markup_range": 0.011805979665924199, - "max_n_entry_orders": 10, - "min_markup": 0.030613429762428288, - "n_close_orders": 8.957725372066655, - "secondary_allocation": 0.469251394373749, - "secondary_pprice_diff": 0.4092147638835546, - "wallet_exposure_limit": 0.05}} From 5fe05f7e903a9010eada6dc3e8379d56565f8827 Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 21 Nov 2022 14:15:49 +0100 Subject: [PATCH 18/27] remove AU disabled recursive example config --- ....json => recursive_grid_mode.example.json} | 0 ...id_mode_auto_unstuck_disabled.example.json | 30 ------------------- 2 files changed, 30 deletions(-) rename configs/live/{recursive_grid_mode_auto_unstuck_enabled.example.json => recursive_grid_mode.example.json} (100%) delete mode 100644 configs/live/recursive_grid_mode_auto_unstuck_disabled.example.json diff --git a/configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json b/configs/live/recursive_grid_mode.example.json similarity index 100% rename from configs/live/recursive_grid_mode_auto_unstuck_enabled.example.json rename to configs/live/recursive_grid_mode.example.json diff --git a/configs/live/recursive_grid_mode_auto_unstuck_disabled.example.json b/configs/live/recursive_grid_mode_auto_unstuck_disabled.example.json deleted file mode 100644 index 1cdaebe55..000000000 --- a/configs/live/recursive_grid_mode_auto_unstuck_disabled.example.json +++ /dev/null @@ -1,30 +0,0 @@ -{"config_name": "recursive_grid_125_symbols_300days_AUdisabled", - "logging_level": 0, - "long": {"auto_unstuck_ema_dist": 0, - "auto_unstuck_wallet_exposure_threshold": 0, - "ddown_factor": 0.2836942996599674, - "ema_span_0": 3573.707148465803, - "ema_span_1": 9371.238150453562, - "enabled": true, - "initial_eprice_ema_dist": 0, - "initial_qty_pct": 0.01234914998889527, - "markup_range": 0.0458465437558736, - "min_markup": 0.013348131482672233, - "n_close_orders": 8, - "rentry_pprice_dist": 0.023239053434194266, - "rentry_pprice_dist_wallet_exposure_weighting": 11.95394791078704, - "wallet_exposure_limit": 0.05}, - "short": {"auto_unstuck_ema_dist": 0, - "auto_unstuck_wallet_exposure_threshold": 0, - "ddown_factor": 2.7565718262412333, - "ema_span_0": 360, - "ema_span_1": 7498.585832673125, - "enabled": false, - "initial_eprice_ema_dist": -0.026302949421201358, - "initial_qty_pct": 0.0219911723842904, - "markup_range": 0.01645520103514452, - "min_markup": 0.007048929135811168, - "n_close_orders": 5, - "rentry_pprice_dist": 0.038052554084690327, - "rentry_pprice_dist_wallet_exposure_weighting": 3.901535888984564, - "wallet_exposure_limit": 0.05}} From 69d271d0a3f240704c17ac211b051c84cb816513 Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 21 Nov 2022 14:17:00 +0100 Subject: [PATCH 19/27] remove neat AU disabled example config --- ...ample.json => neat_grid_mode.example.json} | 0 ...id_mode_auto_unstuck_disabled.example.json | 34 ------------------- 2 files changed, 34 deletions(-) rename configs/live/{neat_grid_mode_auto_unstuck_enabled.example.json => neat_grid_mode.example.json} (100%) delete mode 100644 configs/live/neat_grid_mode_auto_unstuck_disabled.example.json diff --git a/configs/live/neat_grid_mode_auto_unstuck_enabled.example.json b/configs/live/neat_grid_mode.example.json similarity index 100% rename from configs/live/neat_grid_mode_auto_unstuck_enabled.example.json rename to configs/live/neat_grid_mode.example.json diff --git a/configs/live/neat_grid_mode_auto_unstuck_disabled.example.json b/configs/live/neat_grid_mode_auto_unstuck_disabled.example.json deleted file mode 100644 index 927e40235..000000000 --- a/configs/live/neat_grid_mode_auto_unstuck_disabled.example.json +++ /dev/null @@ -1,34 +0,0 @@ -{"config_name": "neat_grid_118_symbols_558days", - "logging_level": 0, - "long": {"auto_unstuck_ema_dist": 0.0, - "auto_unstuck_wallet_exposure_threshold": 0.0, - "backwards_tp": true, - "ema_span_0": 3316.9219129323387, - "ema_span_1": 10080.0, - "enabled": true, - "eprice_exp_base": 1.0192773367952441, - "eqty_exp_base": 2.2451122381485114, - "grid_span": 0.3748808987538866, - "initial_eprice_ema_dist": -0.05650968305205222, - "initial_qty_pct": 0.015, - "markup_range": 0.006814246325306774, - "max_n_entry_orders": 7, - "min_markup": 0.003598694354831559, - "n_close_orders": 8, - "wallet_exposure_limit": 0.1}, - "short": {"auto_unstuck_ema_dist": 0.0, - "auto_unstuck_wallet_exposure_threshold": 0.0, - "backwards_tp": true, - "ema_span_0": 1154.0093012354746, - "ema_span_1": 240.2796340806273, - "enabled": true, - "eprice_exp_base": 0.75, - "eqty_exp_base": 0.885981103800254, - "grid_span": 0.3938211312577981, - "initial_eprice_ema_dist": -0.017450168051132028, - "initial_qty_pct": 0.015, - "markup_range": 0.002122367807571618, - "max_n_entry_orders": 16, - "min_markup": 0.01, - "n_close_orders": 23, - "wallet_exposure_limit": 0.1}} From 773361ea3f96bb9d3c343405199351db919b2721 Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 21 Nov 2022 14:18:54 +0100 Subject: [PATCH 20/27] Update where_to_find_configs.txt --- configs/live/where_to_find_configs.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/live/where_to_find_configs.txt b/configs/live/where_to_find_configs.txt index a077442ec..93a880b99 100644 --- a/configs/live/where_to_find_configs.txt +++ b/configs/live/where_to_find_configs.txt @@ -1,3 +1,7 @@ https://github.com/JohnKearney1/PassivBot-Configurations -https://github.com/hoeckxer/passivbot_configs \ No newline at end of file +https://github.com/hoeckxer/passivbot_configs + +https://github.com/donewiththedollar/passivbot_v5.8.0/tree/main/configs + +https://github.com/tedyptedto/pbos From 07dee63281666509a00009d95c3c1db76065d521 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 16:12:46 +0100 Subject: [PATCH 21/27] bug fix bitget correctly update pos with websocket --- bitget.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/bitget.py b/bitget.py index 221503a1b..633450228 100644 --- a/bitget.py +++ b/bitget.py @@ -14,7 +14,7 @@ import pprint from njit_funcs import round_ -from passivbot import Bot +from passivbot import Bot, logging from procedures import print_async_exception, print_ from pure_funcs import ts_to_date, sort_dict_keys, date_to_ts @@ -83,7 +83,7 @@ def init_market_type(self): self.market_type += "_linear_perpetual" self.product_type = "umcbl" self.inverse = self.config["inverse"] = False - self.min_cost = self.config["min_cost"] = 5.1 + self.min_cost = self.config["min_cost"] = 5.5 elif self.symbol.endswith("USD"): print("inverse perpetual") self.symbol += "_DMCBL" @@ -568,7 +568,11 @@ async def init_exchange_config(self): # set leverage res = await self.private_post( self.endpoints["set_leverage"], - params={"symbol": self.symbol, "marginCoin": self.margin_coin, "leverage": 20}, + params={ + "symbol": self.symbol, + "marginCoin": self.margin_coin, + "leverage": self.leverage, + }, ) print(res) except Exception as e: @@ -715,8 +719,8 @@ def standardize_user_stream_event( events = [] if "event" in event and event["event"] == "login": self.is_logged_into_user_stream = True - return {'logged_in': True} - # print('debug 0', event) + return {"logged_in": True} + # logging.info(f"debug 0 {event}") if "arg" in event and "data" in event and "channel" in event["arg"]: if event["arg"]["channel"] == "orders": for elm in event["data"]: @@ -743,18 +747,23 @@ def standardize_user_stream_event( standardized["filled"] = True events.append(standardized) if event["arg"]["channel"] == "positions": + long_pos = {"psize_long": 0.0, "pprice_long": 0.0} + short_pos = {"psize_short": 0.0, "pprice_short": 0.0} for elm in event["data"]: if elm["instId"] == self.symbol and "averageOpenPrice" in elm: - standardized = { - f"psize_{elm['holdSide']}": round_( - abs(float(elm["total"])), self.qty_step + if elm["holdSide"] == "long": + long_pos["psize_long"] = round_(abs(float(elm["total"])), self.qty_step) + long_pos["pprice_long"] = truncate_float( + float(elm["averageOpenPrice"]), self.price_rounding ) - * (-1 if elm["holdSide"] == "short" else 1), - f"pprice_{elm['holdSide']}": truncate_float( + elif elm["holdSide"] == "short": + short_pos["psize_short"] = -abs(round_(abs(float(elm["total"])), self.qty_step)) + short_pos["pprice_short"] = truncate_float( float(elm["averageOpenPrice"]), self.price_rounding - ), - } - events.append(standardized) + ) + # absence of elemet means no pos + events.append(long_pos) + events.append(short_pos) if event["arg"]["channel"] == "account": for elm in event["data"]: From 7b0b6383743034d31717296951f3945cacfcaf61 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 16:13:39 +0100 Subject: [PATCH 22/27] update pos with REST API before ientry --- passivbot.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/passivbot.py b/passivbot.py index 91e0b12eb..9e3eed62e 100644 --- a/passivbot.py +++ b/passivbot.py @@ -723,7 +723,16 @@ async def cancel_and_create(self): self.ts_locked["cancel_and_create"] = time() try: ideal_orders = [] - for o in self.calc_orders(): + all_orders = self.calc_orders() + for o in all_orders: + if "ientry" in o["custom_id"] and calc_diff(o["price"], self.price) < 0.01: + # call update_position() before making initial entry orders + # in case websocket has failed + logging.info("update_position with REST API before creating initial entries") + await self.update_position() + all_orders = self.calc_orders() + break + for o in all_orders: if any(x in o["custom_id"] for x in ["ientry", "unstuck"]): if calc_diff(o["price"], self.price) < 0.01: # EMA based orders must be closer than 1% of current price From b3c3b141e67290851278761646604ca9ed1e2336 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 16:52:58 +0100 Subject: [PATCH 23/27] change price diff from 1% to 0.3% for pos update don't cancel order if not present in open_orders --- passivbot.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/passivbot.py b/passivbot.py index 9e3eed62e..7dd075986 100644 --- a/passivbot.py +++ b/passivbot.py @@ -376,11 +376,13 @@ async def cancel_orders(self, orders_to_cancel: [dict]) -> [dict]: self.ts_locked["cancel_orders"] = time() try: deletions = [] + oo_ids = {o["order_id"] for o in self.open_orders} for oc in orders_to_cancel: - try: - deletions.append((oc, asyncio.create_task(self.execute_cancellation(oc)))) - except Exception as e: - logging.error(f"error cancelling order c {oc} {e}") + if oc["order_id"] not in oo_ids: + try: + deletions.append((oc, asyncio.create_task(self.execute_cancellation(oc)))) + except Exception as e: + logging.error(f"error cancelling order c {oc} {e}") cancelled_orders = [] for oc, c in deletions: try: @@ -725,7 +727,7 @@ async def cancel_and_create(self): ideal_orders = [] all_orders = self.calc_orders() for o in all_orders: - if "ientry" in o["custom_id"] and calc_diff(o["price"], self.price) < 0.01: + if "ientry" in o["custom_id"] and calc_diff(o["price"], self.price) < 0.003: # call update_position() before making initial entry orders # in case websocket has failed logging.info("update_position with REST API before creating initial entries") From 8941999aac2c2ef6fa525f5c072751dc633079c6 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 17:12:29 +0100 Subject: [PATCH 24/27] bug fix, remove 'not' --- passivbot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/passivbot.py b/passivbot.py index 7dd075986..8f4710091 100644 --- a/passivbot.py +++ b/passivbot.py @@ -378,7 +378,7 @@ async def cancel_orders(self, orders_to_cancel: [dict]) -> [dict]: deletions = [] oo_ids = {o["order_id"] for o in self.open_orders} for oc in orders_to_cancel: - if oc["order_id"] not in oo_ids: + if oc["order_id"] in oo_ids: # only try cancellation if order in self.open_orders try: deletions.append((oc, asyncio.create_task(self.execute_cancellation(oc)))) except Exception as e: @@ -727,7 +727,7 @@ async def cancel_and_create(self): ideal_orders = [] all_orders = self.calc_orders() for o in all_orders: - if "ientry" in o["custom_id"] and calc_diff(o["price"], self.price) < 0.003: + if "ientry" in o["custom_id"] and calc_diff(o["price"], self.price) < 0.002: # call update_position() before making initial entry orders # in case websocket has failed logging.info("update_position with REST API before creating initial entries") From 70f1b74c4b9bde6a5404dbb70915f063eea550d0 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 18:56:27 +0100 Subject: [PATCH 25/27] deduplicate orders_to_cancel and other small changes --- passivbot.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/passivbot.py b/passivbot.py index 8f4710091..d8ce818a6 100644 --- a/passivbot.py +++ b/passivbot.py @@ -369,20 +369,22 @@ async def create_orders(self, orders_to_create: [dict]) -> [dict]: self.ts_released["create_orders"] = time() async def cancel_orders(self, orders_to_cancel: [dict]) -> [dict]: - if not orders_to_cancel: - return if self.ts_locked["cancel_orders"] > self.ts_released["cancel_orders"]: return self.ts_locked["cancel_orders"] = time() try: - deletions = [] - oo_ids = {o["order_id"] for o in self.open_orders} - for oc in orders_to_cancel: - if oc["order_id"] in oo_ids: # only try cancellation if order in self.open_orders - try: - deletions.append((oc, asyncio.create_task(self.execute_cancellation(oc)))) - except Exception as e: - logging.error(f"error cancelling order c {oc} {e}") + if not orders_to_cancel: + return + deletions, orders_to_cancel_dedup, oo_ids = [], [], set() + for o in orders_to_cancel: + if o["order_id"] not in oo_ids: + oo_ids.add(o["order_id"]) + orders_to_cancel_dedup.append(o) + for oc in orders_to_cancel_dedup: + try: + deletions.append((oc, asyncio.create_task(self.execute_cancellation(oc)))) + except Exception as e: + logging.error(f"error cancelling order c {oc} {e}") cancelled_orders = [] for oc, c in deletions: try: @@ -716,14 +718,14 @@ def calc_orders(self): async def cancel_and_create(self): if self.ts_locked["cancel_and_create"] > self.ts_released["cancel_and_create"]: return - if any(self.error_halt.values()): - logging.warning( - f"warning: error in rest api fetch {self.error_halt}, " - + "halting order creations/cancellations" - ) - return self.ts_locked["cancel_and_create"] = time() try: + if any(self.error_halt.values()): + logging.warning( + f"warning: error in rest api fetch {self.error_halt}, " + + "halting order creations/cancellations" + ) + return [] ideal_orders = [] all_orders = self.calc_orders() for o in all_orders: @@ -800,9 +802,9 @@ async def cancel_and_create(self): ) # sleep 10 ms between sending cancellations and sending creations if to_create: results.append(await self.create_orders(to_create[: self.n_orders_per_execution])) - await asyncio.sleep(self.delay_between_executions) # sleep before releasing lock return results finally: + await asyncio.sleep(self.delay_between_executions) # sleep before releasing lock self.ts_released["cancel_and_create"] = time() async def on_market_stream_event(self, ticks: [dict]): @@ -818,7 +820,7 @@ async def on_market_stream_event(self, ticks: [dict]): now = time() if now - self.ts_released["force_update"] > self.force_update_interval: self.ts_released["force_update"] = now - # force update pos and open orders thru rest API every 30 sec + # force update pos and open orders thru rest API every x sec (default 30) await asyncio.gather(self.update_position(), self.update_open_orders()) if now - self.heartbeat_ts > self.heartbeat_interval_seconds: # print heartbeat once an hour From b3303449e5ba91c41046eceba980c242b59c1470 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 20:40:53 +0100 Subject: [PATCH 26/27] update default bounds PSO --- .../particle_swarm_optimization.hjson | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/configs/optimize/particle_swarm_optimization.hjson b/configs/optimize/particle_swarm_optimization.hjson index a1d38c9ef..9c33470b6 100644 --- a/configs/optimize/particle_swarm_optimization.hjson +++ b/configs/optimize/particle_swarm_optimization.hjson @@ -74,7 +74,7 @@ { long: { - grid_span: [0.35, 0.7] + grid_span: [0.34, 0.7] ema_span_0: [240, 4320] ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] @@ -85,15 +85,15 @@ secondary_allocation: [0.0, 0.7] secondary_pprice_diff: [0.05, 0.5] eprice_exp_base: [1.0, 2.1] - min_markup: [0.001, 0.003] + min_markup: [0.001, 0.004] markup_range: [0.0, 0.04] n_close_orders: [7, 20] auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] - auto_unstuck_ema_dist: [0.001, 0.1] + auto_unstuck_ema_dist: [0.001, 0.02] } short: { - grid_span: [0.35, 0.7] + grid_span: [0.34, 0.7] ema_span_0: [240, 4320] ema_span_1: [240, 10080] wallet_exposure_limit: [0.1, 0.1] @@ -104,11 +104,11 @@ secondary_allocation: [0.0, 0.7] secondary_pprice_diff: [0.05, 0.5] eprice_exp_base: [1.0, 2.1] - min_markup: [0.001, 0.003] + min_markup: [0.001, 0.004] markup_range: [0.0, 0.04] n_close_orders: [7, 20] auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] - auto_unstuck_ema_dist: [0.001, 0.1] + auto_unstuck_ema_dist: [0.001, 0.02] } } bounds_recursive_grid: @@ -127,7 +127,7 @@ markup_range: [0.0, 0.04] n_close_orders: [4, 20] auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] - auto_unstuck_ema_dist: [0.001, 0.1] + auto_unstuck_ema_dist: [0.001, 0.02] } short: { @@ -143,7 +143,7 @@ markup_range: [0.0, 0.04] n_close_orders: [4, 20] auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] - auto_unstuck_ema_dist: [0.001, 0.1] + auto_unstuck_ema_dist: [0.001, 0.02] } } bounds_neat_grid: @@ -159,11 +159,11 @@ initial_eprice_ema_dist: [-0.1, 0.01] eqty_exp_base: [0.9, 4.0] eprice_exp_base: [0.9, 4.0] - min_markup: [0.001, 0.003] + min_markup: [0.001, 0.004] markup_range: [0.0, 0.04] n_close_orders: [7, 30] auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] - auto_unstuck_ema_dist: [0.0, 0.02] + auto_unstuck_ema_dist: [0.001, 0.02] } short: { @@ -176,11 +176,11 @@ initial_eprice_ema_dist: [-0.1, 0.01] eqty_exp_base: [0.9, 4.0] eprice_exp_base: [0.9, 4.0] - min_markup: [0.001, 0.003] + min_markup: [0.001, 0.004] markup_range: [0.0, 0.04] n_close_orders: [7, 30] auto_unstuck_wallet_exposure_threshold: [0.01, 0.1] - auto_unstuck_ema_dist: [0.0, 0.02] + auto_unstuck_ema_dist: [0.001, 0.02] } } } From 1ce72fd2b16a355d63cd495798acccef963c5a00 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 23 Nov 2022 23:45:50 +0100 Subject: [PATCH 27/27] bug fix with new symbols --- bitget.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/bitget.py b/bitget.py index 633450228..3b817ee63 100644 --- a/bitget.py +++ b/bitget.py @@ -24,6 +24,8 @@ def first_capitalized(s: str): def truncate_float(x: float, d: int) -> float: + if x is None: + return 0.0 xs = str(x) return float(xs[: xs.find(".") + d + 1]) @@ -255,15 +257,19 @@ async def fetch_position(self) -> dict: if elm["holdSide"] == "long": position["long"] = { "size": round_(float(elm["total"]), self.qty_step), - "price": truncate_float(float(elm["averageOpenPrice"]), self.price_rounding), - "liquidation_price": float(elm["liquidationPrice"]), + "price": truncate_float(elm["averageOpenPrice"], self.price_rounding), + "liquidation_price": 0.0 + if elm["liquidationPrice"] is None + else float(elm["liquidationPrice"]), } elif elm["holdSide"] == "short": position["short"] = { "size": -abs(round_(float(elm["total"]), self.qty_step)), - "price": truncate_float(float(elm["averageOpenPrice"]), self.price_rounding), - "liquidation_price": float(elm["liquidationPrice"]), + "price": truncate_float(elm["averageOpenPrice"], self.price_rounding), + "liquidation_price": 0.0 + if elm["liquidationPrice"] is None + else float(elm["liquidationPrice"]), } for elm in fetched_balance["data"]: if elm["marginCoin"] == self.margin_coin: @@ -754,12 +760,14 @@ def standardize_user_stream_event( if elm["holdSide"] == "long": long_pos["psize_long"] = round_(abs(float(elm["total"])), self.qty_step) long_pos["pprice_long"] = truncate_float( - float(elm["averageOpenPrice"]), self.price_rounding + elm["averageOpenPrice"], self.price_rounding ) elif elm["holdSide"] == "short": - short_pos["psize_short"] = -abs(round_(abs(float(elm["total"])), self.qty_step)) + short_pos["psize_short"] = -abs( + round_(abs(float(elm["total"])), self.qty_step) + ) short_pos["pprice_short"] = truncate_float( - float(elm["averageOpenPrice"]), self.price_rounding + elm["averageOpenPrice"], self.price_rounding ) # absence of elemet means no pos events.append(long_pos)