Stress testing a trading bot against a market crash

article-stress-test.py

"""Stress test a trading bot against crashes it has never seen.

Each scenario file ships with tradefloor and fires on day 50 of the run. The
same seed with no scenario is the control: it is the same market, draw for
draw, until the scenario fires, so any difference after day 50 comes from
the shock. Three versions of one momentum bot are tested, and the question
is which crisis breaks a 15% drawdown limit.
"""
import tradefloor as tf

SEEDS = [21, 22, 23, 24, 25, 26]
DAYS = 80
LIMIT = 15.0                                 # the drawdown limit, in percent
market = tf.Universe.random(20, seed=7)
SCENARIOS = ["liquidity_crisis", "rate_shock", "recession"]


class Momentum:
    """Hold the five names that rose most over the last ten days, at a
    chosen gross exposure. `every` is how many days between rebalances, and
    with `vix_brake` the exposure shrinks as the published VIX rises above 20."""

    def __init__(self, gross, every=1, vix_brake=False):
        self.gross, self.every, self.vix_brake = gross, every, vix_brake

    def act(self, obs):
        if not obs.is_first_step_of_day or obs.day < 10 or obs.day % self.every:
            return None
        bars = {t: obs.history.bars(t, last=10) for t in obs.tickers}
        rise = {t: b[-1]["close"] / b[0]["close"] - 1 for t, b in bars.items() if len(b) == 10}
        top = sorted(rise, key=rise.get, reverse=True)[:5]
        gross = self.gross
        if self.vix_brake:
            gross *= min(1.0, 20 / obs.history.macro()[-1]["vix"])
        worth = obs.portfolio.net_worth()
        orders = {}
        for t in obs.tickers:
            want = gross * worth / 5 / obs.price(t) if t in top else 0
            change = round(want - obs.position(t))
            if change:
                orders[t] = change
        return orders or None


BOTS = {
    "original": dict(gross=1.5),
    "lower": dict(gross=1.0),
    "braked": dict(gross=1.0, every=5, vix_brake=True),
}
LABELS = {"original": "original", "lower": "less leverage",
          "braked": "weekly, VIX brake", "hold": "buy and hold"}


def make(bot):
    if bot == "hold":
        return tf.reference_agents()["buy_and_hold"]
    return Momentum(**BOTS[bot])


def worst_drawdown(bot, scenario):
    worst = 0.0
    for seed in SEEDS:
        card = tf.evaluate({"bot": make(bot)}, seed=seed, universe=market,
                           days=DAYS, history_days=10,
                           scenario=tf.Scenario.load(scenario) if scenario else None)["bot"]
        worst = max(worst, card.max_drawdown_pct)
    return round(worst, 1)


ROWS = ["none"] + SCENARIOS
CALM = "none"
NAMES = list(BOTS) + ["hold"]
results = {bot: {name: worst_drawdown(bot, None if name == "none" else name) for name in ROWS}
           for bot in NAMES}

print(f"worst drawdown over {len(SEEDS)} seeds and {DAYS} days (limit {LIMIT:.0f}%)")
print(f"{'':17}" + "".join(f"{LABELS[bot]:>19}" for bot in NAMES))
for name in ROWS:
    print(f"{name:17}" + "".join(f"{results[bot][name]:>18.1f}%" for bot in NAMES))
breaks = {bot: [n for n in ROWS if results[bot][n] > LIMIT] for bot in NAMES}

# For the charts: the table turned round, and one seed's net worth through
# the liquidity crisis, 100 at the start, for three of the four.
by_scenario = {name: {bot: results[bot][name] for bot in NAMES} for name in ROWS}
curves = {}
for bot in ("original", "braked", "hold"):
    card = tf.evaluate({"bot": make(bot)}, seed=SEEDS[0], universe=market, days=DAYS,
                       history_days=10, scenario=tf.Scenario.load("liquidity_crisis"))["bot"]
    start = card.equity_curve[0]
    curves[bot] = [round(100 * v / start, 2) for v in card.equity_curve]
for bot, names in breaks.items():
    print(f"{LABELS[bot]}: " + ("over the limit in " + ", ".join(names) if names else "inside the limit everywhere"))