diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 58fcfbd98e..38fd9917c1 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -9,16 +9,13 @@ from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.qlib.experiment.quant_experiment import QlibQuantScenario +from rdagent.scenarios.qlib.metrics import ARR_KEY, IC_KEY, MDD_KEY from rdagent.utils import convert2bool from rdagent.utils.agent.tpl import T DIRNAME = Path(__file__).absolute().resolve().parent -IMPORTANT_METRICS = [ - "IC", - "1day.excess_return_with_cost.annualized_return", - "1day.excess_return_with_cost.max_drawdown", -] +IMPORTANT_METRICS = [IC_KEY, ARR_KEY, MDD_KEY] def process_results(current_result, sota_result): diff --git a/rdagent/scenarios/qlib/metrics.py b/rdagent/scenarios/qlib/metrics.py new file mode 100644 index 0000000000..8b6cb36e81 --- /dev/null +++ b/rdagent/scenarios/qlib/metrics.py @@ -0,0 +1,17 @@ +"""Names of the Qlib metrics that RD-Agent reads back from a finished workflow. + +The keys are produced by Qlib, not by RD-Agent: ``IC``/``ICIR``/``Rank IC``/``Rank ICIR`` come from +``SigAnaRecord`` and the ``1day..`` keys from ``PortAnaRecord``. Every place that indexes an +``experiment.result`` Series by metric name should import the constant instead of spelling the key out, so a +typo in one copy cannot silently disagree with another (see #1451). +""" + +IC_KEY = "IC" +ICIR_KEY = "ICIR" +RANK_IC_KEY = "Rank IC" +RANK_ICIR_KEY = "Rank ICIR" + +# Portfolio analysis of the excess return over the benchmark, after transaction cost. +ARR_KEY = "1day.excess_return_with_cost.annualized_return" +IR_KEY = "1day.excess_return_with_cost.information_ratio" +MDD_KEY = "1day.excess_return_with_cost.max_drawdown" # Qlib reports drawdown as a number <= 0 diff --git a/rdagent/scenarios/qlib/proposal/bandit.py b/rdagent/scenarios/qlib/proposal/bandit.py index f5584974fc..3dd41aacfa 100644 --- a/rdagent/scenarios/qlib/proposal/bandit.py +++ b/rdagent/scenarios/qlib/proposal/bandit.py @@ -6,6 +6,17 @@ import numpy as np +from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.qlib.metrics import ( + ARR_KEY, + IC_KEY, + ICIR_KEY, + IR_KEY, + MDD_KEY, + RANK_IC_KEY, + RANK_ICIR_KEY, +) + @dataclass class Metrics: @@ -16,7 +27,7 @@ class Metrics: arr: float = 0.0 ir: float = 0.0 mdd: float = 0.0 - sharpe: float = 0.0 + calmar: float = 0.0 # annualized return / |max drawdown|; not a Sharpe ratio, it carries no volatility term def as_vector(self) -> np.ndarray: return np.array( @@ -28,29 +39,46 @@ def as_vector(self) -> np.ndarray: self.arr, self.ir, -self.mdd, - self.sharpe, + self.calmar, ] ) +def _get_metric(result, key: str, default: float = 0.0) -> float: + """Read one metric from ``experiment.result``, warning instead of silently defaulting when it is absent. + + A missing key, a mistyped key and a genuinely zero metric used to be indistinguishable (see #1451); the + warning makes the first two visible in the log while keeping the loop alive on a partial Qlib result. + """ + if key in result: + return float(result[key]) + logger.warning( + f"Metric {key!r} not found in experiment result, using {default}. Available keys: {list(result.keys())}" + ) + return default + + def extract_metrics_from_experiment(experiment) -> Metrics: - """Extract metrics from experiment feedback""" - try: - result = experiment.result - ic = result.get("IC", 0.0) - icir = result.get("ICIR", 0.0) - rank_ic = result.get("Rank IC", 0.0) - rank_icir = result.get("Rank ICIR", 0.0) - arr = result.get("1day.excess_return_with_cost.annualized_return ", 0.0) - ir = result.get("1day.excess_return_with_cost.information_ratio", 0.0) - mdd = result.get("1day.excess_return_with_cost.max_drawdown", 1.0) # Avoid division by zero - sharpe = arr / -mdd if mdd != 0 else 0.0 - - return Metrics(ic=ic, icir=icir, rank_ic=rank_ic, rank_icir=rank_icir, arr=arr, ir=ir, mdd=mdd, sharpe=sharpe) - except Exception as e: - print(f"Error extracting metrics: {e}") + """Extract the bandit's feature vector from ``experiment.result`` (a Series indexed by Qlib metric name).""" + result = getattr(experiment, "result", None) + if result is None: + # Execution failed, so there is nothing to learn from; a zero vector is neutral for the bandit. + logger.warning("Experiment has no result, using all-zero metrics for the bandit") return Metrics() + ic = _get_metric(result, IC_KEY) + icir = _get_metric(result, ICIR_KEY) + rank_ic = _get_metric(result, RANK_IC_KEY) + rank_icir = _get_metric(result, RANK_ICIR_KEY) + arr = _get_metric(result, ARR_KEY) + ir = _get_metric(result, IR_KEY) + # Qlib reports max drawdown as a number <= 0. A default of 0.0 (guarded below) keeps both the ratio and the + # -mdd vector slot at zero when the key is missing; the previous default of 1.0 flipped the ratio's sign. + mdd = _get_metric(result, MDD_KEY) + calmar = arr / -mdd if mdd != 0 else 0.0 + + return Metrics(ic=ic, icir=icir, rank_ic=rank_ic, rank_icir=rank_icir, arr=arr, ir=ir, mdd=mdd, calmar=calmar) + class LinearThompsonTwoArm: def __init__(self, dim: int, prior_var: float = 1.0, noise_var: float = 1.0): diff --git a/test/qlib/test_bandit_metrics.py b/test/qlib/test_bandit_metrics.py new file mode 100644 index 0000000000..8707125cf1 --- /dev/null +++ b/test/qlib/test_bandit_metrics.py @@ -0,0 +1,115 @@ +"""Tests for the metric extraction that feeds the fin_quant bandit scheduler (see #1451). + +Only ``experiment.result`` is touched, so a ``SimpleNamespace`` carrying a Series with the real Qlib key names is a +faithful stand-in; no Qlib, Docker or LLM is needed. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pandas as pd +import pytest + +from rdagent.scenarios.qlib.developer.feedback import IMPORTANT_METRICS +from rdagent.scenarios.qlib.metrics import ( + ARR_KEY, + IC_KEY, + ICIR_KEY, + IR_KEY, + MDD_KEY, + RANK_IC_KEY, + RANK_ICIR_KEY, +) +from rdagent.scenarios.qlib.proposal.bandit import ( + EnvController, + extract_metrics_from_experiment, +) + +# The exact keys Qlib logs (SigAnaRecord and PortAnaRecord), as read back from qlib_res.csv by QlibFBWorkspace. +QLIB_RESULT = pd.Series( + { + "IC": 0.05, + "ICIR": 0.4, + "Rank IC": 0.06, + "Rank ICIR": 0.5, + "1day.excess_return_with_cost.annualized_return": 0.12, + "1day.excess_return_with_cost.information_ratio": 1.1, + "1day.excess_return_with_cost.max_drawdown": -0.08, + } +) + +LOGGER = "rdagent.scenarios.qlib.proposal.bandit.logger" + + +def _experiment(result): + return SimpleNamespace(result=result) + + +@pytest.mark.offline +def test_metric_keys_match_qlib_output(): + for key in (IC_KEY, ICIR_KEY, RANK_IC_KEY, RANK_ICIR_KEY, ARR_KEY, IR_KEY, MDD_KEY): + assert key == key.strip() + assert key in QLIB_RESULT.index, key + assert set(IMPORTANT_METRICS) <= set(QLIB_RESULT.index) + + +@pytest.mark.offline +def test_extract_reads_every_metric(): + m = extract_metrics_from_experiment(_experiment(QLIB_RESULT)) + + assert m.ic == pytest.approx(0.05) + assert m.arr == pytest.approx(0.12) + assert m.ir == pytest.approx(1.1) + assert m.mdd == pytest.approx(-0.08) + assert m.calmar == pytest.approx(0.12 / 0.08) + + vec = m.as_vector() + assert vec.shape == (8,) + assert vec[4] == pytest.approx(0.12) + assert vec[6] == pytest.approx(0.08) # -mdd, so a deeper drawdown lowers the slot + assert vec[7] > 0 + + +@pytest.mark.offline +def test_missing_key_warns_and_defaults(): + partial = QLIB_RESULT.drop(ARR_KEY) + with patch(LOGGER) as logger: + m = extract_metrics_from_experiment(_experiment(partial)) + + assert m.arr == 0.0 + assert m.calmar == 0.0 + assert m.ic == pytest.approx(0.05) + warnings = [call.args[0] for call in logger.warning.call_args_list] + assert any(ARR_KEY in msg for msg in warnings), warnings + + +@pytest.mark.offline +def test_missing_drawdown_is_neutral(): + partial = QLIB_RESULT.drop(MDD_KEY) + with patch(LOGGER): + m = extract_metrics_from_experiment(_experiment(partial)) + + assert m.mdd == 0.0 + assert m.calmar == 0.0 + assert m.as_vector()[6] == 0.0 + + +@pytest.mark.offline +def test_failed_run_gives_zero_vector(): + with patch(LOGGER): + m = extract_metrics_from_experiment(_experiment(None)) + assert not m.as_vector().any() + + +@pytest.mark.offline +def test_arr_moves_the_reward(): + controller = EnvController() + base = extract_metrics_from_experiment(_experiment(QLIB_RESULT)) + + doubled = QLIB_RESULT.copy() + doubled[ARR_KEY] = 0.24 + better = extract_metrics_from_experiment(_experiment(doubled)) + + # Both the ARR slot (weight 0.25) and the derived ratio (weight 0.2) respond, so 0.45 of the weight is live. + expected_gain = 0.25 * (0.24 - 0.12) + 0.2 * ((0.24 - 0.12) / 0.08) + assert controller.reward(better) - controller.reward(base) == pytest.approx(expected_gain)