{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "\"Open" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2026-01-19T14:28:36.967824880Z", "start_time": "2026-01-19T14:28:36.911827073Z" } }, "outputs": [], "source": [ "# Copyright (c) Microsoft Corporation.\n", "# Licensed under the MIT License." ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2026-01-19T14:28:37.342749633Z", "start_time": "2026-01-19T14:28:36.970492152Z" } }, "outputs": [], "source": [ "import sys, site\n", "from pathlib import Path\n", "\n", "################################# NOTE #################################\n", "# Please be aware that if colab installs the latest numpy and pyqlib #\n", "# in this cell, users should RESTART the runtime in order to run the #\n", "# following cells successfully. #\n", "########################################################################\n", "\n", "try:\n", " import qlib\n", "except ImportError:\n", " # install qlib\n", " ! pip install --upgrade numpy\n", " ! pip install pyqlib\n", " if \"google.colab\" in sys.modules:\n", " # The Google colab environment is a little outdated. We have to downgrade the pyyaml to make it compatible with other packages\n", " ! pip install pyyaml==5.4.1\n", " # reload\n", " site.main()\n", "\n", "scripts_dir = Path.cwd().parent.joinpath(\"scripts\")\n", "if not scripts_dir.joinpath(\"get_data.py\").exists():\n", " # download get_data.py script\n", " scripts_dir = Path(\"~/tmp/qlib_code/scripts\").expanduser().resolve()\n", " scripts_dir.mkdir(parents=True, exist_ok=True)\n", " import requests\n", "\n", " with requests.get(\"https://raw.githubusercontent.com/microsoft/qlib/main/scripts/get_data.py\", timeout=10) as resp:\n", " with open(scripts_dir.joinpath(\"get_data.py\"), \"wb\") as fp:\n", " fp.write(resp.content)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2026-01-19T14:28:39.357732259Z", "start_time": "2026-01-19T14:28:37.344390574Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Gym has been unmaintained since 2022 and does not support NumPy 2.0 amongst other critical functionality.\n", "Please upgrade to Gymnasium, the maintained drop-in replacement of Gym, or contact the authors of your software and request that they upgrade.\n", "Users of this version of Gym should be able to simply replace 'import gym' with 'import gymnasium as gym' in the vast majority of cases.\n", "See the migration guide at https://gymnasium.farama.org/introduction/migration_guide/ for additional information.\n" ] } ], "source": [ "import qlib\n", "import pandas as pd\n", "from qlib.constant import REG_CN\n", "from qlib.utils import exists_qlib_data, init_instance_by_config\n", "from qlib.workflow import R\n", "from qlib.workflow.record_temp import SignalRecord, PortAnaRecord\n", "from qlib.utils import flatten_dict" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2026-01-19T14:28:39.896437579Z", "start_time": "2026-01-19T14:28:39.359349890Z" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[3509038:MainThread](2026-01-27 22:56:42,557) INFO - qlib.Initialization - [config.py:452] - default_conf: client.\n", "[3509038:MainThread](2026-01-27 22:56:42,559) INFO - qlib.Initialization - [__init__.py:82] - qlib successfully initialized based on client settings.\n", "[3509038:MainThread](2026-01-27 22:56:42,560) INFO - qlib.Initialization - [__init__.py:84] - data_path={'__DEFAULT_FREQ': PosixPath('/home/liu/.qlib/qlib_data/cn_data')}\n" ] } ], "source": [ "# use default data\n", "# NOTE: need to download data from remote: python scripts/get_data.py qlib_data_cn --target_dir ~/.qlib/qlib_data/cn_data\n", "provider_uri = \"~/.qlib/qlib_data/cn_data\" # target_dir\n", "if not exists_qlib_data(provider_uri):\n", " print(f\"Qlib data is not found in {provider_uri}\")\n", " sys.path.append(str(scripts_dir))\n", " from get_data import GetData\n", "\n", " GetData().qlib_data(target_dir=provider_uri, region=REG_CN)\n", "qlib.init(provider_uri=provider_uri, region=REG_CN)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2026-01-19T14:28:40.132494481Z", "start_time": "2026-01-19T14:28:39.901180641Z" } }, "outputs": [], "source": [ "market = \"csi300\"\n", "benchmark = \"SH000300\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# train model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2026-01-19T14:29:03.185557418Z", "start_time": "2026-01-19T14:28:40.145127140Z" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[3509038:MainThread](2026-01-27 22:57:14,729) INFO - qlib.timer - [log.py:127] - Time cost: 30.880s | Loading data Done\n" ] } ], "source": [ "###################################\n", "# train model\n", "###################################\n", "start_date = '2008-01-01'\n", "end_date = '2026-01-01'\n", "fit_end_date = \"2022-12-31\"\n", "validation_start_date = \"2023-01-01\"\n", "validation_end_date = \"2024-12-31\"\n", "test_start_date = \"2025-01-01\"\n", "\n", "data_handler_config = {\n", " \"start_time\": start_date,\n", " \"end_time\": end_date,\n", " \"fit_start_time\": start_date,\n", " \"fit_end_time\": fit_end_date,\n", " \"instruments\": market,\n", "}\n", "\n", "task = {\n", " \"model\": {\n", " \"class\": \"LGBModel\",\n", " \"module_path\": \"qlib.contrib.model.gbdt\",\n", " \"kwargs\": {\n", " \"loss\": \"mse\",\n", " \"colsample_bytree\": 0.8879,\n", " \"learning_rate\": 0.0421,\n", " \"subsample\": 0.8789,\n", " \"lambda_l1\": 205.6999,\n", " \"lambda_l2\": 580.9768,\n", " \"max_depth\": 8,\n", " \"num_leaves\": 210,\n", " \"num_threads\": 20,\n", " },\n", " },\n", " \"dataset\": {\n", " \"class\": \"DatasetH\",\n", " \"module_path\": \"qlib.data.dataset\",\n", " \"kwargs\": {\n", " \"handler\": {\n", " \"class\": \"Alpha158\",\n", " \"module_path\": \"qlib.contrib.data.handler\",\n", " \"kwargs\": data_handler_config,\n", " },\n", " \"segments\": {\n", " \"train\": (start_date, fit_end_date),\n", " \"valid\": (validation_start_date, validation_end_date),\n", " \"test\": (test_start_date, end_date),\n", " },\n", " },\n", " },\n", "}\n", "\n", "# model initialization\n", "model = init_instance_by_config(task[\"model\"])\n", "dataset = init_instance_by_config(task[\"dataset\"])\n", "\n", "# start exp to train model\n", "with R.start(experiment_name=\"train_model\"):\n", " R.log_params(**flatten_dict(task))\n", " model.fit(dataset)\n", " R.save_objects(trained_model=model)\n", " rid = R.get_recorder().id" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# prediction, backtest & analysis" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "tags": [] }, "outputs": [], "source": [ "###################################\n", "# prediction, backtest & analysis\n", "###################################\n", "port_analysis_config = {\n", " \"executor\": {\n", " \"class\": \"SimulatorExecutor\",\n", " \"module_path\": \"qlib.backtest.executor\",\n", " \"kwargs\": {\n", " \"time_per_step\": \"day\",\n", " \"generate_portfolio_metrics\": True,\n", " },\n", " },\n", " \"strategy\": {\n", " \"class\": \"TopkDropoutStrategy\",\n", " \"module_path\": \"qlib.contrib.strategy.signal_strategy\",\n", " \"kwargs\": {\n", " \"model\": model,\n", " \"dataset\": dataset,\n", " \"topk\": 50,\n", " \"n_drop\": 5,\n", " },\n", " },\n", " \"backtest\": {\n", " \"start_time\": test_start_date,\n", " \"end_time\": end_date,\n", " \"account\": 1000000,\n", " \"benchmark\": benchmark,\n", " \"exchange_kwargs\": {\n", " \"freq\": \"day\",\n", " \"limit_threshold\": 0.095,\n", " \"deal_price\": \"close\",\n", " \"open_cost\": 0.0005,\n", " \"close_cost\": 0.0015,\n", " \"min_cost\": 5,\n", " },\n", " },\n", "}\n", "\n", "# backtest and analysis\n", "with R.start(experiment_name=\"backtest_analysis\"):\n", " recorder = R.get_recorder(recorder_id=rid, experiment_name=\"train_model\")\n", " model = recorder.load_object(\"trained_model\")\n", "\n", " # prediction\n", " recorder = R.get_recorder()\n", " ba_rid = recorder.id\n", " sr = SignalRecord(model, dataset, recorder)\n", " sr.generate()\n", "\n", " # backtest & analysis\n", " par = PortAnaRecord(recorder, port_analysis_config, \"day\")\n", " par.generate()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# analyze graphs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from qlib.contrib.report import analysis_model, analysis_position\n", "from qlib.data import D\n", "\n", "recorder = R.get_recorder(recorder_id=ba_rid, experiment_name=\"backtest_analysis\")\n", "print(recorder)\n", "pred_df = recorder.load_object(\"pred.pkl\")\n", "report_normal_df = recorder.load_object(\"portfolio_analysis/report_normal_1day.pkl\")\n", "positions = recorder.load_object(\"portfolio_analysis/positions_normal_1day.pkl\")\n", "analysis_df = recorder.load_object(\"portfolio_analysis/port_analysis_1day.pkl\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## analysis position" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### report" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "###################################\n", "# Display trading details\n", "###################################\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"BACKTEST CONFIGURATION\")\n", "print(\"=\" * 80)\n", "print(f\"Initial Balance: ¥{port_analysis_config['backtest']['account']:,.2f}\")\n", "print(f\"Backtest Period: {port_analysis_config['backtest']['start_time']} to {port_analysis_config['backtest']['end_time']}\")\n", "print(f\"Benchmark: {port_analysis_config['backtest']['benchmark']}\")\n", "print(f\"Strategy: Top {port_analysis_config['strategy']['kwargs']['topk']} stocks, drop {port_analysis_config['strategy']['kwargs']['n_drop']}\")\n", "print(\"=\" * 80)\n", "\n", "\n", "# In[ ]:\n", "\n", "\n", "###################################\n", "# Show daily positions and operations\n", "###################################\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"DAILY TRADING POSITIONS\")\n", "print(\"=\" * 80)\n", "\n", "# Positions is a dict, convert to DataFrame for better viewing\n", "if isinstance(positions, dict):\n", " print(f\"\\nTotal trading days: {len(positions)}\")\n", " \n", " # Convert positions dict to a cleaner format\n", " position_summary = []\n", " for date, pos_data in sorted(positions.items())[:10]: # First 10 days\n", " if isinstance(pos_data, dict) and 'position' in pos_data:\n", " pos = pos_data['position']\n", " cash = pos.get('cash', 0)\n", " account_value = pos.get('now_account_value', 0)\n", " \n", " # Count stocks held\n", " stocks_held = {k: v for k, v in pos.items() if k not in ['cash', 'now_account_value']}\n", " \n", " print(f\"\\n{date}:\")\n", " print(f\" Cash: ¥{float(cash):,.2f}\")\n", " print(f\" Account Value: ¥{float(account_value):,.2f}\")\n", " print(f\" Stocks Held: {len(stocks_held)}\")\n", " \n", " if len(stocks_held) > 0:\n", " print(f\"\\n Top 10 Holdings:\")\n", " # Sort by weight if available\n", " holdings_list = []\n", " for stock, data in stocks_held.items():\n", " if isinstance(data, dict):\n", " amount = float(data.get('amount', 0))\n", " price = float(data.get('price', 0))\n", " weight = float(data.get('weight', 0))\n", " value = amount * price\n", " holdings_list.append((stock, amount, price, weight, value))\n", " \n", " # Sort by value descending\n", " holdings_list.sort(key=lambda x: x[4], reverse=True)\n", " \n", " print(f\" {'Stock':<12} {'Shares':<12} {'Price':<12} {'Value':<15} {'Weight':<10}\")\n", " print(f\" {'-'*12} {'-'*12} {'-'*12} {'-'*15} {'-'*10}\")\n", " for stock, amount, price, weight, value in holdings_list[:10]:\n", " print(f\" {stock:<12} {amount:>12,.2f} {price:>12,.2f} ¥{value:>13,.2f} {weight*100:>9.2f}%\")\n", "else:\n", " print(\"\\nPositions DataFrame (first 20 days):\")\n", " print(positions.head(20))\n", "\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"PORTFOLIO PERFORMANCE REPORT\")\n", "print(\"=\" * 80)\n", "print(\"\\nDaily Returns and Portfolio Value (first 20 days):\")\n", "print(report_normal_df.head(20).to_string())\n", "\n", "# Summary statistics\n", "if 'return' in report_normal_df.columns:\n", " print(\"\\n\" + \"=\" * 80)\n", " print(\"PERFORMANCE SUMMARY\")\n", " print(\"=\" * 80)\n", " total_return = (report_normal_df['return'] + 1).prod() - 1\n", " print(f\"Total Return: {total_return:.2%}\")\n", " print(f\"Annualized Return: {report_normal_df['return'].mean() * 252:.2%}\")\n", " print(f\"Sharpe Ratio: {report_normal_df['return'].mean() / report_normal_df['return'].std() * np.sqrt(252):.4f}\")\n", " print(f\"Max Drawdown: {(report_normal_df['return'] + 1).cumprod().div((report_normal_df['return'] + 1).cumprod().cummax()).min() - 1:.2%}\")\n", " print(f\"Volatility: {report_normal_df['return'].std() * np.sqrt(252):.2%}\")\n", "\n", "\n", "# In[ ]:\n", "\n", "\n", "###################################\n", "# Show detailed trading actions\n", "###################################\n", "print(\"\\n\" + \"=\" * 80)\n", "print(\"TRADING ACTIVITY SUMMARY\")\n", "print(\"=\" * 80)\n", "\n", "# Calculate daily trading activity from position changes\n", "if isinstance(positions, dict):\n", " dates = sorted(positions.keys())\n", " print(f\"\\nAnalyzing {len(dates)} trading days...\\n\")\n", " \n", " # Track position changes\n", " prev_stocks = set()\n", " \n", " print(f\"{'Date':<20} {'Action':<15} {'Stocks In':<12} {'Stocks Out':<12} {'Total Held':<12}\")\n", " print(f\"{'-'*20} {'-'*15} {'-'*12} {'-'*12} {'-'*12}\")\n", " \n", " for i, date in enumerate(dates[:20]): # First 20 days\n", " pos_data = positions[date]\n", " \n", " try:\n", " # Access Position object's position attribute\n", " if hasattr(pos_data, 'position'):\n", " pos = pos_data.position\n", " elif hasattr(pos_data, '__dict__'):\n", " pos = pos_data.__dict__.get('position', {})\n", " else:\n", " pos = dict(pos_data) if hasattr(pos_data, '__iter__') else {}\n", " \n", " # If pos is still an object, try to get its dict\n", " if hasattr(pos, '__dict__') and not isinstance(pos, dict):\n", " pos = pos.__dict__\n", " \n", " current_stocks = set([k for k in pos.keys() if k not in ['cash', 'now_account_value']])\n", " \n", " if i == 0:\n", " action = \"Initial\"\n", " stocks_in = len(current_stocks)\n", " stocks_out = 0\n", " else:\n", " stocks_in_set = current_stocks - prev_stocks\n", " stocks_out_set = prev_stocks - current_stocks\n", " stocks_in = len(stocks_in_set)\n", " stocks_out = len(stocks_out_set)\n", " \n", " if stocks_in > 0 and stocks_out > 0:\n", " action = \"Rebalance\"\n", " elif stocks_in > 0:\n", " action = \"Buy\"\n", " elif stocks_out > 0:\n", " action = \"Sell\"\n", " else:\n", " action = \"Hold\"\n", " \n", " print(f\"{str(date):<20} {action:<15} {stocks_in:<12} {stocks_out:<12} {len(current_stocks):<12}\")\n", " \n", " prev_stocks = current_stocks\n", " except Exception as e:\n", " print(f\"{str(date):<20} Error: {str(e)[:30]}\")\n", " \n", " # Show detailed trades for key days\n", " print(\"\\n\" + \"=\" * 80)\n", " print(\"DETAILED POSITION CHANGES (First 3 Trading Days)\")\n", " print(\"=\" * 80)\n", " \n", " prev_holdings = {}\n", " for i, date in enumerate(dates[:3]):\n", " pos_data = positions[date]\n", " \n", " try:\n", " # Access Position object's position attribute\n", " if hasattr(pos_data, 'position'):\n", " pos = pos_data.position\n", " elif hasattr(pos_data, '__dict__'):\n", " pos = pos_data.__dict__.get('position', {})\n", " else:\n", " pos = dict(pos_data) if hasattr(pos_data, '__iter__') else {}\n", " \n", " # If pos is still an object, try to get its dict\n", " if hasattr(pos, '__dict__') and not isinstance(pos, dict):\n", " pos = pos.__dict__\n", " \n", " current_holdings = {k: v for k, v in pos.items() if k not in ['cash', 'now_account_value']}\n", " \n", " print(f\"\\n{date}:\")\n", " \n", " if i == 0:\n", " print(f\" Initial purchase of {len(current_holdings)} stocks\")\n", " else:\n", " # Find what was bought and sold\n", " current_stocks = set(current_holdings.keys())\n", " prev_stocks = set(prev_holdings.keys())\n", " \n", " bought = current_stocks - prev_stocks\n", " sold = prev_stocks - current_stocks\n", " \n", " if bought:\n", " print(f\" Bought ({len(bought)} stocks): {', '.join(list(bought)[:5])}{'...' if len(bought) > 5 else ''}\")\n", " if sold:\n", " print(f\" Sold ({len(sold)} stocks): {', '.join(list(sold)[:5])}{'...' if len(sold) > 5 else ''}\")\n", " if not bought and not sold:\n", " print(f\" Held all {len(current_holdings)} positions\")\n", " \n", " prev_holdings = current_holdings\n", " except Exception as e:\n", " print(f\" Error processing: {e}\")\n", "\n", "else:\n", " print(\"Position data format not recognized\")\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "analysis_position.report_graph(report_normal_df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### risk analysis" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "analysis_position.risk_analysis_graph(analysis_df, report_normal_df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## analysis model" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "label_df = dataset.prepare(\"test\", col_set=\"label\")\n", "label_df.columns = [\"label\"]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### score IC" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pred_label = pd.concat([label_df, pred_df], axis=1, sort=True).reindex(label_df.index)\n", "analysis_position.score_ic_graph(pred_label)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### model performance" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "analysis_model.model_performance_graph(pred_label)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.11" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 }