|
| 1 | +""" |
| 2 | +Getting started with learning to rank |
| 3 | +===================================== |
| 4 | +
|
| 5 | + .. versionadded:: 2.0.0 |
| 6 | +
|
| 7 | +This is a demonstration of using XGBoost for learning to rank tasks using the |
| 8 | +MSLR_10k_letor dataset. For more infomation about the dataset, please visit its |
| 9 | +`description page <https://www.microsoft.com/en-us/research/project/mslr/>`_. |
| 10 | +
|
| 11 | +This is a two-part demo, the first one contains a basic example of using XGBoost to |
| 12 | +train on relevance degree, and the second part simulates click data and enable the |
| 13 | +position debiasing training. |
| 14 | +
|
| 15 | +For an overview of learning to rank in XGBoost, please see |
| 16 | +:doc:`Learning to Rank </tutorials/learning_to_rank>`. |
| 17 | +""" |
| 18 | +from __future__ import annotations |
| 19 | + |
| 20 | +import argparse |
| 21 | +import json |
| 22 | +import os |
| 23 | +import pickle as pkl |
| 24 | + |
| 25 | +import numpy as np |
| 26 | +import pandas as pd |
| 27 | +from sklearn.datasets import load_svmlight_file |
| 28 | + |
| 29 | +import xgboost as xgb |
| 30 | +from xgboost.testing.data import RelDataCV, simulate_clicks, sort_ltr_samples |
| 31 | + |
| 32 | + |
| 33 | +def load_mlsr_10k(data_path: str, cache_path: str) -> RelDataCV: |
| 34 | + """Load the MSLR10k dataset from data_path and cache a pickle object in cache_path. |
| 35 | +
|
| 36 | + Returns |
| 37 | + ------- |
| 38 | +
|
| 39 | + A list of tuples [(X, y, qid), ...]. |
| 40 | +
|
| 41 | + """ |
| 42 | + root_path = os.path.expanduser(args.data) |
| 43 | + cacheroot_path = os.path.expanduser(args.cache) |
| 44 | + cache_path = os.path.join(cacheroot_path, "MSLR_10K_LETOR.pkl") |
| 45 | + |
| 46 | + # Use only the Fold1 for demo: |
| 47 | + # Train, Valid, Test |
| 48 | + # {S1,S2,S3}, S4, S5 |
| 49 | + fold = 1 |
| 50 | + |
| 51 | + if not os.path.exists(cache_path): |
| 52 | + fold_path = os.path.join(root_path, f"Fold{fold}") |
| 53 | + train_path = os.path.join(fold_path, "train.txt") |
| 54 | + valid_path = os.path.join(fold_path, "vali.txt") |
| 55 | + test_path = os.path.join(fold_path, "test.txt") |
| 56 | + X_train, y_train, qid_train = load_svmlight_file( |
| 57 | + train_path, query_id=True, dtype=np.float32 |
| 58 | + ) |
| 59 | + y_train = y_train.astype(np.int32) |
| 60 | + qid_train = qid_train.astype(np.int32) |
| 61 | + |
| 62 | + X_valid, y_valid, qid_valid = load_svmlight_file( |
| 63 | + valid_path, query_id=True, dtype=np.float32 |
| 64 | + ) |
| 65 | + y_valid = y_valid.astype(np.int32) |
| 66 | + qid_valid = qid_valid.astype(np.int32) |
| 67 | + |
| 68 | + X_test, y_test, qid_test = load_svmlight_file( |
| 69 | + test_path, query_id=True, dtype=np.float32 |
| 70 | + ) |
| 71 | + y_test = y_test.astype(np.int32) |
| 72 | + qid_test = qid_test.astype(np.int32) |
| 73 | + |
| 74 | + data = RelDataCV( |
| 75 | + train=(X_train, y_train, qid_train), |
| 76 | + test=(X_test, y_test, qid_test), |
| 77 | + max_rel=4, |
| 78 | + ) |
| 79 | + |
| 80 | + with open(cache_path, "wb") as fd: |
| 81 | + pkl.dump(data, fd) |
| 82 | + |
| 83 | + with open(cache_path, "rb") as fd: |
| 84 | + data = pkl.load(fd) |
| 85 | + |
| 86 | + return data |
| 87 | + |
| 88 | + |
| 89 | +def ranking_demo(args: argparse.Namespace) -> None: |
| 90 | + """Demonstration for learning to rank with relevance degree.""" |
| 91 | + data = load_mlsr_10k(args.data, args.cache) |
| 92 | + |
| 93 | + X_train, y_train, qid_train = data.train |
| 94 | + sorted_idx = np.argsort(qid_train) |
| 95 | + X_train = X_train[sorted_idx] |
| 96 | + y_train = y_train[sorted_idx] |
| 97 | + qid_train = qid_train[sorted_idx] |
| 98 | + |
| 99 | + X_test, y_test, qid_test = data.test |
| 100 | + sorted_idx = np.argsort(qid_test) |
| 101 | + X_test = X_test[sorted_idx] |
| 102 | + y_test = y_test[sorted_idx] |
| 103 | + qid_test = qid_test[sorted_idx] |
| 104 | + |
| 105 | + ranker = xgb.XGBRanker( |
| 106 | + tree_method="gpu_hist", |
| 107 | + lambdarank_pair_method="topk", |
| 108 | + lambdarank_num_pair_per_sample=13, |
| 109 | + eval_metric=["ndcg@1", "ndcg@8"], |
| 110 | + ) |
| 111 | + ranker.fit( |
| 112 | + X_train, |
| 113 | + y_train, |
| 114 | + qid=qid_train, |
| 115 | + eval_set=[(X_test, y_test)], |
| 116 | + eval_qid=[qid_test], |
| 117 | + verbose=True, |
| 118 | + ) |
| 119 | + |
| 120 | + |
| 121 | +def click_data_demo(args: argparse.Namespace) -> None: |
| 122 | + """Demonstration for learning to rank with click data.""" |
| 123 | + data = load_mlsr_10k(args.data, args.cache) |
| 124 | + folds = simulate_clicks(data) |
| 125 | + |
| 126 | + train = [pack[0] for pack in folds] |
| 127 | + test = [pack[1] for pack in folds] |
| 128 | + |
| 129 | + X_train, y_train, qid_train, scores_train, clicks_train, position_train = train |
| 130 | + assert X_train.shape[0] == clicks_train.size |
| 131 | + X_test, y_test, qid_test, scores_test, clicks_test, position_test = test |
| 132 | + assert X_test.shape[0] == clicks_test.size |
| 133 | + assert scores_test.dtype == np.float32 |
| 134 | + assert clicks_test.dtype == np.int32 |
| 135 | + |
| 136 | + X_train, clicks_train, y_train, qid_train = sort_ltr_samples( |
| 137 | + X_train, |
| 138 | + y_train, |
| 139 | + qid_train, |
| 140 | + clicks_train, |
| 141 | + position_train, |
| 142 | + ) |
| 143 | + X_test, clicks_test, y_test, qid_test = sort_ltr_samples( |
| 144 | + X_test, |
| 145 | + y_test, |
| 146 | + qid_test, |
| 147 | + clicks_test, |
| 148 | + position_test, |
| 149 | + ) |
| 150 | + |
| 151 | + class ShowPosition(xgb.callback.TrainingCallback): |
| 152 | + def after_iteration(self, model, epoch, evals_log) -> bool: |
| 153 | + config = json.loads(model.save_config()) |
| 154 | + ti_plus = np.array(config["learner"]["objective"]["ti+"]) |
| 155 | + tj_minus = np.array(config["learner"]["objective"]["tj-"]) |
| 156 | + df = pd.DataFrame({"ti+": ti_plus, "tj-": tj_minus}) |
| 157 | + print(df) |
| 158 | + return False |
| 159 | + |
| 160 | + ranker = xgb.XGBRanker( |
| 161 | + n_estimators=512, |
| 162 | + tree_method="gpu_hist", |
| 163 | + learning_rate=0.01, |
| 164 | + reg_lambda=1.5, |
| 165 | + subsample=0.8, |
| 166 | + sampling_method="gradient_based", |
| 167 | + # LTR specific parameters |
| 168 | + objective="rank:ndcg", |
| 169 | + # - Enable bias estimation |
| 170 | + lambdarank_unbiased=True, |
| 171 | + # - normalization (1 / (norm + 1)) |
| 172 | + lambdarank_bias_norm=1, |
| 173 | + # - Focus on the top 12 documents |
| 174 | + lambdarank_num_pair_per_sample=12, |
| 175 | + lambdarank_pair_method="topk", |
| 176 | + ndcg_exp_gain=True, |
| 177 | + eval_metric=["ndcg@1", "ndcg@3", "ndcg@5", "ndcg@10"], |
| 178 | + callbacks=[ShowPosition()], |
| 179 | + ) |
| 180 | + ranker.fit( |
| 181 | + X_train, |
| 182 | + clicks_train, |
| 183 | + qid=qid_train, |
| 184 | + eval_set=[(X_test, y_test), (X_test, clicks_test)], |
| 185 | + eval_qid=[qid_test, qid_test], |
| 186 | + verbose=True, |
| 187 | + ) |
| 188 | + ranker.predict(X_test) |
| 189 | + |
| 190 | + |
| 191 | +if __name__ == "__main__": |
| 192 | + parser = argparse.ArgumentParser( |
| 193 | + description="Demonstration of learning to rank using XGBoost." |
| 194 | + ) |
| 195 | + parser.add_argument( |
| 196 | + "--data", |
| 197 | + type=str, |
| 198 | + help="Root directory of the MSLR-WEB10K data.", |
| 199 | + required=True, |
| 200 | + ) |
| 201 | + parser.add_argument( |
| 202 | + "--cache", |
| 203 | + type=str, |
| 204 | + help="Directory for caching processed data.", |
| 205 | + required=True, |
| 206 | + ) |
| 207 | + args = parser.parse_args() |
| 208 | + |
| 209 | + ranking_demo(args) |
| 210 | + click_data_demo(args) |
0 commit comments