Skip to content

Commit c3a15c2

Browse files
trivialfistrivialfis
committed
initial work on precision.
1 parent 0462569 commit c3a15c2

File tree

3 files changed

+107
-38
lines changed

3 files changed

+107
-38
lines changed

‎src/metric/rank_metric.cc

+51-23
Original file line numberDiff line numberDiff line change
@@ -253,23 +253,6 @@ struct EvalRank : public MetricNoCache, public EvalRankConfig {
253253
virtual double EvalGroup(PredIndPairContainer *recptr) const = 0;
254254
};
255255

256-
/*! \brief Precision at N, for both classification and rank */
257-
struct EvalPrecision : public EvalRank {
258-
public:
259-
explicit EvalPrecision(const char* name, const char* param) : EvalRank(name, param) {}
260-
261-
double EvalGroup(PredIndPairContainer *recptr) const override {
262-
PredIndPairContainer &rec(*recptr);
263-
// calculate Precision
264-
std::stable_sort(rec.begin(), rec.end(), common::CmpFirst);
265-
unsigned nhit = 0;
266-
for (size_t j = 0; j < rec.size() && j < this->topn; ++j) {
267-
nhit += (rec[j].second != 0);
268-
}
269-
return static_cast<double>(nhit) / this->topn;
270-
}
271-
};
272-
273256
/*! \brief Cox: Partial likelihood of the Cox proportional hazards model */
274257
struct EvalCox : public MetricNoCache {
275258
public:
@@ -321,10 +304,6 @@ XGBOOST_REGISTER_METRIC(AMS, "ams")
321304
.describe("AMS metric for higgs.")
322305
.set_body([](const char* param) { return new EvalAMS(param); });
323306

324-
XGBOOST_REGISTER_METRIC(Precision, "pre")
325-
.describe("precision@k for rank.")
326-
.set_body([](const char* param) { return new EvalPrecision("pre", param); });
327-
328307
XGBOOST_REGISTER_METRIC(Cox, "cox-nloglik")
329308
.describe("Negative log partial likelihood of Cox proportional hazards model.")
330309
.set_body([](const char*) { return new EvalCox(); });
@@ -387,6 +366,8 @@ class EvalRankWithCache : public Metric {
387366
return result;
388367
}
389368

369+
[[nodiscard]] const char* Name() const override { return name_.c_str(); }
370+
390371
virtual double Eval(HostDeviceVector<float> const& preds, MetaInfo const& info,
391372
std::shared_ptr<Cache> p_cache) = 0;
392373
};
@@ -408,6 +389,51 @@ double Finalize(MetaInfo const& info, double score, double sw) {
408389
}
409390
} // namespace
410391

392+
class EvalPrecision : public EvalRankWithCache<ltr::MAPCache> {
393+
public:
394+
using EvalRankWithCache::EvalRankWithCache;
395+
double Eval(HostDeviceVector<float> const& predt, MetaInfo const& info,
396+
std::shared_ptr<ltr::MAPCache> p_cache) final {
397+
// Fixme: check whether minus is applicable here.
398+
if (ctx_->IsCUDA()) {
399+
auto pre = cuda_impl::PreScore(ctx_, info, predt, minus_, p_cache);
400+
return Finalize(info, pre.Residue(), pre.Weights());
401+
}
402+
403+
auto gptr = p_cache->DataGroupPtr(ctx_);
404+
auto h_label = info.labels.HostView().Slice(linalg::All(), 0);
405+
auto h_predt = linalg::MakeTensorView(ctx_, &predt, predt.Size());
406+
auto rank_idx = p_cache->SortedIdx(ctx_, predt.ConstHostSpan());
407+
408+
auto pre = p_cache->Map(ctx_);
409+
auto topk = p_cache->Param().TopK();
410+
411+
common::ParallelFor(p_cache->Groups(), ctx_->Threads(), [&](auto g) {
412+
auto g_label = h_label.Slice(linalg::Range(gptr[g], gptr[g + 1]));
413+
auto g_rank = rank_idx.subspan(gptr[g]);
414+
415+
auto n = std::min(static_cast<std::size_t>(param_.TopK()), g_label.Size());
416+
double n_hits{0.0};
417+
for (std::size_t i = 0; i < n; ++i) {
418+
n_hits += g_label(g_rank[i]);
419+
}
420+
pre[g] = n_hits / topk;
421+
});
422+
423+
auto sw = 0.0;
424+
auto weight = common::MakeOptionalWeights(ctx_, info.weights_);
425+
if (!weight.Empty()) {
426+
CHECK_EQ(weight.weights.size(), p_cache->Groups());
427+
}
428+
for (std::size_t i = 0; i < pre.size(); ++i) {
429+
pre[i] = pre[i] * weight[i];
430+
sw += weight[i];
431+
}
432+
auto sum = std::accumulate(pre.cbegin(), pre.cend(), 0.0);
433+
return Finalize(info, sum, sw);
434+
}
435+
};
436+
411437
/**
412438
* \brief Implement the NDCG score function for learning to rank.
413439
*
@@ -416,7 +442,6 @@ double Finalize(MetaInfo const& info, double score, double sw) {
416442
class EvalNDCG : public EvalRankWithCache<ltr::NDCGCache> {
417443
public:
418444
using EvalRankWithCache::EvalRankWithCache;
419-
const char* Name() const override { return name_.c_str(); }
420445

421446
double Eval(HostDeviceVector<float> const& preds, MetaInfo const& info,
422447
std::shared_ptr<ltr::NDCGCache> p_cache) override {
@@ -475,7 +500,6 @@ class EvalNDCG : public EvalRankWithCache<ltr::NDCGCache> {
475500
class EvalMAPScore : public EvalRankWithCache<ltr::MAPCache> {
476501
public:
477502
using EvalRankWithCache::EvalRankWithCache;
478-
const char* Name() const override { return name_.c_str(); }
479503

480504
double Eval(HostDeviceVector<float> const& predt, MetaInfo const& info,
481505
std::shared_ptr<ltr::MAPCache> p_cache) override {
@@ -527,6 +551,10 @@ class EvalMAPScore : public EvalRankWithCache<ltr::MAPCache> {
527551
}
528552
};
529553

554+
XGBOOST_REGISTER_METRIC(Precision, "pre")
555+
.describe("precision@k for rank.")
556+
.set_body([](const char* param) { return new EvalPrecision("pre", param); });
557+
530558
XGBOOST_REGISTER_METRIC(EvalMAP, "map")
531559
.describe("map@k for ranking.")
532560
.set_body([](char const* param) {

‎src/metric/rank_metric.cu

+42-8
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ struct EvalRankGpu : public GPUMetric, public EvalRankConfig {
5757
return EvalMetricT::EvalMetric(segment_pred_sorter, dlabels.Values().data(), *this);
5858
}
5959

60-
const char* Name() const override {
60+
[[nodiscard]] const char* Name() const override {
6161
return name.c_str();
6262
}
6363

@@ -133,16 +133,50 @@ namespace cuda_impl {
133133
PackedReduceResult PreScore(Context const *ctx, MetaInfo const &info,
134134
HostDeviceVector<float> const &predt, bool minus,
135135
std::shared_ptr<ltr::MAPCache> p_cache) {
136-
auto d_rank_idx = p_cache->SortedIdx(ctx, predt.ConstDeviceSpan());
137136
auto d_group_ptr = p_cache->DataGroupPtr(ctx);
138-
auto it = dh::MakeTransformIterator<PackedReduceResult>(
139-
thrust::make_counting_iterator(0ul),
140-
[=] XGBOOST_DEVICE(std::size_t i) {
137+
auto d_label = info.labels.View(ctx->gpu_id).Slice(linalg::All(), 0);
138+
139+
predt.SetDevice(ctx->gpu_id);
140+
auto d_rank_idx = p_cache->SortedIdx(ctx, predt.ConstDeviceSpan());
141+
auto topk = p_cache->Param().TopK();
142+
auto d_weight = common::MakeOptionalWeights(ctx, info.weights_);
143+
144+
auto it = dh::MakeTransformIterator<double>(
145+
thrust::make_counting_iterator(0ul), [=] XGBOOST_DEVICE(std::size_t i) {
141146
auto group_idx = dh::SegmentId(d_group_ptr, i);
147+
auto g_begin = d_group_ptr[group_idx];
148+
auto g_end = d_group_ptr[group_idx + 1];
149+
i -= g_begin;
150+
auto g_label = d_label.Slice(linalg::Range(g_begin, g_end));
151+
auto g_rank = d_rank_idx.subspan(g_begin, g_end - g_begin);
152+
auto y = g_label(g_rank[i]);
153+
if (i >= topk) {
154+
return 0.0;
155+
}
156+
return y / static_cast<double>(topk);
142157
});
143-
PackedReduceResult n_hits = thrust::reduce(it, it + info.num_row_);
144-
double topk = p_cache->Param().TopK();
145-
return n_hits / topk;
158+
159+
auto cuctx = ctx->CUDACtx();
160+
auto pre = p_cache->Map(ctx);
161+
thrust::fill_n(cuctx->CTP(), pre.data(), pre.size(), 0.0);
162+
163+
std::size_t bytes;
164+
cub::DeviceSegmentedReduce::Sum(nullptr, bytes, it, pre.data(), p_cache->Groups(),
165+
d_group_ptr.data(), d_group_ptr.data() + 1, cuctx->Stream());
166+
dh::TemporaryArray<char> temp(bytes);
167+
cub::DeviceSegmentedReduce::Sum(nullptr, bytes, it, pre.data(), p_cache->Groups(),
168+
d_group_ptr.data(), d_group_ptr.data() + 1, cuctx->Stream());
169+
170+
if (!d_weight.Empty()) {
171+
CHECK_EQ(d_weight.weights.size(), p_cache->Groups());
172+
}
173+
auto val_it = dh::MakeTransformIterator<PackedReduceResult>(
174+
thrust::make_counting_iterator(0ul), [=] XGBOOST_DEVICE(std::size_t g) {
175+
return PackedReduceResult{pre[g] * d_weight[g], static_cast<double>(d_weight[g])};
176+
});
177+
auto result =
178+
thrust::reduce(cuctx->CTP(), val_it, val_it + pre.size(), PackedReduceResult{0.0, 0.0});
179+
return result;
146180
}
147181

148182
PackedReduceResult NDCGScore(Context const *ctx, MetaInfo const &info,

‎src/metric/rank_metric.h

+14-7
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
/**
44
* Copyright 2023 by XGBoost Contributors
55
*/
6-
#include <memory> // for shared_ptr
6+
#include <memory> // for shared_ptr
77

88
#include "../common/common.h" // for AssertGPUSupport
99
#include "../common/ranking_utils.h" // for NDCGCache, MAPCache
@@ -12,9 +12,7 @@
1212
#include "xgboost/data.h" // for MetaInfo
1313
#include "xgboost/host_device_vector.h" // for HostDeviceVector
1414

15-
namespace xgboost {
16-
namespace metric {
17-
namespace cuda_impl {
15+
namespace xgboost::metric::cuda_impl {
1816
PackedReduceResult NDCGScore(Context const *ctx, MetaInfo const &info,
1917
HostDeviceVector<float> const &predt, bool minus,
2018
std::shared_ptr<ltr::NDCGCache> p_cache);
@@ -23,6 +21,10 @@ PackedReduceResult MAPScore(Context const *ctx, MetaInfo const &info,
2321
HostDeviceVector<float> const &predt, bool minus,
2422
std::shared_ptr<ltr::MAPCache> p_cache);
2523

24+
PackedReduceResult PreScore(Context const *ctx, MetaInfo const &info,
25+
HostDeviceVector<float> const &predt, bool minus,
26+
std::shared_ptr<ltr::MAPCache> p_cache);
27+
2628
#if !defined(XGBOOST_USE_CUDA)
2729
inline PackedReduceResult NDCGScore(Context const *, MetaInfo const &,
2830
HostDeviceVector<float> const &, bool,
@@ -37,8 +39,13 @@ inline PackedReduceResult MAPScore(Context const *, MetaInfo const &,
3739
common::AssertGPUSupport();
3840
return {};
3941
}
42+
43+
inline PackedReduceResult PreScore(Context const *, MetaInfo const &,
44+
HostDeviceVector<float> const &, bool,
45+
std::shared_ptr<ltr::MAPCache>) {
46+
common::AssertGPUSupport();
47+
return {};
48+
}
4049
#endif
41-
} // namespace cuda_impl
42-
} // namespace metric
43-
} // namespace xgboost
50+
} // namespace xgboost::metric::cuda_impl
4451
#endif // XGBOOST_METRIC_RANK_METRIC_H_

0 commit comments

Comments
 (0)