Merge branch 'master' into docs/install-py

Author: StrikerRUS

Diff for: .ci/conda-envs/ci-core-py38.txt

@@ -39,6 +39,10 @@ pytest=8.2.*
 # pinned here to help speed up solves
 bokeh=3.1.*
 fsspec=2024.5.*
+# pinning 'libabseil' and 'libre2' to specific build numbers for pyarrow compatibility:
+# ref: https://github.com/microsoft/LightGBM/issues/6772
+libabseil=20240722.0=*_1
+libre2-11=2024.07.02=*_1
 msgpack-python=1.0.*
 pluggy=1.5.*
 pyparsing=3.1.4

Diff for: .github/workflows/lock.yml

@@ -2,8 +2,10 @@ name: 'Lock Inactive Threads'
 
 on:
   schedule:
-    # midnight UTC, every Wednesday
+    # midnight UTC, every Wednesday, for Issues
     - cron: '0 0 * * 3'
+    # midnight UTC, every Thursday, for PRs
+    - cron: '0 0 * * 4'
   # allow manual triggering from GitHub UI
   workflow_dispatch:
 
@@ -42,4 +44,4 @@ jobs:
           # what should the locking status be?
           issue-lock-reason: 'resolved'
           pr-lock-reason: 'resolved'
-          process-only: 'issues, prs'
+          process-only: ${{ github.event.schedule == '0 0 * * 3' && 'issues' || 'prs' }}

Diff for: .pre-commit-config.yaml

@@ -17,6 +17,11 @@ repos:
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.35.1
+    hooks:
+      - id: yamllint
+        args: ["--strict"]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
     rev: v0.8.3

Diff for: .yamllint.yml

@@ -0,0 +1,14 @@
+# default config: https://yamllint.readthedocs.io/en/stable/configuration.html#default-configuration
+extends: default
+
+rules:
+  document-start: disable
+  line-length:
+    max: 999  # temporarily increase allowed line length
+  truthy:
+    # prevent treating GitHub Workflow "on" key as boolean value
+    check-keys: false
+
+  # temporarily disabled rules
+  indentation: disable
+  comments-indentation: disable

Diff for: CMakeLists.txt

@@ -252,54 +252,6 @@ if(USE_CUDA)
       set(CMAKE_CUDA_STANDARD 11)
       set(CMAKE_CUDA_STANDARD_REQUIRED ON)
     endif()
-
-    set(
-      BASE_DEFINES
-      -DPOWER_FEATURE_WORKGROUPS=12
-      -DUSE_CONSTANT_BUF=0
-    )
-    set(
-      ALLFEATS_DEFINES
-      ${BASE_DEFINES}
-      -DENABLE_ALL_FEATURES
-    )
-    set(
-      FULLDATA_DEFINES
-      ${ALLFEATS_DEFINES}
-      -DIGNORE_INDICES
-    )
-
-    message(STATUS "ALLFEATS_DEFINES: ${ALLFEATS_DEFINES}")
-    message(STATUS "FULLDATA_DEFINES: ${FULLDATA_DEFINES}")
-
-    function(add_histogram hsize hname hadd hconst hdir)
-      add_library(histo${hsize}${hname} OBJECT src/treelearner/kernels/histogram${hsize}.cu)
-      set_target_properties(
-        histo${hsize}${hname}
-        PROPERTIES
-          CUDA_SEPARABLE_COMPILATION ON
-          CUDA_ARCHITECTURES ${CUDA_ARCHS}
-      )
-      if(hadd)
-        list(APPEND histograms histo${hsize}${hname})
-        set(histograms ${histograms} PARENT_SCOPE)
-      endif()
-      target_compile_definitions(
-        histo${hsize}${hname}
-        PRIVATE
-        -DCONST_HESSIAN=${hconst}
-        ${hdir}
-      )
-    endfunction()
-
-    foreach(hsize _16_64_256)
-      add_histogram("${hsize}" "_sp_const" "True" "1" "${BASE_DEFINES}")
-      add_histogram("${hsize}" "_sp" "True" "0" "${BASE_DEFINES}")
-      add_histogram("${hsize}" "-allfeats_sp_const" "False" "1" "${ALLFEATS_DEFINES}")
-      add_histogram("${hsize}" "-allfeats_sp" "False" "0" "${ALLFEATS_DEFINES}")
-      add_histogram("${hsize}" "-fulldata_sp_const" "True" "1" "${FULLDATA_DEFINES}")
-      add_histogram("${hsize}" "-fulldata_sp" "True" "0" "${FULLDATA_DEFINES}")
-    endforeach()
 endif()
 
 include(CheckCXXSourceCompiles)
@@ -634,14 +586,6 @@ if(USE_CUDA)
         CUDA_RESOLVE_DEVICE_SYMBOLS ON
     )
   endif()
-
-  # histograms are list of object libraries. Linking object library to other
-  # object libraries only gets usage requirements, the linked objects won't be
-  # used. Thus we have to call target_link_libraries on final targets here.
-  if(BUILD_CLI)
-    target_link_libraries(lightgbm PRIVATE ${histograms})
-  endif()
-  target_link_libraries(_lightgbm PRIVATE ${histograms})
 endif()
 
 if(WIN32)

Diff for: python-package/lightgbm/basic.py

@@ -1248,7 +1248,7 @@ def predict(
         if pred_leaf:
             preds = preds.astype(np.int32)
         is_sparse = isinstance(preds, (list, scipy.sparse.spmatrix))
-        if not is_sparse and preds.size != nrow:
+        if not is_sparse and (preds.size != nrow or pred_leaf or pred_contrib):
             if preds.size % nrow == 0:
                 preds = preds.reshape(nrow, -1)
             else:
@@ -2126,6 +2126,8 @@ def _lazy_init(
                 categorical_feature=categorical_feature,
                 pandas_categorical=self.pandas_categorical,
             )
+        elif _is_pyarrow_table(data) and feature_name == "auto":
+            feature_name = data.column_names
 
         # process for args
         params = {} if params is None else params
@@ -2185,7 +2187,6 @@ def _lazy_init(
             self.__init_from_np2d(data, params_str, ref_dataset)
         elif _is_pyarrow_table(data):
             self.__init_from_pyarrow_table(data, params_str, ref_dataset)
-            feature_name = data.column_names
         elif isinstance(data, list) and len(data) > 0:
             if _is_list_of_numpy_arrays(data):
                 self.__init_from_list_np2d(data, params_str, ref_dataset)

Diff for: python-package/lightgbm/callback.py

@@ -71,6 +71,14 @@ class CallbackEnv:
     evaluation_result_list: Optional[_ListOfEvalResultTuples]
 
 
+def _is_using_cv(env: CallbackEnv) -> bool:
+    """Check if model in callback env is a CVBooster."""
+    # this import is here to avoid a circular import
+    from .engine import CVBooster
+
+    return isinstance(env.model, CVBooster)
+
+
 def _format_eval_result(value: _EvalResultTuple, show_stdv: bool) -> str:
     """Format metric string."""
     dataset_name, metric_name, metric_value, *_ = value
@@ -143,16 +151,13 @@ def _init(self, env: CallbackEnv) -> None:
             )
         self.eval_result.clear()
         for item in env.evaluation_result_list:
-            if len(item) == 4:  # regular train
-                data_name, eval_name = item[:2]
-            else:  # cv
-                data_name, eval_name = item[1].split()
-            self.eval_result.setdefault(data_name, OrderedDict())
+            dataset_name, metric_name, *_ = item
+            self.eval_result.setdefault(dataset_name, OrderedDict())
             if len(item) == 4:
-                self.eval_result[data_name].setdefault(eval_name, [])
+                self.eval_result[dataset_name].setdefault(metric_name, [])
             else:
-                self.eval_result[data_name].setdefault(f"{eval_name}-mean", [])
-                self.eval_result[data_name].setdefault(f"{eval_name}-stdv", [])
+                self.eval_result[dataset_name].setdefault(f"{metric_name}-mean", [])
+                self.eval_result[dataset_name].setdefault(f"{metric_name}-stdv", [])
 
     def __call__(self, env: CallbackEnv) -> None:
         if env.iteration == env.begin_iteration:
@@ -163,15 +168,16 @@ def __call__(self, env: CallbackEnv) -> None:
                 "Please report it at https://github.com/microsoft/LightGBM/issues"
             )
         for item in env.evaluation_result_list:
+            # for cv(), 'metric_value' is actually a mean of metric values over all CV folds
+            dataset_name, metric_name, metric_value, *_ = item
             if len(item) == 4:
-                data_name, eval_name, result = item[:3]
-                self.eval_result[data_name][eval_name].append(result)
+                # train()
+                self.eval_result[dataset_name][metric_name].append(metric_value)
             else:
-                data_name, eval_name = item[1].split()
-                res_mean = item[2]
-                res_stdv = item[4]  # type: ignore[misc]
-                self.eval_result[data_name][f"{eval_name}-mean"].append(res_mean)
-                self.eval_result[data_name][f"{eval_name}-stdv"].append(res_stdv)
+                # cv()
+                metric_std_dev = item[4]  # type: ignore[misc]
+                self.eval_result[dataset_name][f"{metric_name}-mean"].append(metric_value)
+                self.eval_result[dataset_name][f"{metric_name}-stdv"].append(metric_std_dev)
 
 
 def record_evaluation(eval_result: Dict[str, Dict[str, List[Any]]]) -> Callable:
@@ -304,15 +310,15 @@ def _gt_delta(self, curr_score: float, best_score: float, delta: float) -> bool:
     def _lt_delta(self, curr_score: float, best_score: float, delta: float) -> bool:
         return curr_score < best_score - delta
 
-    def _is_train_set(self, ds_name: str, eval_name: str, env: CallbackEnv) -> bool:
+    def _is_train_set(self, dataset_name: str, env: CallbackEnv) -> bool:
         """Check, by name, if a given Dataset is the training data."""
         # for lgb.cv() with eval_train_metric=True, evaluation is also done on the training set
         # and those metrics are considered for early stopping
-        if ds_name == "cv_agg" and eval_name == "train":
+        if _is_using_cv(env) and dataset_name == "train":
             return True
 
         # for lgb.train(), it's possible to pass the training data via valid_sets with any eval_name
-        if isinstance(env.model, Booster) and ds_name == env.model._train_data_name:
+        if isinstance(env.model, Booster) and dataset_name == env.model._train_data_name:
             return True
 
         return False
@@ -327,11 +333,13 @@ def _init(self, env: CallbackEnv) -> None:
             _log_warning("Early stopping is not available in dart mode")
             return
 
+        # get details of the first dataset
+        first_dataset_name, first_metric_name, *_ = env.evaluation_result_list[0]
+
         # validation sets are guaranteed to not be identical to the training data in cv()
         if isinstance(env.model, Booster):
             only_train_set = len(env.evaluation_result_list) == 1 and self._is_train_set(
-                ds_name=env.evaluation_result_list[0][0],
-                eval_name=env.evaluation_result_list[0][1].split(" ")[0],
+                dataset_name=first_dataset_name,
                 env=env,
             )
             if only_train_set:
@@ -370,8 +378,7 @@ def _init(self, env: CallbackEnv) -> None:
                 _log_info(f"Using {self.min_delta} as min_delta for all metrics.")
             deltas = [self.min_delta] * n_datasets * n_metrics
 
-        # split is needed for "<dataset type> <metric>" case (e.g. "train l1")
-        self.first_metric = env.evaluation_result_list[0][1].split(" ")[-1]
+        self.first_metric = first_metric_name
         for eval_ret, delta in zip(env.evaluation_result_list, deltas):
             self.best_iter.append(0)
             if eval_ret[3]:  # greater is better
@@ -381,15 +388,15 @@ def _init(self, env: CallbackEnv) -> None:
                 self.best_score.append(float("inf"))
                 self.cmp_op.append(partial(self._lt_delta, delta=delta))
 
-    def _final_iteration_check(self, env: CallbackEnv, eval_name_splitted: List[str], i: int) -> None:
+    def _final_iteration_check(self, *, env: CallbackEnv, metric_name: str, i: int) -> None:
         if env.iteration == env.end_iteration - 1:
             if self.verbose:
                 best_score_str = "\t".join([_format_eval_result(x, show_stdv=True) for x in self.best_score_list[i]])
                 _log_info(
                     "Did not meet early stopping. " f"Best iteration is:\n[{self.best_iter[i] + 1}]\t{best_score_str}"
                 )
                 if self.first_metric_only:
-                    _log_info(f"Evaluated only: {eval_name_splitted[-1]}")
+                    _log_info(f"Evaluated only: {metric_name}")
             raise EarlyStopException(self.best_iter[i], self.best_score_list[i])
 
     def __call__(self, env: CallbackEnv) -> None:
@@ -405,21 +412,18 @@ def __call__(self, env: CallbackEnv) -> None:
         # self.best_score_list is initialized to an empty list
         first_time_updating_best_score_list = self.best_score_list == []
         for i in range(len(env.evaluation_result_list)):
-            score = env.evaluation_result_list[i][2]
-            if first_time_updating_best_score_list or self.cmp_op[i](score, self.best_score[i]):
-                self.best_score[i] = score
+            dataset_name, metric_name, metric_value, *_ = env.evaluation_result_list[i]
+            if first_time_updating_best_score_list or self.cmp_op[i](metric_value, self.best_score[i]):
+                self.best_score[i] = metric_value
                 self.best_iter[i] = env.iteration
                 if first_time_updating_best_score_list:
                     self.best_score_list.append(env.evaluation_result_list)
                 else:
                     self.best_score_list[i] = env.evaluation_result_list
-            # split is needed for "<dataset type> <metric>" case (e.g. "train l1")
-            eval_name_splitted = env.evaluation_result_list[i][1].split(" ")
-            if self.first_metric_only and self.first_metric != eval_name_splitted[-1]:
+            if self.first_metric_only and self.first_metric != metric_name:
                 continue  # use only the first metric for early stopping
             if self._is_train_set(
-                ds_name=env.evaluation_result_list[i][0],
-                eval_name=eval_name_splitted[0],
+                dataset_name=dataset_name,
                 env=env,
             ):
                 continue  # train data for lgb.cv or sklearn wrapper (underlying lgb.train)
@@ -430,9 +434,9 @@ def __call__(self, env: CallbackEnv) -> None:
                     )
                     _log_info(f"Early stopping, best iteration is:\n[{self.best_iter[i] + 1}]\t{eval_result_str}")
                     if self.first_metric_only:
-                        _log_info(f"Evaluated only: {eval_name_splitted[-1]}")
+                        _log_info(f"Evaluated only: {metric_name}")
                 raise EarlyStopException(self.best_iter[i], self.best_score_list[i])
-            self._final_iteration_check(env, eval_name_splitted, i)
+            self._final_iteration_check(env=env, metric_name=metric_name, i=i)
 
 
 def _should_enable_early_stopping(stopping_rounds: Any) -> bool:

Diff for: python-package/lightgbm/engine.py

@@ -581,15 +581,31 @@ def _agg_cv_result(
     raw_results: List[List[_LGBM_BoosterEvalMethodResultType]],
 ) -> List[_LGBM_BoosterEvalMethodResultWithStandardDeviationType]:
     """Aggregate cross-validation results."""
-    cvmap: Dict[str, List[float]] = OrderedDict()
-    metric_type: Dict[str, bool] = {}
+    # build up 2 maps, of the form:
+    #
+    # OrderedDict{
+    #     (<dataset_name>, <metric_name>): <is_higher_better>
+    # }
+    #
+    # OrderedDict{
+    #     (<dataset_name>, <metric_name>): list[<metric_value>]
+    # }
+    #
+    metric_types: Dict[Tuple[str, str], bool] = OrderedDict()
+    metric_values: Dict[Tuple[str, str], List[float]] = OrderedDict()
     for one_result in raw_results:
-        for one_line in one_result:
-            key = f"{one_line[0]} {one_line[1]}"
-            metric_type[key] = one_line[3]
-            cvmap.setdefault(key, [])
-            cvmap[key].append(one_line[2])
-    return [("cv_agg", k, float(np.mean(v)), metric_type[k], float(np.std(v))) for k, v in cvmap.items()]
+        for dataset_name, metric_name, metric_value, is_higher_better in one_result:
+            key = (dataset_name, metric_name)
+            metric_types[key] = is_higher_better
+            metric_values.setdefault(key, [])
+            metric_values[key].append(metric_value)
+
+    # turn that into a list of tuples of the form:
+    #
+    # [
+    #     (<dataset_name>, <metric_name>, mean(<values>), <is_higher_better>, std_dev(<values>))
+    # ]
+    return [(k[0], k[1], float(np.mean(v)), metric_types[k], float(np.std(v))) for k, v in metric_values.items()]
 
 
 def cv(
@@ -812,9 +828,9 @@ def cv(
             )
         cvbooster.update(fobj=fobj)  # type: ignore[call-arg]
         res = _agg_cv_result(cvbooster.eval_valid(feval))  # type: ignore[call-arg]
-        for _, key, mean, _, std in res:
-            results[f"{key}-mean"].append(mean)
-            results[f"{key}-stdv"].append(std)
+        for dataset_name, metric_name, metric_mean, _, metric_std_dev in res:
+            results[f"{dataset_name} {metric_name}-mean"].append(metric_mean)
+            results[f"{dataset_name} {metric_name}-stdv"].append(metric_std_dev)
         try:
             for cb in callbacks_after_iter:
                 cb(