Changed Ranker to Ranking in evaluation related files. (#2675)

Author: zeahmed

src/Microsoft.ML.Data/EntryPoints/InputBuilder.cs

@@ -850,8 +850,8 @@ public static class PipelineSweeperSupportedMetrics
             public const string RSquared = RegressionLossEvaluatorBase<MultiOutputRegressionEvaluator.Aggregator>.RSquared;
             public const string LogLoss = BinaryClassifierEvaluator.LogLoss;
             public const string LogLossReduction = BinaryClassifierEvaluator.LogLossReduction;
-            public const string Ndcg = RankerEvaluator.Ndcg;
-            public const string Dcg = RankerEvaluator.Dcg;
+            public const string Ndcg = RankingEvaluator.Ndcg;
+            public const string Dcg = RankingEvaluator.Dcg;
             public const string PositivePrecision = BinaryClassifierEvaluator.PosPrecName;
             public const string PositiveRecall = BinaryClassifierEvaluator.PosRecallName;
             public const string NegativePrecision = BinaryClassifierEvaluator.NegPrecName;

src/Microsoft.ML.Data/Evaluators/EvaluatorUtils.cs

@@ -44,7 +44,7 @@ public static Dictionary<string, Func<IHostEnvironment, IMamlEvaluator>> Instanc
                             { MetadataUtils.Const.ScoreColumnKind.Regression, env => new RegressionMamlEvaluator(env, new RegressionMamlEvaluator.Arguments()) },
                             { MetadataUtils.Const.ScoreColumnKind.MultiOutputRegression, env => new MultiOutputRegressionMamlEvaluator(env, new MultiOutputRegressionMamlEvaluator.Arguments()) },
                             { MetadataUtils.Const.ScoreColumnKind.QuantileRegression, env => new QuantileRegressionMamlEvaluator(env, new QuantileRegressionMamlEvaluator.Arguments()) },
-                            { MetadataUtils.Const.ScoreColumnKind.Ranking, env => new RankerMamlEvaluator(env, new RankerMamlEvaluator.Arguments()) },
+                            { MetadataUtils.Const.ScoreColumnKind.Ranking, env => new RankingMamlEvaluator(env, new RankingMamlEvaluator.Arguments()) },
                             { MetadataUtils.Const.ScoreColumnKind.Clustering, env => new ClusteringMamlEvaluator(env, new ClusteringMamlEvaluator.Arguments()) },
                             { MetadataUtils.Const.ScoreColumnKind.AnomalyDetection, env => new AnomalyDetectionMamlEvaluator(env, new AnomalyDetectionMamlEvaluator.Arguments()) }
                         };

src/Microsoft.ML.Data/Evaluators/Metrics/RankerMetrics.cs renamed to src/Microsoft.ML.Data/Evaluators/Metrics/RankingMetrics.cs

@@ -6,7 +6,7 @@
 
 namespace Microsoft.ML.Data
 {
-    public sealed class RankerMetrics
+    public sealed class RankingMetrics
     {
         /// <summary>
         /// Array of normalized discounted cumulative gains where i-th element represent NDCG@i.
@@ -32,15 +32,15 @@ private static T Fetch<T>(IExceptionContext ectx, DataViewRow row, string name)
             return val;
         }
 
-        internal RankerMetrics(IExceptionContext ectx, DataViewRow overallResult)
+        internal RankingMetrics(IExceptionContext ectx, DataViewRow overallResult)
         {
             VBuffer<double> Fetch(string name) => Fetch<VBuffer<double>>(ectx, overallResult, name);
 
-            Dcg = Fetch(RankerEvaluator.Dcg).GetValues().ToArray();
-            Ndcg = Fetch(RankerEvaluator.Ndcg).GetValues().ToArray();
+            Dcg = Fetch(RankingEvaluator.Dcg).GetValues().ToArray();
+            Ndcg = Fetch(RankingEvaluator.Ndcg).GetValues().ToArray();
         }
 
-        internal RankerMetrics(double[] dcg, double[] ndcg)
+        internal RankingMetrics(double[] dcg, double[] ndcg)
         {
             Dcg = new double[dcg.Length];
             dcg.CopyTo(Dcg, 0);

src/Microsoft.ML.Data/Evaluators/RankerEvaluator.cs renamed to src/Microsoft.ML.Data/Evaluators/RankingEvaluator.cs

@@ -16,19 +16,19 @@
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.Model;
 
-[assembly: LoadableClass(typeof(RankerEvaluator), typeof(RankerEvaluator), typeof(RankerEvaluator.Arguments), typeof(SignatureEvaluator),
-    "Ranking Evaluator", RankerEvaluator.LoadName, "Ranking", "rank")]
+[assembly: LoadableClass(typeof(RankingEvaluator), typeof(RankingEvaluator), typeof(RankingEvaluator.Arguments), typeof(SignatureEvaluator),
+    "Ranking Evaluator", RankingEvaluator.LoadName, "Ranking", "rank")]
 
-[assembly: LoadableClass(typeof(RankerMamlEvaluator), typeof(RankerMamlEvaluator), typeof(RankerMamlEvaluator.Arguments), typeof(SignatureMamlEvaluator),
-    "Ranking Evaluator", RankerEvaluator.LoadName, "Ranking", "rank")]
+[assembly: LoadableClass(typeof(RankingMamlEvaluator), typeof(RankingMamlEvaluator), typeof(RankingMamlEvaluator.Arguments), typeof(SignatureMamlEvaluator),
+    "Ranking Evaluator", RankingEvaluator.LoadName, "Ranking", "rank")]
 
-[assembly: LoadableClass(typeof(RankerPerInstanceTransform), null, typeof(SignatureLoadDataTransform),
-    "", RankerPerInstanceTransform.LoaderSignature)]
+[assembly: LoadableClass(typeof(RankingPerInstanceTransform), null, typeof(SignatureLoadDataTransform),
+    "", RankingPerInstanceTransform.LoaderSignature)]
 
 namespace Microsoft.ML.Data
 {
     [BestFriend]
-    internal sealed class RankerEvaluator : EvaluatorBase<RankerEvaluator.Aggregator>
+    internal sealed class RankingEvaluator : EvaluatorBase<RankingEvaluator.Aggregator>
     {
         public sealed class Arguments
         {
@@ -61,7 +61,7 @@ public sealed class Arguments
         private readonly bool _groupSummary;
         private readonly Double[] _labelGains;
 
-        public RankerEvaluator(IHostEnvironment env, Arguments args)
+        public RankingEvaluator(IHostEnvironment env, Arguments args)
             : base(env, LoadName)
         {
             // REVIEW: What kind of checking should be applied to labelGains?
@@ -89,13 +89,13 @@ private protected override void CheckScoreAndLabelTypes(RoleMappedSchema schema)
             var t = schema.Label.Value.Type;
             if (t != NumberDataViewType.Single && !(t is KeyType))
             {
-                throw Host.ExceptSchemaMismatch(nameof(RankerMamlEvaluator.Arguments.LabelColumn),
+                throw Host.ExceptSchemaMismatch(nameof(RankingMamlEvaluator.Arguments.LabelColumn),
                     "label", schema.Label.Value.Name, "R4 or a key", t.ToString());
             }
             var scoreCol = schema.GetUniqueColumn(MetadataUtils.Const.ScoreValueKind.Score);
             if (scoreCol.Type != NumberDataViewType.Single)
             {
-                throw Host.ExceptSchemaMismatch(nameof(RankerMamlEvaluator.Arguments.ScoreColumn),
+                throw Host.ExceptSchemaMismatch(nameof(RankingMamlEvaluator.Arguments.ScoreColumn),
                     "score", scoreCol.Name, "R4", t.ToString());
             }
         }
@@ -105,7 +105,7 @@ private protected override void CheckCustomColumnTypesCore(RoleMappedSchema sche
             var t = schema.Group.Value.Type;
             if (!(t is KeyType))
             {
-                throw Host.ExceptSchemaMismatch(nameof(RankerMamlEvaluator.Arguments.GroupIdColumn),
+                throw Host.ExceptSchemaMismatch(nameof(RankingMamlEvaluator.Arguments.GroupIdColumn),
                     "group", schema.Group.Value.Name, "key", t.ToString());
             }
         }
@@ -129,7 +129,7 @@ internal override IDataTransform GetPerInstanceMetricsCore(RoleMappedData data)
             var scoreInfo = data.Schema.GetUniqueColumn(MetadataUtils.Const.ScoreValueKind.Score);
             Host.CheckParam(data.Schema.Group.HasValue, nameof(data), "Schema must contain a group column");
 
-            return new RankerPerInstanceTransform(Host, data.Data,
+            return new RankingPerInstanceTransform(Host, data.Data,
                 data.Schema.Label.Value.Name, scoreInfo.Name, data.Schema.Group.Value.Name, _truncationLevel, _labelGains);
         }
 
@@ -242,7 +242,7 @@ private protected override void GetAggregatorConsolidationFuncs(Aggregator aggre
         /// <param name="groupId">The name of the groupId column.</param>
         /// <param name="score">The name of the predicted score column.</param>
         /// <returns>The evaluation metrics for these outputs.</returns>
-        public RankerMetrics Evaluate(IDataView data, string label, string groupId, string score)
+        public RankingMetrics Evaluate(IDataView data, string label, string groupId, string score)
         {
             Host.CheckValue(data, nameof(data));
             Host.CheckNonEmpty(label, nameof(label));
@@ -256,12 +256,12 @@ public RankerMetrics Evaluate(IDataView data, string label, string groupId, stri
             Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
             var overall = resultDict[MetricKinds.OverallMetrics];
 
-            RankerMetrics result;
+            RankingMetrics result;
             using (var cursor = overall.GetRowCursorForAllColumns())
             {
                 var moved = cursor.MoveNext();
                 Host.Assert(moved);
-                result = new RankerMetrics(Host, cursor);
+                result = new RankingMetrics(Host, cursor);
                 moved = cursor.MoveNext();
                 Host.Assert(!moved);
             }
@@ -374,15 +374,15 @@ public void Update(short label, Single output)
 
                 public void UpdateGroup(Single weight)
                 {
-                    RankerUtils.QueryMaxDcg(_labelGains, TruncationLevel, _queryLabels, _queryOutputs, _groupMaxDcgCur);
+                    RankingUtils.QueryMaxDcg(_labelGains, TruncationLevel, _queryLabels, _queryOutputs, _groupMaxDcgCur);
                     if (_groupMaxDcg != null)
                     {
                         var maxDcg = new Double[TruncationLevel];
                         Array.Copy(_groupMaxDcgCur, maxDcg, TruncationLevel);
                         _groupMaxDcg.Add(maxDcg);
                     }
 
-                    RankerUtils.QueryDcg(_labelGains, TruncationLevel, _queryLabels, _queryOutputs, _groupDcgCur);
+                    RankingUtils.QueryDcg(_labelGains, TruncationLevel, _queryLabels, _queryOutputs, _groupDcgCur);
                     if (_groupDcg != null)
                     {
                         var groupDcg = new Double[TruncationLevel];
@@ -539,7 +539,7 @@ public void GetSlotNames(ref VBuffer<ReadOnlyMemory<char>> slotNames)
         }
     }
 
-    internal sealed class RankerPerInstanceTransform : IDataTransform
+    internal sealed class RankingPerInstanceTransform : IDataTransform
     {
         public const string LoaderSignature = "RankerPerInstTransform";
         private const string RegistrationName = LoaderSignature;
@@ -552,7 +552,7 @@ private static VersionInfo GetVersionInfo()
                 verReadableCur: 0x00010001,
                 verWeCanReadBack: 0x00010001,
                 loaderSignature: LoaderSignature,
-                loaderAssemblyName: typeof(RankerPerInstanceTransform).Assembly.FullName);
+                loaderAssemblyName: typeof(RankingPerInstanceTransform).Assembly.FullName);
         }
 
         public const string Ndcg = "NDCG";
@@ -576,25 +576,25 @@ private static VersionInfo GetVersionInfo()
         /// </summary>
         public DataViewSchema OutputSchema => _transform.OutputSchema;
 
-        public RankerPerInstanceTransform(IHostEnvironment env, IDataView input, string labelCol, string scoreCol, string groupCol,
+        public RankingPerInstanceTransform(IHostEnvironment env, IDataView input, string labelCol, string scoreCol, string groupCol,
                 int truncationLevel, Double[] labelGains)
         {
             _transform = new Transform(env, input, labelCol, scoreCol, groupCol, truncationLevel, labelGains);
         }
 
-        private RankerPerInstanceTransform(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
+        private RankingPerInstanceTransform(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
         {
             _transform = new Transform(env, ctx, input);
         }
 
-        public static RankerPerInstanceTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
+        public static RankingPerInstanceTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
         {
             Contracts.CheckValue(env, nameof(env));
             var h = env.Register(RegistrationName);
             h.CheckValue(ctx, nameof(ctx));
             ctx.CheckAtModel(GetVersionInfo());
             h.CheckValue(input, nameof(input));
-            return h.Apply("Loading Model", ch => new RankerPerInstanceTransform(h, ctx, input));
+            return h.Apply("Loading Model", ch => new RankingPerInstanceTransform(h, ctx, input));
         }
 
         void ICanSaveModel.Save(ModelSaveContext ctx)
@@ -801,9 +801,9 @@ protected override void ProcessExample(RowCursorState state, short label, Single
             protected override void UpdateState(RowCursorState state)
             {
                 // Calculate the current group DCG, NDCG and MaxDcg.
-                RankerUtils.QueryMaxDcg(_labelGains, _truncationLevel, state.QueryLabels, state.QueryOutputs,
+                RankingUtils.QueryMaxDcg(_labelGains, _truncationLevel, state.QueryLabels, state.QueryOutputs,
                     state.MaxDcgCur);
-                RankerUtils.QueryDcg(_labelGains, _truncationLevel, state.QueryLabels, state.QueryOutputs, state.DcgCur);
+                RankingUtils.QueryDcg(_labelGains, _truncationLevel, state.QueryLabels, state.QueryOutputs, state.DcgCur);
                 for (int t = 0; t < _truncationLevel; t++)
                 {
                     Double ndcg = state.MaxDcgCur[t] > 0 ? state.DcgCur[t] / state.MaxDcgCur[t] * 100 : 0;
@@ -838,7 +838,7 @@ public RowCursorState(int truncationLevel)
     }
 
     [BestFriend]
-    internal sealed class RankerMamlEvaluator : MamlEvaluatorBase
+    internal sealed class RankingMamlEvaluator : MamlEvaluatorBase
     {
         public sealed class Arguments : ArgumentsBase
         {
@@ -855,25 +855,25 @@ public sealed class Arguments : ArgumentsBase
             public string GroupSummaryFilename;
         }
 
-        private readonly RankerEvaluator _evaluator;
+        private readonly RankingEvaluator _evaluator;
         private readonly string _groupIdCol;
 
         private readonly string _groupSummaryFilename;
 
         private protected override IEvaluator Evaluator => _evaluator;
 
-        public RankerMamlEvaluator(IHostEnvironment env, Arguments args)
+        public RankingMamlEvaluator(IHostEnvironment env, Arguments args)
             : base(args, env, MetadataUtils.Const.ScoreColumnKind.Ranking, "RankerMamlEvaluator")
         {
             Host.CheckValue(args, nameof(args));
             Utils.CheckOptionalUserDirectory(args.GroupSummaryFilename, nameof(args.GroupSummaryFilename));
 
-            var evalArgs = new RankerEvaluator.Arguments();
+            var evalArgs = new RankingEvaluator.Arguments();
             evalArgs.DcgTruncationLevel = args.DcgTruncationLevel;
             evalArgs.LabelGains = args.LabelGains;
             evalArgs.OutputGroupSummary = !string.IsNullOrEmpty(args.GroupSummaryFilename);
 
-            _evaluator = new RankerEvaluator(Host, evalArgs);
+            _evaluator = new RankingEvaluator(Host, evalArgs);
             _groupSummaryFilename = args.GroupSummaryFilename;
             _groupIdCol = args.GroupIdColumn;
         }
@@ -908,14 +908,14 @@ private bool TryGetGroupSummaryMetrics(Dictionary<string, IDataView>[] metrics,
             Host.AssertNonEmpty(metrics);
 
             if (metrics.Length == 1)
-                return metrics[0].TryGetValue(RankerEvaluator.GroupSummary, out gs);
+                return metrics[0].TryGetValue(RankingEvaluator.GroupSummary, out gs);
 
             gs = null;
             var gsList = new List<IDataView>();
             for (int i = 0; i < metrics.Length; i++)
             {
                 IDataView idv;
-                if (!metrics[i].TryGetValue(RankerEvaluator.GroupSummary, out idv))
+                if (!metrics[i].TryGetValue(RankingEvaluator.GroupSummary, out idv))
                     return false;
 
                 idv = EvaluateUtils.AddFoldIndex(Host, idv, i, metrics.Length);
@@ -939,13 +939,13 @@ private protected override IEnumerable<string> GetPerInstanceColumnsToSave(RoleM
             yield return scoreCol.Name;
 
             // Return the output columns.
-            yield return RankerPerInstanceTransform.Ndcg;
-            yield return RankerPerInstanceTransform.Dcg;
-            yield return RankerPerInstanceTransform.MaxDcg;
+            yield return RankingPerInstanceTransform.Ndcg;
+            yield return RankingPerInstanceTransform.Dcg;
+            yield return RankingPerInstanceTransform.MaxDcg;
         }
     }
 
-    internal static class RankerUtils
+    internal static class RankingUtils
     {
         private static volatile Double[] _discountMap;
         public static Double[] DiscountMap
@@ -1054,8 +1054,8 @@ private static Comparison<int> GetCompareItems(List<short> queryLabels, List<Sin
 
     internal static partial class Evaluate
     {
-        [TlcModule.EntryPoint(Name = "Models.RankerEvaluator", Desc = "Evaluates a ranking scored dataset.")]
-        public static CommonOutputs.CommonEvaluateOutput Ranking(IHostEnvironment env, RankerMamlEvaluator.Arguments input)
+        [TlcModule.EntryPoint(Name = "Models.RankingEvaluator", Desc = "Evaluates a ranking scored dataset.")]
+        public static CommonOutputs.CommonEvaluateOutput Ranking(IHostEnvironment env, RankingMamlEvaluator.Arguments input)
         {
             Contracts.CheckValue(env, nameof(env));
             var host = env.Register("EvaluateRanker");
@@ -1068,9 +1068,9 @@ public static CommonOutputs.CommonEvaluateOutput Ranking(IHostEnvironment env, R
             MatchColumns(host, input, out label, out weight, out name);
             var schema = input.Data.Schema;
             string groupId = TrainUtils.MatchNameOrDefaultOrNull(host, schema,
-                nameof(RankerMamlEvaluator.Arguments.GroupIdColumn),
+                nameof(RankingMamlEvaluator.Arguments.GroupIdColumn),
                 input.GroupIdColumn, DefaultColumnNames.GroupId);
-            IMamlEvaluator evaluator = new RankerMamlEvaluator(host, input);
+            IMamlEvaluator evaluator = new RankingMamlEvaluator(host, input);
             var data = new RoleMappedData(input.Data, label, null, groupId, weight, name);
             var metrics = evaluator.Evaluate(data);
 

src/Microsoft.ML.Data/TrainCatalog.cs

@@ -623,7 +623,7 @@ internal RankingTrainers(RankingCatalog catalog)
         /// <param name="groupId">The name of the groupId column in <paramref name="data"/>.</param>
         /// <param name="score">The name of the score column in <paramref name="data"/>.</param>
         /// <returns>The evaluation results for these calibrated outputs.</returns>
-        public RankerMetrics Evaluate(IDataView data,
+        public RankingMetrics Evaluate(IDataView data,
             string label = DefaultColumnNames.Label,
             string groupId = DefaultColumnNames.GroupId,
             string score = DefaultColumnNames.Score)
@@ -633,7 +633,7 @@ public RankerMetrics Evaluate(IDataView data,
             Environment.CheckNonEmpty(score, nameof(score));
             Environment.CheckNonEmpty(groupId, nameof(groupId));
 
-            var eval = new RankerEvaluator(Environment, new RankerEvaluator.Arguments() { });
+            var eval = new RankingEvaluator(Environment, new RankingEvaluator.Arguments() { });
             return eval.Evaluate(data, label, groupId, score);
         }
     }

src/Microsoft.ML.EntryPoints/CrossValidationMacro.cs

@@ -430,7 +430,7 @@ private static IMamlEvaluator GetEvaluator(IHostEnvironment env, MacroUtils.Trai
             case MacroUtils.TrainerKinds.SignatureRegressorTrainer:
                 return new RegressionMamlEvaluator(env, new RegressionMamlEvaluator.Arguments());
             case MacroUtils.TrainerKinds.SignatureRankerTrainer:
-                return new RankerMamlEvaluator(env, new RankerMamlEvaluator.Arguments());
+                return new RankingMamlEvaluator(env, new RankingMamlEvaluator.Arguments());
             case MacroUtils.TrainerKinds.SignatureAnomalyDetectorTrainer:
                 return new AnomalyDetectionMamlEvaluator(env, new AnomalyDetectionMamlEvaluator.Arguments());
             case MacroUtils.TrainerKinds.SignatureClusteringTrainer: