diff --git a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs
index 71c08eecd0..a567abef39 100644
--- a/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs
+++ b/src/Microsoft.ML.Data/Evaluators/BinaryClassifierEvaluator.cs
@@ -5,6 +5,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
+using Microsoft.ML.Data.StaticPipe;
+using Microsoft.ML.Data.StaticPipe.Runtime;
using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.CommandLine;
using Microsoft.ML.Runtime.Data;
@@ -410,48 +412,42 @@ public sealed class Counters
public Double Acc
{
- get
- {
+ get {
return (NumTrueNeg + NumTruePos) / (NumTruePos + NumTrueNeg + NumFalseNeg + NumFalsePos);
}
}
public Double RecallPos
{
- get
- {
+ get {
return (NumTruePos + NumFalseNeg > 0) ? NumTruePos / (NumTruePos + NumFalseNeg) : 0;
}
}
public Double PrecisionPos
{
- get
- {
+ get {
return (NumTruePos + NumFalsePos > 0) ? NumTruePos / (NumTruePos + NumFalsePos) : 0;
}
}
public Double RecallNeg
{
- get
- {
+ get {
return (NumTrueNeg + NumFalsePos > 0) ? NumTrueNeg / (NumTrueNeg + NumFalsePos) : 0;
}
}
public Double PrecisionNeg
{
- get
- {
+ get {
return (NumTrueNeg + NumFalseNeg > 0) ? NumTrueNeg / (NumTrueNeg + NumFalseNeg) : 0;
}
}
public Double Entropy
{
- get
- {
+ get {
return MathUtils.Entropy((NumTruePos + NumFalseNeg) /
(NumTruePos + NumTrueNeg + NumFalseNeg + NumFalsePos));
}
@@ -459,8 +455,7 @@ public Double Entropy
public Double LogLoss
{
- get
- {
+ get {
return Double.IsNaN(_logLoss) ? Double.NaN : (_numLogLossPositives + _numLogLossNegatives > 0)
? _logLoss / (_numLogLossPositives + _numLogLossNegatives) : 0;
}
@@ -468,8 +463,7 @@ public Double LogLoss
public Double LogLossReduction
{
- get
- {
+ get {
if (_numLogLossPositives + _numLogLossNegatives == 0)
return 0;
var logLoss = _logLoss / (_numLogLossPositives + _numLogLossNegatives);
@@ -787,6 +781,246 @@ private void ComputePrCurves()
}
}
}
+
+ ///
+ /// Evaluation results for binary classifiers, excluding probabilistic metrics.
+ ///
+ public class Result
+ {
+ ///
+ /// Gets the area under the ROC curve.
+ ///
+ ///
+ /// The area under the ROC curve is equal to the probability that the classifier ranks
+ /// a randomly chosen positive instance higher than a randomly chosen negative one
+ /// (assuming 'positive' ranks higher than 'negative').
+ ///
+ public double Auc { get; }
+
+ ///
+ /// Gets the accuracy of a classifier which is the proportion of correct predictions in the test set.
+ ///
+ public double Accuracy { get; }
+
+ ///
+ /// Gets the positive precision of a classifier which is the proportion of correctly predicted
+ /// positive instances among all the positive predictions (i.e., the number of positive instances
+ /// predicted as positive, divided by the total number of instances predicted as positive).
+ ///
+ public double PositivePrecision { get; }
+
+ ///
+ /// Gets the positive recall of a classifier which is the proportion of correctly predicted
+ /// positive instances among all the positive instances (i.e., the number of positive instances
+ /// predicted as positive, divided by the total number of positive instances).
+ ///
+ public double PositiveRecall { get; private set; }
+
+ ///
+ /// Gets the negative precision of a classifier which is the proportion of correctly predicted
+ /// negative instances among all the negative predictions (i.e., the number of negative instances
+ /// predicted as negative, divided by the total number of instances predicted as negative).
+ ///
+ public double NegativePrecision { get; }
+
+ ///
+ /// Gets the negative recall of a classifier which is the proportion of correctly predicted
+ /// negative instances among all the negative instances (i.e., the number of negative instances
+ /// predicted as negative, divided by the total number of negative instances).
+ ///
+ public double NegativeRecall { get; }
+
+ ///
+ /// Gets the F1 score of the classifier.
+ ///
+ ///
+ /// F1 score is the harmonic mean of precision and recall: 2 * precision * recall / (precision + recall).
+ ///
+ public double F1Score { get; }
+
+ ///
+ /// Gets the area under the precision/recall curve of the classifier.
+ ///
+ ///
+ /// The area under the precision/recall curve is a single number summary of the information in the
+ /// precision/recall curve. It is increasingly used in the machine learning community, particularly
+ /// for imbalanced datasets where one class is observed more frequently than the other. On these
+ /// datasets, AUPRC can highlight performance differences that are lost with AUC.
+ ///
+ public double Auprc { get; }
+
+ protected private static T Fetch(IExceptionContext ectx, IRow row, string name)
+ {
+ if (!row.Schema.TryGetColumnIndex(name, out int col))
+ throw ectx.Except($"Could not find column '{name}'");
+ T val = default;
+ row.GetGetter(col)(ref val);
+ return val;
+ }
+
+ internal Result(IExceptionContext ectx, IRow overallResult)
+ {
+ double Fetch(string name) => Fetch(ectx, overallResult, name);
+ Auc = Fetch(BinaryClassifierEvaluator.Auc);
+ Accuracy = Fetch(BinaryClassifierEvaluator.Accuracy);
+ PositivePrecision = Fetch(BinaryClassifierEvaluator.PosPrecName);
+ PositiveRecall = Fetch(BinaryClassifierEvaluator.PosRecallName);
+ NegativePrecision = Fetch(BinaryClassifierEvaluator.NegPrecName);
+ NegativeRecall = Fetch(BinaryClassifierEvaluator.NegRecallName);
+ F1Score = Fetch(BinaryClassifierEvaluator.F1);
+ Auprc = Fetch(BinaryClassifierEvaluator.AuPrc);
+ }
+ }
+
+ ///
+ /// Evaluation results for binary classifiers, including probabilistic metrics.
+ ///
+ public sealed class CalibratedResult : Result
+ {
+ ///
+ /// Gets the log-loss of the classifier.
+ ///
+ ///
+ /// The log-loss metric, is computed as follows:
+ /// LL = - (1/m) * sum( log(p[i]))
+ /// where m is the number of instances in the test set.
+ /// p[i] is the probability returned by the classifier if the instance belongs to class 1,
+ /// and 1 minus the probability returned by the classifier if the instance belongs to class 0.
+ ///
+ public double LogLoss { get; }
+
+ ///
+ /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG)
+ /// of the classifier.
+ ///
+ ///
+ /// The log-loss reduction is scaled relative to a classifier that predicts the prior for every example:
+ /// (LL(prior) - LL(classifier)) / LL(prior)
+ /// This metric can be interpreted as the advantage of the classifier over a random prediction.
+ /// E.g., if the RIG equals 20, it can be interpreted as "the probability of a correct prediction is
+ /// 20% better than random guessing."
+ ///
+ public double LogLossReduction { get; }
+
+ ///
+ /// Gets the test-set entropy (prior Log-Loss/instance) of the classifier.
+ ///
+ public double Entropy { get; }
+
+ internal CalibratedResult(IExceptionContext ectx, IRow overallResult)
+ : base(ectx, overallResult)
+ {
+ double Fetch(string name) => Fetch(ectx, overallResult, name);
+ LogLoss = Fetch(BinaryClassifierEvaluator.LogLoss);
+ LogLossReduction = Fetch(BinaryClassifierEvaluator.LogLossReduction);
+ Entropy = Fetch(BinaryClassifierEvaluator.Entropy);
+ }
+ }
+
+ ///
+ /// Evaluates scored binary classification data.
+ ///
+ /// The shape type for the input data.
+ /// The data to evaluate.
+ /// The index delegate for the label column.
+ /// The index delegate for columns from calibrated prediction of a binary classifier.
+ /// Under typical scenarios, this will just be the same tuple of results returned from the trainer.
+ /// The evaluation results for these calibrated outputs.
+ public static CalibratedResult Evaluate(
+ DataView data,
+ Func> label,
+ Func score, Scalar probability, Scalar predictedLabel)> pred)
+ {
+ Contracts.CheckValue(data, nameof(data));
+ var env = StaticPipeUtils.GetEnvironment(data);
+ Contracts.AssertValue(env);
+ env.CheckValue(label, nameof(label));
+ env.CheckValue(pred, nameof(pred));
+
+ var indexer = StaticPipeUtils.GetIndexer(data);
+ string labelName = indexer.Get(label(indexer.Indices));
+ (var scoreCol, var probCol, var predCol) = pred(indexer.Indices);
+ Contracts.CheckParam(scoreCol != null, nameof(pred), "Indexing delegate resulted in null score column.");
+ Contracts.CheckParam(probCol != null, nameof(pred), "Indexing delegate resulted in null probability column.");
+ Contracts.CheckParam(predCol != null, nameof(pred), "Indexing delegate resulted in null predicted label column.");
+ string scoreName = indexer.Get(scoreCol);
+ string probName = indexer.Get(probCol);
+ string predName = indexer.Get(predCol);
+
+ var eval = new BinaryClassifierEvaluator(env, new Arguments() { });
+
+ var roles = new RoleMappedData(data.AsDynamic, opt: false,
+ RoleMappedSchema.ColumnRole.Label.Bind(labelName),
+ RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreName),
+ RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Probability, probName),
+ RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.PredictedLabel, predName));
+
+ var resultDict = eval.Evaluate(roles);
+ env.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
+ var overall = resultDict[MetricKinds.OverallMetrics];
+
+ CalibratedResult result;
+ using (var cursor = overall.GetRowCursor(i => true))
+ {
+ var moved = cursor.MoveNext();
+ env.Assert(moved);
+ result = new CalibratedResult(env, cursor);
+ moved = cursor.MoveNext();
+ env.Assert(!moved);
+ }
+ return result;
+ }
+
+ ///
+ /// Evaluates scored binary classification data.
+ ///
+ /// The shape type for the input data.
+ /// The data to evaluate.
+ /// The index delegate for the label column.
+ /// The index delegate for columns from calibrated prediction of a binary classifier.
+ /// Under typical scenarios, this will just be the same tuple of results returned from the trainer.
+ /// The evaluation results for these uncalibrated outputs.
+ public static Result Evaluate(
+ DataView data,
+ Func> label,
+ Func score, Scalar predictedLabel)> pred)
+ {
+ Contracts.CheckValue(data, nameof(data));
+ var env = StaticPipeUtils.GetEnvironment(data);
+ Contracts.AssertValue(env);
+ env.CheckValue(label, nameof(label));
+ env.CheckValue(pred, nameof(pred));
+
+ var indexer = StaticPipeUtils.GetIndexer(data);
+ string labelName = indexer.Get(label(indexer.Indices));
+ (var scoreCol, var predCol) = pred(indexer.Indices);
+ Contracts.CheckParam(scoreCol != null, nameof(pred), "Indexing delegate resulted in null score column.");
+ Contracts.CheckParam(predCol != null, nameof(pred), "Indexing delegate resulted in null predicted label column.");
+ string scoreName = indexer.Get(scoreCol);
+ string predName = indexer.Get(predCol);
+
+ var eval = new BinaryClassifierEvaluator(env, new Arguments() { });
+
+ var roles = new RoleMappedData(data.AsDynamic, opt: false,
+ RoleMappedSchema.ColumnRole.Label.Bind(labelName),
+ RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreName),
+ RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.PredictedLabel, predName));
+
+ var resultDict = eval.Evaluate(roles);
+ env.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
+ var overall = resultDict[MetricKinds.OverallMetrics];
+
+ Result result;
+ using (var cursor = overall.GetRowCursor(i => true))
+ {
+ var moved = cursor.MoveNext();
+ env.Assert(moved);
+ result = new Result(env, cursor);
+ moved = cursor.MoveNext();
+ env.Assert(!moved);
+ }
+ return result;
+ }
}
public sealed class BinaryPerInstanceEvaluator : PerInstanceEvaluatorBase
@@ -1526,4 +1760,4 @@ private static IDataView ExtractConfusionMatrix(IHost host, Dictionary
+ /// Gets the absolute loss of the model.
+ ///
+ ///
+ /// The absolute loss is defined as
+ /// L1 = (1/m) * sum( abs( yi - y'i))
+ /// where m is the number of instances in the test set.
+ /// y'i are the predicted labels for each instance.
+ /// yi are the correct labels of each instance.
+ ///
+ public double L1 { get; }
+
+ ///
+ /// Gets the squared loss of the model.
+ ///
+ ///
+ /// The squared loss is defined as
+ /// L2 = (1/m) * sum(( yi - y'i)^2)
+ /// where m is the number of instances in the test set.
+ /// y'i are the predicted labels for each instance.
+ /// yi are the correct labels of each instance.
+ ///
+ public double L2 { get; }
+
+ ///
+ /// Gets the root mean square loss (or RMS) which is the square root of the L2 loss.
+ ///
+ public double Rms { get; }
+
+ ///
+ /// Gets the user defined loss function.
+ ///
+ ///
+ /// This is the average of a loss function defined by the user,
+ /// computed over all the instances in the test set.
+ ///
+ public double LossFn { get; }
+
+ ///
+ /// Gets the R squared value of the model, which is also known as
+ /// the coefficient of determination.
+ ///
+ public double RSquared { get; }
+
+ private static T Fetch(IExceptionContext ectx, IRow row, string name)
+ {
+ if (!row.Schema.TryGetColumnIndex(name, out int col))
+ throw ectx.Except($"Could not find column '{name}'");
+ T val = default;
+ row.GetGetter(col)(ref val);
+ return val;
+ }
+
+ internal Result(IExceptionContext ectx, IRow overallResult)
+ {
+ double Fetch(string name) => Fetch(ectx, overallResult, name);
+ L1 = Fetch(RegressionEvaluator.L1);
+ L2 = Fetch(RegressionEvaluator.L2);
+ Rms= Fetch(RegressionEvaluator.Rms);
+ LossFn = Fetch(RegressionEvaluator.Loss);
+ RSquared = Fetch(RegressionEvaluator.RSquared);
+ }
+ }
+
+ private sealed class TrivialLossFactory : ISupportRegressionLossFactory
+ {
+ private readonly IRegressionLoss _loss;
+ public TrivialLossFactory(IRegressionLoss loss) => _loss = loss;
+ public IRegressionLoss CreateComponent(IHostEnvironment env) => _loss;
+ }
+
+ ///
+ /// Evaluates scored binary classification data.
+ ///
+ /// The shape type for the input data.
+ /// The data to evaluate.
+ /// The index delegate for the label column.
+ /// The index delegate for the predicted score column.
+ /// Potentially custom loss function. If left unspecified defaults to .
+ /// The evaluation results for these outputs.
+ public static Result Evaluate(
+ DataView data,
+ Func> label,
+ Func> score,
+ IRegressionLoss loss = null)
+ {
+ Contracts.CheckValue(data, nameof(data));
+ var env = StaticPipeUtils.GetEnvironment(data);
+ Contracts.AssertValue(env);
+ env.CheckValue(label, nameof(label));
+ env.CheckValue(score, nameof(score));
+
+ var indexer = StaticPipeUtils.GetIndexer(data);
+ string labelName = indexer.Get(label(indexer.Indices));
+ string scoreName = indexer.Get(score(indexer.Indices));
+
+ var args = new Arguments() { };
+ if (loss != null)
+ args.LossFunction = new TrivialLossFactory(loss);
+ var eval = new RegressionEvaluator(env, args);
+
+ var roles = new RoleMappedData(data.AsDynamic, opt: false,
+ RoleMappedSchema.ColumnRole.Label.Bind(labelName),
+ RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, scoreName));
+
+ var resultDict = eval.Evaluate(roles);
+ env.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
+ var overall = resultDict[MetricKinds.OverallMetrics];
+
+ Result result;
+ using (var cursor = overall.GetRowCursor(i => true))
+ {
+ var moved = cursor.MoveNext();
+ env.Assert(moved);
+ result = new Result(env, cursor);
+ moved = cursor.MoveNext();
+ env.Assert(!moved);
+ }
+ return result;
+ }
}
public sealed class RegressionPerInstanceEvaluator : PerInstanceEvaluatorBase
diff --git a/src/Microsoft.ML.Data/Microsoft.ML.Data.csproj b/src/Microsoft.ML.Data/Microsoft.ML.Data.csproj
index 16d004f8c1..125ee8f479 100644
--- a/src/Microsoft.ML.Data/Microsoft.ML.Data.csproj
+++ b/src/Microsoft.ML.Data/Microsoft.ML.Data.csproj
@@ -7,9 +7,18 @@
CORECLR
+
+
+ True
+ True
+ TermStaticExtensions.tt
+
+
+
+
@@ -18,4 +27,25 @@
+
+
+ TextTemplatingFileGenerator
+ TermStaticExtensions.cs
+
+
+
+
+
+
+
+
+
+ True
+ True
+ TermStaticExtensions.tt
+ TextTemplatingFileGenerator
+ TermStaticExtensions.cs
+
+
+
diff --git a/src/Microsoft.ML.Data/StaticPipe/SchemaBearing.cs b/src/Microsoft.ML.Data/StaticPipe/SchemaBearing.cs
index e68813a6c1..bc03d2b61f 100644
--- a/src/Microsoft.ML.Data/StaticPipe/SchemaBearing.cs
+++ b/src/Microsoft.ML.Data/StaticPipe/SchemaBearing.cs
@@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
+using System.Threading;
using Microsoft.ML.Core.Data;
using Microsoft.ML.Data.StaticPipe.Runtime;
using Microsoft.ML.Runtime;
@@ -18,9 +19,23 @@ namespace Microsoft.ML.Data.StaticPipe
///
public abstract class SchemaBearing
{
- private protected readonly IHostEnvironment Env;
+ protected internal readonly IHostEnvironment Env;
internal readonly StaticSchemaShape Shape;
+ private StaticPipeUtils.IndexHelper _indexer;
+ ///
+ /// The indexer for the object. Note component authors will not access this directly but should instead
+ /// work via the public method
+ ///
+ internal StaticPipeUtils.IndexHelper Indexer
+ {
+ get {
+ if (_indexer == null)
+ Interlocked.CompareExchange(ref _indexer, new StaticPipeUtils.IndexHelper(this), null);
+ return _indexer;
+ }
+ }
+
///
/// Constructor for a block maker.
///
diff --git a/src/Microsoft.ML.Data/StaticPipe/StaticPipeInternalUtils.cs b/src/Microsoft.ML.Data/StaticPipe/StaticPipeInternalUtils.cs
index 54119f0d0a..3aba94c2f1 100644
--- a/src/Microsoft.ML.Data/StaticPipe/StaticPipeInternalUtils.cs
+++ b/src/Microsoft.ML.Data/StaticPipe/StaticPipeInternalUtils.cs
@@ -161,6 +161,23 @@ public static KeyValuePair[] GetNamesTypes(ParameterInfo
return retVal;
}
+ ///
+ /// Given a schema shape defining instance, return the pairs of names and values, based on a recursive
+ /// traversal of the value-tuple. If in that list the value a.b.c is paired with an item x,
+ /// then programmatically when accessed as .a.b.c
+ /// would be that item x.
+ ///
+ /// The schema shape defining type.
+ /// The instance of that schema shape defining type, whose items will
+ /// populate the fields of the returned items.
+ /// It is an implementation detail of the value-tuple type that the names
+ /// are not associated with the type at all, but there is instead an illusion propagated within
+ /// Visual Studio, that works via attributes. Programmatic access to this is limited, except that
+ /// a is attached to the type in appropriate places, e.g.,
+ /// in a delegate one of the parameters, or the return parameter, or somesuch. If present, the names
+ /// will be extracted from that structure, and if not the default names of Item1, Item2,
+ /// etc. will be used.
+ /// The set of names and corresponding values.
public static KeyValuePair[] GetNamesValues(T record, ParameterInfo pInfo)
=> GetNamesValues(record, pInfo);
diff --git a/src/Microsoft.ML.Data/StaticPipe/StaticPipeUtils.cs b/src/Microsoft.ML.Data/StaticPipe/StaticPipeUtils.cs
index 6da97abe8a..09b7102d27 100644
--- a/src/Microsoft.ML.Data/StaticPipe/StaticPipeUtils.cs
+++ b/src/Microsoft.ML.Data/StaticPipe/StaticPipeUtils.cs
@@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
+using System.Collections.Immutable;
using System.Linq;
using Microsoft.ML.Core.Data;
using Microsoft.ML.Runtime;
@@ -13,7 +14,9 @@ namespace Microsoft.ML.Data.StaticPipe.Runtime
{
///
/// Utility methods for components that want to expose themselves in the idioms of the statically-typed pipelines.
- /// These utilities are meant to be called by and useful to component authors, not users of those components.
+ /// These utilities are meant to be called by and useful to component authors, not users of those components. The
+ /// purpose is not to keep them hidden per se, but rather in a place less conspicuous to users that are just trying
+ /// to use the library without writing additional components of their own.
///
public static class StaticPipeUtils
{
@@ -312,8 +315,7 @@ public BidirectionalDictionary()
public T1 this[T2 key]
{
get => _d21[key];
- set
- {
+ set {
Contracts.CheckValue((object)key, nameof(key));
Contracts.CheckValue((object)value, nameof(value));
@@ -332,8 +334,7 @@ public T1 this[T2 key]
public T2 this[T1 key]
{
get => _d12[key];
- set
- {
+ set {
Contracts.CheckValue((object)key, nameof(key));
Contracts.CheckValue((object)value, nameof(value));
@@ -366,5 +367,99 @@ public IReadOnlyDictionary AsOther(IEnumerable keys)
return d;
}
}
+
+ ///
+ /// Retrieves the internally stored environment in .
+ /// Intended usecases is component generating code that needs to have access to an
+ /// environment.
+ ///
+ /// The shape type.
+ /// The object for which we get the environment.
+ /// The internal of the object.
+ public static IHostEnvironment GetEnvironment(SchemaBearing schematized)
+ {
+ Contracts.CheckValue(schematized, nameof(schematized));
+ return schematized.Env;
+ }
+
+ ///
+ /// Retrieves the index helper object for .
+ ///
+ /// The shape type.
+ /// The object for which we get the indexer.
+ /// The index helper.
+ public static IndexHelper GetIndexer(SchemaBearing schematized)
+ {
+ Contracts.CheckValue(schematized, nameof(schematized));
+ return schematized.Indexer;
+ }
+
+ ///
+ /// An indexer that can be constructed over a static pipeline object, to enable us to determine
+ /// the names of the columns. This is used by component authors to allow users to "select" a column,
+ /// but the structure is itself not directly used by users of the API as a general rule. Rather,
+ /// one might imagine the component exposing some sort of delegate taking method that given the
+ /// instance , returns one of the instances stored
+ /// therein, which the component can use to do specific operations.
+ ///
+ /// The shape type.
+ public sealed class IndexHelper
+ {
+ ///
+ /// An instance of the shape type whose items can be used to index to find the names of column.
+ ///
+ public T Indices { get; }
+
+ ///
+ /// Maps the items inside to the names of the associated data's column's name.
+ /// Components can use this structure, but it may be preferable to use
+ /// to ensure uniformity in the exception messages, if possible.
+ ///
+ private ImmutableDictionary Map { get; }
+
+ ///
+ /// Performs a lookup on . If the key is not present this will throw an exception
+ /// more generally helpful in context than that of a direct failure of index on .
+ ///
+ /// The column to look up.
+ /// The optional exception context.
+ /// If successful the name of the column.
+ public string Get(PipelineColumn key, IExceptionContext ectx = null)
+ {
+ Contracts.CheckValueOrNull(ectx);
+ ectx.CheckValue(key, nameof(key));
+ if (!Map.TryGetValue(key, out string name))
+ {
+ // The most obvious reason this might happen is if the user did something like try to attempt to
+ // apply an estimator inside the index delegate, which obviously will not work.
+ throw ectx.ExceptParam(nameof(key), "Column does not appear to be valid for this structure. " +
+ "Please use columns in the provided indexing object without attempting modification.");
+ }
+ return name;
+ }
+
+ ///
+ /// Constructor for the index helper. Note that any public or component code will instead use
+ /// , to fetch the lazily constructed instance of this
+ /// object, since its construction is somewhat expensive.
+ ///
+ /// Constructs the helper for this object.
+ internal IndexHelper(SchemaBearing schematized)
+ {
+ Indices = StaticPipeInternalUtils.MakeAnalysisInstance(out var rec);
+ // We define this delegate just to get its return parameter, so the name extractor has something
+ // to work over. Because this is defined without the names the names will be default, which is not
+ // useful, except we can get the "real" names from the schematized object's shape.
+ Func dummyFunc = () => default;
+ var pairs = StaticPipeInternalUtils.GetNamesValues(Indices, dummyFunc.Method.ReturnParameter);
+ Contracts.Assert(pairs.Length == schematized.Shape.Pairs.Length);
+
+ var builder = ImmutableDictionary.CreateBuilder();
+ for (int i = 0; i < pairs.Length; ++i)
+ // Each "index" come from the analysis of the indices object, but we get the names from the shape.
+ builder.Add(pairs[i].Value, schematized.Shape.Pairs[i].Key);
+ Map = builder.ToImmutable();
+ }
+ }
}
}
diff --git a/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs b/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs
index 0c68dde347..e9d3c59e06 100644
--- a/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs
+++ b/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs
@@ -8,6 +8,7 @@
using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.Internal.Utilities;
+using static Microsoft.ML.Data.StaticPipe.Runtime.TrainerEstimatorReconciler;
namespace Microsoft.ML.Data.StaticPipe.Runtime
{
@@ -336,5 +337,71 @@ private sealed class ImplBool : Scalar
public ImplBool(BinaryClassifierNoCalibration rec) : base(rec, rec._inputs) { }
}
}
+
+ ///
+ /// A reconciler for regression capable of handling the most common cases for regression.
+ ///
+ public sealed class MulticlassClassifier : TrainerEstimatorReconciler
+ {
+ ///
+ /// The delegate to create the instance.
+ ///
+ /// The environment with which to create the estimator
+ /// The label column name
+ /// The features column name
+ /// The weights column name, or null if the reconciler was constructed with null weights
+ /// A estimator producing columns with the fixed name and .
+ public delegate IEstimator EstimatorFactory(IHostEnvironment env, string label, string features, string weights);
+
+ private readonly EstimatorFactory _estFact;
+
+ ///
+ /// The general output for multiclass classifiers.
+ ///
+ public (Vector score, Key predictedLabel) Output { get; }
+
+ protected override IEnumerable Outputs => new PipelineColumn[] { Output.score, Output.predictedLabel };
+
+ private static readonly string[] _fixedOutputNames = new[] { DefaultColumnNames.Score, DefaultColumnNames.PredictedLabel };
+
+ ///
+ /// Constructs a new general regression reconciler.
+ ///
+ /// The delegate to create the training estimator. It is assumed that this estimator
+ /// will produce a single new scalar column named .
+ /// The input label column.
+ /// The input features column.
+ /// The input weights column, or null if there are no weights.
+ public MulticlassClassifier(EstimatorFactory estimatorFactory, Key label, Vector features, Scalar weights)
+ : base(MakeInputs(Contracts.CheckRef(label, nameof(label)), Contracts.CheckRef(features, nameof(features)), weights),
+ _fixedOutputNames)
+ {
+ Contracts.CheckValue(estimatorFactory, nameof(estimatorFactory));
+ _estFact = estimatorFactory;
+ Contracts.Assert(_inputs.Length == 2 || _inputs.Length == 3);
+ Output = (new ImplScore(this), new ImplLabel(this));
+ }
+
+ private static PipelineColumn[] MakeInputs(Key label, Vector features, Scalar weights)
+ => weights == null ? new PipelineColumn[] { label, features } : new PipelineColumn[] { label, features, weights };
+
+ protected override IEstimator ReconcileCore(IHostEnvironment env, string[] inputNames)
+ {
+ Contracts.AssertValue(env);
+ env.Assert(Utils.Size(inputNames) == _inputs.Length);
+ return _estFact(env, inputNames[0], inputNames[1], inputNames.Length > 2 ? inputNames[2] : null);
+ }
+
+ private sealed class ImplLabel : Key
+ {
+ public ImplLabel(MulticlassClassifier rec) : base(rec, rec._inputs) { }
+ }
+
+ private sealed class ImplScore : Vector
+ {
+ public ImplScore(MulticlassClassifier rec) : base(rec, rec._inputs) { }
+ }
+ }
+
}
}
diff --git a/src/Microsoft.ML.Data/Training/TrainerEstimatorBase.cs b/src/Microsoft.ML.Data/Training/TrainerEstimatorBase.cs
index d8ff793979..95800afe4c 100644
--- a/src/Microsoft.ML.Data/Training/TrainerEstimatorBase.cs
+++ b/src/Microsoft.ML.Data/Training/TrainerEstimatorBase.cs
@@ -40,11 +40,6 @@ public abstract class TrainerEstimatorBase : ITrainerEstim
///
public readonly SchemaShape.Column WeightColumn;
- ///
- /// The columns that will be created by the fitted transformer.
- ///
- protected abstract SchemaShape.Column[] OutputColumns { get; }
-
protected readonly IHost Host;
///
@@ -76,12 +71,17 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
CheckInputSchema(inputSchema);
var outColumns = inputSchema.Columns.ToDictionary(x => x.Name);
- foreach (var col in OutputColumns)
+ foreach (var col in GetOutputColumnsCore(inputSchema))
outColumns[col.Name] = col;
return new SchemaShape(outColumns.Values);
}
+ ///
+ /// The columns that will be created by the fitted transformer.
+ ///
+ protected abstract SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema);
+
public TModel Train(TrainContext context)
{
Host.CheckValue(context, nameof(context));
diff --git a/src/Microsoft.ML.Data/Transforms/TermEstimator.cs b/src/Microsoft.ML.Data/Transforms/TermEstimator.cs
index ce358e0760..44e873e9e7 100644
--- a/src/Microsoft.ML.Data/Transforms/TermEstimator.cs
+++ b/src/Microsoft.ML.Data/Transforms/TermEstimator.cs
@@ -3,6 +3,10 @@
// See the LICENSE file in the project root for more information.
using Microsoft.ML.Core.Data;
+using Microsoft.ML.Data.StaticPipe.Runtime;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
using System.Linq;
namespace Microsoft.ML.Runtime.Data
@@ -57,4 +61,137 @@ public SchemaShape GetOutputSchema(SchemaShape inputSchema)
return new SchemaShape(result.Values);
}
}
+
+ public enum KeyValueOrder : byte
+ {
+ ///
+ /// Terms will be assigned ID in the order in which they appear.
+ ///
+ Occurence = TermTransform.SortOrder.Occurrence,
+
+ ///
+ /// Terms will be assigned ID according to their sort via an ordinal comparison for the type.
+ ///
+ Value = TermTransform.SortOrder.Value
+ }
+
+ ///
+ /// Information on the result of fitting a to-key transform.
+ ///
+ /// The type of the values.
+ public sealed class ToKeyFitResult
+ {
+ ///
+ /// For user defined delegates that accept instances of the containing type.
+ ///
+ ///
+ public delegate void OnFit(ToKeyFitResult result);
+
+ // At the moment this is empty. Once PR #863 clears, we can change this class to hold the output
+ // key-values metadata.
+
+ internal ToKeyFitResult(TermTransform.TermMap map)
+ {
+ }
+ }
+
+ public static partial class TermStaticExtensions
+ {
+ // I am not certain I see a good way to cover the distinct types beyond complete enumeration.
+ // Raw generics would allow illegal possible inputs, e.g., Scalar. So, this is a partial
+ // class, and all the public facing extension methods for each possible type are in a T4 generated result.
+
+ private const KeyValueOrder DefSort = (KeyValueOrder)TermTransform.Defaults.Sort;
+ private const int DefMax = TermTransform.Defaults.MaxNumTerms;
+
+ private struct Config
+ {
+ public readonly KeyValueOrder Order;
+ public readonly int Max;
+ public readonly Action OnFit;
+
+ public Config(KeyValueOrder order, int max, Action onFit)
+ {
+ Order = order;
+ Max = max;
+ OnFit = onFit;
+ }
+ }
+
+ private static Action Wrap(ToKeyFitResult.OnFit onFit)
+ {
+ if (onFit == null)
+ return null;
+ // The type T asociated with the delegate will be the actual value type once #863 goes in.
+ // However, until such time as #863 goes in, it would be too awkward to attempt to extract the metadata.
+ // For now construct the useless object then pass it into the delegate.
+ return map => onFit(new ToKeyFitResult(map));
+ }
+
+ private interface ITermCol
+ {
+ PipelineColumn Input { get; }
+ Config Config { get; }
+ }
+
+ private sealed class ImplScalar : Key, ITermCol
+ {
+ public PipelineColumn Input { get; }
+ public Config Config { get; }
+ public ImplScalar(PipelineColumn input, Config config) : base(Rec.Inst, input)
+ {
+ Input = input;
+ Config = config;
+ }
+ }
+
+ private sealed class ImplVector : Vector>, ITermCol
+ {
+ public PipelineColumn Input { get; }
+ public Config Config { get; }
+ public ImplVector(PipelineColumn input, Config config) : base(Rec.Inst, input)
+ {
+ Input = input;
+ Config = config;
+ }
+ }
+
+ private sealed class ImplVarVector : VarVector>, ITermCol
+ {
+ public PipelineColumn Input { get; }
+ public Config Config { get; }
+ public ImplVarVector(PipelineColumn input, Config config) : base(Rec.Inst, input)
+ {
+ Input = input;
+ Config = config;
+ }
+ }
+
+ private sealed class Rec : EstimatorReconciler
+ {
+ public static readonly Rec Inst = new Rec();
+
+ public override IEstimator Reconcile(IHostEnvironment env, PipelineColumn[] toOutput,
+ IReadOnlyDictionary inputNames, IReadOnlyDictionary outputNames, IReadOnlyCollection usedNames)
+ {
+ var infos = new TermTransform.ColumnInfo[toOutput.Length];
+ Action onFit = null;
+ for (int i=0; i tcol.Config.OnFit(tt.GetTermMap(ii));
+ }
+ }
+ var est = new TermEstimator(env, infos);
+ if (onFit == null)
+ return est;
+ return est.WithOnFitDelegate(onFit);
+ }
+ }
+ }
}
diff --git a/src/Microsoft.ML.Data/Transforms/TermStaticExtensions.cs b/src/Microsoft.ML.Data/Transforms/TermStaticExtensions.cs
new file mode 100644
index 0000000000..1da73ff2d8
--- /dev/null
+++ b/src/Microsoft.ML.Data/Transforms/TermStaticExtensions.cs
@@ -0,0 +1,1140 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using Microsoft.ML.Data.StaticPipe.Runtime;
+
+namespace Microsoft.ML.Runtime.Data
+{
+ public static partial class TermStaticExtensions
+ {
+ // Do not edit this file directly. Rather, it is generated out of TermStaticExtensions.tt.
+
+ #region For string inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult>.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult>.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult>.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Key input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult>.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult>.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because the empty string is never entered into the dictionary, it will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult>.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+ #endregion
+
+ #region For float inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Key input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+ #endregion
+
+ #region For double inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Key input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Because NaN floating point values are never entered into the dictionary, and they will always map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+ #endregion
+
+ #region For sbyte inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Key input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+ #endregion
+
+ #region For short inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Key input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+ #endregion
+
+ #region For int inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Key input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// We are inputting a key type with values, and in that case the dictionary is considered to be built over the
+ /// values of the keys, rather than the keys themselves. This also mean the key-values learned for the output
+ /// will be a subset of the key-values in the input.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static VarVector> ToKey(this VarVector> input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVarVector(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+ #endregion
+
+ #region For long inputs.
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Key ToKey(this Scalar input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplScalar(Contracts.CheckRef(input, nameof(input)), new Config(order, maxItems, Wrap(onFit)));
+
+ ///
+ /// Map values to a key-value representation, where the key type's values are those values observed in the input
+ /// during fitting. During transformation, any values unobserved during fitting will map to the missing key.
+ /// Zero is considered a valid value and so will be entered into the dictionary if observed. The potential perf
+ /// implication in that case is that sparse input numeric vectors will map to dense output key vectors.
+ ///
+ /// The input column.
+ /// The ordering policy for what order values will appear in the enumerated set.
+ /// The maximum number of items.
+ /// Called upon fitting with the learnt enumeration on the dataset.
+ /// The key-valued column.
+ public static Vector> ToKey(this Vector input,
+ KeyValueOrder order = DefSort, int maxItems = DefMax, ToKeyFitResult.OnFit onFit = null)
+ => new ImplVector