diff --git a/src/Microsoft.ML.Core/Utilities/Contracts.cs b/src/Microsoft.ML.Core/Utilities/Contracts.cs index bcfc43164e..89c18b3bd4 100644 --- a/src/Microsoft.ML.Core/Utilities/Contracts.cs +++ b/src/Microsoft.ML.Core/Utilities/Contracts.cs @@ -570,7 +570,14 @@ public static T CheckRef(this IExceptionContext ctx, T val, string paramName) return val; } - public static void CheckValue(T val, string paramName) where T : class + public static T CheckRef(this IExceptionContext ctx, T val, string paramName, string msg) where T : class + { + if (object.ReferenceEquals(val, null)) + throw ExceptValue(ctx, paramName, msg); + return val; + } + + public static void CheckValue(T val, string paramName) where T : class { if (object.ReferenceEquals(val, null)) throw ExceptValue(paramName); diff --git a/src/Microsoft.ML.Data/Data/RowCursorUtils.cs b/src/Microsoft.ML.Data/Data/RowCursorUtils.cs index 9f09e0da51..e47873c865 100644 --- a/src/Microsoft.ML.Data/Data/RowCursorUtils.cs +++ b/src/Microsoft.ML.Data/Data/RowCursorUtils.cs @@ -472,6 +472,19 @@ public static IRow CloneRow(IRow row) Utils.BuildArray(row.Schema.ColumnCount, c => RowColumnUtils.GetColumn(row, c))); } + /// + /// Fetches the value of the column by name, in the given row. + /// Used by the evaluators to retrieve the metrics from the results IDataView. + /// + public static T Fetch(IExceptionContext ectx, IRow row, string name) + { + if (!row.Schema.TryGetColumnIndex(name, out int col)) + throw ectx.Except($"Could not find column '{name}'"); + T val = default; + row.GetGetter(col)(ref val); + return val; + } + /// /// Given a row, returns a one-row data view. This is useful for cases where you have a row, and you /// wish to use some facility normally only exposed to dataviews. (E.g., you have an diff --git a/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs index 2c476eb468..05dcba53c7 100644 --- a/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs @@ -52,6 +52,46 @@ public ClusteringEvaluator(IHostEnvironment env, Arguments args) _calculateDbi = args.CalculateDbi; } + /// + /// Evaluates scored clustering data. + /// + /// The scored data. + /// The name of the score column in . + /// The name of the optional label column in . + /// The name of the optional feature column in . + /// The evaluation results. + public Result Evaluate(IDataView data, string score, string label = null, string features = null) + { + Host.CheckValue(data, nameof(data)); + Host.CheckNonEmpty(score, nameof(score)); + + var roles = new List>(); + roles.Add(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, score)); + + if (label != null) + roles.Add(RoleMappedSchema.ColumnRole.Label.Bind(label)); + + if (features != null) + roles.Add(RoleMappedSchema.ColumnRole.Feature.Bind(features)); + + var rolesMappedData = new RoleMappedData(data, opt: false, roles.ToArray()); + + var resultDict = Evaluate(rolesMappedData); + Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics)); + var overall = resultDict[MetricKinds.OverallMetrics]; + + Result result; + using (var cursor = overall.GetRowCursor(i => true)) + { + var moved = cursor.MoveNext(); + Host.Assert(moved); + result = new Result(Host, cursor, _calculateDbi); + moved = cursor.MoveNext(); + Host.Assert(!moved); + } + return result; + } + protected override void CheckScoreAndLabelTypes(RoleMappedSchema schema) { ColumnType type; @@ -517,6 +557,46 @@ private void AssertValid(bool assertGetters) } } } + + /// + /// The metrics generated after evaluating the clustering predictions. + /// + public sealed class Result + { + /// + /// Normalized Mutual Information + /// NMI is a measure of the mutual dependence of the variables. + /// Normalized variants work on data that already has cluster labels. + /// Its value ranged from 0 to 1, where higher numbers are better. + /// + public double Nmi { get; } + + /// + /// Average Score. For the K-Means algorithm, the 'score' is the distance from the centroid to the example. + /// The average score is, therefore, a measure of proximity of the examples to cluster centroids. + /// In other words, it's the 'cluster tightness' measure. + /// Note however, that this metric will only decrease if the number of clusters is increased, + /// and in the extreme case (where each distinct example is its own cluster) it will be equal to zero. + /// + public double AvgMinScore { get; } + + /// + /// Davies-Bouldin Index + /// DBI is a measure of the how much scatter is in the cluster and the cluster separation. + /// + public double Dbi { get; } + + internal Result(IExceptionContext ectx, IRow overallResult, bool calculateDbi) + { + double Fetch(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); + + Nmi = Fetch(ClusteringEvaluator.Nmi); + AvgMinScore = Fetch(ClusteringEvaluator.AvgMinScore); + + if(calculateDbi) + Dbi = Fetch(ClusteringEvaluator.Dbi); + } + } } public sealed class ClusteringPerInstanceEvaluator : PerInstanceEvaluatorBase diff --git a/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs b/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs index 8fed434a94..0867967d51 100644 --- a/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs +++ b/src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs @@ -84,6 +84,39 @@ public static BinaryClassifierEvaluator.Result Evaluate( return eval.Evaluate(data.AsDynamic, labelName, scoreName, predName); } + /// + /// Evaluates scored clustering prediction data. + /// + /// The shape type for the input data. + /// The clustering context. + /// The data to evaluate. + /// The index delegate for the predicted score column. + /// The optional index delegate for the label column. + /// The optional index delegate for the features column. + /// The evaluation metrics. + public static ClusteringEvaluator.Result Evaluate( + this ClusteringContext ctx, + DataView data, + Func> score, + Func> label = null, + Func> features = null) + { + Contracts.CheckValue(data, nameof(data)); + var env = StaticPipeUtils.GetEnvironment(data); + Contracts.AssertValue(env); + env.CheckValue(score, nameof(score)); + + var indexer = StaticPipeUtils.GetIndexer(data); + string scoreName = indexer.Get(score(indexer.Indices)); + + string labelName = (label != null)? indexer.Get(label(indexer.Indices)) : null; + string featuresName = (features!= null) ? indexer.Get(features(indexer.Indices)): null; + + var args = new ClusteringEvaluator.Arguments() { CalculateDbi = !string.IsNullOrEmpty(featuresName) }; + + return new ClusteringEvaluator(env, args).Evaluate(data.AsDynamic, scoreName, labelName, featuresName); + } + /// /// Evaluates scored multiclass classification data. /// @@ -136,7 +169,7 @@ private sealed class TrivialRegressionLossFactory : ISupportRegressionLossFactor } /// - /// Evaluates scored multiclass classification data. + /// Evaluates scored regression data. /// /// The shape type for the input data. /// The regression context. diff --git a/src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs index fe839ee108..5248539d9c 100644 --- a/src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs @@ -570,26 +570,18 @@ public sealed class Result /// public double[] PerClassLogLoss { get; } - private static T Fetch(IExceptionContext ectx, IRow row, string name) - { - if (!row.Schema.TryGetColumnIndex(name, out int col)) - throw ectx.Except($"Could not find column '{name}'"); - T val = default; - row.GetGetter(col)(ref val); - return val; - } internal Result(IExceptionContext ectx, IRow overallResult, int topK) { - double Fetch(string name) => Fetch(ectx, overallResult, name); - AccuracyMicro = Fetch(MultiClassClassifierEvaluator.AccuracyMicro); - AccuracyMacro = Fetch(MultiClassClassifierEvaluator.AccuracyMacro); - LogLoss = Fetch(MultiClassClassifierEvaluator.LogLoss); - LogLossReduction = Fetch(MultiClassClassifierEvaluator.LogLossReduction); + double FetchDouble(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); + AccuracyMicro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMicro); + AccuracyMacro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMacro); + LogLoss = FetchDouble(MultiClassClassifierEvaluator.LogLoss); + LogLossReduction = FetchDouble(MultiClassClassifierEvaluator.LogLossReduction); TopK = topK; if (topK > 0) - TopKAccuracy = Fetch(MultiClassClassifierEvaluator.TopKAccuracy); + TopKAccuracy = FetchDouble(MultiClassClassifierEvaluator.TopKAccuracy); - var perClassLogLoss = Fetch>(ectx, overallResult, MultiClassClassifierEvaluator.PerClassLogLoss); + var perClassLogLoss = RowCursorUtils.Fetch>(ectx, overallResult, MultiClassClassifierEvaluator.PerClassLogLoss); PerClassLogLoss = new double[perClassLogLoss.Length]; perClassLogLoss.CopyTo(PerClassLogLoss); } diff --git a/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs b/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs index d210a8e52a..d02819c4ee 100644 --- a/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs +++ b/src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs @@ -203,18 +203,9 @@ public sealed class Result /// public double RSquared { get; } - private static T Fetch(IExceptionContext ectx, IRow row, string name) - { - if (!row.Schema.TryGetColumnIndex(name, out int col)) - throw ectx.Except($"Could not find column '{name}'"); - T val = default; - row.GetGetter(col)(ref val); - return val; - } - internal Result(IExceptionContext ectx, IRow overallResult) { - double Fetch(string name) => Fetch(ectx, overallResult, name); + double Fetch(string name) => RowCursorUtils.Fetch(ectx, overallResult, name); L1 = Fetch(RegressionEvaluator.L1); L2 = Fetch(RegressionEvaluator.L2); Rms = Fetch(RegressionEvaluator.Rms); @@ -230,10 +221,7 @@ internal Result(IExceptionContext ectx, IRow overallResult) /// The name of the label column. /// The name of the predicted score column. /// The evaluation metrics for these outputs. - public Result Evaluate( - IDataView data, - string label, - string score) + public Result Evaluate(IDataView data, string label, string score) { Host.CheckValue(data, nameof(data)); Host.CheckNonEmpty(label, nameof(label)); diff --git a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs index 3e9208ff5b..faedb6fb85 100644 --- a/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs +++ b/src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs @@ -23,6 +23,9 @@ [assembly: LoadableClass(typeof(AnomalyPredictionTransformer>), typeof(AnomalyPredictionTransformer), null, typeof(SignatureLoadModel), "", AnomalyPredictionTransformer.LoaderSignature)] +[assembly: LoadableClass(typeof(ClusteringPredictionTransformer>>), typeof(ClusteringPredictionTransformer), null, typeof(SignatureLoadModel), + "", ClusteringPredictionTransformer.LoaderSignature)] + namespace Microsoft.ML.Runtime.Data { @@ -508,6 +511,58 @@ private static VersionInfo GetVersionInfo() } } + /// + /// Base class for the working on clustering tasks. + /// + /// An implementation of the + public sealed class ClusteringPredictionTransformer : SingleFeaturePredictionTransformerBase + where TModel : class, IPredictorProducing> + { + public ClusteringPredictionTransformer(IHostEnvironment env, TModel model, ISchema inputSchema, string featureColumn, + float threshold = 0f, string thresholdColumn = DefaultColumnNames.Score) + : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ClusteringPredictionTransformer)), model, inputSchema, featureColumn) + { + Host.CheckNonEmpty(thresholdColumn, nameof(thresholdColumn)); + var schema = new RoleMappedSchema(inputSchema, null, featureColumn); + + var args = new ClusteringScorer.Arguments(); + Scorer = new ClusteringScorer(Host, args, new EmptyDataView(Host, inputSchema), BindableMapper.Bind(Host, schema), schema); + } + + public ClusteringPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx) + : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ClusteringPredictionTransformer)), ctx) + { + // *** Binary format *** + // + + var schema = new RoleMappedSchema(TrainSchema, null, FeatureColumn); + var args = new ClusteringScorer.Arguments(); + Scorer = new ClusteringScorer(Host, args, new EmptyDataView(Host, TrainSchema), BindableMapper.Bind(Host, schema), schema); + } + + protected override void SaveCore(ModelSaveContext ctx) + { + Contracts.AssertValue(ctx); + ctx.SetVersionInfo(GetVersionInfo()); + + // *** Binary format *** + // + // id of string: scorer threshold column + base.SaveCore(ctx); + } + + private static VersionInfo GetVersionInfo() + { + return new VersionInfo( + modelSignature: "CLUSPRED", + verWrittenCur: 0x00010001, // Initial + verReadableCur: 0x00010001, + verWeCanReadBack: 0x00010001, + loaderSignature: ClusteringPredictionTransformer.LoaderSignature, + loaderAssemblyName: typeof(ClusteringPredictionTransformer<>).Assembly.FullName); + } + } + internal static class BinaryPredictionTransformer { public const string LoaderSignature = "BinaryPredXfer"; @@ -547,4 +602,12 @@ internal static class AnomalyPredictionTransformer public static AnomalyPredictionTransformer> Create(IHostEnvironment env, ModelLoadContext ctx) => new AnomalyPredictionTransformer>(env, ctx); } + + internal static class ClusteringPredictionTransformer + { + public const string LoaderSignature = "ClusteringPredXfer"; + + public static ClusteringPredictionTransformer>> Create(IHostEnvironment env, ModelLoadContext ctx) + => new ClusteringPredictionTransformer>>(env, ctx); + } } diff --git a/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs b/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs index ff2b9d7c02..da5fb225c7 100644 --- a/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs +++ b/src/Microsoft.ML.Data/StaticPipe/TrainerEstimatorReconciler.cs @@ -337,6 +337,73 @@ public ImplBool(BinaryClassifierNoCalibration rec) : base(rec, rec.Inputs) { } } } + /// + /// A reconciler capable of handling clustering. + /// + public sealed class Clustering : TrainerEstimatorReconciler + { + /// + /// The delegate to create the clustering trainer instance. + /// + /// The environment with which to create the estimator. + /// The features column name. + /// The weights column name, or null if the reconciler was constructed with null weights. + /// A clustering trainer estimator. + public delegate IEstimator EstimatorFactory(IHostEnvironment env, string features, string weights); + + private readonly EstimatorFactory _estFact; + private static readonly string[] _fixedOutputNames = new[] { DefaultColumnNames.Score, DefaultColumnNames.PredictedLabel }; + + /// + /// The general output for clustering. + /// + public (Vector score, Key predictedLabel) Output { get; } + + /// + /// The output columns, which will contain the columns produced by . + /// + protected override IEnumerable Outputs => new PipelineColumn[] { Output.score, Output.predictedLabel }; + + /// + /// Constructs a new general clustering reconciler. + /// + /// The delegate to create the training estimator. It is assumed that this estimator + /// will produce a new scalar column named and a + /// named PredictedLabel. + /// The input features column. + /// The input weights column, or null if there are no weights. + public Clustering(EstimatorFactory estimatorFactory, Vector features, Scalar weights) + : base(MakeInputs(Contracts.CheckRef(features, nameof(features)), weights), + _fixedOutputNames) + { + Contracts.CheckValue(estimatorFactory, nameof(estimatorFactory)); + _estFact = estimatorFactory; + Contracts.Assert(Inputs.Length == 1 || Inputs.Length == 2); + + Output = (new ImplScore(this), new ImplLabel(this)); + } + + private static PipelineColumn[] MakeInputs(Vector features, Scalar weights) + => weights == null ? new PipelineColumn[] { features } : new PipelineColumn[] { features, weights }; + + protected override IEstimator ReconcileCore(IHostEnvironment env, string[] inputNames) + { + Contracts.AssertValue(env); + env.Assert(Utils.Size(inputNames) == Inputs.Length); + return _estFact(env, inputNames[0], inputNames.Length > 1 ? inputNames[1] : null); + } + + private sealed class ImplScore : Vector + { + public ImplScore(Clustering rec) : base(rec, rec.Inputs) { } + } + + private sealed class ImplLabel : Key + { + public ImplLabel(Clustering rec) : base(rec, rec.Inputs) { } + } + } + /// /// A reconciler for regression capable of handling the most common cases for regression. /// diff --git a/src/Microsoft.ML.Data/Training/TrainContext.cs b/src/Microsoft.ML.Data/Training/TrainContext.cs index c2593654cf..087fe4134f 100644 --- a/src/Microsoft.ML.Data/Training/TrainContext.cs +++ b/src/Microsoft.ML.Data/Training/TrainContext.cs @@ -209,9 +209,6 @@ public sealed class BinaryClassificationContext : TrainContextBase /// /// For trainers for performing binary classification. /// - /// - /// Component authors that have written binary classification. They are great people. - /// public BinaryClassificationTrainers Trainers { get; } public BinaryClassificationContext(IHostEnvironment env) @@ -315,6 +312,62 @@ public BinaryClassifierEvaluator.Result EvaluateNonCalibrated(IDataView data, st } } + /// + /// The central context for clustering trainers. + /// + public sealed class ClusteringContext : TrainContextBase + { + /// + /// List of trainers for performing clustering. + /// + public ClusteringTrainers Trainers { get; } + + /// + /// The clustering context. + /// + public ClusteringContext(IHostEnvironment env) + : base(env, nameof(ClusteringContext)) + { + Trainers = new ClusteringTrainers(this); + } + + public sealed class ClusteringTrainers : ContextInstantiatorBase + { + internal ClusteringTrainers(ClusteringContext ctx) + : base(ctx) + { + } + } + + /// + /// Evaluates scored clustering data. + /// + /// The scored data. + /// The name of the score column in . + /// The name of the optional label column in . + /// If present, the metric will be computed. + /// The name of the optional features column in . + /// If present, the metric will be computed. + /// The evaluation result. + public ClusteringEvaluator.Result Evaluate(IDataView data, + string label = null, + string score = DefaultColumnNames.Score, + string features = null ) + { + Host.CheckValue(data, nameof(data)); + Host.CheckNonEmpty(score, nameof(score)); + + if(features != null) + Host.CheckNonEmpty(features, nameof(features), "The features column name should be non-empty, if provided, if you want to calculate the Dbi metric."); + + if (label != null) + Host.CheckNonEmpty(label, nameof(label), "The features column name should be non-empty, if provided, if you want to calculate the Nmi metric."); + + var eval = new ClusteringEvaluator(Host, new ClusteringEvaluator.Arguments() { CalculateDbi = !string.IsNullOrEmpty(features) }); + return eval.Evaluate(data, score, label, features); + } + } + /// /// The central context for multiclass classification trainers. /// diff --git a/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs b/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs index 46f32a7da3..d6b32ad2d6 100644 --- a/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs +++ b/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs @@ -2,21 +2,20 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Float = System.Single; - -using System; -using System.Linq; -using System.Threading.Tasks; +using Microsoft.ML.Core.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.CommandLine; using Microsoft.ML.Runtime.Data; -using Microsoft.ML.Runtime.Internal.Utilities; +using Microsoft.ML.Runtime.EntryPoints; using Microsoft.ML.Runtime.Internal.CpuMath; +using Microsoft.ML.Runtime.Internal.Internallearn; +using Microsoft.ML.Runtime.Internal.Utilities; using Microsoft.ML.Runtime.KMeans; using Microsoft.ML.Runtime.Numeric; using Microsoft.ML.Runtime.Training; -using Microsoft.ML.Runtime.Internal.Internallearn; -using Microsoft.ML.Runtime.EntryPoints; +using System; +using System.Linq; +using System.Threading.Tasks; [assembly: LoadableClass(KMeansPlusPlusTrainer.Summary, typeof(KMeansPlusPlusTrainer), typeof(KMeansPlusPlusTrainer.Arguments), new[] { typeof(SignatureClusteringTrainer), typeof(SignatureTrainer) }, @@ -29,7 +28,7 @@ namespace Microsoft.ML.Runtime.KMeans { /// - public class KMeansPlusPlusTrainer : TrainerBase + public class KMeansPlusPlusTrainer : TrainerEstimatorBase, KMeansPredictor> { public const string LoadNameValue = "KMeansPlusPlus"; internal const string UserNameValue = "KMeans++ Clustering"; @@ -45,12 +44,18 @@ public enum InitAlgorithm KMeansParallel = 2 } + internal static class Defaults{ + + /// The number of clusters. + internal const int K = 5; + } + public class Arguments : UnsupervisedLearnerInputBaseWithWeight { [Argument(ArgumentType.AtMostOnce, HelpText = "The number of clusters", SortOrder = 50)] [TGUI(SuggestedSweeps = "5,10,20,40")] [TlcModule.SweepableDiscreteParam("K", new object[] { 5, 10, 20, 40 })] - public int K = 5; + public int K = Defaults.K; [Argument(ArgumentType.AtMostOnce, HelpText = "Cluster initialization algorithm", ShortName = "init")] public InitAlgorithm InitAlgorithm = InitAlgorithm.KMeansParallel; @@ -58,7 +63,7 @@ public class Arguments : UnsupervisedLearnerInputBaseWithWeight [Argument(ArgumentType.AtMostOnce, HelpText = "Tolerance parameter for trainer convergence. Lower = slower, more accurate", ShortName = "ot")] [TGUI(Label = "Optimization Tolerance", Description = "Threshold for trainer convergence")] - public Float OptTol = (Float)1e-7; + public float OptTol = (float)1e-7; [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum number of iterations.", ShortName = "maxiter")] [TGUI(Label = "Max Number of Iterations")] @@ -76,21 +81,49 @@ public class Arguments : UnsupervisedLearnerInputBaseWithWeight private readonly int _k; private readonly int _maxIterations; // max number of iterations to train - private readonly Float _convergenceThreshold; // convergence thresholds + private readonly float _convergenceThreshold; // convergence thresholds private readonly long _accelMemBudgetMb; private readonly InitAlgorithm _initAlgorithm; private readonly int _numThreads; + private readonly string _featureColumn; public override TrainerInfo Info { get; } public override PredictionKind PredictionKind => PredictionKind.Clustering; - public KMeansPlusPlusTrainer(IHostEnvironment env, Arguments args) - : base(env, LoadNameValue) + /// + /// Initializes a new instance of + /// + /// The private instance of . + /// The name of the feature column. + /// The name for the column containing the example weights. + /// A delegate to apply all the advanced arguments to the algorithm. + /// The number of clusters. + public KMeansPlusPlusTrainer(IHostEnvironment env, string featureColumn, int clustersCount = Defaults.K, string weightColumn = null, Action advancedSettings = null) + : this(env, new Arguments(), featureColumn, weightColumn, advancedSettings) + { + _k = clustersCount; + } + + internal KMeansPlusPlusTrainer(IHostEnvironment env, Arguments args) + : this(env, args, args.FeatureColumn, args.WeightColumn, null) + { + + } + + private KMeansPlusPlusTrainer(IHostEnvironment env, Arguments args, string featureColumn, string weightColumn, Action advancedSettings = null) + : base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), TrainerUtils.MakeR4VecFeature(featureColumn), null, TrainerUtils.MakeR4ScalarWeightColumn(weightColumn)) { Host.CheckValue(args, nameof(args)); + + if (advancedSettings != null) + advancedSettings.Invoke(args); + Host.CheckUserArg(args.K > 0, nameof(args.K), "Must be positive"); + Host.CheckNonEmpty(featureColumn, nameof(featureColumn)); + _featureColumn = featureColumn; + _k = args.K; Host.CheckUserArg(args.MaxIterations > 0, nameof(args.MaxIterations), "Must be positive"); @@ -110,7 +143,7 @@ public KMeansPlusPlusTrainer(IHostEnvironment env, Arguments args) Info = new TrainerInfo(); } - public override KMeansPredictor Train(TrainContext context) + protected override KMeansPredictor TrainModelCore(TrainContext context) { Host.CheckValue(context, nameof(context)); var data = context.TrainingSet; @@ -136,9 +169,9 @@ private KMeansPredictor TrainCore(IChannel ch, RoleMappedData data, int dimensio // The coordinates of the final centroids at the end of the training. During training // it holds the centroids of the previous iteration. - var centroids = new VBuffer[_k]; + var centroids = new VBuffer[_k]; for (int i = 0; i < _k; i++) - centroids[i] = VBufferUtils.CreateDense(dimensionality); + centroids[i] = VBufferUtils.CreateDense(dimensionality); ch.Info("Initializing centroids"); long missingFeatureCount; @@ -220,11 +253,32 @@ public static CommonOutputs.ClusteringOutput TrainKMeans(IHostEnvironment env, A () => new KMeansPlusPlusTrainer(host, input), getWeight: () => LearnerEntryPointsUtils.FindColumn(host, input.TrainingData.Schema, input.WeightColumn)); } + + protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) + { + return new[] + { + new SchemaShape.Column(DefaultColumnNames.Score, + SchemaShape.Column.VectorKind.Vector, + NumberType.R4, + false, + new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())), + + new SchemaShape.Column(DefaultColumnNames.PredictedLabel, + SchemaShape.Column.VectorKind.Scalar, + NumberType.U4, + true, + new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())) + }; + } + + protected override ClusteringPredictionTransformer MakeTransformer(KMeansPredictor model, ISchema trainSchema) + => new ClusteringPredictionTransformer(Host, model, trainSchema, _featureColumn); } internal static class KMeansPlusPlusInit { - private const Float Epsilon = (Float)1e-15; + private const float Epsilon = (float)1e-15; /// /// Initialize starting centroids via KMeans++ algorithm. This algorithm will always run single-threaded, @@ -232,7 +286,7 @@ internal static class KMeansPlusPlusInit /// public static void Initialize( IHost host, int numThreads, IChannel ch, FeatureFloatVectorCursor.Factory cursorFactory, - int k, int dimensionality, VBuffer[] centroids, + int k, int dimensionality, VBuffer[] centroids, out long missingFeatureCount, out long totalTrainingInstances, bool showWarning = true) { missingFeatureCount = 0; @@ -240,11 +294,11 @@ public static void Initialize( var stopWatch = new System.Diagnostics.Stopwatch(); stopWatch.Start(); const int checkIterations = 5; - Float[] centroidL2s = new Float[k]; + float[] centroidL2s = new float[k]; // Need two vbuffers - one for the features of the current row, and one for the features of // the "candidate row". - var candidate = default(VBuffer); + var candidate = default(VBuffer); using (var pCh = host.StartProgressChannel("KMeansPlusPlusInitialize")) { int i = 0; @@ -268,15 +322,15 @@ public static void Initialize( // initializing i-th centroid Double cumulativeWeight = 0; // sum of weights accumulated so far - Float? cachedCandidateL2 = null; + float? cachedCandidateL2 = null; bool haveCandidate = false; // on all iterations except 0's, we calculate L2 norm of every instance and cache the current candidate's using (var cursor = cursorFactory.Create()) { while (cursor.MoveNext()) { - Float probabilityWeight; - Float l2 = 0; + float probabilityWeight; + float l2 = 0; if (i == 0) { // This check is only performed once, at the first pass of initialization @@ -292,7 +346,7 @@ public static void Initialize( else { l2 = VectorUtils.NormSquared(cursor.Features); - probabilityWeight = Float.PositiveInfinity; + probabilityWeight = float.PositiveInfinity; for (int j = 0; j < i; j++) { @@ -455,7 +509,7 @@ private sealed class SharedState // _bestWeight holds the weight of instance x to _bestCluster[x] where weight(x) = dist(x, _bestCluster[x])^2 - norm(x)^2. // Note that this array is only allocated for MaxInstancesToAccelerate elements. - private readonly Float[] _bestWeight; + private readonly float[] _bestWeight; // The distance of each newly added cluster from the previous round to every old cluster // the first dimension of this array is the size of numSamplesPerRound @@ -464,7 +518,7 @@ private sealed class SharedState // where clusterPrevCount-1 is the last index of the old clusters // and new clusters are stored in cluster[cPrevIdx..clusterCount-1] where // clusterCount-1 is the last index of the clusters. - private readonly Float[,] _clusterDistances; + private readonly float[,] _clusterDistances; public SharedState(FeatureFloatVectorCursor.Factory factory, IChannel ch, long baseMaxInstancesToAccelerate, long clusterBytes, bool isParallel, int numRounds, int numSamplesPerRound, long totalTrainingInstances) @@ -483,14 +537,14 @@ public SharedState(FeatureFloatVectorCursor.Factory factory, IChannel ch, long b // and allocate _bestCluster and _bestWeight as much as possible. if (_acceleratedRowMap.MaxInstancesToAccelerate > 0) { - _clusterDistances = new Float[numSamplesPerRound, numRounds * numSamplesPerRound]; + _clusterDistances = new float[numSamplesPerRound, numRounds * numSamplesPerRound]; _bestCluster = new int[_acceleratedRowMap.MaxInstancesToAccelerate]; - _bestWeight = new Float[_acceleratedRowMap.MaxInstancesToAccelerate]; + _bestWeight = new float[_acceleratedRowMap.MaxInstancesToAccelerate]; for (int i = 0; i < _acceleratedRowMap.MaxInstancesToAccelerate; i++) { _bestCluster[i] = -1; - _bestWeight[i] = Float.MaxValue; + _bestWeight[i] = float.MaxValue; } } else @@ -502,7 +556,7 @@ public int GetBestCluster(int idx) return _bestCluster[idx]; } - public Float GetBestWeight(int idx) + public float GetBestWeight(int idx) { return _bestWeight[idx]; } @@ -514,7 +568,7 @@ public Float GetBestWeight(int idx) /// corresponding to the weight of a distance which does not have /// the L2 norm of the point itself. /// - public void SetInstanceCluster(int n, Float bestWeight, int bestCluster) + public void SetInstanceCluster(int n, float bestWeight, int bestCluster) { Contracts.Assert(0 <= n && n < _acceleratedRowMap.MaxInstancesToAccelerate); Contracts.AssertValue(_clusterDistances); @@ -529,8 +583,8 @@ public void SetInstanceCluster(int n, Float bestWeight, int bestCluster) /// Computes and stores the distance of a new cluster to an old cluster /// must be between 0..numSamplesPerRound-1. /// - public void SetClusterDistance(int newClusterIdxWithinSample, ref VBuffer newClusterFeatures, Float newClusterL2, - int oldClusterIdx, ref VBuffer oldClusterFeatures, Float oldClusterL2) + public void SetClusterDistance(int newClusterIdxWithinSample, ref VBuffer newClusterFeatures, float newClusterL2, + int oldClusterIdx, ref VBuffer oldClusterFeatures, float oldClusterL2) { if (_clusterDistances != null) { @@ -548,7 +602,7 @@ public void SetClusterDistance(int newClusterIdxWithinSample, ref VBuffer /// old cluster, cOld, and distance of a new cluster, cNew, to cOld, this function evaluates whether /// the distance computation of dist(x,cNew) can be avoided. /// - public bool CanWeightComputationBeAvoided(Float instanceDistanceToBestOldCluster, int bestOldCluster, int newClusterIdxWithinSample) + public bool CanWeightComputationBeAvoided(float instanceDistanceToBestOldCluster, int bestOldCluster, int newClusterIdxWithinSample) { Contracts.Assert(instanceDistanceToBestOldCluster >= 0); Contracts.Assert(0 <= newClusterIdxWithinSample && newClusterIdxWithinSample < _clusterDistances.GetLength(0), @@ -560,7 +614,7 @@ public bool CanWeightComputationBeAvoided(Float instanceDistanceToBestOldCluster if (_clusterDistances != null && bestOldCluster != -1) { // This is dist(cNew,cOld). - Float distanceBetweenOldAndNewClusters = _clusterDistances[newClusterIdxWithinSample, bestOldCluster]; + float distanceBetweenOldAndNewClusters = _clusterDistances[newClusterIdxWithinSample, bestOldCluster]; // Use triangle inequality to evaluate whether weight computation can be avoided // dist(x,cNew) + dist(x,cOld) > dist(cOld,cNew) => @@ -581,9 +635,9 @@ public bool CanWeightComputationBeAvoided(Float instanceDistanceToBestOldCluster /// Note that is used to avoid the storing the new cluster in /// final round. After the final round, best cluster information will be ignored. /// - private static void FindBestCluster(ref VBuffer point, int pointRowIndex, SharedState initializationState, - int clusterCount, int clusterPrevCount, VBuffer[] clusters, Float[] clustersL2s, bool needRealDistanceSquared, bool needToStoreWeight, - out Float minDistanceSquared, out int bestCluster) + private static void FindBestCluster(ref VBuffer point, int pointRowIndex, SharedState initializationState, + int clusterCount, int clusterPrevCount, VBuffer[] clusters, float[] clustersL2s, bool needRealDistanceSquared, bool needToStoreWeight, + out float minDistanceSquared, out int bestCluster) { Contracts.AssertValue(initializationState); Contracts.Assert(clusterCount > 0); @@ -596,18 +650,18 @@ private static void FindBestCluster(ref VBuffer point, int pointRowIndex, if (pointRowIndex != -1) // if the space was available for cur in initializationState. { // pointNorm is necessary for using triangle inequality. - Float pointNorm = VectorUtils.NormSquared(point); + float pointNorm = VectorUtils.NormSquared(point); // We have cached distance information for this point. bestCluster = initializationState.GetBestCluster(pointRowIndex); - Float bestWeight = initializationState.GetBestWeight(pointRowIndex); + float bestWeight = initializationState.GetBestWeight(pointRowIndex); // This is used by CanWeightComputationBeAvoided function in order to shortcut weight computation. int bestOldCluster = bestCluster; - Float pointDistanceSquared = pointNorm + bestWeight; + float pointDistanceSquared = pointNorm + bestWeight; // Make pointDistanceSquared zero if it is negative, which it can be due to floating point instability. // Do this before taking the square root. pointDistanceSquared = (pointDistanceSquared >= 0.0f) ? pointDistanceSquared : 0.0f; - Float pointDistance = MathUtils.Sqrt(pointDistanceSquared); + float pointDistance = MathUtils.Sqrt(pointDistanceSquared); // bestCluster is the best cluster from 0 to cPrevIdx-1 and bestWeight is the corresponding weight. // So, this loop only needs to process clusters from cPrevIdx. @@ -621,7 +675,7 @@ private static void FindBestCluster(ref VBuffer point, int pointRowIndex, #endif continue; } - Float weight = -2 * VectorUtils.DotProduct(ref point, ref clusters[j]) + clustersL2s[j]; + float weight = -2 * VectorUtils.DotProduct(ref point, ref clusters[j]) + clustersL2s[j]; if (bestWeight >= weight) { bestWeight = weight; @@ -657,14 +711,14 @@ private static void ComputeAccelerationMemoryRequirement(long accelMemBudgetMb, out long maxInstancesToAccelerate, out long clusterBytes) { // Compute the memory requirement for _clusterDistances. - clusterBytes = sizeof(Float) + clusterBytes = sizeof(float) * numSamplesPerRound // for each newly added cluster * numRounds * numSamplesPerRound; // for older cluster // Second, figure out how many instances can be accelerated. int bytesPerInstance = sizeof(int) // for bestCluster - + sizeof(Float) // for bestWeight + + sizeof(float) // for bestWeight + (isParallel ? sizeof(int) + 16 : 0); // for parallel rowCursor index lookup HashArray storage (16 bytes for RowId, 4 bytes for internal 'next' index) maxInstancesToAccelerate = Math.Max(0, (accelMemBudgetMb * 1024 * 1024 - clusterBytes) / bytesPerInstance); @@ -683,7 +737,7 @@ private static void ComputeAccelerationMemoryRequirement(long accelMemBudgetMb, /// We store weight in most cases to avoid unnecessary computation of norm(x). /// public static void Initialize(IHost host, int numThreads, IChannel ch, FeatureFloatVectorCursor.Factory cursorFactory, - int k, int dimensionality, VBuffer[] centroids, long accelMemBudgetMb, + int k, int dimensionality, VBuffer[] centroids, long accelMemBudgetMb, out long missingFeatureCount, out long totalTrainingInstances) { Contracts.CheckValue(host, nameof(host)); @@ -728,9 +782,9 @@ public static void Initialize(IHost host, int numThreads, IChannel ch, FeatureFl // The final chosen points, to be approximately clustered to determine starting // centroids. - VBuffer[] clusters = new VBuffer[totalSamples]; + VBuffer[] clusters = new VBuffer[totalSamples]; // L2s, kept for distance trick. - Float[] clustersL2s = new Float[totalSamples]; + float[] clustersL2s = new float[totalSamples]; int clusterCount = 0; int clusterPrevCount = -1; @@ -741,7 +795,7 @@ public static void Initialize(IHost host, int numThreads, IChannel ch, FeatureFl // sample. Heap[] buffer = null; var rowStats = KMeansUtils.ParallelWeightedReservoirSample(host, numThreads, 1, cursorFactory, - (ref VBuffer point, int pointIndex) => (Float)1.0, (FeatureFloatVectorCursor cur) => -1, + (ref VBuffer point, int pointIndex) => (float)1.0, (FeatureFloatVectorCursor cur) => -1, ref clusters, ref buffer); totalTrainingInstances = rowStats.TotalTrainingInstances; missingFeatureCount = rowStats.MissingFeatureCount; @@ -768,11 +822,11 @@ public static void Initialize(IHost host, int numThreads, IChannel ch, FeatureFl // to their distance from their best cluster. For each round, this gives us // a new set of 'numSamplesPerRound' instances which are very likely to be as // far from our current total running set of instances as possible. - VBuffer[] roundSamples = new VBuffer[numSamplesPerRound]; + VBuffer[] roundSamples = new VBuffer[numSamplesPerRound]; - KMeansUtils.WeightFunc weightFn = (ref VBuffer point, int pointRowIndex) => + KMeansUtils.WeightFunc weightFn = (ref VBuffer point, int pointRowIndex) => { - Float distanceSquared; + float distanceSquared; int discardBestCluster; FindBestCluster(ref point, pointRowIndex, initializationState, clusterCount, clusterPrevCount, clusters, clustersL2s, true, true, out distanceSquared, out discardBestCluster); @@ -809,15 +863,15 @@ public static void Initialize(IHost host, int numThreads, IChannel ch, FeatureFl // Finally, we do one last pass through the dataset, finding for // each instance the closest chosen cluster instance and summing these into buckets to be used // to weight each one of our candidate chosen clusters. - Float[][] weightBuffer = null; - Float[] totalWeights = null; - KMeansUtils.ParallelMapReduce( + float[][] weightBuffer = null; + float[] totalWeights = null; + KMeansUtils.ParallelMapReduce( numThreads, host, cursorFactory, initializationState.RowIndexGetter, - (ref Float[] weights) => weights = new Float[totalSamples], - (ref VBuffer point, int pointRowIndex, Float[] weights, IRandom rand) => + (ref float[] weights) => weights = new float[totalSamples], + (ref VBuffer point, int pointRowIndex, float[] weights, IRandom rand) => { int bestCluster; - Float discardBestWeight; + float discardBestWeight; FindBestCluster(ref point, pointRowIndex, initializationState, clusterCount, clusterPrevCount, clusters, clustersL2s, false, false, out discardBestWeight, out bestCluster); #if DEBUG @@ -826,25 +880,25 @@ public static void Initialize(IHost host, int numThreads, IChannel ch, FeatureFl #endif weights[bestCluster]++; }, - (Float[][] workStateWeights, IRandom rand, ref Float[] weights) => + (float[][] workStateWeights, IRandom rand, ref float[] weights) => { - weights = new Float[totalSamples]; + weights = new float[totalSamples]; for (int i = 0; i < workStateWeights.Length; i++) CpuMathUtils.Add(workStateWeights[i], weights, totalSamples); }, ref weightBuffer, ref totalWeights); #if DEBUG // This is running the original code to make sure that the new code matches the semantic of the original code. - Float[] debugTotalWeights = null; + float[] debugTotalWeights = null; - Float[][] debugWeightBuffer = null; - KMeansUtils.ParallelMapReduce( + float[][] debugWeightBuffer = null; + KMeansUtils.ParallelMapReduce( numThreads, host, cursorFactory, (FeatureFloatVectorCursor cur) => -1, - (ref Float[] weights) => weights = new Float[totalSamples], - (ref VBuffer point, int discard, Float[] weights, IRandom rand) => weights[KMeansUtils.FindBestCluster(ref point, clusters, clustersL2s)]++, - (Float[][] workStateWeights, IRandom rand, ref Float[] weights) => + (ref float[] weights) => weights = new float[totalSamples], + (ref VBuffer point, int discard, float[] weights, IRandom rand) => weights[KMeansUtils.FindBestCluster(ref point, clusters, clustersL2s)]++, + (float[][] workStateWeights, IRandom rand, ref float[] weights) => { - weights = new Float[totalSamples]; + weights = new float[totalSamples]; for (int i = 0; i < workStateWeights.Length; i++) for (int j = 0; j < workStateWeights[i].Length; j++) weights[j] += workStateWeights[i][j]; @@ -889,15 +943,15 @@ internal static class KMeansRandomInit /// public static void Initialize( IHost host, int numThreads, IChannel ch, FeatureFloatVectorCursor.Factory cursorFactory, - int k, VBuffer[] centroids, + int k, VBuffer[] centroids, out long missingFeatureCount, out long totalTrainingInstances) { using (var pCh = host.StartProgressChannel("KMeansRandomInitialize")) { Heap[] buffer = null; - VBuffer[] outCentroids = null; + VBuffer[] outCentroids = null; var rowStats = KMeansUtils.ParallelWeightedReservoirSample(host, numThreads, k, cursorFactory, - (ref VBuffer point, int pointRowIndex) => 1f, (FeatureFloatVectorCursor cur) => -1, + (ref VBuffer point, int pointRowIndex) => 1f, (FeatureFloatVectorCursor cur) => -1, ref outCentroids, ref buffer); missingFeatureCount = rowStats.MissingFeatureCount; totalTrainingInstances = rowStats.TotalTrainingInstances; @@ -916,15 +970,15 @@ private abstract class WorkChunkStateBase protected readonly long MaxInstancesToAccelerate; // Standard KMeans per-cluster data. - protected readonly VBuffer[] Centroids; + protected readonly VBuffer[] Centroids; protected readonly long[] ClusterSizes; // YinYang per-cluster data. // cached sum of the first maxInstancesToAccelerate instances assigned to cluster i for 0 <= i < _k - protected readonly VBuffer[] CachedSum; + protected readonly VBuffer[] CachedSum; #if DEBUG - public VBuffer[] CachedSumDebug { get { return CachedSum; } } + public VBuffer[] CachedSumDebug { get { return CachedSum; } } #endif protected double PreviousAverageScore; @@ -955,17 +1009,17 @@ public static void Initialize( protected WorkChunkStateBase(long maxInstancesToAccelerate, int k, int dimensionality) { - Centroids = new VBuffer[k]; + Centroids = new VBuffer[k]; for (int j = 0; j < k; j++) - Centroids[j] = VBufferUtils.CreateDense(dimensionality); + Centroids[j] = VBufferUtils.CreateDense(dimensionality); ClusterSizes = new long[k]; if (maxInstancesToAccelerate > 0) { - CachedSum = new VBuffer[k]; + CachedSum = new VBuffer[k]; for (int j = 0; j < k; j++) - CachedSum[j] = VBufferUtils.CreateDense(dimensionality); + CachedSum[j] = VBufferUtils.CreateDense(dimensionality); } K = k; @@ -1008,7 +1062,7 @@ public void KeepYinYangAssignment(int bestCluster) GloballyFiltered++; } - public void UpdateClusterAssignment(bool firstIteration, ref VBuffer features, int cluster, int previousCluster, Float distance) + public void UpdateClusterAssignment(bool firstIteration, ref VBuffer features, int cluster, int previousCluster, float distance) { if (firstIteration) { @@ -1029,13 +1083,13 @@ public void UpdateClusterAssignment(bool firstIteration, ref VBuffer feat UpdateClusterAssignmentMetrics(cluster, distance); } - public void UpdateClusterAssignment(ref VBuffer features, int cluster, Float distance) + public void UpdateClusterAssignment(ref VBuffer features, int cluster, float distance) { VectorUtils.Add(ref features, ref Centroids[cluster]); UpdateClusterAssignmentMetrics(cluster, distance); } - private void UpdateClusterAssignmentMetrics(int cluster, Float distance) + private void UpdateClusterAssignmentMetrics(int cluster, float distance) { _n++; ClusterSizes[cluster]++; @@ -1099,7 +1153,7 @@ public ReducedWorkChunkState(long maxInstancesToAccelerate, int k, int dimension /// of cluster assignment. It is assumed that centroids will contain the previous results /// of this call. /// - public void UpdateClusters(VBuffer[] centroids, Float[] centroidL2s, Float[] deltas, ref Float deltaMax) + public void UpdateClusters(VBuffer[] centroids, float[] centroidL2s, float[] deltas, ref float deltaMax) { bool isAccelerated = MaxInstancesToAccelerate > 0; deltaMax = 0; @@ -1110,11 +1164,11 @@ public void UpdateClusters(VBuffer[] centroids, Float[] centroidL2s, Floa VectorUtils.Add(ref CachedSum[i], ref Centroids[i]); if (ClusterSizes[i] > 1) - VectorUtils.ScaleBy(ref Centroids[i], (Float)(1.0 / ClusterSizes[i])); + VectorUtils.ScaleBy(ref Centroids[i], (float)(1.0 / ClusterSizes[i])); if (isAccelerated) { - Float clusterDelta = MathUtils.Sqrt(VectorUtils.L2DistSquared(ref Centroids[i], ref centroids[i])); + float clusterDelta = MathUtils.Sqrt(VectorUtils.L2DistSquared(ref Centroids[i], ref centroids[i])); deltas[i] = clusterDelta; if (deltaMax < clusterDelta) @@ -1147,9 +1201,9 @@ private sealed class SharedState // YinYang data. // the distance between the old and the new center for each cluster - public readonly Float[] Delta; + public readonly float[] Delta; // max value of delta[i] for 0 <= i < _k - public Float DeltaMax; + public float DeltaMax; // Per instance structures @@ -1162,9 +1216,9 @@ public int GetBestCluster(int idx) private readonly int[] _bestCluster; // upper bound on the distance of an instance to its bestCluster - private readonly Float[] _upperBound; + private readonly float[] _upperBound; // lower bound on the distance of an instance to every cluster other than its bestCluster - private readonly Float[] _lowerBound; + private readonly float[] _lowerBound; public SharedState(FeatureFloatVectorCursor.Factory factory, IChannel ch, long baseMaxInstancesToAccelerate, int k, bool isParallel, long totalTrainingInstances) @@ -1181,11 +1235,11 @@ public SharedState(FeatureFloatVectorCursor.Factory factory, IChannel ch, long b if (MaxInstancesToAccelerate > 0) { // allocate data structures - Delta = new Float[k]; + Delta = new float[k]; _bestCluster = new int[MaxInstancesToAccelerate]; - _upperBound = new Float[MaxInstancesToAccelerate]; - _lowerBound = new Float[MaxInstancesToAccelerate]; + _upperBound = new float[MaxInstancesToAccelerate]; + _lowerBound = new float[MaxInstancesToAccelerate]; } } @@ -1194,14 +1248,14 @@ public SharedState(FeatureFloatVectorCursor.Factory factory, IChannel ch, long b /// to its closer and second closed cluster, as well as the identity of the new /// closest cluster. This method returns the last known closest cluster. /// - public int SetYinYangCluster(int n, ref VBuffer features, Float minDistance, int minCluster, Float secMinDistance) + public int SetYinYangCluster(int n, ref VBuffer features, float minDistance, int minCluster, float secMinDistance) { if (n == -1) return -1; // update upper and lower bound // updates have to be true distances to use triangular inequality - Float instanceNormSquared = VectorUtils.NormSquared(features); + float instanceNormSquared = VectorUtils.NormSquared(features); _upperBound[n] = MathUtils.Sqrt(instanceNormSquared + minDistance); _lowerBound[n] = MathUtils.Sqrt(instanceNormSquared + secMinDistance); int previousCluster = _bestCluster[n]; @@ -1230,16 +1284,16 @@ public bool IsYinYangGloballyBound(int n) } #if DEBUG - public void AssertValidYinYangBounds(int n, ref VBuffer features, VBuffer[] centroids) + public void AssertValidYinYangBounds(int n, ref VBuffer features, VBuffer[] centroids) { // Assert that the global filter is indeed doing the right thing - Float bestDistance = MathUtils.Sqrt(VectorUtils.L2DistSquared(ref features, ref centroids[_bestCluster[n]])); + float bestDistance = MathUtils.Sqrt(VectorUtils.L2DistSquared(ref features, ref centroids[_bestCluster[n]])); Contracts.Assert(KMeansLloydsYinYangTrain.AlmostLeq(bestDistance, _upperBound[n])); for (int j = 0; j < centroids.Length; j++) { if (j == _bestCluster[n]) continue; - Float distance = MathUtils.Sqrt(VectorUtils.L2DistSquared(ref features, ref centroids[j])); + float distance = MathUtils.Sqrt(VectorUtils.L2DistSquared(ref features, ref centroids[j])); Contracts.Assert(AlmostLeq(_lowerBound[n], distance)); } @@ -1249,14 +1303,14 @@ public void AssertValidYinYangBounds(int n, ref VBuffer features, VBuffer public static void Train(IHost host, int numThreads, IChannel ch, FeatureFloatVectorCursor.Factory cursorFactory, long totalTrainingInstances, int k, int dimensionality, int maxIterations, - long accelMemBudgetInMb, Float convergenceThreshold, VBuffer[] centroids) + long accelMemBudgetInMb, float convergenceThreshold, VBuffer[] centroids) { SharedState state; WorkChunkState[] workState; ReducedWorkChunkState reducedState; Initialize(ch, cursorFactory, totalTrainingInstances, numThreads, k, dimensionality, accelMemBudgetInMb, out state, out workState, out reducedState); - Float[] centroidL2s = new Float[k]; + float[] centroidL2s = new float[k]; for (int i = 0; i < k; i++) centroidL2s[i] = VectorUtils.NormSquared(centroids[i]); @@ -1309,9 +1363,9 @@ public static void Train(IHost host, int numThreads, IChannel ch, FeatureFloatVe if (state.IsAccelerated) { // Assert that cachedSum[i] is equal to the sum of the first maxInstancesToAccelerate instances assigned to cluster i - var cachedSumCopy = new VBuffer[k]; + var cachedSumCopy = new VBuffer[k]; for (int i = 0; i < k; i++) - cachedSumCopy[i] = VBufferUtils.CreateDense(dimensionality); + cachedSumCopy[i] = VBufferUtils.CreateDense(dimensionality); using (var cursor = cursorFactory.Create()) { @@ -1361,13 +1415,13 @@ private static void Initialize( // These data structures are allocated only as allowed by the _accelMemBudgetMb parameter // if _accelMemBudgetMb is zero, then the algorithm below reduces to the original KMeans++ implementation int bytesPerCluster = - sizeof(Float) + // for delta - sizeof(Float) * dimensionality * (neededPerThreadWorkStates + 1); // for cachedSum + sizeof(float) + // for delta + sizeof(float) * dimensionality * (neededPerThreadWorkStates + 1); // for cachedSum int bytesPerInstance = sizeof(int) + // for bestCluster - sizeof(Float) + // for upperBound - sizeof(Float) + // for lowerBound + sizeof(float) + // for upperBound + sizeof(float) + // for lowerBound (numThreads > 1 ? sizeof(int) + 16 : 0); // for parallel rowCursor index lookup HashArray storage (16 bytes for RowId, 4 bytes for internal 'next' index) long maxInstancesToAccelerate = Math.Max(0, (accelMemBudgetMb * 1024 * 1024 - bytesPerCluster * k) / bytesPerInstance); @@ -1382,7 +1436,7 @@ private static void Initialize( /// this chunk will be one of _numThreads chunks and the RowCursor will be part of a RowCursorSet. In the /// unthreaded version, this chunk will be the final chunk and hold state for the entire data set. /// - private static void ProcessChunk(FeatureFloatVectorCursor cursor, SharedState state, WorkChunkStateBase chunkState, int k, VBuffer[] centroids, Float[] centroidL2s) + private static void ProcessChunk(FeatureFloatVectorCursor cursor, SharedState state, WorkChunkStateBase chunkState, int k, VBuffer[] centroids, float[] centroidL2s) { while (cursor.MoveNext()) { @@ -1403,8 +1457,8 @@ private static void ProcessChunk(FeatureFloatVectorCursor cursor, SharedState st } } - Float minDistance; - Float secMinDistance; + float minDistance; + float secMinDistance; int cluster; int secCluster; KMeansUtils.FindBestCluster(ref cursor.Features, centroids, centroidL2s, k, false, out minDistance, out cluster, out secMinDistance, out secCluster); @@ -1422,7 +1476,7 @@ private static void ProcessChunk(FeatureFloatVectorCursor cursor, SharedState st #if DEBUG private const Double FloatingPointErrorThreshold = 0.1F; - private static bool AlmostEq(Float af, Float bf) + private static bool AlmostEq(float af, float bf) { Double a = (Double)af; Double b = (Double)bf; @@ -1446,7 +1500,7 @@ private static bool AlmostEq(Float af, Float bf) return false; } - private static bool AlmostLeq(Float a, Float b) + private static bool AlmostLeq(float a, float b) { if (AlmostEq(a, b)) return true; @@ -1461,7 +1515,7 @@ internal static class KMeansUtils public struct WeightedPoint { public double Weight; - public VBuffer Point; + public VBuffer Point; } public struct RowStats @@ -1470,7 +1524,7 @@ public struct RowStats public long TotalTrainingInstances; } - public delegate Float WeightFunc(ref VBuffer point, int pointRowIndex); + public delegate float WeightFunc(ref VBuffer point, int pointRowIndex); /// /// Performs a multithreaded version of weighted reservior sampling, returning @@ -1490,7 +1544,7 @@ public static RowStats ParallelWeightedReservoirSample( int numSamples, FeatureFloatVectorCursor.Factory factory, WeightFunc weightFn, RowIndexGetter rowIndexGetter, - ref VBuffer[] dst, + ref VBuffer[] dst, ref Heap[] buffer) { host.AssertValue(host); @@ -1509,22 +1563,22 @@ public static RowStats ParallelWeightedReservoirSample( else heap.Clear(); }, - (ref VBuffer point, int pointRowIndex, Heap heap, IRandom rand) => + (ref VBuffer point, int pointRowIndex, Heap heap, IRandom rand) => { // We use distance as a proxy for 'is the same point'. By excluding // all points that lie within a very small distance of our current set of // centroids we force the algorithm to explore more broadly and avoid creating a // set of centroids containing the same, or very close to the same, point // more than once. - Float sameClusterEpsilon = (Float)1e-15; + float sameClusterEpsilon = (float)1e-15; - Float weight = weightFn(ref point, pointRowIndex); + float weight = weightFn(ref point, pointRowIndex); // If numeric instability has forced it to zero, then we bound it to epsilon to // keep the key valid and avoid NaN, (although the math does tend to work out regardless: // 1 / 0 => Inf, base ^ Inf => 0, when |base| < 1) if (weight == 0) - weight = Float.Epsilon; + weight = float.Epsilon; if (weight <= sameClusterEpsilon) return; @@ -1584,7 +1638,7 @@ public static RowStats ParallelWeightedReservoirSample( public delegate void InitAction(ref TPartitionState val); public delegate int RowIndexGetter(FeatureFloatVectorCursor cur); - public delegate void MapAction(ref VBuffer point, int rowIndex, TPartitionState state, IRandom rand); + public delegate void MapAction(ref VBuffer point, int rowIndex, TPartitionState state, IRandom rand); public delegate void ReduceAction(TPartitionState[] intermediates, IRandom rand, ref TGlobalState result); /// @@ -1648,19 +1702,19 @@ public static RowStats ParallelMapReduce( }; } - public static int FindBestCluster(ref VBuffer features, VBuffer[] centroids, Float[] centroidL2s) + public static int FindBestCluster(ref VBuffer features, VBuffer[] centroids, float[] centroidL2s) { - Float discard1; - Float discard2; + float discard1; + float discard2; int discard3; int cluster; FindBestCluster(ref features, centroids, centroidL2s, centroids.Length, false, out discard1, out cluster, out discard2, out discard3); return cluster; } - public static int FindBestCluster(ref VBuffer features, VBuffer[] centroids, Float[] centroidL2s, int centroidCount, bool realWeight, out Float minDistance) + public static int FindBestCluster(ref VBuffer features, VBuffer[] centroids, float[] centroidL2s, int centroidCount, bool realWeight, out float minDistance) { - Float discard1; + float discard1; int discard2; int cluster; FindBestCluster(ref features, centroids, centroidL2s, centroidCount, realWeight, out minDistance, out cluster, out discard1, out discard2); @@ -1682,15 +1736,15 @@ public static int FindBestCluster(ref VBuffer features, VBuffer[] /// The second nearest distance, or PosInf if only contains a single point. /// The index of the second nearest centroid, or -1 if only contains a single point. public static void FindBestCluster( - ref VBuffer features, - VBuffer[] centroids, Float[] centroidL2s, int centroidCount, bool needRealDistance, - out Float minDistance, out int cluster, out Float secMinDistance, out int secCluster) + ref VBuffer features, + VBuffer[] centroids, float[] centroidL2s, int centroidCount, bool needRealDistance, + out float minDistance, out int cluster, out float secMinDistance, out int secCluster) { Contracts.Assert(centroids.Length >= centroidCount && centroidL2s.Length >= centroidCount && centroidCount > 0); Contracts.Assert(features.Length == centroids[0].Length); - minDistance = Float.PositiveInfinity; - secMinDistance = Float.PositiveInfinity; + minDistance = float.PositiveInfinity; + secMinDistance = float.PositiveInfinity; cluster = 0; // currently assigned cluster to the instance secCluster = -1; @@ -1698,7 +1752,7 @@ public static void FindBestCluster( { // this is not a real distance, since we don't add L2 norm of the instance // This won't affect minimum calculations, and total score will just be lowered by sum(L2 norms) - Float distance = -2 * VectorUtils.DotProduct(ref features, ref centroids[j]) + centroidL2s[j]; + float distance = -2 * VectorUtils.DotProduct(ref features, ref centroids[j]) + centroidL2s[j]; if (distance <= minDistance) { @@ -1721,7 +1775,7 @@ public static void FindBestCluster( if (needRealDistance) { - Float l2 = VectorUtils.NormSquared(features); + float l2 = VectorUtils.NormSquared(features); minDistance += l2; if (secCluster != -1) secMinDistance += l2; @@ -1731,7 +1785,7 @@ public static void FindBestCluster( /// /// Checks that all coordinates of all centroids are finite, and throws otherwise /// - public static void VerifyModelConsistency(VBuffer[] centroids) + public static void VerifyModelConsistency(VBuffer[] centroids) { foreach (var centroid in centroids) Contracts.Check(centroid.Items().Select(x => x.Value).All(FloatUtils.IsFinite), "Model training failed: non-finite coordinates are generated"); diff --git a/src/Microsoft.ML.KMeansClustering/KMeansStatic.cs b/src/Microsoft.ML.KMeansClustering/KMeansStatic.cs new file mode 100644 index 0000000000..2a64ef8d31 --- /dev/null +++ b/src/Microsoft.ML.KMeansClustering/KMeansStatic.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.KMeans; +using Microsoft.ML.StaticPipe; +using Microsoft.ML.StaticPipe.Runtime; +using System; + +namespace Microsoft.ML.Trainers +{ + /// + /// The trainer context extensions for the . + /// + public static class ClusteringTrainers + { + /// + /// KMeans extension method. + /// + /// The regression context trainer object. + /// The features, or independent variables. + /// The optional example weights. + /// The number of clusters to use for KMeans. + /// Algorithm advanced settings. + /// A delegate that is called every time the + /// method is called on the + /// instance created out of this. This delegate will receive + /// the linear model that was trained. Note that this action cannot change the result in any way; it is only a way for the caller to + /// be informed about what was learnt. + /// The predicted output. + public static (Vector score, Key predictedLabel) KMeans(this ClusteringContext.ClusteringTrainers ctx, + Vector features, Scalar weights = null, + int clustersCount = KMeansPlusPlusTrainer.Defaults.K, + Action advancedSettings = null, + Action onFit = null) + { + var rec = new TrainerEstimatorReconciler.Clustering( + (env, featuresName, weightsName) => + { + var trainer = new KMeansPlusPlusTrainer(env, featuresName, clustersCount, weightsName, advancedSettings); + + if (onFit != null) + return trainer.WithOnFitDelegate(trans => onFit(trans.Model)); + else + return trainer; + }, features, weights); + + return rec.Output; + } + } +} diff --git a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs b/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs index de270a373a..c60a8cbb6b 100644 --- a/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs +++ b/test/Microsoft.ML.Benchmarks/KMeansAndLogisticRegressionBench.cs @@ -52,9 +52,9 @@ public ParameterMixingCalibratedPredictor TrainKMeansAndLR() trans = TrainAndScoreTransform.Create(env, new TrainAndScoreTransform.Arguments { Trainer = ComponentFactoryUtils.CreateFromFunction(host => - new KMeansPlusPlusTrainer(host, new KMeansPlusPlusTrainer.Arguments() + new KMeansPlusPlusTrainer(host, "Features", advancedSettings: s=> { - K = 100 + s.K = 100; })), FeatureColumn = "Features" }, trans); diff --git a/test/Microsoft.ML.StaticPipelineTesting/Training.cs b/test/Microsoft.ML.StaticPipelineTesting/Training.cs index bfd9bfa456..8ccea87539 100644 --- a/test/Microsoft.ML.StaticPipelineTesting/Training.cs +++ b/test/Microsoft.ML.StaticPipelineTesting/Training.cs @@ -8,6 +8,7 @@ using Microsoft.ML.Runtime.FastTree; using Microsoft.ML.Runtime.Internal.Calibration; using Microsoft.ML.Runtime.Internal.Internallearn; +using Microsoft.ML.Runtime.KMeans; using Microsoft.ML.Runtime.Learners; using Microsoft.ML.Runtime.LightGBM; using Microsoft.ML.Runtime.RunTests; @@ -451,5 +452,60 @@ public void LightGbmRegression() Assert.Equal(metrics.Rms * metrics.Rms, metrics.L2, 5); Assert.InRange(metrics.LossFn, 0, double.PositiveInfinity); } + + [Fact] + public void KMeans() + { + var env = new ConsoleEnvironment(seed: 0); + var dataPath = GetDataPath(TestDatasets.iris.trainFilename); + var dataSource = new MultiFileSource(dataPath); + + var ctx = new ClusteringContext(env); + var reader = TextLoader.CreateReader(env, + c => (label: c.LoadText(0), features: c.LoadFloat(1, 4))); + + KMeansPredictor pred = null; + + var est = reader.MakeNewEstimator() + .Append(r => (label: r.label.ToKey(), r.features)) + .Append(r => (r.label, r.features, preds: ctx.Trainers.KMeans(r.features, clustersCount: 3, onFit: p => pred = p))); + + var pipe = reader.Append(est); + + Assert.Null(pred); + var model = pipe.Fit(dataSource); + Assert.NotNull(pred); + + VBuffer[] centroids = default; + int k; + pred.GetClusterCentroids(ref centroids, out k); + + Assert.True(k == 3); + + var data = model.Read(dataSource); + + var metrics = ctx.Evaluate(data, r => r.preds.score, r => r.label, r => r.features); + Assert.NotNull(metrics); + + Assert.InRange(metrics.AvgMinScore, 0.5262, 0.5264); + Assert.InRange(metrics.Nmi, 0.73, 0.77); + Assert.InRange(metrics.Dbi, 0.662, 0.667); + + metrics = ctx.Evaluate(data, r => r.preds.score, label: r => r.label); + Assert.NotNull(metrics); + + Assert.InRange(metrics.AvgMinScore, 0.5262, 0.5264); + Assert.True(metrics.Dbi == 0.0); + + metrics = ctx.Evaluate(data, r => r.preds.score, features: r => r.features); + Assert.True(double.IsNaN(metrics.Nmi)); + + metrics = ctx.Evaluate(data, r => r.preds.score); + Assert.NotNull(metrics); + Assert.InRange(metrics.AvgMinScore, 0.5262, 0.5264); + Assert.True(double.IsNaN(metrics.Nmi)); + Assert.True(metrics.Dbi == 0.0); + + } } } diff --git a/test/Microsoft.ML.TestFramework/Datasets.cs b/test/Microsoft.ML.TestFramework/Datasets.cs index b9d0cad9a5..331043769f 100644 --- a/test/Microsoft.ML.TestFramework/Datasets.cs +++ b/test/Microsoft.ML.TestFramework/Datasets.cs @@ -600,15 +600,15 @@ public static class TestDatasets public static TestDataset mnist28 = new TestDataset() { name = "mnist28", - trainFilename = @"..\MNIST\Train-28x28.txt", - testFilename = @"..\MNIST\Test-28x28.txt" + trainFilename = @"Train-28x28.txt", + testFilename = @"Test-28x28.txt" }; public static TestDataset mnistTiny28 = new TestDataset() { name = "mnistTiny28", - trainFilename = @"..\MNIST\Train-Tiny-28x28.txt", - testFilename = @"..\MNIST\Test-Tiny-28x28.txt" + trainFilename = @"Train-Tiny-28x28.txt", + testFilename = @"Test-Tiny-28x28.txt" }; public static TestDataset sampleBingRegression = new TestDataset() diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs index dcde5e65aa..49b74c7fa2 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TrainerEstimators.cs @@ -4,6 +4,7 @@ using Microsoft.ML.Core.Data; using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.KMeans; using Microsoft.ML.Runtime.Learners; using Microsoft.ML.Runtime.PCA; using Microsoft.ML.Runtime.RunTests; @@ -45,6 +46,37 @@ public void PCATrainerEstimator() Done(); } + /// + /// KMeans TrainerEstimator test + /// + [Fact] + public void KMeansEstimator() + { + string featureColumn = "NumericFeatures"; + string weights = "Weights"; + + var reader = new TextLoader(Env, new TextLoader.Arguments + { + HasHeader = true, + Separator = "\t", + Column = new[] + { + new TextLoader.Column(featureColumn, DataKind.R4, new [] { new TextLoader.Range(1, 784) }), + new TextLoader.Column(weights, DataKind.R4, 0) + } + }); + var data = reader.Read(new MultiFileSource(GetDataPath(TestDatasets.mnistTiny28.trainFilename))); + + + // Pipeline. + var pipeline = new KMeansPlusPlusTrainer(Env, featureColumn, weightColumn: weights, + advancedSettings: s => { s.InitAlgorithm = KMeansPlusPlusTrainer.InitAlgorithm.KMeansParallel; }); + + TestEstimatorCore(pipeline, data); + + Done(); + } + private (IEstimator, IDataView) GetBinaryClassificationPipeline() { var data = new TextLoader(Env,