Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/Microsoft.ML.Core/Utilities/Contracts.cs
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,14 @@ public static T CheckRef<T>(this IExceptionContext ctx, T val, string paramName)
return val;
}

public static void CheckValue<T>(T val, string paramName) where T : class
public static T CheckRef<T>(this IExceptionContext ctx, T val, string paramName, string msg) where T : class
{
if (object.ReferenceEquals(val, null))
throw ExceptValue(ctx, paramName, msg);
return val;
}

public static void CheckValue<T>(T val, string paramName) where T : class
{
if (object.ReferenceEquals(val, null))
throw ExceptValue(paramName);
Expand Down
13 changes: 13 additions & 0 deletions src/Microsoft.ML.Data/Data/RowCursorUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,19 @@ public static IRow CloneRow(IRow row)
Utils.BuildArray(row.Schema.ColumnCount, c => RowColumnUtils.GetColumn(row, c)));
}

/// <summary>
/// Fetches the value of the column by name, in the given row.
/// Used by the evaluators to retrieve the metrics from the results IDataView.
/// </summary>
public static T Fetch<T>(IExceptionContext ectx, IRow row, string name)
{
if (!row.Schema.TryGetColumnIndex(name, out int col))
throw ectx.Except($"Could not find column '{name}'");
T val = default;
row.GetGetter<T>(col)(ref val);
return val;
}

/// <summary>
/// Given a row, returns a one-row data view. This is useful for cases where you have a row, and you
/// wish to use some facility normally only exposed to dataviews. (E.g., you have an <see cref="IRow"/>
Expand Down
80 changes: 80 additions & 0 deletions src/Microsoft.ML.Data/Evaluators/ClusteringEvaluator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,46 @@ public ClusteringEvaluator(IHostEnvironment env, Arguments args)
_calculateDbi = args.CalculateDbi;
}

/// <summary>
/// Evaluates scored clustering data.
/// </summary>
/// <param name="data">The scored data.</param>
/// <param name="score">The name of the score column in <paramref name="data"/>.</param>
/// <param name="label">The name of the optional label column in <paramref name="data"/>.</param>
/// <param name="features">The name of the optional feature column in <paramref name="data"/>.</param>
/// <returns>The evaluation results.</returns>
public Result Evaluate(IDataView data, string score, string label = null, string features = null)
{
Host.CheckValue(data, nameof(data));
Host.CheckNonEmpty(score, nameof(score));

var roles = new List<KeyValuePair<RoleMappedSchema.ColumnRole, string>>();
roles.Add(RoleMappedSchema.CreatePair(MetadataUtils.Const.ScoreValueKind.Score, score));

if (label != null)
roles.Add(RoleMappedSchema.ColumnRole.Label.Bind(label));

if (features != null)
roles.Add(RoleMappedSchema.ColumnRole.Feature.Bind(features));

var rolesMappedData = new RoleMappedData(data, opt: false, roles.ToArray());

var resultDict = Evaluate(rolesMappedData);
Host.Assert(resultDict.ContainsKey(MetricKinds.OverallMetrics));
var overall = resultDict[MetricKinds.OverallMetrics];

Result result;
using (var cursor = overall.GetRowCursor(i => true))
{
var moved = cursor.MoveNext();
Host.Assert(moved);
result = new Result(Host, cursor, _calculateDbi);
moved = cursor.MoveNext();
Host.Assert(!moved);
}
return result;
}

protected override void CheckScoreAndLabelTypes(RoleMappedSchema schema)
{
ColumnType type;
Expand Down Expand Up @@ -517,6 +557,46 @@ private void AssertValid(bool assertGetters)
}
}
}

/// <summary>
/// The metrics generated after evaluating the clustering predictions.
/// </summary>
public sealed class Result
{
/// <summary>
/// Normalized Mutual Information
/// NMI is a measure of the mutual dependence of the variables.
/// <a href="http://en.wikipedia.org/wiki/Mutual_information#Normalized_variants">Normalized variants</a> work on data that already has cluster labels.
/// Its value ranged from 0 to 1, where higher numbers are better.
/// </summary>
public double Nmi { get; }

/// <summary>
/// Average Score. For the K-Means algorithm, the 'score' is the distance from the centroid to the example.
/// The average score is, therefore, a measure of proximity of the examples to cluster centroids.
/// In other words, it's the 'cluster tightness' measure.
/// Note however, that this metric will only decrease if the number of clusters is increased,
/// and in the extreme case (where each distinct example is its own cluster) it will be equal to zero.
/// </summary>
public double AvgMinScore { get; }

/// <summary>
/// <a href="https://en.wikipedia.org/wiki/Davies�Bouldin_index">Davies-Bouldin Index</a>
/// DBI is a measure of the how much scatter is in the cluster and the cluster separation.
/// </summary>
public double Dbi { get; }

internal Result(IExceptionContext ectx, IRow overallResult, bool calculateDbi)
{
double Fetch(string name) => RowCursorUtils.Fetch<double>(ectx, overallResult, name);

Nmi = Fetch(ClusteringEvaluator.Nmi);
AvgMinScore = Fetch(ClusteringEvaluator.AvgMinScore);

if(calculateDbi)
Dbi = Fetch(ClusteringEvaluator.Dbi);
}
}
}

public sealed class ClusteringPerInstanceEvaluator : PerInstanceEvaluatorBase
Expand Down
35 changes: 34 additions & 1 deletion src/Microsoft.ML.Data/Evaluators/EvaluatorStaticExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,39 @@ public static BinaryClassifierEvaluator.Result Evaluate<T>(
return eval.Evaluate(data.AsDynamic, labelName, scoreName, predName);
}

/// <summary>
/// Evaluates scored clustering prediction data.
/// </summary>
/// <typeparam name="T">The shape type for the input data.</typeparam>
/// <param name="ctx">The clustering context.</param>
/// <param name="data">The data to evaluate.</param>
/// <param name="score">The index delegate for the predicted score column.</param>
/// <param name="label">The optional index delegate for the label column.</param>
/// <param name="features">The optional index delegate for the features column.</param>
/// <returns>The evaluation metrics.</returns>
public static ClusteringEvaluator.Result Evaluate<T>(
this ClusteringContext ctx,
DataView<T> data,
Func<T, Vector<float>> score,
Func<T, Key<uint>> label = null,
Func<T, Vector<float>> features = null)
{
Contracts.CheckValue(data, nameof(data));
var env = StaticPipeUtils.GetEnvironment(data);
Contracts.AssertValue(env);
env.CheckValue(score, nameof(score));

var indexer = StaticPipeUtils.GetIndexer(data);
string scoreName = indexer.Get(score(indexer.Indices));

string labelName = (label != null)? indexer.Get(label(indexer.Indices)) : null;
string featuresName = (features!= null) ? indexer.Get(features(indexer.Indices)): null;

var args = new ClusteringEvaluator.Arguments() { CalculateDbi = !string.IsNullOrEmpty(featuresName) };

return new ClusteringEvaluator(env, args).Evaluate(data.AsDynamic, scoreName, labelName, featuresName);
}

/// <summary>
/// Evaluates scored multiclass classification data.
/// </summary>
Expand Down Expand Up @@ -136,7 +169,7 @@ private sealed class TrivialRegressionLossFactory : ISupportRegressionLossFactor
}

/// <summary>
/// Evaluates scored multiclass classification data.
/// Evaluates scored regression data.
/// </summary>
/// <typeparam name="T">The shape type for the input data.</typeparam>
/// <param name="ctx">The regression context.</param>
Expand Down
22 changes: 7 additions & 15 deletions src/Microsoft.ML.Data/Evaluators/MulticlassClassifierEvaluator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -570,26 +570,18 @@ public sealed class Result
/// </remarks>
public double[] PerClassLogLoss { get; }

private static T Fetch<T>(IExceptionContext ectx, IRow row, string name)
{
if (!row.Schema.TryGetColumnIndex(name, out int col))
throw ectx.Except($"Could not find column '{name}'");
T val = default;
row.GetGetter<T>(col)(ref val);
return val;
}
internal Result(IExceptionContext ectx, IRow overallResult, int topK)
{
double Fetch(string name) => Fetch<double>(ectx, overallResult, name);
AccuracyMicro = Fetch(MultiClassClassifierEvaluator.AccuracyMicro);
AccuracyMacro = Fetch(MultiClassClassifierEvaluator.AccuracyMacro);
LogLoss = Fetch(MultiClassClassifierEvaluator.LogLoss);
LogLossReduction = Fetch(MultiClassClassifierEvaluator.LogLossReduction);
double FetchDouble(string name) => RowCursorUtils.Fetch<double>(ectx, overallResult, name);
AccuracyMicro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMicro);
AccuracyMacro = FetchDouble(MultiClassClassifierEvaluator.AccuracyMacro);
LogLoss = FetchDouble(MultiClassClassifierEvaluator.LogLoss);
LogLossReduction = FetchDouble(MultiClassClassifierEvaluator.LogLossReduction);
TopK = topK;
if (topK > 0)
TopKAccuracy = Fetch(MultiClassClassifierEvaluator.TopKAccuracy);
TopKAccuracy = FetchDouble(MultiClassClassifierEvaluator.TopKAccuracy);

var perClassLogLoss = Fetch<VBuffer<double>>(ectx, overallResult, MultiClassClassifierEvaluator.PerClassLogLoss);
var perClassLogLoss = RowCursorUtils.Fetch<VBuffer<double>>(ectx, overallResult, MultiClassClassifierEvaluator.PerClassLogLoss);
PerClassLogLoss = new double[perClassLogLoss.Length];
perClassLogLoss.CopyTo(PerClassLogLoss);
}
Expand Down
16 changes: 2 additions & 14 deletions src/Microsoft.ML.Data/Evaluators/RegressionEvaluator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -203,18 +203,9 @@ public sealed class Result
/// </summary>
public double RSquared { get; }

private static T Fetch<T>(IExceptionContext ectx, IRow row, string name)
{
if (!row.Schema.TryGetColumnIndex(name, out int col))
throw ectx.Except($"Could not find column '{name}'");
T val = default;
row.GetGetter<T>(col)(ref val);
return val;
}

internal Result(IExceptionContext ectx, IRow overallResult)
{
double Fetch(string name) => Fetch<double>(ectx, overallResult, name);
double Fetch(string name) => RowCursorUtils.Fetch<double>(ectx, overallResult, name);
L1 = Fetch(RegressionEvaluator.L1);
L2 = Fetch(RegressionEvaluator.L2);
Rms = Fetch(RegressionEvaluator.Rms);
Expand All @@ -230,10 +221,7 @@ internal Result(IExceptionContext ectx, IRow overallResult)
/// <param name="label">The name of the label column.</param>
/// <param name="score">The name of the predicted score column.</param>
/// <returns>The evaluation metrics for these outputs.</returns>
public Result Evaluate(
IDataView data,
string label,
string score)
public Result Evaluate(IDataView data, string label, string score)
{
Host.CheckValue(data, nameof(data));
Host.CheckNonEmpty(label, nameof(label));
Expand Down
63 changes: 63 additions & 0 deletions src/Microsoft.ML.Data/Scorers/PredictionTransformer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
[assembly: LoadableClass(typeof(AnomalyPredictionTransformer<IPredictorProducing<float>>), typeof(AnomalyPredictionTransformer), null, typeof(SignatureLoadModel),
"", AnomalyPredictionTransformer.LoaderSignature)]

[assembly: LoadableClass(typeof(ClusteringPredictionTransformer<IPredictorProducing<VBuffer<float>>>), typeof(ClusteringPredictionTransformer), null, typeof(SignatureLoadModel),
"", ClusteringPredictionTransformer.LoaderSignature)]

namespace Microsoft.ML.Runtime.Data
{

Expand Down Expand Up @@ -508,6 +511,58 @@ private static VersionInfo GetVersionInfo()
}
}

/// <summary>
/// Base class for the <see cref="ISingleFeaturePredictionTransformer{TModel}"/> working on clustering tasks.
/// </summary>
/// <typeparam name="TModel">An implementation of the <see cref="IPredictorProducing{TResult}"/></typeparam>
public sealed class ClusteringPredictionTransformer<TModel> : SingleFeaturePredictionTransformerBase<TModel, ClusteringScorer>
where TModel : class, IPredictorProducing<VBuffer<float>>
{
public ClusteringPredictionTransformer(IHostEnvironment env, TModel model, ISchema inputSchema, string featureColumn,
float threshold = 0f, string thresholdColumn = DefaultColumnNames.Score)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ClusteringPredictionTransformer<TModel>)), model, inputSchema, featureColumn)
{
Host.CheckNonEmpty(thresholdColumn, nameof(thresholdColumn));
var schema = new RoleMappedSchema(inputSchema, null, featureColumn);

var args = new ClusteringScorer.Arguments();
Scorer = new ClusteringScorer(Host, args, new EmptyDataView(Host, inputSchema), BindableMapper.Bind(Host, schema), schema);
}

public ClusteringPredictionTransformer(IHostEnvironment env, ModelLoadContext ctx)
: base(Contracts.CheckRef(env, nameof(env)).Register(nameof(ClusteringPredictionTransformer<TModel>)), ctx)
{
// *** Binary format ***
// <base info>

var schema = new RoleMappedSchema(TrainSchema, null, FeatureColumn);
var args = new ClusteringScorer.Arguments();
Scorer = new ClusteringScorer(Host, args, new EmptyDataView(Host, TrainSchema), BindableMapper.Bind(Host, schema), schema);
}

protected override void SaveCore(ModelSaveContext ctx)
{
Contracts.AssertValue(ctx);
ctx.SetVersionInfo(GetVersionInfo());

// *** Binary format ***
// <base info>
// id of string: scorer threshold column

@Ivanidzo4ka Ivanidzo4ka Sep 21, 2018 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

// id of string: scorer threshold column [](start = 11, length = 41)

doesn't look right #Resolved

base.SaveCore(ctx);
}

private static VersionInfo GetVersionInfo()
{
return new VersionInfo(
modelSignature: "CLUSPRED",
verWrittenCur: 0x00010001, // Initial
verReadableCur: 0x00010001,
verWeCanReadBack: 0x00010001,
loaderSignature: ClusteringPredictionTransformer.LoaderSignature,
loaderAssemblyName: typeof(ClusteringPredictionTransformer<>).Assembly.FullName);
}
}

internal static class BinaryPredictionTransformer
{
public const string LoaderSignature = "BinaryPredXfer";
Expand Down Expand Up @@ -547,4 +602,12 @@ internal static class AnomalyPredictionTransformer
public static AnomalyPredictionTransformer<IPredictorProducing<float>> Create(IHostEnvironment env, ModelLoadContext ctx)
=> new AnomalyPredictionTransformer<IPredictorProducing<float>>(env, ctx);
}

internal static class ClusteringPredictionTransformer
{
public const string LoaderSignature = "ClusteringPredXfer";

public static ClusteringPredictionTransformer<IPredictorProducing<VBuffer<float>>> Create(IHostEnvironment env, ModelLoadContext ctx)
=> new ClusteringPredictionTransformer<IPredictorProducing<VBuffer<float>>>(env, ctx);
}
}
Loading