Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 66 additions & 23 deletions src/Microsoft.ML.HalLearners/OlsLinearRegression.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Float = System.Single;

using System;
using System.Collections.Generic;
using System.IO;
using Microsoft.ML.Core.Data;
using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.HalLearners;
using Microsoft.ML.Runtime.Internal.Internallearn;
Expand Down Expand Up @@ -34,7 +33,7 @@
namespace Microsoft.ML.Runtime.HalLearners

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Microsoft.ML.Runtime.HalLearners [](start = 10, length = 32)

I'm surprised Pete didn't comment about namespace, especially what it's still "Runtime". But I guess we didn't start to change namespaces for learners yet.

{
/// <include file='doc.xml' path='doc/members/member[@name="OLS"]/*' />
public sealed class OlsLinearRegressionTrainer : TrainerBase<OlsLinearRegressionPredictor>
public sealed class OlsLinearRegressionTrainer : TrainerEstimatorBase<RegressionPredictionTransformer<OlsLinearRegressionPredictor>, OlsLinearRegressionPredictor>
{
public sealed class Arguments : LearnerInputBaseWithWeight
{
Expand All @@ -44,7 +43,7 @@ public sealed class Arguments : LearnerInputBaseWithWeight
[Argument(ArgumentType.AtMostOnce, HelpText = "L2 regularization weight", ShortName = "l2", SortOrder = 50)]
[TGUI(SuggestedSweeps = "1e-6,0.1,1")]
[TlcModule.SweepableDiscreteParamAttribute("L2Weight", new object[] { 1e-6f, 0.1f, 1f })]
public Float L2Weight = 1e-6f;
public float L2Weight = 1e-6f;

[Argument(ArgumentType.LastOccurenceWins, HelpText = "Whether to calculate per parameter significance statistics", ShortName = "sig")]
public bool PerParameterSignificance = true;
Expand All @@ -56,7 +55,7 @@ public sealed class Arguments : LearnerInputBaseWithWeight
internal const string Summary = "The ordinary least square regression fits the target function as a linear function of the numerical features "
+ "that minimizes the square loss function.";

private readonly Float _l2Weight;
private readonly float _l2Weight;
private readonly bool _perParameterSignificance;

public override PredictionKind PredictionKind => PredictionKind.Regression;
Expand All @@ -65,15 +64,59 @@ public sealed class Arguments : LearnerInputBaseWithWeight
private static readonly TrainerInfo _info = new TrainerInfo(caching: false);
public override TrainerInfo Info => _info;

public OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args)
: base(env, LoadNameValue)
/// <summary>
/// Initializes a new instance of <see cref="OlsLinearRegressionTrainer"/>
/// </summary>
/// <param name="env">The environment to use.</param>
/// <param name="labelColumn">The name of the label column.</param>
/// <param name="featureColumn">The name of the feature column.</param>
/// <param name="weightColumn">The name for the example weight column.</param>
/// <param name="advancedSettings">A delegate to apply all the advanced arguments to the algorithm.</param>
public OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, string labelColumn,
string weightColumn = null, Action<Arguments> advancedSettings = null)
: this(env, ArgsInit(featureColumn, labelColumn, weightColumn, advancedSettings))
{
Host.CheckNonEmpty(featureColumn, nameof(featureColumn));
Host.CheckNonEmpty(labelColumn, nameof(labelColumn));
}

/// <summary>
/// Initializes a new instance of <see cref="OlsLinearRegressionTrainer"/>
/// </summary>
internal OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args)
: base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), TrainerUtils.MakeR4VecFeature(args.FeatureColumn),
TrainerUtils.MakeR4ScalarLabel(args.LabelColumn), TrainerUtils.MakeR4ScalarWeightColumn(args.WeightColumn))
{
Host.CheckValue(args, nameof(args));
Host.CheckUserArg(args.L2Weight >= 0, nameof(args.L2Weight), "L2 regularization term cannot be negative");
_l2Weight = args.L2Weight;
_perParameterSignificance = args.PerParameterSignificance;
}

private static Arguments ArgsInit(string featureColumn, string labelColumn,
string weightColumn, Action<Arguments> advancedSettings)
{
var args = new Arguments();

// Apply the advanced args, if the user supplied any.
advancedSettings?.Invoke(args);
args.FeatureColumn = featureColumn;
args.LabelColumn = labelColumn;
args.WeightColumn = weightColumn;
return args;
}

protected override RegressionPredictionTransformer<OlsLinearRegressionPredictor> MakeTransformer(OlsLinearRegressionPredictor model, ISchema trainSchema)
=> new RegressionPredictionTransformer<OlsLinearRegressionPredictor>(Host, model, trainSchema, FeatureColumn.Name);

protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
return new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
};
}

/// <summary>
/// In several calculations, we calculate probabilities or other quantities that should range
/// from 0 to 1, but because of numerical imprecision may, in entirely innocent circumstances,
Expand All @@ -84,7 +127,7 @@ public OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args)
private static Double ProbClamp(Double p)
=> Math.Max(0, Math.Min(p, 1));

public override OlsLinearRegressionPredictor Train(TrainContext context)
protected override OlsLinearRegressionPredictor TrainModelCore(TrainContext context)
{
using (var ch = Host.Start("Training"))
{
Expand Down Expand Up @@ -234,24 +277,24 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac
for (int i = 0; i < beta.Length; ++i)
ch.Check(FloatUtils.IsFinite(beta[i]), "Non-finite values detected in OLS solution");

var weights = VBufferUtils.CreateDense<Float>(beta.Length - 1);
var weights = VBufferUtils.CreateDense<float>(beta.Length - 1);
for (int i = 1; i < beta.Length; ++i)
weights.Values[i - 1] = (Float)beta[i];
var bias = (Float)beta[0];
weights.Values[i - 1] = (float)beta[i];
var bias = (float)beta[0];
if (!(_l2Weight > 0) && m == n)
{
// We would expect the solution to the problem to be exact in this case.
ch.Info("Number of examples equals number of parameters, solution is exact but no statistics can be derived");
return new OlsLinearRegressionPredictor(Host, ref weights, bias, null, null, null, 1, Float.NaN);
return new OlsLinearRegressionPredictor(Host, ref weights, bias, null, null, null, 1, float.NaN);
}

Double rss = 0; // residual sum of squares
Double tss = 0; // total sum of squares
using (var cursor = cursorFactory.Create())
{
var lrPredictor = new LinearRegressionPredictor(Host, ref weights, bias);
var lrMap = lrPredictor.GetMapper<VBuffer<Float>, Float>();
Float yh = default;
var lrMap = lrPredictor.GetMapper<VBuffer<float>, float>();
float yh = default;
while (cursor.MoveNext())
{
var features = cursor.Features;
Expand Down Expand Up @@ -298,7 +341,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac
{
// Iterate through all entries of inverse Hessian to make adjustment to variance.
int ioffset = 1;
Float reg = _l2Weight * _l2Weight * n;
float reg = _l2Weight * _l2Weight * n;
for (int iRow = 1; iRow < m; iRow++)
{
for (int iCol = 0; iCol <= iRow; iCol++)
Expand All @@ -321,7 +364,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac
standardErrors[i] = Math.Sqrt(s2 * standardErrors[i]);
ch.Check(FloatUtils.IsFinite(standardErrors[i]), "Non-finite standard error detected from OLS solution");
tValues[i] = beta[i] / standardErrors[i];
pValues[i] = (Float)MathUtils.TStatisticToPValue(tValues[i], n - m);
pValues[i] = (float)MathUtils.TStatisticToPValue(tValues[i], n - m);
ch.Check(0 <= pValues[i] && pValues[i] <= 1, "p-Value calculated outside expected [0,1] range");
}

Expand Down Expand Up @@ -558,7 +601,7 @@ public IReadOnlyCollection<Double> TValues
public IReadOnlyCollection<Double> PValues
{ get { return _pValues.AsReadOnly(); } }

internal OlsLinearRegressionPredictor(IHostEnvironment env, ref VBuffer<Float> weights, Float bias,
internal OlsLinearRegressionPredictor(IHostEnvironment env, ref VBuffer<float> weights, float bias,
Double[] standardErrors, Double[] tValues, Double[] pValues, Double rSquared, Double rSquaredAdjusted)
: base(env, RegistrationName, ref weights, bias)
{
Expand Down Expand Up @@ -726,7 +769,7 @@ public override void SaveSummary(TextWriter writer, RoleMappedSchema schema)
}
}

public override void GetFeatureWeights(ref VBuffer<Float> weights)
public override void GetFeatureWeights(ref VBuffer<float> weights)
{
if (_pValues == null)
{
Expand All @@ -737,15 +780,15 @@ public override void GetFeatureWeights(ref VBuffer<Float> weights)
var values = weights.Values;
var size = _pValues.Length - 1;
if (Utils.Size(values) < size)
values = new Float[size];
values = new float[size];
for (int i = 0; i < size; i++)
{
var score = -(Float)Math.Log(_pValues[i + 1]);
if (score > Float.MaxValue)
score = Float.MaxValue;
var score = -(float)Math.Log(_pValues[i + 1]);
if (score > float.MaxValue)
score = float.MaxValue;
values[i] = score;
}
weights = new VBuffer<Float>(size, values, weights.Indices);
weights = new VBuffer<float>(size, values, weights.Indices);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using Microsoft.ML.Runtime;
using Microsoft.ML.Runtime.Data;
using Microsoft.ML.Runtime.EntryPoints;
using Microsoft.ML.Runtime.HalLearners;
using Microsoft.ML.Runtime.Learners;
using Microsoft.ML.Runtime.RunTests;
using Xunit;

namespace Microsoft.ML.Tests.TrainerEstimators
{
public partial class TrainerEstimators
{
private IDataView GetGeneratedRegressionDataview()
{
return new TextLoader(Env,
new TextLoader.Arguments()
{
Separator = ";",
HasHeader = true,
Column = new[]
{
new TextLoader.Column("Label", DataKind.R4, 11),
new TextLoader.Column("Features", DataKind.R4, new [] { new TextLoader.Range(0, 10) } )
}
}).Read(new MultiFileSource(GetDataPath(TestDatasets.generatedRegressionDataset.trainFilename)));
}

[Fact]
public void TestEstimatorOlsLinearRegression()
{
var dataView = GetGeneratedRegressionDataview();
var pipe = new OlsLinearRegressionTrainer(Env, "Features", "Label");
TestEstimatorCore(pipe, dataView);
Done();
}
}
}