From 38632a897e0ba7991850ca58adcc23f67aefd6d5 Mon Sep 17 00:00:00 2001 From: Artidoro Pagnoni Date: Mon, 24 Sep 2018 11:23:27 -0700 Subject: [PATCH 1/4] converted OLS, need to write tests --- .../OlsLinearRegression.cs | 97 ++++++++++++++----- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index cb7472633f..7aa75cc9bc 100644 --- a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using Float = System.Single; - using System; using System.Collections.Generic; using System.IO; +using Microsoft.ML.Core.Data; using Microsoft.ML.Runtime; using Microsoft.ML.Runtime.HalLearners; using Microsoft.ML.Runtime.Internal.Internallearn; @@ -34,7 +33,7 @@ namespace Microsoft.ML.Runtime.HalLearners { /// - public sealed class OlsLinearRegressionTrainer : TrainerBase + public sealed class OlsLinearRegressionTrainer : TrainerEstimatorBase, OlsLinearRegressionPredictor> { public sealed class Arguments : LearnerInputBaseWithWeight { @@ -44,7 +43,7 @@ public sealed class Arguments : LearnerInputBaseWithWeight [Argument(ArgumentType.AtMostOnce, HelpText = "L2 regularization weight", ShortName = "l2", SortOrder = 50)] [TGUI(SuggestedSweeps = "1e-6,0.1,1")] [TlcModule.SweepableDiscreteParamAttribute("L2Weight", new object[] { 1e-6f, 0.1f, 1f })] - public Float L2Weight = 1e-6f; + public float L2Weight = 1e-6f; [Argument(ArgumentType.LastOccurenceWins, HelpText = "Whether to calculate per parameter significance statistics", ShortName = "sig")] public bool PerParameterSignificance = true; @@ -56,7 +55,7 @@ public sealed class Arguments : LearnerInputBaseWithWeight internal const string Summary = "The ordinary least square regression fits the target function as a linear function of the numerical features " + "that minimizes the square loss function."; - private readonly Float _l2Weight; + private readonly float _l2Weight; private readonly bool _perParameterSignificance; public override PredictionKind PredictionKind => PredictionKind.Regression; @@ -65,8 +64,25 @@ public sealed class Arguments : LearnerInputBaseWithWeight private static readonly TrainerInfo _info = new TrainerInfo(caching: false); public override TrainerInfo Info => _info; + /// + /// Initializes a new instance of + /// + /// The private instance of . + /// The name of the label column. + /// The name of the feature column. + /// The name for the column containing the initial weight. + /// A delegate to apply all the advanced arguments to the algorithm. + internal OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, string labelColumn, + string weightColumn = null, Action advancedSettings = null) + : this(env, ArgsInit(featureColumn, labelColumn, weightColumn, advancedSettings)) + { + } + + /// + /// Initializes a new instance of + /// public OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args) - : base(env, LoadNameValue) + : base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), MakeFeatureColumn(args.FeatureColumn), MakeLabelColumn(args.LabelColumn), MakeWeightColumn(args.WeightColumn)) { Host.CheckValue(args, nameof(args)); Host.CheckUserArg(args.L2Weight >= 0, nameof(args.L2Weight), "L2 regularization term cannot be negative"); @@ -74,6 +90,43 @@ public OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args) _perParameterSignificance = args.PerParameterSignificance; } + private static Arguments ArgsInit(string featureColumn, string labelColumn, + string weightColumn, Action advancedSettings) + { + var args = new Arguments(); + + //apply the advanced args, if the user supplied any + advancedSettings?.Invoke(args); + args.FeatureColumn = featureColumn; + args.LabelColumn = labelColumn; + args.WeightColumn = weightColumn; + return args; + } + + private static SchemaShape.Column MakeFeatureColumn(string featureColumn) + => new SchemaShape.Column(featureColumn, SchemaShape.Column.VectorKind.Vector, NumberType.R4, false); + + private static SchemaShape.Column MakeLabelColumn(string labelColumn) + => new SchemaShape.Column(labelColumn, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false); + + private static SchemaShape.Column MakeWeightColumn(string weightColumn) + { + if (weightColumn == null) + return null; + return new SchemaShape.Column(weightColumn, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false); + } + + protected override RegressionPredictionTransformer MakeTransformer(OlsLinearRegressionPredictor model, ISchema trainSchema) + => new RegressionPredictionTransformer(Host, model, trainSchema, FeatureColumn.Name); + + protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) + { + return new[] + { + new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())) + }; + } + /// /// In several calculations, we calculate probabilities or other quantities that should range /// from 0 to 1, but because of numerical imprecision may, in entirely innocent circumstances, @@ -84,7 +137,7 @@ public OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args) private static Double ProbClamp(Double p) => Math.Max(0, Math.Min(p, 1)); - public override OlsLinearRegressionPredictor Train(TrainContext context) + protected override OlsLinearRegressionPredictor TrainModelCore(TrainContext context) { using (var ch = Host.Start("Training")) { @@ -234,15 +287,15 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac for (int i = 0; i < beta.Length; ++i) ch.Check(FloatUtils.IsFinite(beta[i]), "Non-finite values detected in OLS solution"); - var weights = VBufferUtils.CreateDense(beta.Length - 1); + var weights = VBufferUtils.CreateDense(beta.Length - 1); for (int i = 1; i < beta.Length; ++i) - weights.Values[i - 1] = (Float)beta[i]; - var bias = (Float)beta[0]; + weights.Values[i - 1] = (float)beta[i]; + var bias = (float)beta[0]; if (!(_l2Weight > 0) && m == n) { // We would expect the solution to the problem to be exact in this case. ch.Info("Number of examples equals number of parameters, solution is exact but no statistics can be derived"); - return new OlsLinearRegressionPredictor(Host, ref weights, bias, null, null, null, 1, Float.NaN); + return new OlsLinearRegressionPredictor(Host, ref weights, bias, null, null, null, 1, float.NaN); } Double rss = 0; // residual sum of squares @@ -250,8 +303,8 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac using (var cursor = cursorFactory.Create()) { var lrPredictor = new LinearRegressionPredictor(Host, ref weights, bias); - var lrMap = lrPredictor.GetMapper, Float>(); - Float yh = default; + var lrMap = lrPredictor.GetMapper, float>(); + float yh = default; while (cursor.MoveNext()) { var features = cursor.Features; @@ -298,7 +351,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac { // Iterate through all entries of inverse Hessian to make adjustment to variance. int ioffset = 1; - Float reg = _l2Weight * _l2Weight * n; + float reg = _l2Weight * _l2Weight * n; for (int iRow = 1; iRow < m; iRow++) { for (int iCol = 0; iCol <= iRow; iCol++) @@ -321,7 +374,7 @@ private OlsLinearRegressionPredictor TrainCore(IChannel ch, FloatLabelCursor.Fac standardErrors[i] = Math.Sqrt(s2 * standardErrors[i]); ch.Check(FloatUtils.IsFinite(standardErrors[i]), "Non-finite standard error detected from OLS solution"); tValues[i] = beta[i] / standardErrors[i]; - pValues[i] = (Float)MathUtils.TStatisticToPValue(tValues[i], n - m); + pValues[i] = (float)MathUtils.TStatisticToPValue(tValues[i], n - m); ch.Check(0 <= pValues[i] && pValues[i] <= 1, "p-Value calculated outside expected [0,1] range"); } @@ -558,7 +611,7 @@ public IReadOnlyCollection TValues public IReadOnlyCollection PValues { get { return _pValues.AsReadOnly(); } } - internal OlsLinearRegressionPredictor(IHostEnvironment env, ref VBuffer weights, Float bias, + internal OlsLinearRegressionPredictor(IHostEnvironment env, ref VBuffer weights, float bias, Double[] standardErrors, Double[] tValues, Double[] pValues, Double rSquared, Double rSquaredAdjusted) : base(env, RegistrationName, ref weights, bias) { @@ -726,7 +779,7 @@ public override void SaveSummary(TextWriter writer, RoleMappedSchema schema) } } - public override void GetFeatureWeights(ref VBuffer weights) + public override void GetFeatureWeights(ref VBuffer weights) { if (_pValues == null) { @@ -737,15 +790,15 @@ public override void GetFeatureWeights(ref VBuffer weights) var values = weights.Values; var size = _pValues.Length - 1; if (Utils.Size(values) < size) - values = new Float[size]; + values = new float[size]; for (int i = 0; i < size; i++) { - var score = -(Float)Math.Log(_pValues[i + 1]); - if (score > Float.MaxValue) - score = Float.MaxValue; + var score = -(float)Math.Log(_pValues[i + 1]); + if (score > float.MaxValue) + score = float.MaxValue; values[i] = score; } - weights = new VBuffer(size, values, weights.Indices); + weights = new VBuffer(size, values, weights.Indices); } } } From 24472c9966dba52a6a81956b928e2166a6168f40 Mon Sep 17 00:00:00 2001 From: Artidoro Pagnoni Date: Mon, 24 Sep 2018 13:03:53 -0700 Subject: [PATCH 2/4] added test for estimator --- .../OlsLinearRegression.cs | 4 +- .../OlsLinearRegressionTests.cs | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 test/Microsoft.ML.Tests/TrainerEstimators/OlsLinearRegressionTests.cs diff --git a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index 7aa75cc9bc..8574890fe4 100644 --- a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -72,7 +72,7 @@ public sealed class Arguments : LearnerInputBaseWithWeight /// The name of the feature column. /// The name for the column containing the initial weight. /// A delegate to apply all the advanced arguments to the algorithm. - internal OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, string labelColumn, + public OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, string labelColumn, string weightColumn = null, Action advancedSettings = null) : this(env, ArgsInit(featureColumn, labelColumn, weightColumn, advancedSettings)) { @@ -81,7 +81,7 @@ internal OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, /// /// Initializes a new instance of /// - public OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args) + internal OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args) : base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), MakeFeatureColumn(args.FeatureColumn), MakeLabelColumn(args.LabelColumn), MakeWeightColumn(args.WeightColumn)) { Host.CheckValue(args, nameof(args)); diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/OlsLinearRegressionTests.cs b/test/Microsoft.ML.Tests/TrainerEstimators/OlsLinearRegressionTests.cs new file mode 100644 index 0000000000..3fd13dc729 --- /dev/null +++ b/test/Microsoft.ML.Tests/TrainerEstimators/OlsLinearRegressionTests.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Runtime; +using Microsoft.ML.Runtime.Data; +using Microsoft.ML.Runtime.EntryPoints; +using Microsoft.ML.Runtime.HalLearners; +using Microsoft.ML.Runtime.Learners; +using Microsoft.ML.Runtime.RunTests; +using Xunit; + +namespace Microsoft.ML.Tests.TrainerEstimators +{ + public partial class TrainerEstimators + { + private IDataView GetGeneratedRegressionDataview() + { + return new TextLoader(Env, + new TextLoader.Arguments() + { + Separator = ";", + HasHeader = true, + Column = new[] + { + new TextLoader.Column("Label", DataKind.R4, 11), + new TextLoader.Column("Features", DataKind.R4, new [] { new TextLoader.Range(0, 10) } ) + } + }).Read(new MultiFileSource(GetDataPath(TestDatasets.generatedRegressionDataset.trainFilename))); + } + + [Fact] + public void TestEstimatorOlsLinearRegression() + { + var dataView = GetGeneratedRegressionDataview(); + var pipe = new OlsLinearRegressionTrainer(Env, "Features", "Label"); + TestEstimatorCore(pipe, dataView); + Done(); + } + } +} From df1344a32fe9680ca0ad9c91c51a8aec4984e877 Mon Sep 17 00:00:00 2001 From: Artidoro Pagnoni Date: Mon, 24 Sep 2018 13:34:00 -0700 Subject: [PATCH 3/4] used TrainerUtils to create new columns --- .../OlsLinearRegression.cs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index 8574890fe4..a775987ef7 100644 --- a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -82,7 +82,8 @@ public OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, st /// Initializes a new instance of /// internal OlsLinearRegressionTrainer(IHostEnvironment env, Arguments args) - : base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), MakeFeatureColumn(args.FeatureColumn), MakeLabelColumn(args.LabelColumn), MakeWeightColumn(args.WeightColumn)) + : base(Contracts.CheckRef(env, nameof(env)).Register(LoadNameValue), TrainerUtils.MakeR4VecFeature(args.FeatureColumn), + TrainerUtils.MakeR4ScalarLabel(args.LabelColumn), TrainerUtils.MakeR4ScalarWeightColumn(args.WeightColumn)) { Host.CheckValue(args, nameof(args)); Host.CheckUserArg(args.L2Weight >= 0, nameof(args.L2Weight), "L2 regularization term cannot be negative"); @@ -103,19 +104,6 @@ private static Arguments ArgsInit(string featureColumn, string labelColumn, return args; } - private static SchemaShape.Column MakeFeatureColumn(string featureColumn) - => new SchemaShape.Column(featureColumn, SchemaShape.Column.VectorKind.Vector, NumberType.R4, false); - - private static SchemaShape.Column MakeLabelColumn(string labelColumn) - => new SchemaShape.Column(labelColumn, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false); - - private static SchemaShape.Column MakeWeightColumn(string weightColumn) - { - if (weightColumn == null) - return null; - return new SchemaShape.Column(weightColumn, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false); - } - protected override RegressionPredictionTransformer MakeTransformer(OlsLinearRegressionPredictor model, ISchema trainSchema) => new RegressionPredictionTransformer(Host, model, trainSchema, FeatureColumn.Name); From 76b6fbdc47dae5b90f6e9568d0abc61e316d43d7 Mon Sep 17 00:00:00 2001 From: Artidoro Pagnoni Date: Mon, 24 Sep 2018 16:36:11 -0700 Subject: [PATCH 4/4] fixing review comments --- src/Microsoft.ML.HalLearners/OlsLinearRegression.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs index a775987ef7..b660ecf713 100644 --- a/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs +++ b/src/Microsoft.ML.HalLearners/OlsLinearRegression.cs @@ -67,15 +67,17 @@ public sealed class Arguments : LearnerInputBaseWithWeight /// /// Initializes a new instance of /// - /// The private instance of . + /// The environment to use. /// The name of the label column. /// The name of the feature column. - /// The name for the column containing the initial weight. + /// The name for the example weight column. /// A delegate to apply all the advanced arguments to the algorithm. public OlsLinearRegressionTrainer(IHostEnvironment env, string featureColumn, string labelColumn, string weightColumn = null, Action advancedSettings = null) : this(env, ArgsInit(featureColumn, labelColumn, weightColumn, advancedSettings)) { + Host.CheckNonEmpty(featureColumn, nameof(featureColumn)); + Host.CheckNonEmpty(labelColumn, nameof(labelColumn)); } /// @@ -96,7 +98,7 @@ private static Arguments ArgsInit(string featureColumn, string labelColumn, { var args = new Arguments(); - //apply the advanced args, if the user supplied any + // Apply the advanced args, if the user supplied any. advancedSettings?.Invoke(args); args.FeatureColumn = featureColumn; args.LabelColumn = labelColumn;