From 4ac1ee760cb762291fcb60ad60e593ed343e6f43 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 12 Feb 2020 15:03:45 -0800 Subject: [PATCH 1/5] fix issue in WaiterWaiter --- src/Microsoft.ML.Data/DataView/CacheDataView.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Data/DataView/CacheDataView.cs b/src/Microsoft.ML.Data/DataView/CacheDataView.cs index 251ef0c485..a7d71c6eac 100644 --- a/src/Microsoft.ML.Data/DataView/CacheDataView.cs +++ b/src/Microsoft.ML.Data/DataView/CacheDataView.cs @@ -682,7 +682,13 @@ public bool Wait(long pos) { foreach (var w in _waiters) w.Wait(pos); - return pos < _parent._rowCount || _parent._rowCount == -1; + + // Back ground thread is filling data view and change the _rowCount from -1 to real row count. + // Original is: return pos < _parent._rowCount || _parent._rowCount == -1; + // In some case the change to _rowCount happens + // after pos < _parent._rowCount finish but before _parent._rowCount == -1 + // thus cause flakyness + return _parent._rowCount == -1 || pos < _parent._rowCount; } public static Wrapper Create(CacheDataView parent, Func pred) From ad9a2d70aa13edb4b8872a4ccfbbdfdea31b4882 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:02:15 -0800 Subject: [PATCH 2/5] re-enable tests that affected by the fixed issue --- .../UnitTests/TestEntryPoints.cs | 12 ------------ .../IntrospectiveTraining.cs | 2 -- test/Microsoft.ML.Predictor.Tests/TestPredictors.cs | 4 ---- .../DataPipe/TestDataPipe.cs | 2 -- test/Microsoft.ML.Tests/OnnxConversionTest.cs | 2 -- .../Api/CookbookSamples/CookbookSamplesDynamicApi.cs | 8 -------- .../Scenarios/IrisPlantClassificationTests.cs | 2 -- 7 files changed, 32 deletions(-) diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 8b773d29f6..de8dcee348 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -2640,8 +2640,6 @@ public void EntryPointEvaluateMulticlass() } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void EntryPointEvaluateRegression() { var dataPath = GetDataPath(TestDatasets.generatedRegressionDatasetmacro.trainFilename); @@ -2752,8 +2750,6 @@ public void EntryPointLightGbmMulticlass() } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void EntryPointSdcaBinary() { TestEntryPointRoutine("breast-cancer.txt", "Trainers.StochasticDualCoordinateAscentBinaryClassifier"); @@ -2766,8 +2762,6 @@ public void EntryPointSDCAMulticlass() } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void EntryPointSDCARegression() { TestEntryPointRoutine(TestDatasets.generatedRegressionDatasetmacro.trainFilename, "Trainers.StochasticDualCoordinateAscentRegressor", loader: TestDatasets.generatedRegressionDatasetmacro.loaderSettings); @@ -3855,8 +3849,6 @@ public void EntryPointChainedTrainTestMacros() } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void EntryPointChainedCrossValMacros() { string inputGraph = @" @@ -5506,8 +5498,6 @@ public void TestCrossValidationMacroMulticlassWithWarnings() } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TestCrossValidationMacroWithStratification() { var dataPath = GetDataPath(@"breast-cancer.txt"); @@ -6039,8 +6029,6 @@ public void TestCrossValidationMacroWithNonDefaultNames() } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TestOvaMacro() { var dataPath = GetDataPath(@"iris.txt"); diff --git a/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs b/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs index 76b6cf8a81..31fa7277d1 100644 --- a/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs +++ b/test/Microsoft.ML.Functional.Tests/IntrospectiveTraining.cs @@ -203,8 +203,6 @@ public void InspectLdaModelParameters() /// Introspective Training: Linear model parameters may be inspected. /// [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void InpsectLinearModelParameters() { var mlContext = new MLContext(seed: 1); diff --git a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs index 9fd0897c9b..797831afc0 100644 --- a/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs +++ b/test/Microsoft.ML.Predictor.Tests/TestPredictors.cs @@ -247,8 +247,6 @@ public void KMeansClusteringTest() [X64Fact("x86 output differs from Baseline")] [TestCategory("Binary")] [TestCategory("SDCA")] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void LinearClassifierTest() { var binaryPredictors = new[] @@ -324,8 +322,6 @@ public void BinaryClassifierLogisticRegressionNormTest() /// [LessThanNetCore30OrNotNetCoreAndX64Fact("netcoreapp3.0 and x86 output differs from Baseline")] [TestCategory("Binary")] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void BinaryClassifierLogisticRegressionNonNegativeTest() { var binaryPredictors = new[] { TestLearners.logisticRegressionNonNegative }; diff --git a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs index 39f9932686..20865d7a2c 100644 --- a/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs +++ b/test/Microsoft.ML.TestFramework/DataPipe/TestDataPipe.cs @@ -1022,8 +1022,6 @@ public void SavePipeTrainAndScoreFccFastTree() [TestCategory("DataPipeSerialization")] [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void SavePipeTrainAndScoreFccTransformStr() { TestCore(null, false, diff --git a/test/Microsoft.ML.Tests/OnnxConversionTest.cs b/test/Microsoft.ML.Tests/OnnxConversionTest.cs index 6573b940bd..52cd84c8f8 100644 --- a/test/Microsoft.ML.Tests/OnnxConversionTest.cs +++ b/test/Microsoft.ML.Tests/OnnxConversionTest.cs @@ -696,8 +696,6 @@ public void LogisticRegressionOnnxConversionTest() } [LightGBMFact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void LightGbmBinaryClassificationOnnxConversionTest() { // Step 1: Create and train a ML.NET pipeline. diff --git a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs index d19b35a256..70299beb83 100644 --- a/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs +++ b/test/Microsoft.ML.Tests/Scenarios/Api/CookbookSamples/CookbookSamplesDynamicApi.cs @@ -204,8 +204,6 @@ private void TrainRegression(string trainDataPath, string testDataPath, string m } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TrainRegressionModel() => TrainRegression(GetDataPath(TestDatasets.generatedRegressionDataset.trainFilename), GetDataPath(TestDatasets.generatedRegressionDataset.testFilename), DeleteOutputPath("cook_model.zip")); @@ -293,8 +291,6 @@ private void PredictOnIris(ITransformer model) } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TrainAndPredictOnIris() => PredictOnIris(TrainOnIris(GetDataPath("iris.data"))); @@ -629,8 +625,6 @@ private void CategoricalFeaturizationOn(params string[] dataPath) } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void CrossValidationIris() => CrossValidationOn(GetDataPath("iris.data")); @@ -708,8 +702,6 @@ public class OutputRow } [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void CustomTransformer() { var mlContext = new MLContext(1); diff --git a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs index fcb594d910..4181ce3b19 100644 --- a/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs +++ b/test/Microsoft.ML.Tests/Scenarios/IrisPlantClassificationTests.cs @@ -15,8 +15,6 @@ namespace Microsoft.ML.Scenarios public partial class ScenariosTests : BaseTestClass { [Fact] - //Skipping test temporarily. This test will be re-enabled once the cause of failures has been determined - [Trait("Category", "SkipInCI")] public void TrainAndPredictIrisModelTest() { var mlContext = new MLContext(seed: 1); From 4d76dda9ea837348ac1c1b8ef2b77951a40ec341 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 12 Feb 2020 16:26:08 -0800 Subject: [PATCH 3/5] refine comments --- src/Microsoft.ML.Data/DataView/CacheDataView.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.Data/DataView/CacheDataView.cs b/src/Microsoft.ML.Data/DataView/CacheDataView.cs index a7d71c6eac..c8ac426928 100644 --- a/src/Microsoft.ML.Data/DataView/CacheDataView.cs +++ b/src/Microsoft.ML.Data/DataView/CacheDataView.cs @@ -683,12 +683,10 @@ public bool Wait(long pos) foreach (var w in _waiters) w.Wait(pos); - // Back ground thread is filling data view and change the _rowCount from -1 to real row count. - // Original is: return pos < _parent._rowCount || _parent._rowCount == -1; - // In some case the change to _rowCount happens - // after pos < _parent._rowCount finish but before _parent._rowCount == -1 - // thus cause flakyness - return _parent._rowCount == -1 || pos < _parent._rowCount; + // _parent._rowCount may or may not be initialized at this point. Only read the value once + // to avoid race conditions. + var rowCount = _parent._rowCount; + return rowCount == -1 || pos < rowCount; } public static Wrapper Create(CacheDataView parent, Func pred) From d989ff3058c5a3ac7726e510434f80522924e730 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Wed, 12 Feb 2020 23:41:13 -0800 Subject: [PATCH 4/5] refactor based on discussion --- .../DataView/CacheDataView.cs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.ML.Data/DataView/CacheDataView.cs b/src/Microsoft.ML.Data/DataView/CacheDataView.cs index c8ac426928..7210c76fea 100644 --- a/src/Microsoft.ML.Data/DataView/CacheDataView.cs +++ b/src/Microsoft.ML.Data/DataView/CacheDataView.cs @@ -197,9 +197,13 @@ public int MapInputToCacheColumnIndex(int inputIndex) /// public long? GetRowCount() { - if (_rowCount < 0) + // _rowCount may or may not be initialized at this point. Only read the value once + // to avoid race conditions. + long rowCount = _rowCount; + + if (rowCount < 0) return null; - return _rowCount; + return rowCount; } public DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null) @@ -605,8 +609,8 @@ private sealed class TrivialWaiter : IWaiter private TrivialWaiter(CacheDataView parent) { - Contracts.Assert(parent._rowCount >= 0); - _lim = parent._rowCount; + Contracts.Assert(parent.GetRowCount().HasValue); + _lim = parent.GetRowCount().Value; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -669,7 +673,7 @@ private WaiterWaiter(CacheDataView parent, Func pred) waiters.Add(waiter); } // Make the array of waiters. - if (_parent._rowCount < 0 && waiters.Count == 0) + if (!_parent.GetRowCount().HasValue && waiters.Count == 0) { Contracts.AssertValue(_parent._cacheDefaultWaiter); waiters.Add(_parent._cacheDefaultWaiter); @@ -682,11 +686,7 @@ public bool Wait(long pos) { foreach (var w in _waiters) w.Wait(pos); - - // _parent._rowCount may or may not be initialized at this point. Only read the value once - // to avoid race conditions. - var rowCount = _parent._rowCount; - return rowCount == -1 || pos < rowCount; + return !_parent.GetRowCount().HasValue || pos < _parent.GetRowCount().Value; } public static Wrapper Create(CacheDataView parent, Func pred) @@ -1423,8 +1423,8 @@ public ImplOne(CacheDataView parent, DataViewRowCursor input, int srcCol, Ordere : base(parent, input, srcCol, waiter) { _getter = input.GetGetter(input.Schema[srcCol]); - if (parent._rowCount >= 0) - _values = new T[(int)parent._rowCount]; + if (parent.GetRowCount().HasValue) + _values = new T[(int)parent.GetRowCount().Value]; } public override void CacheCurrent() From dffaaf092fd9e3fe0a13a7df25062fbd62158b43 Mon Sep 17 00:00:00 2001 From: frank-dong-ms <55860649+frank-dong-ms@users.noreply.github.com> Date: Thu, 13 Feb 2020 11:52:20 -0800 Subject: [PATCH 5/5] take comments --- src/Microsoft.ML.Data/DataView/CacheDataView.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.Data/DataView/CacheDataView.cs b/src/Microsoft.ML.Data/DataView/CacheDataView.cs index 7210c76fea..e24ed9c59d 100644 --- a/src/Microsoft.ML.Data/DataView/CacheDataView.cs +++ b/src/Microsoft.ML.Data/DataView/CacheDataView.cs @@ -609,8 +609,9 @@ private sealed class TrivialWaiter : IWaiter private TrivialWaiter(CacheDataView parent) { - Contracts.Assert(parent.GetRowCount().HasValue); - _lim = parent.GetRowCount().Value; + var rowCount = parent.GetRowCount(); + Contracts.Assert(rowCount.HasValue); + _lim = rowCount.Value; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -686,7 +687,9 @@ public bool Wait(long pos) { foreach (var w in _waiters) w.Wait(pos); - return !_parent.GetRowCount().HasValue || pos < _parent.GetRowCount().Value; + + var rowCount = _parent.GetRowCount(); + return !rowCount.HasValue || pos < rowCount.Value; } public static Wrapper Create(CacheDataView parent, Func pred) @@ -1423,8 +1426,8 @@ public ImplOne(CacheDataView parent, DataViewRowCursor input, int srcCol, Ordere : base(parent, input, srcCol, waiter) { _getter = input.GetGetter(input.Schema[srcCol]); - if (parent.GetRowCount().HasValue) - _values = new T[(int)parent.GetRowCount().Value]; + if (parent.GetRowCount() is { } rowCount) + _values = new T[rowCount]; } public override void CacheCurrent()