From a28701b7ae90b491bc322672e6c52cb583675f17 Mon Sep 17 00:00:00 2001 From: J W Date: Mon, 17 Sep 2018 17:31:18 -0400 Subject: [PATCH 01/19] Start renaming arguments --- .../CpuMathUtils.netcoreapp.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index f15f5c3938..9c37cd5109 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -24,14 +24,14 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Avx.IsSupported ? Vector256Alignment : (Sse.IsSupported ? Vector128Alignment : FloatAlignment); - public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) + public static void MatTimesSrc(bool transpose, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) { Contracts.Assert(mat.Size == dst.Size * src.Size); Contracts.Assert(crun >= 0); if (Avx.IsSupported) { - if (!tran) + if (!transpose) { Contracts.Assert(crun <= dst.Size); AvxIntrinsics.MatMulX(add, mat, src, dst, crun, src.Size); @@ -44,7 +44,7 @@ public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArr } else if (Sse.IsSupported) { - if (!tran) + if (!transpose) { Contracts.Assert(crun <= dst.Size); SseIntrinsics.MatMulA(add, mat, src, dst, crun, src.Size); @@ -57,7 +57,7 @@ public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArr } else { - if (!tran) + if (!transpose) { Contracts.Assert(crun <= dst.Size); for (int i = 0; i < crun; i++) @@ -102,7 +102,7 @@ public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArr } } - public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, + public static void MatTimesSrc(bool transpose, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) { Contracts.AssertValue(rgposSrc); @@ -123,7 +123,7 @@ public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgpo if (Avx.IsSupported) { - if (!tran) + if (!transpose) { Contracts.Assert(crun <= dst.Size); AvxIntrinsics.MatMulPX(add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size); @@ -136,7 +136,7 @@ public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgpo } else if (Sse.IsSupported) { - if (!tran) + if (!transpose) { Contracts.Assert(crun <= dst.Size); SseIntrinsics.MatMulPA(add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size); @@ -149,7 +149,7 @@ public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgpo } else { - if (!tran) + if (!transpose) { Contracts.Assert(crun <= dst.Size); for (int i = 0; i < crun; i++) From 177edd98a9b8dbbbcc9f2444d7c68e0991c9583a Mon Sep 17 00:00:00 2001 From: J W Date: Sat, 22 Sep 2018 11:36:49 -0400 Subject: [PATCH 02/19] Start updating parameter names --- .../CpuMathUtils.netcoreapp.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 9c37cd5109..c70d21dd88 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -197,74 +197,74 @@ public static void MatTimesSrc(bool transpose, bool add, AlignedArray mat, int[] } } - public static void Add(float a, float[] dst, int count) + public static void Add(float src, float[] dst, int count) { Contracts.AssertNonEmpty(dst); Contracts.Assert(count > 0); Contracts.Assert(count <= dst.Length); - Add(a, new Span(dst, 0, count)); + Add(src, new Span(dst, 0, count)); } - private static void Add(float a, Span dst) + private static void Add(float src, Span dst) { if (Avx.IsSupported) { - AvxIntrinsics.AddScalarU(a, dst); + AvxIntrinsics.AddScalarU(src, dst); } else if (Sse.IsSupported) { - SseIntrinsics.AddScalarU(a, dst); + SseIntrinsics.AddScalarU(src, dst); } else { for (int i = 0; i < dst.Length; i++) { - dst[i] += a; + dst[i] += src; } } } - public static void Scale(float a, float[] dst, int count) + public static void Scale(float src, float[] dst, int count) { Contracts.AssertNonEmpty(dst); Contracts.Assert(count > 0); Contracts.Assert(count <= dst.Length); - Scale(a, new Span(dst, 0, count)); + Scale(src, new Span(dst, 0, count)); } - public static void Scale(float a, float[] dst, int offset, int count) + public static void Scale(float src, float[] dst, int offset, int count) { Contracts.AssertNonEmpty(dst); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); Contracts.Assert(offset < (dst.Length - count)); - Scale(a, new Span(dst, offset, count)); + Scale(src, new Span(dst, offset, count)); } - private static void Scale(float a, Span dst) + private static void Scale(float src, Span dst) { if (Avx.IsSupported) { - AvxIntrinsics.ScaleU(a, dst); + AvxIntrinsics.ScaleU(src, dst); } else if (Sse.IsSupported) { - SseIntrinsics.ScaleU(a, dst); + SseIntrinsics.ScaleU(src, dst); } else { for (int i = 0; i < dst.Length; i++) { - dst[i] *= a; + dst[i] *= src; } } } // dst = a * src - public static void Scale(float a, float[] src, float[] dst, int count) + public static void Scale(float src, float[] src, float[] dst, int count) { Contracts.AssertNonEmpty(src); Contracts.AssertNonEmpty(dst); From 70226ba76ace1ff56eeed1a94763e18ab7216033 Mon Sep 17 00:00:00 2001 From: J W Date: Tue, 25 Sep 2018 06:09:42 -0400 Subject: [PATCH 03/19] Update parameter names --- .../CpuMathUtils.netcoreapp.cs | 562 +++++++++--------- 1 file changed, 281 insertions(+), 281 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index c70d21dd88..7bc714ae8c 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -24,172 +24,172 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Avx.IsSupported ? Vector256Alignment : (Sse.IsSupported ? Vector128Alignment : FloatAlignment); - public static void MatTimesSrc(bool transpose, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) + public static void MatTimessource(bool transpose, bool add, AlignedArray mat, AlignedArray source, AlignedArray destination, int crun) { - Contracts.Assert(mat.Size == dst.Size * src.Size); + Contracts.Assert(mat.Size == destination.Size * source.Size); Contracts.Assert(crun >= 0); if (Avx.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= dst.Size); - AvxIntrinsics.MatMulX(add, mat, src, dst, crun, src.Size); + Contracts.Assert(crun <= destination.Size); + AvxIntrinsics.MatMulX(add, mat, source, destination, crun, source.Size); } else { - Contracts.Assert(crun <= src.Size); - AvxIntrinsics.MatMulTranX(add, mat, src, dst, dst.Size, crun); + Contracts.Assert(crun <= source.Size); + AvxIntrinsics.MatMulTranX(add, mat, source, destination, destination.Size, crun); } } else if (Sse.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= dst.Size); - SseIntrinsics.MatMulA(add, mat, src, dst, crun, src.Size); + Contracts.Assert(crun <= destination.Size); + SseIntrinsics.MatMulA(add, mat, source, destination, crun, source.Size); } else { - Contracts.Assert(crun <= src.Size); - SseIntrinsics.MatMulTranA(add, mat, src, dst, dst.Size, crun); + Contracts.Assert(crun <= source.Size); + SseIntrinsics.MatMulTranA(add, mat, source, destination, destination.Size, crun); } } else { if (!transpose) { - Contracts.Assert(crun <= dst.Size); + Contracts.Assert(crun <= destination.Size); for (int i = 0; i < crun; i++) { float dotProduct = 0; - for (int j = 0; j < src.Size; j++) + for (int j = 0; j < source.Size; j++) { - dotProduct += mat[i * src.Size + j] * src[j]; + dotProduct += mat[i * source.Size + j] * source[j]; } if (add) { - dst[i] += dotProduct; + destination[i] += dotProduct; } else { - dst[i] = dotProduct; + destination[i] = dotProduct; } } } else { - Contracts.Assert(crun <= src.Size); - for (int i = 0; i < dst.Size; i++) + Contracts.Assert(crun <= source.Size); + for (int i = 0; i < destination.Size; i++) { float dotProduct = 0; for (int j = 0; j < crun; j++) { - dotProduct += mat[j * src.Size + i] * src[j]; + dotProduct += mat[j * source.Size + i] * source[j]; } if (add) { - dst[i] += dotProduct; + destination[i] += dotProduct; } else { - dst[i] = dotProduct; + destination[i] = dotProduct; } } } } } - public static void MatTimesSrc(bool transpose, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, - int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) + public static void MatTimessource(bool transpose, bool add, AlignedArray mat, int[] rgpossource, AlignedArray sourceValues, + int posMin, int iposMin, int iposLim, AlignedArray destination, int crun) { - Contracts.AssertValue(rgposSrc); + Contracts.AssertValue(rgpossource); Contracts.Assert(iposMin >= 0); Contracts.Assert(iposMin <= iposLim); - Contracts.Assert(iposLim <= rgposSrc.Length); - Contracts.Assert(mat.Size == dst.Size * srcValues.Size); + Contracts.Assert(iposLim <= rgpossource.Length); + Contracts.Assert(mat.Size == destination.Size * sourceValues.Size); if (iposMin >= iposLim) { if (!add) - dst.ZeroItems(); + destination.ZeroItems(); return; } - Contracts.AssertNonEmpty(rgposSrc); + Contracts.AssertNonEmpty(rgpossource); Contracts.Assert(crun >= 0); if (Avx.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= dst.Size); - AvxIntrinsics.MatMulPX(add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size); + Contracts.Assert(crun <= destination.Size); + AvxIntrinsics.MatMulPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, crun, sourceValues.Size); } else { - Contracts.Assert(crun <= srcValues.Size); - AvxIntrinsics.MatMulTranPX(add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, dst.Size); + Contracts.Assert(crun <= sourceValues.Size); + AvxIntrinsics.MatMulTranPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, destination.Size); } } else if (Sse.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= dst.Size); - SseIntrinsics.MatMulPA(add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun, srcValues.Size); + Contracts.Assert(crun <= destination.Size); + SseIntrinsics.MatMulPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, crun, sourceValues.Size); } else { - Contracts.Assert(crun <= srcValues.Size); - SseIntrinsics.MatMulTranPA(add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, dst.Size); + Contracts.Assert(crun <= sourceValues.Size); + SseIntrinsics.MatMulTranPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, destination.Size); } } else { if (!transpose) { - Contracts.Assert(crun <= dst.Size); + Contracts.Assert(crun <= destination.Size); for (int i = 0; i < crun; i++) { float dotProduct = 0; for (int j = iposMin; j < iposLim; j++) { - int col = rgposSrc[j] - posMin; - dotProduct += mat[i * srcValues.Size + col] * srcValues[col]; + int col = rgpossource[j] - posMin; + dotProduct += mat[i * sourceValues.Size + col] * sourceValues[col]; } if (add) { - dst[i] += dotProduct; + destination[i] += dotProduct; } else { - dst[i] = dotProduct; + destination[i] = dotProduct; } } } else { - Contracts.Assert(crun <= srcValues.Size); - for (int i = 0; i < dst.Size; i++) + Contracts.Assert(crun <= sourceValues.Size); + for (int i = 0; i < destination.Size; i++) { float dotProduct = 0; for (int j = iposMin; j < iposLim; j++) { - int col = rgposSrc[j] - posMin; - dotProduct += mat[col * dst.Size + i] * srcValues[col]; + int col = rgpossource[j] - posMin; + dotProduct += mat[col * destination.Size + i] * sourceValues[col]; } if (add) { - dst[i] += dotProduct; + destination[i] += dotProduct; } else { - dst[i] = dotProduct; + destination[i] = dotProduct; } } @@ -197,584 +197,584 @@ public static void MatTimesSrc(bool transpose, bool add, AlignedArray mat, int[] } } - public static void Add(float src, float[] dst, int count) + public static void Add(float source, float[] destination, int count) { - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= destination.Length); - Add(src, new Span(dst, 0, count)); + Add(source, new Span(destination, 0, count)); } - private static void Add(float src, Span dst) + private static void Add(float source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddScalarU(src, dst); + AvxIntrinsics.AddScalarU(source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddScalarU(src, dst); + SseIntrinsics.AddScalarU(source, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] += src; + destination[i] += source; } } } - public static void Scale(float src, float[] dst, int count) + public static void Scale(float source, float[] destination, int count) { - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= destination.Length); - Scale(src, new Span(dst, 0, count)); + Scale(source, new Span(destination, 0, count)); } - public static void Scale(float src, float[] dst, int offset, int count) + public static void Scale(float source, float[] destination, int offset, int count) { - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset < (dst.Length - count)); + Contracts.Assert(offset < (destination.Length - count)); - Scale(src, new Span(dst, offset, count)); + Scale(source, new Span(destination, offset, count)); } - private static void Scale(float src, Span dst) + private static void Scale(float source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.ScaleU(src, dst); + AvxIntrinsics.ScaleU(source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScaleU(src, dst); + SseIntrinsics.ScaleU(source, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] *= src; + destination[i] *= source; } } } - // dst = a * src - public static void Scale(float src, float[] src, float[] dst, int count) + // destination = a * source + public static void Scale(float source, float[] source, float[] destination, int count) { - Contracts.AssertNonEmpty(src); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(source); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= source.Length); + Contracts.Assert(count <= destination.Length); - Scale(a, new Span(src, 0, count), new Span(dst, 0, count)); + Scale(a, new Span(source, 0, count), new Span(destination, 0, count)); } - private static void Scale(float a, Span src, Span dst) + private static void Scale(float a, Span source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.ScaleSrcU(a, src, dst); + AvxIntrinsics.ScalesourceU(a, source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScaleSrcU(a, src, dst); + SseIntrinsics.ScalesourceU(a, source, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] = a * src[i]; + destination[i] = a * source[i]; } } } - // dst[i] = a * (dst[i] + b) - public static void ScaleAdd(float a, float b, float[] dst, int count) + // destination[i] = a * (destination[i] + b) + public static void ScaleAdd(float a, float b, float[] destination, int count) { - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= destination.Length); - ScaleAdd(a, b, new Span(dst, 0, count)); + ScaleAdd(a, b, new Span(destination, 0, count)); } - private static void ScaleAdd(float a, float b, Span dst) + private static void ScaleAdd(float a, float b, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.ScaleAddU(a, b, dst); + AvxIntrinsics.ScaleAddU(a, b, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScaleAddU(a, b, dst); + SseIntrinsics.ScaleAddU(a, b, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] = a * (dst[i] + b); + destination[i] = a * (destination[i] + b); } } } - public static void AddScale(float a, float[] src, float[] dst, int count) + public static void AddScale(float a, float[] source, float[] destination, int count) { - Contracts.AssertNonEmpty(src); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(source); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= source.Length); + Contracts.Assert(count <= destination.Length); - AddScale(a, new Span(src, 0, count), new Span(dst, 0, count)); + AddScale(a, new Span(source, 0, count), new Span(destination, 0, count)); } - public static void AddScale(float a, float[] src, float[] dst, int dstOffset, int count) + public static void AddScale(float a, float[] source, float[] destination, int dstOffset, int count) { - Contracts.AssertNonEmpty(src); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(source); + Contracts.AssertNonEmpty(destination); Contracts.Assert(dstOffset >= 0); - Contracts.Assert(dstOffset < dst.Length); + Contracts.Assert(dstOffset < destination.Length); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); - Contracts.Assert(count <= (dst.Length - dstOffset)); + Contracts.Assert(count <= source.Length); + Contracts.Assert(count <= (destination.Length - dstOffset)); - AddScale(a, new Span(src, 0, count), new Span(dst, dstOffset, count)); + AddScale(a, new Span(source, 0, count), new Span(destination, dstOffset, count)); } - private static void AddScale(float a, Span src, Span dst) + private static void AddScale(float a, Span source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleU(a, src, dst); + AvxIntrinsics.AddScaleU(a, source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleU(a, src, dst); + SseIntrinsics.AddScaleU(a, source, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] += a * src[i]; + destination[i] += a * source[i]; } } } - public static void AddScale(float a, float[] src, int[] indices, float[] dst, int count) + public static void AddScale(float a, float[] source, int[] indices, float[] destination, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count < dst.Length); + Contracts.Assert(count < destination.Length); - AddScale(a, new Span(src), new Span(indices, 0, count), new Span(dst)); + AddScale(a, new Span(source), new Span(indices, 0, count), new Span(destination)); } - public static void AddScale(float a, float[] src, int[] indices, float[] dst, int dstOffset, int count) + public static void AddScale(float a, float[] source, int[] indices, float[] destination, int dstOffset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(dstOffset >= 0); - Contracts.Assert(dstOffset < dst.Length); + Contracts.Assert(dstOffset < destination.Length); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count < (dst.Length - dstOffset)); + Contracts.Assert(count < (destination.Length - dstOffset)); - AddScale(a, new Span(src), new Span(indices, 0, count), - new Span(dst, dstOffset, dst.Length - dstOffset)); + AddScale(a, new Span(source), new Span(indices, 0, count), + new Span(destination, dstOffset, destination.Length - dstOffset)); } - private static void AddScale(float a, Span src, Span indices, Span dst) + private static void AddScale(float a, Span source, Span indices, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleSU(a, src, indices, dst); + AvxIntrinsics.AddScaleSU(a, source, indices, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleSU(a, src, indices, dst); + SseIntrinsics.AddScaleSU(a, source, indices, destination); } else { for (int i = 0; i < indices.Length; i++) { int index = indices[i]; - dst[index] += a * src[i]; + destination[index] += a * source[i]; } } } - public static void AddScaleCopy(float a, float[] src, float[] dst, float[] res, int count) + public static void AddScaleCopy(float a, float[] source, float[] destination, float[] res, int count) { - Contracts.AssertNonEmpty(src); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(source); + Contracts.AssertNonEmpty(destination); Contracts.AssertNonEmpty(res); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= source.Length); + Contracts.Assert(count <= destination.Length); Contracts.Assert(count <= res.Length); - AddScaleCopy(a, new Span(src, 0, count), new Span(dst, 0, count), new Span(res, 0, count)); + AddScaleCopy(a, new Span(source, 0, count), new Span(destination, 0, count), new Span(res, 0, count)); } - private static void AddScaleCopy(float a, Span src, Span dst, Span res) + private static void AddScaleCopy(float a, Span source, Span destination, Span res) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleCopyU(a, src, dst, res); + AvxIntrinsics.AddScaleCopyU(a, source, destination, res); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleCopyU(a, src, dst, res); + SseIntrinsics.AddScaleCopyU(a, source, destination, res); } else { for (int i = 0; i < res.Length; i++) { - res[i] = a * src[i] + dst[i]; + res[i] = a * source[i] + destination[i]; } } } - public static void Add(float[] src, float[] dst, int count) + public static void Add(float[] source, float[] destination, int count) { - Contracts.AssertNonEmpty(src); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(source); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); - Contracts.Assert(count <= dst.Length); + Contracts.Assert(count <= source.Length); + Contracts.Assert(count <= destination.Length); - Add(new Span(src, 0, count), new Span(dst, 0, count)); + Add(new Span(source, 0, count), new Span(destination, 0, count)); } - private static void Add(Span src, Span dst) + private static void Add(Span source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddU(src, dst); + AvxIntrinsics.AddU(source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddU(src, dst); + SseIntrinsics.AddU(source, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] += src[i]; + destination[i] += source[i]; } } } - public static void Add(float[] src, int[] indices, float[] dst, int count) + public static void Add(float[] source, int[] indices, float[] destination, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count < dst.Length); + Contracts.Assert(count < destination.Length); - Add(new Span(src), new Span(indices, 0, count), new Span(dst)); + Add(new Span(source), new Span(indices, 0, count), new Span(destination)); } - public static void Add(float[] src, int[] indices, float[] dst, int dstOffset, int count) + public static void Add(float[] source, int[] indices, float[] destination, int dstOffset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(destination); Contracts.Assert(dstOffset >= 0); - Contracts.Assert(dstOffset < dst.Length); + Contracts.Assert(dstOffset < destination.Length); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count <= (dst.Length - dstOffset)); + Contracts.Assert(count <= (destination.Length - dstOffset)); - Add(new Span(src), new Span(indices, 0, count), - new Span(dst, dstOffset, dst.Length - dstOffset)); + Add(new Span(source), new Span(indices, 0, count), + new Span(destination, dstOffset, destination.Length - dstOffset)); } - private static void Add(Span src, Span indices, Span dst) + private static void Add(Span source, Span indices, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddSU(src, indices, dst); + AvxIntrinsics.AddSU(source, indices, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddSU(src, indices, dst); + SseIntrinsics.AddSU(source, indices, destination); } else { for (int i = 0; i < indices.Length; i++) { int index = indices[i]; - dst[index] += src[i]; + destination[index] += source[i]; } } } - public static void MulElementWise(float[] src1, float[] src2, float[] dst, int count) + public static void MulElementWise(float[] source1, float[] source2, float[] destination, int count) { - Contracts.AssertNonEmpty(src1); - Contracts.AssertNonEmpty(src2); - Contracts.AssertNonEmpty(dst); + Contracts.AssertNonEmpty(source1); + Contracts.AssertNonEmpty(source2); + Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= src1.Length); - Contracts.Assert(count <= src2.Length); + Contracts.Assert(count <= source1.Length); + Contracts.Assert(count <= source2.Length); - MulElementWise(new Span(src1, 0, count), new Span(src2, 0, count), - new Span(dst, 0, count)); + MulElementWise(new Span(source1, 0, count), new Span(source2, 0, count), + new Span(destination, 0, count)); } - private static void MulElementWise(Span src1, Span src2, Span dst) + private static void MulElementWise(Span source1, Span source2, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.MulElementWiseU(src1, src2, dst); + AvxIntrinsics.MulElementWiseU(source1, source2, destination); } else if (Sse.IsSupported) { - SseIntrinsics.MulElementWiseU(src1, src2, dst); + SseIntrinsics.MulElementWiseU(source1, source2, destination); } else { - for (int i = 0; i < dst.Length; i++) + for (int i = 0; i < destination.Length; i++) { - dst[i] = src1[i] * src2[i]; + destination[i] = source1[i] * source2[i]; } } } - public static float Sum(float[] src, int count) + public static float Sum(float[] source, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); - return Sum(new Span(src, 0, count)); + return Sum(new Span(source, 0, count)); } - public static float Sum(float[] src, int offset, int count) + public static float Sum(float[] source, int offset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (src.Length - count)); + Contracts.Assert(offset <= (source.Length - count)); - return Sum(new Span(src, offset, count)); + return Sum(new Span(source, offset, count)); } - private static float Sum(Span src) + private static float Sum(Span source) { if (Avx.IsSupported) { - return AvxIntrinsics.SumU(src); + return AvxIntrinsics.SumU(source); } else if (Sse.IsSupported) { - return SseIntrinsics.SumU(src); + return SseIntrinsics.SumU(source); } else { float sum = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - sum += src[i]; + sum += source[i]; } return sum; } } - public static float SumSq(float[] src, int count) + public static float SumSq(float[] source, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); - return SumSq(new Span(src, 0, count)); + return SumSq(new Span(source, 0, count)); } - public static float SumSq(float[] src, int offset, int count) + public static float SumSq(float[] source, int offset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (src.Length - count)); + Contracts.Assert(offset <= (source.Length - count)); - return SumSq(new Span(src, offset, count)); + return SumSq(new Span(source, offset, count)); } - private static float SumSq(Span src) + private static float SumSq(Span source) { if (Avx.IsSupported) { - return AvxIntrinsics.SumSqU(src); + return AvxIntrinsics.SumSqU(source); } else if (Sse.IsSupported) { - return SseIntrinsics.SumSqU(src); + return SseIntrinsics.SumSqU(source); } else { float result = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - result += src[i] * src[i]; + result += source[i] * source[i]; } return result; } } - public static float SumSq(float mean, float[] src, int offset, int count) + public static float SumSq(float mean, float[] source, int offset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (src.Length - count)); + Contracts.Assert(offset <= (source.Length - count)); - return SumSq(mean, new Span(src, offset, count)); + return SumSq(mean, new Span(source, offset, count)); } - private static float SumSq(float mean, Span src) + private static float SumSq(float mean, Span source) { if (Avx.IsSupported) { - return (mean == 0) ? AvxIntrinsics.SumSqU(src) : AvxIntrinsics.SumSqDiffU(mean, src); + return (mean == 0) ? AvxIntrinsics.SumSqU(source) : AvxIntrinsics.SumSqDiffU(mean, source); } else if (Sse.IsSupported) { - return (mean == 0) ? SseIntrinsics.SumSqU(src) : SseIntrinsics.SumSqDiffU(mean, src); + return (mean == 0) ? SseIntrinsics.SumSqU(source) : SseIntrinsics.SumSqDiffU(mean, source); } else { float result = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - result += (src[i] - mean) * (src[i] - mean); + result += (source[i] - mean) * (source[i] - mean); } return result; } } - public static float SumAbs(float[] src, int count) + public static float SumAbs(float[] source, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); - return SumAbs(new Span(src, 0, count)); + return SumAbs(new Span(source, 0, count)); } - public static float SumAbs(float[] src, int offset, int count) + public static float SumAbs(float[] source, int offset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (src.Length - count)); + Contracts.Assert(offset <= (source.Length - count)); - return SumAbs(new Span(src, offset, count)); + return SumAbs(new Span(source, offset, count)); } - private static float SumAbs(Span src) + private static float SumAbs(Span source) { if (Avx.IsSupported) { - return AvxIntrinsics.SumAbsU(src); + return AvxIntrinsics.SumAbsU(source); } else if (Sse.IsSupported) { - return SseIntrinsics.SumAbsU(src); + return SseIntrinsics.SumAbsU(source); } else { float sum = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - sum += Math.Abs(src[i]); + sum += Math.Abs(source[i]); } return sum; } } - public static float SumAbs(float mean, float[] src, int offset, int count) + public static float SumAbs(float mean, float[] source, int offset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (src.Length - count)); + Contracts.Assert(offset <= (source.Length - count)); - return SumAbs(mean, new Span(src, offset, count)); + return SumAbs(mean, new Span(source, offset, count)); } - private static float SumAbs(float mean, Span src) + private static float SumAbs(float mean, Span source) { if (Avx.IsSupported) { - return (mean == 0) ? AvxIntrinsics.SumAbsU(src) : AvxIntrinsics.SumAbsDiffU(mean, src); + return (mean == 0) ? AvxIntrinsics.SumAbsU(source) : AvxIntrinsics.SumAbsDiffU(mean, source); } else if (Sse.IsSupported) { - return (mean == 0) ? SseIntrinsics.SumAbsU(src) : SseIntrinsics.SumAbsDiffU(mean, src); + return (mean == 0) ? SseIntrinsics.SumAbsU(source) : SseIntrinsics.SumAbsDiffU(mean, source); } else { float sum = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - sum += Math.Abs(src[i] - mean); + sum += Math.Abs(source[i] - mean); } return sum; } } - public static float MaxAbs(float[] src, int count) + public static float MaxAbs(float[] source, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); - return MaxAbs(new Span(src, 0, count)); + return MaxAbs(new Span(source, 0, count)); } - public static float MaxAbs(float[] src, int offset, int count) + public static float MaxAbs(float[] source, int offset, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (src.Length - count)); + Contracts.Assert(offset <= (source.Length - count)); - return MaxAbs(new Span(src, offset, count)); + return MaxAbs(new Span(source, offset, count)); } - private static float MaxAbs(Span src) + private static float MaxAbs(Span source) { if (Avx.IsSupported) { - return AvxIntrinsics.MaxAbsU(src); + return AvxIntrinsics.MaxAbsU(source); } else if (Sse.IsSupported) { - return SseIntrinsics.MaxAbsU(src); + return SseIntrinsics.MaxAbsU(source); } else { float max = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - float abs = Math.Abs(src[i]); + float abs = Math.Abs(source[i]); if (abs > max) { max = abs; @@ -784,31 +784,31 @@ private static float MaxAbs(Span src) } } - public static float MaxAbsDiff(float mean, float[] src, int count) + public static float MaxAbsDiff(float mean, float[] source, int count) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); - return MaxAbsDiff(mean, new Span(src, 0, count)); + return MaxAbsDiff(mean, new Span(source, 0, count)); } - private static float MaxAbsDiff(float mean, Span src) + private static float MaxAbsDiff(float mean, Span source) { if (Avx.IsSupported) { - return AvxIntrinsics.MaxAbsDiffU(mean, src); + return AvxIntrinsics.MaxAbsDiffU(mean, source); } else if (Sse.IsSupported) { - return SseIntrinsics.MaxAbsDiffU(mean, src); + return SseIntrinsics.MaxAbsDiffU(mean, source); } else { float max = 0; - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - float abs = Math.Abs(src[i] - mean); + float abs = Math.Abs(source[i] - mean); if (abs > max) { max = abs; @@ -947,24 +947,24 @@ private static float L2DistSquared(Span a, Span b) } } - public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices) + public static void ZeroMatrixItems(AlignedArray destination, int ccol, int cfltRow, int[] indices) { Contracts.Assert(ccol > 0); Contracts.Assert(ccol <= cfltRow); if (ccol == cfltRow) { - ZeroItemsU(dst, dst.Size, indices, indices.Length); + ZeroItemsU(destination, destination.Size, indices, indices.Length); } else { - ZeroMatrixItemsCore(dst, dst.Size, ccol, cfltRow, indices, indices.Length); + ZeroMatrixItemsCore(destination, destination.Size, ccol, cfltRow, indices, indices.Length); } } - private static unsafe void ZeroItemsU(AlignedArray dst, int c, int[] indices, int cindices) + private static unsafe void ZeroItemsU(AlignedArray destination, int c, int[] indices, int cindices) { - fixed (float* pdst = &dst.Items[0]) + fixed (float* pdst = &destination.Items[0]) fixed (int* pidx = &indices[0]) { for (int i = 0; i < cindices; ++i) @@ -977,9 +977,9 @@ private static unsafe void ZeroItemsU(AlignedArray dst, int c, int[] indices, in } } - private static unsafe void ZeroMatrixItemsCore(AlignedArray dst, int c, int ccol, int cfltRow, int[] indices, int cindices) + private static unsafe void ZeroMatrixItemsCore(AlignedArray destination, int c, int ccol, int cfltRow, int[] indices, int cindices) { - fixed (float* pdst = &dst.Items[0]) + fixed (float* pdst = &destination.Items[0]) fixed (int* pidx = &indices[0]) { int ivLogMin = 0; @@ -1012,34 +1012,34 @@ private static unsafe void ZeroMatrixItemsCore(AlignedArray dst, int c, int ccol } } - public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] src, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] source, float threshold, float[] v, float[] w) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(v); Contracts.AssertNonEmpty(w); Contracts.Assert(length > 0); - Contracts.Assert(length <= src.Length); + Contracts.Assert(length <= source.Length); Contracts.Assert(length <= v.Length); Contracts.Assert(length <= w.Length); - SdcaL1UpdateDense(primalUpdate, new Span(src, 0, length), threshold, new Span(v, 0, length), new Span(w, 0, length)); + SdcaL1UpdateDense(primalUpdate, new Span(source, 0, length), threshold, new Span(v, 0, length), new Span(w, 0, length)); } - private static void SdcaL1UpdateDense(float primalUpdate, Span src, float threshold, Span v, Span w) + private static void SdcaL1UpdateDense(float primalUpdate, Span source, float threshold, Span v, Span w) { if (Avx.IsSupported) { - AvxIntrinsics.SdcaL1UpdateU(primalUpdate, src, threshold, v, w); + AvxIntrinsics.SdcaL1UpdateU(primalUpdate, source, threshold, v, w); } else if (Sse.IsSupported) { - SseIntrinsics.SdcaL1UpdateU(primalUpdate, src, threshold, v, w); + SseIntrinsics.SdcaL1UpdateU(primalUpdate, source, threshold, v, w); } else { - for (int i = 0; i < src.Length; i++) + for (int i = 0; i < source.Length; i++) { - v[i] += src[i] * primalUpdate; + v[i] += source[i] * primalUpdate; float value = v[i]; w[i] = Math.Abs(value) > threshold ? (value > 0 ? value - threshold : value + threshold) : 0; } @@ -1047,38 +1047,38 @@ private static void SdcaL1UpdateDense(float primalUpdate, Span src, float } // REVIEW NEEDED: The second argument "length" is unused even in the existing code. - public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] src, int[] indices, int count, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) { - Contracts.AssertNonEmpty(src); + Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); Contracts.AssertNonEmpty(v); Contracts.AssertNonEmpty(w); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); Contracts.Assert(count < length); Contracts.Assert(length <= v.Length); Contracts.Assert(length <= w.Length); - SdcaL1UpdateSparse(primalUpdate, new Span(src, 0, count), new Span(indices, 0, count), threshold, new Span(v), new Span(w)); + SdcaL1UpdateSparse(primalUpdate, new Span(source, 0, count), new Span(indices, 0, count), threshold, new Span(v), new Span(w)); } - private static void SdcaL1UpdateSparse(float primalUpdate, Span src, Span indices, float threshold, Span v, Span w) + private static void SdcaL1UpdateSparse(float primalUpdate, Span source, Span indices, float threshold, Span v, Span w) { if (Avx.IsSupported) { - AvxIntrinsics.SdcaL1UpdateSU(primalUpdate, src, indices, threshold, v, w); + AvxIntrinsics.SdcaL1UpdateSU(primalUpdate, source, indices, threshold, v, w); } else if (Sse.IsSupported) { - SseIntrinsics.SdcaL1UpdateSU(primalUpdate, src, indices, threshold, v, w); + SseIntrinsics.SdcaL1UpdateSU(primalUpdate, source, indices, threshold, v, w); } else { for (int i = 0; i < indices.Length; i++) { int index = indices[i]; - v[index] += src[i] * primalUpdate; + v[index] += source[i] * primalUpdate; float value = v[index]; w[index] = Math.Abs(value) > threshold ? (value > 0 ? value - threshold : value + threshold) : 0; } From 5926e602166e67ce76bee91ac73d3c7d35909cbc Mon Sep 17 00:00:00 2001 From: J W Date: Tue, 25 Sep 2018 10:58:14 -0400 Subject: [PATCH 04/19] Update naming of method --- src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 7bc714ae8c..d05acf998d 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -24,7 +24,7 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Avx.IsSupported ? Vector256Alignment : (Sse.IsSupported ? Vector128Alignment : FloatAlignment); - public static void MatTimessource(bool transpose, bool add, AlignedArray mat, AlignedArray source, AlignedArray destination, int crun) + public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, AlignedArray source, AlignedArray destination, int crun) { Contracts.Assert(mat.Size == destination.Size * source.Size); Contracts.Assert(crun >= 0); @@ -102,7 +102,7 @@ public static void MatTimessource(bool transpose, bool add, AlignedArray mat, Al } } - public static void MatTimessource(bool transpose, bool add, AlignedArray mat, int[] rgpossource, AlignedArray sourceValues, + public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, int[] rgpossource, AlignedArray sourceValues, int posMin, int iposMin, int iposLim, AlignedArray destination, int crun) { Contracts.AssertValue(rgpossource); From 6ee96697199dae0ebcc95edd91ac738077ac5515 Mon Sep 17 00:00:00 2001 From: J W Date: Tue, 25 Sep 2018 12:38:53 -0400 Subject: [PATCH 05/19] Update argument names due to PR feedback --- .../CpuMathUtils.netcoreapp.cs | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index d05acf998d..1591d4ecc5 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -24,43 +24,43 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Avx.IsSupported ? Vector256Alignment : (Sse.IsSupported ? Vector128Alignment : FloatAlignment); - public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, AlignedArray source, AlignedArray destination, int crun) + public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, AlignedArray source, AlignedArray destination, int stride) { Contracts.Assert(mat.Size == destination.Size * source.Size); - Contracts.Assert(crun >= 0); + Contracts.Assert(stride >= 0); if (Avx.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= destination.Size); - AvxIntrinsics.MatMulX(add, mat, source, destination, crun, source.Size); + Contracts.Assert(stride <= destination.Size); + AvxIntrinsics.MatMulX(add, mat, source, destination, stride, source.Size); } else { - Contracts.Assert(crun <= source.Size); - AvxIntrinsics.MatMulTranX(add, mat, source, destination, destination.Size, crun); + Contracts.Assert(stride <= source.Size); + AvxIntrinsics.MatMulTranX(add, mat, source, destination, destination.Size, stride); } } else if (Sse.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= destination.Size); - SseIntrinsics.MatMulA(add, mat, source, destination, crun, source.Size); + Contracts.Assert(stride <= destination.Size); + SseIntrinsics.MatMulA(add, mat, source, destination, stride, source.Size); } else { - Contracts.Assert(crun <= source.Size); - SseIntrinsics.MatMulTranA(add, mat, source, destination, destination.Size, crun); + Contracts.Assert(stride <= source.Size); + SseIntrinsics.MatMulTranA(add, mat, source, destination, destination.Size, stride); } } else { if (!transpose) { - Contracts.Assert(crun <= destination.Size); - for (int i = 0; i < crun; i++) + Contracts.Assert(stride <= destination.Size); + for (int i = 0; i < stride; i++) { float dotProduct = 0; for (int j = 0; j < source.Size; j++) @@ -80,11 +80,11 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, Al } else { - Contracts.Assert(crun <= source.Size); + Contracts.Assert(stride <= source.Size); for (int i = 0; i < destination.Size; i++) { float dotProduct = 0; - for (int j = 0; j < crun; j++) + for (int j = 0; j < stride; j++) { dotProduct += mat[j * source.Size + i] * source[j]; } @@ -102,8 +102,8 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, Al } } - public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, int[] rgpossource, AlignedArray sourceValues, - int posMin, int iposMin, int iposLim, AlignedArray destination, int crun) + public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, int[] rgpossource, AlignedArray sourceValues, + int posMin, int iposMin, int iposLim, AlignedArray destination, int stride) { Contracts.AssertValue(rgpossource); Contracts.Assert(iposMin >= 0); @@ -119,18 +119,18 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, in } Contracts.AssertNonEmpty(rgpossource); - Contracts.Assert(crun >= 0); + Contracts.Assert(stride >= 0); if (Avx.IsSupported) { if (!transpose) { - Contracts.Assert(crun <= destination.Size); - AvxIntrinsics.MatMulPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, crun, sourceValues.Size); + Contracts.Assert(stride <= destination.Size); + AvxIntrinsics.MatMulPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, stride, sourceValues.Size); } else { - Contracts.Assert(crun <= sourceValues.Size); + Contracts.Assert(stride <= sourceValues.Size); AvxIntrinsics.MatMulTranPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, destination.Size); } } @@ -138,12 +138,12 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, in { if (!transpose) { - Contracts.Assert(crun <= destination.Size); - SseIntrinsics.MatMulPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, crun, sourceValues.Size); + Contracts.Assert(stride <= destination.Size); + SseIntrinsics.MatMulPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, stride, sourceValues.Size); } else { - Contracts.Assert(crun <= sourceValues.Size); + Contracts.Assert(stride <= sourceValues.Size); SseIntrinsics.MatMulTranPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, destination.Size); } } @@ -151,8 +151,8 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, in { if (!transpose) { - Contracts.Assert(crun <= destination.Size); - for (int i = 0; i < crun; i++) + Contracts.Assert(stride <= destination.Size); + for (int i = 0; i < stride; i++) { float dotProduct = 0; for (int j = iposMin; j < iposLim; j++) @@ -173,7 +173,7 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, in } else { - Contracts.Assert(crun <= sourceValues.Size); + Contracts.Assert(stride <= sourceValues.Size); for (int i = 0; i < destination.Size; i++) { float dotProduct = 0; @@ -197,74 +197,74 @@ public static void MatTimesSource(bool transpose, bool add, AlignedArray mat, in } } - public static void Add(float source, float[] destination, int count) + public static void Add(float value, float[] destination, int count) { Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); Contracts.Assert(count <= destination.Length); - Add(source, new Span(destination, 0, count)); + Add(value, new Span(destination, 0, count)); } - private static void Add(float source, Span destination) + private static void Add(float value, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddScalarU(source, destination); + AvxIntrinsics.AddScalarU(value, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddScalarU(source, destination); + SseIntrinsics.AddScalarU(value, destination); } else { for (int i = 0; i < destination.Length; i++) { - destination[i] += source; + destination[i] += value; } } } - public static void Scale(float source, float[] destination, int count) + public static void Scale(float value, float[] destination, int count) { Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); Contracts.Assert(count <= destination.Length); - Scale(source, new Span(destination, 0, count)); + Scale(value, new Span(destination, 0, count)); } - public static void Scale(float source, float[] destination, int offset, int count) + public static void Scale(float value, float[] destination, int offset, int count) { Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); Contracts.Assert(offset >= 0); Contracts.Assert(offset < (destination.Length - count)); - Scale(source, new Span(destination, offset, count)); + Scale(value, new Span(destination, offset, count)); } - private static void Scale(float source, Span destination) + private static void Scale(float value, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.ScaleU(source, destination); + AvxIntrinsics.ScaleU(value, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScaleU(source, destination); + SseIntrinsics.ScaleU(value, destination); } else { for (int i = 0; i < destination.Length; i++) { - destination[i] *= source; + destination[i] *= value; } } } // destination = a * source - public static void Scale(float source, float[] source, float[] destination, int count) + public static void Scale(float value, float[] source, float[] destination, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(destination); @@ -272,7 +272,7 @@ public static void Scale(float source, float[] source, float[] destination, int Contracts.Assert(count <= source.Length); Contracts.Assert(count <= destination.Length); - Scale(a, new Span(source, 0, count), new Span(destination, 0, count)); + Scale(value, new Span(source, 0, count), new Span(destination, 0, count)); } private static void Scale(float a, Span source, Span destination) From 82542b74b595b0be9c3c433eaa1e6ae4406047d3 Mon Sep 17 00:00:00 2001 From: J W Date: Tue, 25 Sep 2018 18:36:32 -0400 Subject: [PATCH 06/19] Update more argument names --- .../CpuMathUtils.netcoreapp.cs | 136 +++++++++--------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 1591d4ecc5..2185ef237d 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -24,9 +24,9 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Avx.IsSupported ? Vector256Alignment : (Sse.IsSupported ? Vector128Alignment : FloatAlignment); - public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, AlignedArray source, AlignedArray destination, int stride) + public static void MatrixTimesSource(bool transpose, bool add, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) { - Contracts.Assert(mat.Size == destination.Size * source.Size); + Contracts.Assert(matrix.Size == destination.Size * source.Size); Contracts.Assert(stride >= 0); if (Avx.IsSupported) @@ -34,12 +34,12 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, if (!transpose) { Contracts.Assert(stride <= destination.Size); - AvxIntrinsics.MatMulX(add, mat, source, destination, stride, source.Size); + AvxIntrinsics.MatMulX(add, matrix, source, destination, stride, source.Size); } else { Contracts.Assert(stride <= source.Size); - AvxIntrinsics.MatMulTranX(add, mat, source, destination, destination.Size, stride); + AvxIntrinsics.MatMulTranX(add, matrix, source, destination, destination.Size, stride); } } else if (Sse.IsSupported) @@ -47,12 +47,12 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, if (!transpose) { Contracts.Assert(stride <= destination.Size); - SseIntrinsics.MatMulA(add, mat, source, destination, stride, source.Size); + SseIntrinsics.MatMulA(add, matrix, source, destination, stride, source.Size); } else { Contracts.Assert(stride <= source.Size); - SseIntrinsics.MatMulTranA(add, mat, source, destination, destination.Size, stride); + SseIntrinsics.MatMulTranA(add, matrix, source, destination, destination.Size, stride); } } else @@ -65,7 +65,7 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, float dotProduct = 0; for (int j = 0; j < source.Size; j++) { - dotProduct += mat[i * source.Size + j] * source[j]; + dotProduct += matrix[i * source.Size + j] * source[j]; } if (add) @@ -86,7 +86,7 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, float dotProduct = 0; for (int j = 0; j < stride; j++) { - dotProduct += mat[j * source.Size + i] * source[j]; + dotProduct += matrix[j * source.Size + i] * source[j]; } if (add) @@ -102,23 +102,23 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, } } - public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, int[] rgpossource, AlignedArray sourceValues, - int posMin, int iposMin, int iposLim, AlignedArray destination, int stride) + public static void MatrixTimesSource(bool transpose, bool add, AlignedArray matrix, int[] posSource, AlignedArray sourceValues, + int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) { - Contracts.AssertValue(rgpossource); + Contracts.AssertValue(posSource); Contracts.Assert(iposMin >= 0); - Contracts.Assert(iposMin <= iposLim); - Contracts.Assert(iposLim <= rgpossource.Length); - Contracts.Assert(mat.Size == destination.Size * sourceValues.Size); + Contracts.Assert(iposMin <= iposLimit); + Contracts.Assert(iposLimit <= posSource.Length); + Contracts.Assert(matrix.Size == destination.Size * sourceValues.Size); - if (iposMin >= iposLim) + if (iposMin >= iposLimit) { if (!add) destination.ZeroItems(); return; } - Contracts.AssertNonEmpty(rgpossource); + Contracts.AssertNonEmpty(posSource); Contracts.Assert(stride >= 0); if (Avx.IsSupported) @@ -126,12 +126,12 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, if (!transpose) { Contracts.Assert(stride <= destination.Size); - AvxIntrinsics.MatMulPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, stride, sourceValues.Size); + AvxIntrinsics.MatMulPX(add, matrix, posSource, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size); } else { Contracts.Assert(stride <= sourceValues.Size); - AvxIntrinsics.MatMulTranPX(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, destination.Size); + AvxIntrinsics.MatMulTranPX(add, matrix, posSource, sourceValues, posMin, iposMin, iposLimit, destination, destination.Size); } } else if (Sse.IsSupported) @@ -139,12 +139,12 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, if (!transpose) { Contracts.Assert(stride <= destination.Size); - SseIntrinsics.MatMulPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, stride, sourceValues.Size); + SseIntrinsics.MatMulPA(add, matrix, posSource, sourceValues, posMin, iposMin, iposLimit, destination, stride, sourceValues.Size); } else { Contracts.Assert(stride <= sourceValues.Size); - SseIntrinsics.MatMulTranPA(add, mat, rgpossource, sourceValues, posMin, iposMin, iposLim, destination, destination.Size); + SseIntrinsics.MatMulTranPA(add, matrix, posSource, sourceValues, posMin, iposMin, iposLimit, destination, destination.Size); } } else @@ -155,10 +155,10 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, for (int i = 0; i < stride; i++) { float dotProduct = 0; - for (int j = iposMin; j < iposLim; j++) + for (int j = iposMin; j < iposLimit; j++) { - int col = rgpossource[j] - posMin; - dotProduct += mat[i * sourceValues.Size + col] * sourceValues[col]; + int col = posSource[j] - posMin; + dotProduct += matrix[i * sourceValues.Size + col] * sourceValues[col]; } if (add) @@ -177,10 +177,10 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray mat, for (int i = 0; i < destination.Size; i++) { float dotProduct = 0; - for (int j = iposMin; j < iposLim; j++) + for (int j = iposMin; j < iposLimit; j++) { - int col = rgpossource[j] - posMin; - dotProduct += mat[col * destination.Size + i] * sourceValues[col]; + int col = posSource[j] - posMin; + dotProduct += matrix[col * destination.Size + i] * sourceValues[col]; } if (add) @@ -275,55 +275,55 @@ public static void Scale(float value, float[] source, float[] destination, int c Scale(value, new Span(source, 0, count), new Span(destination, 0, count)); } - private static void Scale(float a, Span source, Span destination) + private static void Scale(float value, Span source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.ScalesourceU(a, source, destination); + AvxIntrinsics.ScalesourceU(value, source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScalesourceU(a, source, destination); + SseIntrinsics.ScalesourceU(value, source, destination); } else { for (int i = 0; i < destination.Length; i++) { - destination[i] = a * source[i]; + destination[i] = value * source[i]; } } } // destination[i] = a * (destination[i] + b) - public static void ScaleAdd(float a, float b, float[] destination, int count) + public static void ScaleAdd(float scale, float addend, float[] destination, int count) { Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); Contracts.Assert(count <= destination.Length); - ScaleAdd(a, b, new Span(destination, 0, count)); + ScaleAdd(scale, addend, new Span(destination, 0, count)); } - private static void ScaleAdd(float a, float b, Span destination) + private static void ScaleAdd(float scale, float addend, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.ScaleAddU(a, b, destination); + AvxIntrinsics.ScaleAddU(scale, addend, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScaleAddU(a, b, destination); + SseIntrinsics.ScaleAddU(scale, addend, destination); } else { for (int i = 0; i < destination.Length; i++) { - destination[i] = a * (destination[i] + b); + destination[i] = scale * (destination[i] + addend); } } } - public static void AddScale(float a, float[] source, float[] destination, int count) + public static void AddScale(float scale, float[] source, float[] destination, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(destination); @@ -331,10 +331,10 @@ public static void AddScale(float a, float[] source, float[] destination, int co Contracts.Assert(count <= source.Length); Contracts.Assert(count <= destination.Length); - AddScale(a, new Span(source, 0, count), new Span(destination, 0, count)); + AddScale(scale, new Span(source, 0, count), new Span(destination, 0, count)); } - public static void AddScale(float a, float[] source, float[] destination, int dstOffset, int count) + public static void AddScale(float scale, float[] source, float[] destination, int dstOffset, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(destination); @@ -344,29 +344,29 @@ public static void AddScale(float a, float[] source, float[] destination, int ds Contracts.Assert(count <= source.Length); Contracts.Assert(count <= (destination.Length - dstOffset)); - AddScale(a, new Span(source, 0, count), new Span(destination, dstOffset, count)); + AddScale(scale, new Span(source, 0, count), new Span(destination, dstOffset, count)); } - private static void AddScale(float a, Span source, Span destination) + private static void AddScale(float scale, Span source, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleU(a, source, destination); + AvxIntrinsics.AddScaleU(scale, source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleU(a, source, destination); + SseIntrinsics.AddScaleU(scale, source, destination); } else { for (int i = 0; i < destination.Length; i++) { - destination[i] += a * source[i]; + destination[i] += scale * source[i]; } } } - public static void AddScale(float a, float[] source, int[] indices, float[] destination, int count) + public static void AddScale(float scale, float[] source, int[] indices, float[] destination, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); @@ -376,10 +376,10 @@ public static void AddScale(float a, float[] source, int[] indices, float[] dest Contracts.Assert(count <= indices.Length); Contracts.Assert(count < destination.Length); - AddScale(a, new Span(source), new Span(indices, 0, count), new Span(destination)); + AddScale(scale, new Span(source), new Span(indices, 0, count), new Span(destination)); } - public static void AddScale(float a, float[] source, int[] indices, float[] destination, int dstOffset, int count) + public static void AddScale(float scale, float[] source, int[] indices, float[] destination, int dstOffset, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); @@ -391,31 +391,31 @@ public static void AddScale(float a, float[] source, int[] indices, float[] dest Contracts.Assert(count <= indices.Length); Contracts.Assert(count < (destination.Length - dstOffset)); - AddScale(a, new Span(source), new Span(indices, 0, count), + AddScale(scale, new Span(source), new Span(indices, 0, count), new Span(destination, dstOffset, destination.Length - dstOffset)); } - private static void AddScale(float a, Span source, Span indices, Span destination) + private static void AddScale(float scale, Span source, Span indices, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleSU(a, source, indices, destination); + AvxIntrinsics.AddScaleSU(scale, source, indices, destination); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleSU(a, source, indices, destination); + SseIntrinsics.AddScaleSU(scale, source, indices, destination); } else { for (int i = 0; i < indices.Length; i++) { int index = indices[i]; - destination[index] += a * source[i]; + destination[index] += scale * source[i]; } } } - public static void AddScaleCopy(float a, float[] source, float[] destination, float[] res, int count) + public static void AddScaleCopy(float scale, float[] source, float[] destination, float[] res, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(destination); @@ -425,24 +425,24 @@ public static void AddScaleCopy(float a, float[] source, float[] destination, fl Contracts.Assert(count <= destination.Length); Contracts.Assert(count <= res.Length); - AddScaleCopy(a, new Span(source, 0, count), new Span(destination, 0, count), new Span(res, 0, count)); + AddScaleCopy(scale, new Span(source, 0, count), new Span(destination, 0, count), new Span(res, 0, count)); } - private static void AddScaleCopy(float a, Span source, Span destination, Span res) + private static void AddScaleCopy(float scale, Span source, Span destination, Span res) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleCopyU(a, source, destination, res); + AvxIntrinsics.AddScaleCopyU(scale, source, destination, res); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleCopyU(a, source, destination, res); + SseIntrinsics.AddScaleCopyU(scale, source, destination, res); } else { for (int i = 0; i < res.Length; i++) { - res[i] = a * source[i] + destination[i]; + res[i] = scale * source[i] + destination[i]; } } } @@ -526,34 +526,34 @@ private static void Add(Span source, Span indices, Span desti } } - public static void MulElementWise(float[] source1, float[] source2, float[] destination, int count) + public static void MulElementWise(float[] left, float[] right, float[] destination, int count) { - Contracts.AssertNonEmpty(source1); - Contracts.AssertNonEmpty(source2); + Contracts.AssertNonEmpty(left); + Contracts.AssertNonEmpty(right); Contracts.AssertNonEmpty(destination); Contracts.Assert(count > 0); - Contracts.Assert(count <= source1.Length); - Contracts.Assert(count <= source2.Length); + Contracts.Assert(count <= left.Length); + Contracts.Assert(count <= right.Length); - MulElementWise(new Span(source1, 0, count), new Span(source2, 0, count), + MulElementWise(new Span(left, 0, count), new Span(right, 0, count), new Span(destination, 0, count)); } - private static void MulElementWise(Span source1, Span source2, Span destination) + private static void MulElementWise(Span left, Span right, Span destination) { if (Avx.IsSupported) { - AvxIntrinsics.MulElementWiseU(source1, source2, destination); + AvxIntrinsics.MulElementWiseU(left, right, destination); } else if (Sse.IsSupported) { - SseIntrinsics.MulElementWiseU(source1, source2, destination); + SseIntrinsics.MulElementWiseU(left, right, destination); } else { for (int i = 0; i < destination.Length; i++) { - destination[i] = source1[i] * source2[i]; + destination[i] = left[i] * right[i]; } } } From 287f7fc22f4d60e639c4eff3834b85102feae87f Mon Sep 17 00:00:00 2001 From: J W Date: Thu, 27 Sep 2018 08:03:48 -0400 Subject: [PATCH 07/19] Remove unused argument --- src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 2185ef237d..557db00763 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -1047,7 +1047,7 @@ private static void SdcaL1UpdateDense(float primalUpdate, Span source, fl } // REVIEW NEEDED: The second argument "length" is unused even in the existing code. - public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateSparse(float primalUpdate, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); @@ -1056,9 +1056,6 @@ public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] so Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count < length); - Contracts.Assert(length <= v.Length); - Contracts.Assert(length <= w.Length); SdcaL1UpdateSparse(primalUpdate, new Span(source, 0, count), new Span(indices, 0, count), threshold, new Span(v), new Span(w)); } From d6a677df033a7431da13606ff1f01d542bb72a51 Mon Sep 17 00:00:00 2001 From: J W Date: Mon, 1 Oct 2018 12:41:46 -0400 Subject: [PATCH 08/19] Update net standard file --- .../CpuAligenedMathUtils.cs | 4 +- .../CpuMathUtils.netstandard.cs | 76 +- src/Microsoft.ML.Transforms/RffTransform.cs | 4 +- .../Common/EntryPoints/core_ep-list.tsv | 22 + .../Common/EntryPoints/core_manifest.json | 2411 +++++++++++++++-- .../UnitTests.cs | 16 +- 6 files changed, 2228 insertions(+), 305 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuAligenedMathUtils.cs b/src/Microsoft.ML.CpuMath/CpuAligenedMathUtils.cs index 30308f219d..e31a9f8909 100644 --- a/src/Microsoft.ML.CpuMath/CpuAligenedMathUtils.cs +++ b/src/Microsoft.ML.CpuMath/CpuAligenedMathUtils.cs @@ -89,7 +89,7 @@ public static void MatTimesSrc(bool add, ICpuFullMatrix mat, ICpuVector src, ICp bool colMajor = typeof(TMatrix) == typeof(CpuAlignedMatrixCol); AssertCompatible(mat, src, dst); var m = A(mat); - CpuMathUtils.MatTimesSrc(colMajor, add, m.Items, A(src).Items, A(dst).Items, m.RunCnt); + CpuMathUtils.MatrixTimesSource(colMajor, add, m.Items, A(src).Items, A(dst).Items, m.RunCnt); } /// @@ -108,7 +108,7 @@ public static void MatTranTimesSrc(bool add, ICpuFullMatrix mat, ICpuVector src, bool colMajor = typeof(TMatrix) == typeof(CpuAlignedMatrixCol); AssertCompatible(mat, dst, src); var m = A(mat); - CpuMathUtils.MatTimesSrc(!colMajor, add, m.Items, A(src).Items, A(dst).Items, m.RunCnt); + CpuMathUtils.MatrixTimesSource(!colMajor, add, m.Items, A(src).Items, A(dst).Items, m.RunCnt); } } diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index b35f171388..293a38acb8 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -15,77 +15,77 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Vector128Alignment; - public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, AlignedArray src, AlignedArray dst, int crun) => SseUtils.MatTimesSrc(tran, add, mat, src, dst, crun); + public static void MatrixTimesSource(bool transpose, bool add, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(transpose, add, matrix, source, destination, stride); - public static void MatTimesSrc(bool tran, bool add, AlignedArray mat, int[] rgposSrc, AlignedArray srcValues, - int posMin, int iposMin, int iposLim, AlignedArray dst, int crun) => SseUtils.MatTimesSrc(tran, add, mat, rgposSrc, srcValues, posMin, iposMin, iposLim, dst, crun); + public static void MatrixTimesSource(bool transpose, bool add, AlignedArray matrix, int[] rgposSrc, AlignedArray srcValues, + int posMin, int iposMin, int iposLim, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(transpose, add, matrix, rgposSrc, srcValues, posMin, iposMin, iposLim, destination, stride); - public static void Add(float a, float[] dst, int count) => SseUtils.Add(a, dst, count); + public static void Add(float value, float[] destination, int count) => SseUtils.Add(value, destination, count); - public static void Scale(float a, float[] dst, int count) => SseUtils.Scale(a, dst, count); + public static void Scale(float value, float[] destination, int count) => SseUtils.Scale(value, destination, count); - public static void Scale(float a, float[] dst, int offset, int count) => SseUtils.Scale(a, dst, offset, count); + public static void Scale(float value, float[] destination, int offset, int count) => SseUtils.Scale(value, destination, offset, count); - public static void Scale(float a, float[] src, float[] dst, int count) => SseUtils.Scale(a, src, dst, count); + public static void Scale(float value, float[] source, float[] destination, int count) => SseUtils.Scale(value, source, destination, count); - public static void ScaleAdd(float a, float b, float[] dst, int count) => SseUtils.ScaleAdd(a, b, dst, count); + public static void ScaleAdd(float value, float addend, float[] destination, int count) => SseUtils.ScaleAdd(value, addend, destination, count); - public static void AddScale(float a, float[] src, float[] dst, int count) => SseUtils.AddScale(a, src, dst, count); + public static void AddScale(float value, float[] source, float[] destination, int count) => SseUtils.AddScale(value, source, destination, count); - public static void AddScale(float a, float[] src, float[] dst, int dstOffset, int count) => SseUtils.AddScale(a, src, dst, dstOffset, count); + public static void AddScale(float value, float[] source, float[] destination, int dstOffset, int count) => SseUtils.AddScale(value, source, destination, dstOffset, count); - public static void AddScale(float a, float[] src, int[] indices, float[] dst, int count) => SseUtils.AddScale(a, src, indices, dst, count); + public static void AddScale(float value, float[] source, int[] indices, float[] destination, int count) => SseUtils.AddScale(value, source, indices, destination, count); - public static void AddScale(float a, float[] src, int[] indices, float[] dst, int dstOffset, int count) => SseUtils.AddScale(a, src, indices, dst, dstOffset, count); + public static void AddScale(float value, float[] source, int[] indices, float[] destination, int dstOffset, int count) => SseUtils.AddScale(value, source, indices, destination, dstOffset, count); - public static void AddScaleCopy(float a, float[] src, float[] dst, float[] res, int count) => SseUtils.AddScaleCopy(a, src, dst, res, count); + public static void AddScaleCopy(float value, float[] source, float[] destination, float[] res, int count) => SseUtils.AddScaleCopy(value, source, destination, res, count); - public static void Add(float[] src, float[] dst, int count) => SseUtils.Add(src, dst, count); + public static void Add(float[] source, float[] destination, int count) => SseUtils.Add(source, destination, count); - public static void Add(float[] src, int[] indices, float[] dst, int count) => SseUtils.Add(src, indices, dst, count); + public static void Add(float[] source, int[] indices, float[] destination, int count) => SseUtils.Add(source, indices, destination, count); - public static void Add(float[] src, int[] indices, float[] dst, int dstOffset, int count) => SseUtils.Add(src, indices, dst, dstOffset, count); + public static void Add(float[] source, int[] indices, float[] destination, int dstOffset, int count) => SseUtils.Add(source, indices, destination, dstOffset, count); - public static void MulElementWise(float[] src1, float[] src2, float[] dst, int count) => SseUtils.MulElementWise(src1, src2, dst, count); + public static void MulElementWise(float[] left, float[] right, float[] destination, int count) => SseUtils.MulElementWise(left, right, destination, count); - public static float Sum(float[] src, int count) => SseUtils.Sum(src, count); + public static float Sum(float[] source, int count) => SseUtils.Sum(source, count); - public static float Sum(float[] src, int offset, int count) => SseUtils.Sum(src, offset, count); + public static float Sum(float[] source, int offset, int count) => SseUtils.Sum(source, offset, count); - public static float SumSq(float[] src, int count) => SseUtils.SumSq(src, count); + public static float SumSq(float[] source, int count) => SseUtils.SumSq(source, count); - public static float SumSq(float[] src, int offset, int count) => SseUtils.SumSq(src, offset, count); + public static float SumSq(float[] source, int offset, int count) => SseUtils.SumSq(source, offset, count); - public static float SumSq(float mean, float[] src, int offset, int count) => SseUtils.SumSq(mean, src, offset, count); + public static float SumSq(float mean, float[] source, int offset, int count) => SseUtils.SumSq(mean, source, offset, count); - public static float SumAbs(float[] src, int count) => SseUtils.SumAbs(src, count); + public static float SumAbs(float[] source, int count) => SseUtils.SumAbs(source, count); - public static float SumAbs(float[] src, int offset, int count) => SseUtils.SumAbs(src, offset, count); + public static float SumAbs(float[] source, int offset, int count) => SseUtils.SumAbs(source, offset, count); - public static float SumAbs(float mean, float[] src, int offset, int count) => SseUtils.SumAbs(mean, src, offset, count); + public static float SumAbs(float mean, float[] source, int offset, int count) => SseUtils.SumAbs(mean, source, offset, count); - public static float MaxAbs(float[] src, int count) => SseUtils.MaxAbs(src, count); + public static float MaxAbs(float[] source, int count) => SseUtils.MaxAbs(source, count); - public static float MaxAbs(float[] src, int offset, int count) => SseUtils.MaxAbs(src, offset, count); + public static float MaxAbs(float[] source, int offset, int count) => SseUtils.MaxAbs(source, offset, count); - public static float MaxAbsDiff(float mean, float[] src, int count) => SseUtils.MaxAbsDiff(mean, src, count); + public static float MaxAbsDiff(float mean, float[] source, int count) => SseUtils.MaxAbsDiff(mean, source, count); - public static float DotProductDense(float[] a, float[] b, int count) => SseUtils.DotProductDense(a, b, count); + public static float DotProductDense(float[] value, float[] b, int count) => SseUtils.DotProductDense(value, b, count); - public static float DotProductDense(float[] a, int offset, float[] b, int count) => SseUtils.DotProductDense(a, offset, b, count); + public static float DotProductDense(float[] value, int offset, float[] b, int count) => SseUtils.DotProductDense(value, offset, b, count); - public static float DotProductSparse(float[] a, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(a, b, indices, count); + public static float DotProductSparse(float[] value, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(value, b, indices, count); - public static float DotProductSparse(float[] a, int offset, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(a, offset, b, indices, count); + public static float DotProductSparse(float[] value, int offset, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(value, offset, b, indices, count); - public static float L2DistSquared(float[] a, float[] b, int count) => SseUtils.L2DistSquared(a, b, count); + public static float L2DistSquared(float[] value, float[] b, int count) => SseUtils.L2DistSquared(value, b, count); - public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[] indices) => SseUtils.ZeroMatrixItems(dst, ccol, cfltRow, indices); + public static void ZeroMatrixItems(AlignedArray destination, int ccol, int cfltRow, int[] indices) => SseUtils.ZeroMatrixItems(destination, ccol, cfltRow, indices); - public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] src, float threshold, float[] v, float[] w) - => SseUtils.SdcaL1UpdateDense(primalUpdate, length, src, threshold, v, w); + public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] source, float threshold, float[] v, float[] w) + => SseUtils.SdcaL1UpdateDense(primalUpdate, length, source, threshold, v, w); - public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] src, int[] indices, int count, float threshold, float[] v, float[] w) - => SseUtils.SdcaL1UpdateSparse(primalUpdate, length, src, indices, count, threshold, v, w); + public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) + => SseUtils.SdcaL1UpdateSparse(primalUpdate, length, source, indices, count, threshold, v, w); } } diff --git a/src/Microsoft.ML.Transforms/RffTransform.cs b/src/Microsoft.ML.Transforms/RffTransform.cs index 7cb51a6f65..6d953a1172 100644 --- a/src/Microsoft.ML.Transforms/RffTransform.cs +++ b/src/Microsoft.ML.Transforms/RffTransform.cs @@ -552,7 +552,7 @@ private static void TransformFeatures(IHost host, ref VBuffer src, ref VB if (src.IsDense) { featuresAligned.CopyFrom(src.Values, 0, src.Length); - CpuMathUtils.MatTimesSrc(false, false, transformInfo.RndFourierVectors, featuresAligned, productAligned, + CpuMathUtils.MatrixTimesSource(false, false, transformInfo.RndFourierVectors, featuresAligned, productAligned, transformInfo.NewDim); } else @@ -560,7 +560,7 @@ private static void TransformFeatures(IHost host, ref VBuffer src, ref VB // This overload of MatTimesSrc ignores the values in slots that are not in src.Indices, so there is // no need to zero them out. featuresAligned.CopyFrom(src.Indices, src.Values, 0, 0, src.Count, zeroItems: false); - CpuMathUtils.MatTimesSrc(false, false, transformInfo.RndFourierVectors, src.Indices, featuresAligned, 0, 0, + CpuMathUtils.MatrixTimesSource(false, false, transformInfo.RndFourierVectors, src.Indices, featuresAligned, 0, 0, src.Count, productAligned, transformInfo.NewDim); } diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index c69bf1171f..e61593029f 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -1,20 +1,31 @@ Data.CustomTextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData ImportText Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output +Data.CustomTextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData ImportText Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output +Data.DataViewReference Pass dataview from memory to experiment Microsoft.ML.Runtime.EntryPoints.DataViewReference ImportData Microsoft.ML.Runtime.EntryPoints.DataViewReference+Input Microsoft.ML.Runtime.EntryPoints.DataViewReference+Output Data.DataViewReference Pass dataview from memory to experiment Microsoft.ML.Runtime.EntryPoints.DataViewReference ImportData Microsoft.ML.Runtime.EntryPoints.DataViewReference+Input Microsoft.ML.Runtime.EntryPoints.DataViewReference+Output Data.IDataViewArrayConverter Create an array variable of IDataView Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput +Data.IDataViewArrayConverter Create an array variable of IDataView Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput Data.PredictorModelArrayConverter Create an array variable of IPredictorModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput +Data.PredictorModelArrayConverter Create an array variable of IPredictorModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput +Data.TextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData TextLoader Microsoft.ML.Runtime.EntryPoints.ImportTextData+LoaderInput Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output Data.TextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData TextLoader Microsoft.ML.Runtime.EntryPoints.ImportTextData+LoaderInput Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output Data.TransformModelArrayConverter Create an array variable of ITransformModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelOutput +Data.TransformModelArrayConverter Create an array variable of ITransformModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelOutput Models.AnomalyDetectionEvaluator Evaluates an anomaly detection scored dataset. Microsoft.ML.Runtime.Data.Evaluate AnomalyDetection Microsoft.ML.Runtime.Data.AnomalyDetectionMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.AnomalyPipelineEnsemble Combine anomaly detection models into an ensemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator CreateAnomalyPipelineEnsemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator+PipelineAnomalyInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Models.BinaryClassificationEvaluator Evaluates a binary classification scored dataset. Microsoft.ML.Runtime.Data.Evaluate Binary Microsoft.ML.Runtime.Data.BinaryClassifierMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClassificationEvaluateOutput Models.BinaryCrossValidator Cross validation for binary classification Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro CrossValidateBinary Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Output] +Models.BinaryCrossValidator Cross validation for binary classification Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro CrossValidateBinary Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Output] Models.BinaryEnsemble Combine binary classifiers into an ensemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator CreateBinaryEnsemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator+ClassifierInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Models.BinaryPipelineEnsemble Combine binary classification models into an ensemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator CreateBinaryPipelineEnsemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator+PipelineClassifierInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Models.ClassificationEvaluator Evaluates a multi class classification scored dataset. Microsoft.ML.Runtime.Data.Evaluate MultiClass Microsoft.ML.Runtime.Data.MultiClassMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClassificationEvaluateOutput Models.ClusterEvaluator Evaluates a clustering scored dataset. Microsoft.ML.Runtime.Data.Evaluate Clustering Microsoft.ML.Runtime.Data.ClusteringMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.CrossValidationResultsCombiner Combine the metric data views returned from cross validation. Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CombineMetrics Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombineMetricsInput Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombinedOutput +Models.CrossValidationResultsCombiner Combine the metric data views returned from cross validation. Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CombineMetrics Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombineMetricsInput Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombinedOutput +Models.CrossValidator Cross validation for general learning Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CrossValidate Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Output] Models.CrossValidator Cross validation for general learning Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CrossValidate Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Output] Models.CrossValidatorDatasetSplitter Split the dataset into the specified number of cross-validation folds (train and test sets) Microsoft.ML.Runtime.EntryPoints.CVSplit Split Microsoft.ML.Runtime.EntryPoints.CVSplit+Input Microsoft.ML.Runtime.EntryPoints.CVSplit+Output +Models.CrossValidatorDatasetSplitter Split the dataset into the specified number of cross-validation folds (train and test sets) Microsoft.ML.Runtime.EntryPoints.CVSplit Split Microsoft.ML.Runtime.EntryPoints.CVSplit+Input Microsoft.ML.Runtime.EntryPoints.CVSplit+Output +Models.DatasetTransformer Applies a TransformModel to a dataset. Microsoft.ML.Runtime.EntryPoints.ModelOperations Apply Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelOutput Models.DatasetTransformer Applies a TransformModel to a dataset. Microsoft.ML.Runtime.EntryPoints.ModelOperations Apply Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelOutput Models.EnsembleSummary Summarize a pipeline ensemble predictor. Microsoft.ML.Runtime.Ensemble.EntryPoints.PipelineEnsemble Summarize Microsoft.ML.Runtime.EntryPoints.SummarizePredictor+Input Microsoft.ML.Runtime.Ensemble.EntryPoints.PipelineEnsemble+SummaryOutput Models.FixedPlattCalibrator Apply a Platt calibrator with a fixed slope and offset to an input model Microsoft.ML.Runtime.Internal.Calibration.Calibrate FixedPlatt Microsoft.ML.Runtime.Internal.Calibration.Calibrate+FixedPlattInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CalibratorOutput @@ -22,6 +33,7 @@ Models.MultiClassPipelineEnsemble Combine multiclass classifiers into an ensembl Models.MultiOutputRegressionEvaluator Evaluates a multi output regression scored dataset. Microsoft.ML.Runtime.Data.Evaluate MultiOutputRegression Microsoft.ML.Runtime.Data.MultiOutputRegressionMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.NaiveCalibrator Apply a Naive calibrator to an input model Microsoft.ML.Runtime.Internal.Calibration.Calibrate Naive Microsoft.ML.Runtime.Internal.Calibration.Calibrate+NoArgumentsInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CalibratorOutput Models.OneVersusAll One-vs-All macro (OVA) Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro OneVersusAll Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Output] +Models.OneVersusAll One-vs-All macro (OVA) Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro OneVersusAll Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Output] Models.OnnxConverter Converts the model to ONNX format. Microsoft.ML.Runtime.Model.Onnx.SaveOnnxCommand Apply Microsoft.ML.Runtime.Model.Onnx.SaveOnnxCommand+Arguments Microsoft.ML.Runtime.Model.Onnx.SaveOnnxCommand+Output Models.OvaModelCombiner Combines a sequence of PredictorModels into a single model Microsoft.ML.Runtime.Learners.OvaPredictor CombineOvaModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineOvaPredictorModelsInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput Models.PAVCalibrator Apply a PAV calibrator to an input model Microsoft.ML.Runtime.Internal.Calibration.Calibrate Pav Microsoft.ML.Runtime.Internal.Calibration.Calibrate+NoArgumentsInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CalibratorOutput @@ -35,6 +47,8 @@ Models.RegressionPipelineEnsemble Combine regression models into an ensemble Mic Models.Summarizer Summarize a linear regression predictor. Microsoft.ML.Runtime.EntryPoints.SummarizePredictor Summarize Microsoft.ML.Runtime.EntryPoints.SummarizePredictor+Input Microsoft.ML.Runtime.EntryPoints.CommonOutputs+SummaryOutput Models.SweepResultExtractor Extracts the sweep result. Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro ExtractSweepResult Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro+ResultInput Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro+Output Models.TrainTestBinaryEvaluator Train test for binary classification Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro TrainTestBinary Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Output] +Models.TrainTestBinaryEvaluator Train test for binary classification Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro TrainTestBinary Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Output] +Models.TrainTestEvaluator General train test for any supported evaluator Microsoft.ML.Runtime.EntryPoints.TrainTestMacro TrainTest Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Output] Models.TrainTestEvaluator General train test for any supported evaluator Microsoft.ML.Runtime.EntryPoints.TrainTestMacro TrainTest Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Output] Trainers.AveragedPerceptronBinaryClassifier Averaged Perceptron Binary Classifier. Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer TrainBinary Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.EnsembleBinaryClassifier Train binary ensemble. Microsoft.ML.Ensemble.EntryPoints.Ensemble CreateBinaryEnsemble Microsoft.ML.Runtime.Ensemble.EnsembleTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput @@ -85,6 +99,7 @@ Transforms.DatasetScorer Score a dataset with a predictor model Microsoft.ML.Run Transforms.DatasetTransformScorer Score a dataset with a transform model Microsoft.ML.Runtime.EntryPoints.ScoreModel ScoreUsingTransform Microsoft.ML.Runtime.EntryPoints.ScoreModel+InputTransformScorer Microsoft.ML.Runtime.EntryPoints.ScoreModel+Output Transforms.Dictionarizer Converts input values (words, numbers, etc.) to index in a dictionary. Microsoft.ML.Runtime.Transforms.TextAnalytics TermTransform Microsoft.ML.Runtime.Data.TermTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.FeatureCombiner Combines all the features into one feature column. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareFeatures Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+FeatureCombinerInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.FeatureCombiner Combines all the features into one feature column. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareFeatures Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+FeatureCombinerInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.FeatureSelectorByCount Selects the slots for which the count of non-default values is greater than or equal to a threshold. Microsoft.ML.Runtime.EntryPoints.SelectFeatures CountSelect Microsoft.ML.Runtime.Data.CountFeatureSelectionTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.FeatureSelectorByMutualInformation Selects the top k slots across all specified columns ordered by their mutual information with the label column. Microsoft.ML.Runtime.EntryPoints.SelectFeatures MutualInformationSelect Microsoft.ML.Runtime.Data.MutualInformationFeatureSelectionTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.GlobalContrastNormalizer Performs a global contrast normalization on input values: Y = (s * X - M) / D, where s is a scale, M is mean and D is either L2 norm or standard deviation. Microsoft.ML.Runtime.Data.LpNormalization GcNormalize Microsoft.ML.Runtime.Data.LpNormNormalizerTransform+GcnArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -95,12 +110,15 @@ Transforms.ImagePixelExtractor Extract color plane(s) from an image. Options inc Transforms.ImageResizer Scales an image to specified dimensions using one of the three scale types: isotropic with padding, isotropic with cropping or anisotropic. In case of isotropic padding, transparent color is used to pad resulting image. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics ImageResizer Microsoft.ML.Runtime.ImageAnalytics.ImageResizerTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.KeyToTextConverter KeyToValueTransform utilizes KeyValues metadata to map key indices to the corresponding values in the KeyValues metadata. Microsoft.ML.Runtime.Data.Categorical KeyToText Microsoft.ML.Runtime.Data.KeyToValueTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelColumnKeyBooleanConverter Transforms the label to either key or bool (if needed) to make it suitable for classification. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareClassificationLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+ClassificationLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.LabelColumnKeyBooleanConverter Transforms the label to either key or bool (if needed) to make it suitable for classification. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareClassificationLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+ClassificationLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelIndicator Label remapper used by OVA Microsoft.ML.Runtime.Data.LabelIndicatorTransform LabelIndicator Microsoft.ML.Runtime.Data.LabelIndicatorTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelToFloatConverter Transforms the label to float to make it suitable for regression. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareRegressionLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+RegressionLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.LabelToFloatConverter Transforms the label to float to make it suitable for regression. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareRegressionLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+RegressionLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LightLda The LDA transform implements LightLDA, a state-of-the-art implementation of Latent Dirichlet Allocation. Microsoft.ML.Runtime.Transforms.TextAnalytics LightLda Microsoft.ML.Runtime.TextAnalytics.LdaTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LogMeanVarianceNormalizer Normalizes the data based on the computed mean and variance of the logarithm of the data. Microsoft.ML.Runtime.Data.Normalize LogMeanVar Microsoft.ML.Runtime.Data.NormalizeTransform+LogMeanVarArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LpNormalizer Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf). Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D is either L2 norm, L1 norm or LInf norm. Microsoft.ML.Runtime.Data.LpNormalization Normalize Microsoft.ML.Runtime.Data.LpNormNormalizerTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.ManyHeterogeneousModelCombiner Combines a sequence of TransformModels and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput +Transforms.ManyHeterogeneousModelCombiner Combines a sequence of TransformModels and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput Transforms.MeanVarianceNormalizer Normalizes the data based on the computed mean and variance of the data. Microsoft.ML.Runtime.Data.Normalize MeanVar Microsoft.ML.Runtime.Data.NormalizeTransform+MeanVarArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.MinMaxNormalizer Normalizes the data based on the observed minimum and maximum values of the data. Microsoft.ML.Runtime.Data.Normalize MinMax Microsoft.ML.Runtime.Data.NormalizeTransform+MinMaxArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.MissingValueHandler Handle missing values by replacing them with either the default value or the mean/min/max value (for non-text columns only). An indicator column can optionally be concatenated, if theinput column type is numeric. Microsoft.ML.Runtime.Data.NAHandling Handle Microsoft.ML.Runtime.Data.NAHandleTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -109,11 +127,13 @@ Transforms.MissingValuesDropper Removes NAs from vector columns. Microsoft.ML.Ru Transforms.MissingValuesRowDropper Filters out rows that contain missing values. Microsoft.ML.Runtime.Data.NAHandling Filter Microsoft.ML.Runtime.Data.NAFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.MissingValueSubstitutor Create an output column of the same type and size of the input column, where missing values are replaced with either the default value or the mean/min/max value (for non-text columns only). Microsoft.ML.Runtime.Data.NAHandling Replace Microsoft.ML.Runtime.Data.NAReplaceTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.ModelCombiner Combines a sequence of TransformModels into a single model Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTransformModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsOutput +Transforms.ModelCombiner Combines a sequence of TransformModels into a single model Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTransformModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsOutput Transforms.NGramTranslator Produces a bag of counts of ngrams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of ngrams and using the id in the dictionary as the index in the bag. Microsoft.ML.Runtime.Transforms.TextAnalytics NGramTransform Microsoft.ML.Runtime.Data.NgramTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.NoOperation Does nothing. Microsoft.ML.Runtime.Data.NopTransform Nop Microsoft.ML.Runtime.Data.NopTransform+NopInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.OptionalColumnCreator If the source column does not exist after deserialization, create a column with the right type and default values. Microsoft.ML.Runtime.DataPipe.OptionalColumnTransform MakeOptional Microsoft.ML.Runtime.DataPipe.OptionalColumnTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.PcaCalculator PCA is a dimensionality-reduction transform which computes the projection of a numeric vector onto a low-rank subspace. Microsoft.ML.Runtime.Data.PcaTransform Calculate Microsoft.ML.Runtime.Data.PcaTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.PredictedLabelColumnOriginalValueConverter Transforms a predicted label column to its original values, unless it is of type bool. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner ConvertPredictedLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+PredictedLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput +Transforms.PredictedLabelColumnOriginalValueConverter Transforms a predicted label column to its original values, unless it is of type bool. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner ConvertPredictedLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+PredictedLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.RandomNumberGenerator Adds a column with a generated number sequence. Microsoft.ML.Runtime.Data.RandomNumberGenerator Generate Microsoft.ML.Runtime.Data.GenerateNumberTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.RowRangeFilter Filters a dataview on a column of type Single, Double or Key (contiguous). Keeps the values that are in the specified min/max range. NaNs are always filtered out. If the input is a Key type, the min/max are considered percentages of the number of values. Microsoft.ML.Runtime.EntryPoints.SelectRows FilterByRange Microsoft.ML.Runtime.Data.RangeFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.RowSkipAndTakeFilter Allows limiting input to a subset of rows at an optional offset. Can be used to implement data paging. Microsoft.ML.Runtime.EntryPoints.SelectRows SkipAndTakeFilter Microsoft.ML.Runtime.Data.SkipTakeFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -127,8 +147,10 @@ Transforms.TensorFlowScorer Transforms the data using the TensorFlow model. Micr Transforms.TextFeaturizer A transform that turns a collection of text documents into numerical feature vectors. The feature vectors are normalized counts of (word and/or character) ngrams in a given tokenized text. Microsoft.ML.Runtime.Transforms.TextAnalytics TextTransform Microsoft.ML.Runtime.Data.TextTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TextToKeyConverter Converts input values (words, numbers, etc.) to index in a dictionary. Microsoft.ML.Runtime.Data.Categorical TextToKey Microsoft.ML.Runtime.Data.TermTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TrainTestDatasetSplitter Split the dataset into train and test sets Microsoft.ML.Runtime.EntryPoints.TrainTestSplit Split Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Input Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Output +Transforms.TrainTestDatasetSplitter Split the dataset into train and test sets Microsoft.ML.Runtime.EntryPoints.TrainTestSplit Split Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Input Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Output Transforms.TreeLeafFeaturizer Trains a tree ensemble, or loads it from a file, then maps a numeric feature vector to three outputs: 1. A vector containing the individual tree outputs of the tree ensemble. 2. A vector indicating the leaves that the feature vector falls on in the tree ensemble. 3. A vector indicating the paths that the feature vector falls on in the tree ensemble. If a both a model file and a trainer are specified - will use the model file. If neither are specified, will train a default FastTree model. This can handle key labels by training a regression model towards their optionally permuted indices. Microsoft.ML.Runtime.Data.TreeFeaturize Featurizer Microsoft.ML.Runtime.Data.TreeEnsembleFeaturizerTransform+ArgumentsForEntryPoint Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TwoHeterogeneousModelCombiner Combines a TransformModel and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTwoModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+SimplePredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput +Transforms.TwoHeterogeneousModelCombiner Combines a TransformModel and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTwoModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+SimplePredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput Transforms.VectorToImage Converts vector array into image type. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics VectorToImage Microsoft.ML.Runtime.ImageAnalytics.VectorToImageTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.WordEmbeddings Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model Microsoft.ML.Runtime.Transforms.TextAnalytics WordEmbeddings Microsoft.ML.Runtime.Data.WordEmbeddingsTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.WordTokenizer The input to this transform is text, and the output is a vector of text containing the words (tokens) in the original text. The separator is space, but can be specified as any other character (or multiple characters) if needed. Microsoft.ML.Runtime.Transforms.TextAnalytics DelimitedTokenizeTransform Microsoft.ML.Runtime.Data.DelimitedTokenizeTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index 1ff41a090b..86345aac70 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -38,6 +38,67 @@ } ] }, + { + "Name": "Data.CustomTextLoader", + "Desc": "Import a dataset from a text file", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "InputFile", + "Type": "FileHandle", + "Desc": "Location of the input file", + "Aliases": [ + "data" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "CustomSchema", + "Type": "String", + "Desc": "Custom schema to use for parsing", + "Aliases": [ + "schema" + ], + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The resulting data view" + } + ] + }, + { + "Name": "Data.DataViewReference", + "Desc": "Pass dataview from memory to experiment", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Pointer to IDataView in memory", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The resulting data view" + } + ] + }, { "Name": "Data.DataViewReference", "Desc": "Pass dataview from memory to experiment", @@ -90,6 +151,64 @@ } ] }, + { + "Name": "Data.IDataViewArrayConverter", + "Desc": "Create an array variable of IDataView", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "The data sets", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "The data set array" + } + ] + }, + { + "Name": "Data.PredictorModelArrayConverter", + "Desc": "Create an array variable of IPredictorModel", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Model", + "Type": { + "Kind": "Array", + "ItemType": "PredictorModel" + }, + "Desc": "The models", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputModel", + "Type": { + "Kind": "Array", + "ItemType": "PredictorModel" + }, + "Desc": "The model array" + } + ] + }, { "Name": "Data.PredictorModelArrayConverter", "Desc": "Create an array variable of IPredictorModel", @@ -470,44 +589,15 @@ ] }, { - "Name": "Data.TransformModelArrayConverter", - "Desc": "Create an array variable of ITransformModel", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TransformModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The models", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The model array" - } - ] - }, - { - "Name": "Models.AnomalyDetectionEvaluator", - "Desc": "Evaluates an anomaly detection scored dataset.", + "Name": "Data.TextLoader", + "Desc": "Import a dataset from a text file", "FriendlyName": null, "ShortName": null, "Inputs": [ { - "Name": "Data", - "Type": "DataView", - "Desc": "The data to be used for evaluation.", + "Name": "InputFile", + "Type": "FileHandle", + "Desc": "Location of the input file", "Aliases": [ "data" ], @@ -516,104 +606,512 @@ "IsNullable": false }, { - "Name": "NameColumn", - "Type": "String", - "Desc": "Name column name.", - "Aliases": [ - "name" - ], - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": "Name" - }, - { - "Name": "K", - "Type": "Int", - "Desc": "Expected number of false positives", - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 10 - }, - { - "Name": "P", - "Type": "Float", - "Desc": "Expected false positive rate", - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 0.01 - }, - { - "Name": "NumTopResults", - "Type": "Int", - "Desc": "Number of top-scored predictions to display", - "Aliases": [ - "n" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 50 - }, - { - "Name": "Stream", - "Type": "Bool", - "Desc": "Whether to calculate metrics in one pass", - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": true - }, - { - "Name": "MaxAucExamples", - "Type": "Int", - "Desc": "The number of samples to use for AUC calculation. If 0, AUC is not computed. If -1, the whole dataset is used", - "Aliases": [ - "numauc" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": -1 - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels.", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Weight column name.", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "ScoreColumn", - "Type": "String", - "Desc": "Score column name.", - "Aliases": [ - "score" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, + "Name": "Arguments", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Column", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Name", + "Type": "String", + "Desc": "Name of the column", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Type", + "Type": { + "Kind": "Enum", + "Values": [ + "I1", + "U1", + "I2", + "U2", + "I4", + "U4", + "I8", + "U8", + "R4", + "Num", + "R8", + "TX", + "Text", + "TXT", + "BL", + "Bool", + "TimeSpan", + "TS", + "DT", + "DateTime", + "DZ", + "DateTimeZone", + "UG", + "U16" + ] + }, + "Desc": "Type of the items in the column", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Source", + "Type": { + "Kind": "Array", + "ItemType": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Min", + "Type": "Int", + "Desc": "First index in the range", + "Required": true, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Max", + "Type": "Int", + "Desc": "Last index in the range", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "AutoEnd", + "Type": "Bool", + "Desc": "This range extends to the end of the line, but should be a fixed number of items", + "Aliases": [ + "auto" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "VariableEnd", + "Type": "Bool", + "Desc": "This range extends to the end of the line, which can vary from line to line", + "Aliases": [ + "var" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "AllOther", + "Type": "Bool", + "Desc": "This range includes only other indices not specified", + "Aliases": [ + "other" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "ForceVector", + "Type": "Bool", + "Desc": "Force scalar columns to be treated as vectors of length one", + "Aliases": [ + "vector" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + } + ] + } + }, + "Desc": "Source index range(s) of the column", + "Aliases": [ + "src" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "KeyRange", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Min", + "Type": "UInt", + "Desc": "First index in the range", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0 + }, + { + "Name": "Max", + "Type": "UInt", + "Desc": "Last index in the range", + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Contiguous", + "Type": "Bool", + "Desc": "Whether the key is contiguous", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + } + ] + }, + "Desc": "For a key column, this defines the range of values", + "Aliases": [ + "key" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "Column groups. Each group is specified as name:type:numeric-ranges, eg, col=Features:R4:1-17,26,35-40", + "Aliases": [ + "col" + ], + "Required": false, + "SortOrder": 1.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "UseThreads", + "Type": "Bool", + "Desc": "Use separate parsing threads?", + "Aliases": [ + "threads" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "HeaderFile", + "Type": "String", + "Desc": "File containing a header with feature names. If specified, header defined in the data file (header+) is ignored.", + "Aliases": [ + "hf" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "MaxRows", + "Type": "Int", + "Desc": "Maximum number of rows to produce", + "Aliases": [ + "rows" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "AllowQuoting", + "Type": "Bool", + "Desc": "Whether the input may include quoted values, which can contain separator characters, colons, and distinguish empty values from missing values. When true, consecutive separators denote a missing value and an empty value is denoted by \"\". When false, consecutive separators denote an empty value.", + "Aliases": [ + "quote" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "AllowSparse", + "Type": "Bool", + "Desc": "Whether the input may include sparse representations", + "Aliases": [ + "sparse" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "InputSize", + "Type": "Int", + "Desc": "Number of source columns in the text data. Default is that sparse rows contain their size information.", + "Aliases": [ + "size" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": true, + "Default": null + }, + { + "Name": "Separator", + "Type": { + "Kind": "Array", + "ItemType": "Char" + }, + "Desc": "Source column separator.", + "Aliases": [ + "sep" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": [ + "\t" + ] + }, + { + "Name": "TrimWhitespace", + "Type": "Bool", + "Desc": "Remove trailing whitespace from lines", + "Aliases": [ + "trim" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "HasHeader", + "Type": "Bool", + "Desc": "Data file has header with feature names. Header is read only if options 'hs' and 'hf' are not specified.", + "Aliases": [ + "header" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": false + } + ] + }, + "Desc": "Arguments", + "Aliases": [ + "args" + ], + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The resulting data view" + } + ], + "InputKind": [ + "ILearningPipelineLoader" + ] + }, + { + "Name": "Data.TransformModelArrayConverter", + "Desc": "Create an array variable of ITransformModel", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "TransformModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The models", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The model array" + } + ] + }, + { + "Name": "Data.TransformModelArrayConverter", + "Desc": "Create an array variable of ITransformModel", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "TransformModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The models", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The model array" + } + ] + }, + { + "Name": "Models.AnomalyDetectionEvaluator", + "Desc": "Evaluates an anomaly detection scored dataset.", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data to be used for evaluation.", + "Aliases": [ + "data" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "NameColumn", + "Type": "String", + "Desc": "Name column name.", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": "Name" + }, + { + "Name": "K", + "Type": "Int", + "Desc": "Expected number of false positives", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 10 + }, + { + "Name": "P", + "Type": "Float", + "Desc": "Expected false positive rate", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 0.01 + }, + { + "Name": "NumTopResults", + "Type": "Int", + "Desc": "Number of top-scored predictions to display", + "Aliases": [ + "n" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 50 + }, + { + "Name": "Stream", + "Type": "Bool", + "Desc": "Whether to calculate metrics in one pass", + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "MaxAucExamples", + "Type": "Int", + "Desc": "The number of samples to use for AUC calculation. If 0, AUC is not computed. If -1, the whole dataset is used", + "Aliases": [ + "numauc" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": -1 + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels.", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Weight column name.", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "ScoreColumn", + "Type": "String", + "Desc": "Score column name.", + "Aliases": [ + "score" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, { "Name": "StratColumn", "Type": { @@ -804,85 +1302,218 @@ "Default": 100000 }, { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels.", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels.", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Weight column name.", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "ScoreColumn", + "Type": "String", + "Desc": "Score column name.", + "Aliases": [ + "score" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "StratColumn", + "Type": { + "Kind": "Array", + "ItemType": "String" + }, + "Desc": "Stratification column name.", + "Aliases": [ + "strat" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "Warnings", + "Type": "DataView", + "Desc": "Warning dataset" + }, + { + "Name": "OverallMetrics", + "Type": "DataView", + "Desc": "Overall metrics dataset" + }, + { + "Name": "PerInstanceMetrics", + "Type": "DataView", + "Desc": "Per instance metrics dataset" + }, + { + "Name": "ConfusionMatrix", + "Type": "DataView", + "Desc": "Confusion matrix dataset" + } + ], + "InputKind": [ + "IEvaluatorInput" + ], + "OutputKind": [ + "IClassificationEvaluatorOutput", + "IEvaluatorOutput" + ] + }, + { + "Name": "Models.BinaryCrossValidator", + "Desc": "Cross validation for binary classification", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data set", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Nodes", + "Type": { + "Kind": "Array", + "ItemType": "Node" + }, + "Desc": "The training subgraph", + "Required": true, + "SortOrder": 3.0, + "IsNullable": false + }, + { + "Name": "Inputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data to be used for training", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph inputs", + "Required": true, + "SortOrder": 4.0, + "IsNullable": false + }, + { + "Name": "Outputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Model", + "Type": "PredictorModel", + "Desc": "The model", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph outputs", + "Required": true, + "SortOrder": 5.0, + "IsNullable": false }, { - "Name": "WeightColumn", + "Name": "StratificationColumn", "Type": "String", - "Desc": "Weight column name.", + "Desc": "Column to use for stratification", "Aliases": [ - "weight" + "strat" ], "Required": false, - "SortOrder": 150.0, + "SortOrder": 7.0, "IsNullable": false, "Default": null }, { - "Name": "ScoreColumn", - "Type": "String", - "Desc": "Score column name.", + "Name": "NumFolds", + "Type": "Int", + "Desc": "Number of folds in k-fold cross-validation", "Aliases": [ - "score" + "k" ], "Required": false, - "SortOrder": 150.0, + "SortOrder": 8.0, "IsNullable": false, - "Default": null - }, + "Default": 2 + } + ], + "Outputs": [ { - "Name": "StratColumn", + "Name": "PredictorModel", "Type": { "Kind": "Array", - "ItemType": "String" + "ItemType": "PredictorModel" }, - "Desc": "Stratification column name.", - "Aliases": [ - "strat" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ + "Desc": "The trained model" + }, { "Name": "Warnings", - "Type": "DataView", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, "Desc": "Warning dataset" }, { "Name": "OverallMetrics", - "Type": "DataView", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, "Desc": "Overall metrics dataset" }, { "Name": "PerInstanceMetrics", - "Type": "DataView", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, "Desc": "Per instance metrics dataset" }, { "Name": "ConfusionMatrix", - "Type": "DataView", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, "Desc": "Confusion matrix dataset" } - ], - "InputKind": [ - "IEvaluatorInput" - ], - "OutputKind": [ - "IClassificationEvaluatorOutput", - "IEvaluatorOutput" ] }, { @@ -1356,58 +1987,205 @@ "Required": false, "SortOrder": 150.0, "IsNullable": false, - "Default": 3 + "Default": 3 + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels.", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Weight column name.", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "ScoreColumn", + "Type": "String", + "Desc": "Score column name.", + "Aliases": [ + "score" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "StratColumn", + "Type": { + "Kind": "Array", + "ItemType": "String" + }, + "Desc": "Stratification column name.", + "Aliases": [ + "strat" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "Warnings", + "Type": "DataView", + "Desc": "Warning dataset" + }, + { + "Name": "OverallMetrics", + "Type": "DataView", + "Desc": "Overall metrics dataset" + }, + { + "Name": "PerInstanceMetrics", + "Type": "DataView", + "Desc": "Per instance metrics dataset" + } + ], + "InputKind": [ + "IEvaluatorInput" + ], + "OutputKind": [ + "IEvaluatorOutput" + ] + }, + { + "Name": "Models.CrossValidationResultsCombiner", + "Desc": "Combine the metric data views returned from cross validation.", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "OverallMetrics", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "Overall metrics datasets", + "Required": false, + "SortOrder": 1.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "PerInstanceMetrics", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "Per instance metrics datasets", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "ConfusionMatrix", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "Confusion matrix datasets", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Warnings", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "Warning datasets", + "Required": false, + "SortOrder": 4.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Kind", + "Type": { + "Kind": "Enum", + "Values": [ + "SignatureBinaryClassifierTrainer", + "SignatureMultiClassClassifierTrainer", + "SignatureRankerTrainer", + "SignatureRegressorTrainer", + "SignatureMultiOutputRegressorTrainer", + "SignatureAnomalyDetectorTrainer", + "SignatureClusteringTrainer" + ] + }, + "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", + "Required": true, + "SortOrder": 5.0, + "IsNullable": false, + "Default": "SignatureBinaryClassifierTrainer" }, { "Name": "LabelColumn", "Type": "String", - "Desc": "Column to use for labels.", + "Desc": "The label column name", "Aliases": [ - "lab" + "Label" ], "Required": false, - "SortOrder": 150.0, + "SortOrder": 6.0, "IsNullable": false, - "Default": null + "Default": "Label" }, { "Name": "WeightColumn", "Type": "String", - "Desc": "Weight column name.", + "Desc": "Column to use for example weight", "Aliases": [ "weight" ], "Required": false, - "SortOrder": 150.0, + "SortOrder": 7.0, "IsNullable": false, - "Default": null + "Default": "Weight" }, { - "Name": "ScoreColumn", + "Name": "GroupColumn", "Type": "String", - "Desc": "Score column name.", + "Desc": "Column to use for grouping", "Aliases": [ - "score" + "group" ], "Required": false, - "SortOrder": 150.0, + "SortOrder": 8.0, "IsNullable": false, - "Default": null + "Default": "GroupId" }, { - "Name": "StratColumn", - "Type": { - "Kind": "Array", - "ItemType": "String" - }, - "Desc": "Stratification column name.", + "Name": "NameColumn", + "Type": "String", + "Desc": "Name column name", "Aliases": [ - "strat" + "name" ], "Required": false, - "SortOrder": 150.0, + "SortOrder": 9.0, "IsNullable": false, - "Default": null + "Default": "Name" } ], "Outputs": [ @@ -1425,13 +2203,12 @@ "Name": "PerInstanceMetrics", "Type": "DataView", "Desc": "Per instance metrics dataset" + }, + { + "Name": "ConfusionMatrix", + "Type": "DataView", + "Desc": "Confusion matrix dataset" } - ], - "InputKind": [ - "IEvaluatorInput" - ], - "OutputKind": [ - "IEvaluatorOutput" ] }, { @@ -1504,19 +2281,219 @@ }, "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", "Required": true, - "SortOrder": 5.0, + "SortOrder": 5.0, + "IsNullable": false, + "Default": "SignatureBinaryClassifierTrainer" + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "The label column name", + "Aliases": [ + "Label" + ], + "Required": false, + "SortOrder": 6.0, + "IsNullable": false, + "Default": "Label" + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Column to use for example weight", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 7.0, + "IsNullable": false, + "Default": "Weight" + }, + { + "Name": "GroupColumn", + "Type": "String", + "Desc": "Column to use for grouping", + "Aliases": [ + "group" + ], + "Required": false, + "SortOrder": 8.0, + "IsNullable": false, + "Default": "GroupId" + }, + { + "Name": "NameColumn", + "Type": "String", + "Desc": "Name column name", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 9.0, + "IsNullable": false, + "Default": "Name" + } + ], + "Outputs": [ + { + "Name": "Warnings", + "Type": "DataView", + "Desc": "Warning dataset" + }, + { + "Name": "OverallMetrics", + "Type": "DataView", + "Desc": "Overall metrics dataset" + }, + { + "Name": "PerInstanceMetrics", + "Type": "DataView", + "Desc": "Per instance metrics dataset" + }, + { + "Name": "ConfusionMatrix", + "Type": "DataView", + "Desc": "Confusion matrix dataset" + } + ] + }, + { + "Name": "Models.CrossValidator", + "Desc": "Cross validation for general learning", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data set", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "The transform model from the pipeline before this command. It gets included in the Output.PredictorModel.", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "Nodes", + "Type": { + "Kind": "Array", + "ItemType": "Node" + }, + "Desc": "The training subgraph", + "Required": true, + "SortOrder": 3.0, + "IsNullable": false + }, + { + "Name": "Inputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data to be used for training", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph inputs", + "Required": true, + "SortOrder": 4.0, + "IsNullable": false + }, + { + "Name": "Outputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "The predictor model", + "Required": false, + "SortOrder": 1.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "The transform model", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null + } + ] + }, + "Desc": "The training subgraph outputs", + "Required": true, + "SortOrder": 5.0, + "IsNullable": false + }, + { + "Name": "StratificationColumn", + "Type": "String", + "Desc": "Column to use for stratification", + "Aliases": [ + "strat" + ], + "Required": false, + "SortOrder": 6.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "NumFolds", + "Type": "Int", + "Desc": "Number of folds in k-fold cross-validation", + "Aliases": [ + "k" + ], + "Required": false, + "SortOrder": 7.0, + "IsNullable": false, + "Default": 2 + }, + { + "Name": "Kind", + "Type": { + "Kind": "Enum", + "Values": [ + "SignatureBinaryClassifierTrainer", + "SignatureMultiClassClassifierTrainer", + "SignatureRankerTrainer", + "SignatureRegressorTrainer", + "SignatureMultiOutputRegressorTrainer", + "SignatureAnomalyDetectorTrainer", + "SignatureClusteringTrainer" + ] + }, + "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", + "Required": true, + "SortOrder": 8.0, "IsNullable": false, "Default": "SignatureBinaryClassifierTrainer" }, { "Name": "LabelColumn", "Type": "String", - "Desc": "The label column name", + "Desc": "Column to use for labels", "Aliases": [ - "Label" + "lab" ], "Required": false, - "SortOrder": 6.0, + "SortOrder": 9.0, "IsNullable": false, "Default": "Label" }, @@ -1528,7 +2505,7 @@ "weight" ], "Required": false, - "SortOrder": 7.0, + "SortOrder": 10.0, "IsNullable": false, "Default": "Weight" }, @@ -1540,7 +2517,7 @@ "group" ], "Required": false, - "SortOrder": 8.0, + "SortOrder": 11.0, "IsNullable": false, "Default": "GroupId" }, @@ -1552,12 +2529,28 @@ "name" ], "Required": false, - "SortOrder": 9.0, + "SortOrder": 12.0, "IsNullable": false, "Default": "Name" } ], "Outputs": [ + { + "Name": "PredictorModel", + "Type": { + "Kind": "Array", + "ItemType": "PredictorModel" + }, + "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." + }, + { + "Name": "TransformModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." + }, { "Name": "Warnings", "Type": "DataView", @@ -1851,6 +2844,95 @@ } ] }, + { + "Name": "Models.CrossValidatorDatasetSplitter", + "Desc": "Split the dataset into the specified number of cross-validation folds (train and test sets)", + "FriendlyName": "Dataset CV Split", + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "NumFolds", + "Type": "Int", + "Desc": "Number of folds to split into", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 2 + }, + { + "Name": "StratificationColumn", + "Type": "String", + "Desc": "Stratification column", + "Aliases": [ + "strat" + ], + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "TrainData", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "Training data (one dataset per fold)" + }, + { + "Name": "TestData", + "Type": { + "Kind": "Array", + "ItemType": "DataView" + }, + "Desc": "Testing data (one dataset per fold)" + } + ] + }, + { + "Name": "Models.DatasetTransformer", + "Desc": "Applies a TransformModel to a dataset.", + "FriendlyName": "Apply Transform Model Output", + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "Transform model", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + } + ], + "InputKind": [ + "ITransformInput" + ] + }, { "Name": "Models.DatasetTransformer", "Desc": "Applies a TransformModel to a dataset.", @@ -2205,47 +3287,194 @@ "IsNullable": false }, { - "Name": "UncalibratedPredictorModel", - "Type": "PredictorModel", - "Desc": "The predictor to calibrate", + "Name": "UncalibratedPredictorModel", + "Type": "PredictorModel", + "Desc": "The predictor to calibrate", + "Aliases": [ + "uncalibratedPredictorModel" + ], + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "MaxRows", + "Type": "Int", + "Desc": "The maximum number of examples to train the calibrator on", + "Aliases": [ + "maxRows" + ], + "Required": true, + "SortOrder": 3.0, + "IsNullable": false, + "Default": 1000000000, + "Range": { + "Inf": 0, + "Max": 2147483647 + } + } + ], + "Outputs": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "The trained model" + } + ], + "InputKind": [ + "ICalibratorInput", + "ITransformInput" + ], + "OutputKind": [ + "ICalibratorOutput", + "ITrainerOutput" + ] + }, + { + "Name": "Models.OneVersusAll", + "Desc": "One-vs-All macro (OVA)", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Nodes", + "Type": { + "Kind": "Array", + "ItemType": "Node" + }, + "Desc": "The subgraph for the binary trainer used to construct the OVA learner. This should be a TrainBinary node.", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "TrainingData", + "Type": "DataView", + "Desc": "The data to be used for training", + "Aliases": [ + "data" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "OutputForSubGraph", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Model", + "Type": "PredictorModel", + "Desc": "The predictor model for the subgraph exemplar.", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph output.", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "FeatureColumn", + "Type": "String", + "Desc": "Column to use for features", + "Aliases": [ + "feat" + ], + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": "Features" + }, + { + "Name": "UseProbabilities", + "Type": "Bool", + "Desc": "Use probabilities in OVA combiner", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": true + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": "Label" + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Column to use for example weight", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 4.0, + "IsNullable": false, + "Default": "Weight" + }, + { + "Name": "NormalizeFeatures", + "Type": { + "Kind": "Enum", + "Values": [ + "No", + "Warn", + "Auto", + "Yes" + ] + }, + "Desc": "Normalize option for the feature column", "Aliases": [ - "uncalibratedPredictorModel" + "norm" ], - "Required": true, - "SortOrder": 2.0, - "IsNullable": false + "Required": false, + "SortOrder": 5.0, + "IsNullable": false, + "Default": "Auto" }, { - "Name": "MaxRows", - "Type": "Int", - "Desc": "The maximum number of examples to train the calibrator on", + "Name": "Caching", + "Type": { + "Kind": "Enum", + "Values": [ + "Auto", + "Memory", + "Disk", + "None" + ] + }, + "Desc": "Whether learner should cache input training data", "Aliases": [ - "maxRows" + "cache" ], - "Required": true, - "SortOrder": 3.0, + "Required": false, + "SortOrder": 6.0, "IsNullable": false, - "Default": 1000000000, - "Range": { - "Inf": 0, - "Max": 2147483647 - } + "Default": "Auto" } ], "Outputs": [ { "Name": "PredictorModel", "Type": "PredictorModel", - "Desc": "The trained model" + "Desc": "The trained multiclass model" } ], "InputKind": [ - "ICalibratorInput", - "ITransformInput" - ], - "OutputKind": [ - "ICalibratorOutput", - "ITrainerOutput" + "ITrainerInputWithWeight", + "ITrainerInputWithLabel", + "ITrainerInput" ] }, { @@ -3488,23 +4717,239 @@ ], "Outputs": [ { - "Name": "State", - "Type": { - "Kind": "C# Object", - "ItemType": "Microsoft.ML.Runtime.EntryPoints.IMlState" - }, - "Desc": "Stateful autoML object, keeps track of where the search in progress." + "Name": "State", + "Type": { + "Kind": "C# Object", + "ItemType": "Microsoft.ML.Runtime.EntryPoints.IMlState" + }, + "Desc": "Stateful autoML object, keeps track of where the search in progress." + }, + { + "Name": "Results", + "Type": "DataView", + "Desc": "Results of the sweep, including pipelines (as graph strings), IDs, and metric values." + } + ] + }, + { + "Name": "Models.TrainTestBinaryEvaluator", + "Desc": "Train test for binary classification", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "TrainingData", + "Type": "DataView", + "Desc": "The data to be used for training", + "Aliases": [ + "train" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "TestingData", + "Type": "DataView", + "Desc": "The data to be used for testing", + "Aliases": [ + "test" + ], + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "Nodes", + "Type": { + "Kind": "Array", + "ItemType": "Node" + }, + "Desc": "The training subgraph", + "Required": true, + "SortOrder": 3.0, + "IsNullable": false + }, + { + "Name": "Inputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data to be used for training", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph inputs", + "Required": true, + "SortOrder": 4.0, + "IsNullable": false + }, + { + "Name": "Outputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Model", + "Type": "PredictorModel", + "Desc": "The model", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph outputs", + "Required": true, + "SortOrder": 5.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "The trained model" + }, + { + "Name": "Warnings", + "Type": "DataView", + "Desc": "Warning dataset" + }, + { + "Name": "OverallMetrics", + "Type": "DataView", + "Desc": "Overall metrics dataset" + }, + { + "Name": "PerInstanceMetrics", + "Type": "DataView", + "Desc": "Per instance metrics dataset" + }, + { + "Name": "ConfusionMatrix", + "Type": "DataView", + "Desc": "Confusion matrix dataset" + } + ] + }, + { + "Name": "Models.TrainTestBinaryEvaluator", + "Desc": "Train test for binary classification", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "TrainingData", + "Type": "DataView", + "Desc": "The data to be used for training", + "Aliases": [ + "train" + ], + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "TestingData", + "Type": "DataView", + "Desc": "The data to be used for testing", + "Aliases": [ + "test" + ], + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "Nodes", + "Type": { + "Kind": "Array", + "ItemType": "Node" + }, + "Desc": "The training subgraph", + "Required": true, + "SortOrder": 3.0, + "IsNullable": false + }, + { + "Name": "Inputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data to be used for training", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph inputs", + "Required": true, + "SortOrder": 4.0, + "IsNullable": false + }, + { + "Name": "Outputs", + "Type": { + "Kind": "Struct", + "Fields": [ + { + "Name": "Model", + "Type": "PredictorModel", + "Desc": "The model", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ] + }, + "Desc": "The training subgraph outputs", + "Required": true, + "SortOrder": 5.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "The trained model" }, { - "Name": "Results", + "Name": "Warnings", "Type": "DataView", - "Desc": "Results of the sweep, including pipelines (as graph strings), IDs, and metric values." + "Desc": "Warning dataset" + }, + { + "Name": "OverallMetrics", + "Type": "DataView", + "Desc": "Overall metrics dataset" + }, + { + "Name": "PerInstanceMetrics", + "Type": "DataView", + "Desc": "Per instance metrics dataset" + }, + { + "Name": "ConfusionMatrix", + "Type": "DataView", + "Desc": "Confusion matrix dataset" } ] }, { - "Name": "Models.TrainTestBinaryEvaluator", - "Desc": "Train test for binary classification", + "Name": "Models.TrainTestEvaluator", + "Desc": "General train test for any supported evaluator", "FriendlyName": null, "ShortName": null, "Inputs": [ @@ -3530,6 +4975,15 @@ "SortOrder": 2.0, "IsNullable": false }, + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "The aggregated transform model from the pipeline before this command, to apply to the test data, and also include in the final model, together with the predictor model.", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": null + }, { "Name": "Nodes", "Type": { @@ -3538,7 +4992,7 @@ }, "Desc": "The training subgraph", "Required": true, - "SortOrder": 3.0, + "SortOrder": 4.0, "IsNullable": false }, { @@ -3558,7 +5012,7 @@ }, "Desc": "The training subgraph inputs", "Required": true, - "SortOrder": 4.0, + "SortOrder": 5.0, "IsNullable": false }, { @@ -3567,26 +5021,127 @@ "Kind": "Struct", "Fields": [ { - "Name": "Model", + "Name": "PredictorModel", "Type": "PredictorModel", - "Desc": "The model", - "Required": true, + "Desc": "The predictor model", + "Required": false, "SortOrder": 1.0, - "IsNullable": false + "IsNullable": false, + "Default": null + }, + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "Transform model", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null } ] }, "Desc": "The training subgraph outputs", "Required": true, - "SortOrder": 5.0, + "SortOrder": 6.0, "IsNullable": false + }, + { + "Name": "Kind", + "Type": { + "Kind": "Enum", + "Values": [ + "SignatureBinaryClassifierTrainer", + "SignatureMultiClassClassifierTrainer", + "SignatureRankerTrainer", + "SignatureRegressorTrainer", + "SignatureMultiOutputRegressorTrainer", + "SignatureAnomalyDetectorTrainer", + "SignatureClusteringTrainer" + ] + }, + "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", + "Required": false, + "SortOrder": 7.0, + "IsNullable": false, + "Default": "SignatureBinaryClassifierTrainer" + }, + { + "Name": "PipelineId", + "Type": "String", + "Desc": "Identifies which pipeline was run for this train test.", + "Required": false, + "SortOrder": 8.0, + "IsNullable": false, + "Default": null + }, + { + "Name": "IncludeTrainingMetrics", + "Type": "Bool", + "Desc": "Indicates whether to include and output training dataset metrics.", + "Required": false, + "SortOrder": 9.0, + "IsNullable": false, + "Default": false + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 10.0, + "IsNullable": false, + "Default": "Label" + }, + { + "Name": "WeightColumn", + "Type": "String", + "Desc": "Column to use for example weight", + "Aliases": [ + "weight" + ], + "Required": false, + "SortOrder": 11.0, + "IsNullable": false, + "Default": "Weight" + }, + { + "Name": "GroupColumn", + "Type": "String", + "Desc": "Column to use for grouping", + "Aliases": [ + "group" + ], + "Required": false, + "SortOrder": 12.0, + "IsNullable": false, + "Default": "GroupId" + }, + { + "Name": "NameColumn", + "Type": "String", + "Desc": "Name column name", + "Aliases": [ + "name" + ], + "Required": false, + "SortOrder": 13.0, + "IsNullable": false, + "Default": "Name" } ], "Outputs": [ { "Name": "PredictorModel", "Type": "PredictorModel", - "Desc": "The trained model" + "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." + }, + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." }, { "Name": "Warnings", @@ -3607,6 +5162,26 @@ "Name": "ConfusionMatrix", "Type": "DataView", "Desc": "Confusion matrix dataset" + }, + { + "Name": "TrainingWarnings", + "Type": "DataView", + "Desc": "Warning dataset for training" + }, + { + "Name": "TrainingOverallMetrics", + "Type": "DataView", + "Desc": "Overall metrics dataset for training" + }, + { + "Name": "TrainingPerInstanceMetrics", + "Type": "DataView", + "Desc": "Per instance metrics dataset for training" + }, + { + "Name": "TrainingConfusionMatrix", + "Type": "DataView", + "Desc": "Confusion matrix dataset for training" } ] }, @@ -17842,6 +19417,52 @@ "ITransformOutput" ] }, + { + "Name": "Transforms.FeatureCombiner", + "Desc": "Combines all the features into one feature column.", + "FriendlyName": "Feature Combiner", + "ShortName": "fc", + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Features", + "Type": { + "Kind": "Array", + "ItemType": "String" + }, + "Desc": "Features", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "Transforms.FeatureSelectorByCount", "Desc": "Selects the slots for which the count of non-default values is greater than or equal to a threshold.", @@ -19039,6 +20660,57 @@ "ITransformOutput" ] }, + { + "Name": "Transforms.LabelColumnKeyBooleanConverter", + "Desc": "Transforms the label to either key or bool (if needed) to make it suitable for classification.", + "FriendlyName": "Prepare Classification Label", + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "The label column", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "TextKeyValues", + "Type": "Bool", + "Desc": "Convert the key values to text", + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": true + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "Transforms.LabelIndicator", "Desc": "Label remapper used by OVA", @@ -19182,6 +20854,48 @@ "ITransformOutput" ] }, + { + "Name": "Transforms.LabelToFloatConverter", + "Desc": "Transforms the label to float to make it suitable for regression.", + "FriendlyName": "Prepare Regression Label", + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "LabelColumn", + "Type": "String", + "Desc": "The label column", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "Transforms.LightLda", "Desc": "The LDA transform implements LightLDA, a state-of-the-art implementation of Latent Dirichlet Allocation.", @@ -19845,6 +21559,40 @@ } ] }, + { + "Name": "Transforms.ManyHeterogeneousModelCombiner", + "Desc": "Combines a sequence of TransformModels and a PredictorModel into a single PredictorModel.", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "TransformModels", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "Transform model", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "Predictor model", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "Predictor model" + } + ] + }, { "Name": "Transforms.MeanVarianceNormalizer", "Desc": "Normalizes the data based on the computed mean and variance of the data.", @@ -20650,6 +22398,33 @@ } ] }, + { + "Name": "Transforms.ModelCombiner", + "Desc": "Combines a sequence of TransformModels into a single model", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Models", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "Input models", + "Required": false, + "SortOrder": 1.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "OutputModel", + "Type": "TransformModel", + "Desc": "Combined model" + } + ] + }, { "Name": "Transforms.NGramTranslator", "Desc": "Produces a bag of counts of ngrams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of ngrams and using the id in the dictionary as the index in the bag.", @@ -21177,6 +22952,48 @@ "ITransformOutput" ] }, + { + "Name": "Transforms.PredictedLabelColumnOriginalValueConverter", + "Desc": "Transforms a predicted label column to its original values, unless it is of type bool.", + "FriendlyName": "Convert Predicted Label", + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "PredictedLabelColumn", + "Type": "String", + "Desc": "The predicted label column", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputData", + "Type": "DataView", + "Desc": "Transformed dataset" + }, + { + "Name": "Model", + "Type": "TransformModel", + "Desc": "Transform model" + } + ], + "InputKind": [ + "ITransformInput" + ], + "OutputKind": [ + "ITransformOutput" + ] + }, { "Name": "Transforms.RandomNumberGenerator", "Desc": "Adds a column with a generated number sequence.", @@ -22327,6 +24144,55 @@ } ] }, + { + "Name": "Transforms.TrainTestDatasetSplitter", + "Desc": "Split the dataset into train and test sets", + "FriendlyName": "Dataset Train-Test Split", + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "Fraction", + "Type": "Float", + "Desc": "Fraction of training data", + "Required": false, + "SortOrder": 2.0, + "IsNullable": false, + "Default": 0.8 + }, + { + "Name": "StratificationColumn", + "Type": "String", + "Desc": "Stratification column", + "Aliases": [ + "strat" + ], + "Required": false, + "SortOrder": 3.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ + { + "Name": "TrainData", + "Type": "DataView", + "Desc": "Training data" + }, + { + "Name": "TestData", + "Type": "DataView", + "Desc": "Testing data" + } + ] + }, { "Name": "Transforms.TreeLeafFeaturizer", "Desc": "Trains a tree ensemble, or loads it from a file, then maps a numeric feature vector to three outputs: 1. A vector containing the individual tree outputs of the tree ensemble. 2. A vector indicating the leaves that the feature vector falls on in the tree ensemble. 3. A vector indicating the paths that the feature vector falls on in the tree ensemble. If a both a model file and a trainer are specified - will use the model file. If neither are specified, will train a default FastTree model. This can handle key labels by training a regression model towards their optionally permuted indices.", @@ -22425,6 +24291,37 @@ } ] }, + { + "Name": "Transforms.TwoHeterogeneousModelCombiner", + "Desc": "Combines a TransformModel and a PredictorModel into a single PredictorModel.", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "TransformModel", + "Type": "TransformModel", + "Desc": "Transform model", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + }, + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "Predictor model", + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "PredictorModel", + "Type": "PredictorModel", + "Desc": "Predictor model" + } + ] + }, { "Name": "Transforms.VectorToImage", "Desc": "Converts vector array into image type.", @@ -28907,6 +30804,10 @@ "Kind": "ILearningPipelineLoader", "Settings": [] }, + { + "Kind": "ILearningPipelineLoader", + "Settings": [] + }, { "Kind": "IMulticlassClassificationOutput", "Settings": [] diff --git a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs index 7284ce6c1a..ad7266bbee 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs @@ -90,7 +90,7 @@ public void MatMulATest(int matTest, int srcTest, int dstTest, float[] expected) AlignedArray src = _testSrcVectors[srcTest]; AlignedArray dst = _testDstVectors[dstTest]; - CpuMathUtils.MatTimesSrc(false, false, mat, src, dst, dst.Size); + CpuMathUtils.MatrixTimesSource(false, false, mat, src, dst, dst.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -106,7 +106,7 @@ public void MatMulAAddTest(int matTest, int srcTest, int dstTest, float[] expect AlignedArray src = _testSrcVectors[srcTest]; AlignedArray dst = _testDstVectors[dstTest]; - CpuMathUtils.MatTimesSrc(false, true, mat, src, dst, dst.Size); + CpuMathUtils.MatrixTimesSource(false, true, mat, src, dst, dst.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -122,7 +122,7 @@ public void MatMulTranATest(int matTest, int srcTest, int dstTest, float[] expec AlignedArray src = _testSrcVectors[srcTest]; AlignedArray dst = _testDstVectors[dstTest]; - CpuMathUtils.MatTimesSrc(true, false, mat, src, dst, src.Size); + CpuMathUtils.MatrixTimesSource(true, false, mat, src, dst, src.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -138,7 +138,7 @@ public void MatMulTranAAddTest(int matTest, int srcTest, int dstTest, float[] ex AlignedArray src = _testSrcVectors[srcTest]; AlignedArray dst = _testDstVectors[dstTest]; - CpuMathUtils.MatTimesSrc(true, true, mat, src, dst, src.Size); + CpuMathUtils.MatrixTimesSource(true, true, mat, src, dst, src.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -155,7 +155,7 @@ public void MatMulPATest(int matTest, int srcTest, int dstTest, float[] expected AlignedArray dst = _testDstVectors[dstTest]; int[] idx = _testIndexArray; - CpuMathUtils.MatTimesSrc(false, false, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, dst.Size); + CpuMathUtils.MatrixTimesSource(false, false, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, dst.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -172,7 +172,7 @@ public void MatMulPAAddTest(int matTest, int srcTest, int dstTest, float[] expec AlignedArray dst = _testDstVectors[dstTest]; int[] idx = _testIndexArray; - CpuMathUtils.MatTimesSrc(false, true, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, dst.Size); + CpuMathUtils.MatrixTimesSource(false, true, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, dst.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -189,7 +189,7 @@ public void MatMulTranPATest(int matTest, int srcTest, int dstTest, float[] expe AlignedArray dst = _testDstVectors[dstTest]; int[] idx = _testIndexArray; - CpuMathUtils.MatTimesSrc(true, false, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, src.Size); + CpuMathUtils.MatrixTimesSource(true, false, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, src.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); @@ -206,7 +206,7 @@ public void MatMulTranPAAddTest(int matTest, int srcTest, int dstTest, float[] e AlignedArray dst = _testDstVectors[dstTest]; int[] idx = _testIndexArray; - CpuMathUtils.MatTimesSrc(true, true, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, src.Size); + CpuMathUtils.MatrixTimesSource(true, true, mat, idx, src, 0, 0, (srcTest == 0) ? 4 : 9, dst, src.Size); float[] actual = new float[dst.Size]; dst.CopyTo(actual, 0, dst.Size); Assert.Equal(expected, actual, _matMulComparer); From aa93772a31e4c7bd940827496123611e6340d8aa Mon Sep 17 00:00:00 2001 From: J W Date: Mon, 1 Oct 2018 14:29:07 -0400 Subject: [PATCH 09/19] Regen entry points after merging --- .../Common/EntryPoints/core_ep-list.tsv | 22 - .../Common/EntryPoints/core_manifest.json | 2239 ++--------------- .../UnitTests/TestEntryPoints.cs | 2 +- 3 files changed, 170 insertions(+), 2093 deletions(-) diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index e61593029f..c69bf1171f 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -1,31 +1,20 @@ Data.CustomTextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData ImportText Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output -Data.CustomTextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData ImportText Microsoft.ML.Runtime.EntryPoints.ImportTextData+Input Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output -Data.DataViewReference Pass dataview from memory to experiment Microsoft.ML.Runtime.EntryPoints.DataViewReference ImportData Microsoft.ML.Runtime.EntryPoints.DataViewReference+Input Microsoft.ML.Runtime.EntryPoints.DataViewReference+Output Data.DataViewReference Pass dataview from memory to experiment Microsoft.ML.Runtime.EntryPoints.DataViewReference ImportData Microsoft.ML.Runtime.EntryPoints.DataViewReference+Input Microsoft.ML.Runtime.EntryPoints.DataViewReference+Output Data.IDataViewArrayConverter Create an array variable of IDataView Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput -Data.IDataViewArrayConverter Create an array variable of IDataView Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput Data.PredictorModelArrayConverter Create an array variable of IPredictorModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput -Data.PredictorModelArrayConverter Create an array variable of IPredictorModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput -Data.TextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData TextLoader Microsoft.ML.Runtime.EntryPoints.ImportTextData+LoaderInput Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output Data.TextLoader Import a dataset from a text file Microsoft.ML.Runtime.EntryPoints.ImportTextData TextLoader Microsoft.ML.Runtime.EntryPoints.ImportTextData+LoaderInput Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output Data.TransformModelArrayConverter Create an array variable of ITransformModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelOutput -Data.TransformModelArrayConverter Create an array variable of ITransformModel Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro MakeArray Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelInput Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelOutput Models.AnomalyDetectionEvaluator Evaluates an anomaly detection scored dataset. Microsoft.ML.Runtime.Data.Evaluate AnomalyDetection Microsoft.ML.Runtime.Data.AnomalyDetectionMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.AnomalyPipelineEnsemble Combine anomaly detection models into an ensemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator CreateAnomalyPipelineEnsemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator+PipelineAnomalyInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Models.BinaryClassificationEvaluator Evaluates a binary classification scored dataset. Microsoft.ML.Runtime.Data.Evaluate Binary Microsoft.ML.Runtime.Data.BinaryClassifierMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClassificationEvaluateOutput Models.BinaryCrossValidator Cross validation for binary classification Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro CrossValidateBinary Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Output] -Models.BinaryCrossValidator Cross validation for binary classification Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro CrossValidateBinary Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Output] Models.BinaryEnsemble Combine binary classifiers into an ensemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator CreateBinaryEnsemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator+ClassifierInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Models.BinaryPipelineEnsemble Combine binary classification models into an ensemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator CreateBinaryPipelineEnsemble Microsoft.ML.Runtime.EntryPoints.EnsembleCreator+PipelineClassifierInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Models.ClassificationEvaluator Evaluates a multi class classification scored dataset. Microsoft.ML.Runtime.Data.Evaluate MultiClass Microsoft.ML.Runtime.Data.MultiClassMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClassificationEvaluateOutput Models.ClusterEvaluator Evaluates a clustering scored dataset. Microsoft.ML.Runtime.Data.Evaluate Clustering Microsoft.ML.Runtime.Data.ClusteringMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.CrossValidationResultsCombiner Combine the metric data views returned from cross validation. Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CombineMetrics Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombineMetricsInput Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombinedOutput -Models.CrossValidationResultsCombiner Combine the metric data views returned from cross validation. Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CombineMetrics Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombineMetricsInput Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+CombinedOutput -Models.CrossValidator Cross validation for general learning Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CrossValidate Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Output] Models.CrossValidator Cross validation for general learning Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro CrossValidate Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationMacro+Output] Models.CrossValidatorDatasetSplitter Split the dataset into the specified number of cross-validation folds (train and test sets) Microsoft.ML.Runtime.EntryPoints.CVSplit Split Microsoft.ML.Runtime.EntryPoints.CVSplit+Input Microsoft.ML.Runtime.EntryPoints.CVSplit+Output -Models.CrossValidatorDatasetSplitter Split the dataset into the specified number of cross-validation folds (train and test sets) Microsoft.ML.Runtime.EntryPoints.CVSplit Split Microsoft.ML.Runtime.EntryPoints.CVSplit+Input Microsoft.ML.Runtime.EntryPoints.CVSplit+Output -Models.DatasetTransformer Applies a TransformModel to a dataset. Microsoft.ML.Runtime.EntryPoints.ModelOperations Apply Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelOutput Models.DatasetTransformer Applies a TransformModel to a dataset. Microsoft.ML.Runtime.EntryPoints.ModelOperations Apply Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+ApplyTransformModelOutput Models.EnsembleSummary Summarize a pipeline ensemble predictor. Microsoft.ML.Runtime.Ensemble.EntryPoints.PipelineEnsemble Summarize Microsoft.ML.Runtime.EntryPoints.SummarizePredictor+Input Microsoft.ML.Runtime.Ensemble.EntryPoints.PipelineEnsemble+SummaryOutput Models.FixedPlattCalibrator Apply a Platt calibrator with a fixed slope and offset to an input model Microsoft.ML.Runtime.Internal.Calibration.Calibrate FixedPlatt Microsoft.ML.Runtime.Internal.Calibration.Calibrate+FixedPlattInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CalibratorOutput @@ -33,7 +22,6 @@ Models.MultiClassPipelineEnsemble Combine multiclass classifiers into an ensembl Models.MultiOutputRegressionEvaluator Evaluates a multi output regression scored dataset. Microsoft.ML.Runtime.Data.Evaluate MultiOutputRegression Microsoft.ML.Runtime.Data.MultiOutputRegressionMamlEvaluator+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput Models.NaiveCalibrator Apply a Naive calibrator to an input model Microsoft.ML.Runtime.Internal.Calibration.Calibrate Naive Microsoft.ML.Runtime.Internal.Calibration.Calibrate+NoArgumentsInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CalibratorOutput Models.OneVersusAll One-vs-All macro (OVA) Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro OneVersusAll Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Output] -Models.OneVersusAll One-vs-All macro (OVA) Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro OneVersusAll Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.OneVersusAllMacro+Output] Models.OnnxConverter Converts the model to ONNX format. Microsoft.ML.Runtime.Model.Onnx.SaveOnnxCommand Apply Microsoft.ML.Runtime.Model.Onnx.SaveOnnxCommand+Arguments Microsoft.ML.Runtime.Model.Onnx.SaveOnnxCommand+Output Models.OvaModelCombiner Combines a sequence of PredictorModels into a single model Microsoft.ML.Runtime.Learners.OvaPredictor CombineOvaModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineOvaPredictorModelsInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput Models.PAVCalibrator Apply a PAV calibrator to an input model Microsoft.ML.Runtime.Internal.Calibration.Calibrate Pav Microsoft.ML.Runtime.Internal.Calibration.Calibrate+NoArgumentsInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CalibratorOutput @@ -47,8 +35,6 @@ Models.RegressionPipelineEnsemble Combine regression models into an ensemble Mic Models.Summarizer Summarize a linear regression predictor. Microsoft.ML.Runtime.EntryPoints.SummarizePredictor Summarize Microsoft.ML.Runtime.EntryPoints.SummarizePredictor+Input Microsoft.ML.Runtime.EntryPoints.CommonOutputs+SummaryOutput Models.SweepResultExtractor Extracts the sweep result. Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro ExtractSweepResult Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro+ResultInput Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro+Output Models.TrainTestBinaryEvaluator Train test for binary classification Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro TrainTestBinary Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Output] -Models.TrainTestBinaryEvaluator Train test for binary classification Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro TrainTestBinary Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Output] -Models.TrainTestEvaluator General train test for any supported evaluator Microsoft.ML.Runtime.EntryPoints.TrainTestMacro TrainTest Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Output] Models.TrainTestEvaluator General train test for any supported evaluator Microsoft.ML.Runtime.EntryPoints.TrainTestMacro TrainTest Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Output] Trainers.AveragedPerceptronBinaryClassifier Averaged Perceptron Binary Classifier. Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer TrainBinary Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.EnsembleBinaryClassifier Train binary ensemble. Microsoft.ML.Ensemble.EntryPoints.Ensemble CreateBinaryEnsemble Microsoft.ML.Runtime.Ensemble.EnsembleTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput @@ -99,7 +85,6 @@ Transforms.DatasetScorer Score a dataset with a predictor model Microsoft.ML.Run Transforms.DatasetTransformScorer Score a dataset with a transform model Microsoft.ML.Runtime.EntryPoints.ScoreModel ScoreUsingTransform Microsoft.ML.Runtime.EntryPoints.ScoreModel+InputTransformScorer Microsoft.ML.Runtime.EntryPoints.ScoreModel+Output Transforms.Dictionarizer Converts input values (words, numbers, etc.) to index in a dictionary. Microsoft.ML.Runtime.Transforms.TextAnalytics TermTransform Microsoft.ML.Runtime.Data.TermTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.FeatureCombiner Combines all the features into one feature column. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareFeatures Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+FeatureCombinerInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput -Transforms.FeatureCombiner Combines all the features into one feature column. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareFeatures Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+FeatureCombinerInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.FeatureSelectorByCount Selects the slots for which the count of non-default values is greater than or equal to a threshold. Microsoft.ML.Runtime.EntryPoints.SelectFeatures CountSelect Microsoft.ML.Runtime.Data.CountFeatureSelectionTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.FeatureSelectorByMutualInformation Selects the top k slots across all specified columns ordered by their mutual information with the label column. Microsoft.ML.Runtime.EntryPoints.SelectFeatures MutualInformationSelect Microsoft.ML.Runtime.Data.MutualInformationFeatureSelectionTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.GlobalContrastNormalizer Performs a global contrast normalization on input values: Y = (s * X - M) / D, where s is a scale, M is mean and D is either L2 norm or standard deviation. Microsoft.ML.Runtime.Data.LpNormalization GcNormalize Microsoft.ML.Runtime.Data.LpNormNormalizerTransform+GcnArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -110,15 +95,12 @@ Transforms.ImagePixelExtractor Extract color plane(s) from an image. Options inc Transforms.ImageResizer Scales an image to specified dimensions using one of the three scale types: isotropic with padding, isotropic with cropping or anisotropic. In case of isotropic padding, transparent color is used to pad resulting image. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics ImageResizer Microsoft.ML.Runtime.ImageAnalytics.ImageResizerTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.KeyToTextConverter KeyToValueTransform utilizes KeyValues metadata to map key indices to the corresponding values in the KeyValues metadata. Microsoft.ML.Runtime.Data.Categorical KeyToText Microsoft.ML.Runtime.Data.KeyToValueTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelColumnKeyBooleanConverter Transforms the label to either key or bool (if needed) to make it suitable for classification. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareClassificationLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+ClassificationLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput -Transforms.LabelColumnKeyBooleanConverter Transforms the label to either key or bool (if needed) to make it suitable for classification. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareClassificationLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+ClassificationLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelIndicator Label remapper used by OVA Microsoft.ML.Runtime.Data.LabelIndicatorTransform LabelIndicator Microsoft.ML.Runtime.Data.LabelIndicatorTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LabelToFloatConverter Transforms the label to float to make it suitable for regression. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareRegressionLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+RegressionLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput -Transforms.LabelToFloatConverter Transforms the label to float to make it suitable for regression. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner PrepareRegressionLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+RegressionLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LightLda The LDA transform implements LightLDA, a state-of-the-art implementation of Latent Dirichlet Allocation. Microsoft.ML.Runtime.Transforms.TextAnalytics LightLda Microsoft.ML.Runtime.TextAnalytics.LdaTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LogMeanVarianceNormalizer Normalizes the data based on the computed mean and variance of the logarithm of the data. Microsoft.ML.Runtime.Data.Normalize LogMeanVar Microsoft.ML.Runtime.Data.NormalizeTransform+LogMeanVarArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.LpNormalizer Normalize vectors (rows) individually by rescaling them to unit norm (L2, L1 or LInf). Performs the following operation on a vector X: Y = (X - M) / D, where M is mean and D is either L2 norm, L1 norm or LInf norm. Microsoft.ML.Runtime.Data.LpNormalization Normalize Microsoft.ML.Runtime.Data.LpNormNormalizerTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.ManyHeterogeneousModelCombiner Combines a sequence of TransformModels and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput -Transforms.ManyHeterogeneousModelCombiner Combines a sequence of TransformModels and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput Transforms.MeanVarianceNormalizer Normalizes the data based on the computed mean and variance of the data. Microsoft.ML.Runtime.Data.Normalize MeanVar Microsoft.ML.Runtime.Data.NormalizeTransform+MeanVarArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.MinMaxNormalizer Normalizes the data based on the observed minimum and maximum values of the data. Microsoft.ML.Runtime.Data.Normalize MinMax Microsoft.ML.Runtime.Data.NormalizeTransform+MinMaxArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.MissingValueHandler Handle missing values by replacing them with either the default value or the mean/min/max value (for non-text columns only). An indicator column can optionally be concatenated, if theinput column type is numeric. Microsoft.ML.Runtime.Data.NAHandling Handle Microsoft.ML.Runtime.Data.NAHandleTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -127,13 +109,11 @@ Transforms.MissingValuesDropper Removes NAs from vector columns. Microsoft.ML.Ru Transforms.MissingValuesRowDropper Filters out rows that contain missing values. Microsoft.ML.Runtime.Data.NAHandling Filter Microsoft.ML.Runtime.Data.NAFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.MissingValueSubstitutor Create an output column of the same type and size of the input column, where missing values are replaced with either the default value or the mean/min/max value (for non-text columns only). Microsoft.ML.Runtime.Data.NAHandling Replace Microsoft.ML.Runtime.Data.NAReplaceTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.ModelCombiner Combines a sequence of TransformModels into a single model Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTransformModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsOutput -Transforms.ModelCombiner Combines a sequence of TransformModels into a single model Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTransformModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+CombineTransformModelsOutput Transforms.NGramTranslator Produces a bag of counts of ngrams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of ngrams and using the id in the dictionary as the index in the bag. Microsoft.ML.Runtime.Transforms.TextAnalytics NGramTransform Microsoft.ML.Runtime.Data.NgramTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.NoOperation Does nothing. Microsoft.ML.Runtime.Data.NopTransform Nop Microsoft.ML.Runtime.Data.NopTransform+NopInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.OptionalColumnCreator If the source column does not exist after deserialization, create a column with the right type and default values. Microsoft.ML.Runtime.DataPipe.OptionalColumnTransform MakeOptional Microsoft.ML.Runtime.DataPipe.OptionalColumnTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.PcaCalculator PCA is a dimensionality-reduction transform which computes the projection of a numeric vector onto a low-rank subspace. Microsoft.ML.Runtime.Data.PcaTransform Calculate Microsoft.ML.Runtime.Data.PcaTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.PredictedLabelColumnOriginalValueConverter Transforms a predicted label column to its original values, unless it is of type bool. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner ConvertPredictedLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+PredictedLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput -Transforms.PredictedLabelColumnOriginalValueConverter Transforms a predicted label column to its original values, unless it is of type bool. Microsoft.ML.Runtime.EntryPoints.FeatureCombiner ConvertPredictedLabel Microsoft.ML.Runtime.EntryPoints.FeatureCombiner+PredictedLabelInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.RandomNumberGenerator Adds a column with a generated number sequence. Microsoft.ML.Runtime.Data.RandomNumberGenerator Generate Microsoft.ML.Runtime.Data.GenerateNumberTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.RowRangeFilter Filters a dataview on a column of type Single, Double or Key (contiguous). Keeps the values that are in the specified min/max range. NaNs are always filtered out. If the input is a Key type, the min/max are considered percentages of the number of values. Microsoft.ML.Runtime.EntryPoints.SelectRows FilterByRange Microsoft.ML.Runtime.Data.RangeFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.RowSkipAndTakeFilter Allows limiting input to a subset of rows at an optional offset. Can be used to implement data paging. Microsoft.ML.Runtime.EntryPoints.SelectRows SkipAndTakeFilter Microsoft.ML.Runtime.Data.SkipTakeFilter+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput @@ -147,10 +127,8 @@ Transforms.TensorFlowScorer Transforms the data using the TensorFlow model. Micr Transforms.TextFeaturizer A transform that turns a collection of text documents into numerical feature vectors. The feature vectors are normalized counts of (word and/or character) ngrams in a given tokenized text. Microsoft.ML.Runtime.Transforms.TextAnalytics TextTransform Microsoft.ML.Runtime.Data.TextTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TextToKeyConverter Converts input values (words, numbers, etc.) to index in a dictionary. Microsoft.ML.Runtime.Data.Categorical TextToKey Microsoft.ML.Runtime.Data.TermTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TrainTestDatasetSplitter Split the dataset into train and test sets Microsoft.ML.Runtime.EntryPoints.TrainTestSplit Split Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Input Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Output -Transforms.TrainTestDatasetSplitter Split the dataset into train and test sets Microsoft.ML.Runtime.EntryPoints.TrainTestSplit Split Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Input Microsoft.ML.Runtime.EntryPoints.TrainTestSplit+Output Transforms.TreeLeafFeaturizer Trains a tree ensemble, or loads it from a file, then maps a numeric feature vector to three outputs: 1. A vector containing the individual tree outputs of the tree ensemble. 2. A vector indicating the leaves that the feature vector falls on in the tree ensemble. 3. A vector indicating the paths that the feature vector falls on in the tree ensemble. If a both a model file and a trainer are specified - will use the model file. If neither are specified, will train a default FastTree model. This can handle key labels by training a regression model towards their optionally permuted indices. Microsoft.ML.Runtime.Data.TreeFeaturize Featurizer Microsoft.ML.Runtime.Data.TreeEnsembleFeaturizerTransform+ArgumentsForEntryPoint Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.TwoHeterogeneousModelCombiner Combines a TransformModel and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTwoModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+SimplePredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput -Transforms.TwoHeterogeneousModelCombiner Combines a TransformModel and a PredictorModel into a single PredictorModel. Microsoft.ML.Runtime.EntryPoints.ModelOperations CombineTwoModels Microsoft.ML.Runtime.EntryPoints.ModelOperations+SimplePredictorModelInput Microsoft.ML.Runtime.EntryPoints.ModelOperations+PredictorModelOutput Transforms.VectorToImage Converts vector array into image type. Microsoft.ML.Runtime.ImageAnalytics.EntryPoints.ImageAnalytics VectorToImage Microsoft.ML.Runtime.ImageAnalytics.VectorToImageTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.WordEmbeddings Word Embeddings transform is a text featurizer which converts vectors of text tokens into sentence vectors using a pre-trained model Microsoft.ML.Runtime.Transforms.TextAnalytics WordEmbeddings Microsoft.ML.Runtime.Data.WordEmbeddingsTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.WordTokenizer The input to this transform is text, and the output is a vector of text containing the words (tokens) in the original text. The separator is space, but can be specified as any other character (or multiple characters) if needed. Microsoft.ML.Runtime.Transforms.TextAnalytics DelimitedTokenizeTransform Microsoft.ML.Runtime.Data.DelimitedTokenizeTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index a439e09a27..03f8cb170f 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -38,67 +38,6 @@ } ] }, - { - "Name": "Data.CustomTextLoader", - "Desc": "Import a dataset from a text file", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "InputFile", - "Type": "FileHandle", - "Desc": "Location of the input file", - "Aliases": [ - "data" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "CustomSchema", - "Type": "String", - "Desc": "Custom schema to use for parsing", - "Aliases": [ - "schema" - ], - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The resulting data view" - } - ] - }, - { - "Name": "Data.DataViewReference", - "Desc": "Pass dataview from memory to experiment", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Pointer to IDataView in memory", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The resulting data view" - } - ] - }, { "Name": "Data.DataViewReference", "Desc": "Pass dataview from memory to experiment", @@ -151,64 +90,6 @@ } ] }, - { - "Name": "Data.IDataViewArrayConverter", - "Desc": "Create an array variable of IDataView", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "The data sets", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputData", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "The data set array" - } - ] - }, - { - "Name": "Data.PredictorModelArrayConverter", - "Desc": "Create an array variable of IPredictorModel", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Model", - "Type": { - "Kind": "Array", - "ItemType": "PredictorModel" - }, - "Desc": "The models", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputModel", - "Type": { - "Kind": "Array", - "ItemType": "PredictorModel" - }, - "Desc": "The model array" - } - ] - }, { "Name": "Data.PredictorModelArrayConverter", "Desc": "Create an array variable of IPredictorModel", @@ -589,423 +470,44 @@ ] }, { - "Name": "Data.TextLoader", - "Desc": "Import a dataset from a text file", + "Name": "Data.TransformModelArrayConverter", + "Desc": "Create an array variable of ITransformModel", "FriendlyName": null, "ShortName": null, "Inputs": [ { - "Name": "InputFile", - "Type": "FileHandle", - "Desc": "Location of the input file", - "Aliases": [ - "data" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "Arguments", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Column", - "Type": { - "Kind": "Array", - "ItemType": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Name", - "Type": "String", - "Desc": "Name of the column", - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "Type", - "Type": { - "Kind": "Enum", - "Values": [ - "I1", - "U1", - "I2", - "U2", - "I4", - "U4", - "I8", - "U8", - "R4", - "Num", - "R8", - "TX", - "Text", - "TXT", - "BL", - "Bool", - "TimeSpan", - "TS", - "DT", - "DateTime", - "DZ", - "DateTimeZone", - "UG", - "U16" - ] - }, - "Desc": "Type of the items in the column", - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null - }, - { - "Name": "Source", - "Type": { - "Kind": "Array", - "ItemType": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Min", - "Type": "Int", - "Desc": "First index in the range", - "Required": true, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 0 - }, - { - "Name": "Max", - "Type": "Int", - "Desc": "Last index in the range", - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null - }, - { - "Name": "AutoEnd", - "Type": "Bool", - "Desc": "This range extends to the end of the line, but should be a fixed number of items", - "Aliases": [ - "auto" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, - { - "Name": "VariableEnd", - "Type": "Bool", - "Desc": "This range extends to the end of the line, which can vary from line to line", - "Aliases": [ - "var" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, - { - "Name": "AllOther", - "Type": "Bool", - "Desc": "This range includes only other indices not specified", - "Aliases": [ - "other" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, - { - "Name": "ForceVector", - "Type": "Bool", - "Desc": "Force scalar columns to be treated as vectors of length one", - "Aliases": [ - "vector" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - } - ] - } - }, - "Desc": "Source index range(s) of the column", - "Aliases": [ - "src" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "KeyRange", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Min", - "Type": "UInt", - "Desc": "First index in the range", - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 0 - }, - { - "Name": "Max", - "Type": "UInt", - "Desc": "Last index in the range", - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null - }, - { - "Name": "Contiguous", - "Type": "Bool", - "Desc": "Whether the key is contiguous", - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": true - } - ] - }, - "Desc": "For a key column, this defines the range of values", - "Aliases": [ - "key" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - } - ] - } - }, - "Desc": "Column groups. Each group is specified as name:type:numeric-ranges, eg, col=Features:R4:1-17,26,35-40", - "Aliases": [ - "col" - ], - "Required": false, - "SortOrder": 1.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "UseThreads", - "Type": "Bool", - "Desc": "Use separate parsing threads?", - "Aliases": [ - "threads" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": true - }, - { - "Name": "HeaderFile", - "Type": "String", - "Desc": "File containing a header with feature names. If specified, header defined in the data file (header+) is ignored.", - "Aliases": [ - "hf" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "MaxRows", - "Type": "Int", - "Desc": "Maximum number of rows to produce", - "Aliases": [ - "rows" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null - }, - { - "Name": "AllowQuoting", - "Type": "Bool", - "Desc": "Whether the input may include quoted values, which can contain separator characters, colons, and distinguish empty values from missing values. When true, consecutive separators denote a missing value and an empty value is denoted by \"\". When false, consecutive separators denote an empty value.", - "Aliases": [ - "quote" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": true - }, - { - "Name": "AllowSparse", - "Type": "Bool", - "Desc": "Whether the input may include sparse representations", - "Aliases": [ - "sparse" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": true - }, - { - "Name": "InputSize", - "Type": "Int", - "Desc": "Number of source columns in the text data. Default is that sparse rows contain their size information.", - "Aliases": [ - "size" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": true, - "Default": null - }, - { - "Name": "Separator", - "Type": { - "Kind": "Array", - "ItemType": "Char" - }, - "Desc": "Source column separator.", - "Aliases": [ - "sep" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": [ - "\t" - ] - }, - { - "Name": "TrimWhitespace", - "Type": "Bool", - "Desc": "Remove trailing whitespace from lines", - "Aliases": [ - "trim" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - }, - { - "Name": "HasHeader", - "Type": "Bool", - "Desc": "Data file has header with feature names. Header is read only if options 'hs' and 'hf' are not specified.", - "Aliases": [ - "header" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": false - } - ] - }, - "Desc": "Arguments", - "Aliases": [ - "args" - ], - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The resulting data view" - } - ], - "InputKind": [ - "ILearningPipelineLoader" - ] - }, - { - "Name": "Data.TransformModelArrayConverter", - "Desc": "Create an array variable of ITransformModel", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TransformModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The models", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The model array" - } - ] - }, - { - "Name": "Data.TransformModelArrayConverter", - "Desc": "Create an array variable of ITransformModel", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TransformModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The models", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The model array" - } - ] - }, - { - "Name": "Models.AnomalyDetectionEvaluator", - "Desc": "Evaluates an anomaly detection scored dataset.", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data to be used for evaluation.", + "Name": "TransformModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The models", + "Required": true, + "SortOrder": 1.0, + "IsNullable": false + } + ], + "Outputs": [ + { + "Name": "OutputModel", + "Type": { + "Kind": "Array", + "ItemType": "TransformModel" + }, + "Desc": "The model array" + } + ] + }, + { + "Name": "Models.AnomalyDetectionEvaluator", + "Desc": "Evaluates an anomaly detection scored dataset.", + "FriendlyName": null, + "ShortName": null, + "Inputs": [ + { + "Name": "Data", + "Type": "DataView", + "Desc": "The data to be used for evaluation.", "Aliases": [ "data" ], @@ -1290,230 +792,97 @@ "Default": -1 }, { - "Name": "NumAuPrcExamples", - "Type": "Int", - "Desc": "The number of samples to use for AUPRC calculation. Specify 0 for no AUPRC calculation", - "Aliases": [ - "numauprc" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": 100000 - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels.", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Weight column name.", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "ScoreColumn", - "Type": "String", - "Desc": "Score column name.", - "Aliases": [ - "score" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "StratColumn", - "Type": { - "Kind": "Array", - "ItemType": "String" - }, - "Desc": "Stratification column name.", - "Aliases": [ - "strat" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "Warnings", - "Type": "DataView", - "Desc": "Warning dataset" - }, - { - "Name": "OverallMetrics", - "Type": "DataView", - "Desc": "Overall metrics dataset" - }, - { - "Name": "PerInstanceMetrics", - "Type": "DataView", - "Desc": "Per instance metrics dataset" - }, - { - "Name": "ConfusionMatrix", - "Type": "DataView", - "Desc": "Confusion matrix dataset" - } - ], - "InputKind": [ - "IEvaluatorInput" - ], - "OutputKind": [ - "IClassificationEvaluatorOutput", - "IEvaluatorOutput" - ] - }, - { - "Name": "Models.BinaryCrossValidator", - "Desc": "Cross validation for binary classification", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data set", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "Nodes", - "Type": { - "Kind": "Array", - "ItemType": "Node" - }, - "Desc": "The training subgraph", - "Required": true, - "SortOrder": 3.0, - "IsNullable": false - }, - { - "Name": "Inputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data to be used for training", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph inputs", - "Required": true, - "SortOrder": 4.0, - "IsNullable": false + "Name": "NumAuPrcExamples", + "Type": "Int", + "Desc": "The number of samples to use for AUPRC calculation. Specify 0 for no AUPRC calculation", + "Aliases": [ + "numauprc" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": 100000 }, { - "Name": "Outputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Model", - "Type": "PredictorModel", - "Desc": "The model", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph outputs", - "Required": true, - "SortOrder": 5.0, - "IsNullable": false + "Name": "LabelColumn", + "Type": "String", + "Desc": "Column to use for labels.", + "Aliases": [ + "lab" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null }, { - "Name": "StratificationColumn", + "Name": "WeightColumn", "Type": "String", - "Desc": "Column to use for stratification", + "Desc": "Weight column name.", "Aliases": [ - "strat" + "weight" ], "Required": false, - "SortOrder": 7.0, + "SortOrder": 150.0, "IsNullable": false, "Default": null }, { - "Name": "NumFolds", - "Type": "Int", - "Desc": "Number of folds in k-fold cross-validation", + "Name": "ScoreColumn", + "Type": "String", + "Desc": "Score column name.", "Aliases": [ - "k" + "score" ], "Required": false, - "SortOrder": 8.0, + "SortOrder": 150.0, "IsNullable": false, - "Default": 2 - } - ], - "Outputs": [ + "Default": null + }, { - "Name": "PredictorModel", + "Name": "StratColumn", "Type": { "Kind": "Array", - "ItemType": "PredictorModel" + "ItemType": "String" }, - "Desc": "The trained model" - }, + "Desc": "Stratification column name.", + "Aliases": [ + "strat" + ], + "Required": false, + "SortOrder": 150.0, + "IsNullable": false, + "Default": null + } + ], + "Outputs": [ { "Name": "Warnings", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, + "Type": "DataView", "Desc": "Warning dataset" }, { "Name": "OverallMetrics", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, + "Type": "DataView", "Desc": "Overall metrics dataset" }, { "Name": "PerInstanceMetrics", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, + "Type": "DataView", "Desc": "Per instance metrics dataset" }, { "Name": "ConfusionMatrix", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, + "Type": "DataView", "Desc": "Confusion matrix dataset" } + ], + "InputKind": [ + "IEvaluatorInput" + ], + "OutputKind": [ + "IClassificationEvaluatorOutput", + "IEvaluatorOutput" ] }, { @@ -2009,183 +1378,36 @@ "weight" ], "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "ScoreColumn", - "Type": "String", - "Desc": "Score column name.", - "Aliases": [ - "score" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "StratColumn", - "Type": { - "Kind": "Array", - "ItemType": "String" - }, - "Desc": "Stratification column name.", - "Aliases": [ - "strat" - ], - "Required": false, - "SortOrder": 150.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "Warnings", - "Type": "DataView", - "Desc": "Warning dataset" - }, - { - "Name": "OverallMetrics", - "Type": "DataView", - "Desc": "Overall metrics dataset" - }, - { - "Name": "PerInstanceMetrics", - "Type": "DataView", - "Desc": "Per instance metrics dataset" - } - ], - "InputKind": [ - "IEvaluatorInput" - ], - "OutputKind": [ - "IEvaluatorOutput" - ] - }, - { - "Name": "Models.CrossValidationResultsCombiner", - "Desc": "Combine the metric data views returned from cross validation.", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "OverallMetrics", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "Overall metrics datasets", - "Required": false, - "SortOrder": 1.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "PerInstanceMetrics", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "Per instance metrics datasets", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "ConfusionMatrix", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "Confusion matrix datasets", - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "Warnings", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "Warning datasets", - "Required": false, - "SortOrder": 4.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "Kind", - "Type": { - "Kind": "Enum", - "Values": [ - "SignatureBinaryClassifierTrainer", - "SignatureMultiClassClassifierTrainer", - "SignatureRankerTrainer", - "SignatureRegressorTrainer", - "SignatureMultiOutputRegressorTrainer", - "SignatureAnomalyDetectorTrainer", - "SignatureClusteringTrainer" - ] - }, - "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", - "Required": true, - "SortOrder": 5.0, - "IsNullable": false, - "Default": "SignatureBinaryClassifierTrainer" - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "The label column name", - "Aliases": [ - "Label" - ], - "Required": false, - "SortOrder": 6.0, - "IsNullable": false, - "Default": "Label" - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Column to use for example weight", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 7.0, + "SortOrder": 150.0, "IsNullable": false, - "Default": "Weight" + "Default": null }, { - "Name": "GroupColumn", + "Name": "ScoreColumn", "Type": "String", - "Desc": "Column to use for grouping", + "Desc": "Score column name.", "Aliases": [ - "group" + "score" ], "Required": false, - "SortOrder": 8.0, + "SortOrder": 150.0, "IsNullable": false, - "Default": "GroupId" + "Default": null }, { - "Name": "NameColumn", - "Type": "String", - "Desc": "Name column name", + "Name": "StratColumn", + "Type": { + "Kind": "Array", + "ItemType": "String" + }, + "Desc": "Stratification column name.", "Aliases": [ - "name" + "strat" ], "Required": false, - "SortOrder": 9.0, + "SortOrder": 150.0, "IsNullable": false, - "Default": "Name" + "Default": null } ], "Outputs": [ @@ -2203,12 +1425,13 @@ "Name": "PerInstanceMetrics", "Type": "DataView", "Desc": "Per instance metrics dataset" - }, - { - "Name": "ConfusionMatrix", - "Type": "DataView", - "Desc": "Confusion matrix dataset" } + ], + "InputKind": [ + "IEvaluatorInput" + ], + "OutputKind": [ + "IEvaluatorOutput" ] }, { @@ -2573,222 +1796,6 @@ } ] }, - { - "Name": "Models.CrossValidator", - "Desc": "Cross validation for general learning", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data set", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "The transform model from the pipeline before this command. It gets included in the Output.PredictorModel.", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "Nodes", - "Type": { - "Kind": "Array", - "ItemType": "Node" - }, - "Desc": "The training subgraph", - "Required": true, - "SortOrder": 3.0, - "IsNullable": false - }, - { - "Name": "Inputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data to be used for training", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph inputs", - "Required": true, - "SortOrder": 4.0, - "IsNullable": false - }, - { - "Name": "Outputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "The predictor model", - "Required": false, - "SortOrder": 1.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "The transform model", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null - } - ] - }, - "Desc": "The training subgraph outputs", - "Required": true, - "SortOrder": 5.0, - "IsNullable": false - }, - { - "Name": "StratificationColumn", - "Type": "String", - "Desc": "Column to use for stratification", - "Aliases": [ - "strat" - ], - "Required": false, - "SortOrder": 6.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "NumFolds", - "Type": "Int", - "Desc": "Number of folds in k-fold cross-validation", - "Aliases": [ - "k" - ], - "Required": false, - "SortOrder": 7.0, - "IsNullable": false, - "Default": 2 - }, - { - "Name": "Kind", - "Type": { - "Kind": "Enum", - "Values": [ - "SignatureBinaryClassifierTrainer", - "SignatureMultiClassClassifierTrainer", - "SignatureRankerTrainer", - "SignatureRegressorTrainer", - "SignatureMultiOutputRegressorTrainer", - "SignatureAnomalyDetectorTrainer", - "SignatureClusteringTrainer" - ] - }, - "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", - "Required": true, - "SortOrder": 8.0, - "IsNullable": false, - "Default": "SignatureBinaryClassifierTrainer" - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 9.0, - "IsNullable": false, - "Default": "Label" - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Column to use for example weight", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 10.0, - "IsNullable": false, - "Default": "Weight" - }, - { - "Name": "GroupColumn", - "Type": "String", - "Desc": "Column to use for grouping", - "Aliases": [ - "group" - ], - "Required": false, - "SortOrder": 11.0, - "IsNullable": false, - "Default": "GroupId" - }, - { - "Name": "NameColumn", - "Type": "String", - "Desc": "Name column name", - "Aliases": [ - "name" - ], - "Required": false, - "SortOrder": 12.0, - "IsNullable": false, - "Default": "Name" - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": { - "Kind": "Array", - "ItemType": "PredictorModel" - }, - "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." - }, - { - "Name": "TransformModel", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." - }, - { - "Name": "Warnings", - "Type": "DataView", - "Desc": "Warning dataset" - }, - { - "Name": "OverallMetrics", - "Type": "DataView", - "Desc": "Overall metrics dataset" - }, - { - "Name": "PerInstanceMetrics", - "Type": "DataView", - "Desc": "Per instance metrics dataset" - }, - { - "Name": "ConfusionMatrix", - "Type": "DataView", - "Desc": "Confusion matrix dataset" - } - ] - }, { "Name": "Models.CrossValidatorDatasetSplitter", "Desc": "Split the dataset into the specified number of cross-validation folds (train and test sets)", @@ -2844,95 +1851,6 @@ } ] }, - { - "Name": "Models.CrossValidatorDatasetSplitter", - "Desc": "Split the dataset into the specified number of cross-validation folds (train and test sets)", - "FriendlyName": "Dataset CV Split", - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "NumFolds", - "Type": "Int", - "Desc": "Number of folds to split into", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": 2 - }, - { - "Name": "StratificationColumn", - "Type": "String", - "Desc": "Stratification column", - "Aliases": [ - "strat" - ], - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "TrainData", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "Training data (one dataset per fold)" - }, - { - "Name": "TestData", - "Type": { - "Kind": "Array", - "ItemType": "DataView" - }, - "Desc": "Testing data (one dataset per fold)" - } - ] - }, - { - "Name": "Models.DatasetTransformer", - "Desc": "Applies a TransformModel to a dataset.", - "FriendlyName": "Apply Transform Model Output", - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "Transform model", - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputData", - "Type": "DataView", - "Desc": "Transformed dataset" - } - ], - "InputKind": [ - "ITransformInput" - ] - }, { "Name": "Models.DatasetTransformer", "Desc": "Applies a TransformModel to a dataset.", @@ -3288,193 +2206,46 @@ }, { "Name": "UncalibratedPredictorModel", - "Type": "PredictorModel", - "Desc": "The predictor to calibrate", - "Aliases": [ - "uncalibratedPredictorModel" - ], - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - }, - { - "Name": "MaxRows", - "Type": "Int", - "Desc": "The maximum number of examples to train the calibrator on", - "Aliases": [ - "maxRows" - ], - "Required": true, - "SortOrder": 3.0, - "IsNullable": false, - "Default": 1000000000, - "Range": { - "Inf": 0, - "Max": 2147483647 - } - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "The trained model" - } - ], - "InputKind": [ - "ICalibratorInput", - "ITransformInput" - ], - "OutputKind": [ - "ICalibratorOutput", - "ITrainerOutput" - ] - }, - { - "Name": "Models.OneVersusAll", - "Desc": "One-vs-All macro (OVA)", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Nodes", - "Type": { - "Kind": "Array", - "ItemType": "Node" - }, - "Desc": "The subgraph for the binary trainer used to construct the OVA learner. This should be a TrainBinary node.", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "TrainingData", - "Type": "DataView", - "Desc": "The data to be used for training", - "Aliases": [ - "data" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "OutputForSubGraph", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Model", - "Type": "PredictorModel", - "Desc": "The predictor model for the subgraph exemplar.", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph output.", - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - }, - { - "Name": "FeatureColumn", - "Type": "String", - "Desc": "Column to use for features", - "Aliases": [ - "feat" - ], - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": "Features" - }, - { - "Name": "UseProbabilities", - "Type": "Bool", - "Desc": "Use probabilities in OVA combiner", - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": true - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": "Label" - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Column to use for example weight", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 4.0, - "IsNullable": false, - "Default": "Weight" - }, - { - "Name": "NormalizeFeatures", - "Type": { - "Kind": "Enum", - "Values": [ - "No", - "Warn", - "Auto", - "Yes" - ] - }, - "Desc": "Normalize option for the feature column", - "Aliases": [ - "norm" - ], - "Required": false, - "SortOrder": 5.0, - "IsNullable": false, - "Default": "Auto" - }, - { - "Name": "Caching", - "Type": { - "Kind": "Enum", - "Values": [ - "Auto", - "Memory", - "Disk", - "None" - ] - }, - "Desc": "Whether learner should cache input training data", + "Type": "PredictorModel", + "Desc": "The predictor to calibrate", "Aliases": [ - "cache" + "uncalibratedPredictorModel" ], - "Required": false, - "SortOrder": 6.0, + "Required": true, + "SortOrder": 2.0, + "IsNullable": false + }, + { + "Name": "MaxRows", + "Type": "Int", + "Desc": "The maximum number of examples to train the calibrator on", + "Aliases": [ + "maxRows" + ], + "Required": true, + "SortOrder": 3.0, "IsNullable": false, - "Default": "Auto" + "Default": 1000000000, + "Range": { + "Inf": 0, + "Max": 2147483647 + } } ], "Outputs": [ { "Name": "PredictorModel", "Type": "PredictorModel", - "Desc": "The trained multiclass model" + "Desc": "The trained model" } ], "InputKind": [ - "ITrainerInputWithWeight", - "ITrainerInputWithLabel", - "ITrainerInput" + "ICalibratorInput", + "ITransformInput" + ], + "OutputKind": [ + "ICalibratorOutput", + "ITrainerOutput" ] }, { @@ -4719,237 +3490,21 @@ { "Name": "State", "Type": { - "Kind": "C# Object", - "ItemType": "Microsoft.ML.Runtime.EntryPoints.IMlState" - }, - "Desc": "Stateful autoML object, keeps track of where the search in progress." - }, - { - "Name": "Results", - "Type": "DataView", - "Desc": "Results of the sweep, including pipelines (as graph strings), IDs, and metric values." - } - ] - }, - { - "Name": "Models.TrainTestBinaryEvaluator", - "Desc": "Train test for binary classification", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TrainingData", - "Type": "DataView", - "Desc": "The data to be used for training", - "Aliases": [ - "train" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "TestingData", - "Type": "DataView", - "Desc": "The data to be used for testing", - "Aliases": [ - "test" - ], - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - }, - { - "Name": "Nodes", - "Type": { - "Kind": "Array", - "ItemType": "Node" - }, - "Desc": "The training subgraph", - "Required": true, - "SortOrder": 3.0, - "IsNullable": false - }, - { - "Name": "Inputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data to be used for training", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph inputs", - "Required": true, - "SortOrder": 4.0, - "IsNullable": false - }, - { - "Name": "Outputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Model", - "Type": "PredictorModel", - "Desc": "The model", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph outputs", - "Required": true, - "SortOrder": 5.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "The trained model" - }, - { - "Name": "Warnings", - "Type": "DataView", - "Desc": "Warning dataset" - }, - { - "Name": "OverallMetrics", - "Type": "DataView", - "Desc": "Overall metrics dataset" - }, - { - "Name": "PerInstanceMetrics", - "Type": "DataView", - "Desc": "Per instance metrics dataset" - }, - { - "Name": "ConfusionMatrix", - "Type": "DataView", - "Desc": "Confusion matrix dataset" - } - ] - }, - { - "Name": "Models.TrainTestBinaryEvaluator", - "Desc": "Train test for binary classification", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TrainingData", - "Type": "DataView", - "Desc": "The data to be used for training", - "Aliases": [ - "train" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "TestingData", - "Type": "DataView", - "Desc": "The data to be used for testing", - "Aliases": [ - "test" - ], - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - }, - { - "Name": "Nodes", - "Type": { - "Kind": "Array", - "ItemType": "Node" - }, - "Desc": "The training subgraph", - "Required": true, - "SortOrder": 3.0, - "IsNullable": false - }, - { - "Name": "Inputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "The data to be used for training", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph inputs", - "Required": true, - "SortOrder": 4.0, - "IsNullable": false - }, - { - "Name": "Outputs", - "Type": { - "Kind": "Struct", - "Fields": [ - { - "Name": "Model", - "Type": "PredictorModel", - "Desc": "The model", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ] - }, - "Desc": "The training subgraph outputs", - "Required": true, - "SortOrder": 5.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "The trained model" - }, - { - "Name": "Warnings", - "Type": "DataView", - "Desc": "Warning dataset" - }, - { - "Name": "OverallMetrics", - "Type": "DataView", - "Desc": "Overall metrics dataset" - }, - { - "Name": "PerInstanceMetrics", - "Type": "DataView", - "Desc": "Per instance metrics dataset" + "Kind": "C# Object", + "ItemType": "Microsoft.ML.Runtime.EntryPoints.IMlState" + }, + "Desc": "Stateful autoML object, keeps track of where the search in progress." }, { - "Name": "ConfusionMatrix", + "Name": "Results", "Type": "DataView", - "Desc": "Confusion matrix dataset" + "Desc": "Results of the sweep, including pipelines (as graph strings), IDs, and metric values." } ] }, { - "Name": "Models.TrainTestEvaluator", - "Desc": "General train test for any supported evaluator", + "Name": "Models.TrainTestBinaryEvaluator", + "Desc": "Train test for binary classification", "FriendlyName": null, "ShortName": null, "Inputs": [ @@ -4975,15 +3530,6 @@ "SortOrder": 2.0, "IsNullable": false }, - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "The aggregated transform model from the pipeline before this command, to apply to the test data, and also include in the final model, together with the predictor model.", - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": null - }, { "Name": "Nodes", "Type": { @@ -4992,7 +3538,7 @@ }, "Desc": "The training subgraph", "Required": true, - "SortOrder": 4.0, + "SortOrder": 3.0, "IsNullable": false }, { @@ -5012,7 +3558,7 @@ }, "Desc": "The training subgraph inputs", "Required": true, - "SortOrder": 5.0, + "SortOrder": 4.0, "IsNullable": false }, { @@ -5021,127 +3567,26 @@ "Kind": "Struct", "Fields": [ { - "Name": "PredictorModel", + "Name": "Model", "Type": "PredictorModel", - "Desc": "The predictor model", - "Required": false, + "Desc": "The model", + "Required": true, "SortOrder": 1.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "Transform model", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null + "IsNullable": false } ] }, "Desc": "The training subgraph outputs", "Required": true, - "SortOrder": 6.0, + "SortOrder": 5.0, "IsNullable": false - }, - { - "Name": "Kind", - "Type": { - "Kind": "Enum", - "Values": [ - "SignatureBinaryClassifierTrainer", - "SignatureMultiClassClassifierTrainer", - "SignatureRankerTrainer", - "SignatureRegressorTrainer", - "SignatureMultiOutputRegressorTrainer", - "SignatureAnomalyDetectorTrainer", - "SignatureClusteringTrainer" - ] - }, - "Desc": "Specifies the trainer kind, which determines the evaluator to be used.", - "Required": false, - "SortOrder": 7.0, - "IsNullable": false, - "Default": "SignatureBinaryClassifierTrainer" - }, - { - "Name": "PipelineId", - "Type": "String", - "Desc": "Identifies which pipeline was run for this train test.", - "Required": false, - "SortOrder": 8.0, - "IsNullable": false, - "Default": null - }, - { - "Name": "IncludeTrainingMetrics", - "Type": "Bool", - "Desc": "Indicates whether to include and output training dataset metrics.", - "Required": false, - "SortOrder": 9.0, - "IsNullable": false, - "Default": false - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "Column to use for labels", - "Aliases": [ - "lab" - ], - "Required": false, - "SortOrder": 10.0, - "IsNullable": false, - "Default": "Label" - }, - { - "Name": "WeightColumn", - "Type": "String", - "Desc": "Column to use for example weight", - "Aliases": [ - "weight" - ], - "Required": false, - "SortOrder": 11.0, - "IsNullable": false, - "Default": "Weight" - }, - { - "Name": "GroupColumn", - "Type": "String", - "Desc": "Column to use for grouping", - "Aliases": [ - "group" - ], - "Required": false, - "SortOrder": 12.0, - "IsNullable": false, - "Default": "GroupId" - }, - { - "Name": "NameColumn", - "Type": "String", - "Desc": "Name column name", - "Aliases": [ - "name" - ], - "Required": false, - "SortOrder": 13.0, - "IsNullable": false, - "Default": "Name" } ], "Outputs": [ { "Name": "PredictorModel", "Type": "PredictorModel", - "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." - }, - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel." + "Desc": "The trained model" }, { "Name": "Warnings", @@ -5162,26 +3607,6 @@ "Name": "ConfusionMatrix", "Type": "DataView", "Desc": "Confusion matrix dataset" - }, - { - "Name": "TrainingWarnings", - "Type": "DataView", - "Desc": "Warning dataset for training" - }, - { - "Name": "TrainingOverallMetrics", - "Type": "DataView", - "Desc": "Overall metrics dataset for training" - }, - { - "Name": "TrainingPerInstanceMetrics", - "Type": "DataView", - "Desc": "Per instance metrics dataset for training" - }, - { - "Name": "TrainingConfusionMatrix", - "Type": "DataView", - "Desc": "Confusion matrix dataset for training" } ] }, @@ -19417,52 +17842,6 @@ "ITransformOutput" ] }, - { - "Name": "Transforms.FeatureCombiner", - "Desc": "Combines all the features into one feature column.", - "FriendlyName": "Feature Combiner", - "ShortName": "fc", - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "Features", - "Type": { - "Kind": "Array", - "ItemType": "String" - }, - "Desc": "Features", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "OutputData", - "Type": "DataView", - "Desc": "Transformed dataset" - }, - { - "Name": "Model", - "Type": "TransformModel", - "Desc": "Transform model" - } - ], - "InputKind": [ - "ITransformInput" - ], - "OutputKind": [ - "ITransformOutput" - ] - }, { "Name": "Transforms.FeatureSelectorByCount", "Desc": "Selects the slots for which the count of non-default values is greater than or equal to a threshold.", @@ -20567,78 +18946,27 @@ ], "Required": false, "SortOrder": 150.0, - "IsNullable": false, - "Default": null - } - ] - } - }, - "Desc": "New column definition(s) (optional form: name:src)", - "Aliases": [ - "col" - ], - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputData", - "Type": "DataView", - "Desc": "Transformed dataset" - }, - { - "Name": "Model", - "Type": "TransformModel", - "Desc": "Transform model" - } - ], - "InputKind": [ - "ITransformInput" - ], - "OutputKind": [ - "ITransformOutput" - ] - }, - { - "Name": "Transforms.LabelColumnKeyBooleanConverter", - "Desc": "Transforms the label to either key or bool (if needed) to make it suitable for classification.", - "FriendlyName": "Prepare Classification Label", - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", + "IsNullable": false, + "Default": null + } + ] + } + }, + "Desc": "New column definition(s) (optional form: name:src)", + "Aliases": [ + "col" + ], "Required": true, "SortOrder": 1.0, "IsNullable": false }, { - "Name": "LabelColumn", - "Type": "String", - "Desc": "The label column", + "Name": "Data", + "Type": "DataView", + "Desc": "Input dataset", "Required": true, - "SortOrder": 2.0, + "SortOrder": 1.0, "IsNullable": false - }, - { - "Name": "TextKeyValues", - "Type": "Bool", - "Desc": "Convert the key values to text", - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": true } ], "Outputs": [ @@ -20854,48 +19182,6 @@ "ITransformOutput" ] }, - { - "Name": "Transforms.LabelToFloatConverter", - "Desc": "Transforms the label to float to make it suitable for regression.", - "FriendlyName": "Prepare Regression Label", - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "LabelColumn", - "Type": "String", - "Desc": "The label column", - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputData", - "Type": "DataView", - "Desc": "Transformed dataset" - }, - { - "Name": "Model", - "Type": "TransformModel", - "Desc": "Transform model" - } - ], - "InputKind": [ - "ITransformInput" - ], - "OutputKind": [ - "ITransformOutput" - ] - }, { "Name": "Transforms.LightLda", "Desc": "The LDA transform implements LightLDA, a state-of-the-art implementation of Latent Dirichlet Allocation.", @@ -21559,40 +19845,6 @@ } ] }, - { - "Name": "Transforms.ManyHeterogeneousModelCombiner", - "Desc": "Combines a sequence of TransformModels and a PredictorModel into a single PredictorModel.", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TransformModels", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "Transform model", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "Predictor model", - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "Predictor model" - } - ] - }, { "Name": "Transforms.MeanVarianceNormalizer", "Desc": "Normalizes the data based on the computed mean and variance of the data.", @@ -22398,33 +20650,6 @@ } ] }, - { - "Name": "Transforms.ModelCombiner", - "Desc": "Combines a sequence of TransformModels into a single model", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "Models", - "Type": { - "Kind": "Array", - "ItemType": "TransformModel" - }, - "Desc": "Input models", - "Required": false, - "SortOrder": 1.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "OutputModel", - "Type": "TransformModel", - "Desc": "Combined model" - } - ] - }, { "Name": "Transforms.NGramTranslator", "Desc": "Produces a bag of counts of ngrams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of ngrams and using the id in the dictionary as the index in the bag.", @@ -22952,48 +21177,6 @@ "ITransformOutput" ] }, - { - "Name": "Transforms.PredictedLabelColumnOriginalValueConverter", - "Desc": "Transforms a predicted label column to its original values, unless it is of type bool.", - "FriendlyName": "Convert Predicted Label", - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "PredictedLabelColumn", - "Type": "String", - "Desc": "The predicted label column", - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "OutputData", - "Type": "DataView", - "Desc": "Transformed dataset" - }, - { - "Name": "Model", - "Type": "TransformModel", - "Desc": "Transform model" - } - ], - "InputKind": [ - "ITransformInput" - ], - "OutputKind": [ - "ITransformOutput" - ] - }, { "Name": "Transforms.RandomNumberGenerator", "Desc": "Adds a column with a generated number sequence.", @@ -24141,55 +22324,6 @@ } ] }, - { - "Name": "Transforms.TrainTestDatasetSplitter", - "Desc": "Split the dataset into train and test sets", - "FriendlyName": "Dataset Train-Test Split", - "ShortName": null, - "Inputs": [ - { - "Name": "Data", - "Type": "DataView", - "Desc": "Input dataset", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "Fraction", - "Type": "Float", - "Desc": "Fraction of training data", - "Required": false, - "SortOrder": 2.0, - "IsNullable": false, - "Default": 0.8 - }, - { - "Name": "StratificationColumn", - "Type": "String", - "Desc": "Stratification column", - "Aliases": [ - "strat" - ], - "Required": false, - "SortOrder": 3.0, - "IsNullable": false, - "Default": null - } - ], - "Outputs": [ - { - "Name": "TrainData", - "Type": "DataView", - "Desc": "Training data" - }, - { - "Name": "TestData", - "Type": "DataView", - "Desc": "Testing data" - } - ] - }, { "Name": "Transforms.TreeLeafFeaturizer", "Desc": "Trains a tree ensemble, or loads it from a file, then maps a numeric feature vector to three outputs: 1. A vector containing the individual tree outputs of the tree ensemble. 2. A vector indicating the leaves that the feature vector falls on in the tree ensemble. 3. A vector indicating the paths that the feature vector falls on in the tree ensemble. If a both a model file and a trainer are specified - will use the model file. If neither are specified, will train a default FastTree model. This can handle key labels by training a regression model towards their optionally permuted indices.", @@ -24288,37 +22422,6 @@ } ] }, - { - "Name": "Transforms.TwoHeterogeneousModelCombiner", - "Desc": "Combines a TransformModel and a PredictorModel into a single PredictorModel.", - "FriendlyName": null, - "ShortName": null, - "Inputs": [ - { - "Name": "TransformModel", - "Type": "TransformModel", - "Desc": "Transform model", - "Required": true, - "SortOrder": 1.0, - "IsNullable": false - }, - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "Predictor model", - "Required": true, - "SortOrder": 2.0, - "IsNullable": false - } - ], - "Outputs": [ - { - "Name": "PredictorModel", - "Type": "PredictorModel", - "Desc": "Predictor model" - } - ] - }, { "Name": "Transforms.VectorToImage", "Desc": "Converts vector array into image type.", @@ -30801,10 +28904,6 @@ "Kind": "ILearningPipelineLoader", "Settings": [] }, - { - "Kind": "ILearningPipelineLoader", - "Settings": [] - }, { "Kind": "IMulticlassClassificationOutput", "Settings": [] diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 9cdfdaae0d..80747d5a0e 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -240,7 +240,7 @@ private string GetBuildPrefix() #endif } - [Fact(Skip = "Execute this test if you want to regenerate ep-list and _manifest.json")] + [Fact] public void RegenerateEntryPointCatalog() { var (epListContents, jObj) = BuildManifests(); From 0ab973bd36227cd6065b2bb3e315a1919d0b1cef Mon Sep 17 00:00:00 2001 From: J W Date: Mon, 1 Oct 2018 16:50:31 -0400 Subject: [PATCH 10/19] Update more argument names and revert test skip --- .../CpuMathUtils.netcoreapp.cs | 48 +++++++++---------- .../CpuMathUtils.netstandard.cs | 2 +- .../UnitTests/TestEntryPoints.cs | 2 +- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 557db00763..2c9968a36c 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -862,45 +862,45 @@ private static float DotProductDense(Span a, Span b) } } - public static float DotProductSparse(float[] a, float[] b, int[] indices, int count) + public static float DotProductSparse(float[] value, float[] b, int[] indices, int count) { - Contracts.AssertNonEmpty(a); + Contracts.AssertNonEmpty(value); Contracts.AssertNonEmpty(b); Contracts.AssertNonEmpty(indices); Contracts.Assert(count > 0); - Contracts.Assert(count < a.Length); + Contracts.Assert(count < value.Length); Contracts.Assert(count <= b.Length); Contracts.Assert(count <= indices.Length); - return DotProductSparse(new Span(a), new Span(b), + return DotProductSparse(new Span(value), new Span(b), new Span(indices, 0, count)); } - public static float DotProductSparse(float[] a, int offset, float[] b, int[] indices, int count) + public static float DotProductSparse(float[] value, int offset, float[] b, int[] indices, int count) { - Contracts.AssertNonEmpty(a); + Contracts.AssertNonEmpty(value); Contracts.AssertNonEmpty(b); Contracts.AssertNonEmpty(indices); Contracts.Assert(count > 0); - Contracts.Assert(count < (a.Length - offset)); + Contracts.Assert(count < (value.Length - offset)); Contracts.Assert(count <= b.Length); Contracts.Assert(count <= indices.Length); Contracts.Assert(offset >= 0); - Contracts.Assert(offset < a.Length); + Contracts.Assert(offset < value.Length); - return DotProductSparse(new Span(a, offset, a.Length - offset), + return DotProductSparse(new Span(value, offset, value.Length - offset), new Span(b), new Span(indices, 0, count)); } - private static float DotProductSparse(Span a, Span b, Span indices) + private static float DotProductSparse(Span value, Span b, Span indices) { if (Avx.IsSupported) { - return AvxIntrinsics.DotSU(a, b, indices); + return AvxIntrinsics.DotSU(value, b, indices); } else if (Sse.IsSupported) { - return SseIntrinsics.DotSU(a, b, indices); + return SseIntrinsics.DotSU(value, b, indices); } else { @@ -908,39 +908,39 @@ private static float DotProductSparse(Span a, Span b, Span in for (int i = 0; i < indices.Length; i++) { int index = indices[i]; - result += a[index] * b[i]; + result += value[index] * b[i]; } return result; } } - public static float L2DistSquared(float[] a, float[] b, int count) + public static float L2DistSquared(float[] left, float[] right, int count) { - Contracts.AssertNonEmpty(a); - Contracts.AssertNonEmpty(b); + Contracts.AssertNonEmpty(left); + Contracts.AssertNonEmpty(right); Contracts.Assert(count > 0); - Contracts.Assert(count <= a.Length); - Contracts.Assert(count <= b.Length); + Contracts.Assert(count <= left.Length); + Contracts.Assert(count <= right.Length); - return L2DistSquared(new Span(a, 0, count), new Span(b, 0, count)); + return L2DistSquared(new Span(left, 0, count), new Span(right, 0, count)); } - private static float L2DistSquared(Span a, Span b) + private static float L2DistSquared(Span left, Span right) { if (Avx.IsSupported) { - return AvxIntrinsics.Dist2(a, b); + return AvxIntrinsics.Dist2(left, right); } else if (Sse.IsSupported) { - return SseIntrinsics.Dist2(a, b); + return SseIntrinsics.Dist2(left, right); } else { float norm = 0; - for (int i = 0; i < b.Length; i++) + for (int i = 0; i < right.Length; i++) { - float distance = a[i] - b[i]; + float distance = left[i] - right[i]; norm += distance * distance; } return norm; diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index 293a38acb8..fca785cf4d 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -78,7 +78,7 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray matr public static float DotProductSparse(float[] value, int offset, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(value, offset, b, indices, count); - public static float L2DistSquared(float[] value, float[] b, int count) => SseUtils.L2DistSquared(value, b, count); + public static float L2DistSquared(float[] left, float[] right, int count) => SseUtils.L2DistSquared(left, right, count); public static void ZeroMatrixItems(AlignedArray destination, int ccol, int cfltRow, int[] indices) => SseUtils.ZeroMatrixItems(destination, ccol, cfltRow, indices); diff --git a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs index 80747d5a0e..9cdfdaae0d 100644 --- a/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs +++ b/test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs @@ -240,7 +240,7 @@ private string GetBuildPrefix() #endif } - [Fact] + [Fact(Skip = "Execute this test if you want to regenerate ep-list and _manifest.json")] public void RegenerateEntryPointCatalog() { var (epListContents, jObj) = BuildManifests(); From dc99725f2c66b4f853fe4e8eb46481b02e4ad504 Mon Sep 17 00:00:00 2001 From: J W Date: Tue, 2 Oct 2018 21:52:10 -0400 Subject: [PATCH 11/19] Fix compile issues in .NET Core compile --- src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs | 4 ++-- test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 2c9968a36c..dbb87c62c7 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -279,11 +279,11 @@ private static void Scale(float value, Span source, Span destinati { if (Avx.IsSupported) { - AvxIntrinsics.ScalesourceU(value, source, destination); + AvxIntrinsics.ScaleSrcU(value, source, destination); } else if (Sse.IsSupported) { - SseIntrinsics.ScalesourceU(value, source, destination); + SseIntrinsics.ScaleSrcU(value, source, destination); } else { diff --git a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs index ad7266bbee..f2e5ea05fd 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs @@ -617,7 +617,7 @@ public void SdcaL1UpdateSUTest(int test) expected[index] = Math.Abs(value) > DefaultScale ? (value > 0 ? value - DefaultScale : value + DefaultScale) : 0; } - CpuMathUtils.SdcaL1UpdateSparse(DefaultScale, src.Length, src, idx, idx.Length, DefaultScale, v, w); + CpuMathUtils.SdcaL1UpdateSparse(DefaultScale, src, idx, idx.Length, DefaultScale, v, w); var actual = w; Assert.Equal(expected, actual, _comparer); } From 82dfdcf8b8eef64fffca94c626036f5fe18044a4 Mon Sep 17 00:00:00 2001 From: J W Date: Tue, 2 Oct 2018 22:12:02 -0400 Subject: [PATCH 12/19] Revert removal of argument --- src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs | 4 +++- src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs | 2 +- src/Microsoft.ML.CpuMath/Sse.cs | 5 +---- test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index dbb87c62c7..0efd176b58 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -1047,7 +1047,7 @@ private static void SdcaL1UpdateDense(float primalUpdate, Span source, fl } // REVIEW NEEDED: The second argument "length" is unused even in the existing code. - public static void SdcaL1UpdateSparse(float primalUpdate, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); @@ -1056,6 +1056,8 @@ public static void SdcaL1UpdateSparse(float primalUpdate, float[] source, int[] Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); + Contracts.Assert(length <= v.Length); + Contracts.Assert(length <= w.Length); SdcaL1UpdateSparse(primalUpdate, new Span(source, 0, count), new Span(indices, 0, count), threshold, new Span(v), new Span(w)); } diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index fca785cf4d..7337d2600a 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -86,6 +86,6 @@ public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] sou => SseUtils.SdcaL1UpdateDense(primalUpdate, length, source, threshold, v, w); public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) - => SseUtils.SdcaL1UpdateSparse(primalUpdate, length, source, indices, count, threshold, v, w); + => SseUtils.SdcaL1UpdateSparse(primalUpdate, source, indices, count, threshold, v, w); } } diff --git a/src/Microsoft.ML.CpuMath/Sse.cs b/src/Microsoft.ML.CpuMath/Sse.cs index 13de22dd5b..b503d15a07 100644 --- a/src/Microsoft.ML.CpuMath/Sse.cs +++ b/src/Microsoft.ML.CpuMath/Sse.cs @@ -1493,18 +1493,15 @@ public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] src } } - public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] src, int[] indices, int count, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateSparse(float primalUpdate, float[] src, int[] indices, int count, float threshold, float[] v, float[] w) { Contracts.AssertNonEmpty(src); Contracts.Assert(count <= src.Length); Contracts.AssertNonEmpty(indices); Contracts.Assert(count <= indices.Length); Contracts.AssertNonEmpty(w); - Contracts.Assert(length <= w.Length); Contracts.AssertNonEmpty(v); - Contracts.Assert(length <= v.Length); Contracts.Assert(0 < count); - Contracts.Assert(count < length); unsafe { diff --git a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs index f2e5ea05fd..ad7266bbee 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs @@ -617,7 +617,7 @@ public void SdcaL1UpdateSUTest(int test) expected[index] = Math.Abs(value) > DefaultScale ? (value > 0 ? value - DefaultScale : value + DefaultScale) : 0; } - CpuMathUtils.SdcaL1UpdateSparse(DefaultScale, src, idx, idx.Length, DefaultScale, v, w); + CpuMathUtils.SdcaL1UpdateSparse(DefaultScale, src.Length, src, idx, idx.Length, DefaultScale, v, w); var actual = w; Assert.Equal(expected, actual, _comparer); } From a21dcf3af5163ae5fe235aa66acb5e36516ec4e2 Mon Sep 17 00:00:00 2001 From: J W Date: Fri, 5 Oct 2018 18:23:41 -0400 Subject: [PATCH 13/19] Update more parameter names --- .../CpuMathUtils.netcoreapp.cs | 104 +++++++++--------- .../CpuMathUtils.netstandard.cs | 8 +- 2 files changed, 56 insertions(+), 56 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 0efd176b58..47e5a3bd1c 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -263,7 +263,7 @@ private static void Scale(float value, Span destination) } } - // destination = a * source + // destination = value * source public static void Scale(float value, float[] source, float[] destination, int count) { Contracts.AssertNonEmpty(source); @@ -294,7 +294,7 @@ private static void Scale(float value, Span source, Span destinati } } - // destination[i] = a * (destination[i] + b) + // destination[i] = scale * (destination[i] + addend) public static void ScaleAdd(float scale, float addend, float[] destination, int count) { Contracts.AssertNonEmpty(destination); @@ -334,17 +334,17 @@ public static void AddScale(float scale, float[] source, float[] destination, in AddScale(scale, new Span(source, 0, count), new Span(destination, 0, count)); } - public static void AddScale(float scale, float[] source, float[] destination, int dstOffset, int count) + public static void AddScale(float scale, float[] source, float[] destination, int destinationOffset, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(destination); - Contracts.Assert(dstOffset >= 0); - Contracts.Assert(dstOffset < destination.Length); + Contracts.Assert(destinationOffset >= 0); + Contracts.Assert(destinationOffset < destination.Length); Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); - Contracts.Assert(count <= (destination.Length - dstOffset)); + Contracts.Assert(count <= (destination.Length - destinationOffset)); - AddScale(scale, new Span(source, 0, count), new Span(destination, dstOffset, count)); + AddScale(scale, new Span(source, 0, count), new Span(destination, destinationOffset, count)); } private static void AddScale(float scale, Span source, Span destination) @@ -379,20 +379,20 @@ public static void AddScale(float scale, float[] source, int[] indices, float[] AddScale(scale, new Span(source), new Span(indices, 0, count), new Span(destination)); } - public static void AddScale(float scale, float[] source, int[] indices, float[] destination, int dstOffset, int count) + public static void AddScale(float scale, float[] source, int[] indices, float[] destination, int destinationOffset, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); Contracts.AssertNonEmpty(destination); - Contracts.Assert(dstOffset >= 0); - Contracts.Assert(dstOffset < destination.Length); + Contracts.Assert(destinationOffset >= 0); + Contracts.Assert(destinationOffset < destination.Length); Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count < (destination.Length - dstOffset)); + Contracts.Assert(count < (destination.Length - destinationOffset)); AddScale(scale, new Span(source), new Span(indices, 0, count), - new Span(destination, dstOffset, destination.Length - dstOffset)); + new Span(destination, destinationOffset, destination.Length - destinationOffset)); } private static void AddScale(float scale, Span source, Span indices, Span destination) @@ -490,20 +490,20 @@ public static void Add(float[] source, int[] indices, float[] destination, int c Add(new Span(source), new Span(indices, 0, count), new Span(destination)); } - public static void Add(float[] source, int[] indices, float[] destination, int dstOffset, int count) + public static void Add(float[] source, int[] indices, float[] destination, int destinationOffset, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); Contracts.AssertNonEmpty(destination); - Contracts.Assert(dstOffset >= 0); - Contracts.Assert(dstOffset < destination.Length); + Contracts.Assert(destinationOffset >= 0); + Contracts.Assert(destinationOffset < destination.Length); Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(count <= (destination.Length - dstOffset)); + Contracts.Assert(count <= (destination.Length - destinationOffset)); Add(new Span(source), new Span(indices, 0, count), - new Span(destination, dstOffset, destination.Length - dstOffset)); + new Span(destination, destinationOffset, destination.Length - destinationOffset)); } private static void Add(Span source, Span indices, Span destination) @@ -818,89 +818,89 @@ private static float MaxAbsDiff(float mean, Span source) } } - public static float DotProductDense(float[] a, float[] b, int count) + public static float DotProductDense(float[] left, float[] right, int count) { - Contracts.AssertNonEmpty(a); - Contracts.AssertNonEmpty(b); + Contracts.AssertNonEmpty(left); + Contracts.AssertNonEmpty(right); Contracts.Assert(count > 0); - Contracts.Assert(a.Length >= count); - Contracts.Assert(b.Length >= count); + Contracts.Assert(left.Length >= count); + Contracts.Assert(right.Length >= count); - return DotProductDense(new Span(a, 0, count), new Span(b, 0, count)); + return DotProductDense(new Span(left, 0, count), new Span(right, 0, count)); } - public static float DotProductDense(float[] a, int offset, float[] b, int count) + public static float DotProductDense(float[] left, int offset, float[] right, int count) { - Contracts.AssertNonEmpty(a); - Contracts.AssertNonEmpty(b); + Contracts.AssertNonEmpty(left); + Contracts.AssertNonEmpty(right); Contracts.Assert(count > 0); - Contracts.Assert(count <= b.Length); + Contracts.Assert(count <= right.Length); Contracts.Assert(offset >= 0); - Contracts.Assert(offset <= (a.Length - count)); + Contracts.Assert(offset <= (left.Length - count)); - return DotProductDense(new Span(a, offset, count), new Span(b, 0, count)); + return DotProductDense(new Span(left, offset, count), new Span(right, 0, count)); } - private static float DotProductDense(Span a, Span b) + private static float DotProductDense(Span left, Span right) { if (Avx.IsSupported) { - return AvxIntrinsics.DotU(a, b); + return AvxIntrinsics.DotU(left, right); } else if (Sse.IsSupported) { - return SseIntrinsics.DotU(a, b); + return SseIntrinsics.DotU(left, right); } else { float result = 0; - for (int i = 0; i < b.Length; i++) + for (int i = 0; i < right.Length; i++) { - result += a[i] * b[i]; + result += left[i] * right[i]; } return result; } } - public static float DotProductSparse(float[] value, float[] b, int[] indices, int count) + public static float DotProductSparse(float[] left, float[] right, int[] indices, int count) { - Contracts.AssertNonEmpty(value); - Contracts.AssertNonEmpty(b); + Contracts.AssertNonEmpty(left); + Contracts.AssertNonEmpty(right); Contracts.AssertNonEmpty(indices); Contracts.Assert(count > 0); - Contracts.Assert(count < value.Length); - Contracts.Assert(count <= b.Length); + Contracts.Assert(count < left.Length); + Contracts.Assert(count <= right.Length); Contracts.Assert(count <= indices.Length); - return DotProductSparse(new Span(value), new Span(b), + return DotProductSparse(new Span(left), new Span(right), new Span(indices, 0, count)); } - public static float DotProductSparse(float[] value, int offset, float[] b, int[] indices, int count) + public static float DotProductSparse(float[] left, int offset, float[] right, int[] indices, int count) { - Contracts.AssertNonEmpty(value); - Contracts.AssertNonEmpty(b); + Contracts.AssertNonEmpty(left); + Contracts.AssertNonEmpty(right); Contracts.AssertNonEmpty(indices); Contracts.Assert(count > 0); - Contracts.Assert(count < (value.Length - offset)); - Contracts.Assert(count <= b.Length); + Contracts.Assert(count < (left.Length - offset)); + Contracts.Assert(count <= right.Length); Contracts.Assert(count <= indices.Length); Contracts.Assert(offset >= 0); - Contracts.Assert(offset < value.Length); + Contracts.Assert(offset < left.Length); - return DotProductSparse(new Span(value, offset, value.Length - offset), - new Span(b), new Span(indices, 0, count)); + return DotProductSparse(new Span(left, offset, left.Length - offset), + new Span(right), new Span(indices, 0, count)); } - private static float DotProductSparse(Span value, Span b, Span indices) + private static float DotProductSparse(Span left, Span right, Span indices) { if (Avx.IsSupported) { - return AvxIntrinsics.DotSU(value, b, indices); + return AvxIntrinsics.DotSU(left, right, indices); } else if (Sse.IsSupported) { - return SseIntrinsics.DotSU(value, b, indices); + return SseIntrinsics.DotSU(left, right, indices); } else { @@ -908,7 +908,7 @@ private static float DotProductSparse(Span value, Span b, Span SseUtils.MaxAbsDiff(mean, source, count); - public static float DotProductDense(float[] value, float[] b, int count) => SseUtils.DotProductDense(value, b, count); + public static float DotProductDense(float[] left, float[] right, int count) => SseUtils.DotProductDense(left, right, count); - public static float DotProductDense(float[] value, int offset, float[] b, int count) => SseUtils.DotProductDense(value, offset, b, count); + public static float DotProductDense(float[] left, int offset, float[] right, int count) => SseUtils.DotProductDense(left, offset, right, count); - public static float DotProductSparse(float[] value, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(value, b, indices, count); + public static float DotProductSparse(float[] left, float[] right, int[] indices, int count) => SseUtils.DotProductSparse(left, right, indices, count); - public static float DotProductSparse(float[] value, int offset, float[] b, int[] indices, int count) => SseUtils.DotProductSparse(value, offset, b, indices, count); + public static float DotProductSparse(float[] left, int offset, float[] right, int[] indices, int count) => SseUtils.DotProductSparse(left, offset, right, indices, count); public static float L2DistSquared(float[] left, float[] right, int count) => SseUtils.L2DistSquared(left, right, count); From 0a6ce775a17b20bac9d5e7e817cf14df31456fb9 Mon Sep 17 00:00:00 2001 From: J W Date: Fri, 5 Oct 2018 18:44:56 -0400 Subject: [PATCH 14/19] Remove unused parameter --- src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs | 5 +---- test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 47e5a3bd1c..2ac63f14fd 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -1046,8 +1046,7 @@ private static void SdcaL1UpdateDense(float primalUpdate, Span source, fl } } - // REVIEW NEEDED: The second argument "length" is unused even in the existing code. - public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateSparse(float primalUpdate, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(indices); @@ -1056,8 +1055,6 @@ public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] so Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); Contracts.Assert(count <= indices.Length); - Contracts.Assert(length <= v.Length); - Contracts.Assert(length <= w.Length); SdcaL1UpdateSparse(primalUpdate, new Span(source, 0, count), new Span(indices, 0, count), threshold, new Span(v), new Span(w)); } diff --git a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs index ad7266bbee..f2e5ea05fd 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs @@ -617,7 +617,7 @@ public void SdcaL1UpdateSUTest(int test) expected[index] = Math.Abs(value) > DefaultScale ? (value > 0 ? value - DefaultScale : value + DefaultScale) : 0; } - CpuMathUtils.SdcaL1UpdateSparse(DefaultScale, src.Length, src, idx, idx.Length, DefaultScale, v, w); + CpuMathUtils.SdcaL1UpdateSparse(DefaultScale, src, idx, idx.Length, DefaultScale, v, w); var actual = w; Assert.Equal(expected, actual, _comparer); } From ca153ebb1a995c2f87f9643255537c137fc8f099 Mon Sep 17 00:00:00 2001 From: J W Date: Sat, 6 Oct 2018 08:18:42 -0400 Subject: [PATCH 15/19] Fix build errors for removing length parameter --- src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs | 6 +++--- src/Microsoft.ML.CpuMath/Sse.cs | 8 ++------ src/Microsoft.ML.CpuMath/Thunk.cs | 2 +- .../Standard/LinearClassificationTrainer.cs | 4 ++-- .../Standard/SdcaMultiClass.cs | 8 ++++---- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index 37f20248db..27b51d4262 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -82,10 +82,10 @@ public static void MatrixTimesSource(bool transpose, bool add, AlignedArray matr public static void ZeroMatrixItems(AlignedArray destination, int ccol, int cfltRow, int[] indices) => SseUtils.ZeroMatrixItems(destination, ccol, cfltRow, indices); - public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] source, float threshold, float[] v, float[] w) - => SseUtils.SdcaL1UpdateDense(primalUpdate, length, source, threshold, v, w); + public static void SdcaL1UpdateDense(float primalUpdate, float[] source, float threshold, float[] v, float[] w) + => SseUtils.SdcaL1UpdateDense(primalUpdate, source, threshold, v, w); - public static void SdcaL1UpdateSparse(float primalUpdate, int length, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateSparse(float primalUpdate, float[] source, int[] indices, int count, float threshold, float[] v, float[] w) => SseUtils.SdcaL1UpdateSparse(primalUpdate, source, indices, count, threshold, v, w); } } diff --git a/src/Microsoft.ML.CpuMath/Sse.cs b/src/Microsoft.ML.CpuMath/Sse.cs index b503d15a07..72609caab6 100644 --- a/src/Microsoft.ML.CpuMath/Sse.cs +++ b/src/Microsoft.ML.CpuMath/Sse.cs @@ -1474,22 +1474,18 @@ public static void ZeroMatrixItems(AlignedArray dst, int ccol, int cfltRow, int[ } } - public static void SdcaL1UpdateDense(float primalUpdate, int length, float[] src, float threshold, float[] v, float[] w) + public static void SdcaL1UpdateDense(float primalUpdate, float[] src, float threshold, float[] v, float[] w) { Contracts.AssertNonEmpty(src); - Contracts.Assert(length <= src.Length); Contracts.AssertNonEmpty(v); - Contracts.Assert(length <= v.Length); Contracts.AssertNonEmpty(w); - Contracts.Assert(length <= w.Length); - Contracts.Assert(length > 0); unsafe { fixed (float* psrc = &src[0]) fixed (float* pd1 = &v[0]) fixed (float* pd2 = &w[0]) - Thunk.SdcaL1UpdateU(primalUpdate, psrc, threshold, pd1, pd2, length); + Thunk.SdcaL1UpdateU(primalUpdate, psrc, threshold, pd1, pd2); } } diff --git a/src/Microsoft.ML.CpuMath/Thunk.cs b/src/Microsoft.ML.CpuMath/Thunk.cs index 1053f75b75..1fc6dff0d1 100644 --- a/src/Microsoft.ML.CpuMath/Thunk.cs +++ b/src/Microsoft.ML.CpuMath/Thunk.cs @@ -419,7 +419,7 @@ public static void ApplyBoundedRectifiedLinearDerivativeX(/*const*/ float* pv, f public static extern void ZeroMatrixItemsCore(float* pd, int c, int ccol, int cfltRow, /*const*/ int* pindices, int cindices); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void SdcaL1UpdateU(float primalUpdate, /*const*/ float* ps, float threshold, float* pd1, float* pd2, int c); + public static extern void SdcaL1UpdateU(float primalUpdate, /*const*/ float* ps, float threshold, float* pd1, float* pd2); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] public static extern void SdcaL1UpdateSU(float primalUpdate, /*const*/ float* ps, /*const*/ int* pi, float threshold, float* pd1, float* pd2, int c); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs b/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs index 7bcb8ef04c..25b5cc895f 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs @@ -807,9 +807,9 @@ protected virtual void TrainWithoutLock(IProgressChannelProvider progress, Float } if (features.IsDense) - CpuMathUtils.SdcaL1UpdateDense(primalUpdate, features.Length, features.Values, l1Threshold, l1IntermediateWeights[0].Values, weights[0].Values); + CpuMathUtils.SdcaL1UpdateDense(primalUpdate, features.Values, l1Threshold, l1IntermediateWeights[0].Values, weights[0].Values); else if (features.Count > 0) - CpuMathUtils.SdcaL1UpdateSparse(primalUpdate, features.Length, features.Values, features.Indices, features.Count, l1Threshold, l1IntermediateWeights[0].Values, weights[0].Values); + CpuMathUtils.SdcaL1UpdateSparse(primalUpdate, features.Values, features.Indices, features.Count, l1Threshold, l1IntermediateWeights[0].Values, weights[0].Values); } break; diff --git a/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs b/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs index 4b8b0aa6bf..2561cba440 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs @@ -210,9 +210,9 @@ protected override void TrainWithoutLock(IProgressChannelProvider progress, Floa } if (features.IsDense) - CpuMathUtils.SdcaL1UpdateDense(-primalUpdate, features.Length, features.Values, l1Threshold, l1IntermediateWeights[iClass].Values, weights[iClass].Values); + CpuMathUtils.SdcaL1UpdateDense(-primalUpdate, features.Values, l1Threshold, l1IntermediateWeights[iClass].Values, weights[iClass].Values); else if (features.Count > 0) - CpuMathUtils.SdcaL1UpdateSparse(-primalUpdate, features.Length, features.Values, features.Indices, features.Count, l1Threshold, l1IntermediateWeights[iClass].Values, weights[iClass].Values); + CpuMathUtils.SdcaL1UpdateSparse(-primalUpdate, features.Values, features.Indices, features.Count, l1Threshold, l1IntermediateWeights[iClass].Values, weights[iClass].Values); } break; @@ -237,9 +237,9 @@ protected override void TrainWithoutLock(IProgressChannelProvider progress, Floa : 0; if (features.IsDense) - CpuMathUtils.SdcaL1UpdateDense(labelPrimalUpdate, features.Length, features.Values, l1Threshold, l1IntermediateWeights[label].Values, weights[label].Values); + CpuMathUtils.SdcaL1UpdateDense(labelPrimalUpdate, features.Values, l1Threshold, l1IntermediateWeights[label].Values, weights[label].Values); else if (features.Count > 0) - CpuMathUtils.SdcaL1UpdateSparse(labelPrimalUpdate, features.Length, features.Values, features.Indices, features.Count, l1Threshold, l1IntermediateWeights[label].Values, weights[label].Values); + CpuMathUtils.SdcaL1UpdateSparse(labelPrimalUpdate, features.Values, features.Indices, features.Count, l1Threshold, l1IntermediateWeights[label].Values, weights[label].Values); } rowCount++; From dcbaf69ec3133a2fa154e94fa90a235ae13385c6 Mon Sep 17 00:00:00 2001 From: J W Date: Sat, 13 Oct 2018 08:29:32 -0400 Subject: [PATCH 16/19] Update test arguments --- test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs index c29b7e4917..50efd1bd97 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs @@ -594,7 +594,7 @@ public void SdcaL1UpdateUTest(int test) expected[i] = Math.Abs(value) > DefaultScale ? (value > 0 ? value - DefaultScale : value + DefaultScale) : 0; } - CpuMathUtils.SdcaL1UpdateDense(DefaultScale, src.Length, src, DefaultScale, v, w); + CpuMathUtils.SdcaL1UpdateDense(DefaultScale, src, DefaultScale, v, w); var actual = w; Assert.Equal(expected, actual, _comparer); } From 2a1a8ddd6322662157ed818fe3681304ed17c4fa Mon Sep 17 00:00:00 2001 From: J W Date: Sat, 13 Oct 2018 08:30:52 -0400 Subject: [PATCH 17/19] Update more argument names --- .../CpuMathUtils.netcoreapp.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 2fc5deb3d7..98a8c515a8 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -415,34 +415,34 @@ private static void AddScale(float scale, Span source, Span indices, } } - public static void AddScaleCopy(float scale, float[] source, float[] destination, float[] res, int count) + public static void AddScaleCopy(float scale, float[] source, float[] destination, float[] result, int count) { Contracts.AssertNonEmpty(source); Contracts.AssertNonEmpty(destination); - Contracts.AssertNonEmpty(res); + Contracts.AssertNonEmpty(result); Contracts.Assert(count > 0); Contracts.Assert(count <= source.Length); Contracts.Assert(count <= destination.Length); - Contracts.Assert(count <= res.Length); + Contracts.Assert(count <= result.Length); - AddScaleCopy(scale, new Span(source, 0, count), new Span(destination, 0, count), new Span(res, 0, count)); + AddScaleCopy(scale, new Span(source, 0, count), new Span(destination, 0, count), new Span(result, 0, count)); } - private static void AddScaleCopy(float scale, Span source, Span destination, Span res) + private static void AddScaleCopy(float scale, Span source, Span destination, Span result) { if (Avx.IsSupported) { - AvxIntrinsics.AddScaleCopyU(scale, source, destination, res); + AvxIntrinsics.AddScaleCopyU(scale, source, destination, result); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleCopyU(scale, source, destination, res); + SseIntrinsics.AddScaleCopyU(scale, source, destination, result); } else { - for (int i = 0; i < res.Length; i++) + for (int i = 0; i < result.Length; i++) { - res[i] = scale * source[i] + destination[i]; + result[i] = scale * source[i] + destination[i]; } } } From 46b31a9aeb6d8dc584b8c2126782b287095b8076 Mon Sep 17 00:00:00 2001 From: J W Date: Mon, 29 Oct 2018 17:48:47 -0400 Subject: [PATCH 18/19] Fix merge issues --- .../CpuMathUtils.netcoreapp.cs | 16 ++++++++-------- .../CpuMathUtils.netstandard.cs | 6 +++--- .../UnitTests.cs | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index f1bbc71c4c..763350d914 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -33,12 +33,12 @@ public static void MatrixTimesSource(bool transpose, AlignedArray matrix, Aligne { if (!transpose) { - Contracts.Assert(stride <= dst.Size); + Contracts.Assert(stride <= destination.Size); AvxIntrinsics.MatMul(matrix, source, destination, stride, source.Size); } else { - Contracts.Assert(stride <= src.Size); + Contracts.Assert(stride <= source.Size); AvxIntrinsics.MatMulTran(matrix, source, destination, destination.Size, stride); } } @@ -91,10 +91,10 @@ public static void MatrixTimesSource(bool transpose, AlignedArray matrix, Aligne public static void MatTimesSrc(AlignedArray matrix, int[] rgposSrc, AlignedArray sourceValues, int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) { - Contracts.AssertValue(posSource); + Contracts.AssertValue(rgposSrc); Contracts.Assert(iposMin >= 0); Contracts.Assert(iposMin <= iposLimit); - Contracts.Assert(iposLimit <= posSource.Length); + Contracts.Assert(iposLimit <= rgposSrc.Length); Contracts.Assert(matrix.Size == destination.Size * sourceValues.Size); if (iposMin >= iposLimit) @@ -103,7 +103,7 @@ public static void MatTimesSrc(AlignedArray matrix, int[] rgposSrc, AlignedArray return; } - Contracts.AssertNonEmpty(posSource); + Contracts.AssertNonEmpty(rgposSrc); Contracts.Assert(stride >= 0); if (Avx.IsSupported) @@ -287,11 +287,11 @@ public static void AddScaleCopy(float scale, ReadOnlySpan source, ReadOnl if (Avx.IsSupported) { - AvxIntrinsics.AddScaleCopyU(a, src, dst, res, count); + AvxIntrinsics.AddScaleCopyU(scale, source, destination, result, count); } else if (Sse.IsSupported) { - SseIntrinsics.AddScaleCopyU(a, src, dst, res, count); + SseIntrinsics.AddScaleCopyU(scale, source, destination, result, count); } else { @@ -707,7 +707,7 @@ public static void SdcaL1UpdateDense(float primalUpdate, int count, ReadOnlySpan Contracts.AssertNonEmpty(v); Contracts.AssertNonEmpty(w); Contracts.Assert(count > 0); - Contracts.Assert(count <= src.Length); + Contracts.Assert(count <= source.Length); Contracts.Assert(count <= v.Length); Contracts.Assert(count <= w.Length); diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index ff3fcaca97..bc9569d390 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -16,10 +16,10 @@ public static partial class CpuMathUtils public static int GetVectorAlignment() => Vector128Alignment; - public static void MatTimesSrc(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(transpose, matrix, source, destination, stride); + public static void MatrixTimesSource(bool transpose, AlignedArray matrix, AlignedArray source, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(transpose, matrix, source, destination, stride); - public static void MatTimesSrc(AlignedArray matrix, int[] rgposSrc, AlignedArray sourceValues, - int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(matrix, rgposSrc, srcValues, posMin, iposMin, iposLimit, destination, stride); + public static void MatrixTimesSource(AlignedArray matrix, int[] rgposSrc, AlignedArray sourceValues, + int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) => SseUtils.MatTimesSrc(matrix, rgposSrc, sourceValues, posMin, iposMin, iposLimit, destination, stride); public static void Add(float value, Span destination) => SseUtils.Add(value, destination); diff --git a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs index 7e9df73262..1d2b53bb79 100644 --- a/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs +++ b/test/Microsoft.ML.CpuMath.UnitTests.netcoreapp/UnitTests.cs @@ -511,7 +511,7 @@ public void SdcaL1UpdateUTest(int test) expected[i] = Math.Abs(value) > DefaultScale ? (value > 0 ? value - DefaultScale : value + DefaultScale) : 0; } - CpuMathUtils.SdcaL1UpdateDense(DefaultScale, src, DefaultScale, v, w); + CpuMathUtils.SdcaL1UpdateDense(DefaultScale, src.Length, src, DefaultScale, v, w); var actual = w; Assert.Equal(expected, actual, _comparer); } From 3fe14e6ed6eb6fe2c59171716686b315f8e5e4c9 Mon Sep 17 00:00:00 2001 From: J W Date: Mon, 29 Oct 2018 17:58:29 -0400 Subject: [PATCH 19/19] Further PR updates --- src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs | 2 +- src/Microsoft.ML.CpuMath/Thunk.cs | 1 - src/Microsoft.ML.Transforms/RffTransform.cs | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs index 763350d914..e9d95ccc1d 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netcoreapp.cs @@ -88,7 +88,7 @@ public static void MatrixTimesSource(bool transpose, AlignedArray matrix, Aligne } } - public static void MatTimesSrc(AlignedArray matrix, int[] rgposSrc, AlignedArray sourceValues, + public static void MatrixTimesSource(AlignedArray matrix, int[] rgposSrc, AlignedArray sourceValues, int posMin, int iposMin, int iposLimit, AlignedArray destination, int stride) { Contracts.AssertValue(rgposSrc); diff --git a/src/Microsoft.ML.CpuMath/Thunk.cs b/src/Microsoft.ML.CpuMath/Thunk.cs index f177a8329c..86e3992f62 100644 --- a/src/Microsoft.ML.CpuMath/Thunk.cs +++ b/src/Microsoft.ML.CpuMath/Thunk.cs @@ -92,7 +92,6 @@ public static extern void MatMulP(/*const*/ float* pmat, /*const*/ int* pposSrc, public static extern void ZeroMatrixItemsCore(float* pd, int c, int ccol, int cfltRow, /*const*/ int* pindices, int cindices); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] - public static extern void SdcaL1UpdateU(float primalUpdate, /*const*/ float* ps, float threshold, float* pd1, float* pd2, int c); [DllImport(NativePath), SuppressUnmanagedCodeSecurity] diff --git a/src/Microsoft.ML.Transforms/RffTransform.cs b/src/Microsoft.ML.Transforms/RffTransform.cs index 025cf807ba..30c311ca68 100644 --- a/src/Microsoft.ML.Transforms/RffTransform.cs +++ b/src/Microsoft.ML.Transforms/RffTransform.cs @@ -568,7 +568,6 @@ private ValueGetter> GetterFromVectorType(IRow input, int iinfo) private ValueGetter> GetterFromFloatType(IRow input, int iinfo) { - var getSrc = input.GetGetter(_srcCols[iinfo]); var src = default(float);