Estimators for Timeseries SSA / IID ChangepointDetection and SpikeDetection transforms

src/Microsoft.ML.TimeSeries/AdaptiveSingularSpectrumSequenceModeler.cs

@@ -221,13 +221,16 @@ private static VersionInfo GetVersionInfo()
         {
             return new VersionInfo(
                 modelSignature: "SSAMODLR",
-                verWrittenCur: 0x00010001, // Initial
-                verReadableCur: 0x00010001,
+                //verWrittenCur: 0x00010001, // Initial
+                verWrittenCur: 0x00010002, // Added saving _state and _nextPrediction
+                verReadableCur: 0x00010002,
                 verWeCanReadBack: 0x00010001,
                 loaderSignature: LoaderSignature,
                 loaderAssemblyName: typeof(AdaptiveSingularSpectrumSequenceModeler).Assembly.FullName);
         }
 
+        private const int VersionSavingStateAndPrediction = 0x00010002;
+
         /// <summary>
         /// The constructor for Adaptive SSA model.
         /// </summary>
@@ -333,6 +336,7 @@ private AdaptiveSingularSpectrumSequenceModeler(AdaptiveSingularSpectrumSequence
             _autoregressionNoiseVariance = model._autoregressionNoiseVariance;
             _observationNoiseMean = model._observationNoiseMean;
             _autoregressionNoiseMean = model._autoregressionNoiseMean;
+            _nextPrediction = model._nextPrediction;
             _maxTrendRatio = model._maxTrendRatio;
             _shouldStablize = model._shouldStablize;
             _shouldMaintainInfo = model._shouldMaintainInfo;
@@ -368,11 +372,13 @@ public AdaptiveSingularSpectrumSequenceModeler(IHostEnvironment env, ModelLoadCo
             // RankSelectionMethod: _rankSelectionMethod
             // bool: isWeightSet
             // float[]: _alpha
+            // float[]: _state
             // bool: ShouldComputeForecastIntervals
             // float: _observationNoiseVariance
             // float: _autoregressionNoiseVariance
             // float: _observationNoiseMean
             // float: _autoregressionNoiseMean
+            // float: _nextPrediction
             // int: _maxRank
             // bool: _shouldStablize
             // bool: _shouldMaintainInfo
@@ -404,6 +410,14 @@ public AdaptiveSingularSpectrumSequenceModeler(IHostEnvironment env, ModelLoadCo
             _alpha = ctx.Reader.ReadFloatArray();
             _host.CheckDecode(Utils.Size(_alpha) == _windowSize - 1);
 
+            if (ctx.Header.ModelVerReadable >= VersionSavingStateAndPrediction)
+            {
+                _state = ctx.Reader.ReadFloatArray();
+                _host.CheckDecode(Utils.Size(_state) == _windowSize - 1);
+            }
+            else
+                _state = new Single[_windowSize - 1];
+
             ShouldComputeForecastIntervals = ctx.Reader.ReadBoolByte();
 
             _observationNoiseVariance = ctx.Reader.ReadSingle();
@@ -414,6 +428,8 @@ public AdaptiveSingularSpectrumSequenceModeler(IHostEnvironment env, ModelLoadCo
 
             _observationNoiseMean = ctx.Reader.ReadSingle();
             _autoregressionNoiseMean = ctx.Reader.ReadSingle();
+            if (ctx.Header.ModelVerReadable >= VersionSavingStateAndPrediction)
+                _nextPrediction = ctx.Reader.ReadSingle();
 
             _maxRank = ctx.Reader.ReadInt32();
             _host.CheckDecode(1 <= _maxRank && _maxRank <= _windowSize - 1);
@@ -444,7 +460,7 @@ public AdaptiveSingularSpectrumSequenceModeler(IHostEnvironment env, ModelLoadCo
             }
 
             _buffer = new FixedSizeQueue<Single>(_seriesLength);
-            _state = new Single[_windowSize - 1];
+
             _x = new CpuAlignedVector(_windowSize, SseUtils.CbAlign);
             _xSmooth = new CpuAlignedVector(_windowSize, SseUtils.CbAlign);
         }
@@ -475,11 +491,13 @@ public void Save(ModelSaveContext ctx)
             // RankSelectionMethod: _rankSelectionMethod
             // bool: _isWeightSet
             // float[]: _alpha
+            // float[]: _state
             // bool: ShouldComputeForecastIntervals
             // float: _observationNoiseVariance
             // float: _autoregressionNoiseVariance
             // float: _observationNoiseMean
             // float: _autoregressionNoiseMean
+            // float: _nextPrediction
             // int: _maxRank
             // bool: _shouldStablize
             // bool: _shouldMaintainInfo
@@ -494,11 +512,13 @@ public void Save(ModelSaveContext ctx)
             ctx.Writer.Write((byte)_rankSelectionMethod);
             ctx.Writer.WriteBoolByte(_wTrans != null);
             ctx.Writer.WriteFloatArray(_alpha);
+            ctx.Writer.WriteFloatArray(_state);
             ctx.Writer.WriteBoolByte(ShouldComputeForecastIntervals);
             ctx.Writer.Write(_observationNoiseVariance);
             ctx.Writer.Write(_autoregressionNoiseVariance);
             ctx.Writer.Write(_observationNoiseMean);
             ctx.Writer.Write(_autoregressionNoiseMean);
+            ctx.Writer.Write(_nextPrediction);
             ctx.Writer.Write(_maxRank);
             ctx.Writer.WriteBoolByte(_shouldStablize);
             ctx.Writer.WriteBoolByte(_shouldMaintainInfo);

src/Microsoft.ML.TimeSeries/IidAnomalyDetectionBase.cs

@@ -15,18 +15,32 @@ namespace Microsoft.ML.Runtime.TimeSeriesProcessing
     /// </summary>
     public abstract class IidAnomalyDetectionBase : SequentialAnomalyDetectionTransformBase<Single, IidAnomalyDetectionBase.State>
     {
-        public IidAnomalyDetectionBase(ArgumentsBase args, string name, IHostEnvironment env, IDataView input)
-            : base(args, name, env, input)
+        public IidAnomalyDetectionBase(ArgumentsBase args, string name, IHostEnvironment env)
+            : base(args, name, env)
         {
             InitialWindowSize = 0;
         }
 
-        public IidAnomalyDetectionBase(IHostEnvironment env, ModelLoadContext ctx, string name, IDataView input)
-            : base(env, ctx, name, input)
+        public IidAnomalyDetectionBase(IHostEnvironment env, ModelLoadContext ctx, string name)
+            : base(env, ctx, name)
         {
             Host.CheckDecode(InitialWindowSize == 0);
         }
 
+        public override Schema GetOutputSchema(Schema inputSchema)
+        {
+            Host.CheckValue(inputSchema, nameof(inputSchema));
+
+            if (!inputSchema.TryGetColumnIndex(InputColumnName, out var col))
+                throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", InputColumnName);
+
+            var colType = inputSchema.GetColumnType(col);
+            if (colType != NumberType.R4)
+                throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", InputColumnName, NumberType.R4.ToString(), colType.ToString());
+
+            return Transform(new EmptyDataView(Host, inputSchema)).Schema;
+        }
+
         public override void Save(ModelSaveContext ctx)
         {
             ctx.CheckAtModel();

src/Microsoft.ML.TimeSeries/IidChangePointDetector.cs

@@ -3,24 +3,35 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Core.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.CommandLine;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.EntryPoints;
 using Microsoft.ML.Runtime.Model;
 using Microsoft.ML.Runtime.TimeSeriesProcessing;
+using static Microsoft.ML.Runtime.TimeSeriesProcessing.SequentialAnomalyDetectionTransformBase<System.Single, Microsoft.ML.Runtime.TimeSeriesProcessing.IidAnomalyDetectionBase.State>;
 
-[assembly: LoadableClass(IidChangePointDetector.Summary, typeof(IidChangePointDetector), typeof(IidChangePointDetector.Arguments), typeof(SignatureDataTransform),
+[assembly: LoadableClass(IidChangePointDetector.Summary, typeof(IDataTransform), typeof(IidChangePointDetector), typeof(IidChangePointDetector.Arguments), typeof(SignatureDataTransform),
     IidChangePointDetector.UserName, IidChangePointDetector.LoaderSignature, IidChangePointDetector.ShortName)]
-[assembly: LoadableClass(IidChangePointDetector.Summary, typeof(IidChangePointDetector), null, typeof(SignatureLoadDataTransform),
+
+[assembly: LoadableClass(IidChangePointDetector.Summary, typeof(IDataTransform), typeof(IidChangePointDetector), null, typeof(SignatureLoadDataTransform),
+    IidChangePointDetector.UserName, IidChangePointDetector.LoaderSignature)]
+
+[assembly: LoadableClass(IidChangePointDetector.Summary, typeof(IidChangePointDetector), null, typeof(SignatureLoadModel),
     IidChangePointDetector.UserName, IidChangePointDetector.LoaderSignature)]
 
+[assembly: LoadableClass(typeof(IRowMapper), typeof(IidChangePointDetector), null, typeof(SignatureLoadRowMapper),
+   IidChangePointDetector.UserName, IidChangePointDetector.LoaderSignature)]
+
 namespace Microsoft.ML.Runtime.TimeSeriesProcessing
 {
     /// <summary>
     /// This class implements the change point detector transform for an i.i.d. sequence based on adaptive kernel density estimation and martingales.
     /// </summary>
-    public sealed class IidChangePointDetector : IidAnomalyDetectionBase, ITransformTemplate
+    public sealed class IidChangePointDetector : IidAnomalyDetectionBase
     {
         internal const string Summary = "This transform detects the change-points in an i.i.d. sequence using adaptive kernel density estimation and martingales.";
         public const string LoaderSignature = "IidChangePointDetector";
@@ -89,8 +100,18 @@ private static VersionInfo GetVersionInfo()
                 loaderAssemblyName: typeof(IidChangePointDetector).Assembly.FullName);
         }
 
-        public IidChangePointDetector(IHostEnvironment env, Arguments args, IDataView input)
-            : base(new BaseArguments(args), LoaderSignature, env, input)
+        // Factory method for SignatureDataTransform.
+        private static IDataTransform Create(IHostEnvironment env, Arguments args, IDataView input)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValue(args, nameof(args));
+            env.CheckValue(input, nameof(input));
+
+            return new IidChangePointDetector(env, args).MakeDataTransform(input);
+        }
+
+        internal IidChangePointDetector(IHostEnvironment env, Arguments args)
+            : base(new BaseArguments(args), LoaderSignature, env)
         {
             switch (Martingale)
             {
@@ -109,8 +130,28 @@ public IidChangePointDetector(IHostEnvironment env, Arguments args, IDataView in
             }
         }
 
-        public IidChangePointDetector(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
-            : base(env, ctx, LoaderSignature, input)
+        // Factory method for SignatureLoadDataTransform.
+        private static IDataTransform Create(IHostEnvironment env, ModelLoadContext ctx, IDataView input)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValue(ctx, nameof(ctx));
+            env.CheckValue(input, nameof(input));
+
+            return new IidChangePointDetector(env, ctx).MakeDataTransform(input);
+        }
+
+        // Factory method for SignatureLoadModel.
+        private static IidChangePointDetector Create(IHostEnvironment env, ModelLoadContext ctx)
+        {
+            Contracts.CheckValue(env, nameof(env));
+            env.CheckValue(ctx, nameof(ctx));
+            ctx.CheckAtModel(GetVersionInfo());
+
+            return new IidChangePointDetector(env, ctx);
+        }
+
+        internal IidChangePointDetector(IHostEnvironment env, ModelLoadContext ctx)
+            : base(env, ctx, LoaderSignature)
         {
             // *** Binary format ***
             // <base>
@@ -119,8 +160,8 @@ public IidChangePointDetector(IHostEnvironment env, ModelLoadContext ctx, IDataV
             Host.CheckDecode(Side == AnomalySide.TwoSided);
         }
 
-        private IidChangePointDetector(IHostEnvironment env, IidChangePointDetector transform, IDataView newSource)
-            : base(new BaseArguments(transform), LoaderSignature, env, newSource)
+        private IidChangePointDetector(IHostEnvironment env, IidChangePointDetector transform)
+            : base(new BaseArguments(transform), LoaderSignature, env)
         {
         }
 
@@ -139,9 +180,65 @@ public override void Save(ModelSaveContext ctx)
             base.Save(ctx);
         }
 
-        public IDataTransform ApplyToData(IHostEnvironment env, IDataView newSource)
+        // Factory method for SignatureLoadRowMapper.
+        private static IRowMapper Create(IHostEnvironment env, ModelLoadContext ctx, ISchema inputSchema)
+            => Create(env, ctx).MakeRowMapper(inputSchema);
+    }
+
+    /// <summary>
+    /// Estimator for <see cref="IidChangePointDetector"/>
+    /// </summary>
+    public sealed class IidChangePointEstimator : TrivialEstimator<IidChangePointDetector>
+    {
+        /// <summary>
+        /// Create a new instance of <see cref="IidChangePointEstimator"/>
+        /// </summary>
+        /// <param name="env">Host Environment.</param>
+        /// <param name="inputColumn">Name of the input column.</param>
+        /// <param name="outputColumn">The name of the new column.</param>
+        /// <param name="confidence">The confidence for change point detection in the range [0, 100].</param>
+        /// <param name="changeHistoryLength">The change history length.</param>
+        /// <param name="martingale">The martingale used for scoring.</param>
+        /// <param name="eps">The epsilon parameter for the Power martingale.</param>
+        public IidChangePointEstimator(IHostEnvironment env, string inputColumn, string outputColumn, int confidence,
+            int changeHistoryLength, MartingaleType martingale = MartingaleType.Power, double eps = 0.1)
+            : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(IidChangePointEstimator)),
+                new IidChangePointDetector(env, new IidChangePointDetector.Arguments
+                {
+                    Name = outputColumn,
+                    Source = inputColumn,
+                    Confidence = confidence,
+                    ChangeHistoryLength = changeHistoryLength,
+                    Martingale = martingale,
+                    PowerMartingaleEpsilon = eps
+                }))
+        {
+        }
+
+        public IidChangePointEstimator(IHostEnvironment env, IidChangePointDetector.Arguments args)
+            : base(Contracts.CheckRef(env, nameof(env)).Register(nameof(IidChangePointEstimator)),
+                new IidChangePointDetector(env, args))
         {
-            return new IidChangePointDetector(env, this, newSource);
+        }
+
+        public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
+        {
+            Host.CheckValue(inputSchema, nameof(inputSchema));
+
+            if (!inputSchema.TryFindColumn(Transformer.InputColumnName, out var col))
+                throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", Transformer.InputColumnName);
+            if (col.ItemType != NumberType.R4)
+                throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", Transformer.InputColumnName, NumberType.R4.ToString(), col.GetTypeString());
+
+            var metadata = new List<SchemaShape.Column>() {
+                new SchemaShape.Column(MetadataUtils.Kinds.SlotNames, SchemaShape.Column.VectorKind.Vector, TextType.Instance, false)
+            };
+            var resultDic = inputSchema.Columns.ToDictionary(x => x.Name);
+
+            resultDic[Transformer.OutputColumnName] = new SchemaShape.Column(
+                Transformer.OutputColumnName, SchemaShape.Column.VectorKind.Vector, NumberType.R8, false, new SchemaShape(metadata));
+
+            return new SchemaShape(resultDic.Values);
         }
     }
 }