Movement and Internalization Phase 2 (#1626)

* Internalize ITrainer
* Internalize TrainContext
* Internalization of environment implementations
* Internalize HostEnvironmentBase/ConsoleEnvironment
* Limit usage of ConsoleEnvironment
* Internalize ICommandLineComponentFactory
* Internalize ICommand and all implementations
* Internalize much of the entry-point attribute support
* Internalize much of the channel and message infrastructure
* Fix two tests from failing by writing to same path from two concurrently running tests.

Author: TomFinley

Microsoft.ML.sln

@@ -17,8 +17,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.CpuMath", "src
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PipelineInference", "src\Microsoft.ML.PipelineInference\Microsoft.ML.PipelineInference.csproj", "{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.InferenceTesting", "test\Microsoft.ML.InferenceTesting\Microsoft.ML.InferenceTesting.csproj", "{E278EC99-E6EE-49FE-92E6-0A309A478D98}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Data", "src\Microsoft.ML.Data\Microsoft.ML.Data.csproj", "{AD92D96B-0E96-4F22-8DCE-892E13B1F282}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Onnx", "src\Microsoft.ML.Onnx\Microsoft.ML.Onnx.csproj", "{65D0603E-B96C-4DFC-BDD1-705891B88C18}"
@@ -175,14 +173,6 @@ Global
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release|Any CPU.Build.0 = Release|Any CPU
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Debug-Intrinsics|Any CPU.Build.0 = Debug-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release|Any CPU.Build.0 = Release|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release-Intrinsics|Any CPU.ActiveCfg = Release-Intrinsics|Any CPU
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98}.Release-Intrinsics|Any CPU.Build.0 = Release-Intrinsics|Any CPU
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282}.Debug-Intrinsics|Any CPU.ActiveCfg = Debug-Intrinsics|Any CPU
@@ -520,7 +510,6 @@ Global
 		{EC743D1D-7691-43B7-B9B0-5F2F7018A8F6} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{46F2F967-C23F-4076-858D-33F7DA9BD2DA} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{2D7391C9-8254-4B8F-BF26-FADAF8F02F44} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
-		{E278EC99-E6EE-49FE-92E6-0A309A478D98} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 		{AD92D96B-0E96-4F22-8DCE-892E13B1F282} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{65D0603E-B96C-4DFC-BDD1-705891B88C18} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{707BB22C-7E5F-497A-8C2F-74578F675705} = {09EADF06-BE25-4228-AB53-95AE3E15B530}

src/Microsoft.ML.Api/GenerateCodeCommand.cs

@@ -24,7 +24,7 @@ namespace Microsoft.ML.Runtime.Api
     ///
     /// REVIEW: Consider adding support for generating VBuffers instead of arrays, maybe for high dimensionality vectors.
     /// </summary>
-    public sealed class GenerateCodeCommand : ICommand
+    internal sealed class GenerateCodeCommand : ICommand
     {
         public const string LoadName = "GenerateSamplePredictionCode";
         private const string CodeTemplatePath = "Microsoft.ML.Api.GeneratedCodeTemplate.csresource";

src/Microsoft.ML.Api/TypedCursor.cs

@@ -528,8 +528,6 @@ public ICursor GetRootCursor()
     /// </summary>
     public static class CursoringUtils
     {
-        private const string NeedEnvObsoleteMessage = "This method is obsolete. Please use the overload that takes an additional 'env' argument. An environment can be created via new LocalEnvironment().";
-
         /// <summary>
         /// Generate a strongly-typed cursorable wrapper of the <see cref="IDataView"/>.
         /// </summary>
@@ -550,24 +548,6 @@ public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, IHostEnv
             return TypedCursorable<TRow>.Create(env, data, ignoreMissingColumns, schemaDefinition);
         }
 
-        /// <summary>
-        /// Generate a strongly-typed cursorable wrapper of the <see cref="IDataView"/>.
-        /// </summary>
-        /// <typeparam name="TRow">The user-defined row type.</typeparam>
-        /// <param name="data">The underlying data view.</param>
-        /// <param name="ignoreMissingColumns">Whether to ignore the case when a requested column is not present in the data view.</param>
-        /// <param name="schemaDefinition">Optional user-provided schema definition. If it is not present, the schema is inferred from the definition of T.</param>
-        /// <returns>The cursorable wrapper of <paramref name="data"/>.</returns>
-        [Obsolete(NeedEnvObsoleteMessage)]
-        public static ICursorable<TRow> AsCursorable<TRow>(this IDataView data, bool ignoreMissingColumns = false,
-            SchemaDefinition schemaDefinition = null)
-            where TRow : class, new()
-        {
-            // REVIEW: Take an env as a parameter.
-            var env = new ConsoleEnvironment();
-            return data.AsCursorable<TRow>(env, ignoreMissingColumns, schemaDefinition);
-        }
-
         /// <summary>
         /// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
         /// </summary>
@@ -589,24 +569,5 @@ public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, IHostEnv
             var engine = new PipeEngine<TRow>(env, data, ignoreMissingColumns, schemaDefinition);
             return engine.RunPipe(reuseRowObject);
         }
-
-        /// <summary>
-        /// Convert an <see cref="IDataView"/> into a strongly-typed <see cref="IEnumerable{TRow}"/>.
-        /// </summary>
-        /// <typeparam name="TRow">The user-defined row type.</typeparam>
-        /// <param name="data">The underlying data view.</param>
-        /// <param name="reuseRowObject">Whether to return the same object on every row, or allocate a new one per row.</param>
-        /// <param name="ignoreMissingColumns">Whether to ignore the case when a requested column is not present in the data view.</param>
-        /// <param name="schemaDefinition">Optional user-provided schema definition. If it is not present, the schema is inferred from the definition of T.</param>
-        /// <returns>The <see cref="IEnumerable{TRow}"/> that holds the data in <paramref name="data"/>. It can be enumerated multiple times.</returns>
-        [Obsolete(NeedEnvObsoleteMessage)]
-        public static IEnumerable<TRow> AsEnumerable<TRow>(this IDataView data, bool reuseRowObject,
-            bool ignoreMissingColumns = false, SchemaDefinition schemaDefinition = null)
-            where TRow : class, new()
-        {
-            // REVIEW: Take an env as a parameter.
-            var env = new ConsoleEnvironment();
-            return data.AsEnumerable<TRow>(env, reuseRowObject, ignoreMissingColumns, schemaDefinition);
-        }
     }
 }

src/Microsoft.ML.Core/CommandLine/CmdParser.cs

@@ -40,7 +40,8 @@ internal enum SettingsFlags
     ///
     /// This allows components to be created by name, signature type, and a settings string.
     /// </summary>
-    public interface ICommandLineComponentFactory : IComponentFactory
+    [BestFriend]
+    internal interface ICommandLineComponentFactory : IComponentFactory
     {
         Type SignatureType { get; }
         string Name { get; }

src/Common/AssemblyLoadingUtils.cs renamed to src/Microsoft.ML.Core/ComponentModel/AssemblyLoadingUtils.cs

@@ -6,11 +6,13 @@
 using System;
 using System.IO;
 using System.IO.Compression;
-using System.Linq;
 using System.Reflection;
 
 namespace Microsoft.ML.Runtime
 {
+    [Obsolete("The usage for this is intended for the internal command line utilities and is not intended for anything related to the API. " +
+        "Please consider another way of doing whatever it is you're attempting to accomplish.")]
+    [BestFriend]
     internal static class AssemblyLoadingUtils
     {
         /// <summary>

src/Microsoft.ML.Core/Data/ICommand.cs

@@ -11,9 +11,11 @@ namespace Microsoft.ML.Runtime.Command
     /// <summary>
     /// The signature for commands.
     /// </summary>
-    public delegate void SignatureCommand();
+    [BestFriend]
+    internal delegate void SignatureCommand();
 
-    public interface ICommand
+    [BestFriend]
+    internal interface ICommand
     {
         void Run();
     }

src/Microsoft.ML.Core/Data/IFileHandle.cs

@@ -61,7 +61,7 @@ public sealed class SimpleFileHandle : IFileHandle
         // handle has been disposed.
         private List<Stream> _streams;
 
-        private bool IsDisposed { get { return _streams == null; } }
+        private bool IsDisposed => _streams == null;
 
         public SimpleFileHandle(IExceptionContext ectx, string path, bool needsWrite, bool autoDelete)
         {
@@ -84,15 +84,9 @@ public SimpleFileHandle(IExceptionContext ectx, string path, bool needsWrite, bo
             _streams = new List<Stream>();
         }
 
-        public bool CanWrite
-        {
-            get { return !_wrote && !IsDisposed; }
-        }
+        public bool CanWrite => !_wrote && !IsDisposed;
 
-        public bool CanRead
-        {
-            get { return _wrote && !IsDisposed; }
-        }
+        public bool CanRead => _wrote && !IsDisposed;
 
         public void Dispose()
         {

src/Microsoft.ML.Core/Data/IHostEnvironment.cs

@@ -72,6 +72,8 @@ public interface IHostEnvironment : IChannelProvider, IProgressChannelProvider
         /// The suffix and prefix are optional. A common use for suffix is to specify an extension, eg, ".txt".
         /// The use of suffix and prefix, including whether they have any affect, is up to the host environment.
         /// </summary>
+        [Obsolete("The host environment is not disposable, so it is inappropriate to use this method. " +
+            "Please handle your own temporary files within the component yourself, including their proper disposal and deletion.")]
         IFileHandle CreateTempFile(string suffix = null, string prefix = null);
 
         /// <summary>
@@ -188,7 +190,8 @@ public readonly struct ChannelMessage
         /// </summary>
         public string Message => _args != null ? string.Format(_message, _args) : _message;
 
-        public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string message)
+        [BestFriend]
+        internal ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string message)
         {
             Contracts.CheckNonEmpty(message, nameof(message));
             Kind = kind;
@@ -197,7 +200,8 @@ public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, s
             _args = null;
         }
 
-        public ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string fmt, params object[] args)
+        [BestFriend]
+        internal ChannelMessage(ChannelMessageKind kind, MessageSensitivity sensitivity, string fmt, params object[] args)
         {
             Contracts.CheckNonEmpty(fmt, nameof(fmt));
             Contracts.CheckNonEmpty(args, nameof(args));

src/Microsoft.ML.Core/Data/ProgressReporter.cs

@@ -14,7 +14,8 @@ namespace Microsoft.ML.Runtime.Data
     /// <summary>
     /// The progress reporting classes used by <see cref="HostEnvironmentBase{THostEnvironmentBase}"/> descendants.
     /// </summary>
-    public static class ProgressReporting
+    [BestFriend]
+    internal static class ProgressReporting
     {
         /// <summary>
         /// The progress channel for <see cref="ConsoleEnvironment"/>.

src/Microsoft.ML.Core/Data/ServerChannel.cs

@@ -19,7 +19,8 @@ namespace Microsoft.ML.Runtime
     /// delegates will be published in some fashion, with the target scenario being
     /// that the library will publish some sort of restful API.
     /// </summary>
-    public sealed class ServerChannel : ServerChannel.IPendingBundleNotification, IDisposable
+    [BestFriend]
+    internal sealed class ServerChannel : ServerChannel.IPendingBundleNotification, IDisposable
     {
         // See ServerChannel.md for a more elaborate discussion of high level usage and design.
         private readonly IChannelProvider _chp;
@@ -250,7 +251,8 @@ public void AddDoneAction(Action onDone)
         }
     }
 
-    public static class ServerChannelUtilities
+    [BestFriend]
+    internal static class ServerChannelUtilities
     {
         /// <summary>
         /// Convenience method for <see cref="ServerChannel.Start"/> that looks more idiomatic to typical