Add support for Linux and global python installations

src/Bonsai.Scripting.Python/EnvironmentConfig.cs

@@ -0,0 +1,66 @@
+using System.IO;
+
+namespace Bonsai.Scripting.Python
+{
+    internal class EnvironmentConfig
+    {
+        private EnvironmentConfig()
+        {
+        }
+
+        public EnvironmentConfig(string pythonHome, string pythonVersion)
+        {
+            Path = pythonHome;
+            PythonHome = pythonHome;
+            PythonVersion = pythonVersion;
+            IncludeSystemSitePackages = true;
+        }
+
+        public string Path { get; private set; }
+
+        public string PythonHome { get; private set; }
+
+        public string PythonVersion { get; private set; }
+
+        public bool IncludeSystemSitePackages { get; private set; }
+
+        public static EnvironmentConfig FromConfigFile(string configFileName)
+        {
+            var config = new EnvironmentConfig
+            {
+                Path = System.IO.Path.GetDirectoryName(configFileName)
+            };
+            using var configReader = new StreamReader(File.OpenRead(configFileName));
+            while (!configReader.EndOfStream)
+            {
+                var line = configReader.ReadLine();
+
+                if (line.StartsWith("home"))
+                {
+                    config.PythonHome = GetConfigValue(line);
+                }
+                else if (line.StartsWith("include-system-site-packages"))
+                {
+                    config.IncludeSystemSitePackages = bool.Parse(GetConfigValue(line));
+                }
+                else if (line.StartsWith("version"))
+                {
+                    var pythonVersion = GetConfigValue(line);
+                    if (!string.IsNullOrEmpty(pythonVersion))
+                    {
+                        pythonVersion = pythonVersion.Substring(0, pythonVersion.LastIndexOf('.'));
+                    }
+                    config.PythonVersion = pythonVersion;
+                }
+            }
+
+            return config;
+        }
+
+        private static string GetConfigValue(string line)
+        {
+            var parts = line.Split('=');
+            return parts.Length > 1 ? parts[1].Trim() : string.Empty;
+        }
+    }
+}

src/Bonsai.Scripting.Python/EnvironmentHelper.cs

@@ -1,63 +1,134 @@
 using System;
 using System.IO;
-using System.Linq;
+using System.Runtime.InteropServices;
 using Python.Runtime;
 
 namespace Bonsai.Scripting.Python
 {
     static class EnvironmentHelper
     {
-        public static string GetPythonDLL(string path)
+        public static string GetPythonDLL(EnvironmentConfig config)
         {
-            return Directory
-                .EnumerateFiles(path, searchPattern: "python3?*.*")
-                .Select(Path.GetFileNameWithoutExtension)
-                .Where(match => match.Length > "python3".Length)
-                .Select(match => match.Replace(".", string.Empty))
-                .FirstOrDefault();
+            return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+                ? $"python{config.PythonVersion.Replace(".", string.Empty)}.dll"
+                : $"libpython{config.PythonVersion}.so";
         }
 
         public static void SetRuntimePath(string pythonHome)
         {
-            var path = Environment.GetEnvironmentVariable("PATH", EnvironmentVariableTarget.Process).TrimEnd(Path.PathSeparator);
-            path = string.IsNullOrEmpty(path) ? pythonHome : pythonHome + Path.PathSeparator + path;
-            Environment.SetEnvironmentVariable("PATH", path, EnvironmentVariableTarget.Process);
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                var path = Environment.GetEnvironmentVariable("PATH", EnvironmentVariableTarget.Process).TrimEnd(Path.PathSeparator);
+                path = string.IsNullOrEmpty(path) ? pythonHome : pythonHome + Path.PathSeparator + path;
+                Environment.SetEnvironmentVariable("PATH", path, EnvironmentVariableTarget.Process);
+            }
+        }
+
+        static string FindPythonHome()
+        {
+            var systemPath = Environment.GetEnvironmentVariable("PATH");
+            var searchPaths = systemPath.Split(Path.PathSeparator);
+            var isRunningOnWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+            var pythonExecutableName = isRunningOnWindows ? "python.exe" : "python3";
+
+            var pythonHome = Array.Find(searchPaths, path => File.Exists(Path.Combine(path, pythonExecutableName)));
+            if (pythonHome != null && !isRunningOnWindows && MonoHelper.IsRunningOnMono)
+            {
+                var pythonExecutablePath = Path.Combine(pythonHome, pythonExecutableName);
+                pythonExecutablePath = MonoHelper.GetRealPath(pythonExecutablePath);
+                var baseDirectory = Directory.GetParent(pythonExecutablePath).Parent;
+                if (baseDirectory != null)
+                {
+                    pythonHome = Path.Combine(baseDirectory.FullName, "lib", Path.GetFileName(pythonExecutablePath));
+                }
+            }
+
+            return pythonHome;
         }
 
-        public static string GetPythonHome(string path)
+        public static string GetEnvironmentPath(string path)
+        {
+            if (string.IsNullOrEmpty(path))
+            {
+                path = Environment.GetEnvironmentVariable("VIRTUAL_ENV", EnvironmentVariableTarget.Process);
+                if (path == null) return FindPythonHome();
+            }
+
+            return Path.GetFullPath(path);
+        }
+
+        public static EnvironmentConfig GetEnvironmentConfig(string path)
         {
             var configFileName = Path.Combine(path, "pyvenv.cfg");
             if (File.Exists(configFileName))
             {
-                using var configReader = new StreamReader(File.OpenRead(configFileName));
-                while (!configReader.EndOfStream)
+                return EnvironmentConfig.FromConfigFile(configFileName);
+            }
+            else
+            {
+                var pythonHome = path;
+                var pythonVersion = string.Empty;
+                const string DefaultPythonName = "python";
+                if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
                 {
-                    var line = configReader.ReadLine();
-                    if (line.StartsWith("home"))
-                    {
-                        var parts = line.Split('=');
-                        return parts[parts.Length - 1].Trim();
-                    }
+                    var baseDirectory = Directory.GetParent(path).Parent;
+                    pythonHome = Path.Combine(baseDirectory.FullName, "bin");
+                }
+
+                var pythonName = Path.GetFileName(path);
+                var pythonVersionIndex = pythonName.LastIndexOf(DefaultPythonName, StringComparison.OrdinalIgnoreCase);
+                if (pythonVersionIndex >= 0)
+                {
+                    pythonVersion = pythonName.Substring(pythonVersionIndex + DefaultPythonName.Length);
                 }
-            }
 
-            return path;
+                return new EnvironmentConfig(pythonHome, pythonVersion);
+            }
         }
 
-        public static string GetPythonPath(string pythonHome, string path)
+        public static string GetPythonPath(EnvironmentConfig config)
         {
+            string sitePackages;
             var basePath = PythonEngine.PythonPath;
-            if (string.IsNullOrEmpty(basePath))
+            var baseDirectory = AppDomain.CurrentDomain.BaseDirectory;
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                if (string.IsNullOrEmpty(basePath))
+                {
+                    var pythonZip = Path.Combine(config.PythonHome, Path.ChangeExtension(Runtime.PythonDLL, ".zip"));
+                    var pythonDLLs = Path.Combine(config.PythonHome, "DLLs");
+                    var pythonLib = Path.Combine(config.PythonHome, "Lib");
+                    basePath = string.Join(Path.PathSeparator.ToString(), pythonZip, pythonDLLs, pythonLib, baseDirectory);
+                }
+
+                sitePackages = Path.Combine(config.Path, "Lib", "site-packages");
+                if (config.IncludeSystemSitePackages && config.Path != config.PythonHome)
+                {
+                    var systemSitePackages = Path.Combine(config.PythonHome, "Lib", "site-packages");
+                    sitePackages = $"{sitePackages}{Path.PathSeparator}{systemSitePackages}";
+                }
+            }
+            else
             {
-                var pythonZip = Path.Combine(pythonHome, Path.ChangeExtension(Runtime.PythonDLL, ".zip"));
-                var pythonDLLs = Path.Combine(pythonHome, "DLLs");
-                var pythonLib = Path.Combine(pythonHome, "Lib");
-                var baseDirectory = AppDomain.CurrentDomain.BaseDirectory;
-                basePath = string.Join(Path.PathSeparator.ToString(), pythonZip, pythonDLLs, pythonLib, baseDirectory);
+                if (string.IsNullOrEmpty(basePath))
+                {
+                    var pythonBase = Path.GetDirectoryName(config.PythonHome);
+                    pythonBase = Path.Combine(pythonBase, "lib", $"python{config.PythonVersion}");
+                    var pythonLibDynload = Path.Combine(pythonBase, "lib-dynload");
+                    basePath = string.Join(Path.PathSeparator.ToString(), pythonBase, pythonLibDynload, baseDirectory);
+                }
+
+                sitePackages = Path.Combine(config.Path, "lib", $"python{config.PythonVersion}", "site-packages");
+                if (config.IncludeSystemSitePackages)
+                {
+                    var localAppData = Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData);
+                    var localFolder = Directory.GetParent(localAppData).FullName;
+                    var systemSitePackages = Path.Combine(localFolder, "lib", $"python{config.PythonVersion}", "site-packages");
+                    sitePackages = $"{sitePackages}{Path.PathSeparator}{systemSitePackages}";
+                }
             }
 
-            var sitePackages = Path.Combine(path, "Lib", "site-packages");
-            return $"{basePath}{Path.PathSeparator}{path}{Path.PathSeparator}{sitePackages}";
+            return $"{basePath}{Path.PathSeparator}{config.Path}{Path.PathSeparator}{sitePackages}";
         }
     }
-}
+}

src/Bonsai.Scripting.Python/MonoHelper.cs

@@ -0,0 +1,26 @@
+using System;
+
+namespace Bonsai.Scripting.Python
+{
+    static class MonoHelper
+    {
+        internal static readonly bool IsRunningOnMono = Type.GetType("Mono.Runtime") != null;
+
+        public static string GetRealPath(string path)
+        {
+            var unixPath = Type.GetType("Mono.Unix.UnixPath, Mono.Posix");
+            if (unixPath == null)
+            {
+                throw new InvalidOperationException("No compatible Mono.Posix implementation was found.");
+            }
+
+            var getRealPath = unixPath.GetMethod(nameof(GetRealPath));
+            if (getRealPath == null)
+            {
+                throw new InvalidOperationException($"No compatible {nameof(GetRealPath)} method was found.");
+            }
+
+            return (string)getRealPath.Invoke(null, new[] { path });
+        }
+    }
+}

src/Bonsai.Scripting.Python/PythonInterpreterLock.cs

@@ -0,0 +1,68 @@
+using Bonsai.Expressions;
+using System.Linq.Expressions;
+using System.Collections.Generic;
+using System.Reactive.Linq;
+using System;
+using System.Reflection;
+using System.Linq;
+using Python.Runtime;
+using System.Reactive;
+
+namespace Bonsai.Scripting.Python
+{
+    public class PythonInterpreterLock : WorkflowExpressionBuilder
+    {
+        static readonly Range<int> argumentRange = Range.Create(lowerBound: 1, upperBound: 1);
+        private static readonly object _lock = new object();
+
+        public PythonInterpreterLock()
+            : this(new ExpressionBuilderGraph())
+        {
+        }
+
+        public PythonInterpreterLock(ExpressionBuilderGraph workflow)
+            : base(workflow)
+        {
+        }
+
+        public override Range<int> ArgumentRange => argumentRange;
+
+        public override Expression Build(IEnumerable<Expression> arguments)
+        {
+            var source = arguments.FirstOrDefault();
+            if (source == null)
+            {
+                throw new InvalidOperationException("There must be at least one input.");
+            }
+            var sourceType = source.Type.GetGenericArguments()[0]; // Get TSource from IObservable<TSource>
+            var factoryParameter = Expression.Parameter(typeof(IObservable<>).MakeGenericType(sourceType), "factoryParam");
+
+            return BuildWorkflow(arguments, factoryParameter, selectorBody =>
+            {
+                var selector = Expression.Lambda(selectorBody, factoryParameter);
+                var resultType = selectorBody.Type.GetGenericArguments()[0];
+                return Expression.Call(GetType(), nameof(Process), new Type[] {sourceType, resultType}, source, selector);
+            });
+        }
+
+        static IObservable<TResult> Process<TSource, TResult>(IObservable<TSource> source, Func<IObservable<TSource>, IObservable<TResult>> selector)
+        {
+            var gilProtectedSource = Observable.Create<TSource>(observer =>
+            {
+                var sourceObserver = Observer.Create<TSource>(
+                    value =>
+                    {
+                        using (Py.GIL())
+                        {
+                            observer.OnNext(value); //locking around downstream effects
+                        }
+                    },
+                    observer.OnError,
+                    observer.OnCompleted);
+                return source.SubscribeSafe(observer);
+            });
+
+            return selector(gilProtectedSource);
+        }
+    }
+}

src/Bonsai.Scripting.Python/RuntimeManager.cs

@@ -28,10 +28,7 @@ internal RuntimeManager(string pythonHome, string scriptPath, IObserver<RuntimeM
             {
                 Initialize(pythonHome);
                 threadState = PythonEngine.BeginAllowThreads();
-                using (Py.GIL())
-                {
-                    MainModule = CreateModule(scriptPath: scriptPath);
-                }
+                MainModule = CreateModule(scriptPath: scriptPath);
                 observer.OnNext(this);
             });
         }
@@ -115,21 +112,14 @@ static void Initialize(string path)
         {
             if (!PythonEngine.IsInitialized)
             {
-                if (string.IsNullOrEmpty(path))
-                {
-                    path = Environment.GetEnvironmentVariable("VIRTUAL_ENV", EnvironmentVariableTarget.Process);
-                    if (string.IsNullOrEmpty(path)) path = Environment.CurrentDirectory;
-                }
-
-                path = Path.GetFullPath(path);
-                var pythonHome = EnvironmentHelper.GetPythonHome(path);
-                Runtime.PythonDLL = EnvironmentHelper.GetPythonDLL(pythonHome);
-                EnvironmentHelper.SetRuntimePath(pythonHome);
-                PythonEngine.PythonHome = pythonHome;
-                if (pythonHome != path)
+                path = EnvironmentHelper.GetEnvironmentPath(path);
+                var config = EnvironmentHelper.GetEnvironmentConfig(path);
+                Runtime.PythonDLL = EnvironmentHelper.GetPythonDLL(config);
+                EnvironmentHelper.SetRuntimePath(config.PythonHome);
+                PythonEngine.PythonHome = config.PythonHome;
+                if (config.PythonHome != path)
                 {
-                    var version = PythonEngine.Version;
-                    PythonEngine.PythonPath = EnvironmentHelper.GetPythonPath(pythonHome, path);
+                    PythonEngine.PythonPath = EnvironmentHelper.GetPythonPath(config);
                 }
                 PythonEngine.Initialize();
             }