Resolve WGC technical limitation (#303)

Avasam · web-flow · commit ef52113073fe · 2024-10-20T14:39:20.000-04:00
diff --git a/docs/tutorial.md b/docs/tutorial.md
@@ -43,7 +43,6 @@
 
 - **Windows Graphics Capture** (fast, most compatible, capped at 60fps)  
     Only available in Windows 10.0.17134 and up.  
-    Due to current technical limitations, Windows versions below 10.0.0.17763 require having at least one audio or video Capture Device connected and enabled.
     Allows recording UWP apps, Hardware Accelerated and Exclusive Fullscreen windows.  
     Adds a yellow border on Windows 10 (not on Windows 11).
     Caps at around 60 FPS.  
diff --git a/ruff.toml b/ruff.toml
@@ -146,6 +146,10 @@ max-branches = 15
   # Issues with using a star-imported name will be caught by type-checkers.
   "F405", # may be undefined, or defined from star imports
 ]
+"src/d3d11.py" = [
+  # Following windows API/ctypes like naming conventions
+  "N801", # invalid-class-name
+]
 
 [lint.flake8-tidy-imports.banned-api]
 "cv2.imread".msg = """\
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
@@ -17,26 +17,28 @@ PySide6-Essentials>=6.6.0 # Python 3.12 support
 scipy>=1.11.2 # Python 3.12 support
 tomli-w>=1.1.0  # Typing fixes
 typing-extensions>=4.4.0  # @override decorator support
+
 #
 # Build and compile resources
 pyinstaller>=5.13  # Python 3.12 support
+
 #
 # https://peps.python.org/pep-0508/#environment-markers
 #
 # Windows-only dependencies:
 comtypes<1.4.5 ; sys_platform == 'win32'  # https://github.com/pyinstaller/pyinstaller-hooks-contrib/issues/807
 pygrabber>=0.2 ; sys_platform == 'win32'  # Completed types
 pywin32>=301 ; sys_platform == 'win32'
-winrt-Windows.AI.MachineLearning>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
+typed-D3DShot[numpy]>=1.0.1 ; sys_platform == 'win32'
 winrt-Windows.Foundation>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-winrt-Windows.Graphics.Capture>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-winrt-Windows.Graphics.Capture.Interop>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-winrt-Windows.Graphics.DirectX>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-winrt-Windows.Graphics.DirectX.Direct3D11>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-winrt-Windows.Graphics.Imaging>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
+winrt-Windows.Graphics.Capture>=2.3.0 ; sys_platform == 'win32'  # Python 3.13 support
+winrt-Windows.Graphics.Capture.Interop>=2.3.0 ; sys_platform == 'win32'  # Python 3.13 support
+winrt-Windows.Graphics.DirectX>=2.3.0 ; sys_platform == 'win32'  # Python 3.13 support
+winrt-Windows.Graphics.DirectX.Direct3D11>=2.3.0 ; sys_platform == 'win32'  # Python 3.13 support
+winrt-Windows.Graphics.DirectX.Direct3D11.Interop>=2.3.0 ; sys_platform == 'win32'
 winrt-Windows.Graphics>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-winrt-Windows.Media.Capture>=2.2.0 ; sys_platform == 'win32'  # Python 3.13 support
-typed-D3DShot[numpy]>=1.0.1 ; sys_platform == 'win32'
+winrt-Windows.Graphics.Imaging>=2.3.0 ; sys_platform == 'win32'  # Python 3.13 support
+
 #
 # Linux-only dependencies
 PyScreeze ; sys_platform == 'linux'
diff --git a/src/capture_method/WindowsGraphicsCaptureMethod.py b/src/capture_method/WindowsGraphicsCaptureMethod.py
@@ -14,23 +14,19 @@
 from winrt.windows.graphics.capture.interop import create_for_window
 from winrt.windows.graphics.directx import DirectXPixelFormat
 from winrt.windows.graphics.directx.direct3d11 import IDirect3DSurface
+from winrt.windows.graphics.directx.direct3d11.interop import (
+    create_direct3d11_device_from_dxgi_device,
+)
 from winrt.windows.graphics.imaging import BitmapBufferAccessMode, SoftwareBitmap
 
 from capture_method.CaptureMethodBase import CaptureMethodBase
-from utils import (
-    BGRA_CHANNEL_COUNT,
-    WGC_MIN_BUILD,
-    WINDOWS_BUILD_NUMBER,
-    get_direct3d_device,
-    is_valid_hwnd,
-)
+from d3d11 import D3D11_CREATE_DEVICE_FLAG, D3D_DRIVER_TYPE, D3D11CreateDevice
+from utils import BGRA_CHANNEL_COUNT, WGC_MIN_BUILD, WINDOWS_BUILD_NUMBER, is_valid_hwnd
 
 if TYPE_CHECKING:
     from AutoSplit import AutoSplit
 
 WGC_NO_BORDER_MIN_BUILD = 20348
-LEARNING_MODE_DEVICE_BUILD = 17763
-"""https://learn.microsoft.com/en-us/uwp/api/windows.ai.machinelearning.learningmodeldevice"""
 
 
 async def convert_d3d_surface_to_software_bitmap(surface: IDirect3DSurface | None):
@@ -42,13 +38,11 @@ class WindowsGraphicsCaptureMethod(CaptureMethodBase):
     short_description = "fast, most compatible, capped at 60fps"
     description = f"""
 Only available in Windows 10.0.{WGC_MIN_BUILD} and up.
-Due to current technical limitations, Windows versions below 10.0.0.{LEARNING_MODE_DEVICE_BUILD}
-require having at least one audio or video Capture Device connected and enabled.
 Allows recording UWP apps, Hardware Accelerated and Exclusive Fullscreen windows.
 Adds a yellow border on Windows 10 (not on Windows 11).
 Caps at around 60 FPS."""
 
-    size: SizeInt32
+    size: "SizeInt32"
     frame_pool: Direct3D11CaptureFramePool | None = None
     session: GraphicsCaptureSession | None = None
     """This is stored to prevent session from being garbage collected"""
@@ -59,11 +53,16 @@ def __init__(self, autosplit: "AutoSplit"):
         if not is_valid_hwnd(autosplit.hwnd):
             return
 
+        dxgi, *_ = D3D11CreateDevice(
+            DriverType=D3D_DRIVER_TYPE.HARDWARE,
+            Flags=D3D11_CREATE_DEVICE_FLAG.BGRA_SUPPORT,
+        )
+        direct3d_device = create_direct3d11_device_from_dxgi_device(dxgi.value)
         item = create_for_window(autosplit.hwnd)
         frame_pool = Direct3D11CaptureFramePool.create_free_threaded(
-            get_direct3d_device(),
+            direct3d_device,
             DirectXPixelFormat.B8_G8_R8_A8_UINT_NORMALIZED,
-            1,
+            1,  # number_of_buffers
             item.size,
         )
         if not frame_pool:
@@ -114,6 +113,8 @@ def get_frame(self) -> MatLike | None:
             return None
 
         # We were too fast and the next frame wasn't ready yet
+        # TODO: Consider "add_frame_arrive" instead !
+        # https://github.com/pywinrt/pywinrt/blob/5bf1ac5ff4a77cf343e11d7c841c368fa9235d81/samples/screen_capture/__main__.py#L67-L78
         if not frame:
             return self.last_converted_frame
 
diff --git a/src/capture_method/__init__.py b/src/capture_method/__init__.py
@@ -10,13 +10,7 @@
 
 from capture_method.CaptureMethodBase import CaptureMethodBase
 from capture_method.VideoCaptureDeviceCaptureMethod import VideoCaptureDeviceCaptureMethod
-from utils import (
-    WGC_MIN_BUILD,
-    WINDOWS_BUILD_NUMBER,
-    first,
-    get_input_device_resolution,
-    try_get_direct3d_device,
-)
+from utils import WGC_MIN_BUILD, WINDOWS_BUILD_NUMBER, first, get_input_device_resolution
 
 if sys.platform == "win32":
     from _ctypes import COMError  # noqa: PLC2701 # comtypes is untyped
@@ -125,12 +119,8 @@ def get(self, key: CaptureMethodEnum, default: object = None, /):
 
 CAPTURE_METHODS = CaptureMethodDict()
 if sys.platform == "win32":
-    if (  # Windows Graphics Capture requires a minimum Windows Build
-        WINDOWS_BUILD_NUMBER >= WGC_MIN_BUILD
-        # Our current implementation of Windows Graphics Capture
-        # does not ensure we can get an ID3DDevice
-        and try_get_direct3d_device()
-    ):
+    # Windows Graphics Capture requires a minimum Windows Build
+    if WINDOWS_BUILD_NUMBER >= WGC_MIN_BUILD:
         CAPTURE_METHODS[CaptureMethodEnum.WINDOWS_GRAPHICS_CAPTURE] = WindowsGraphicsCaptureMethod
     CAPTURE_METHODS[CaptureMethodEnum.BITBLT] = BitBltCaptureMethod
     try:  # Test for laptop cross-GPU Desktop Duplication issue
diff --git a/src/d3d11.py b/src/d3d11.py
@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2024 David Lechner <david@pybricks.com>
+import sys
+
+if sys.platform != "win32":
+    raise OSError
+
+import ctypes
+import enum
+import uuid
+from ctypes import wintypes
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ctypes import _FuncPointer  # pyright: ignore[reportPrivateUsage]
+
+
+###
+# https://github.com/pywinrt/pywinrt/blob/main/samples/screen_capture/iunknown.py
+###
+
+
+class GUID(ctypes.Structure):
+    _fields_ = (
+        ("Data1", ctypes.c_ulong),
+        ("Data2", ctypes.c_ushort),
+        ("Data3", ctypes.c_ushort),
+        ("Data4", ctypes.c_ubyte * 8),
+    )
+
+
+class IUnknown(ctypes.c_void_p):
+    QueryInterface = ctypes.WINFUNCTYPE(
+        # _CData is incompatible with int
+        int,  # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
+        ctypes.POINTER(GUID),
+        ctypes.POINTER(wintypes.LPVOID),
+    )(0, "QueryInterface")
+    AddRef = ctypes.WINFUNCTYPE(wintypes.ULONG)(1, "AddRef")
+    Release = ctypes.WINFUNCTYPE(wintypes.ULONG)(2, "Release")
+
+    def query_interface(self, iid: uuid.UUID | str) -> "IUnknown":
+        if isinstance(iid, str):
+            iid = uuid.UUID(iid)
+
+        ppv = wintypes.LPVOID()
+        _iid = GUID.from_buffer_copy(iid.bytes_le)
+        ret = self.QueryInterface(self, ctypes.byref(_iid), ctypes.byref(ppv))
+
+        if ret:
+            raise ctypes.WinError(ret)
+
+        return IUnknown(ppv.value)
+
+    def __del__(self):
+        IUnknown.Release(self)
+
+
+###
+# https://github.com/pywinrt/pywinrt/blob/main/samples/screen_capture/d3d11.py
+###
+
+
+__all__ = [
+    "D3D11_CREATE_DEVICE_FLAG",
+    "D3D_DRIVER_TYPE",
+    "D3D_FEATURE_LEVEL",
+    "D3D11CreateDevice",
+]
+
+IN = 1
+OUT = 2
+
+# https://learn.microsoft.com/en-us/windows/win32/api/d3dcommon/ne-d3dcommon-d3d_driver_type
+#
+# typedef enum D3D_DRIVER_TYPE {
+#   D3D_DRIVER_TYPE_UNKNOWN = 0,
+#   D3D_DRIVER_TYPE_HARDWARE,
+#   D3D_DRIVER_TYPE_REFERENCE,
+#   D3D_DRIVER_TYPE_NULL,
+#   D3D_DRIVER_TYPE_SOFTWARE,
+#   D3D_DRIVER_TYPE_WARP
+# } ;
+
+
+class D3D_DRIVER_TYPE(enum.IntEnum):
+    UNKNOWN = 0
+    HARDWARE = 1
+    REFERENCE = 2
+    NULL = 3
+    SOFTWARE = 4
+    WARP = 5
+
+
+# https://learn.microsoft.com/en-us/windows/win32/api/d3d11/ne-d3d11-d3d11_create_device_flag
+#
+# typedef enum D3D11_CREATE_DEVICE_FLAG {
+#   D3D11_CREATE_DEVICE_SINGLETHREADED = 0x1,
+#   D3D11_CREATE_DEVICE_DEBUG = 0x2,
+#   D3D11_CREATE_DEVICE_SWITCH_TO_REF = 0x4,
+#   D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS = 0x8,
+#   D3D11_CREATE_DEVICE_BGRA_SUPPORT = 0x20,
+#   D3D11_CREATE_DEVICE_DEBUGGABLE = 0x40,
+#   D3D11_CREATE_DEVICE_PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY = 0x80,
+#   D3D11_CREATE_DEVICE_DISABLE_GPU_TIMEOUT = 0x100,
+#   D3D11_CREATE_DEVICE_VIDEO_SUPPORT = 0x800
+# } ;
+
+
+class D3D11_CREATE_DEVICE_FLAG(enum.IntFlag):
+    SINGLETHREADED = 0x1
+    DEBUG = 0x2
+    SWITCH_TO_REF = 0x4
+    PREVENT_INTERNAL_THREADING_OPTIMIZATIONS = 0x8
+    BGRA_SUPPORT = 0x20
+    DEBUGGABLE = 0x40
+    PREVENT_ALTERING_LAYER_SETTINGS_FROM_REGISTRY = 0x80
+    DISABLE_GPU_TIMEOUT = 0x100
+    VIDEO_SUPPORT = 0x800
+
+
+# https://learn.microsoft.com/en-us/windows/win32/api/d3dcommon/ne-d3dcommon-d3d_feature_level
+#
+# typedef enum D3D_FEATURE_LEVEL {
+#   D3D_FEATURE_LEVEL_1_0_GENERIC,
+#   D3D_FEATURE_LEVEL_1_0_CORE,
+#   D3D_FEATURE_LEVEL_9_1,
+#   D3D_FEATURE_LEVEL_9_2,
+#   D3D_FEATURE_LEVEL_9_3,
+#   D3D_FEATURE_LEVEL_10_0,
+#   D3D_FEATURE_LEVEL_10_1,
+#   D3D_FEATURE_LEVEL_11_0,
+#   D3D_FEATURE_LEVEL_11_1,
+#   D3D_FEATURE_LEVEL_12_0,
+#   D3D_FEATURE_LEVEL_12_1,
+#   D3D_FEATURE_LEVEL_12_2
+# } ;
+
+
+class D3D_FEATURE_LEVEL(enum.IntEnum):
+    LEVEL_1_0_GENERIC = 0x1000
+    LEVEL_1_0_CORE = 0x1001
+    LEVEL_9_1 = 0x9100
+    LEVEL_9_2 = 0x9200
+    LEVEL_9_3 = 0x9300
+    LEVEL_10_0 = 0xA000
+    LEVEL_10_1 = 0xA100
+    LEVEL_11_0 = 0xB000
+    LEVEL_11_1 = 0xB100
+    LEVEL_12_0 = 0xC000
+    LEVEL_12_1 = 0xC100
+    LEVEL_12_2 = 0xC200
+
+
+# not sure where this is officially defined or if the value would ever change
+
+D3D11_SDK_VERSION = 7
+
+# https://learn.microsoft.com/en-us/windows/win32/api/d3d11/nf-d3d11-d3d11createdevice
+#
+# HRESULT D3D11CreateDevice(
+#   [in, optional]  IDXGIAdapter            *pAdapter,
+#                   D3D_DRIVER_TYPE         DriverType,
+#                   HMODULE                 Software,
+#                   UINT                    Flags,
+#   [in, optional]  const D3D_FEATURE_LEVEL *pFeatureLevels,
+#                   UINT                    FeatureLevels,
+#                   UINT                    SDKVersion,
+#   [out, optional] ID3D11Device            **ppDevice,
+#   [out, optional] D3D_FEATURE_LEVEL       *pFeatureLevel,
+#   [out, optional] ID3D11DeviceContext     **ppImmediateContext
+# );
+
+
+def errcheck(
+    result: int,
+    _func: "_FuncPointer",  # Actually WinFunctionType but that's an internal class
+    args: tuple[
+        IUnknown | None,  # IDXGIAdapter
+        D3D_DRIVER_TYPE,
+        wintypes.HMODULE | None,
+        D3D11_CREATE_DEVICE_FLAG,
+        D3D_FEATURE_LEVEL | None,
+        int,
+        int,
+        IUnknown,  # ID3D11Device
+        wintypes.UINT,
+        IUnknown,  # ID3D11DeviceContext
+    ],
+):
+    if result:
+        raise ctypes.WinError(result)
+
+    return (args[7], D3D_FEATURE_LEVEL(args[8].value), args[9])
+
+
+D3D11CreateDevice = ctypes.WINFUNCTYPE(
+    # _CData is incompatible with int
+    int,  # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
+    wintypes.LPVOID,
+    wintypes.UINT,
+    wintypes.LPVOID,
+    wintypes.UINT,
+    ctypes.POINTER(wintypes.UINT),
+    wintypes.UINT,
+    wintypes.UINT,
+    ctypes.POINTER(IUnknown),
+    ctypes.POINTER(wintypes.UINT),
+    ctypes.POINTER(IUnknown),
+)(
+    ("D3D11CreateDevice", ctypes.windll.d3d11),
+    (
+        (IN, "pAdapter", None),
+        (IN, "DriverType", D3D_DRIVER_TYPE.UNKNOWN),
+        (IN, "Software", None),
+        (IN, "Flags", 0),
+        (IN, "pFeatureLevels", None),
+        (IN, "FeatureLevels", 0),
+        (IN, "SDKVersion", D3D11_SDK_VERSION),
+        (OUT, "ppDevice"),
+        (OUT, "pFeatureLevel"),
+        (OUT, "ppImmediateContext"),
+    ),
+)
+# _CData is incompatible with int
+D3D11CreateDevice.errcheck = errcheck  # type: ignore[assignment] # pyright: ignore[reportAttributeAccessIssue]
diff --git a/src/region_selection.py b/src/region_selection.py
diff --git a/src/utils.py b/src/utils.py
