CmdPal: Add hidden window as owner for tool windows (#42902)

## Summary of the Pull Request

This PR changes the method used to hide tool windows from the taskbar
and Alt+Tab to a more reliable approach.
Previously, this was achieved by adding `WS_EX_TOOLWINDOW` to an unowned
top-level window, which proved unreliable in several scenarios.

The new implementation assigns a hidden window as the owner of each tool
window.
This ensures that the window does not appear on the taskbar even when
the Windows setting
**Settings → System → Multitasking → On the taskbar, show all opened
windows** is set to **On all desktops**.

## Change log one-liner

Fixes Command Palette windows occasionally appearing on the taskbar
under certain system settings.

<!-- Please review the items on the PR checklist before submitting-->
## PR Checklist

- [x] Closes: #42395
- [x] **Communication:** I've discussed this with core contributors
already. If the work hasn't been agreed, this work might be rejected
- [x] **Tests:** Added/updated and all pass
- [x] **Localization:** All end-user-facing strings can be localized
- [x] **Dev docs:** Added/updated
- [x] **New binaries:** none
- [x] **Documentation updated:** no need

<!-- Provide a more detailed description of the PR, other things fixed,
or any additional comments/features here -->
## Detailed Description of the Pull Request / Additional comments

<!-- Describe how you validated the behavior. Add automated tests
wherever possible, but list manual validation steps taken as well -->
## Validation Steps Performed

Tested alongside the stable CmdPal on a system with

Author: jiripolasek

src/modules/cmdpal/Microsoft.CmdPal.UI/Helpers/WindowExtensions.cs

@@ -22,7 +22,7 @@ public static void SetIcon(this Window window)
         appWindow.SetIcon(@"Assets\icon.ico");
     }
 
-    private static HWND GetWindowHwnd(this Window window)
+    public static HWND GetWindowHwnd(this Window window)
     {
         return window is null
             ? throw new ArgumentNullException(nameof(window))

src/modules/cmdpal/Microsoft.CmdPal.UI/HiddenOwnerWindowBehavior.cs

@@ -0,0 +1,89 @@
+// Copyright (c) Microsoft Corporation
+// The Microsoft Corporation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.CmdPal.UI.Helpers;
+using Microsoft.UI.Xaml;
+using Windows.Win32;
+using Windows.Win32.Foundation;
+using Windows.Win32.Graphics.Dwm;
+using Windows.Win32.UI.WindowsAndMessaging;
+
+namespace Microsoft.CmdPal.UI;
+
+/// <summary>
+/// Provides behavior to control taskbar and Alt+Tab presence by assigning a hidden owner
+/// and toggling extended window styles for a target window.
+/// </summary>
+internal sealed class HiddenOwnerWindowBehavior
+{
+    private HWND _hiddenOwnerHwnd;
+    private Window? _hiddenWindow;
+
+    /// <summary>
+    /// Shows or hides a window in the taskbar (and Alt+Tab) by updating ownership and extended window styles.
+    /// </summary>
+    /// <param name="target">The <see cref="Microsoft.UI.Xaml.Window"/> to update.</param>
+    /// <param name="isVisibleInTaskbar"> True to show the window in the taskbar (and Alt+Tab); false to hide it from both. </param>
+    /// <remarks>
+    /// When hiding the window, a hidden owner is assigned and <see cref="WINDOW_EX_STYLE.WS_EX_TOOLWINDOW"/>
+    /// is enabled to keep it out of the taskbar and Alt+Tab. When showing, the owner is cleared and
+    /// <see cref="WINDOW_EX_STYLE.WS_EX_APPWINDOW"/> is enabled to ensure taskbar presence. Since tool
+    /// windows use smaller corner radii, the normal rounded corners are enforced via
+    /// <see cref="DWM_WINDOW_CORNER_PREFERENCE.DWMWCP_ROUND"/>.
+    /// </remarks>
+    /// <seealso href="https://learn.microsoft.com/en-us/windows/win32/shell/taskbar#managing-taskbar-buttons" />
+    public void ShowInTaskbar(Window target, bool isVisibleInTaskbar)
+    {
+        /*
+         * There are the three main ways to control whether a window appears on the taskbar:
+         * https://learn.microsoft.com/en-us/windows/win32/shell/taskbar#managing-taskbar-buttons
+         *
+         * 1. Set the window's owner. Owned windows do not appear on the taskbar:
+         *    Turns out this is the most reliable way to hide a window from the taskbar and ALT+TAB. WinForms and WPF uses this method
+         *    to back their ShowInTaskbar property as well.
+         *
+         * 2. Use the WS_EX_TOOLWINDOW extended window style:
+         *    This mostly works, with some reports that it silently fails in some cases. The biggest issue
+         *    is that for certain Windows settings (like Multitasking -> Show taskbar buttons on all displays = On all desktops),
+         *    the taskbar button is always shown even for tool windows.
+         *
+         * 3. Using ITaskbarList:
+         *    This is what AppWindow.IsShownInSwitchers uses, but it's COM-based and more complex, and can
+         *    fail if Explorer isn't running or responding. It could be a good backup, if needed.
+         */
+
+        var visibleHwnd = target.GetWindowHwnd();
+
+        if (isVisibleInTaskbar)
+        {
+            // remove any owner window
+            PInvoke.SetWindowLongPtr(visibleHwnd, WINDOW_LONG_PTR_INDEX.GWLP_HWNDPARENT, HWND.Null);
+        }
+        else
+        {
+            // Set the hidden window as the owner of the target window
+            var hiddenHwnd = EnsureHiddenOwner();
+            PInvoke.SetWindowLongPtr(visibleHwnd, WINDOW_LONG_PTR_INDEX.GWLP_HWNDPARENT, hiddenHwnd);
+        }
+
+        // Tool windows don't show up in ALT+TAB, and don't show up in the taskbar
+        // Tool window and app window styles are mutually exclusive, change both just to be safe
+        target.ToggleExtendedWindowStyle(WINDOW_EX_STYLE.WS_EX_TOOLWINDOW, !isVisibleInTaskbar);
+        target.ToggleExtendedWindowStyle(WINDOW_EX_STYLE.WS_EX_APPWINDOW, isVisibleInTaskbar);
+
+        // Since tool windows have smaller corner radii, we need to force the normal ones
+        target.SetCornerPreference(DWM_WINDOW_CORNER_PREFERENCE.DWMWCP_ROUND);
+    }
+
+    private HWND EnsureHiddenOwner()
+    {
+        if (_hiddenOwnerHwnd.IsNull)
+        {
+            _hiddenWindow = new Window();
+            _hiddenOwnerHwnd = _hiddenWindow.GetWindowHwnd();
+        }
+
+        return _hiddenOwnerHwnd;
+    }
+}

src/modules/cmdpal/Microsoft.CmdPal.UI/MainWindow.xaml.cs

@@ -57,6 +57,7 @@ public sealed partial class MainWindow : WindowEx,
     private readonly List<TopLevelHotkey> _hotkeys = [];
     private readonly KeyboardListener _keyboardListener;
     private readonly LocalKeyboardListener _localKeyboardListener;
+    private readonly HiddenOwnerWindowBehavior _hiddenOwnerBehavior = new();
     private bool _ignoreHotKeyWhenFullScreen = true;
 
     private DesktopAcrylicController? _acrylicController;
@@ -65,6 +66,7 @@ public sealed partial class MainWindow : WindowEx,
     public MainWindow()
     {
         InitializeComponent();
+        HideWindow();
 
         _hwnd = new HWND(WinRT.Interop.WindowNative.GetWindowHandle(this).ToInt32());
 
@@ -73,6 +75,8 @@ public MainWindow()
             CommandPaletteHost.SetHostHwnd((ulong)_hwnd.Value);
         }
 
+        _hiddenOwnerBehavior.ShowInTaskbar(this, Debugger.IsAttached);
+
         _keyboardListener = new KeyboardListener();
         _keyboardListener.Start();
 
@@ -126,16 +130,6 @@ public MainWindow()
 
         // Force window to be created, and then cloaked. This will offset initial animation when the window is shown.
         HideWindow();
-
-        ApplyWindowStyle();
-    }
-
-    private void ApplyWindowStyle()
-    {
-        // Tool windows don't show up in ALT+TAB, and don't show up in the taskbar
-        // Since tool windows have smaller corner radii, we need to force the normal ones
-        this.ToggleExtendedWindowStyle(WINDOW_EX_STYLE.WS_EX_TOOLWINDOW, !Debugger.IsAttached);
-        this.SetCornerPreference(DWM_WINDOW_CORNER_PREFERENCE.DWMWCP_ROUND);
     }
 
     private static void LocalKeyboardListener_OnKeyPressed(object? sender, LocalKeyboardListenerKeyPressedEventArgs e)
@@ -264,7 +258,7 @@ private void ShowHwnd(IntPtr hwndValue, MonitorBehavior target)
         // because that would make it hard to debug the app
         if (Debugger.IsAttached)
         {
-            ApplyWindowStyle();
+            _hiddenOwnerBehavior.ShowInTaskbar(this, true);
         }
 
         // Just to be sure, SHOW our hwnd.

src/modules/cmdpal/Microsoft.CmdPal.UI/NativeMethods.txt

@@ -58,4 +58,9 @@ GetModuleHandle
 
 GetWindowLong
 SetWindowLong
-WINDOW_EX_STYLE
+WINDOW_EX_STYLE
+CreateWindowEx
+WNDCLASSEXW
+RegisterClassEx
+GetStockObject
+GetModuleHandle

src/modules/cmdpal/Microsoft.CmdPal.UI/ToastWindow.xaml.cs

@@ -25,11 +25,11 @@ public sealed partial class ToastWindow : WindowEx,
     IRecipient<QuitMessage>
 {
     private readonly HWND _hwnd;
+    private readonly DispatcherQueueTimer _debounceTimer = DispatcherQueue.GetForCurrentThread().CreateTimer();
+    private readonly HiddenOwnerWindowBehavior _hiddenOwnerWindowBehavior = new();
 
     public ToastViewModel ViewModel { get; } = new();
 
-    private readonly DispatcherQueueTimer _debounceTimer = DispatcherQueue.GetForCurrentThread().CreateTimer();
-
     public ToastWindow()
     {
         this.InitializeComponent();
@@ -39,12 +39,7 @@ public ToastWindow()
         this.SetIcon();
         AppWindow.Title = RS_.GetString("ToastWindowTitle");
         AppWindow.TitleBar.PreferredHeightOption = TitleBarHeightOption.Collapsed;
-
-        // Tool windows don't show up in ALT+TAB, and don't show up in the taskbar
-        // Since tool windows have smaller corner radii, we need to force the normal ones
-        // to visually match system toasts.
-        this.ToggleExtendedWindowStyle(WINDOW_EX_STYLE.WS_EX_TOOLWINDOW, true);
-        this.SetCornerPreference(DWM_WINDOW_CORNER_PREFERENCE.DWMWCP_ROUND);
+        _hiddenOwnerWindowBehavior.ShowInTaskbar(this, false);
 
         _hwnd = new HWND(WinRT.Interop.WindowNative.GetWindowHandle(this).ToInt32());
         PInvoke.EnableWindow(_hwnd, false);