From 54d71cf51ba204f63bc3e8d4f3e7dd3893d8425a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Quang=20Ng=C3=B4?= <ngoquang2708@gmail.com>
Date: Wed, 25 Dec 2024 01:43:27 +0700
Subject: [PATCH] renderer_vulkan: Fix deadlock when resizing the SDL window

 Commit 400da1a add ability to handle recreation of the swapchain which
fix black screen under XWayland. Unfortunately, that commit doesn't
account for rapid swapchain recreation caused by resizing window.

 This commit aim to solve that issue by delay resetting the frame's
present done fence until after we know for sure we will be submitting
work with it.
---
 .../renderer_vulkan/vk_presenter.cpp           | 18 ++++++------------
 .../renderer_vulkan/vk_swapchain.cpp           |  4 +---
 src/video_core/renderer_vulkan/vk_swapchain.h  |  2 +-
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/src/video_core/renderer_vulkan/vk_presenter.cpp b/src/video_core/renderer_vulkan/vk_presenter.cpp
index bc55cde2309..32e7007ade7 100644
--- a/src/video_core/renderer_vulkan/vk_presenter.cpp
+++ b/src/video_core/renderer_vulkan/vk_presenter.cpp
@@ -628,15 +628,14 @@ Frame* Presenter::PrepareFrameInternal(VideoCore::ImageId image_id, bool is_eop)
 }
 
 void Presenter::Present(Frame* frame) {
-    // Recreate the swapchain if the window was resized.
-    if (window.GetWidth() != swapchain.GetExtent().width ||
-        window.GetHeight() != swapchain.GetExtent().height) {
+    // Recreate the swapchain if the window was resized or failed to acquire next image.
+    while (window.GetWidth() != swapchain.GetExtent().width ||
+           window.GetHeight() != swapchain.GetExtent().height || !swapchain.AcquireNextImage()) {
         swapchain.Recreate(window.GetWidth(), window.GetHeight());
     }
 
-    if (!swapchain.AcquireNextImage()) {
-        swapchain.Recreate(window.GetWidth(), window.GetHeight());
-    }
+    // Reset fence for queue submission.
+    instance.GetDevice().resetFences(frame->present_done);
 
     ImGui::Core::NewFrame();
 
@@ -733,9 +732,7 @@ void Presenter::Present(Frame* frame) {
 
     // Present to swapchain.
     std::scoped_lock submit_lock{Scheduler::submit_mutex};
-    if (!swapchain.Present()) {
-        swapchain.Recreate(window.GetWidth(), window.GetHeight());
-    }
+    swapchain.Present();
 
     // Free the frame for reuse
     std::scoped_lock fl{free_mutex};
@@ -776,9 +773,6 @@ Frame* Presenter::GetRenderFrame() {
         }
     }
 
-    // Reset fence for next queue submission.
-    device.resetFences(frame->present_done);
-
     // If the window dimensions changed, recreate this frame
     if (frame->width != window.GetWidth() || frame->height != window.GetHeight()) {
         RecreateFrame(frame, window.GetWidth(), window.GetHeight());
diff --git a/src/video_core/renderer_vulkan/vk_swapchain.cpp b/src/video_core/renderer_vulkan/vk_swapchain.cpp
index 380660a2f06..d0bc7ebdce6 100644
--- a/src/video_core/renderer_vulkan/vk_swapchain.cpp
+++ b/src/video_core/renderer_vulkan/vk_swapchain.cpp
@@ -112,7 +112,7 @@ bool Swapchain::AcquireNextImage() {
     return !needs_recreation;
 }
 
-bool Swapchain::Present() {
+void Swapchain::Present() {
 
     const vk::PresentInfoKHR present_info = {
         .waitSemaphoreCount = 1,
@@ -131,8 +131,6 @@ bool Swapchain::Present() {
     }
 
     frame_index = (frame_index + 1) % image_count;
-
-    return !needs_recreation;
 }
 
 void Swapchain::FindPresentFormat() {
diff --git a/src/video_core/renderer_vulkan/vk_swapchain.h b/src/video_core/renderer_vulkan/vk_swapchain.h
index 19ae9b2d2e0..a41b3ca76af 100644
--- a/src/video_core/renderer_vulkan/vk_swapchain.h
+++ b/src/video_core/renderer_vulkan/vk_swapchain.h
@@ -32,7 +32,7 @@ class Swapchain {
     bool AcquireNextImage();
 
     /// Presents the current image and move to the next one
-    bool Present();
+    void Present();
 
     vk::SurfaceKHR GetSurface() const {
         return surface;