GPUDevice: Add recovery from lost device

src/core/gpu.cpp

@@ -1993,7 +1993,7 @@ void GPU::SetDisplayTexture(GPUTexture* texture, GPUTexture* depth_buffer, s32 v
   m_display_texture_view_height = view_height;
 }
 
-bool GPU::PresentDisplay()
+GPUDevice::PresentResult GPU::PresentDisplay()
 {
   FlushRender();
 
@@ -2004,7 +2004,8 @@ bool GPU::PresentDisplay()
   return RenderDisplay(nullptr, display_rect, draw_rect, !g_settings.debugging.show_vram);
 }
 
-bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const GSVector4i draw_rect, bool postfx)
+GPUDevice::PresentResult GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect,
+                                            const GSVector4i draw_rect, bool postfx)
 {
   GL_SCOPE_FMT("RenderDisplay: {}", draw_rect);
 
@@ -2027,10 +2028,15 @@ bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const
 
     // Now we can apply the post chain.
     GPUTexture* post_output_texture = PostProcessing::InternalChain.GetOutputTexture();
-    if (PostProcessing::InternalChain.Apply(display_texture, m_display_depth_buffer, post_output_texture,
-                                            GSVector4i(0, 0, display_texture_view_width, display_texture_view_height),
-                                            display_texture_view_width, display_texture_view_height,
-                                            m_crtc_state.display_width, m_crtc_state.display_height))
+    if (const GPUDevice::PresentResult pres = PostProcessing::InternalChain.Apply(
+          display_texture, m_display_depth_buffer, post_output_texture,
+          GSVector4i(0, 0, display_texture_view_width, display_texture_view_height), display_texture_view_width,
+          display_texture_view_height, m_crtc_state.display_width, m_crtc_state.display_height);
+        pres != GPUDevice::PresentResult::OK)
+    {
+      return pres;
+    }
+    else
     {
       display_texture_view_x = 0;
       display_texture_view_y = 0;
@@ -2057,8 +2063,9 @@ bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const
   {
     if (target)
       g_gpu_device->SetRenderTarget(target);
-    else if (!g_gpu_device->BeginPresent(false))
-      return false;
+    else if (const GPUDevice::PresentResult pres = g_gpu_device->BeginPresent(false);
+             pres != GPUDevice::PresentResult::OK)
+      return pres;
   }
 
   if (display_texture)
@@ -2167,7 +2174,9 @@ bool GPU::RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const
                                               m_crtc_state.display_height);
   }
   else
-    return true;
+  {
+    return GPUDevice::PresentResult::OK;
+  }
 }
 
 bool GPU::SendDisplayToMediaCapture(MediaCapture* cap)
@@ -2186,7 +2195,7 @@ bool GPU::SendDisplayToMediaCapture(MediaCapture* cap)
   // Not cleared by RenderDisplay().
   g_gpu_device->ClearRenderTarget(target, GPUDevice::DEFAULT_CLEAR_COLOR);
 
-  if (!RenderDisplay(target, display_rect, draw_rect, postfx)) [[unlikely]]
+  if (RenderDisplay(target, display_rect, draw_rect, postfx) != GPUDevice::PresentResult::OK) [[unlikely]]
     return false;
 
   return cap->DeliverVideoFrame(target);

src/core/gpu.h

@@ -8,6 +8,7 @@
 #include "timing_event.h"
 #include "types.h"
 
+#include "util/gpu_device.h"
 #include "util/gpu_texture.h"
 
 #include "common/bitfield.h"
@@ -233,7 +234,7 @@ public:
                               bool show_osd_message);
 
   /// Draws the current display texture, with any post-processing.
-  bool PresentDisplay();
+  GPUDevice::PresentResult PresentDisplay();
 
   /// Sends the current frame to media capture.
   bool SendDisplayToMediaCapture(MediaCapture* cap);
@@ -630,7 +631,8 @@ protected:
   void SetDisplayTexture(GPUTexture* texture, GPUTexture* depth_texture, s32 view_x, s32 view_y, s32 view_width,
                          s32 view_height);
 
-  bool RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const GSVector4i draw_rect, bool postfx);
+  GPUDevice::PresentResult RenderDisplay(GPUTexture* target, const GSVector4i display_rect, const GSVector4i draw_rect,
+                                         bool postfx);
 
   bool Deinterlace(u32 field, u32 line_skip);
   bool DeinterlaceExtractField(u32 dst_bufidx, GPUTexture* src, u32 x, u32 y, u32 width, u32 height, u32 line_skip);

src/core/imgui_overlays.cpp

@@ -160,7 +160,7 @@ void Host::DisplayLoadingScreen(const char* message, int progress_min /*= -1*/,
 
   // TODO: Glass effect or something.
 
-  if (g_gpu_device->BeginPresent(false))
+  if (g_gpu_device->BeginPresent(false) == GPUDevice::PresentResult::OK)
   {
     g_gpu_device->RenderImGui();
     g_gpu_device->EndPresent(false);

src/core/system.cpp

@@ -162,6 +162,7 @@ static void DestroySystem();
 
 static bool CreateGPU(GPURenderer renderer, bool is_switching, Error* error);
 static bool RecreateGPU(GPURenderer renderer, bool force_recreate_device = false, bool update_display = true);
+static void HandleHostGPUDeviceLost();
 
 /// Updates the throttle period, call when target emulation speed changes.
 static void UpdateThrottlePeriod();
@@ -1202,6 +1203,45 @@ bool System::RecreateGPU(GPURenderer renderer, bool force_recreate_device, bool
   return true;
 }
 
+void System::HandleHostGPUDeviceLost()
+{
+  static Common::Timer::Value s_last_gpu_reset_time = 0;
+  static constexpr float MIN_TIME_BETWEEN_RESETS = 15.0f;
+
+  // If we're constantly crashing on something in particular, we don't want to end up in an
+  // endless reset loop.. that'd probably end up leaking memory and/or crashing us for other
+  // reasons. So just abort in such case.
+  const Common::Timer::Value current_time = Common::Timer::GetCurrentValue();
+  if (s_last_gpu_reset_time != 0 &&
+      Common::Timer::ConvertValueToSeconds(current_time - s_last_gpu_reset_time) < MIN_TIME_BETWEEN_RESETS)
+  {
+    Panic("Host GPU lost too many times, device is probably completely wedged.");
+  }
+  s_last_gpu_reset_time = current_time;
+
+  // Little bit janky, but because the device is lost, the VRAM readback is going to give us garbage.
+  // So back up what we have, it's probably missing bits, but whatever...
+  DynamicHeapArray<u8> vram_backup(VRAM_SIZE);
+  std::memcpy(vram_backup.data(), g_vram, VRAM_SIZE);
+
+  // Device lost, something went really bad.
+  // Let's just toss out everything, and try to hobble on.
+  if (!RecreateGPU(g_gpu->IsHardwareRenderer() ? g_settings.gpu_renderer : GPURenderer::Software, true, false))
+  {
+    Panic("Failed to recreate GS device after loss.");
+    return;
+  }
+
+  // Restore backed-up VRAM.
+  std::memcpy(g_vram, vram_backup.data(), VRAM_SIZE);
+
+  // First frame after reopening is definitely going to be trash, so skip it.
+  Host::AddIconOSDMessage(
+    "HostGPUDeviceLost", ICON_EMOJI_WARNING,
+    TRANSLATE_STR("System", "Host GPU device encountered an error and has recovered. This may cause broken rendering."),
+    Host::OSD_CRITICAL_ERROR_DURATION);
+}
+
 void System::LoadSettings(bool display_osd_messages)
 {
   std::unique_lock<std::mutex> lock = Host::GetSettingsLock();
@@ -5710,13 +5750,13 @@ bool System::PresentDisplay(bool skip_present, bool explicit_present)
   ImGuiManager::RenderOverlayWindows();
   ImGuiManager::RenderDebugWindows();
 
-  bool do_present;
+  GPUDevice::PresentResult pres;
   if (g_gpu && !skip_present)
-    do_present = g_gpu->PresentDisplay();
+    pres = g_gpu->PresentDisplay();
   else
-    do_present = g_gpu_device->BeginPresent(skip_present);
+    pres = g_gpu_device->BeginPresent(skip_present);
 
-  if (do_present)
+  if (pres == GPUDevice::PresentResult::OK)
   {
     g_gpu_device->RenderImGui();
     g_gpu_device->EndPresent(explicit_present);
@@ -5729,13 +5769,16 @@ bool System::PresentDisplay(bool skip_present, bool explicit_present)
   }
   else
   {
+    if (pres == GPUDevice::PresentResult::DeviceLost) [[unlikely]]
+      HandleHostGPUDeviceLost();
+
     // Still need to kick ImGui or it gets cranky.
     ImGui::Render();
   }
 
   ImGuiManager::NewFrame();
 
-  return do_present;
+  return (pres == GPUDevice::PresentResult::OK);
 }
 
 void System::InvalidateDisplay()