From 424be5264520fbd67008003f8ded9665fcbe924b Mon Sep 17 00:00:00 2001
From: Timothy Warren <twarren@nabancard.com>
Date: Wed, 5 Apr 2023 10:13:39 -0400
Subject: [PATCH] Add validation layers for debugging

---
 src/app.rs  | 82 +++++++++++++++++++++++++++++++++++++++++++++++++----
 src/main.rs | 14 ++++-----
 2 files changed, 83 insertions(+), 13 deletions(-)

diff --git a/src/app.rs b/src/app.rs
index 4bf9be4..63d5d7a 100644
--- a/src/app.rs
+++ b/src/app.rs
@@ -1,10 +1,41 @@
 use anyhow::{anyhow, Result};
+use log::*;
+use std::collections::HashSet;
+use std::ffi::CStr;
+use std::os::raw::c_void;
 use vulkanalia::loader::{LibloadingLoader, LIBRARY};
 use vulkanalia::prelude::v1_0::*;
+use vulkanalia::vk::ExtDebugUtilsExtension;
 use vulkanalia::window as vk_window;
 use winit::window::Window;
 
-unsafe fn create_instance(window: &Window, entry: &Entry) -> Result<Instance> {
+const VALIDATION_ENABLED: bool = cfg!(debug_assertions);
+const VALIDATION_LAYER: vk::ExtensionName =
+    vk::ExtensionName::from_bytes(b"VK_LAYER_KHRONOS_validation");
+
+extern "system" fn debug_callback(
+    severity: vk::DebugUtilsMessageSeverityFlagsEXT,
+    type_: vk::DebugUtilsMessageTypeFlagsEXT,
+    data: *const vk::DebugUtilsMessengerCallbackDataEXT,
+    _: *mut c_void,
+) -> vk::Bool32 {
+    let data = unsafe { *data };
+    let message = unsafe { CStr::from_ptr(data.message) }.to_string_lossy();
+
+    if severity >= vk::DebugUtilsMessageSeverityFlagsEXT::ERROR {
+        error!("({:?}) {}", type_, message);
+    } else if severity >= vk::DebugUtilsMessageSeverityFlagsEXT::WARNING {
+        warn!("({:?}) {}", type_, message);
+    } else if severity >= vk::DebugUtilsMessageSeverityFlagsEXT::INFO {
+        debug!("({:?}) {}", type_, message);
+    } else {
+        trace!("({:?}) {}", type_, message);
+    }
+
+    vk::FALSE
+}
+
+unsafe fn create_instance(window: &Window, entry: &Entry, data: &mut AppData) -> Result<Instance> {
     let application_info = vk::ApplicationInfo::builder()
         .application_name(b"Vulkan Tutorial\0")
         .application_version(vk::make_version(1, 0, 0))
@@ -12,10 +43,29 @@ unsafe fn create_instance(window: &Window, entry: &Entry) -> Result<Instance> {
         .engine_version(vk::make_version(1, 0, 0))
         .api_version(vk::make_version(1, 0, 0));
 
+    let available_layers = entry
+        .enumerate_instance_layer_properties()?
+        .iter()
+        .map(|l| l.layer_name)
+        .collect::<HashSet<_>>();
+
+    if VALIDATION_ENABLED && !available_layers.contains(&VALIDATION_LAYER) {
+        return Err(anyhow!("Validation layer requested but not supported."));
+    }
+
+    let layers = if VALIDATION_ENABLED {
+        vec![VALIDATION_LAYER.as_ptr()]
+    } else {
+        Vec::new()
+    };
+
     let mut extensions = vk_window::get_required_instance_extensions(window)
         .iter()
         .map(|e| e.as_ptr())
         .collect::<Vec<_>>();
+    if VALIDATION_ENABLED {
+        extensions.push(vk::EXT_DEBUG_UTILS_EXTENSION.name.as_ptr());
+    }
 
     // Compatibility extension for macOS
     let flags = if entry
@@ -29,12 +79,24 @@ unsafe fn create_instance(window: &Window, entry: &Entry) -> Result<Instance> {
         vk::InstanceCreateFlags::empty()
     };
 
-    let info = vk::InstanceCreateInfo::builder()
+    let mut info = vk::InstanceCreateInfo::builder()
         .application_info(&application_info)
+        .enabled_layer_names(&layers)
         .enabled_extension_names(&extensions)
         .flags(flags);
 
-    Ok(entry.create_instance(&info, None)?)
+    let mut debug_info = vk::DebugUtilsMessengerCreateInfoEXT::builder()
+        .message_severity(vk::DebugUtilsMessageSeverityFlagsEXT::all())
+        .message_type(vk::DebugUtilsMessageTypeFlagsEXT::all())
+        .user_callback(Some(debug_callback));
+
+    if VALIDATION_ENABLED {
+        info = info.push_next(&mut debug_info);
+    }
+
+    let instance = entry.create_instance(&info, None)?;
+
+    Ok(instance)
 }
 
 /// Our Vulkan app.
@@ -42,6 +104,7 @@ unsafe fn create_instance(window: &Window, entry: &Entry) -> Result<Instance> {
 pub struct App {
     entry: Entry,
     instance: Instance,
+    data: AppData,
 }
 
 impl App {
@@ -49,9 +112,10 @@ impl App {
     pub unsafe fn create(window: &Window) -> Result<Self> {
         let loader = LibloadingLoader::new(LIBRARY)?;
         let entry = Entry::new(loader).map_err(|b| anyhow!("{}", b))?;
-        let instance = create_instance(window, &entry)?;
+        let mut data = AppData::default();
+        let instance = create_instance(window, &entry, &mut data)?;
 
-        Ok(Self { entry, instance })
+        Ok(Self { entry, instance, data })
     }
 
     /// Renders a frame for our Vulkan app.
@@ -61,10 +125,16 @@ impl App {
 
     /// Destroys our Vulkan app.
     pub unsafe fn destroy(&mut self) {
+        if VALIDATION_ENABLED {
+            self.instance.destroy_debug_utils_messenger_ext(self.data.messenger, None);
+        }
+
         self.instance.destroy_instance(None);
     }
 }
 
 /// The Vulkan handles and associated properties used by our Vulkan app.
 #[derive(Clone, Debug, Default)]
-pub struct AppData {}
+struct AppData {
+    messenger: vk::DebugUtilsMessengerEXT,
+}
diff --git a/src/main.rs b/src/main.rs
index 68b3184..9cd8711 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,12 +1,12 @@
-#![allow(
-    dead_code,
-    unused_variables,
-    clippy::too_many_arguments,
-    clippy::unnecessary_wraps
-)]
+// #![allow(
+//     dead_code,
+//     unused_variables,
+//     clippy::too_many_arguments,
+//     clippy::unnecessary_wraps
+// )]
 
 mod app;
-use app::{App, AppData};
+use app::App;
 
 use anyhow::Result;
 use winit::dpi::LogicalSize;