WIP validation layers

TODO: "Debugging instance creation and destruction" on
https://vulkan-tutorial.com/Drawing_a_triangle/Setup/Validation_layers

src/main.rs

@@ -26,11 +26,95 @@ const ENABLE_VALIDATION_LAYERS: bool = true;
 const ENABLE_VALIDATION_LAYERS: bool = false;
 
 const VALIDATION_LAYER_STR_0: &str = "VK_LAYER_KHRONOS_validation\x00";
-const VALIDATION_LAYERS: [&str; 1] = [VALIDATION_LAYER_STR_0];
+const VALIDATION_LAYERS: [*const u8; 1] = [VALIDATION_LAYER_STR_0.as_ptr()];
+
+fn check_validation_layer_support() -> bool {
+    let mut layer_count: u32 = 0;
+    unsafe {
+        ffi::vkEnumerateInstanceLayerProperties(
+            std::ptr::addr_of_mut!(layer_count),
+            std::ptr::null_mut(),
+        );
+    }
+
+    let mut layers: Vec<ffi::VkLayerProperties> = Vec::with_capacity(layer_count as usize);
+    layers.resize(
+        layer_count as usize,
+        ffi::VkLayerProperties {
+            layerName: [0; 256],
+            specVersion: 0,
+            implementationVersion: 0,
+            description: [0; 256],
+        },
+    );
+
+    unsafe {
+        ffi::vkEnumerateInstanceLayerProperties(
+            std::ptr::addr_of_mut!(layer_count),
+            layers.as_mut_ptr(),
+        );
+    }
+
+    for layer_name in VALIDATION_LAYERS {
+        let mut layer_found = false;
+        let ln_cstr = unsafe { CStr::from_ptr(layer_name as *const i8) };
+        for layer_prop in &layers {
+            let lp_cstr: &CStr = unsafe { CStr::from_ptr(layer_prop.layerName.as_ptr()) };
+            if ln_cstr == lp_cstr {
+                layer_found = true;
+                break;
+            }
+        }
+
+        if !layer_found {
+            return false;
+        }
+    }
+
+    println!("Validation layers available");
+    true
+}
+
+extern "C" fn validation_debug_callback(
+    _message_severity: ffi::VkDebugUtilsMessageSeverityFlagBitsEXT,
+    _message_type: ffi::VkDebugUtilsMessageTypeFlagsEXT,
+    callback_data: *const ffi::VkDebugUtilsMessengerCallbackDataEXT,
+    _user_data: *mut std::ffi::c_void,
+) -> u32 {
+    let message: &CStr = unsafe { CStr::from_ptr((*callback_data).pMessage) };
+
+    println!(
+        "validation layer: {}",
+        message.to_str().unwrap_or("INVALID UTF-8 STRING")
+    );
+
+    ffi::VK_FALSE
+}
+
+fn create_debug_utils_messenger_ext(
+    instance: ffi::VkInstance,
+    create_info: *const ffi::VkDebugUtilsMessengerCreateInfoEXT,
+    allocator: *const ffi::VkAllocationCallbacks,
+    debug_messenger: *mut ffi::VkDebugUtilsMessengerEXT,
+) -> i32 {
+    let func_opt: ffi::PFN_vkCreateDebugUtilsMessengerEXT = unsafe {
+        std::mem::transmute(ffi::vkGetInstanceProcAddr(
+            instance,
+            "vkCreateDebugUtilsMessengerEXT\x00".as_ptr() as *const i8,
+        ))
+    };
+
+    if let Some(func) = func_opt {
+        unsafe { func(instance, create_info, allocator, debug_messenger) }
+    } else {
+        ffi::VkResult_VK_ERROR_EXTENSION_NOT_PRESENT
+    }
+}
 
 struct VulkanApp {
     window: *mut ffi::GLFWwindow,
     vk_instance: ffi::VkInstance,
+    debug_messenger: ffi::VkDebugUtilsMessengerEXT,
 }
 
 impl VulkanApp {
@@ -38,6 +122,7 @@ impl VulkanApp {
         Self {
             window: std::ptr::null_mut(),
             vk_instance: std::ptr::null_mut(),
+            debug_messenger: std::ptr::null_mut(),
         }
     }
 
@@ -62,7 +147,7 @@ impl VulkanApp {
 
     fn init_vulkan(&mut self) {
         // Check validation layers before creating instance.
-        if ENABLE_VALIDATION_LAYERS && !Self::check_validation_layer_support() {
+        if ENABLE_VALIDATION_LAYERS && !check_validation_layer_support() {
             panic!("Validation layers requested, but not available!");
         }
 
@@ -88,16 +173,45 @@ impl VulkanApp {
             exts = ffi::glfwGetRequiredInstanceExtensions(std::ptr::addr_of_mut!(ext_count));
         }
 
+        let mut exts_with_validation: Vec<*const std::ffi::c_char> =
+            Vec::with_capacity(ext_count as usize + 1);
+        let validation_string: *const std::ffi::c_char =
+            ffi::VK_EXT_DEBUG_UTILS_EXTENSION_NAME.as_ptr() as *const i8;
+        if ENABLE_VALIDATION_LAYERS {
+            let exts_slice: &[*const std::ffi::c_char] =
+                unsafe { std::slice::from_raw_parts(exts, ext_count as usize) };
+            for i in 0..(ext_count as usize) {
+                exts_with_validation.push(exts_slice[i]);
+            }
+            exts_with_validation.push(validation_string);
+        }
+
         // Second populate the struct with necessary info.
         let create_info = ffi::VkInstanceCreateInfo {
             sType: ffi::VkStructureType_VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
             pNext: std::ptr::null(),
             flags: 0,
             pApplicationInfo: std::ptr::addr_of!(app_info),
-            enabledLayerCount: 0,
+            enabledLayerCount: if ENABLE_VALIDATION_LAYERS {
-            ppEnabledLayerNames: std::ptr::null(),
+                VALIDATION_LAYERS.len() as u32
-            enabledExtensionCount: ext_count,
+            } else {
-            ppEnabledExtensionNames: exts,
+                0
+            },
+            ppEnabledLayerNames: if ENABLE_VALIDATION_LAYERS {
+                VALIDATION_LAYERS.as_ptr() as *const *const i8
+            } else {
+                std::ptr::null()
+            },
+            enabledExtensionCount: if ENABLE_VALIDATION_LAYERS {
+                ext_count + 1
+            } else {
+                ext_count
+            },
+            ppEnabledExtensionNames: if ENABLE_VALIDATION_LAYERS {
+                exts_with_validation.as_ptr()
+            } else {
+                exts
+            },
         };
 
         let vk_result = unsafe {
@@ -111,53 +225,38 @@ impl VulkanApp {
         if vk_result != ffi::VkResult_VK_SUCCESS {
             panic!("ERROR: Failed to create vk instance!");
         }
+
+        self.setup_debug_messenger();
     }
 
-    fn check_validation_layer_support() -> bool {
+    fn setup_debug_messenger(&mut self) {
-        let mut layer_count: u32 = 0;
+        if !ENABLE_VALIDATION_LAYERS {
-        unsafe {
+            return;
-            ffi::vkEnumerateInstanceLayerProperties(
-                std::ptr::addr_of_mut!(layer_count),
-                std::ptr::null_mut(),
-            );
         }
 
-        let mut layers: Vec<ffi::VkLayerProperties> = Vec::with_capacity(layer_count as usize);
+        if self.vk_instance.is_null() {
-        layers.resize(
+            panic!("ERROR: Cannot set up debug messenger if vk_instance is not initialized!");
-            layer_count as usize,
+        }
-            ffi::VkLayerProperties {
+
-                layerName: [0; 256],
+        let create_info = ffi::VkDebugUtilsMessengerCreateInfoEXT {
-                specVersion: 0,
+            sType: ffi::VkStructureType_VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
-                implementationVersion: 0,
+            pNext: std::ptr::null(),
-                description: [0; 256],
+            flags: 0,
-            },
+            messageSeverity: ffi::VkDebugUtilsMessageSeverityFlagBitsEXT_VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT | ffi::VkDebugUtilsMessageSeverityFlagBitsEXT_VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | ffi::VkDebugUtilsMessageSeverityFlagBitsEXT_VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
+            messageType: ffi::VkDebugUtilsMessageTypeFlagBitsEXT_VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | ffi::VkDebugUtilsMessageTypeFlagBitsEXT_VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | ffi::VkDebugUtilsMessageTypeFlagBitsEXT_VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
+            pfnUserCallback: Some(validation_debug_callback),
+            pUserData: std::ptr::null_mut(),
+        };
+
+        let result = create_debug_utils_messenger_ext(
+            self.vk_instance,
+            std::ptr::addr_of!(create_info),
+            std::ptr::null(),
+            std::ptr::addr_of_mut!(self.debug_messenger),
         );
-
+        if result != ffi::VkResult_VK_SUCCESS {
-        unsafe {
+            panic!("Failed to set up debug messenger!");
-            ffi::vkEnumerateInstanceLayerProperties(
-                std::ptr::addr_of_mut!(layer_count),
-                layers.as_mut_ptr(),
-            );
         }
-
-        for layer_name in VALIDATION_LAYERS {
-            let mut layer_found = false;
-            let ln_cstr = unsafe { CStr::from_ptr(layer_name.as_ptr() as *const i8) };
-            for layer_prop in &layers {
-                let lp_cstr: &CStr = unsafe { CStr::from_ptr(layer_prop.layerName.as_ptr()) };
-                if ln_cstr == lp_cstr {
-                    layer_found = true;
-                    break;
-                }
-            }
-
-            if !layer_found {
-                return false;
-            }
-        }
-
-        println!("Validation layers available");
-        true
     }
 
     fn main_loop(&mut self) {
@@ -182,6 +281,23 @@ impl VulkanApp {
 
 impl Drop for VulkanApp {
     fn drop(&mut self) {
+        if ENABLE_VALIDATION_LAYERS && !self.debug_messenger.is_null() {
+            let func_opt: ffi::PFN_vkDestroyDebugUtilsMessengerEXT = unsafe {
+                std::mem::transmute(ffi::vkGetInstanceProcAddr(
+                    self.vk_instance,
+                    "vkDestroyDebugUtilsMessengerEXT\x00".as_ptr() as *const i8,
+                ))
+            };
+
+            if let Some(func) = func_opt {
+                unsafe {
+                    func(self.vk_instance, self.debug_messenger, std::ptr::null());
+                }
+            } else {
+                println!("WARNING: Failed to load fn to unload debug messenger!");
+            }
+        }
+
         if !self.vk_instance.is_null() {
             unsafe {
                 ffi::vkDestroyInstance(self.vk_instance, std::ptr::null());