ImGui flickering when use mutable swapchain format for two dynamic renderings.

[stripe2933]

I'm not sure if this error is related to ImGui or MoltenVK. However, since this flickering is not observed when testing the same code in a Windows environment with NVIDIA drivers, I am raising this issue with MoltenVK.

ImGui does not support proper gamma correction for the B8G8R8A8_SRGB format. Therefore, I am using the VK_KHR_mutable_swapchain_format extension and specifying the two formats, B8G8R8A8_SRGB and B8G8R8A8_UNORM, in VkImageFormatListCreateInfo in the pNext of VkSwapchainCreateInfoKHR. During ImGui rendering, I render to the swapchain image view using the B8G8R8A8_UNORM format. Here is some of the code I am using:

• Swapchain creation

auto createSwapchain(
    vk::SwapchainKHR oldSwapchain
) -> vk::raii::SwapchainKHR {
    const vk::SurfaceCapabilitiesKHR surfaceCapabilities = gpu.physicalDevice.getSurfaceCapabilitiesKHR(surface);
    return { gpu.device, vk::StructureChain {
        vk::SwapchainCreateInfoKHR {
            vk::SwapchainCreateFlagBitsKHR::eMutableFormat,
            surface,
            std::min(surfaceCapabilities.minImageCount + 1, surfaceCapabilities.maxImageCount),
            vk::Format::eB8G8R8A8Srgb,
            vk::ColorSpaceKHR::eSrgbNonlinear,
            (swapchainImageExtent = surfaceCapabilities.currentExtent),
            1,
            vk::ImageUsageFlagBits::eColorAttachment,
            vk::SharingMode::eExclusive, {},
            surfaceCapabilities.currentTransform,
            vk::CompositeAlphaFlagBitsKHR::eOpaque,
            vk::PresentModeKHR::eFifo,
            {},
            oldSwapchain,
        },
        vk::ImageFormatListCreateInfo {
            unsafeProxy({
                vk::Format::eB8G8R8A8Srgb,
                vk::Format::eB8G8R8A8Unorm,
            })
        },
    }.get() };
}

• ImGui initialization

    IMGUI_CHECKVERSION();
    ImGui::CreateContext();

    ImGuiIO &io = ImGui::GetIO();
    int framebufferWidth, framebufferHeight;
    glfwGetFramebufferSize(window, &framebufferWidth, &framebufferHeight);
    io.DisplaySize = { static_cast<float>(framebufferWidth), static_cast<float>(framebufferHeight) };
    float contentScaleX, contentScaleY;
    glfwGetWindowContentScale(window, &contentScaleX, &contentScaleY);
    io.DisplayFramebufferScale = { contentScaleX, contentScaleY };
    ImGui_ImplGlfw_InitForVulkan(window, true);

    ImGui_ImplVulkan_InitInfo initInfo {
        .Instance = *instance,
        .PhysicalDevice = *gpu.physicalDevice,
        .Device = *gpu.device,
        .QueueFamily = 0,
        .Queue = gpu.graphicsPresentQueue,
        .DescriptorPool = *imGuiDescriptorPool,
        .MinImageCount = 2,
        .ImageCount = 2, // = MAX_FRAMES_IN_FLIGHT
        .MSAASamples = VK_SAMPLE_COUNT_1_BIT,
        .UseDynamicRendering = true,
        .PipelineRenderingCreateInfo = vk::PipelineRenderingCreateInfo {
            0,
            unsafeProxy({ vk::Format::eB8G8R8A8Unorm }),
        },
    };
    ImGui_ImplVulkan_Init(&initInfo);

• Rendering loop

// Change the current swapchain image layout from PRESENT_SRC_KHR  to COLOR_ATTACHMENT_OPTIMAL for rendering.
cb.pipelineBarrier(
    vk::PipelineStageFlagBits::eColorAttachmentOutput, vk::PipelineStageFlagBits::eColorAttachmentOutput,
    {},
    {}, {},
    vk::ImageMemoryBarrier {
        {}, vk::AccessFlagBits::eColorAttachmentWrite,
        vk::ImageLayout::ePresentSrcKHR, vk::ImageLayout::eColorAttachmentOptimal,
        vk::QueueFamilyIgnored, vk::QueueFamilyIgnored,
        sharedData.swapchainImages[swapchainImageIndex], { vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1 },
    });

// Begin dynamic rendering with B8G8R8A8_SRGB format.
cb.beginRenderingKHR(vk::RenderingInfo {
    {},
    { { 0, 0 }, sharedData.swapchainImageExtent },
    1,
    0,
    unsafeProxy({
        vk::RenderingAttachmentInfo {
            *sharedData.swapchainImageViews[swapchainImageIndex]  /* format=B8G8R8A8_SRGB */, vk::ImageLayout::eColorAttachmentOptimal,
            {}, {}, {},
            vk::AttachmentLoadOp::eClear, vk::AttachmentStoreOp::eStore, vk::ClearColorValue { 0.f, 0.f, 0.f, 1.f },
        },
    }),
});

cb.setViewport(0, vk::Viewport {
    0.f, 0.f,
    static_cast<float>(sharedData.swapchainImageExtent.width), static_cast<float>(sharedData.swapchainImageExtent.height),
    0.f, 1.f
});
cb.setScissor(0, vk::Rect2D { { 0, 0 }, sharedData.swapchainImageExtent });

// Draw a simple triangle.
cb.bindPipeline(vk::PipelineBindPoint::eGraphics, *sharedData.triangleRenderer.pipeline);
cb.draw(3, 1, 0, 0);

cb.endRenderingKHR();

// Memory barrier that ensure the triangle rendering pass is done before imgui do.
cb.pipelineBarrier(
    vk::PipelineStageFlagBits::eColorAttachmentOutput, vk::PipelineStageFlagBits::eColorAttachmentOutput,
    {},
    {}, {},
    vk::ImageMemoryBarrier {
        vk::AccessFlagBits::eColorAttachmentWrite, vk::AccessFlagBits::eColorAttachmentRead,
        vk::ImageLayout::eColorAttachmentOptimal, vk::ImageLayout::eColorAttachmentOptimal,
        vk::QueueFamilyIgnored, vk::QueueFamilyIgnored,
        sharedData.swapchainImages[swapchainImageIndex], { vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1 },
    });

// Begin dynamic rendering with B8G8R8A8_UNORM format.
cb.beginRenderingKHR(vk::RenderingInfo {
    {},
    { { 0, 0 }, sharedData.swapchainImageExtent },
    1,
    0,
    unsafeProxy({
        vk::RenderingAttachmentInfo {
            *sharedData.imGuiSwapchainImageViews[swapchainImageIndex]  /* format=B8G8R8A8_UNORM */, vk::ImageLayout::eColorAttachmentOptimal,
            {}, {}, {},
            vk::AttachmentLoadOp::eLoad /* previously rendered image must not be cleared */, vk::AttachmentStoreOp::eStore, vk::ClearColorValue { 0.f, 0.f, 0.f, 1.f },
        },
    }),
});

// Draw ImGui data.
ImGui_ImplVulkan_RenderDrawData(ImGui::GetDrawData(), cb);

cb.endRenderingKHR();

// Change the current swapchain image layout from COLOR_ATTACHMENT_OPTIMAL to PRESENT_SRC_KHR.
cb.pipelineBarrier(
    vk::PipelineStageFlagBits::eColorAttachmentOutput, vk::PipelineStageFlagBits::eBottomOfPipe,
    {},
    {}, {},
    vk::ImageMemoryBarrier {
        vk::AccessFlagBits::eColorAttachmentWrite, {},
        vk::ImageLayout::eColorAttachmentOptimal, vk::ImageLayout::ePresentSrcKHR,
        vk::QueueFamilyIgnored, vk::QueueFamilyIgnored,
        sharedData.swapchainImages[swapchainImageIndex], { vk::ImageAspectFlagBits::eColor, 0, 1, 0, 1 },
    });

When using this code, flickering occurs as shown in the video below. The validation layers, including synchronization validation, do not raise any issues.

One puzzling point is that if I do not use the mutable format for rendering (i.e., use B8G8R8A8_SRGB in the PipelineRenderingCreateInfo of ImGui_ImplVulkan_InitInfo and use swapchainImageViews instead of imGuiSwapchainImageViews), this flickering does not occur.

• With mutable format

mutable-format.mov

• Without mutable format (flickering not occurred)

[stripe2933]

flickering.mov

Based on several experiments, it seems to be an issue with MoltenVK. When I replaced the ImGui rendering part of the code with another rectangle rendering code, the same flickering was observed. The flickering becomes more frequent as the frame time approaches the display refresh rate (1s/120=8.3ms).

One peculiar point that not shown in the above video is that the red rectangle in the video often hide the Metal HUD.