rhi: Add the basic infrastructure for geometry shader support

.. but this will only be supported on Vulkan, OpenGL 3.2+, and Open GL
ES 3.2+ for the time being.

The situation is:

- Vulkan is working.  qsb accepts .geom files already, and QShader has
  existing geometry shader support.

- OpenGL 3.2 and OpenGL ES 3.2 are working.

- D3D11 is not working.  D3D11 supports geometry shaders, but SPIRV-
  Cross does not support translating geometry shaders to HLSL.

- Metal is not working.  Metal does not directly support geometry
  shaders.

Change-Id: Ieb7c44c58b8be5f2e2197bf5133cf6847e6c132d
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

src/gui/rhi/qrhi.cpp

@@ -711,6 +711,15 @@ Q_LOGGING_CATEGORY(QRHI_LOG_INFO, "qt.rhi.general")
     the way hull shaders are structured, whereas Metal uses a somewhat
     different tessellation pipeline than others), and therefore no guarantees
     can be given for a universal solution for now.
+
+    \value GeometryShader Indicates that the geometry shader stage is
+    supported. When supported, a geometry shader can be specified in the
+    QRhiShaderStage list. \b{Geometry Shaders are considered an experimental
+    feature in QRhi and can only be expected to be supported with Vulkan,
+    OpenGL (3.2+) and OpenGL ES (3.2+) for the time being}, assuming the
+    implementation reports it as supported at run time. Geometry shaders have
+    portability issues between APIs, and therefore no guarantees can be given
+    for a universal solution for now.
  */
 
 /*!
@@ -1470,6 +1479,9 @@ QDebug operator<<(QDebug dbg, const QRhiVertexInputLayout &v)
 
     \value Compute Compute stage. Must be used only when the QRhi::Compute
     feature is supported.
+
+    \value Geometry Geometry stage. Must be used only when the
+    QRhi::GeometryShader feature is supported.
  */
 
 /*!
@@ -3269,6 +3281,7 @@ void QRhiImplementation::updateLayoutDesc(QRhiShaderResourceBindings *srb)
     \value TessellationEvaluationStage Tessellation evaluation (domain shader) stage
     \value FragmentStage Fragment (pixel shader) stage
     \value ComputeStage Compute stage
+    \value GeometryStage Geometry stage
  */
 
 /*!
@@ -7454,6 +7467,8 @@ QRhiPassResourceTracker::BufferStage QRhiPassResourceTracker::toPassTrackerBuffe
         return QRhiPassResourceTracker::BufFragmentStage;
     if (stages.testFlag(QRhiShaderResourceBinding::ComputeStage))
         return QRhiPassResourceTracker::BufComputeStage;
+    if (stages.testFlag(QRhiShaderResourceBinding::GeometryStage))
+        return QRhiPassResourceTracker::BufGeometryStage;
 
     Q_UNREACHABLE();
     return QRhiPassResourceTracker::BufVertexStage;
@@ -7472,6 +7487,8 @@ QRhiPassResourceTracker::TextureStage QRhiPassResourceTracker::toPassTrackerText
         return QRhiPassResourceTracker::TexFragmentStage;
     if (stages.testFlag(QRhiShaderResourceBinding::ComputeStage))
         return QRhiPassResourceTracker::TexComputeStage;
+    if (stages.testFlag(QRhiShaderResourceBinding::GeometryStage))
+        return QRhiPassResourceTracker::TexGeometryStage;
 
     Q_UNREACHABLE();
     return QRhiPassResourceTracker::TexVertexStage;

src/gui/rhi/qrhi_p.h

@@ -297,6 +297,7 @@ public:
         Vertex,
         TessellationControl,
         TessellationEvaluation,
+        Geometry,
         Fragment,
         Compute
     };
@@ -351,8 +352,9 @@ public:
         VertexStage = 1 << 0,
         TessellationControlStage = 1 << 1,
         TessellationEvaluationStage = 1 << 2,
-        FragmentStage = 1 << 3,
-        ComputeStage = 1 << 4
+        GeometryStage = 1 << 3,
+        FragmentStage = 1 << 4,
+        ComputeStage = 1 << 5
     };
     Q_DECLARE_FLAGS(StageFlags, StageFlag)
 
@@ -1673,7 +1675,8 @@ public:
         ThreeDimensionalTextures,
         RenderTo3DTextureSlice,
         TextureArrays,
-        Tessellation
+        Tessellation,
+        GeometryShader
     };
 
     enum BeginFrameFlag {

src/gui/rhi/qrhi_p_p.h

@@ -613,7 +613,8 @@ public:
         BufTCStage,
         BufTEStage,
         BufFragmentStage,
-        BufComputeStage
+        BufComputeStage,
+        BufGeometryStage
     };
 
     enum BufferAccess {
@@ -635,7 +636,8 @@ public:
         TexFragmentStage,
         TexColorOutputStage,
         TexDepthOutputStage,
-        TexComputeStage
+        TexComputeStage,
+        TexGeometryStage
     };
 
     enum TextureAccess {

src/gui/rhi/qrhid3d11.cpp

@@ -565,6 +565,8 @@ bool QRhiD3D11::isFeatureSupported(QRhi::Feature feature) const
         return true;
     case QRhi::Tessellation:
         return false;
+    case QRhi::GeometryShader:
+        return false;
     default:
         Q_UNREACHABLE();
         return false;

src/gui/rhi/qrhigles2.cpp

@@ -466,6 +466,10 @@ QT_BEGIN_NAMESPACE
 #define GL_PATCHES                        0x000E
 #endif
 
+#ifndef GL_GEOMETRY_SHADER
+#define GL_GEOMETRY_SHADER                0x8DD9
+#endif
+
 /*!
     Constructs a new QRhiGles2InitParams.
 
@@ -867,6 +871,11 @@ bool QRhiGles2::create(QRhi::Flags flags)
     else
         caps.tessellation = caps.ctxMajor >= 4; // 4.0
 
+    if (caps.gles)
+        caps.geometryShader = caps.ctxMajor > 3 || (caps.ctxMajor == 3 && caps.ctxMinor >= 2); // ES 3.2
+    else
+        caps.geometryShader = caps.ctxMajor > 3 || (caps.ctxMajor == 3 && caps.ctxMinor >= 2); // 3.2
+
     if (caps.ctxMajor >= 3) { // 3.0 or ES 3.0
         GLint maxArraySize = 0;
         f->glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &maxArraySize);
@@ -1266,6 +1275,8 @@ bool QRhiGles2::isFeatureSupported(QRhi::Feature feature) const
         return caps.maxTextureArraySize > 0;
     case QRhi::Tessellation:
         return caps.tessellation;
+    case QRhi::GeometryShader:
+        return caps.geometryShader;
     default:
         Q_UNREACHABLE();
         return false;
@@ -4238,6 +4249,8 @@ static inline GLenum toGlShaderType(QRhiShaderStage::Type type)
         return GL_TESS_CONTROL_SHADER;
     case QRhiShaderStage::TessellationEvaluation:
         return GL_TESS_EVALUATION_SHADER;
+    case QRhiShaderStage::Geometry:
+        return GL_GEOMETRY_SHADER;
     case QRhiShaderStage::Fragment:
         return GL_FRAGMENT_SHADER;
     case QRhiShaderStage::Compute:
@@ -4519,6 +4532,8 @@ static inline QShader::Stage toShaderStage(QRhiShaderStage::Type type)
         return QShader::TessellationControlStage;
     case QRhiShaderStage::TessellationEvaluation:
         return QShader::TessellationEvaluationStage;
+    case QRhiShaderStage::Geometry:
+        return QShader::GeometryStage;
     case QRhiShaderStage::Fragment:
         return QShader::FragmentStage;
     case QRhiShaderStage::Compute:
@@ -5447,6 +5462,7 @@ static inline bool isGraphicsStage(const QRhiShaderStage &shaderStage)
     return t == QRhiShaderStage::Vertex
             || t == QRhiShaderStage::TessellationControl
             || t == QRhiShaderStage::TessellationEvaluation
+            || t == QRhiShaderStage::Geometry
             || t == QRhiShaderStage::Fragment;
 }
 
@@ -5469,8 +5485,9 @@ bool QGles2GraphicsPipeline::create()
 
     enum {
         VtxIdx = 0,
-        TEIdx,
         TCIdx,
+        TEIdx,
+        GeomIdx,
         FragIdx,
         LastIdx
     };
@@ -5482,6 +5499,8 @@ bool QGles2GraphicsPipeline::create()
             return TCIdx;
         case QRhiShaderStage::TessellationEvaluation:
             return TEIdx;
+        case QRhiShaderStage::Geometry:
+            return GeomIdx;
         case QRhiShaderStage::Fragment:
             return FragIdx;
         default:

src/gui/rhi/qrhigles2_p_p.h

@@ -968,7 +968,8 @@ public:
               screenSpaceDerivatives(false),
               programBinary(false),
               texture3D(false),
-              tessellation(false)
+              tessellation(false),
+              geometryShader(false)
         { }
         int ctxMajor;
         int ctxMinor;
@@ -1018,6 +1019,7 @@ public:
         uint programBinary : 1;
         uint texture3D : 1;
         uint tessellation : 1;
+        uint geometryShader : 1;
     } caps;
     QGles2SwapChain *currentSwapChain = nullptr;
     QSet<GLint> supportedCompressedFormats;

src/gui/rhi/qrhimetal.mm

@@ -620,6 +620,8 @@ bool QRhiMetal::isFeatureSupported(QRhi::Feature feature) const
         return true;
     case QRhi::Tessellation:
         return false;
+    case QRhi::GeometryShader:
+        return false;
     default:
         Q_UNREACHABLE();
         return false;

src/gui/rhi/qrhivulkan.cpp

@@ -633,6 +633,8 @@ bool QRhiVulkan::create(QRhi::Flags flags)
             features.textureCompressionASTC_LDR = VK_TRUE;
         if (physDevFeatures.textureCompressionBC)
             features.textureCompressionBC = VK_TRUE;
+        if (physDevFeatures.geometryShader)
+            features.geometryShader = VK_TRUE;
         devInfo.pEnabledFeatures = &features;
 
         VkResult err = f->vkCreateDevice(physDev, &devInfo, nullptr, &dev);
@@ -696,6 +698,7 @@ bool QRhiVulkan::create(QRhi::Flags flags)
     caps.texture3DSliceAs2D = caps.vulkan11OrHigher;
 
     caps.tessellation = physDevFeatures.tessellationShader;
+    caps.geometryShader = physDevFeatures.geometryShader;
 
     if (!importedAllocator) {
         VmaVulkanFunctions afuncs;
@@ -3987,6 +3990,8 @@ static inline VkPipelineStageFlags toVkPipelineStage(QRhiPassResourceTracker::Bu
         return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
     case QRhiPassResourceTracker::BufComputeStage:
         return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
+    case QRhiPassResourceTracker::BufGeometryStage:
+        return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
     default:
         Q_UNREACHABLE();
         break;
@@ -4061,6 +4066,8 @@ static inline VkPipelineStageFlags toVkPipelineStage(QRhiPassResourceTracker::Te
         return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
     case QRhiPassResourceTracker::TexComputeStage:
         return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
+    case QRhiPassResourceTracker::TexGeometryStage:
+        return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
     default:
         Q_UNREACHABLE();
         break;
@@ -4317,6 +4324,8 @@ bool QRhiVulkan::isFeatureSupported(QRhi::Feature feature) const
         return true;
     case QRhi::Tessellation:
         return caps.tessellation;
+    case QRhi::GeometryShader:
+        return caps.geometryShader;
     default:
         Q_UNREACHABLE();
         return false;
@@ -5267,6 +5276,8 @@ static inline VkShaderStageFlagBits toVkShaderStage(QRhiShaderStage::Type type)
         return VK_SHADER_STAGE_FRAGMENT_BIT;
     case QRhiShaderStage::Compute:
         return VK_SHADER_STAGE_COMPUTE_BIT;
+    case QRhiShaderStage::Geometry:
+        return VK_SHADER_STAGE_GEOMETRY_BIT;
     default:
         Q_UNREACHABLE();
         return VK_SHADER_STAGE_VERTEX_BIT;
@@ -5559,6 +5570,8 @@ static inline VkShaderStageFlags toVkShaderStageFlags(QRhiShaderResourceBinding:
         s |= VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT;
     if (stage.testFlag(QRhiShaderResourceBinding::TessellationEvaluationStage))
         s |= VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT;
+    if (stage.testFlag(QRhiShaderResourceBinding::GeometryStage))
+        s |= VK_SHADER_STAGE_GEOMETRY_BIT;
     return VkShaderStageFlags(s);
 }
 

src/gui/rhi/qrhivulkan_p_p.h

@@ -891,6 +891,7 @@ public:
         bool texture3DSliceAs2D = false;
         bool tessellation = false;
         bool vulkan11OrHigher = false;
+        bool geometryShader = false;
     } caps;
 
     VkPipelineCache pipelineCache = VK_NULL_HANDLE;

tests/manual/rhi/CMakeLists.txt

@@ -25,3 +25,4 @@ add_subdirectory(tex3d)
 add_subdirectory(texturearray)
 add_subdirectory(polygonmode)
 add_subdirectory(tessellation)
+add_subdirectory(geometryshader)

tests/manual/rhi/geometryshader/CMakeLists.txt

@@ -0,0 +1,21 @@
+qt_internal_add_manual_test(geometryshader
+    GUI
+    SOURCES
+        geometryshader.cpp
+    PUBLIC_LIBRARIES
+        Qt::Gui
+        Qt::GuiPrivate
+)
+
+set(geometryshader_resource_files
+    "test.vert.qsb"
+    "test.geom.qsb"
+    "test.frag.qsb"
+)
+
+qt_internal_add_resource(geometryshader "geometryshader"
+    PREFIX
+        "/"
+    FILES
+        ${geometryshader_resource_files}
+)

tests/manual/rhi/geometryshader/buildshaders.bat

@@ -0,0 +1,3 @@
+qsb --glsl 320es,410 test.vert -o test.vert.qsb
+qsb --glsl 320es,410 test.geom -o test.geom.qsb
+qsb --glsl 320es,410 test.frag -o test.frag.qsb

tests/manual/rhi/geometryshader/geometryshader.cpp

@@ -0,0 +1,141 @@
+/****************************************************************************
+**
+** Copyright (C) 2022 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the examples of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "../shared/examplefw.h"
+
+static const float points[] = { 0.0f, 0.0f, 0.0f };
+
+struct
+{
+    QVector<QRhiResource *> releasePool;
+    QRhiBuffer *vbuf = nullptr;
+    QRhiBuffer *ubuf = nullptr;
+    QRhiShaderResourceBindings *srb = nullptr;
+    QRhiGraphicsPipeline *ps = nullptr;
+    QRhiResourceUpdateBatch *initialUpdates = nullptr;
+    float radius = 0.0f;
+} d;
+
+void Window::customInit()
+{
+    if (!m_r->isFeatureSupported(QRhi::GeometryShader))
+        qFatal("Geometry shaders are not supported");
+
+    m_clearColor.setRgb(0, 0, 0);
+
+    d.vbuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(points));
+    d.vbuf->create();
+    d.releasePool << d.vbuf;
+
+    d.ubuf = m_r->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, 4);
+    d.ubuf->create();
+    d.releasePool << d.ubuf;
+
+    d.srb = m_r->newShaderResourceBindings();
+    d.releasePool << d.srb;
+    const QRhiShaderResourceBinding::StageFlags geom = QRhiShaderResourceBinding::GeometryStage;
+    d.srb->setBindings({ QRhiShaderResourceBinding::uniformBuffer(0, geom, d.ubuf) });
+    d.srb->create();
+
+    d.ps = m_r->newGraphicsPipeline();
+    d.releasePool << d.ps;
+
+    d.ps->setTopology(QRhiGraphicsPipeline::Points);
+
+    d.ps->setShaderStages(
+            { { QRhiShaderStage::Vertex, getShader(QLatin1String(":/test.vert.qsb")) },
+              { QRhiShaderStage::Geometry, getShader(QLatin1String(":/test.geom.qsb")) },
+              { QRhiShaderStage::Fragment, getShader(QLatin1String(":/test.frag.qsb")) } });
+
+    d.ps->setCullMode(QRhiGraphicsPipeline::Back);
+    d.ps->setDepthTest(true);
+    d.ps->setDepthWrite(true);
+    QRhiVertexInputLayout inputLayout;
+    inputLayout.setBindings({ { 3 * sizeof(float) } });
+    inputLayout.setAttributes({ { 0, 0, QRhiVertexInputAttribute::Float3, 0 } });
+    d.ps->setVertexInputLayout(inputLayout);
+    d.ps->setShaderResourceBindings(d.srb);
+    d.ps->setRenderPassDescriptor(m_rp);
+    d.ps->create();
+
+    d.initialUpdates = m_r->nextResourceUpdateBatch();
+    d.initialUpdates->uploadStaticBuffer(d.vbuf, points);
+    d.initialUpdates->updateDynamicBuffer(d.ubuf, 0, 4, &d.radius);
+}
+
+void Window::customRelease()
+{
+    qDeleteAll(d.releasePool);
+    d.releasePool.clear();
+}
+
+void Window::customRender()
+{
+    const QSize outputSizeInPixels = m_sc->currentPixelSize();
+    QRhiCommandBuffer *cb = m_sc->currentFrameCommandBuffer();
+    QRhiResourceUpdateBatch *u = m_r->nextResourceUpdateBatch();
+    if (d.initialUpdates) {
+        u->merge(d.initialUpdates);
+        d.initialUpdates->release();
+        d.initialUpdates = nullptr;
+    }
+
+    u->updateDynamicBuffer(d.ubuf, 0, 4, &d.radius);
+    d.radius = std::fmod(d.radius + 0.01f, 1.0f);
+
+    cb->beginPass(m_sc->currentFrameRenderTarget(), m_clearColor, { 1.0f, 0 }, u);
+    cb->setGraphicsPipeline(d.ps);
+    cb->setViewport({ 0, 0, float(outputSizeInPixels.width()), float(outputSizeInPixels.height()) });
+    cb->setShaderResources();
+    QRhiCommandBuffer::VertexInput vbufBinding(d.vbuf, 0);
+    cb->setVertexInput(0, 1, &vbufBinding);
+    cb->draw(1);
+    cb->endPass();
+}

tests/manual/rhi/geometryshader/test.frag

@@ -0,0 +1,8 @@
+#version 440
+
+layout(location = 0) out vec4 fragColor;
+
+void main()
+{
+    fragColor = vec4(1.0);
+}

tests/manual/rhi/geometryshader/test.geom

@@ -0,0 +1,26 @@
+#version 430
+
+# define M_PI 3.14159265358979323846
+
+layout(points) in;
+layout(line_strip, max_vertices = 7) out;
+
+layout(std140, binding = 0) uniform buf {
+    float radius;
+};
+
+void main(void)
+{
+
+    for(int i=0;i<7;++i)
+    {
+        float theta = float(i) / 6.0f * 2.0 * M_PI;
+
+        gl_Position = gl_in[0].gl_Position;
+        gl_Position.xy += radius * vec2(cos(theta), sin(theta));
+
+        EmitVertex();
+    }
+    EndPrimitive();
+
+}

tests/manual/rhi/geometryshader/test.vert

@@ -0,0 +1,8 @@
+#version 440
+
+layout(location = 0) in vec3 position;
+
+void main()
+{
+    gl_Position = vec4(position, 1.0);
+}