rhi: Add support for polygon fill mode

Support for Polygon Mode (Triangle Fill Mode in Metal, Fill Mode in D3D)
in the RHI graphics pipeline.

Options are Fill and Line

Status:
    OpenGL - ok
    Vulkan - ok
    Metal - ok
    D3D11 - ok
    OpenGL ES - does not support glPolygonMode.

Change-Id: I20b7ef416624700c3dc8d1cbe6474f4ca3889db8
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

src/gui/rhi/qrhi.cpp

@@ -4255,6 +4255,20 @@ QDebug operator<<(QDebug dbg, const QRhiShaderResourceBindings &srb)
     \value DecrementAndWrap
  */
 
+/*!
+    \enum QRhiGraphicsPipeline::PolygonMode
+    \brief Specifies the polygon rasterization mode
+
+    Polygon Mode (Triangle Fill Mode in Metal, Fill Mode in D3D) specifies
+    the fill mode used when rasterizing polygons.  Polygons may be drawn as
+    solids (Fill), or as a wire mesh (Line).
+
+    \note OpenGL ES does not support Polygon Mode
+
+    \value Fill The interior of the polygon is filled (default)
+    \value Line Boundary edges of the polygon are drawn as line segments.
+ */
+
 /*!
     \class QRhiGraphicsPipeline::TargetBlend
     \internal

src/gui/rhi/qrhi_p.h

@@ -1223,6 +1223,11 @@ public:
         CompareOp compareOp = Always;
     };
 
+    enum PolygonMode {
+        Fill,
+        Line
+    };
+
     QRhiResource::Type resourceType() const override;
 
     Flags flags() const { return m_flags; }
@@ -1305,6 +1310,9 @@ public:
     int patchControlPointCount() const { return m_patchControlPointCount; }
     void setPatchControlPointCount(int count) { m_patchControlPointCount = count; }
 
+    PolygonMode polygonMode() const {return m_polygonMode; }
+    void setPolygonMode(PolygonMode mode) {m_polygonMode = mode; }
+
     virtual bool create() = 0;
 
 protected:
@@ -1327,6 +1335,7 @@ protected:
     int m_depthBias = 0;
     float m_slopeScaledDepthBias = 0.0f;
     int m_patchControlPointCount = 3;
+    PolygonMode m_polygonMode = Fill;
     QVarLengthArray<QRhiShaderStage, 4> m_shaderStages;
     QRhiVertexInputLayout m_vertexInputLayout;
     QRhiShaderResourceBindings *m_shaderResourceBindings = nullptr;

src/gui/rhi/qrhid3d11.cpp

@@ -3783,6 +3783,19 @@ static inline D3D11_CULL_MODE toD3DCullMode(QRhiGraphicsPipeline::CullMode c)
     }
 }
 
+static inline D3D11_FILL_MODE toD3DFillMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::Fill:
+        return D3D11_FILL_SOLID;
+    case QRhiGraphicsPipeline::Line:
+        return D3D11_FILL_WIREFRAME;
+    default:
+        Q_UNREACHABLE();
+        return D3D11_FILL_SOLID;
+    }
+}
+
 static inline D3D11_COMPARISON_FUNC toD3DCompareOp(QRhiGraphicsPipeline::CompareOp op)
 {
     switch (op) {
@@ -4072,7 +4085,7 @@ bool QD3D11GraphicsPipeline::create()
 
     D3D11_RASTERIZER_DESC rastDesc;
     memset(&rastDesc, 0, sizeof(rastDesc));
-    rastDesc.FillMode = D3D11_FILL_SOLID;
+    rastDesc.FillMode = toD3DFillMode(m_polygonMode);
     rastDesc.CullMode = toD3DCullMode(m_cullMode);
     rastDesc.FrontCounterClockwise = m_frontFace == CCW;
     rastDesc.DepthBias = m_depthBias;

src/gui/rhi/qrhigles2.cpp

@@ -454,6 +454,14 @@ QT_BEGIN_NAMESPACE
 #define GL_PATCH_VERTICES                 0x8E72
 #endif
 
+#ifndef GL_LINE
+#define GL_LINE                           0x1B01
+#endif
+
+#ifndef GL_FILL
+#define GL_FILL                           0x1B02
+#endif
+
 #ifndef GL_PATCHES
 #define GL_PATCHES                        0x000E
 #endif
@@ -661,6 +669,8 @@ bool QRhiGles2::create(QRhi::Flags flags)
         return false;
 
     f = static_cast<QOpenGLExtensions *>(ctx->extraFunctions());
+    glPolygonMode = reinterpret_cast<void(QOPENGLF_APIENTRYP)(GLenum, GLenum)>(
+            ctx->getProcAddress(QByteArrayLiteral("glPolygonMode")));
 
     const char *vendor = reinterpret_cast<const char *>(f->glGetString(GL_VENDOR));
     const char *renderer = reinterpret_cast<const char *>(f->glGetString(GL_RENDERER));
@@ -2506,6 +2516,19 @@ static inline GLenum toGlStencilOp(QRhiGraphicsPipeline::StencilOp op)
     }
 }
 
+static inline GLenum toGlPolygonMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::PolygonMode::Fill:
+        return GL_FILL;
+    case QRhiGraphicsPipeline::PolygonMode::Line:
+        return GL_LINE;
+    default:
+        Q_UNREACHABLE();
+        return GL_FILL;
+    }
+}
+
 static inline GLenum toGlMinFilter(QRhiSampler::Filter f, QRhiSampler::Filter m)
 {
     switch (f) {
@@ -3318,6 +3341,12 @@ void QRhiGles2::executeBindGraphicsPipeline(QGles2CommandBuffer *cbD, QGles2Grap
         f->glFrontFace(frontFace);
     }
 
+    const GLenum polygonMode = toGlPolygonMode(psD->m_polygonMode);
+    if (glPolygonMode && (forceUpdate || polygonMode != state.polygonMode)) {
+        state.polygonMode = polygonMode;
+        glPolygonMode(GL_FRONT_AND_BACK, polygonMode);
+    }
+
     if (!psD->m_targetBlends.isEmpty()) {
         // We do not have MRT support here, meaning all targets use the blend
         // params from the first one. This is technically incorrect, even if

src/gui/rhi/qrhigles2_p_p.h

@@ -603,6 +603,7 @@ struct QGles2CommandBuffer : public QRhiCommandBuffer
         float polyOffsetUnits;
         float lineWidth;
         int cpCount;
+        GLenum polygonMode;
         void reset() { valid = false; }
         struct {
             // not part of QRhiGraphicsPipeline but used by setGraphicsPipeline()
@@ -918,6 +919,7 @@ public:
     QOpenGLContext *maybeShareContext = nullptr;
     mutable bool needsMakeCurrentDueToSwap = false;
     QOpenGLExtensions *f = nullptr;
+    void (QOPENGLF_APIENTRYP glPolygonMode) (GLenum, GLenum) = nullptr;
     uint vao = 0;
     struct Caps {
         Caps()

src/gui/rhi/qrhimetal.mm

@@ -307,6 +307,7 @@ struct QMetalGraphicsPipelineData
     MTLPrimitiveType primitiveType;
     MTLWinding winding;
     MTLCullMode cullMode;
+    MTLTriangleFillMode triangleFillMode;
     float depthBias;
     float slopeScaledDepthBias;
     QMetalShader vs;
@@ -1064,6 +1065,10 @@ void QRhiMetal::setGraphicsPipeline(QRhiCommandBuffer *cb, QRhiGraphicsPipeline
             [cbD->d->currentRenderPassEncoder setCullMode: psD->d->cullMode];
             cbD->currentCullMode = int(psD->d->cullMode);
         }
+        if (cbD->currentTriangleFillMode == -1 || psD->d->triangleFillMode != uint(cbD->currentTriangleFillMode)) {
+            [cbD->d->currentRenderPassEncoder setTriangleFillMode: psD->d->triangleFillMode];
+            cbD->currentTriangleFillMode = int(psD->d->triangleFillMode);
+        }
         if (cbD->currentFrontFaceWinding == -1 || psD->d->winding != uint(cbD->currentFrontFaceWinding)) {
             [cbD->d->currentRenderPassEncoder setFrontFacingWinding: psD->d->winding];
             cbD->currentFrontFaceWinding = int(psD->d->winding);
@@ -3498,6 +3503,19 @@ static inline MTLCullMode toMetalCullMode(QRhiGraphicsPipeline::CullMode c)
     }
 }
 
+static inline MTLTriangleFillMode toMetalTriangleFillMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::Fill:
+        return MTLTriangleFillModeFill;
+    case QRhiGraphicsPipeline::Line:
+        return MTLTriangleFillModeLines;
+    default:
+        Q_UNREACHABLE();
+        return MTLTriangleFillModeFill;
+    }
+}
+
 id<MTLLibrary> QRhiMetalData::createMetalLib(const QShader &shader, QShader::Variant shaderVariant,
                                              QString *error, QByteArray *entryPoint, QShaderKey *activeKey)
 {
@@ -3754,6 +3772,7 @@ bool QMetalGraphicsPipeline::create()
     d->primitiveType = toMetalPrimitiveType(m_topology);
     d->winding = m_frontFace == CCW ? MTLWindingCounterClockwise : MTLWindingClockwise;
     d->cullMode = toMetalCullMode(m_cullMode);
+    d->triangleFillMode = toMetalTriangleFillMode(m_polygonMode);
     d->depthBias = float(m_depthBias);
     d->slopeScaledDepthBias = m_slopeScaledDepthBias;
 
@@ -3902,6 +3921,7 @@ void QMetalCommandBuffer::resetPerPassCachedState()
     currentIndexOffset = 0;
     currentIndexFormat = QRhiCommandBuffer::IndexUInt16;
     currentCullMode = -1;
+    currentTriangleFillMode = -1;
     currentFrontFaceWinding = -1;
     currentDepthBiasValues = { 0.0f, 0.0f };
 

src/gui/rhi/qrhimetal_p_p.h

@@ -303,6 +303,7 @@ struct QMetalCommandBuffer : public QRhiCommandBuffer
     quint32 currentIndexOffset;
     QRhiCommandBuffer::IndexFormat currentIndexFormat;
     int currentCullMode;
+    int currentTriangleFillMode;
     int currentFrontFaceWinding;
     QPair<float, float> currentDepthBiasValues;
 

src/gui/rhi/qrhivulkan.cpp

@@ -5493,6 +5493,19 @@ static inline VkStencilOp toVkStencilOp(QRhiGraphicsPipeline::StencilOp op)
     }
 }
 
+static inline VkPolygonMode toVkPolygonMode(QRhiGraphicsPipeline::PolygonMode mode)
+{
+    switch (mode) {
+    case QRhiGraphicsPipeline::Fill:
+        return VK_POLYGON_MODE_FILL;
+    case QRhiGraphicsPipeline::Line:
+        return VK_POLYGON_MODE_LINE;
+    default:
+        Q_UNREACHABLE();
+        return VK_POLYGON_MODE_FILL;
+    }
+}
+
 static inline void fillVkStencilOpState(VkStencilOpState *dst, const QRhiGraphicsPipeline::StencilOpState &src)
 {
     dst->failOp = toVkStencilOp(src.failOp);
@@ -7009,6 +7022,7 @@ bool QVkGraphicsPipeline::create()
         rastInfo.depthBiasSlopeFactor = m_slopeScaledDepthBias;
     }
     rastInfo.lineWidth = rhiD->caps.wideLines ? m_lineWidth : 1.0f;
+    rastInfo.polygonMode = toVkPolygonMode(m_polygonMode);
     pipelineInfo.pRasterizationState = &rastInfo;
 
     VkPipelineMultisampleStateCreateInfo msInfo;

tests/manual/rhi/CMakeLists.txt

@@ -23,4 +23,5 @@ add_subdirectory(instancing)
 add_subdirectory(noninstanced)
 add_subdirectory(tex3d)
 add_subdirectory(texturearray)
+add_subdirectory(polygonmode)
 add_subdirectory(tessellation)

tests/manual/rhi/polygonmode/CMakeLists.txt

@@ -0,0 +1,20 @@
+qt_internal_add_manual_test(polygonmode
+    GUI
+    SOURCES
+        polygonmode.cpp
+    PUBLIC_LIBRARIES
+        Qt::Gui
+        Qt::GuiPrivate
+)
+
+set(polygonmode_resource_files
+    "test.vert.qsb"
+    "test.frag.qsb"
+)
+
+qt_internal_add_resource(polygonmode "polygonmode"
+    PREFIX
+        "/"
+    FILES
+        ${polygonmode_resource_files}
+)

tests/manual/rhi/polygonmode/buildshaders.bat

@@ -0,0 +1,2 @@
+qsb --glsl 320es,410,120 test.vert --msl 12 --hlsl 50 -o test.vert.qsb
+qsb --glsl 320es,410,120 test.frag --msl 12 --hlsl 50 -o test.frag.qsb

tests/manual/rhi/polygonmode/polygonmode.cpp

@@ -0,0 +1,148 @@
+/****************************************************************************
+**
+** Copyright (C) 2021 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the examples of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:BSD$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** BSD License Usage
+** Alternatively, you may use this file under the terms of the BSD license
+** as follows:
+**
+** "Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions are
+** met:
+**   * Redistributions of source code must retain the above copyright
+**     notice, this list of conditions and the following disclaimer.
+**   * Redistributions in binary form must reproduce the above copyright
+**     notice, this list of conditions and the following disclaimer in
+**     the documentation and/or other materials provided with the
+**     distribution.
+**   * Neither the name of The Qt Company Ltd nor the names of its
+**     contributors may be used to endorse or promote products derived
+**     from this software without specific prior written permission.
+**
+**
+** THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+** "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+** LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+** A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+** OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+** SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+** LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+** OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "../shared/examplefw.h"
+
+static const float geom[] = {
+    -0.5f, 0.5f, 0.0f,   1.0f, 0.0f, 0.0f,
+    -0.5f, -0.5f, 0.0f,  0.0f, 1.0f, 0.0f,
+    0.0f, 0.5f, 0.0f,    0.0f, 0.0f, 1.0f,
+    0.0f, -0.5f, 0.0f,   1.0f, 0.0f, 1.0f,
+    0.5f, 0.5f, 0.0f,    1.0f, 1.0f, 0.0f,
+    0.5f, -0.5f, 0.0f,   0.0f, 1.0f, 1.0f,
+
+};
+
+struct {
+    QVector<QRhiResource *> releasePool;
+    QRhiBuffer *vbuf = nullptr;
+    QRhiShaderResourceBindings *srb = nullptr;
+    QRhiGraphicsPipeline *ps = nullptr;
+    QRhiResourceUpdateBatch *initialUpdates = nullptr;
+    int count = 0;
+} d;
+
+void Window::customInit()
+{
+    m_clearColor = QColor("black");
+
+    d.vbuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(geom));
+    d.vbuf->create();
+    d.releasePool << d.vbuf;
+
+    d.srb = m_r->newShaderResourceBindings();
+    d.releasePool << d.srb;
+    d.srb->create();
+
+    d.ps = m_r->newGraphicsPipeline();
+    d.releasePool << d.ps;
+
+    d.ps->setTopology(QRhiGraphicsPipeline::TriangleStrip);
+
+    d.ps->setShaderStages({
+        { QRhiShaderStage::Vertex, getShader(QLatin1String(":/test.vert.qsb")) },
+        { QRhiShaderStage::Fragment, getShader(QLatin1String(":/test.frag.qsb")) }
+    });
+
+    d.ps->setCullMode(QRhiGraphicsPipeline::None);
+    d.ps->setPolygonMode(QRhiGraphicsPipeline::Line);
+    d.ps->setDepthTest(true);
+    d.ps->setDepthWrite(true);
+    QRhiVertexInputLayout inputLayout;
+    inputLayout.setBindings({
+        { 6 * sizeof(float) }
+    });
+    inputLayout.setAttributes({
+        { 0, 0, QRhiVertexInputAttribute::Float3, 0 },
+        { 0, 1, QRhiVertexInputAttribute::Float3, 3 * sizeof(float) }
+    });
+    d.ps->setVertexInputLayout(inputLayout);
+    d.ps->setShaderResourceBindings(d.srb);
+    d.ps->setRenderPassDescriptor(m_rp);
+    d.ps->create();
+
+    d.initialUpdates = m_r->nextResourceUpdateBatch();
+    d.initialUpdates->uploadStaticBuffer(d.vbuf, geom);
+}
+
+void Window::customRelease()
+{
+    qDeleteAll(d.releasePool);
+    d.releasePool.clear();
+}
+
+void Window::customRender()
+{
+    const QSize outputSizeInPixels = m_sc->currentPixelSize();
+    QRhiCommandBuffer *cb = m_sc->currentFrameCommandBuffer();
+    QRhiResourceUpdateBatch *u = m_r->nextResourceUpdateBatch();
+    if (d.initialUpdates) {
+        u->merge(d.initialUpdates);
+        d.initialUpdates->release();
+        d.initialUpdates = nullptr;
+    }
+
+    QRhiGraphicsPipeline::PolygonMode polygonMode =
+            (++d.count / 60) % 2 ? QRhiGraphicsPipeline::Fill : QRhiGraphicsPipeline::Line;
+
+    if (d.ps->polygonMode() != polygonMode) {
+        d.ps->setPolygonMode(polygonMode);
+        d.ps->create();
+    }
+
+    cb->beginPass(m_sc->currentFrameRenderTarget(), m_clearColor, { 1.0f, 0 }, u);
+    cb->setGraphicsPipeline(d.ps);
+    cb->setViewport({ 0, 0, float(outputSizeInPixels.width()), float(outputSizeInPixels.height()) });
+    cb->setShaderResources();
+    QRhiCommandBuffer::VertexInput vbufBinding(d.vbuf, 0);
+    cb->setVertexInput(0, 1, &vbufBinding);
+    cb->draw(6);
+    cb->endPass();
+}

tests/manual/rhi/polygonmode/test.frag

@@ -0,0 +1,10 @@
+#version 440
+
+layout(location = 0) in vec3 v_color;
+
+layout(location = 0) out vec4 fragColor;
+
+void main()
+{
+    fragColor = vec4(v_color, 1.0);
+}

tests/manual/rhi/polygonmode/test.vert

@@ -0,0 +1,12 @@
+#version 440
+
+layout(location = 0) in vec3 position;
+layout(location = 1) in vec3 color;
+
+layout(location = 0) out vec3 v_color;
+
+void main()
+{
+    gl_Position = vec4(position, 1.0);
+    v_color = color;
+}

tests/manual/rhi/tessellation/tessellation.cpp

@@ -100,6 +100,7 @@ void Window::customInit()
     });
 
     d.ps->setCullMode(QRhiGraphicsPipeline::Back);
+    d.ps->setPolygonMode(QRhiGraphicsPipeline::Line);
     d.ps->setDepthTest(true);
     d.ps->setDepthWrite(true);
     QRhiVertexInputLayout inputLayout;