rhi: Add a displacement / tessellation manual test

There is something odd when running on Metal: note how the uv
is vec3 instead of vec2, in order to make the vertex-tesc-tese
data to look like this:

struct main0_out
{
    float3 out_uv;
    float3 out_normal;
    float4 gl_Position;
};

if out_uv was float2 we'd get some strange rendering results,
perhaps due to something related to alignment. But have no means
to investigate this further.

Change-Id: I79d4edb2ddde3971c599c4326d98e99a49aa7122
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

tests/manual/rhi/CMakeLists.txt

@@ -32,6 +32,7 @@ add_subdirectory(geometryshader)
 add_subdirectory(stenciloutline)
 add_subdirectory(stereo)
 add_subdirectory(tex1d)
+add_subdirectory(displacement)
 if(QT_FEATURE_widgets)
     add_subdirectory(rhiwidget)
 endif()

tests/manual/rhi/displacement/CMakeLists.txt

@@ -0,0 +1,22 @@
+# Copyright (C) 2022 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+qt_internal_add_manual_test(displacement
+    GUI
+    SOURCES
+        displacement.cpp
+    LIBRARIES
+        Qt::Gui
+        Qt::GuiPrivate
+)
+
+qt_internal_add_resource(displacement "displacement"
+    PREFIX
+        "/"
+    FILES
+        "material.vert.qsb"
+        "material.tesc.qsb"
+        "material.tese.qsb"
+        "material.frag.qsb"
+        "heightmap.png"
+)

tests/manual/rhi/displacement/buildshaders

@@ -0,0 +1,6 @@
+qsb --glsl 320es,410 --hlsl 50 --msl 12 --msltess material.vert -o material.vert.qsb
+qsb --glsl 320es,410 --hlsl 50 --msl 12 material.frag -o material.frag.qsb
+qsb --glsl 320es,410 --msl 12 --tess-mode triangles material.tesc -o material.tesc.qsb
+qsb --glsl 320es,410 --msl 12  --tess-vertex-count 3 material.tese -o material.tese.qsb
+qsb -r hlsl,50,material_hull.hlsl material.tesc.qsb
+qsb -r hlsl,50,material_domain.hlsl material.tese.qsb

tests/manual/rhi/displacement/buildshaders.bat

@@ -0,0 +1,8 @@
+qsb --glsl 320es,410 --hlsl 50 --msl 12 --msltess material.vert -o material.vert.qsb
+qsb --glsl 320es,410 --hlsl 50 --msl 12 material.frag -o material.frag.qsb
+
+qsb --glsl 320es,410 --msl 12 --tess-mode triangles material.tesc -o material.tesc.qsb
+qsb --glsl 320es,410 --msl 12  --tess-vertex-count 3 material.tese -o material.tese.qsb
+
+qsb -r hlsl,50,material_hull.hlsl material.tesc.qsb
+qsb -r hlsl,50,material_domain.hlsl material.tese.qsb

tests/manual/rhi/displacement/displacement.cpp

@@ -0,0 +1,222 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
+
+#define EXAMPLEFW_KEYPRESS_EVENTS
+#include "../shared/examplefw.h"
+#include "../shared/cube.h"
+
+// Another tessellation test. Use the keys (info printed on debug output) to
+// change the tessellation and displacement factors. Compatible with Direct 3D
+// via hand-written hull and domain shaders, but this already pushes the limits
+// of what is sensible when it comes to injecting hand-written HLSL code to get
+// tessellation functional (cbuffer layout, resource registers all need to be
+// figured out manually and works only as long as the GLSL source is not
+// changing, etc.). Note that the domain shader must use SampleLevel
+// (textureLod), it won't compile for ds_5_0 otherwise.
+
+static const quint32 UBUF_SIZE = 80;
+
+struct {
+    QList<QRhiResource *> releasePool;
+
+    QRhiBuffer *vbuf;
+    QRhiBuffer *ubuf;
+    QRhiTexture *tex;
+    QRhiSampler *sampler;
+    QRhiShaderResourceBindings *srb;
+    QRhiGraphicsPipeline *psWire;
+    QRhiGraphicsPipeline *psSolid;
+    bool rotate = true;
+    float rotation = 0.0f;
+    float viewZ = 0.0f;
+    float displacementAmount = 0.0f;
+    float tessInner = 4;
+    float tessOuter = 4;
+    bool useTex = false;
+    bool wireframe = true;
+
+    QRhiResourceUpdateBatch *initialUpdates = nullptr;
+} d;
+
+void Window::customInit()
+{
+    if (!m_r->isFeatureSupported(QRhi::Tessellation))
+        qFatal("Tessellation is not supported");
+
+    qDebug("Left: decrease inner tessellation factor (default is 4)\n"
+           "Right: increase inner tessellation factor\n"
+           "Up: decrease outer tessellation factor (default is 4)\n"
+           "Down: increase outer tessellation factor\n"
+           "W: move camera forward\n"
+           "S: move camera backwards\n"
+           "[: decrease displacement amount (default is 0)\n"
+           "]: increase displacement amount\n"
+           "Tab: toggle displacement texture usage (off by default)\n"
+           "Backspace: toggle wireframe (on by default)\n"
+           );
+
+    d.initialUpdates = m_r->nextResourceUpdateBatch();
+
+    d.vbuf = m_r->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(cube));
+    d.vbuf->create();
+    d.releasePool << d.vbuf;
+
+    d.initialUpdates->uploadStaticBuffer(d.vbuf, cube);
+
+    d.ubuf = m_r->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, m_r->ubufAligned(UBUF_SIZE));
+    d.ubuf->create();
+    d.releasePool << d.ubuf;
+
+    QImage image;
+    image.load(":/heightmap.png");
+    if (image.isNull())
+        qFatal("Failed to load displacement map");
+
+    d.tex = m_r->newTexture(QRhiTexture::RGBA8, image.size());
+    d.tex->create();
+    d.releasePool << d.tex;
+
+    d.initialUpdates->uploadTexture(d.tex, image);
+
+    d.sampler = m_r->newSampler(QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
+                                QRhiSampler::Repeat, QRhiSampler::Repeat);
+    d.releasePool << d.sampler;
+    d.sampler->create();
+
+    d.srb = m_r->newShaderResourceBindings();
+    d.releasePool << d.srb;
+    d.srb->setBindings({
+            QRhiShaderResourceBinding::uniformBuffer(0,
+                           QRhiShaderResourceBinding::TessellationControlStage
+                           | QRhiShaderResourceBinding::TessellationEvaluationStage,
+                           d.ubuf),
+            QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::TessellationEvaluationStage, d.tex, d.sampler)
+        });
+    d.srb->create();
+
+    QRhiVertexInputLayout inputLayout;
+    inputLayout.setBindings({
+        { 3 * sizeof(float) },
+        { 2 * sizeof(float) },
+        { 3 * sizeof(float) }
+    });
+    inputLayout.setAttributes({
+        { 0, 0, QRhiVertexInputAttribute::Float3, 0 },
+        { 1, 1, QRhiVertexInputAttribute::Float2, 0 },
+        { 2, 2, QRhiVertexInputAttribute::Float3, 0 }
+    });
+
+    const QRhiShaderStage stages[] = {
+        { QRhiShaderStage::Vertex, getShader(QLatin1String(":/material.vert.qsb")) },
+        { QRhiShaderStage::TessellationControl, getShader(QLatin1String(":/material.tesc.qsb")) },
+        { QRhiShaderStage::TessellationEvaluation, getShader(QLatin1String(":/material.tese.qsb")) },
+        { QRhiShaderStage::Fragment, getShader(QLatin1String(":/material.frag.qsb")) }
+    };
+
+    d.psWire = m_r->newGraphicsPipeline();
+    d.releasePool << d.psWire;
+    d.psWire->setTopology(QRhiGraphicsPipeline::Patches);
+    d.psWire->setPatchControlPointCount(3);
+    d.psWire->setShaderStages(stages, stages + 4);
+    d.psWire->setDepthTest(true);
+    d.psWire->setDepthWrite(true);
+    d.psWire->setCullMode(QRhiGraphicsPipeline::Back);
+    d.psWire->setPolygonMode(QRhiGraphicsPipeline::Line);
+    d.psWire->setVertexInputLayout(inputLayout);
+    d.psWire->setShaderResourceBindings(d.srb);
+    d.psWire->setRenderPassDescriptor(m_rp);
+    d.psWire->create();
+
+    d.psSolid = m_r->newGraphicsPipeline();
+    d.releasePool << d.psSolid;
+    d.psSolid->setTopology(QRhiGraphicsPipeline::Patches);
+    d.psSolid->setPatchControlPointCount(3);
+    d.psSolid->setShaderStages(stages, stages + 4);
+    d.psSolid->setDepthTest(true);
+    d.psSolid->setDepthWrite(true);
+    d.psSolid->setCullMode(QRhiGraphicsPipeline::Back);
+    d.psSolid->setVertexInputLayout(inputLayout);
+    d.psSolid->setShaderResourceBindings(d.srb);
+    d.psSolid->setRenderPassDescriptor(m_rp);
+    d.psSolid->create();
+}
+
+void Window::customRelease()
+{
+    qDeleteAll(d.releasePool);
+    d.releasePool.clear();
+}
+
+void Window::customRender()
+{
+    const QSize outputSizeInPixels = m_sc->currentPixelSize();
+    QRhiCommandBuffer *cb = m_sc->currentFrameCommandBuffer();
+    QRhiResourceUpdateBatch *u = nullptr;
+    if (d.initialUpdates) {
+        u = d.initialUpdates;
+        d.initialUpdates = nullptr;
+    }
+
+    char *p = d.ubuf->beginFullDynamicBufferUpdateForCurrentFrame();
+    QMatrix4x4 mvp = m_proj;
+    mvp.translate(0, 0, d.viewZ);
+    mvp.rotate(d.rotation, 1, 1, 0);
+    mvp.scale(0.5f);
+
+    memcpy(p, mvp.constData(), 64);
+    memcpy(p + 64, &d.displacementAmount, sizeof(float));
+    memcpy(p + 68, &d.tessInner, sizeof(float));
+    memcpy(p + 72, &d.tessOuter, sizeof(float));
+    qint32 useTex = d.useTex ? 1 : 0;
+    memcpy(p + 76, &useTex, sizeof(qint32));
+
+    d.ubuf->endFullDynamicBufferUpdateForCurrentFrame();
+
+    const QRhiCommandBuffer::VertexInput vbufBinding[] = {
+        { d.vbuf, 0 },
+        { d.vbuf, quint32(36 * 3 * sizeof(float)) },
+        { d.vbuf, quint32(36 * (3 + 2) * sizeof(float)) }
+    };
+
+    cb->beginPass(m_sc->currentFrameRenderTarget(), m_clearColor, { 1.0f, 0 }, u, QRhiCommandBuffer::DoNotTrackResourcesForCompute);
+
+    cb->setGraphicsPipeline(d.wireframe ? d.psWire : d.psSolid);
+    cb->setViewport({ 0, 0, float(outputSizeInPixels.width()), float(outputSizeInPixels.height()) });
+    cb->setShaderResources(d.srb);
+    cb->setVertexInput(0, 3, vbufBinding);
+    cb->draw(36);
+
+    cb->endPass();
+
+    if (d.rotate)
+        d.rotation += 1;
+}
+
+void Window::keyPressEvent(QKeyEvent *e)
+{
+    if (e->key() == Qt::Key_Right)
+        d.tessInner += 1.0f;
+    else if (e->key() == Qt::Key_Left)
+        d.tessInner -= 1.0f;
+    else if (e->key() == Qt::Key_Down)
+        d.tessOuter += 1.0f;
+    else if (e->key() == Qt::Key_Up)
+        d.tessOuter -= 1.0f;
+    else if (e->key() == Qt::Key_W)
+        d.viewZ += 0.1f;
+    else if (e->key() == Qt::Key_S)
+        d.viewZ -= 0.1f;
+    else if (e->key() == Qt::Key_Space)
+        d.rotate = !d.rotate;
+    else if (e->key() == Qt::Key_BracketLeft)
+        d.displacementAmount -= 0.1f;
+    else if (e->key() == Qt::Key_BracketRight)
+        d.displacementAmount += 0.1f;
+    else if (e->key() == Qt::Key_Tab)
+        d.useTex = !d.useTex;
+    else if (e->key() == Qt::Key_Backspace)
+        d.wireframe = !d.wireframe;
+
+    qDebug("Inner: %f Outer: %f Displacement amount: %f Use displacement map: %d",
+           d.tessInner, d.tessOuter, d.displacementAmount, d.useTex);
+}

tests/manual/rhi/displacement/material.frag

@@ -0,0 +1,10 @@
+#version 440
+
+layout(location = 0) out vec4 fragColor;
+
+layout(location = 1) in vec3 in_normal;
+
+void main()
+{
+    fragColor = vec4((normalize(in_normal) + 1.0) / 2.0, 1.0);
+}

tests/manual/rhi/displacement/material.tesc

@@ -0,0 +1,32 @@
+#version 440
+
+layout(vertices = 3) out;
+
+layout(location = 0) in vec3 in_uv[];
+layout(location = 1) in vec3 in_normal[];
+
+layout(location = 0) out vec3 out_uv[];
+layout(location = 1) out vec3 out_normal[];
+
+layout(std140, binding = 0) uniform buf {
+    mat4 mvp;
+    float displacementAmount;
+    float tessInner;
+    float tessOuter;
+    int useTex;
+};
+
+void main()
+{
+    if (gl_InvocationID == 0) {
+        gl_TessLevelOuter[0] = tessOuter;
+        gl_TessLevelOuter[1] = tessOuter;
+        gl_TessLevelOuter[2] = tessOuter;
+
+        gl_TessLevelInner[0] = tessInner;
+    }
+
+    gl_out[gl_InvocationID].gl_Position = gl_in[gl_InvocationID].gl_Position;
+    out_uv[gl_InvocationID] = in_uv[gl_InvocationID];
+    out_normal[gl_InvocationID] = in_normal[gl_InvocationID];
+}

tests/manual/rhi/displacement/material.tese

@@ -0,0 +1,37 @@
+#version 440
+
+layout(triangles, fractional_odd_spacing, ccw) in;
+
+layout(location = 0) in vec3 in_uv[];
+layout(location = 1) in vec3 in_normal[];
+
+//layout(location = 0) out vec2 out_uv;
+layout(location = 1) out vec3 out_normal;
+
+layout(std140, binding = 0) uniform buf {
+    mat4 mvp;
+    float displacementAmount;
+    float tessInner;
+    float tessOuter;
+    int useTex;
+};
+
+layout(binding = 1) uniform sampler2D displacementMap;
+
+void main()
+{
+    vec4 pos = (gl_TessCoord.x * gl_in[0].gl_Position) + (gl_TessCoord.y * gl_in[1].gl_Position) + (gl_TessCoord.z * gl_in[2].gl_Position);
+    vec2 uv = gl_TessCoord.x * in_uv[0].xy + gl_TessCoord.y * in_uv[1].xy;
+    vec3 normal = normalize(gl_TessCoord.x * in_normal[0] + gl_TessCoord.y * in_normal[1] + gl_TessCoord.z * in_normal[2]);
+
+    vec4 c = texture(displacementMap, uv);
+    const vec3 yCoeff_709 = vec3(0.2126, 0.7152, 0.0722);
+    float df = dot(c.rgb, yCoeff_709);
+    if (useTex == 0)
+        df = 1.0;
+    vec3 displacedPos = pos.xyz + normal * df * displacementAmount;
+    gl_Position = mvp * vec4(displacedPos, 1.0);
+
+//    out_uv = uv;
+    out_normal = normal;
+}

tests/manual/rhi/displacement/material.vert

@@ -0,0 +1,15 @@
+#version 440
+
+layout(location = 0) in vec3 position;
+layout(location = 1) in vec2 uv;
+layout(location = 2) in vec3 normal;
+
+layout(location = 0) out vec3 out_uv;
+layout(location = 1) out vec3 out_normal;
+
+void main()
+{
+    gl_Position = vec4(position, 1.0);
+    out_uv = vec3(uv, 0.0);
+    out_normal = normal;
+}

tests/manual/rhi/displacement/material_domain.hlsl

@@ -0,0 +1,54 @@
+struct Input
+{
+    float edges[3] : SV_TessFactor;
+    float inside : SV_InsideTessFactor;
+};
+
+struct PatchInput
+{
+    float3 position : POSITION;
+    float2 uv : TEXCOORD0;
+    float3 normal : TEXCOORD1;
+};
+
+struct PixelInput
+{
+    //float2 uv : TEXCOORD0;
+    float3 normal : TEXCOORD1;
+    float4 position : SV_POSITION;
+};
+
+cbuffer buf : register(b0)
+{
+    row_major float4x4 mvp : packoffset(c0);
+    float displacementAmount : packoffset(c4);
+    float tessInner : packoffset(c4.y);
+    float tessOuter : packoffset(c4.z);
+    int useTex : packoffset(c4.w);
+};
+
+Texture2D<float4> tex : register(t1);
+SamplerState texsampler : register(s1);
+
+[domain("tri")]
+PixelInput main(Input input, float3 uvwCoord : SV_DomainLocation, const OutputPatch<PatchInput, 3> patch)
+{
+    PixelInput output;
+
+    float3 pos = uvwCoord.x * patch[0].position + uvwCoord.y * patch[1].position + uvwCoord.z * patch[2].position;
+    float2 uv =  uvwCoord.x * patch[0].uv + uvwCoord.y * patch[1].uv;
+    float3 normal = normalize(uvwCoord.x * patch[0].normal + uvwCoord.y * patch[1].normal + uvwCoord.z * patch[2].normal);
+
+    float4 c = tex.SampleLevel(texsampler, uv, 0.0);
+    const float3 yCoeff_709 = float3(0.2126, 0.7152, 0.0722);
+    float df = dot(c.rgb, yCoeff_709);
+    if (useTex == 0)
+        df = 1.0;
+    float3 displacedPos = pos + normal * df * displacementAmount;
+
+    output.position = mul(float4(displacedPos, 1.0), mvp);
+    //output.uv = uv;
+    output.normal = normal;
+
+    return output;
+}

tests/manual/rhi/displacement/material_hull.hlsl

@@ -0,0 +1,52 @@
+struct Input
+{
+    float2 uv : TEXCOORD0;
+    float3 normal : TEXCOORD1;
+    float4 position : SV_Position;
+};
+
+struct Output
+{
+    float3 position : POSITION;
+    float2 uv : TEXCOORD0;
+    float3 normal : TEXCOORD1;
+};
+
+struct ConstantData
+{
+    float edges[3] : SV_TessFactor;
+    float inside : SV_InsideTessFactor;
+};
+
+cbuffer buf : register(b0)
+{
+    row_major float4x4 mvp : packoffset(c0);
+    float displacementAmount : packoffset(c4);
+    float tessInner : packoffset(c4.y);
+    float tessOuter : packoffset(c4.z);
+    int useTex : packoffset(c4.w);
+};
+
+ConstantData patchConstFunc(InputPatch<Input, 3> ip, uint PatchID : SV_PrimitiveID )
+{
+    ConstantData d;
+    d.edges[0] = tessOuter;
+    d.edges[1] = tessOuter;
+    d.edges[2] = tessOuter;
+    d.inside = tessInner;
+    return d;
+}
+
+[domain("tri")]
+[partitioning("integer")]
+[outputtopology("triangle_cw")]
+[outputcontrolpoints(3)]
+[patchconstantfunc("patchConstFunc")]
+Output main(InputPatch<Input, 3> patch, uint pointId : SV_OutputControlPointID, uint patchId : SV_PrimitiveID)
+{
+    Output output;
+    output.position = patch[pointId].position;
+    output.uv = patch[pointId].uv;
+    output.normal = patch[pointId].normal;
+    return output;
+}

tests/manual/rhi/shared/cube.h

@@ -25,6 +25,8 @@
  * Author: Bill Hollings <bill.hollings@brenwill.com>
  */
 
+// Normals added by Qt.
+
 #ifndef CUBE_H
 #define CUBE_H
 
@@ -72,6 +74,7 @@ static const float cube[] = {
      1.0f,-1.0f, 1.0f,
      1.0f, 1.0f, 1.0f,
 
+    // UVs
     0.0f, 1.0f,  // -X side
     1.0f, 1.0f,
     1.0f, 0.0f,
@@ -113,6 +116,49 @@ static const float cube[] = {
     0.0f, 1.0f,
     1.0f, 1.0f,
     1.0f, 0.0f,
+
+    // normals
+    -1.0, 0.0, 0.0, //  -X side
+    -1.0, 0.0, 0.0,
+    -1.0, 0.0, 0.0,
+    -1.0, 0.0, 0.0,
+    -1.0, 0.0, 0.0,
+    -1.0, 0.0, 0.0,
+
+    0.0, 0.0, -1.0, //  -Z side
+    0.0, 0.0, -1.0,
+    0.0, 0.0, -1.0,
+    0.0, 0.0, -1.0,
+    0.0, 0.0, -1.0,
+    0.0, 0.0, -1.0,
+
+    0.0, -1.0, 0.0, //  -Y side
+    0.0, -1.0, 0.0,
+    0.0, -1.0, 0.0,
+    0.0, -1.0, 0.0,
+    0.0, -1.0, 0.0,
+    0.0, -1.0, 0.0,
+
+    0.0, 1.0, 0.0, //  +Y side
+    0.0, 1.0, 0.0,
+    0.0, 1.0, 0.0,
+    0.0, 1.0, 0.0,
+    0.0, 1.0, 0.0,
+    0.0, 1.0, 0.0,
+
+    1.0, 0.0, 0.0, //  +X side
+    1.0, 0.0, 0.0,
+    1.0, 0.0, 0.0,
+    1.0, 0.0, 0.0,
+    1.0, 0.0, 0.0,
+    1.0, 0.0, 0.0,
+
+    0.0, 0.0, 1.0, //  +Z side
+    0.0, 0.0, 1.0,
+    0.0, 0.0, 1.0,
+    0.0, 0.0, 1.0,
+    0.0, 0.0, 1.0,
+    0.0, 0.0, 1.0
 };
 // clang-format on