RHI: Store texture handle as 64-bit int

When storing a void* pointer to the texture handle, we had
to ensure that the variable would exist until the build phase,
which is error prone and caused errors in QQuickWidget because
we copied the texture ID from the FBO into a local variable
before passing it into QQuickWindow::setRenderTarget().

The reason for using a void* was that we cannot know the width
of the handles in the different backends, but we do know that
they are 64-bit at maximum, so instead of storing potentially
dangling pointers, we just make it a 64-bit integer and cast
it back and forth in the backends.

Task-number: QTBUG-78638
Change-Id: I7951e24351ddb209045ab6197d81eb1290b4da67
Reviewed-by: Laszlo Agocs <laszlo.agocs@qt.io>

src/gui/rhi/qrhi.cpp

@@ -2261,16 +2261,12 @@ QRhiResource::Type QRhiRenderBuffer::resourceType() const
 
 /*!
     \variable QRhiTexture::NativeTexture::object
-    \brief a pointer to the native object handle.
+    \brief 64-bit integer containing the native object handle.
 
-    With OpenGL, the native handle is a GLuint value, so \c object is then a
-    pointer to a GLuint. With Vulkan, the native handle is a VkImage, so \c
-    object is a pointer to a VkImage. With Direct3D 11 and Metal \c
-    object is a pointer to a ID3D11Texture2D or MTLTexture pointer, respectively.
-
-    \note Pay attention to the fact that \a object is always a pointer
-    to the native texture handle type, even if the native type itself is a
-    pointer.
+    With OpenGL, the native handle is a GLuint value, so \c object can then be
+    cast to a GLuint. With Vulkan, the native handle is a VkImage, so \c
+    object can be cast to a VkImage. With Direct3D 11 and Metal \c
+    object contains a ID3D11Texture2D or MTLTexture pointer, respectively.
  */
 
 /*!

src/gui/rhi/qrhi_p.h

@@ -777,7 +777,7 @@ public:
     };
 
     struct NativeTexture {
-        const void *object;
+        quint64 object;
         int layout;
     };
 

src/gui/rhi/qrhid3d11.cpp

@@ -2937,14 +2937,14 @@ bool QD3D11Texture::build()
 
 bool QD3D11Texture::buildFrom(QRhiTexture::NativeTexture src)
 {
-    auto *srcTex = static_cast<ID3D11Texture2D * const *>(src.object);
-    if (!srcTex || !*srcTex)
+    ID3D11Texture2D *srcTex = reinterpret_cast<ID3D11Texture2D *>(src.object);
+    if (srcTex == nullptr)
         return false;
 
     if (!prepareBuild())
         return false;
 
-    tex = *srcTex;
+    tex = srcTex;
 
     if (!finishBuild())
         return false;
@@ -2960,7 +2960,7 @@ bool QD3D11Texture::buildFrom(QRhiTexture::NativeTexture src)
 
 QRhiTexture::NativeTexture QD3D11Texture::nativeTexture()
 {
-    return {&tex, 0};
+    return {quint64(tex), 0};
 }
 
 ID3D11UnorderedAccessView *QD3D11Texture::unorderedAccessViewForLevel(int level)

src/gui/rhi/qrhigles2.cpp

@@ -3776,14 +3776,14 @@ bool QGles2Texture::build()
 
 bool QGles2Texture::buildFrom(QRhiTexture::NativeTexture src)
 {
-    const uint *textureId = static_cast<const uint *>(src.object);
-    if (!textureId || !*textureId)
+    const uint textureId = uint(src.object);
+    if (textureId == 0)
         return false;
 
     if (!prepareBuild())
         return false;
 
-    texture = *textureId;
+    texture = textureId;
     specified = true;
 
     QRHI_RES_RHI(QRhiGles2);
@@ -3799,7 +3799,7 @@ bool QGles2Texture::buildFrom(QRhiTexture::NativeTexture src)
 
 QRhiTexture::NativeTexture QGles2Texture::nativeTexture()
 {
-    return {&texture, 0};
+    return {texture, 0};
 }
 
 QGles2Sampler::QGles2Sampler(QRhiImplementation *rhi, Filter magFilter, Filter minFilter, Filter mipmapMode,

src/gui/rhi/qrhimetal.mm

@@ -2592,14 +2592,14 @@ bool QMetalTexture::build()
 
 bool QMetalTexture::buildFrom(QRhiTexture::NativeTexture src)
 {
-    void * const * tex = (void * const *) src.object;
-    if (!tex || !*tex)
+    id<MTLTexture> tex = id<MTLTexture>(src.object);
+    if (tex == 0)
         return false;
 
     if (!prepareBuild())
         return false;
 
-    d->tex = (id<MTLTexture>) *tex;
+    d->tex = tex;
 
     d->owns = false;
 
@@ -2615,7 +2615,7 @@ bool QMetalTexture::buildFrom(QRhiTexture::NativeTexture src)
 
 QRhiTexture::NativeTexture QMetalTexture::nativeTexture()
 {
-    return {&d->tex, 0};
+    return {quint64(d->tex), 0};
 }
 
 id<MTLTexture> QMetalTextureData::viewForLevel(int level)

src/gui/rhi/qrhivulkan.cpp

@@ -5570,14 +5570,14 @@ bool QVkTexture::build()
 
 bool QVkTexture::buildFrom(QRhiTexture::NativeTexture src)
 {
-    auto *img = static_cast<const VkImage*>(src.object);
-    if (!img || !*img)
+    VkImage img = VkImage(src.object);
+    if (img == 0)
         return false;
 
     if (!prepareBuild())
         return false;
 
-    image = *img;
+    image = img;
 
     if (!finishBuild())
         return false;
@@ -5595,7 +5595,7 @@ bool QVkTexture::buildFrom(QRhiTexture::NativeTexture src)
 
 QRhiTexture::NativeTexture QVkTexture::nativeTexture()
 {
-    return {&image, usageState.layout};
+    return {quint64(image), usageState.layout};
 }
 
 void QVkTexture::setNativeLayout(int layout)

tests/auto/gui/rhi/qrhi/tst_qrhi.cpp

@@ -519,9 +519,8 @@ void tst_QRhi::nativeTexture()
 #ifdef TST_VK
     case QRhi::Vulkan:
     {
-        auto *image = static_cast<const VkImage *>(nativeTex.object);
+        auto image = VkImage(nativeTex.object);
         QVERIFY(image);
-        QVERIFY(*image);
         QVERIFY(nativeTex.layout >= 1); // VK_IMAGE_LAYOUT_GENERAL
         QVERIFY(nativeTex.layout <= 8); // VK_IMAGE_LAYOUT_PREINITIALIZED
     }
@@ -530,27 +529,24 @@ void tst_QRhi::nativeTexture()
 #ifdef TST_GL
     case QRhi::OpenGLES2:
     {
-        auto *textureId = static_cast<const uint *>(nativeTex.object);
+        auto textureId = uint(nativeTex.object);
         QVERIFY(textureId);
-        QVERIFY(*textureId);
     }
         break;
 #endif
 #ifdef TST_D3D11
     case QRhi::D3D11:
     {
-        auto *texture = static_cast<void * const *>(nativeTex.object);
+        auto *texture = reinterpret_cast<void *>(nativeTex.object);
         QVERIFY(texture);
-        QVERIFY(*texture);
     }
         break;
 #endif
 #ifdef TST_MTL
     case QRhi::Metal:
     {
-        void * const * texture = (void * const *)nativeTex.object;
+        auto texture = (void *)nativeTex.object;
         QVERIFY(texture);
-        QVERIFY(*texture);
     }
         break;
 #endif

tests/manual/rhi/texuploads/texuploads.cpp

@@ -241,7 +241,7 @@ void Window::customRender()
             if (nativeTexture.object) {
 #if defined(Q_OS_MACOS) || defined(Q_OS_IOS)
                 if (graphicsApi == Metal) {
-                    qDebug() << "Metal texture: " << *(void**)nativeTexture.object;
+                    qDebug() << "Metal texture: " << nativeTexture.object;
                     // Now could cast to id<MTLTexture> and do something with
                     // it, keeping in mind that copy operations are only done
                     // in beginPass, while rendering into a texture may only