Ugly fix for handling QT_SOURCE_TREE

QT_SOURCE_TREE is a variable that is set in qtbase/.qmake.conf.
In qtbase, it's used throughout various
projects to find cpp sources when building standalone tests (among
other things).
Everything works fine with qmake, because even if qmake is invoked
on the tests subfolder, qmake searches up the source directory tree
until it finds a .qmake.conf file, and uses that.

When building qttools with qmake, the qdoc project expects
to have a QT_SOURCE_TREE value, but it's not actually set in the
qttools/.qmake.conf file, so the generated include paths that use
that value are incorrect. Curiously the build still succeeds.

Now in CMake land we replaced QT_SOURCE_TREE with
CMAKE_SOURCE_DIR, but that does not work properly when doing a
standalone tests build, because the project in that case is the
tests one, and not the qtbase one, so configuration fails in a
developer build when trying to configure some private tests.

So far I've found that only qtbase actively uses this value.
A temporary fix is to save the qtbase source directory into a
QT_SOURCE_TREE variable inside the generated
BuildInternalsExtra.cmake file.

The pro2cmake script is changed to handle presence of QT_SOURCE_TREE
in a qrc file path. This is handled by finding the location of a
.qmake.conf file starting from the project file absolute path.
This is needed to stop the script from crashing when handling
the mimedatabase test projects for example.

The change also regenerates the relevant failing test projects, and
thus standalone tests (when doing developer builds aka private_tests
enabled) now configure and build successfully.

Change-Id: I15adc6f4ab6e3056c43ed850196204e2229c4d98
Reviewed-by: Simon Hausmann <simon.hausmann@qt.io>

cmake/QtBuildInternalsExtra.cmake.in

@@ -19,3 +19,6 @@ endif()
 # Propagate non-prefix builds.
 set(QT_WILL_INSTALL @QT_WILL_INSTALL@ CACHE BOOL
     "Boolean indicating if doing a Qt prefix build (vs non-prefix build)." FORCE)
+
+set(QT_SOURCE_TREE "@QT_SOURCE_TREE@" CACHE PATH
+"A path to the source tree of the previously configured QtBase project." FORCE)

cmake/QtPostProcess.cmake

@@ -265,6 +265,8 @@ endfunction()
 
 function(qt_generate_build_internals_extra_cmake_code)
     if(PROJECT_NAME STREQUAL "QtBase")
+
+        set(QT_SOURCE_TREE "${QtBase_SOURCE_DIR}")
         qt_path_join(extra_file_path
                      ${QT_CONFIG_BUILD_DIR}
                      ${INSTALL_CMAKE_NAMESPACE}BuildInternals/QtBuildInternalsExtra.cmake)

cmake/QtSetup.cmake

@@ -63,6 +63,14 @@ if (PROJECT_NAME STREQUAL "QtBase")
     unset(__qt_will_install_value)
 endif()
 
+# Specify the QT_SOURCE_TREE only when building qtbase. Needed by some tests when the tests are
+# built as part of the project, and not standalone. For standalone tests, the value is set in
+# QtBuildInternalsExtra.cmake.
+if(PROJECT_NAME STREQUAL "QtBase")
+    set(QT_SOURCE_TREE "${QtBase_SOURCE_DIR}" CACHE PATH
+        "A path to the source tree of the previously configured QtBase project." FORCE)
+endif()
+
 if(FEATURE_developer_build)
     if(DEFINED QT_CMAKE_EXPORT_COMPILE_COMMANDS)
         set(CMAKE_EXPORT_COMPILE_COMMANDS ${QT_CMAKE_EXPORT_COMPILE_COMMANDS})

tests/auto/corelib/io/qfilesystementry/CMakeLists.txt

@@ -1,7 +1,17 @@
-add_qt_test(tst_qfilesystementry SOURCES
+# Generated from qfilesystementry.pro.
+
+#####################################################################
+## tst_qfilesystementry Test:
+#####################################################################
+
+add_qt_test(tst_qfilesystementry
+    GUI
+    SOURCES
+        ${QT_SOURCE_TREE}/src/corelib/io/qfilesystementry.cpp ${QT_SOURCE_TREE}/src/corelib/io/qfilesystementry_p.h
         tst_qfilesystementry.cpp
-        ${PROJECT_SOURCE_DIR}/src/corelib/io/qfilesystementry.cpp
-        ${PROJECT_SOURCE_DIR}/src/corelib/io/qfilesystementry_p.h
     LIBRARIES
         Qt::CorePrivate
 )
+
+#### Keys ignored in scope 1:.:.:qfilesystementry.pro:<TRUE>:
+# CONFIG = "testcase"

tests/auto/corelib/mimetypes/qmimedatabase/qmimedatabase-cache/.prev_CMakeLists.txt

@@ -5,6 +5,7 @@
 #####################################################################
 
 add_qt_test(tst_qmimedatabase-cache
+    GUI
     SOURCES
         ../tst_qmimedatabase.h
         tst_qmimedatabase-cache.cpp
@@ -13,17 +14,28 @@ add_qt_test(tst_qmimedatabase-cache
 )
 
 # Resources:
+set_source_files_properties("/Volumes/T3/Dev/qt/qt5_cmake/qtbase/src/corelib/mimetypes/mime/packages/freedesktop.org.xml"
+    PROPERTIES alias "freedesktop.org.xml"
+)
+add_qt_resource(tst_qmimedatabase-cache "mimetypes"
+    PREFIX
+        "/qt-project.org/qmime/packages"
+    BASE
+        "/Volumes/T3/Dev/qt/qt5_cmake/qtbase/src/corelib/mimetypes"
+    FILES
+        mime/packages/freedesktop.org.xml
+)
 set_source_files_properties("../qml-again.xml"
-    PROPERTIES alias "qml-again.xml"]
+    PROPERTIES alias "qml-again.xml"
 )
 set_source_files_properties("../test.qml"
-    PROPERTIES alias "test.qml"]
+    PROPERTIES alias "test.qml"
 )
 set_source_files_properties("../text-x-objcsrc.xml"
-    PROPERTIES alias "text-x-objcsrc.xml"]
+    PROPERTIES alias "text-x-objcsrc.xml"
 )
 set_source_files_properties("../yast2-metapackage-handler-mimetypes.xml"
-    PROPERTIES alias "yast2-metapackage-handler-mimetypes.xml"]
+    PROPERTIES alias "yast2-metapackage-handler-mimetypes.xml"
 )
 add_qt_resource(tst_qmimedatabase-cache "testdata"
     PREFIX
@@ -60,7 +72,7 @@ extend_target(tst_qmimedatabase-cache CONDITION 2*_x_ - g
         -Wshadow
 )
 
-extend_target(tst_qmimedatabase-cache CONDITION UNIX AND NOT APPLE_OSX AND NOT QNX
+extend_target(tst_qmimedatabase-cache CONDITION UNIX AND NOT APPLE AND NOT QNX
     DEFINES
         USE_XDG_DATA_DIRS
 )

tests/auto/corelib/mimetypes/qmimedatabase/qmimedatabase-cache/CMakeLists.txt

@@ -5,6 +5,7 @@
 #####################################################################
 
 add_qt_test(tst_qmimedatabase-cache
+    GUI
     SOURCES
         ../tst_qmimedatabase.h
         tst_qmimedatabase-cache.cpp
@@ -66,7 +67,7 @@ extend_target(tst_qmimedatabase-cache CONDITION CMAKE_COMPILER_IS_GNUCXX
         -Wshadow
 )
 
-extend_target(tst_qmimedatabase-cache CONDITION UNIX AND NOT APPLE_OSX AND NOT QNX
+extend_target(tst_qmimedatabase-cache CONDITION UNIX AND NOT APPLE AND NOT QNX
     DEFINES
         USE_XDG_DATA_DIRS
 )

tests/auto/corelib/mimetypes/qmimedatabase/qmimedatabase-xml/.prev_CMakeLists.txt

@@ -0,0 +1,78 @@
+# Generated from qmimedatabase-xml.pro.
+
+#####################################################################
+## tst_qmimedatabase-xml Test:
+#####################################################################
+
+add_qt_test(tst_qmimedatabase-xml
+    GUI
+    SOURCES
+        ../tst_qmimedatabase.h
+        tst_qmimedatabase-xml.cpp
+    PUBLIC_LIBRARIES
+        Qt::Concurrent
+)
+
+# Resources:
+set_source_files_properties("/Volumes/T3/Dev/qt/qt5_cmake/qtbase/src/corelib/mimetypes/mime/packages/freedesktop.org.xml"
+    PROPERTIES alias "freedesktop.org.xml"
+)
+add_qt_resource(tst_qmimedatabase-xml "mimetypes"
+    PREFIX
+        "/qt-project.org/qmime/packages"
+    BASE
+        "/Volumes/T3/Dev/qt/qt5_cmake/qtbase/src/corelib/mimetypes"
+    FILES
+        mime/packages/freedesktop.org.xml
+)
+set_source_files_properties("../qml-again.xml"
+    PROPERTIES alias "qml-again.xml"
+)
+set_source_files_properties("../test.qml"
+    PROPERTIES alias "test.qml"
+)
+set_source_files_properties("../text-x-objcsrc.xml"
+    PROPERTIES alias "text-x-objcsrc.xml"
+)
+set_source_files_properties("../yast2-metapackage-handler-mimetypes.xml"
+    PROPERTIES alias "yast2-metapackage-handler-mimetypes.xml"
+)
+add_qt_resource(tst_qmimedatabase-xml "testdata"
+    PREFIX
+        "/qt-project.org/qmime"
+    BASE
+        ".."
+    FILES
+        invalid-magic1.xml
+        invalid-magic2.xml
+        invalid-magic3.xml
+        magic-and-hierarchy.foo
+        magic-and-hierarchy.xml
+        magic-and-hierarchy2.foo
+        qml-again.xml
+        test.qml
+        text-x-objcsrc.xml
+        yast2-metapackage-handler-mimetypes.xml
+)
+
+
+#### Keys ignored in scope 1:.:.:qmimedatabase-xml.pro:<TRUE>:
+# CONFIG = "testcase"
+
+## Scopes:
+#####################################################################
+
+extend_target(tst_qmimedatabase-xml CONDITION 2*_x_ - g
+    COMPILE_OPTIONS
+        -W
+        -Wall
+        -Wextra
+        -Wno-long-long
+        -Wnon-virtual-dtor
+        -Wshadow
+)
+
+extend_target(tst_qmimedatabase-xml CONDITION UNIX AND NOT APPLE AND NOT QNX
+    DEFINES
+        USE_XDG_DATA_DIRS
+)

tests/auto/corelib/mimetypes/qmimedatabase/qmimedatabase-xml/CMakeLists.txt

@@ -5,11 +5,12 @@
 #####################################################################
 
 add_qt_test(tst_qmimedatabase-xml
+    GUI
     SOURCES
         ../tst_qmimedatabase.h
         tst_qmimedatabase-xml.cpp
     PUBLIC_LIBRARIES
-        Qt::ConcurrentPrivate
+        Qt::Concurrent
 )
 
 # Resources:
@@ -25,7 +26,6 @@ set_source_files_properties("../text-x-objcsrc.xml"
 set_source_files_properties("../yast2-metapackage-handler-mimetypes.xml"
     PROPERTIES alias "yast2-metapackage-handler-mimetypes.xml"
 )
-
 add_qt_resource(tst_qmimedatabase-xml "testdata"
     PREFIX
         "/qt-project.org/qmime"
@@ -56,7 +56,7 @@ corelib_add_mimetypes_resources(tst_qmimedatabase-xml)
 ## Scopes:
 #####################################################################
 
-extend_target(tst_qmimedatabase-xml CONDITION CMAKE_COMPILER_IS_GNUCXX
+extend_target(tst_qmimedatabase-xml CONDITION CMAKE_COMPILER_IS_GNUCXX # special case
     COMPILE_OPTIONS
         -W
         -Wall
@@ -66,7 +66,7 @@ extend_target(tst_qmimedatabase-xml CONDITION CMAKE_COMPILER_IS_GNUCXX
         -Wshadow
 )
 
-extend_target(tst_qmimedatabase-xml CONDITION UNIX AND NOT APPLE_OSX AND NOT QNX
+extend_target(tst_qmimedatabase-xml CONDITION UNIX AND NOT APPLE AND NOT QNX
     DEFINES
         USE_XDG_DATA_DIRS
 )

tests/auto/network/socket/platformsocketengine/CMakeLists.txt

@@ -9,7 +9,7 @@ add_qt_test(tst_platformsocketengine
     SOURCES
         tst_platformsocketengine.cpp
     INCLUDE_DIRECTORIES
-        ${PROJECT_SOURCE_DIR}/src/network
+        ${QT_SOURCE_TREE}/src/network
     LIBRARIES
         Qt::CorePrivate
         Qt::NetworkPrivate
@@ -31,7 +31,7 @@ extend_target(tst_platformsocketengine CONDITION WIN32
 
 extend_target(tst_platformsocketengine CONDITION QT_FEATURE_reduce_exports AND UNIX
     SOURCES
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qabstractsocketengine.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qabstractsocketengine.cpp
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qnativesocketengine.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qnativesocketengine.cpp
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qnativesocketengine_unix.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qnativesocketengine_unix.cpp
 )

tests/auto/network/socket/qhttpsocketengine/CMakeLists.txt

@@ -9,7 +9,7 @@ add_qt_test(tst_qhttpsocketengine
     SOURCES
         tst_qhttpsocketengine.cpp
     INCLUDE_DIRECTORIES
-        ${PROJECT_SOURCE_DIR}/src/network
+        ${QT_SOURCE_TREE}/src/network
     LIBRARIES
         Qt::CorePrivate
         Qt::NetworkPrivate
@@ -35,7 +35,7 @@ extend_target(tst_qhttpsocketengine CONDITION WIN32
 
 extend_target(tst_qhttpsocketengine CONDITION QT_FEATURE_reduce_exports AND UNIX
     SOURCES
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qabstractsocketengine.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qabstractsocketengine.cpp
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qnativesocketengine.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qnativesocketengine.cpp
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qnativesocketengine_unix.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qnativesocketengine_unix.cpp
 )

tests/auto/network/socket/qsocks5socketengine/CMakeLists.txt

@@ -9,7 +9,7 @@ add_qt_test(tst_qsocks5socketengine
     SOURCES
         tst_qsocks5socketengine.cpp
     INCLUDE_DIRECTORIES
-        ${PROJECT_SOURCE_DIR}/src/network
+        ${QT_SOURCE_TREE}/src/network
     LIBRARIES
         Qt::CorePrivate
         Qt::NetworkPrivate
@@ -35,7 +35,7 @@ extend_target(tst_qsocks5socketengine CONDITION WIN32
 
 extend_target(tst_qsocks5socketengine CONDITION QT_FEATURE_reduce_exports AND UNIX
     SOURCES
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qabstractsocketengine.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qabstractsocketengine.cpp
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qnativesocketengine.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qnativesocketengine.cpp
-        ${PROJECT_SOURCE_DIR}/src/network/socket/qnativesocketengine_unix.cpp
+        ${QT_SOURCE_TREE}/src/network/socket/qnativesocketengine_unix.cpp
 )

util/cmake/pro2cmake.py

@@ -96,11 +96,48 @@ def _parse_commandline():
     return parser.parse_args()
 
 
-def process_qrc_file(target: str, filepath: str, base_dir: str = '') -> str:
+def find_qmake_conf(project_file_path: str = '') -> typing.Optional[str]:
+    if not os.path.isabs(project_file_path):
+        print('Warning: could not find .qmake.conf file, given path is not an absolute path: {}'
+              .format(project_file_path))
+        return None
+
+    cwd = os.path.dirname(project_file_path)
+    file_name = '.qmake.conf'
+
+    while os.path.isdir(cwd):
+        maybe_file = os.path.join(cwd, file_name)
+        if os.path.isfile(maybe_file):
+            return maybe_file
+        else:
+            cwd = os.path.dirname(cwd)
+
+    return None
+
+
+def process_qrc_file(target: str, filepath: str, base_dir: str = '', project_file_path: str = '') -> str:
     assert(target)
+
+    # Hack to handle QT_SOURCE_TREE. Assume currently that it's the same
+    # as the qtbase source path.
+    qt_source_tree_literal = '${QT_SOURCE_TREE}'
+    if qt_source_tree_literal in filepath:
+        qmake_conf = find_qmake_conf(project_file_path)
+
+        if qmake_conf:
+            qt_source_tree = os.path.dirname(qmake_conf)
+            filepath = filepath.replace(qt_source_tree_literal, qt_source_tree)
+        else:
+            print('Warning, could not determine QT_SOURCE_TREE location while trying to find: {}'
+                  .format(filepath))
+
+
     resource_name = os.path.splitext(os.path.basename(filepath))[0]
     base_dir = os.path.join('' if base_dir == '.' else base_dir, os.path.dirname(filepath))
 
+    if not os.path.isfile(filepath):
+        raise RuntimeError('Invalid file path given to process_qrc_file: {}'.format(filepath))
+
     tree = ET.parse(filepath)
     root = tree.getroot()
     assert(root.tag == 'RCC')
@@ -338,7 +375,7 @@ class Scope(object):
                  file: typing.Optional[str] = None, condition: str = '',
                  base_dir: str = '',
                  operations: typing.Mapping[str, typing.List[Operation]] = {
-                    'QT_SOURCE_TREE': [SetOperation(['${PROJECT_SOURCE_DIR}'])],
+                    'QT_SOURCE_TREE': [SetOperation(['${QT_SOURCE_TREE}'])],
                     'QT_BUILD_TREE': [SetOperation(['${PROJECT_BUILD_DIR}'])],
                  }) -> None:
         if parent_scope:
@@ -357,6 +394,7 @@ class Scope(object):
         self._scope_id = Scope.SCOPE_ID
         Scope.SCOPE_ID += 1
         self._file = file
+        self._file_absolute_path = os.path.abspath(file)
         self._condition = map_condition(condition)
         self._children = []  # type: typing.List[Scope]
         self._included_children = []  # type: typing.List[Scope]
@@ -494,6 +532,10 @@ class Scope(object):
     def file(self) -> str:
         return self._file or ''
 
+    @property
+    def file_absolute_path(self) -> str:
+        return self._file_absolute_path or ''
+
     @property
     def generated_cmake_lists_path(self) -> str:
         assert self.basedir
@@ -1464,7 +1506,7 @@ def write_resources(cm_fh: typing.IO[str], target: str, scope: Scope, indent: in
         qrc_only = True
         for r in resources:
             if r.endswith('.qrc'):
-                qrc_output += process_qrc_file(target, r, scope.basedir)
+                qrc_output += process_qrc_file(target, r, scope.basedir, scope.file_absolute_path)
             else:
                 qrc_only = False