1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CMake: Rework SBOM python dependency lookup and handling

Browse Source 
Introduce a bunch of helper functions to split out the finding of a
specific python interpreter, a specific python dependency within a
given python interpreter, and the handling of dependencies for each of
the possible SBOM operations: GENERATE_JSON, VERIFY_SBOM, and
RUN_NTIA.

In a future change this will allow us to conditionally enable certain
operations, depending on which dependencies are available, without
failing the build outright.

One behavior change is better logging of what goes wrong if a
dependency is missing.

Another is the double look up of python on macOS, to account for the
system framework python not having the required dependencies.
These was previously an internal opt in, but now it is done
automatically.

Pick-to: 6.8
Task-number: QTBUG-122899
Change-Id: I3054f2649c0092dc5f2d3e299065f0f62dc6c5fb
Reviewed-by: Joerg Bornemann <joerg.bornemann@qt.io>
This commit is contained in:
Alexandru Croitor 2024-10-16 15:21:34 +02:00
parent 08d5d9e9da
commit 29a435faf8
1 changed files with 288 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

338
cmake/QtPublicSbomGenerationHelpers.cmake
View File

@ -256,16 +256,14 @@ function(_qt_internal_sbom_end_project_generate)
_qt_internal_get_staging_area_spdx_file_path(staging_area_spdx_file)
if((arg_GENERATE_JSON OR arg_VERIFY) AND NOT QT_INTERNAL_NO_SBOM_PYTHON_OPS)
_qt_internal_sbom_find_python()
_qt_internal_sbom_find_python_dependencies()
endif()
if(arg_GENERATE_JSON AND NOT QT_INTERNAL_NO_SBOM_PYTHON_OPS)
_qt_internal_sbom_find_and_handle_sbom_op_dependencies(REQUIRED OP_KEY "GENERATE_JSON")
_qt_internal_sbom_generate_json()
endif()
if(arg_VERIFY AND NOT QT_INTERNAL_NO_SBOM_PYTHON_OPS)
_qt_internal_sbom_find_and_handle_sbom_op_dependencies(REQUIRED OP_KEY "VERIFY_SBOM")
_qt_internal_sbom_find_and_handle_sbom_op_dependencies(REQUIRED OP_KEY "RUN_NTIA")
_qt_internal_sbom_verify_valid_and_ntia_compliant()
endif()
@ -978,82 +976,311 @@ function(_qt_internal_sbom_get_and_check_spdx_id)
set(${arg_VARIABLE} "${id}" PARENT_SCOPE)
endfunction()
# Helper to find the python interpreter, to be able to run post-installation steps like NTIA
# verification.
# Helper to find a python interpreter and a specific python dependency, e.g. to be able to generate
# a SPDX JSON SBOM, or run post-installation steps like NTIA verification.
# The exact dependency should be specified as the OP_KEY.
#
# Caches the found python executable in a separate cache var QT_INTERNAL_SBOM_PYTHON_EXECUTABLE, to
# avoid conflicts with any other found python package.
# avoid conflicts with any other found python interpreter.
function(_qt_internal_sbom_find_and_handle_sbom_op_dependencies)
set(opt_args
REQUIRED
)
set(single_args
OP_KEY
OUT_VAR_DEPS_FOUND
)
set(multi_args "")
cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}")
_qt_internal_validate_all_args_are_parsed(arg)
if(NOT arg_OP_KEY)
message(FATAL_ERROR "OP_KEY is required")
endif()
set(supported_ops "GENERATE_JSON" "VERIFY_SBOM" "RUN_NTIA")
if(arg_OP_KEY STREQUAL "GENERATE_JSON" OR arg_OP_KEY STREQUAL "VERIFY_SBOM")
set(import_statement "import spdx_tools.spdx.clitools.pyspdxtools")
elseif(arg_OP_KEY STREQUAL "RUN_NTIA")
set(import_statement "import ntia_conformance_checker.main")
else()
message(FATAL_ERROR "OP_KEY must be one of ${supported_ops}")
endif()
# Return early if we found the dependencies.
if(QT_INTERNAL_SBOM_DEPS_FOUND_FOR_${arg_OP_KEY})
if(arg_OUT_VAR_DEPS_FOUND)
set(${arg_OUT_VAR_DEPS_FOUND} TRUE PARENT_SCOPE)
endif()
return()
endif()
# NTIA-compliance checker requires Python 3.9 or later, so we use it as the minimum for all
# SBOM OPs.
set(required_version "3.9")
set(python_common_args
VERSION "${required_version}"
)
set(everything_found FALSE)
# On macOS FindPython prefers looking in the system framework location, but that usually would
# not have the required dependencies. So we first look in it, and then fallback to any other
# non-framework python found.
if(CMAKE_HOST_APPLE)
set(extra_python_args SEARCH_IN_FRAMEWORKS QUIET)
_qt_internal_sbom_find_python_and_dependency_helper_lambda()
endif()
if(NOT everything_found)
set(extra_python_args QUIET)
_qt_internal_sbom_find_python_and_dependency_helper_lambda()
endif()
if(NOT everything_found)
if(arg_REQUIRED)
set(message_type "FATAL_ERROR")
else()
set(message_type "DEBUG")
endif()
if(NOT python_found)
# Look for python one more time, this time without QUIET, to show an error why it
# wasn't found.
if(arg_REQUIRED)
_qt_internal_sbom_find_python_helper(${python_common_args}
OUT_VAR_PYTHON_PATH unused_python
OUT_VAR_PYTHON_FOUND unused_found
)
endif()
message(${message_type} "Python ${required_version} for running SBOM ops not found.")
elseif(NOT dep_found)
message(${message_type} "Python dependency for running SBOM op ${arg_OP_KEY} "
"not found:\n Python: ${python_path} \n Output: \n${dep_find_output}")
endif()
else()
message(DEBUG "Using Python ${python_path} for running SBOM ops.")
if(NOT QT_INTERNAL_SBOM_PYTHON_EXECUTABLE)
set(QT_INTERNAL_SBOM_PYTHON_EXECUTABLE "${python_path}" CACHE INTERNAL
"Python interpeter used for SBOM generation.")
endif()
set(QT_INTERNAL_SBOM_DEPS_FOUND_FOR_${arg_OP_KEY} "TRUE" CACHE INTERNAL
"All dependencies found to run SBOM OP ${arg_OP_KEY}")
endif()
if(arg_OUT_VAR_DEPS_FOUND)
set(${arg_OUT_VAR_DEPS_FOUND} "${QT_INTERNAL_SBOM_DEPS_FOUND_FOR_${arg_OP_KEY}}"
PARENT_SCOPE)
endif()
endfunction()
# Helper macro to find python and a given dependency. Expects the caller to set all of the vars.
# Meant to reduce the line noise due to the repeated calls.
macro(_qt_internal_sbom_find_python_and_dependency_helper_lambda)
_qt_internal_sbom_find_python_and_dependency_helper(
PYTHON_ARGS
${extra_python_args}
${python_common_args}
DEPENDENCY_ARGS
DEPENDENCY_IMPORT_STATEMENT "${import_statement}"
OUT_VAR_PYTHON_PATH python_path
OUT_VAR_PYTHON_FOUND python_found
OUT_VAR_DEP_FOUND dep_found
OUT_VAR_PYTHON_AND_DEP_FOUND everything_found
OUT_VAR_DEP_FIND_OUTPUT dep_find_output
)
endmacro()
# Tries to find python and a given dependency based on the args passed to PYTHON_ARGS and
# DEPENDENCY_ARGS which are forwarded to the respective finding functions.
# Returns the path to the python interpreter, whether it was found, whether the dependency was
# found, whether both were found, and the reason why the dependency might not be found.
function(_qt_internal_sbom_find_python_and_dependency_helper)
set(opt_args)
set(single_args
OUT_VAR_PYTHON_PATH
OUT_VAR_PYTHON_FOUND
OUT_VAR_DEP_FOUND
OUT_VAR_PYTHON_AND_DEP_FOUND
OUT_VAR_DEP_FIND_OUTPUT
)
set(multi_args
PYTHON_ARGS
DEPENDENCY_ARGS
)
cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}")
_qt_internal_validate_all_args_are_parsed(arg)
set(everything_found_inner FALSE)
set(deps_find_output_inner "")
if(NOT arg_OUT_VAR_PYTHON_PATH)
message(FATAL_ERROR "OUT_VAR_PYTHON_PATH var is required")
endif()
if(NOT arg_OUT_VAR_PYTHON_FOUND)
message(FATAL_ERROR "OUT_VAR_PYTHON_FOUND var is required")
endif()
if(NOT arg_OUT_VAR_DEP_FOUND)
message(FATAL_ERROR "OUT_VAR_DEP_FOUND var is required")
endif()
if(NOT arg_OUT_VAR_PYTHON_AND_DEP_FOUND)
message(FATAL_ERROR "OUT_VAR_PYTHON_AND_DEP_FOUND var is required")
endif()
if(NOT arg_OUT_VAR_DEP_FIND_OUTPUT)
message(FATAL_ERROR "OUT_VAR_DEP_FIND_OUTPUT var is required")
endif()
_qt_internal_sbom_find_python_helper(
${arg_PYTHON_ARGS}
OUT_VAR_PYTHON_PATH python_path_inner
OUT_VAR_PYTHON_FOUND python_found_inner
)
if(python_found_inner AND python_path_inner)
_qt_internal_sbom_find_python_dependency_helper(
${arg_DEPENDENCY_ARGS}
PYTHON_PATH "${python_path_inner}"
OUT_VAR_FOUND dep_found_inner
OUT_VAR_OUTPUT dep_find_output_inner
)
if(dep_found_inner)
set(everything_found_inner TRUE)
endif()
endif()
set(${arg_OUT_VAR_PYTHON_PATH} "${python_path_inner}" PARENT_SCOPE)
set(${arg_OUT_VAR_PYTHON_FOUND} "${python_found_inner}" PARENT_SCOPE)
set(${arg_OUT_VAR_DEP_FOUND} "${dep_found_inner}" PARENT_SCOPE)
set(${arg_OUT_VAR_PYTHON_AND_DEP_FOUND} "${everything_found_inner}" PARENT_SCOPE)
set(${arg_OUT_VAR_DEP_FIND_OUTPUT} "${dep_find_output_inner}" PARENT_SCOPE)
endfunction()
# Tries to find the python intrepreter, given the QT_SBOM_PYTHON_INTERP path hint, as well as
# other options.
# Ignores any previously found python.
# Returns the python interpreter path and whether it was successfully found.
#
# This is intentionally a function, and not a macro, to prevent overriding the Python3_EXECUTABLE
# non-cache variable in a global scope in case if a different python is found and used for a
# different purpose (e.g. qtwebengine or qtinterfaceframework).
# The reason to use a different python is that an already found python might not be the version we
# need.
# need, or might lack the dependencies we need.
# https://gitlab.kitware.com/cmake/cmake/-/issues/21797#note_901621 claims that finding multiple
# python versions in separate directory scopes is possible, and I claim a function scope is as
# good as a directory scope.
function(_qt_internal_sbom_find_python)
# Return early if we found a suitable python.
if(QT_INTERNAL_SBOM_PYTHON_EXECUTABLE)
return()
function(_qt_internal_sbom_find_python_helper)
set(opt_args
SEARCH_IN_FRAMEWORKS
QUIET
)
set(single_args
VERSION
OUT_VAR_PYTHON_PATH
OUT_VAR_PYTHON_FOUND
)
set(multi_args "")
cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}")
_qt_internal_validate_all_args_are_parsed(arg)
if(NOT arg_OUT_VAR_PYTHON_PATH)
message(FATAL_ERROR "OUT_VAR_PYTHON_PATH var is required")
endif()
if(NOT arg_OUT_VAR_PYTHON_FOUND)
message(FATAL_ERROR "OUT_VAR_PYTHON_FOUND var is required")
endif()
# Allow disabling looking for a python interpreter shipped as part of a macOS system framework.
if(QT_INTERNAL_NO_SBOM_FIND_PYTHON_FRAMEWORK)
set(Python_FIND_FRAMEWORK NEVER)
if(NOT arg_SEARCH_IN_FRAMEWORKS)
set(Python3_FIND_FRAMEWORK NEVER)
endif()
# NTIA-compliance checker requires Python 3.9 or later.
set(required_version "3.9")
# Python3_VERSION would have been set by a previous find_package(Python3) call.
set(already_found_python_version "${Python3_VERSION}")
if(NOT already_found_python_version
OR "${already_found_python_version}" VERSION_LESS "${required_version}")
# Locally reset any executable that was possibly already found and is a lower version.
# We do this to ensure we re-do the lookup, rather than error out saying a low version was
# found.
set(Python3_EXECUTABLE "")
if(QT_SBOM_PYTHON_INTERP)
set(Python3_ROOT_DIR ${QT_SBOM_PYTHON_INTERP})
endif()
find_package(Python3 ${required_version} REQUIRED COMPONENTS Interpreter)
# We won't get here unless a version was found, because of the REQUIRED.
set(QT_INTERNAL_SBOM_PYTHON_EXECUTABLE "${Python3_EXECUTABLE}" CACHE STRING
"Python interpeter used for SBOM steps")
set(required_version "")
if(arg_VERSION)
set(required_version "${arg_VERSION}")
endif()
set(find_quiet "")
if(arg_QUIET)
set(find_quiet "QUIET")
endif()
# Locally reset any executable that was possibly already found.
# We do this to ensure we always re-do the lookup/
# This needs to be set to an empty string, to override any cache variable
set(Python3_EXECUTABLE "")
# This needs to be unset, because the Python module checks whether the variable is defined, not
# whether it is empty.
unset(_Python3_EXECUTABLE)
if(QT_SBOM_PYTHON_INTERP)
set(Python3_ROOT_DIR ${QT_SBOM_PYTHON_INTERP})
endif()
find_package(Python3 ${required_version} ${find_quiet} COMPONENTS Interpreter)
set(${arg_OUT_VAR_PYTHON_PATH} "${Python3_EXECUTABLE}" PARENT_SCOPE)
set(${arg_OUT_VAR_PYTHON_FOUND} "${Python3_Interpreter_FOUND}" PARENT_SCOPE)
endfunction()
# Helper to find the various python package dependencies needed to run the post-installation NTIA
# verification and the spdx format validation step.
function(_qt_internal_sbom_find_python_dependencies)
if(NOT QT_INTERNAL_SBOM_PYTHON_EXECUTABLE)
message(FATAL_ERROR "Python interpreter not found for sbom dependencies.")
# Helper that takes an python import statement to run using the given python interpreter path,
# to confirm that the given python dependency can be found.
# Returns whether the dependency was found and the output of running the import, for error handling.
function(_qt_internal_sbom_find_python_dependency_helper)
set(opt_args "")
set(single_args
DEPENDENCY_IMPORT_STATEMENT
PYTHON_PATH
OUT_VAR_FOUND
OUT_VAR_OUTPUT
)
set(multi_args "")
cmake_parse_arguments(PARSE_ARGV 0 arg "${opt_args}" "${single_args}" "${multi_args}")
_qt_internal_validate_all_args_are_parsed(arg)
if(NOT arg_PYTHON_PATH)
message(FATAL_ERROR "Python interpreter path not given.")
endif()
if(QT_SBOM_HAVE_PYTHON_DEPS)
return()
if(NOT arg_DEPENDENCY_IMPORT_STATEMENT)
message(FATAL_ERROR "Python depdendency import statement not given.")
endif()
if(NOT arg_OUT_VAR_FOUND)
message(FATAL_ERROR "Out var found variable not given.")
endif()
set(python_path "${arg_PYTHON_PATH}")
execute_process(
COMMAND
${QT_INTERNAL_SBOM_PYTHON_EXECUTABLE} -c "
import spdx_tools.spdx.clitools.pyspdxtools
import ntia_conformance_checker.main
"
${python_path} -c "${arg_DEPENDENCY_IMPORT_STATEMENT}"
RESULT_VARIABLE res
OUTPUT_VARIABLE output
ERROR_VARIABLE output
)
if("${res}" STREQUAL "0")
set(QT_SBOM_HAVE_PYTHON_DEPS TRUE CACHE INTERNAL "")
set(found TRUE)
set(output "${output}")
else()
message(FATAL_ERROR "SBOM Python dependencies not found. Error:\n${output}")
set(found FALSE)
string(CONCAT output "SBOM Python dependency ${arg_DEPENDENCY_IMPORT_STATEMENT} not found. "
"Error:\n${output}")
endif()
set(${arg_OUT_VAR_FOUND} "${found}" PARENT_SCOPE)
if(arg_OUT_VAR_OUTPUT)
set(${arg_OUT_VAR_OUTPUT} "${output}" PARENT_SCOPE)
endif()
endfunction()
@ -1105,6 +1332,9 @@ function(_qt_internal_sbom_generate_json)
if(NOT QT_INTERNAL_SBOM_PYTHON_EXECUTABLE)
message(FATAL_ERROR "Python interpreter not found for generating SBOM json file.")
endif()
if(NOT QT_INTERNAL_SBOM_DEPS_FOUND_FOR_GENERATE_JSON)
message(FATAL_ERROR "Python dependencies not found for generating SBOM json file.")
endif()
set(content "
message(STATUS \"Generating JSON: \${QT_SBOM_OUTPUT_PATH}.json\")
@ -1131,6 +1361,14 @@ function(_qt_internal_sbom_verify_valid_and_ntia_compliant)
message(FATAL_ERROR "Python interpreter not found for verifying SBOM file.")
endif()
if(NOT QT_INTERNAL_SBOM_DEPS_FOUND_FOR_VERIFY_SBOM)
message(FATAL_ERROR "Python dependencies not found for verifying SBOM file")
endif()
if(NOT QT_INTERNAL_SBOM_DEPS_FOUND_FOR_RUN_NTIA)
message(FATAL_ERROR "Python dependencies not found for running the SBOM NTIA checker.")
endif()
set(content "
message(STATUS \"Verifying: \${QT_SBOM_OUTPUT_PATH}\")
execute_process(