MDEV-30831 Cannot compile AWS KMS Plugin

Fix AWS SDK build, it has changed substantionally since the plugin was introduced. There is now a bunch of intermediate C libraries, aws-cpp-crt and others, and for static linking, the link dependency must be declared. Also support AWS C++ SDK in vcpkg package manager.
2025-06-06 16:26:27 +02:00 · 2025-06-06 16:26:27 +02:00 · dd2982dc33
commit dd2982dc33
parent 5729d89cd5
4 changed files with 125 additions and 67 deletions
--- a/cmake/aws_sdk.cmake
+++ b/cmake/aws_sdk.cmake
@ -4,19 +4,27 @@ MACRO (SKIP_AWS_SDK MSG)
  RETURN()
 ENDMACRO()

-FUNCTION (CHECK_AWS_SDK RETVAL REASON)
+FUNCTION (CHECK_AWS_SDK COMPONENTS RETVAL REASON)
  # AWS_SDK_EXTERNAL_PROJECT must be ON
  IF(NOT AWS_SDK_EXTERNAL_PROJECT)
-    SKIP_AWS_SDK("AWS_SDK_EXTERNAL_PROJECT is not ON")
+   FOREACH(comp ${COMPONENTS})
+    FIND_PACKAGE(aws-cpp-sdk-${comp} CONFIG QUIET)
+    IF (NOT aws-cpp-sdk-${comp}_FOUND)
+      SKIP_AWS_SDK("AWS_SDK_EXTERNAL_PROJECT is not ON and aws-cpp-sdk-${comp} not found")
+    ENDIF()
+   ENDFOREACH()
+   SET(${RETVAL} ON PARENT_SCOPE)
  ENDIF()
  IF(NOT NOT_FOR_DISTRIBUTION)
    SKIP_AWS_SDK("AWS SDK has Apache 2.0 License which is not compatible with GPLv2. Set -DNOT_FOR_DISTRIBUTION=ON if you need it")
  ENDIF()
+  IF(CMAKE_VERSION VERSION_LESS "3.15")
+    SKIP_AWS_SDK("CMake too old")
+  ENDIF()
  # Check compiler support
  IF(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
-    EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION)
-    IF (GCC_VERSION VERSION_LESS 4.8)
-      SKIP_AWS_SDK("GCC VERSION too old (${GCC_VERSION}, required is 4.8 or later")
+    IF (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9)
+      SKIP_AWS_SDK("GCC VERSION too old (${GCC_VERSION}, required is 4.9 or later")
    ENDIF()
  ELSEIF (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
    IF ((CMAKE_CXX_COMPILER_VERSION AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS  3.3) OR
@ -36,35 +44,27 @@ FUNCTION (CHECK_AWS_SDK RETVAL REASON)
    SKIP_AWS_SDK("OS unsupported by AWS SDK")
  ENDIF()

-  # Build from source, using ExternalProject_Add
-  # AWS C++ SDK requires cmake 2.8.12
-  IF(CMAKE_VERSION VERSION_LESS "2.8.12")
-    SKIP_AWS_SDK("CMake is too old")
-  ENDIF()
-
  IF(UNIX)
-    # Check librairies required for building SDK
-    FIND_PACKAGE(CURL)
-    SET_PACKAGE_PROPERTIES(CURL PROPERTIES TYPE REQUIRED)
-    IF(NOT CURL_FOUND)
-      SKIP_AWS_SDK("AWS C++ SDK requires libcurl development package")
+    IF("${WITH_ZLIB}" STREQUAL "bundled")
+      # Breaks FIND_PACKAGE(ZLIB)
+      SKIP_AWS_SDK("Incompatible with WITH_ZLIB=bundled")
    ENDIF()
-    FIND_PATH(UUID_INCLUDE_DIR uuid/uuid.h)
-    IF(NOT UUID_INCLUDE_DIR)
-      SKIP_AWS_SDK("AWS C++ SDK requires uuid development package")
-    ENDIF()
-    IF(NOT APPLE)
-      FIND_LIBRARY(UUID_LIBRARIES uuid)
-      SET_PACKAGE_PROPERTIES(UUID_LIBRARIES PROPERTIES TYPE REQUIRED)
-      IF(NOT UUID_LIBRARIES)
-        SKIP_AWS_SDK("AWS C++ SDK requires uuid development package")
+    # Check libraries required for building SDK
+    FOREACH(pkg CURL ZLIB OpenSSL)
+      FIND_PACKAGE(${pkg})
+      IF(NOT ${pkg}_FOUND)
+        SKIP_AWS_SDK("AWS C++ SDK requires ${pkg}")
      ENDIF()
-      FIND_PACKAGE(OpenSSL)
-      SET_PACKAGE_PROPERTIES(OpenSSL PROPERTIES TYPE REQUIRED)
-      IF(NOT OPENSSL_FOUND)
-        SKIP_AWS_SDK("AWS C++ SDK requires openssl development package")
+      SET_PACKAGE_PROPERTIES(${pkg} PROPERTIES TYPE REQUIRED)
+    ENDFOREACH()
+    # Also check for required libraries explicitely - they might be
+    # missing, even if check above succeeds, e.g when using own copy
+    # of zlib
+    FOREACH(lib OpenSSL::Crypto ZLIB::ZLIB CURL::libcurl)
+      IF(NOT TARGET ${lib})
+        SKIP_AWS_SDK("AWS C++ SDK requires ${lib}")
      ENDIF()
-    ENDIF()
+    ENDFOREACH()
  ENDIF()
  SET(${RETVAL} ON PARENT_SCOPE)
 ENDFUNCTION()
@ -85,14 +85,4 @@ FUNCTION(USE_AWS_SDK_LIBS)
    SET_PROPERTY(GLOBAL PROPERTY AWS_SDK_LIBS ${comp} APPEND)
    TARGET_LINK_LIBRARIES(${target} aws-cpp-sdk-${comp})
  ENDFOREACH()
-  TARGET_LINK_LIBRARIES(${target} aws-cpp-sdk-core)
-  TARGET_INCLUDE_DIRECTORIES(${target} PRIVATE ${PROJECT_BINARY_DIR}/extra/aws_sdk/aws_sdk_cpp/include)
-  # Link OS libraries that AWS SDK depends on
-  IF(WIN32)
-    TARGET_LINK_LIBRARIES(${target} bcrypt winhttp wininet userenv version)
-  ELSE()
-    FIND_PACKAGE(CURL REQUIRED)
-    FIND_PACKAGE(OpenSSL REQUIRED)
-    TARGET_LINK_LIBRARIES(${target} ${OPENSSL_LIBRARIES} ${CURL_LIBRARIES} ${UUID_LIBRARIES})
-  ENDIF()
 ENDFUNCTION()
--- a/extra/aws_sdk/CMakeLists.txt
+++ b/extra/aws_sdk/CMakeLists.txt
@ -8,19 +8,46 @@ IF(SDK_LIBS_COUNT EQUAL 0)
  RETURN()
 ENDIF()

-CHECK_AWS_SDK(RETVAL REASON)
+CHECK_AWS_SDK("${SDK_LIBS}" RETVAL REASON)
 IF(NOT RETVAL)
  MESSAGE(FATAL_ERROR 
   "AWS C++ will not be built (${REASON}), but dependency on following components is found ${SDK_LIBS}.
   Use CHECK_AWS_SDK() function before trying to build with SDK components")
 ENDIF()
 
+IF(NOT AWS_SDK_EXTERNAL_PROJECT)
+  # System aws sdk libraries found
+  RETURN()
+ENDIF()
+
+SET(all_libs
+  aws-crt-cpp
+  aws-c-event-stream
+  aws-c-common
+  aws-c-http
+  aws-c-mqtt
+  aws-c-cal
+  aws-c-auth
+  aws-c-io
+  aws-checksums
+  aws-c-s3
+  aws-c-sdkutils
+  aws-c-compression
+  aws-cpp-sdk-core
+)
+IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  LIST(APPEND all_libs s2n)
+ENDIF()
+
+FOREACH(lib ${SDK_LIBS})
+  LIST(APPEND all_libs aws-cpp-sdk-${lib})
+ENDFOREACH()
+LIST(REMOVE_DUPLICATES all_libs)

 SET(byproducts)
-
-FOREACH(lib ${SDK_LIBS} core)
-  SET(lib aws-cpp-sdk-${lib})
-  ADD_LIBRARY(${lib} STATIC IMPORTED GLOBAL)
+MAKE_DIRECTORY(${PROJECT_BINARY_DIR}/extra/aws_sdk/aws_sdk_cpp/include)
+FOREACH(lib ${all_libs})
+  ADD_LIBRARY(${lib} UNKNOWN IMPORTED GLOBAL)
  ADD_DEPENDENCIES(${lib} aws_sdk_cpp)
  
  SET (loc "${CMAKE_CURRENT_BINARY_DIR}/aws_sdk_cpp/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${lib}${CMAKE_STATIC_LIBRARY_SUFFIX}")
@ -28,47 +55,85 @@ FOREACH(lib ${SDK_LIBS} core)
    SET(byproducts ${byproducts} BUILD_BYPRODUCTS ${loc})
  ENDIF()
  SET_TARGET_PROPERTIES(${lib} PROPERTIES IMPORTED_LOCATION ${loc})
+  TARGET_INCLUDE_DIRECTORIES(${lib} INTERFACE ${PROJECT_BINARY_DIR}/extra/aws_sdk/aws_sdk_cpp/include)
 ENDFOREACH()

-# To be compatible with older cmake, we use older version of the SDK
-IF(CMAKE_VERSION LESS "3.0")
-  SET(GIT_TAG "1.0.8")
+# Define link dependencies  between AWS C libs
+TARGET_LINK_LIBRARIES(aws-cpp-sdk-core INTERFACE aws-crt-cpp aws-c-event-stream aws-c-common)
+TARGET_LINK_LIBRARIES(aws-crt-cpp INTERFACE aws-c-http aws-c-mqtt aws-c-cal aws-c-auth aws-c-common aws-c-io
+  aws-checksums aws-c-event-stream aws-c-s3 aws-c-sdkutils)
+TARGET_LINK_LIBRARIES(aws-c-event-stream INTERFACE  aws-c-io aws-c-common aws-checksums)
+TARGET_LINK_LIBRARIES(aws-c-http INTERFACE aws-c-io aws-c-compression aws-c-cal aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-mqtt INTERFACE aws-c-http aws-c-io aws-c-common aws-c-cal aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-auth INTERFACE aws-c-sdkutils  aws-c-cal  aws-c-http  aws-c-io  aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-io INTERFACE aws-c-common  aws-c-cal)
+TARGET_LINK_LIBRARIES(aws-checksums INTERFACE aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-compression INTERFACE aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-s3 INTERFACE aws-c-auth  aws-checksums aws-c-http aws-c-io aws-c-cal aws-c-common aws-c-sdkutils  aws-c-common)
+TARGET_LINK_LIBRARIES(aws-c-sdkutils INTERFACE aws-c-common)
+
+IF(WIN32)
+  TARGET_LINK_LIBRARIES(aws-c-io INTERFACE bcrypt crypt32 secur32 ncrypt ws2_32)
+  TARGET_LINK_LIBRARIES(aws-c-common INTERFACE  shlwapi)
+  TARGET_LINK_LIBRARIES(aws-cpp-sdk-core INTERFACE winhttp wininet version userenv)
 ELSE()
-  SET(GIT_TAG "1.8.29")
+  TARGET_LINK_LIBRARIES(aws-c-cal INTERFACE OpenSSL::Crypto)
+  TARGET_LINK_LIBRARIES(aws-cpp-sdk-core INTERFACE ZLIB::ZLIB CURL::libcurl)
+
+  # Dependencies below are from CMakeLists.txt for aws-c-common
+  SET(THREADS_PREFER_PTHREAD_FLAG ON)
+  FIND_PACKAGE(Threads REQUIRED)
+  TARGET_LINK_LIBRARIES(aws-c-common INTERFACE ${CMAKE_DL_LIBS} Threads::Threads)
+  IF(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    TARGET_LINK_LIBRARIES(aws-c-common INTERFACE s2n m rt)
+  ELSEIF(APPLE)
+    TARGET_LINK_LIBRARIES(aws-c-common INTERFACE "-framework CoreFoundation")
+    TARGET_LINK_LIBRARIES(aws-c-io INTERFACE "-framework Security" "-framework Network")
+  ENDIF()
 ENDIF()

-IF(MSVC_CRT_TYPE MATCHES "/MD")
-  SET(FORCE_SHARED_CRT ON)
-ELSE()
-  SET(FORCE_SHARED_CRT OFF)
-ENDIF()
+FOREACH(lib ${SDK_LIBS})
+  TARGET_LINK_LIBRARIES(aws-cpp-sdk-${lib} INTERFACE aws-cpp-sdk-core)
+ENDFOREACH()
+
+SET(GIT_TAG "1.11.582")

 LIST(REMOVE_DUPLICATES SDK_LIBS)
 STRING( REPLACE ";" "!" SDK_LIBS_STR "${SDK_LIBS}")
-#MESSAGE("SDK_LIBS_STR=${SDK_LIBS_STR}")
+
+IF(MSVC)
+ SET(BUILD_AND_INSTALL_COMMANDS
+   BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --config RelWithDebInfo --verbose --parallel
+   INSTALL_COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR> --config RelWithDebInfo)
+ SET(BUILD_TYPE -DCMAKE_BUILD_TYPE=RelWithDebInfo)
+ELSE()
+ SET(BUILD_AND_INSTALL_COMMANDS)
+ SET(BUILD_TYPE)
+ENDIF()

 ExternalProject_Add(
  aws_sdk_cpp
  GIT_REPOSITORY "https://github.com/awslabs/aws-sdk-cpp.git"
  GIT_TAG ${GIT_TAG}
+  GIT_SHALLOW TRUE
  UPDATE_COMMAND ""
  SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-cpp"
  LIST_SEPARATOR !
  ${byproducts}
  CMAKE_ARGS
+    ${BUILD_TYPE}
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
    -DBUILD_ONLY=${SDK_LIBS_STR}
    -DBUILD_SHARED_LIBS=OFF
-    -DFORCE_SHARED_CRT=${FORCE_SHARED_CRT}
+    -DFORCE_SHARED_CRT=ON
    -DENABLE_TESTING=OFF
-    "-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} ${PIC_FLAG}"
-    "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${PIC_FLAG}"
-    "-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} ${PIC_FLAG}"
-    "-DCMAKE_CXX_FLAGS_MINSIZEREL=${CMAKE_CXX_FLAGS_MINSIZEREL} ${PIC_FLAG}"
-    "-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
-    "-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
+    -DAWS_WARNINGS_ARE_ERRORS=OFF
    ${EXTRA_SDK_CMAKE_FLAGS}
    -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/aws_sdk_cpp
    -DCMAKE_INSTALL_LIBDIR=lib
-    TEST_COMMAND ""
+  TEST_COMMAND ""
+  ${BUILD_AND_INSTALL_COMMANDS}
 )
+
 SET_TARGET_PROPERTIES(aws_sdk_cpp PROPERTIES EXCLUDE_FROM_ALL TRUE)
--- a/plugin/aws_key_management/CMakeLists.txt
+++ b/plugin/aws_key_management/CMakeLists.txt
@ -1,7 +1,10 @@
 INCLUDE(aws_sdk)
-CHECK_AWS_SDK(HAVE_AWS_SDK REASON)
+CHECK_AWS_SDK("kms;core" HAVE_AWS_SDK REASON)
 IF(NOT HAVE_AWS_SDK)
  MESSAGE_ONCE(AWS_KEY_MANAGEMENT_NO_AWS_SDK "Can't build aws_key_management - AWS SDK not available (${REASON})")
+  IF(PLUGIN_AWS_KEY_MANAGEMENT MATCHES "^(STATIC|DYNAMIC)$")
+    MESSAGE(FATAL_ERROR "Can't build reqired plugin aws_key_management: ${REASON}")
+  ENDIF()
  ADD_FEATURE_INFO(AWS_KEY_MANAGEMENT "OFF" "AWS Encryption Key Management Plugin")
  RETURN()
 ENDIF()
@ -11,7 +14,7 @@ MYSQL_ADD_PLUGIN(aws_key_management
  COMPONENT aws-key-management)

 IF(TARGET aws_key_management)
- USE_AWS_SDK_LIBS(aws_key_management kms)
+  USE_AWS_SDK_LIBS(aws_key_management kms core)
+  ADD_FEATURE_INFO(AWS_KEY_MANAGEMENT "ON" "AWS Encryption Key Management Plugin")
 ENDIF()

-ADD_FEATURE_INFO(AWS_KEY_MANAGEMENT "ON" "AWS Encryption Key Management Plugin")
--- a/plugin/aws_key_management/aws_key_management_plugin.cc
+++ b/plugin/aws_key_management/aws_key_management_plugin.cc
@ -540,7 +540,7 @@ static int generate_and_save_datakey(uint keyid, uint version)
    return(-1);
  }
  unsigned int len= (unsigned int)byteBuffer.GetLength();
-  if (write(fd, byteBuffer.GetUnderlyingData(), len) != len)
+  if ((unsigned int)write(fd, byteBuffer.GetUnderlyingData(), len) != len)
  {
    my_printf_error(ER_UNKNOWN_ERROR, "AWS KMS plugin: can't write to %s", ME_ERROR_LOG_ONLY, filename);
    close(fd);