1berry/qtbase
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support multi-stage builds to provision the configurations and test data

Browse Source 
In order to reuse the test server to the external modules, it is much
easier to share the common configurations (scripts) and test data via
Dockerfile. In addition, the external module can create more layers
depending on their needs. Therefore, supporting multi-stage builds is
needed. The disadvantage is that the docker-compose needs to re-build
the images every time. However, it is just a one-time effort. If the
Dockerfile doesn't get changed, the extra build time can be ignored.

Because of multi-stage builds, the test server will keep a Dockerfile at
least. Therefore, the volume sharing is no more needed. The test data of
a service can be added into the images by using COPY/ADD commands.

NOTE:
This patch relies on docker-compose v1.21.0 (docker-compose build now
supports the use of Dockerfile from outside the build context).

Change-Id: Ib3f6a5fcf6979732ae8a40a494a1360fca4ac7bf
Reviewed-by: Maurice Kalinowski <maurice.kalinowski@qt.io>
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>
This commit is contained in:
Ryan Chu 2019-03-22 16:20:52 +01:00
parent 0c404fd6f3
commit 713f77176e
7 changed files with 185 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
tests/auto/Dockerfile Normal file
View File

@ -0,0 +1,27 @@
# This Dockerfile is used to provision the shared scripts (e.g. startup.sh) and configurations. It
# relies on the arguments passed by docker-compose file to build additional images for each service.
# To lean more how it works, please check the topic "Use multi-stage builds".
# https://docs.docker.com/develop/develop-images/multistage-build/
ARG provisioningImage
FROM $provisioningImage as testserver_tier2
# Add and merge the testdata into service folder
ARG serviceDir
ARG shareDir=$serviceDir
COPY $serviceDir $shareDir service/
# create the shared script of testserver
RUN echo "#!/usr/bin/env bash\n" \
"set -ex\n" \
"for RUN_CMD; do \$RUN_CMD; done\n" \
"service dbus restart\n" \
"service avahi-daemon restart\n" \
"sleep infinity\n" > startup.sh
RUN chmod +x startup.sh
# rewrite the default configurations of avahi-daemon
ARG test_domain
RUN sed -i -e "s,#domain-name=local,domain-name=${test_domain:-test-net.qt.local}," \
-e "s,#publish-aaaa-on-ipv4=yes,publish-aaaa-on-ipv4=no," \
/etc/avahi/avahi-daemon.conf

5
tests/auto/testserver.pri
View File

@ -102,6 +102,7 @@ isEmpty(TESTSERVER_VERSION) {
# The environment variables passed to the docker-compose file
TEST_ENV = 'MACHINE_IP=$(shell docker-machine ip qt-test-server)'
TEST_ENV += 'TEST_DOMAIN=$$DNSDOMAIN'
TEST_ENV += 'SHARED_DATA=$$PWD'
TEST_CMD = env
} else:equals(QMAKE_HOST.os, Windows) {
# There is no docker bridge on Windows. It is impossible to ping a container.
@ -115,6 +116,7 @@ isEmpty(TESTSERVER_VERSION) {
# The environment variables passed to the docker-compose file
TEST_ENV = '\$\$env:MACHINE_IP = docker-machine ip qt-test-server;'
TEST_ENV += '\$\$env:TEST_DOMAIN = $$shell_quote(\"$$DNSDOMAIN\");'
TEST_ENV += '\$\$env:SHARED_DATA = $$shell_quote(\"$$PWD\");'
# Docker-compose CLI environment variables:
# Enable path conversion from Windows-style to Unix-style in volume definitions.
@ -127,6 +129,7 @@ isEmpty(TESTSERVER_VERSION) {
$$dirname(_QMAKE_CONF_)/tests/testserver/docker-compose.yml
# The environment variables passed to the docker-compose file
TEST_ENV = 'TEST_DOMAIN=$$DNSDOMAIN'
TEST_ENV += 'SHARED_DATA=$$PWD'
TEST_CMD = env
}
!exists($$TESTSERVER_COMPOSE_FILE): error("Invalid TESTSERVER_COMPOSE_FILE specified")
@ -200,7 +203,7 @@ isEmpty(TESTSERVER_VERSION) {
# Bring up test servers and make sure the services are ready.
!isEmpty(TEST_CMD): testserver_test.commands = $$TEST_CMD $$TEST_ENV
testserver_test.commands += docker-compose $$MACHINE_CONFIG -f $$TESTSERVER_COMPOSE_FILE up \
--detach --force-recreate --timeout 1 $${QT_TEST_SERVER_LIST} &&
--build -d --force-recreate --timeout 1 $${QT_TEST_SERVER_LIST} &&
# Check test cases with docker-based test servers.
testserver_test.commands += $(MAKE) -f $(MAKEFILE) check_network &&

11
tests/testserver/Dockerfile
View File

@ -1,11 +0,0 @@
# This Dockerfile is used on windows as volume sharing / mounting does not work in conjunction
# docker-machine. Windows Update 1809 might solve some of those issues when using docker0
# network switch, but nothing has been reported in regards to the combination with docker-machine.
ARG provisioningImage
FROM $provisioningImage
# Common is used for all test images so far, no need for a variable
COPY ./common /common
ARG servicedir
COPY $servicedir /service

17
tests/testserver/common/startup.sh
View File

@ -34,7 +34,7 @@ set -ex
# export variables
export USER=qt-test-server
export PASS=password
export CONFIG=common/testdata
export CONFIG=service/testdata
export TESTDATA=service/testdata
# add users
@ -43,17 +43,4 @@ useradd -m -s /bin/bash $USER; echo "$USER:$PASS" | chpasswd
# install configurations and test data
su $USER -c "cp $CONFIG/system/passwords ~/"
# modules initialization (apache2.sh, ftp-proxy.sh ...)
for RUN_CMD
do $RUN_CMD
done
# start multicast DNS service discovery (mDNS)
sed -i -e "s,#domain-name=local,domain-name=${test_domain:-test-net.qt.local}," \
-e "s,#publish-aaaa-on-ipv4=yes,publish-aaaa-on-ipv4=no," \
/etc/avahi/avahi-daemon.conf
service dbus restart
service avahi-daemon restart
# keep-alive in docker detach mode
sleep infinity
./startup.sh "$@"

81
tests/testserver/docker-compose-for-macOS.yml
View File

@ -12,82 +12,95 @@ version: '3.4'
services:
apache2:
image: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
container_name: qt-test-server-apache2
domainname: ${TEST_DOMAIN}
volumes:
- ./common:/common:ro
- ./apache2:/service:ro
entrypoint: common/startup.sh
command: [common/ssl.sh, service/apache2.sh]
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
shareDir: ./common
serviceDir: ./apache2
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: [service/ssl.sh, service/apache2.sh]
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
- test_cert="qt-test-server-host-network-cacert.pem"
squid:
image: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
container_name: qt-test-server-squid
domainname: ${TEST_DOMAIN}
depends_on:
- apache2
volumes:
- ./common:/common:ro
- ./squid:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
shareDir: ./common
serviceDir: ./squid
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/squid.sh
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
vsftpd:
image: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
container_name: qt-test-server-vsftpd
domainname: ${TEST_DOMAIN}
volumes:
- ./common:/common:ro
- ./vsftpd:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
shareDir: ./common
serviceDir: ./vsftpd
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/vsftpd.sh
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
ftp-proxy:
image: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
container_name: qt-test-server-ftp-proxy
domainname: ${TEST_DOMAIN}
depends_on:
- vsftpd
volumes:
- ./common:/common:ro
- ./ftp-proxy:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
shareDir: ./common
serviceDir: ./ftp-proxy
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/ftp-proxy.sh
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
danted:
image: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
container_name: qt-test-server-danted
domainname: ${TEST_DOMAIN}
depends_on:
- apache2
- vsftpd
- ftp-proxy
volumes:
- ./common:/common:ro
- ./danted:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
shareDir: ./common
serviceDir: ./danted
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/danted.sh
network_mode: "host"
extra_hosts:
@ -97,4 +110,4 @@ services:
- danted_external=${MACHINE_IP}
- danted_auth_internal=${MACHINE_IP}
- danted_auth_external=${MACHINE_IP}
- test_domain=${TEST_DOMAIN}

46
tests/testserver/docker-compose-for-windows.yml
View File

@ -16,16 +16,18 @@ services:
domainname: ${TEST_DOMAIN}
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
servicedir: ./apache2
entrypoint: common/startup.sh
command: [common/ssl.sh, service/apache2.sh]
shareDir: ./common
serviceDir: ./apache2
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: [service/ssl.sh, service/apache2.sh]
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
- test_cert="qt-test-server-host-network-cacert.pem"
squid:
@ -35,32 +37,34 @@ services:
- apache2
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
servicedir: ./squid
entrypoint: common/startup.sh
shareDir: ./common
serviceDir: ./squid
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/squid.sh
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
vsftpd:
container_name: qt-test-server-vsftpd
domainname: ${TEST_DOMAIN}
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
servicedir: ./vsftpd
entrypoint: common/startup.sh
shareDir: ./common
serviceDir: ./vsftpd
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/vsftpd.sh
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
ftp-proxy:
container_name: qt-test-server-ftp-proxy
@ -69,16 +73,17 @@ services:
- vsftpd
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
servicedir: ./ftp-proxy
entrypoint: common/startup.sh
shareDir: ./common
serviceDir: ./ftp-proxy
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/ftp-proxy.sh
network_mode: "host"
extra_hosts:
- "qt-test-server.${TEST_DOMAIN}:${MACHINE_IP}"
environment:
- test_domain=${TEST_DOMAIN}
danted:
container_name: qt-test-server-danted
@ -89,10 +94,13 @@ services:
- ftp-proxy
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
servicedir: ./danted
entrypoint: common/startup.sh
shareDir: ./common
serviceDir: ./danted
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/danted.sh
network_mode: "host"
extra_hosts:
@ -102,4 +110,4 @@ services:
- danted_external=${MACHINE_IP}
- danted_auth_internal=${MACHINE_IP}
- danted_auth_external=${MACHINE_IP}
- test_domain=${TEST_DOMAIN}

124
tests/testserver/docker-compose.yml
View File

@ -12,18 +12,21 @@ version: '3.4'
services:
apache2:
image: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
container_name: qt-test-server-apache2
domainname: ${TEST_DOMAIN}
hostname: apache2
volumes:
- ./common:/common:ro
- ./apache2:/service:ro
entrypoint: common/startup.sh
command: [common/ssl.sh, service/apache2.sh]
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-apache2:537fe302f61851d1663f41495230d8e3554a4a13
shareDir: ./common
serviceDir: ./apache2
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: [service/ssl.sh, service/apache2.sh]
squid:
image: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
container_name: qt-test-server-squid
domainname: ${TEST_DOMAIN}
hostname: squid
@ -35,25 +38,33 @@ services:
- iptables:iptables.${TEST_DOMAIN}
- vsftpd:vsftpd.${TEST_DOMAIN}
- echo:echo.${TEST_DOMAIN}
volumes:
- ./common:/common:ro
- ./squid:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-squid:9c32f41b19aca3d778733c4d8fb0ecc5955e893c
shareDir: ./common
serviceDir: ./squid
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/squid.sh
vsftpd:
image: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
container_name: qt-test-server-vsftpd
domainname: ${TEST_DOMAIN}
hostname: vsftpd
volumes:
- ./common:/common:ro
- ./vsftpd:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-vsftpd:f3a9c8d793a77cc007c0e4e481bec01f9e3eeb7e
shareDir: ./common
serviceDir: ./vsftpd
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/vsftpd.sh
ftp-proxy:
image: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
container_name: qt-test-server-ftp-proxy
domainname: ${TEST_DOMAIN}
hostname: ftp-proxy
@ -61,14 +72,18 @@ services:
- vsftpd
external_links:
- vsftpd:vsftpd.${TEST_DOMAIN}
volumes:
- ./common:/common:ro
- ./ftp-proxy:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-ftp-proxy:d7de8b28392d173db512a558ccc84ead8bece2ae
shareDir: ./common
serviceDir: ./ftp-proxy
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/ftp-proxy.sh
danted:
image: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
container_name: qt-test-server-danted
domainname: ${TEST_DOMAIN}
hostname: danted
@ -82,44 +97,61 @@ services:
- ftp-proxy:ftp-proxy.${TEST_DOMAIN}
- cyrus:cyrus.${TEST_DOMAIN}
- echo:echo.${TEST_DOMAIN}
volumes:
- ./common:/common:ro
- ./danted:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-danted:35607f9b790524cf9690c7d12a9a401696b7b6b5
shareDir: ./common
serviceDir: ./danted
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/danted.sh
cyrus:
image: qt-test-server-cyrus:c8d72754abc0e501afd624ce838e4df35505abc9
container_name: qt-test-server-cyrus
domainname: ${TEST_DOMAIN}
hostname: cyrus
volumes:
- ./common:/common:ro
- ./cyrus:/service:ro
entrypoint: common/startup.sh
command: [common/ssl.sh, service/cyrus.sh]
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-cyrus:c8d72754abc0e501afd624ce838e4df35505abc9
shareDir: ./common
serviceDir: ./cyrus
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: [service/ssl.sh, service/cyrus.sh]
iptables:
image: qt-test-server-iptables:cb7a8bd6d28602085a88c8ced7d67e28e75781e2
container_name: qt-test-server-iptables
domainname: ${TEST_DOMAIN}
hostname: iptables
volumes:
- ./common:/common:ro
- ./iptables:/service:ro
entrypoint: common/startup.sh
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-iptables:cb7a8bd6d28602085a88c8ced7d67e28e75781e2
shareDir: ./common
serviceDir: ./iptables
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/iptables.sh
cap_add:
- NET_ADMIN
- NET_RAW
echo:
image: qt-test-server-echo:b29ad409e746a834c1055fd0f7a55fd5056da6ea
container_name: qt-test-server-echo
domainname: ${TEST_DOMAIN}
hostname: echo
volumes:
- ./common:/common:ro
- ./echo:/service:ro
entrypoint: common/startup.sh
command: service/echo.sh
container_name: qt-test-server-echo
domainname: ${TEST_DOMAIN}
hostname: echo
build:
context: .
dockerfile: ${SHARED_DATA}/Dockerfile
args:
provisioningImage: qt-test-server-echo:b29ad409e746a834c1055fd0f7a55fd5056da6ea
shareDir: ./common
serviceDir: ./echo
test_domain: ${TEST_DOMAIN}
entrypoint: service/startup.sh
command: service/echo.sh