From 74440b2f921e52746be20071ac5a451029a6d66d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 23 Feb 2015 11:01:37 +0000 Subject: [PATCH 1/2] Run tests using Docker-in-Docker so we can test multiple versions Signed-off-by: Aanand Prasad --- CONTRIBUTING.md | 13 +++++++ Dockerfile | 15 +++++++- Dockerfile.tests | 5 +++ script/test | 20 ++++++++-- script/wrapdocker | 98 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 Dockerfile.tests create mode 100755 script/wrapdocker diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00bb7f43..22cbdcf8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,8 +24,21 @@ that should get you started. ## Running the test suite +Use the test script to run linting checks and then the full test suite: + $ script/test +Tests are run against a Docker daemon inside a container, so that we can test against multiple Docker versions. By default they'll run against only the latest Docker version - set the `DOCKER_VERSIONS` environment variable to "all" to run against all supported versions: + + $ DOCKER_VERSIONS=all script/test + +Arguments to `script/test` are passed through to the `nosetests` executable, so you can specify a test directory, file, module, class or method: + + $ script/test tests/unit + $ script/test tests/unit/cli_test.py + $ script/test tests.integration.service_test + $ script/test tests.integration.service_test:ServiceTest.test_containers + ## Building binaries Linux: diff --git a/Dockerfile b/Dockerfile index ee9fb4a2..fc553cef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,18 @@ FROM debian:wheezy -RUN apt-get update -qq && apt-get install -qy python python-pip python-dev git && apt-get clean + +RUN apt-get update -qq + +# Compose dependencies +RUN apt-get install -qqy python python-pip python-dev git + +# Test dependencies +RUN apt-get install -qqy apt-transport-https ca-certificates curl lxc iptables +RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.3.3 > /usr/local/bin/docker-1.3.3 && chmod +x /usr/local/bin/docker-1.3.3 +RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.4.1 > /usr/local/bin/docker-1.4.1 && chmod +x /usr/local/bin/docker-1.4.1 +RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.5.0 > /usr/local/bin/docker-1.5.0 && chmod +x /usr/local/bin/docker-1.5.0 + +RUN apt-get clean + RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/Dockerfile.tests b/Dockerfile.tests new file mode 100644 index 00000000..f30564d2 --- /dev/null +++ b/Dockerfile.tests @@ -0,0 +1,5 @@ +FROM docker-compose + +ADD script/wrapdocker /usr/local/bin/wrapdocker +VOLUME /var/lib/docker +ENTRYPOINT ["/usr/local/bin/wrapdocker"] diff --git a/script/test b/script/test index fef16b80..461fe7d1 100755 --- a/script/test +++ b/script/test @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/bash +# See CONTRIBUTING.md for usage. + set -ex + docker build -t docker-compose . -docker run -v /var/run/docker.sock:/var/run/docker.sock --rm --entrypoint flake8 docker-compose compose -docker run -v /var/run/docker.sock:/var/run/docker.sock --rm --entrypoint nosetests docker-compose $@ +docker run --privileged --rm --entrypoint flake8 docker-compose compose + +docker build -f Dockerfile.tests -t docker-compose-tests . + +if [ "$DOCKER_VERSIONS" == "" ]; then + DOCKER_VERSIONS="1.5.0" +elif [ "$DOCKER_VERSIONS" == "all" ]; then + DOCKER_VERSIONS="1.3.3 1.4.1 1.5.0" +fi + +for version in $DOCKER_VERSIONS; do + docker run --privileged --rm -e "DOCKER_VERSION=$version" docker-compose-tests nosetests "$@" +done diff --git a/script/wrapdocker b/script/wrapdocker new file mode 100755 index 00000000..e5bcbad6 --- /dev/null +++ b/script/wrapdocker @@ -0,0 +1,98 @@ +#!/bin/bash +# Adapted from https://github.com/jpetazzo/dind + +# First, make sure that cgroups are mounted correctly. +CGROUP=/sys/fs/cgroup +: {LOG:=stdio} + +[ -d $CGROUP ] || + mkdir $CGROUP + +mountpoint -q $CGROUP || + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo "Could not make a tmpfs mount. Did you use --privileged?" + exit 1 + } + +if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security +then + mount -t securityfs none /sys/kernel/security || { + echo "Could not mount /sys/kernel/security." + echo "AppArmor detection and --privileged mode might break." + } +fi + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) +do + [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS + mountpoint -q $CGROUP/$SUBSYS || + mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS + + # The two following sections address a bug which manifests itself + # by a cryptic "lxc-start: no ns_cgroup option specified" when + # trying to start containers withina container. + # The bug seems to appear when the cgroup hierarchies are not + # mounted on the exact same directories in the host, and in the + # container. + + # Named, control-less cgroups are mounted with "-o name=foo" + # (and appear as such under /proc//cgroup) but are usually + # mounted on a directory named "foo" (without the "name=" prefix). + # Systemd and OpenRC (and possibly others) both create such a + # cgroup. To avoid the aforementioned bug, we symlink "foo" to + # "name=foo". This shouldn't have any adverse effect. + echo $SUBSYS | grep -q ^name= && { + NAME=$(echo $SUBSYS | sed s/^name=//) + ln -s $SUBSYS $CGROUP/$NAME + } + + # Likewise, on at least one system, it has been reported that + # systemd would mount the CPU and CPU accounting controllers + # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" + # but on a directory called "cpu,cpuacct" (note the inversion + # in the order of the groups). This tries to work around it. + [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +grep -q :devices: /proc/1/cgroup || + echo "WARNING: the 'devices' cgroup should be in its own hierarchy." +grep -qw devices /proc/1/cgroup || + echo "WARNING: it looks like the 'devices' cgroup is not mounted." + +# Now, close extraneous file descriptors. +pushd /proc/self/fd >/dev/null +for FD in * +do + case "$FD" in + # Keep stdin/stdout/stderr + [012]) + ;; + # Nuke everything else + *) + eval exec "$FD>&-" + ;; + esac +done +popd >/dev/null + +if [ "$DOCKER_VERSION" == "" ]; then + DOCKER_VERSION="1.5.0" +fi + +ln -s "/usr/local/bin/docker-$DOCKER_VERSION" "/usr/local/bin/docker" + +# If a pidfile is still around (for example after a container restart), +# delete it so that docker can start. +rm -rf /var/run/docker.pid +docker -d $DOCKER_DAEMON_ARGS &>/var/log/docker.log & + +>&2 echo "Waiting for Docker to start..." +while ! docker ps &>/dev/null; do + sleep 1 +done + +exec "$@" From 42e6296b0ee9e9c1ccb1c9ff249e33775108eba2 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 25 Feb 2015 13:52:55 +0000 Subject: [PATCH 2/2] Kick everything off from a single container Signed-off-by: Aanand Prasad --- Dockerfile | 30 ++++++++++----- Dockerfile.tests | 5 --- script/dind | 88 ++++++++++++++++++++++++++++++++++++++++++++ script/test | 23 +++++------- script/test-versions | 26 +++++++++++++ script/wrapdocker | 80 +--------------------------------------- 6 files changed, 145 insertions(+), 107 deletions(-) delete mode 100644 Dockerfile.tests create mode 100755 script/dind create mode 100755 script/test-versions diff --git a/Dockerfile b/Dockerfile index fc553cef..d7a6019a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,27 @@ FROM debian:wheezy -RUN apt-get update -qq +RUN set -ex; \ + apt-get update -qq; \ + apt-get install -y \ + python \ + python-pip \ + python-dev \ + git \ + apt-transport-https \ + ca-certificates \ + curl \ + lxc \ + iptables \ + ; \ + rm -rf /var/lib/apt/lists/* -# Compose dependencies -RUN apt-get install -qqy python python-pip python-dev git +ENV ALL_DOCKER_VERSIONS 1.3.3 1.4.1 1.5.0 -# Test dependencies -RUN apt-get install -qqy apt-transport-https ca-certificates curl lxc iptables -RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.3.3 > /usr/local/bin/docker-1.3.3 && chmod +x /usr/local/bin/docker-1.3.3 -RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.4.1 > /usr/local/bin/docker-1.4.1 && chmod +x /usr/local/bin/docker-1.4.1 -RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.5.0 > /usr/local/bin/docker-1.5.0 && chmod +x /usr/local/bin/docker-1.5.0 - -RUN apt-get clean +RUN set -ex; \ + for v in ${ALL_DOCKER_VERSIONS}; do \ + curl https://get.docker.com/builds/Linux/x86_64/docker-$v -o /usr/local/bin/docker-$v; \ + chmod +x /usr/local/bin/docker-$v; \ + done RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ diff --git a/Dockerfile.tests b/Dockerfile.tests deleted file mode 100644 index f30564d2..00000000 --- a/Dockerfile.tests +++ /dev/null @@ -1,5 +0,0 @@ -FROM docker-compose - -ADD script/wrapdocker /usr/local/bin/wrapdocker -VOLUME /var/lib/docker -ENTRYPOINT ["/usr/local/bin/wrapdocker"] diff --git a/script/dind b/script/dind new file mode 100755 index 00000000..f8fae637 --- /dev/null +++ b/script/dind @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +# DinD: a wrapper script which allows docker to be run inside a docker container. +# Original version by Jerome Petazzoni +# See the blog post: http://blog.docker.com/2013/09/docker-can-now-run-within-docker/ +# +# This script should be executed inside a docker container in privilieged mode +# ('docker run --privileged', introduced in docker 0.6). + +# Usage: dind CMD [ARG...] + +# apparmor sucks and Docker needs to know that it's in a container (c) @tianon +export container=docker + +# First, make sure that cgroups are mounted correctly. +CGROUP=/cgroup + +mkdir -p "$CGROUP" + +if ! mountpoint -q "$CGROUP"; then + mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { + echo >&2 'Could not make a tmpfs mount. Did you use --privileged?' + exit 1 + } +fi + +if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security; then + mount -t securityfs none /sys/kernel/security || { + echo >&2 'Could not mount /sys/kernel/security.' + echo >&2 'AppArmor detection and -privileged mode might break.' + } +fi + +# Mount the cgroup hierarchies exactly as they are in the parent system. +for SUBSYS in $(cut -d: -f2 /proc/1/cgroup); do + mkdir -p "$CGROUP/$SUBSYS" + if ! mountpoint -q $CGROUP/$SUBSYS; then + mount -n -t cgroup -o "$SUBSYS" cgroup "$CGROUP/$SUBSYS" + fi + + # The two following sections address a bug which manifests itself + # by a cryptic "lxc-start: no ns_cgroup option specified" when + # trying to start containers withina container. + # The bug seems to appear when the cgroup hierarchies are not + # mounted on the exact same directories in the host, and in the + # container. + + # Named, control-less cgroups are mounted with "-o name=foo" + # (and appear as such under /proc//cgroup) but are usually + # mounted on a directory named "foo" (without the "name=" prefix). + # Systemd and OpenRC (and possibly others) both create such a + # cgroup. To avoid the aforementioned bug, we symlink "foo" to + # "name=foo". This shouldn't have any adverse effect. + name="${SUBSYS#name=}" + if [ "$name" != "$SUBSYS" ]; then + ln -s "$SUBSYS" "$CGROUP/$name" + fi + + # Likewise, on at least one system, it has been reported that + # systemd would mount the CPU and CPU accounting controllers + # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" + # but on a directory called "cpu,cpuacct" (note the inversion + # in the order of the groups). This tries to work around it. + if [ "$SUBSYS" = 'cpuacct,cpu' ]; then + ln -s "$SUBSYS" "$CGROUP/cpu,cpuacct" + fi +done + +# Note: as I write those lines, the LXC userland tools cannot setup +# a "sub-container" properly if the "devices" cgroup is not in its +# own hierarchy. Let's detect this and issue a warning. +if ! grep -q :devices: /proc/1/cgroup; then + echo >&2 'WARNING: the "devices" cgroup should be in its own hierarchy.' +fi +if ! grep -qw devices /proc/1/cgroup; then + echo >&2 'WARNING: it looks like the "devices" cgroup is not mounted.' +fi + +# Mount /tmp +mount -t tmpfs none /tmp + +if [ $# -gt 0 ]; then + exec "$@" +fi + +echo >&2 'ERROR: No command specified.' +echo >&2 'You probably want to run hack/make.sh, or maybe a shell?' diff --git a/script/test b/script/test index 461fe7d1..f278023a 100755 --- a/script/test +++ b/script/test @@ -3,17 +3,14 @@ set -ex -docker build -t docker-compose . -docker run --privileged --rm --entrypoint flake8 docker-compose compose +TAG="docker-compose:$(git rev-parse --short HEAD)" -docker build -f Dockerfile.tests -t docker-compose-tests . - -if [ "$DOCKER_VERSIONS" == "" ]; then - DOCKER_VERSIONS="1.5.0" -elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="1.3.3 1.4.1 1.5.0" -fi - -for version in $DOCKER_VERSIONS; do - docker run --privileged --rm -e "DOCKER_VERSION=$version" docker-compose-tests nosetests "$@" -done +docker build -t "$TAG" . +docker run \ + --rm \ + --volume="/var/run/docker.sock:/var/run/docker.sock" \ + -e DOCKER_VERSIONS \ + -e "TAG=$TAG" \ + --entrypoint="script/test-versions" \ + "$TAG" \ + "$@" diff --git a/script/test-versions b/script/test-versions new file mode 100755 index 00000000..9f30eeca --- /dev/null +++ b/script/test-versions @@ -0,0 +1,26 @@ +#!/bin/bash +# This should be run inside a container built from the Dockerfile +# at the root of the repo - script/test will do it automatically. + +set -e + +>&2 echo "Running lint checks" +flake8 compose + +if [ "$DOCKER_VERSIONS" == "" ]; then + DOCKER_VERSIONS="1.5.0" +elif [ "$DOCKER_VERSIONS" == "all" ]; then + DOCKER_VERSIONS="$ALL_DOCKER_VERSIONS" +fi + +for version in $DOCKER_VERSIONS; do + >&2 echo "Running tests against Docker $version" + docker-1.5.0 run \ + --rm \ + --privileged \ + --volume="/var/lib/docker" \ + -e "DOCKER_VERSION=$version" \ + --entrypoint="script/dind" \ + "$TAG" \ + script/wrapdocker nosetests "$@" +done diff --git a/script/wrapdocker b/script/wrapdocker index e5bcbad6..20dc9e3c 100755 --- a/script/wrapdocker +++ b/script/wrapdocker @@ -1,83 +1,4 @@ #!/bin/bash -# Adapted from https://github.com/jpetazzo/dind - -# First, make sure that cgroups are mounted correctly. -CGROUP=/sys/fs/cgroup -: {LOG:=stdio} - -[ -d $CGROUP ] || - mkdir $CGROUP - -mountpoint -q $CGROUP || - mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup $CGROUP || { - echo "Could not make a tmpfs mount. Did you use --privileged?" - exit 1 - } - -if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security -then - mount -t securityfs none /sys/kernel/security || { - echo "Could not mount /sys/kernel/security." - echo "AppArmor detection and --privileged mode might break." - } -fi - -# Mount the cgroup hierarchies exactly as they are in the parent system. -for SUBSYS in $(cut -d: -f2 /proc/1/cgroup) -do - [ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS - mountpoint -q $CGROUP/$SUBSYS || - mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS - - # The two following sections address a bug which manifests itself - # by a cryptic "lxc-start: no ns_cgroup option specified" when - # trying to start containers withina container. - # The bug seems to appear when the cgroup hierarchies are not - # mounted on the exact same directories in the host, and in the - # container. - - # Named, control-less cgroups are mounted with "-o name=foo" - # (and appear as such under /proc//cgroup) but are usually - # mounted on a directory named "foo" (without the "name=" prefix). - # Systemd and OpenRC (and possibly others) both create such a - # cgroup. To avoid the aforementioned bug, we symlink "foo" to - # "name=foo". This shouldn't have any adverse effect. - echo $SUBSYS | grep -q ^name= && { - NAME=$(echo $SUBSYS | sed s/^name=//) - ln -s $SUBSYS $CGROUP/$NAME - } - - # Likewise, on at least one system, it has been reported that - # systemd would mount the CPU and CPU accounting controllers - # (respectively "cpu" and "cpuacct") with "-o cpuacct,cpu" - # but on a directory called "cpu,cpuacct" (note the inversion - # in the order of the groups). This tries to work around it. - [ $SUBSYS = cpuacct,cpu ] && ln -s $SUBSYS $CGROUP/cpu,cpuacct -done - -# Note: as I write those lines, the LXC userland tools cannot setup -# a "sub-container" properly if the "devices" cgroup is not in its -# own hierarchy. Let's detect this and issue a warning. -grep -q :devices: /proc/1/cgroup || - echo "WARNING: the 'devices' cgroup should be in its own hierarchy." -grep -qw devices /proc/1/cgroup || - echo "WARNING: it looks like the 'devices' cgroup is not mounted." - -# Now, close extraneous file descriptors. -pushd /proc/self/fd >/dev/null -for FD in * -do - case "$FD" in - # Keep stdin/stdout/stderr - [012]) - ;; - # Nuke everything else - *) - eval exec "$FD>&-" - ;; - esac -done -popd >/dev/null if [ "$DOCKER_VERSION" == "" ]; then DOCKER_VERSION="1.5.0" @@ -95,4 +16,5 @@ while ! docker ps &>/dev/null; do sleep 1 done +>&2 echo ">" "$@" exec "$@"