From: Dessalines Date: Sat, 25 Jul 2020 02:06:00 +0000 (-0400) Subject: Automated travis docker testing and deploys. (#1023) X-Git-Url: http://these/git/%7B%60%24%7BarchiveTodayUrl%7D/%7B%60/feeds/inbox/static/git-favicon.png?a=commitdiff_plain;h=edb2e40383b0201cb5be3f462e9374ff1c5c8ab2;p=lemmy.git Automated travis docker testing and deploys. (#1023) * Testing out a travis 1. * Testing out a travis 2. * Testing out a travis 3. * Testing out a travis 4. * Testing out a travis 5. * Testing out a travis 6. Trying a failing test * Testing out a travis 7. Trying to cache the docker images. * Testing out a travis 8. Still not correctly failing. * Trying full path for docker_images dir. * Fixing test again, trying a dockerhub deploy. * Fixing test again, trying a dockerhub deploy 2. * Remove caching. * Trying to fix docker deploy. * Try using the travis tag * Using dessalines/lemmy:travis * Try using the prod dockerfile now. * Update http sigs dependency * Finally getting this mostly working. Co-authored-by: asonix --- diff --git a/.travis.yml b/.travis.yml index 9541afaa..8120c83f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,28 @@ -language: rust -rust: - - stable -matrix: - allow_failures: - - rust: nightly - fast_finish: true -cache: cargo -before_cache: - - rm -rfv target/debug/incremental/lemmy_server-* - - rm -rfv target/debug/.fingerprint/lemmy_server-* - - rm -rfv target/debug/build/lemmy_server-* - - rm -rfv target/debug/deps/lemmy_server-* - - rm -rfv target/debug/lemmy_server.d -before_script: - - psql -c "create user lemmy with password 'password' superuser;" -U postgres - - psql -c 'create database lemmy with owner lemmy;' -U postgres - - rustup component add clippy --toolchain stable-x86_64-unknown-linux-gnu -before_install: - - cd server -script: - # Default checks, but fail if anything is detected - - cargo build - - cargo clippy -- -D clippy::style -D clippy::correctness -D clippy::complexity -D clippy::perf - - cargo install diesel_cli --no-default-features --features postgres --force - - diesel migration run - - cargo test --workspace +sudo: required +language: node_js +node_js: +- 14 +services: +- docker env: + matrix: + - DOCKER_COMPOSE_VERSION=1.25.5 global: - - DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy - - LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy - - RUST_TEST_THREADS=1 - -addons: - postgresql: "9.4" + - secure: nzmFoTxPn7OT+qcTULezSCT6B44j/q8RxERBQSr1FVXaCcDrBr6q9ewhGy7BHWP74r4qbif4m9r3sNELZCoFYFP3JwLnrZfX/xUwU8p61eFD2PMOJAdOywDxb94SvooOSnjBmxNvRsuqf6Zmnw378mbsSVCi9Xbx9jpoV4Jq8zKgO0M8WIl/lj2dijD95WIMrHcorbzKS3+2zW3LkPiC2bnfDAUmUDfaCj1gh9FCvzZMtrSxu7kxAeFCkR16TJUciIcGgag8rLHfxwG0h2uEJJ+3/62qCWUdgnj171oTE4ZRi0hdvt2HOY5wjHfS2y1ZxWYgo31uws3pyoTNeQZi0o7Q9Xe/4JXYZXvDfuscSZ9RiuhAstCVswtXPJJVVJQ9cdl5eX1TI0bz8eVRvRy4p40OIBjKiobkmRjl8sXjFbpYAIvFr+TgSa/K/bxm3POfI0B8bIHI85zFxUMrWt5i2IJ0dWvDNHrz+CWWKn1vVFYbBNPgDDHtE0P3LWLEioWFf+ULycjW8DefWc+b63Lf9SSaEE7FnX2mc+BaHCgubCDkJy9Au4xP8zQlJjgZwOdTedw5jvmwz3fqMZBpHypVUXzZs7cRhMWtQ7TAoGb8TOqXNgPEVW+BARNXl0wAamTgjt9v20x0wkp+/SLJwMNY+zvwmzxzd5R9TPgDOqyIRTU= + - secure: ALZqC4OYV315P7EZyk+c/PLJdneeU7jMC30TTzMcX3hospIu7naWekZ+HUnziFDQKZxIHWKZsq1R52DWhsERLrPF3SVa+QiXu8vTTPrETBWnu9VgyFzgdEbUKRas1X3qerEAHcNBms1EAl2FOiQM1k5EDygrClv4KWgyzntEtKJbN2UCFKxtoBSdMZA6fcGtCwffcj8uIAIP2NhZixbU+smVgVbpMpe6QEuuEoVlVrfH8iXxb8Gi+qkd0YIYAHkjtTqQ/nHuAUhcuEE0mORTNGPv7CmTwpuQiGCCdtySZc7Qq8z1x2y7RLy0+RVxM0PR8UV6iy4ipyTgZ6wTF30ksLDxOI3GlRaKF3F6kLErOiEiEUOqa+zLgUM0OLGTn+KLATQDx74in5NcKjKUAnkuxdZyuDbifvQb5tqfrGdXd22pzVZbielRJRW59ig0Nr5cxEpRtoRkoFKNk7o3XlD6JmIBjKn1UHkZ4H/oLUKIXT2qOP2fIEzgLjfpSuGwhvJRz1KRP49HYVl7Gkd45/RdZ519W0gnMkIrEaod90iXSFNTgmJTGeH0Mv0jHameN47PIT3c49MOy5Hj0XCHUPfc6qqrdGnliS5hTnrFThCfn5ZuSZxVdgGLJUQvV+D+5KDqjFdGyNGVGoEg0YdrDtGXmpojbyQDJAT7ToL3yIBF7co= +before_install: +# Install docker-compose +- sudo rm /usr/local/bin/docker-compose +- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname + -s`-`uname -m` > docker-compose +- chmod +x docker-compose +- sudo mv docker-compose /usr/local/bin +# Change dir +- cd docker/travis +script: +- "./run-tests.sh" +deploy: + provider: script + script: bash docker_push.sh + on: + branch: main diff --git a/docker/prod/deploy.sh b/docker/prod/deploy.sh index 2e872818..3c12df20 100755 --- a/docker/prod/deploy.sh +++ b/docker/prod/deploy.sh @@ -24,35 +24,39 @@ cd docker/prod || exit # Changing the docker-compose prod sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../prod/docker-compose.yml sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../../ansible/templates/docker-compose.yml +sed -i "s/dessalines\/lemmy:v.*/dessalines\/lemmy:$new_tag/" ../travis/docker_push.sh git add ../prod/docker-compose.yml git add ../../ansible/templates/docker-compose.yml +git add ../travis/docker_push.sh # The commit git commit -m"Version $new_tag" git tag $new_tag -export COMPOSE_DOCKER_CLI_BUILD=1 -export DOCKER_BUILDKIT=1 - -# Rebuilding docker -if [ $third_semver -eq 0 ]; then - # TODO get linux/arm/v7 build working - # Build for Raspberry Pi / other archs too - docker buildx build --platform linux/amd64,linux/arm64 ../../ \ - --file Dockerfile \ - --tag dessalines/lemmy:$new_tag \ - --push -else - docker buildx build --platform linux/amd64 ../../ \ - --file Dockerfile \ - --tag dessalines/lemmy:$new_tag \ - --push -fi +# Now doing the building on travis, but leave this in for when you need to do an arm build + +# export COMPOSE_DOCKER_CLI_BUILD=1 +# export DOCKER_BUILDKIT=1 + +# # Rebuilding docker +# if [ $third_semver -eq 0 ]; then +# # TODO get linux/arm/v7 build working +# # Build for Raspberry Pi / other archs too +# docker buildx build --platform linux/amd64,linux/arm64 ../../ \ +# --file Dockerfile \ +# --tag dessalines/lemmy:$new_tag \ +# --push +# else +# docker buildx build --platform linux/amd64 ../../ \ +# --file Dockerfile \ +# --tag dessalines/lemmy:$new_tag \ +# --push +# fi # Push git push origin $new_tag git push # Pushing to any ansible deploys -cd ../../../lemmy-ansible || exit -ansible-playbook -i prod playbooks/site.yml --vault-password-file vault_pass +# cd ../../../lemmy-ansible || exit +# ansible-playbook -i prod playbooks/site.yml --vault-password-file vault_pass diff --git a/docker/travis/docker-compose.yml b/docker/travis/docker-compose.yml new file mode 100644 index 00000000..5248553a --- /dev/null +++ b/docker/travis/docker-compose.yml @@ -0,0 +1,113 @@ +version: '3.3' + +services: + nginx: + image: nginx:1.17-alpine + ports: + - "8540:8540" + - "8550:8550" + - "8560:8560" + volumes: + # Hack to make this work from both docker/federation/ and docker/federation-test/ + - ../federation/nginx.conf:/etc/nginx/nginx.conf + restart: on-failure + depends_on: + - lemmy-alpha + - pictrs + - lemmy-beta + - lemmy-gamma + - iframely + + pictrs: + restart: always + image: asonix/pictrs:v0.1.13-r0 + user: 991:991 + volumes: + - ./volumes/pictrs_alpha:/mnt + + lemmy-alpha: + image: dessalines/lemmy:travis + environment: + - LEMMY_HOSTNAME=lemmy-alpha:8540 + - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_alpha:5432/lemmy + - LEMMY_JWT_SECRET=changeme + - LEMMY_FRONT_END_DIR=/app/dist + - LEMMY_FEDERATION__ENABLED=true + - LEMMY_FEDERATION__TLS_ENABLED=false + - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-beta,lemmy-gamma + - LEMMY_PORT=8540 + - LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha + - LEMMY_SETUP__ADMIN_PASSWORD=lemmy + - LEMMY_SETUP__SITE_NAME=lemmy-alpha + - RUST_BACKTRACE=1 + - RUST_LOG=debug + depends_on: + - postgres_alpha + postgres_alpha: + image: postgres:12-alpine + environment: + - POSTGRES_USER=lemmy + - POSTGRES_PASSWORD=password + - POSTGRES_DB=lemmy + volumes: + - ./volumes/postgres_alpha:/var/lib/postgresql/data + + lemmy-beta: + image: dessalines/lemmy:travis + environment: + - LEMMY_HOSTNAME=lemmy-beta:8550 + - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_beta:5432/lemmy + - LEMMY_JWT_SECRET=changeme + - LEMMY_FRONT_END_DIR=/app/dist + - LEMMY_FEDERATION__ENABLED=true + - LEMMY_FEDERATION__TLS_ENABLED=false + - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-gamma + - LEMMY_PORT=8550 + - LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta + - LEMMY_SETUP__ADMIN_PASSWORD=lemmy + - LEMMY_SETUP__SITE_NAME=lemmy-beta + - RUST_BACKTRACE=1 + - RUST_LOG=debug + depends_on: + - postgres_beta + postgres_beta: + image: postgres:12-alpine + environment: + - POSTGRES_USER=lemmy + - POSTGRES_PASSWORD=password + - POSTGRES_DB=lemmy + volumes: + - ./volumes/postgres_beta:/var/lib/postgresql/data + + lemmy-gamma: + image: dessalines/lemmy:travis + environment: + - LEMMY_HOSTNAME=lemmy-gamma:8560 + - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_gamma:5432/lemmy + - LEMMY_JWT_SECRET=changeme + - LEMMY_FRONT_END_DIR=/app/dist + - LEMMY_FEDERATION__ENABLED=true + - LEMMY_FEDERATION__TLS_ENABLED=false + - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-beta + - LEMMY_PORT=8560 + - LEMMY_SETUP__ADMIN_USERNAME=lemmy_gamma + - LEMMY_SETUP__ADMIN_PASSWORD=lemmy + - LEMMY_SETUP__SITE_NAME=lemmy-gamma + - RUST_BACKTRACE=1 + - RUST_LOG=debug + depends_on: + - postgres_gamma + postgres_gamma: + image: postgres:12-alpine + environment: + - POSTGRES_USER=lemmy + - POSTGRES_PASSWORD=password + - POSTGRES_DB=lemmy + volumes: + - ./volumes/postgres_gamma:/var/lib/postgresql/data + + iframely: + image: dogbin/iframely:latest + volumes: + - ../iframely.config.local.js:/iframely/config.local.js:ro + restart: always diff --git a/docker/travis/docker_push.sh b/docker/travis/docker_push.sh new file mode 100644 index 00000000..bd6dfca6 --- /dev/null +++ b/docker/travis/docker_push.sh @@ -0,0 +1,5 @@ +#!/bin/sh +echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin +docker tag dessalines/lemmy:travis \ + dessalines/lemmy:v0.7.28 +docker push dessalines/lemmy:v0.7.28 diff --git a/docker/travis/run-tests.sh b/docker/travis/run-tests.sh new file mode 100755 index 00000000..f09b73c2 --- /dev/null +++ b/docker/travis/run-tests.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +# make sure there are no old containers or old data around +sudo docker-compose down +sudo rm -rf volumes + +mkdir -p volumes/pictrs_{alpha,beta,gamma} +sudo chown -R 991:991 volumes/pictrs_{alpha,beta,gamma} + +sudo docker build ../../ --file ../prod/Dockerfile --tag dessalines/lemmy:travis + +sudo docker-compose up -d + +pushd ../../ui +echo "Waiting for Lemmy to start..." +while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8540/api/v1/site')" != "200" ]]; do sleep 1; done +while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8550/api/v1/site')" != "200" ]]; do sleep 1; done +while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8560/api/v1/site')" != "200" ]]; do sleep 1; done +yarn +yarn api-test +popd + +sudo docker-compose down + +sudo rm -r volumes/