Automated travis docker testing and deploys. (#1023)

* Testing out a travis 1.

* Testing out a travis 2.

* Testing out a travis 3.

* Testing out a travis 4.

* Testing out a travis 5.

* Testing out a travis 6. Trying a failing test

* Testing out a travis 7. Trying to cache the docker images.

* Testing out a travis 8. Still not correctly failing.

* Trying full path for docker_images dir.

* Fixing test again, trying a dockerhub deploy.

* Fixing test again, trying a dockerhub deploy 2.

* Remove caching.

* Trying to fix docker deploy.

* Try using the travis tag

* Using dessalines/lemmy:travis

* Try using the prod dockerfile now.

* Update http sigs dependency

* Finally getting this mostly working.

Co-authored-by: asonix <asonix@asonix.dog>

.travis.yml

@@ -1,35 +1,28 @@
-language: rust
+sudo: required
-rust:
+language: node_js
-  - stable
+node_js:
-matrix:
+- 14
-  allow_failures:
+services:
-    - rust: nightly
+- docker
-  fast_finish: true
-cache: cargo
-before_cache:
-  - rm -rfv target/debug/incremental/lemmy_server-*
-  - rm -rfv target/debug/.fingerprint/lemmy_server-*
-  - rm -rfv target/debug/build/lemmy_server-*
-  - rm -rfv target/debug/deps/lemmy_server-*
-  - rm -rfv target/debug/lemmy_server.d
-before_script:
-  - psql -c "create user lemmy with password 'password' superuser;" -U postgres
-  - psql -c 'create database lemmy with owner lemmy;' -U postgres
-  - rustup component add clippy --toolchain stable-x86_64-unknown-linux-gnu
-before_install:
-  - cd server
-script:
-  # Default checks, but fail if anything is detected
-  - cargo build
-  - cargo clippy -- -D clippy::style -D clippy::correctness -D clippy::complexity -D clippy::perf
-  - cargo install diesel_cli --no-default-features --features postgres --force
-  - diesel migration run
-  - cargo test --workspace
 env:
+  matrix:
+  - DOCKER_COMPOSE_VERSION=1.25.5
   global:
-    - DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
+  - secure: nzmFoTxPn7OT+qcTULezSCT6B44j/q8RxERBQSr1FVXaCcDrBr6q9ewhGy7BHWP74r4qbif4m9r3sNELZCoFYFP3JwLnrZfX/xUwU8p61eFD2PMOJAdOywDxb94SvooOSnjBmxNvRsuqf6Zmnw378mbsSVCi9Xbx9jpoV4Jq8zKgO0M8WIl/lj2dijD95WIMrHcorbzKS3+2zW3LkPiC2bnfDAUmUDfaCj1gh9FCvzZMtrSxu7kxAeFCkR16TJUciIcGgag8rLHfxwG0h2uEJJ+3/62qCWUdgnj171oTE4ZRi0hdvt2HOY5wjHfS2y1ZxWYgo31uws3pyoTNeQZi0o7Q9Xe/4JXYZXvDfuscSZ9RiuhAstCVswtXPJJVVJQ9cdl5eX1TI0bz8eVRvRy4p40OIBjKiobkmRjl8sXjFbpYAIvFr+TgSa/K/bxm3POfI0B8bIHI85zFxUMrWt5i2IJ0dWvDNHrz+CWWKn1vVFYbBNPgDDHtE0P3LWLEioWFf+ULycjW8DefWc+b63Lf9SSaEE7FnX2mc+BaHCgubCDkJy9Au4xP8zQlJjgZwOdTedw5jvmwz3fqMZBpHypVUXzZs7cRhMWtQ7TAoGb8TOqXNgPEVW+BARNXl0wAamTgjt9v20x0wkp+/SLJwMNY+zvwmzxzd5R9TPgDOqyIRTU=
-    - LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
+  - secure: ALZqC4OYV315P7EZyk+c/PLJdneeU7jMC30TTzMcX3hospIu7naWekZ+HUnziFDQKZxIHWKZsq1R52DWhsERLrPF3SVa+QiXu8vTTPrETBWnu9VgyFzgdEbUKRas1X3qerEAHcNBms1EAl2FOiQM1k5EDygrClv4KWgyzntEtKJbN2UCFKxtoBSdMZA6fcGtCwffcj8uIAIP2NhZixbU+smVgVbpMpe6QEuuEoVlVrfH8iXxb8Gi+qkd0YIYAHkjtTqQ/nHuAUhcuEE0mORTNGPv7CmTwpuQiGCCdtySZc7Qq8z1x2y7RLy0+RVxM0PR8UV6iy4ipyTgZ6wTF30ksLDxOI3GlRaKF3F6kLErOiEiEUOqa+zLgUM0OLGTn+KLATQDx74in5NcKjKUAnkuxdZyuDbifvQb5tqfrGdXd22pzVZbielRJRW59ig0Nr5cxEpRtoRkoFKNk7o3XlD6JmIBjKn1UHkZ4H/oLUKIXT2qOP2fIEzgLjfpSuGwhvJRz1KRP49HYVl7Gkd45/RdZ519W0gnMkIrEaod90iXSFNTgmJTGeH0Mv0jHameN47PIT3c49MOy5Hj0XCHUPfc6qqrdGnliS5hTnrFThCfn5ZuSZxVdgGLJUQvV+D+5KDqjFdGyNGVGoEg0YdrDtGXmpojbyQDJAT7ToL3yIBF7co=
-    - RUST_TEST_THREADS=1
+before_install:
-
+# Install docker-compose
-addons:
+- sudo rm /usr/local/bin/docker-compose
-  postgresql: "9.4"
+- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname
+  -s`-`uname -m` > docker-compose
+- chmod +x docker-compose
+- sudo mv docker-compose /usr/local/bin
+# Change dir
+- cd docker/travis
+script:
+- "./run-tests.sh"
+deploy:
+  provider: script
+  script: bash docker_push.sh
+  on:
+    branch: main

docker/prod/deploy.sh

@@ -24,35 +24,39 @@ cd docker/prod || exit
 # Changing the docker-compose prod
 sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../prod/docker-compose.yml
 sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../../ansible/templates/docker-compose.yml
+sed -i "s/dessalines\/lemmy:v.*/dessalines\/lemmy:$new_tag/" ../travis/docker_push.sh
 git add ../prod/docker-compose.yml
 git add ../../ansible/templates/docker-compose.yml
+git add ../travis/docker_push.sh
 
 # The commit
 git commit -m"Version $new_tag"
 git tag $new_tag
 
-export COMPOSE_DOCKER_CLI_BUILD=1
+# Now doing the building on travis, but leave this in for when you need to do an arm build
-export DOCKER_BUILDKIT=1
 
-# Rebuilding docker
+# export COMPOSE_DOCKER_CLI_BUILD=1
-if [ $third_semver -eq 0 ]; then
+# export DOCKER_BUILDKIT=1
-  # TODO get linux/arm/v7 build working
+
-  # Build for Raspberry Pi / other archs too
+# # Rebuilding docker
-  docker buildx build --platform linux/amd64,linux/arm64 ../../ \
+# if [ $third_semver -eq 0 ]; then
-    --file Dockerfile \
+#   # TODO get linux/arm/v7 build working
-    --tag dessalines/lemmy:$new_tag \
+#   # Build for Raspberry Pi / other archs too
-    --push
+#   docker buildx build --platform linux/amd64,linux/arm64 ../../ \
-else
+#     --file Dockerfile \
-  docker buildx build --platform linux/amd64 ../../ \
+#     --tag dessalines/lemmy:$new_tag \
-    --file Dockerfile \
+#     --push
-    --tag dessalines/lemmy:$new_tag \
+# else
-    --push
+#   docker buildx build --platform linux/amd64 ../../ \
-fi
+#     --file Dockerfile \
+#     --tag dessalines/lemmy:$new_tag \
+#     --push
+# fi
 
 # Push
 git push origin $new_tag
 git push
 
 # Pushing to any ansible deploys
-cd ../../../lemmy-ansible || exit
+# cd ../../../lemmy-ansible || exit
-ansible-playbook -i prod playbooks/site.yml --vault-password-file vault_pass
+# ansible-playbook -i prod playbooks/site.yml --vault-password-file vault_pass

docker/travis/docker-compose.yml

@@ -0,0 +1,113 @@
+version: '3.3'
+
+services:
+  nginx:
+    image: nginx:1.17-alpine
+    ports:
+      - "8540:8540"
+      - "8550:8550"
+      - "8560:8560"
+    volumes:
+      # Hack to make this work from both docker/federation/ and docker/federation-test/
+      - ../federation/nginx.conf:/etc/nginx/nginx.conf
+    restart: on-failure
+    depends_on:
+      - lemmy-alpha
+      - pictrs
+      - lemmy-beta
+      - lemmy-gamma
+      - iframely
+
+  pictrs:
+    restart: always
+    image: asonix/pictrs:v0.1.13-r0
+    user: 991:991
+    volumes:
+      - ./volumes/pictrs_alpha:/mnt
+
+  lemmy-alpha:
+    image: dessalines/lemmy:travis
+    environment:
+      - LEMMY_HOSTNAME=lemmy-alpha:8540
+      - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_alpha:5432/lemmy
+      - LEMMY_JWT_SECRET=changeme
+      - LEMMY_FRONT_END_DIR=/app/dist
+      - LEMMY_FEDERATION__ENABLED=true
+      - LEMMY_FEDERATION__TLS_ENABLED=false
+      - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-beta,lemmy-gamma
+      - LEMMY_PORT=8540
+      - LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha
+      - LEMMY_SETUP__ADMIN_PASSWORD=lemmy
+      - LEMMY_SETUP__SITE_NAME=lemmy-alpha
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    depends_on:
+      - postgres_alpha
+  postgres_alpha:
+    image: postgres:12-alpine
+    environment:
+      - POSTGRES_USER=lemmy
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=lemmy
+    volumes:
+      - ./volumes/postgres_alpha:/var/lib/postgresql/data
+
+  lemmy-beta:
+    image: dessalines/lemmy:travis
+    environment:
+      - LEMMY_HOSTNAME=lemmy-beta:8550
+      - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_beta:5432/lemmy
+      - LEMMY_JWT_SECRET=changeme
+      - LEMMY_FRONT_END_DIR=/app/dist
+      - LEMMY_FEDERATION__ENABLED=true
+      - LEMMY_FEDERATION__TLS_ENABLED=false
+      - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-gamma
+      - LEMMY_PORT=8550
+      - LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta
+      - LEMMY_SETUP__ADMIN_PASSWORD=lemmy
+      - LEMMY_SETUP__SITE_NAME=lemmy-beta
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    depends_on:
+      - postgres_beta
+  postgres_beta:
+    image: postgres:12-alpine
+    environment:
+      - POSTGRES_USER=lemmy
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=lemmy
+    volumes:
+      - ./volumes/postgres_beta:/var/lib/postgresql/data
+
+  lemmy-gamma:
+    image: dessalines/lemmy:travis
+    environment:
+      - LEMMY_HOSTNAME=lemmy-gamma:8560
+      - LEMMY_DATABASE_URL=postgres://lemmy:password@postgres_gamma:5432/lemmy
+      - LEMMY_JWT_SECRET=changeme
+      - LEMMY_FRONT_END_DIR=/app/dist
+      - LEMMY_FEDERATION__ENABLED=true
+      - LEMMY_FEDERATION__TLS_ENABLED=false
+      - LEMMY_FEDERATION__ALLOWED_INSTANCES=lemmy-alpha,lemmy-beta
+      - LEMMY_PORT=8560
+      - LEMMY_SETUP__ADMIN_USERNAME=lemmy_gamma
+      - LEMMY_SETUP__ADMIN_PASSWORD=lemmy
+      - LEMMY_SETUP__SITE_NAME=lemmy-gamma
+      - RUST_BACKTRACE=1
+      - RUST_LOG=debug
+    depends_on:
+      - postgres_gamma
+  postgres_gamma:
+    image: postgres:12-alpine
+    environment:
+      - POSTGRES_USER=lemmy
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=lemmy
+    volumes:
+      - ./volumes/postgres_gamma:/var/lib/postgresql/data
+
+  iframely:
+    image: dogbin/iframely:latest
+    volumes:
+      - ../iframely.config.local.js:/iframely/config.local.js:ro
+    restart: always

docker/travis/docker_push.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+docker tag dessalines/lemmy:travis \
+  dessalines/lemmy:v0.7.28
+docker push dessalines/lemmy:v0.7.28

docker/travis/run-tests.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+# make sure there are no old containers or old data around
+sudo docker-compose down
+sudo rm -rf volumes
+
+mkdir -p volumes/pictrs_{alpha,beta,gamma}
+sudo chown -R 991:991 volumes/pictrs_{alpha,beta,gamma}
+
+sudo docker build ../../ --file ../prod/Dockerfile --tag dessalines/lemmy:travis
+
+sudo docker-compose up -d
+
+pushd ../../ui
+echo "Waiting for Lemmy to start..."
+while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8540/api/v1/site')" != "200" ]]; do sleep 1; done
+while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8550/api/v1/site')" != "200" ]]; do sleep 1; done
+while [[ "$(curl -s -o /dev/null -w '%{http_code}' 'localhost:8560/api/v1/site')" != "200" ]]; do sleep 1; done
+yarn
+yarn api-test
+popd
+
+sudo docker-compose down
+
+sudo rm -r volumes/