fedi/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-22 16:47:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Split CI pipelines (#23385)

Browse source 
- This PR attempts to split our various DB tests into separate
pipelines.
- It splits up some of the extra feature-related tests rather than
having most of them in the MySQL test.
- It disables the race detector for some of the pipelines as well, as it
can cause slower runs and is mostly redundant when the pipelines just
swap DBs.
- It builds without SQLite support for any of the non-SQLite pipelines.
- It moves the e2e test to using SQLite rather than PG (partially
because I moved the minio tests to PG and that mucked up the test
config, and partially because it avoids another running service)
- It splits up the `go mod download` task in the Makefile from the tool
installation, as the tools are only needed in the compliance pipeline.
(Arguably even some of the tools aren't needed there, but that could be
a follow-up PR)
- SQLite is now the only arm64 pipeline, moving PG back to amd64 which
can leverage autoscaler

Should resolve #22010 - one thing that wasn't changed here but is
mentioned in that issue, unit tests are needed in the same pipeline as
an integration test in order to form a complete coverage report (at
least as far as I could tell), so for now it remains in a pipeline with
a DB integration test.

Please let me know if I've inadvertently changed something that was how
it was on purpose.

---

I will say sometimes it's hard to pin down the average time, as a
pipeline could be waiting for a runner for X minutes and that brings the
total up by X minutes as well, but overall this does seem to be faster
on average.

---------

Signed-off-by: jolheiser <john.olheiser@gmail.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
John Olheiser 2023-03-10 00:13:17 -06:00 committed by GitHub
parent d647e74502
commit f92e0a4018
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 374 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

482
.drone.yml
View file

@ -32,6 +32,7 @@ steps:
pull: always
commands:
- make deps-backend
- make deps-tools
volumes:
- name: deps
path: /go
@ -168,7 +169,112 @@ steps:
---
kind: pipeline
type: docker
name: testing-amd64
name: testing-pgsql
platform:
os: linux
arch: amd64
depends_on:
- compliance
trigger:
event:
- push
- tag
- pull_request
paths:
exclude:
- docs/**
volumes:
- name: deps
temp: {}
services:
- name: pgsql
pull: default
image: postgres:15
environment:
POSTGRES_DB: test
POSTGRES_PASSWORD: postgres
- name: ldap
image: gitea/test-openldap:latest
pull: always
- name: minio
image: minio/minio:RELEASE.2021-03-12T00-00-47Z
pull: always
commands:
- minio server /data
environment:
MINIO_ACCESS_KEY: 123456
MINIO_SECRET_KEY: 12345678
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git config --global --add safe.directory /drone/src
- git fetch --tags --force
when:
event:
exclude:
- pull_request
- name: deps-backend
image: golang:1.20
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: prepare-test-env
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
pull: always
commands:
- ./build/test-env-prepare.sh
- name: build
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- ./build/test-env-check.sh
- make backend
environment:
GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: test-pgsql
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- timeout -s ABRT 50m make test-pgsql-migration test-pgsql
environment:
GOPROXY: https://goproxy.io
TAGS: bindata gogit
RACE_ENABLED: true
TEST_TAGS: gogit
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on: [build]
volumes:
- name: deps
path: /go
---
kind: pipeline
type: docker
name: testing-mysql
platform:
os: linux
@ -198,40 +304,12 @@ services:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: test
- name: mysql8
image: mysql:8
pull: always
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: testgitea
- name: mssql
image: mcr.microsoft.com/mssql/server:latest
pull: always
environment:
ACCEPT_EULA: Y
MSSQL_PID: Standard
SA_PASSWORD: MwantsaSecurePassword1
- name: ldap
image: gitea/test-openldap:latest
pull: always
- name: elasticsearch
image: elasticsearch:7.5.0
pull: always
environment:
discovery.type: single-node
- name: minio
image: minio/minio:RELEASE.2021-03-12T00-00-47Z
pull: always
commands:
- minio server /data
environment:
MINIO_ACCESS_KEY: 123456
MINIO_SECRET_KEY: 12345678
- name: smtpimap
image: tabascoterrier/docker-imap-devel:latest
pull: always
@ -257,12 +335,6 @@ steps:
- name: deps
path: /go
- name: tag-pre-condition
image: drone/git
pull: always
commands:
- git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
- name: prepare-test-env
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
pull: always
@ -278,7 +350,7 @@ steps:
environment:
GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
TAGS: bindata
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
@ -291,7 +363,7 @@ steps:
- make unit-test-coverage test-check
environment:
GOPROXY: https://goproxy.io
TAGS: bindata sqlite sqlite_unlock_notify
TAGS: bindata
RACE_ENABLED: true
GITHUB_READ_TOKEN:
from_secret: github_read_token
@ -307,7 +379,7 @@ steps:
- make unit-test-coverage test-check
environment:
GOPROXY: https://goproxy.io
TAGS: bindata gogit sqlite sqlite_unlock_notify
TAGS: bindata gogit
RACE_ENABLED: true
GITHUB_READ_TOKEN:
from_secret: github_read_token
@ -325,7 +397,6 @@ steps:
GOPROXY: https://goproxy.io
TAGS: bindata
RACE_ENABLED: true
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200"
depends_on: [build]
@ -333,38 +404,6 @@ steps:
- name: deps
path: /go
- name: test-mysql8
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- timeout -s ABRT 50m make test-mysql8-migration test-mysql8
environment:
GOPROXY: https://goproxy.io
TAGS: bindata
RACE_ENABLED: true
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on: [build]
volumes:
- name: deps
path: /go
- name: test-mssql
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- make test-mssql-migration test-mssql
environment:
GOPROXY: https://goproxy.io
TAGS: bindata
RACE_ENABLED: true
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on: [build]
volumes:
- name: deps
path: /go
- name: generate-coverage
image: golang:1.20
commands:
@ -398,7 +437,186 @@ steps:
---
kind: pipeline
name: testing-arm64
type: docker
name: testing-mysql8
platform:
os: linux
arch: amd64
depends_on:
- compliance
trigger:
event:
- push
- tag
- pull_request
paths:
exclude:
- docs/**
volumes:
- name: deps
temp: {}
services:
- name: mysql8
image: mysql:8
pull: always
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: yes
MYSQL_DATABASE: testgitea
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git config --global --add safe.directory /drone/src
- git fetch --tags --force
when:
event:
exclude:
- pull_request
- name: deps-backend
image: golang:1.20
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: prepare-test-env
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
pull: always
commands:
- ./build/test-env-prepare.sh
- name: build
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- ./build/test-env-check.sh
- make backend
environment:
GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: test-mysql8
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- timeout -s ABRT 50m make test-mysql8-migration test-mysql8
environment:
GOPROXY: https://goproxy.io
TAGS: bindata
USE_REPO_TEST_DIR: 1
depends_on: [build]
volumes:
- name: deps
path: /go
---
kind: pipeline
type: docker
name: testing-mssql
platform:
os: linux
arch: amd64
depends_on:
- compliance
trigger:
event:
- push
- tag
- pull_request
paths:
exclude:
- docs/**
volumes:
- name: deps
temp: {}
services:
- name: mssql
image: mcr.microsoft.com/mssql/server:latest
pull: always
environment:
ACCEPT_EULA: Y
MSSQL_PID: Standard
SA_PASSWORD: MwantsaSecurePassword1
steps:
- name: fetch-tags
image: docker:git
pull: always
commands:
- git config --global --add safe.directory /drone/src
- git fetch --tags --force
when:
event:
exclude:
- pull_request
- name: deps-backend
image: golang:1.20
pull: always
commands:
- make deps-backend
volumes:
- name: deps
path: /go
- name: prepare-test-env
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
pull: always
commands:
- ./build/test-env-prepare.sh
- name: build
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- ./build/test-env-check.sh
- make backend
environment:
GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata
depends_on: [deps-backend, prepare-test-env]
volumes:
- name: deps
path: /go
- name: test-mssql
image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- make test-mssql-migration test-mssql
environment:
GOPROXY: https://goproxy.io
TAGS: bindata
USE_REPO_TEST_DIR: 1
depends_on: [build]
volumes:
- name: deps
path: /go
---
kind: pipeline
name: testing-sqlite
platform:
os: linux
@ -420,18 +638,6 @@ volumes:
- name: deps
temp: {}
services:
- name: pgsql
pull: default
image: postgres:10
environment:
POSTGRES_DB: test
POSTGRES_PASSWORD: postgres
- name: ldap
pull: default
image: gitea/test-openldap:latest
steps:
- name: fetch-tags
image: docker:git
@ -490,23 +696,6 @@ steps:
- name: deps
path: /go
- name: test-pgsql
image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env
user: gitea
commands:
- timeout -s ABRT 50m make test-pgsql-migration test-pgsql
environment:
GOPROXY: https://goproxy.io
TAGS: bindata gogit
RACE_ENABLED: true
TEST_TAGS: gogit
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on: [build]
volumes:
- name: deps
path: /go
---
kind: pipeline
type: docker
@ -530,15 +719,6 @@ volumes:
- name: deps
temp: {}
services:
- name: pgsql
pull: default
image: postgres:10
environment:
POSTGRES_DB: testgitea-e2e
POSTGRES_PASSWORD: postgres
POSTGRES_INITDB_ARGS: --encoding=UTF8 --lc-collate='en_US.UTF-8' --lc-ctype='en_US.UTF-8'
steps:
- name: deps-frontend
image: node:18
@ -568,14 +748,12 @@ steps:
- curl -sLO https://go.dev/dl/go1.20.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.20.linux-amd64.tar.gz
- groupadd --gid 1001 gitea && useradd -m --gid 1001 --uid 1001 gitea
- apt-get -qq update && apt-get -qqy install build-essential
- export TEST_PGSQL_SCHEMA=''
- ./build/test-env-prepare.sh
- su gitea bash -c "export PATH=$PATH:/usr/local/go/bin && timeout -s ABRT 40m make test-e2e-pgsql"
- su gitea bash -c "export PATH=$PATH:/usr/local/go/bin && timeout -s ABRT 40m make test-e2e-sqlite"
environment:
GOPROXY: https://goproxy.io
GOSUMDB: sum.golang.org
USE_REPO_TEST_DIR: 1
TEST_PGSQL_DBNAME: 'testgitea-e2e'
DEBIAN_FRONTEND: noninteractive
depends_on: [build-frontend, deps-backend]
volumes:
@ -709,8 +887,11 @@ trigger:
- docs/**
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
volumes:
- name: deps
@ -842,8 +1023,11 @@ trigger:
- tag
depends_on:
- testing-arm64
- testing-amd64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
volumes:
- name: deps
@ -994,8 +1178,11 @@ platform:
arch: amd64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1064,8 +1251,11 @@ platform:
arch: amd64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1129,8 +1319,11 @@ platform:
arch: amd64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1192,8 +1385,11 @@ platform:
arch: amd64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1292,8 +1488,11 @@ platform:
arch: arm64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1362,8 +1561,11 @@ platform:
arch: arm64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1427,8 +1629,11 @@ platform:
arch: arm64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1493,8 +1698,11 @@ platform:
arch: arm64
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
trigger:
ref:
@ -1607,7 +1815,6 @@ platform:
steps:
- name: manifest-rootless
pull: always
image: plugins/manifest
pull: always
settings:
@ -1671,8 +1878,11 @@ trigger:
- failure
depends_on:
- testing-amd64
- testing-arm64
- testing-mysql
- testing-mysql8
- testing-mssql
- testing-pgsql
- testing-sqlite
- release-version
- release-latest
- docker-linux-amd64-release

6
Makefile
View file

@ -190,6 +190,7 @@ help:
@echo " - deps install dependencies"
@echo " - deps-frontend install frontend dependencies"
@echo " - deps-backend install backend dependencies"
@echo " - deps-tools install tool dependencies"
@echo " - lint lint everything"
@echo " - lint-frontend lint frontend files"
@echo " - lint-backend lint backend files"
@ -821,7 +822,7 @@ docs:
cd docs; make trans-copy clean build-offline;
.PHONY: deps
deps: deps-frontend deps-backend
deps: deps-frontend deps-backend deps-tools
.PHONY: deps-frontend
deps-frontend: node_modules
@ -829,6 +830,9 @@ deps-frontend: node_modules
.PHONY: deps-backend
deps-backend:
$(GO) mod download
.PHONY: deps-tools
deps-tools:
$(GO) install $(AIR_PACKAGE)
$(GO) install $(EDITORCONFIG_CHECKER_PACKAGE)
$(GO) install $(ERRCHECK_PACKAGE)

25
tests/mysql.ini.tmpl
View file

@ -55,28 +55,6 @@ LFS_START_SERVER = true
LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
SSH_TRUSTED_USER_CA_KEYS = ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCb4DC1dMFnJ6pXWo7GMxTchtzmJHYzfN6sZ9FAPFR4ijMLfGki+olvOMO5Fql1/yGnGfbELQa1S6y4shSvj/5K+zUFScmEXYf3Gcr87RqilLkyk16RS+cHNB1u87xTHbETaa3nyCJeGQRpd4IQ4NKob745mwDZ7jQBH8AZEng50Oh8y8fi8skBBBzaYp1ilgvzG740L7uex6fHV62myq0SXeCa+oJUjq326FU8y+Vsa32H8A3e7tOgXZPdt2TVNltx2S9H2WO8RMi7LfaSwARNfy1zu+bfR50r6ef8Yx5YKCMz4wWb1SHU1GS800mjOjlInLQORYRNMlSwR1+vLlVDciOqFapDSbj+YOVOawR0R1aqlSKpZkt33DuOBPx9qe6CVnIi7Z+Px/KqM+OLCzlLY/RS+LbxQpDWcfTVRiP+S5qRTcE3M3UioN/e0BE/1+MpX90IGpvVkA63ILYbKEa4bM3ASL7ChTCr6xN5XT+GpVJveFKK1cfNx9ExHI4rzYE=
[lfs]
MINIO_BASE_PATH = lfs/
[attachment]
MINIO_BASE_PATH = attachments/
[avatars]
MINIO_BASE_PATH = avatars/
[repo-avatars]
MINIO_BASE_PATH = repo-avatars/
[storage]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = minio:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = gitea
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[mailer]
ENABLED = true
MAILER_TYPE = dummy
@ -122,6 +100,9 @@ INSTALL_LOCK = true
SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
[lfs]
PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/data/lfs
[packages]
ENABLED = true

21
tests/pgsql.ini.tmpl
View file

@ -105,7 +105,26 @@ SECRET_KEY = 9pCviYTWSb
INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ
[lfs]
PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-pgsql/data/lfs
MINIO_BASE_PATH = lfs/
[attachment]
MINIO_BASE_PATH = attachments/
[avatars]
MINIO_BASE_PATH = avatars/
[repo-avatars]
MINIO_BASE_PATH = repo-avatars/
[storage]
STORAGE_TYPE = minio
SERVE_DIRECT = false
MINIO_ENDPOINT = minio:9000
MINIO_ACCESS_KEY_ID = 123456
MINIO_SECRET_ACCESS_KEY = 12345678
MINIO_BUCKET = gitea
MINIO_LOCATION = us-east-1
MINIO_USE_SSL = false
[packages]
ENABLED = true