From d06da0d9c6de8e77f8548fafcbaa99c9db0d954a Mon Sep 17 00:00:00 2001 From: Otto Richter Date: Thu, 5 Sep 2024 18:48:18 +0200 Subject: [PATCH 1/2] Update integration test README and remove outdated chinese version --- tests/integration/README.md | 50 +++++++++++++++------- tests/integration/README_ZH.md | 77 ---------------------------------- 2 files changed, 36 insertions(+), 91 deletions(-) delete mode 100644 tests/integration/README_ZH.md diff --git a/tests/integration/README.md b/tests/integration/README.md index c40f47452b..f0fbf94df9 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,5 +1,23 @@ # Integration tests +Thank you for your effort to provide good software tests for Forgejo. +Please also read the general testing instructions in the +[Forgejo contributor documentation](https://forgejo.org/docs/next/contributor/testing/). + +This file is meant to provide specific information for the integration tests +as well as some tips and tricks you should know. + +Feel free to extend this file with more instructions if you feel like you have something to share! + + +## How to run the tests? + +Before running any tests, please ensure you perform a clean build: + +``` +make clean build +``` + Integration tests can be run with make commands for the appropriate backends, namely: ```shell @@ -8,40 +26,39 @@ make test-pgsql make test-mysql ``` -Make sure to perform a clean build before running tests: -``` -make clean build -``` -## Run tests via local act_runner +### Run tests via local forgejo runner -### Run all jobs +If you have a [forgejo runner](https://code.forgejo.org/forgejo/runner/), +you can use it to run the test jobs: + +#### Run all jobs ``` -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest +forgejo-runner exec -W .forgejo/workflows/testing.yml --event=pull_request ``` Warning: This file defines many jobs, so it will be resource-intensive and therefore not recommended. -### Run single job +#### Run single job ```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -j +forgejo-runner exec -W .forgejo/workflows/testing.yml --event=pull_request -j ``` You can list all job names via: ```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -l +forgejo-runner exec -W .forgejo/workflows/testing.yml --event=pull_request -l ``` -## Run sqlite integration tests +### Run sqlite integration tests Start tests ``` make test-sqlite ``` -## Run MySQL integration tests +### Run MySQL integration tests Setup a MySQL database inside docker ``` docker run -e "MYSQL_DATABASE=test" -e "MYSQL_ALLOW_EMPTY_PASSWORD=yes" -p 3306:3306 --rm --name mysql mysql:latest #(just ctrl-c to stop db and clean the container) @@ -52,7 +69,7 @@ Start tests based on the database container TEST_MYSQL_HOST=localhost:3306 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=root TEST_MYSQL_PASSWORD='' make test-mysql ``` -## Run pgsql integration tests +### Run pgsql integration tests Setup a pgsql database inside docker ``` docker run -e "POSTGRES_DB=test" -p 5432:5432 --rm --name pgsql postgres:latest #(just ctrl-c to stop db and clean the container) @@ -62,7 +79,7 @@ Start tests based on the database container TEST_PGSQL_HOST=localhost:5432 TEST_PGSQL_DBNAME=test TEST_PGSQL_USERNAME=postgres TEST_PGSQL_PASSWORD=postgres make test-pgsql ``` -## Running individual tests +### Running individual tests Example command to run GPG test: @@ -99,3 +116,8 @@ SLOW_FLUSH = 5S ; 5s is the default value ```bash GITEA_SLOW_TEST_TIME="10s" GITEA_SLOW_FLUSH_TIME="5s" make test-sqlite ``` + +## Tips and tricks + +If you know noteworthy tests that can act as an inspiration for new tests, +please add some details here. diff --git a/tests/integration/README_ZH.md b/tests/integration/README_ZH.md deleted file mode 100644 index d0debf2fd6..0000000000 --- a/tests/integration/README_ZH.md +++ /dev/null @@ -1,77 +0,0 @@ -# 关于集成测试 - -使用如下 make 命令可以运行指定的集成测试： -```shell -make test-mysql -make test-pgsql -make test-sqlite -``` - -在执行集成测试命令前请确保清理了之前的构建环境，清理命令如下： -``` -make clean build -``` - -## 如何在本地 act_runner 上运行测试 - -### 运行所有任务 - -``` -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -``` - -警告:由于在此文件中定义了许多任务，因此此操作将花费太多的CPU和内存来运行。所以不建议这样做。 - -### 运行单个任务 - -```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -j -``` - -您可以通过以下方式列出所有任务名称: -```SHELL -act_runner exec -W ./.github/workflows/pull-db-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest -l -``` - -## 如何使用 sqlite 数据库进行集成测试 -使用该命令执行集成测试 -``` -make test-sqlite -``` - -## 如何使用 mysql 数据库进行集成测试 -首先在docker容器里部署一个 mysql 数据库 -``` -docker run -e "MYSQL_DATABASE=test" -e "MYSQL_ALLOW_EMPTY_PASSWORD=yes" -p 3306:3306 --rm --name mysql mysql:8 #(just ctrl-c to stop db and clean the container) -``` -之后便可以基于这个数据库进行集成测试 -``` -TEST_MYSQL_HOST=localhost:3306 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=root TEST_MYSQL_PASSWORD='' make test-mysql -``` - -## 如何使用 pgsql 数据库进行集成测试 -同上，首先在 docker 容器里部署一个 pgsql 数据库 -``` -docker run -e "POSTGRES_DB=test" -p 5432:5432 --rm --name pgsql postgres:14 #(just ctrl-c to stop db and clean the container) -``` -之后便可以基于这个数据库进行集成测试 -``` -TEST_PGSQL_HOST=localhost:5432 TEST_PGSQL_DBNAME=test TEST_PGSQL_USERNAME=postgres TEST_PGSQL_PASSWORD=postgres make test-pgsql -``` - -## 如何进行自定义的集成测试 - -下面的示例展示了怎样在集成测试中只进行 GPG 测试： - -sqlite 数据库: - -``` -make test-sqlite#GPG -``` - -其它数据库 (用 PGSQL 取代 MYSQL）: - -``` -TEST_MYSQL_HOST=localhost:1433 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=sa TEST_MYSQL_PASSWORD=MwantsaSecurePassword1 make test-mysql#GPG -``` - From 0f3ba99ff09931e384ba0153bcc3aafdc7d8a61c Mon Sep 17 00:00:00 2001 From: Otto Richter Date: Thu, 5 Sep 2024 19:21:21 +0200 Subject: [PATCH 2/2] Add detailed instructions for e2e testing - rewrite parts of and improve README - simplify repo declaration - list coverage in workflow path filter --- .forgejo/workflows/e2e.yml | 3 + tests/e2e/README.md | 185 ++++++++++++++++++++++++-------- tests/e2e/declare_repos_test.go | 12 ++- 3 files changed, 154 insertions(+), 46 deletions(-) diff --git a/.forgejo/workflows/e2e.yml b/.forgejo/workflows/e2e.yml index 949177eb8b..c6956560ca 100644 --- a/.forgejo/workflows/e2e.yml +++ b/.forgejo/workflows/e2e.yml @@ -8,6 +8,9 @@ on: - .forgejo/workflows/e2e.yml - tests/e2e/** - web_src/js/** + - web_src/css/form.css + - templates/webhook/shared-settings.tmpl + - templates/org/team/new.tmpl jobs: test-e2e: diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 3d17ee3a72..65520115e6 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,42 +1,93 @@ # End to end tests -E2e tests largely follow the same syntax as [integration tests](../integration). -Whereas integration tests are intended to mock and stress the back-end, server-side code, e2e tests the interface between front-end and back-end, as well as visual regressions with both assertions and visual comparisons. -They can be run with make commands for the appropriate backends, namely: -```shell -make test-sqlite -make test-pgsql -make test-mysql -``` +Thank you for your effort to provide good software tests for Forgejo. +Please also read the general testing instructions in the +[Forgejo contributor documentation](https://forgejo.org/docs/next/contributor/testing/) +and make sure to also check the +[Playwright documentation](https://playwright.dev/docs/intro) +for further information. + +This file is meant to provide specific information for the integration tests +as well as some tips and tricks you should know. + +Feel free to extend this file with more instructions if you feel like you have something to share! + + +## How to run the tests? + +Before running any tests, please ensure you perform a clean frontend build: -Make sure to perform a clean front-end build before running tests: ``` make clean frontend ``` -## Install playwright system dependencies +Whenever you modify frontend code (i.e. JavaScript and CSS files), +you need to create a new frontend build. + +For tests that require interactive Git repos, +you also need to ensure a Forgejo binary is ready to be used by Git hooks. +For this, you additionally need to run + +~~~ +make TAGS="sqlite sqlite_unlock_notify" backend +~~~ + +### Install dependencies + +Browsertesting is performed by playwright. +You need certain system libraries and playwright will download required browsers. +Playwright takes care of this when you run: + ``` npx playwright install-deps ``` -## Interactive testing +> **Note** +> On some operating systems, the installation of missing libraries can complicate testing certain browsers. +> It is often not necessary to test with all browsers locally. +> Choosing either Firefox or Chromium is fine. -You can make use of Playwright's integrated UI mode to run individual tests, -get feedback and visually trace what your browser is doing. -To do so, launch the debugserver using: +### Run all tests + +If you want to run the full test suite, you can use + +``` +make test-e2e-sqlite +``` + +### Interactive testing + +We recommend that you use interactive testing for the development. +After you performed the required builds, +you should use one shell to start the debugserver (and leave it running): ``` make test-e2e-debugserver ``` -Then launch the Playwright UI: +It allows you to explore the test data in your local browser, +and playwright to perform tests on it. + +> **Note** +> The modifications persist while the debugserver is running. +> If you modified things, it might be useful to restart it to get back to a fresh state. +> While writing playwright tests, you either +> need to ensure they are resilient against repeated runs +> (e.g. when only creating new content), +> or that they restore the initial state for the next browser run. + +#### With the playwright UI: + +Playwright ships with an integrated UI mode which allows you to +run individual tests and to debug them by seeing detailed traces of what playwright does. +Launch it with: ``` npx playwright test --ui ``` -You can also run individual tests while the debugserver using: +#### Running individual tests ``` npx playwright test actions.test.e2e.js:9 @@ -46,18 +97,29 @@ First, specify the complete test filename, and after the colon you can put the linenumber where the test is defined. -## Run all tests via local act_runner +#### With VSCodium or VSCode + +To debug a test, you can also use "Playwright Test" for +[VScodium](https://open-vsx.org/extension/ms-playwright/playwright) +or [VSCode](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). + + +### Run all tests via local act_runner + +If you have a [forgejo runner](https://code.forgejo.org/forgejo/runner/), +you can use it to run the test jobs: + ``` -act_runner exec -W ./.github/workflows/pull-e2e-tests.yml --event=pull_request --default-actions-url="https://github.com" -i catthehacker/ubuntu:runner-latest +forgejo-runner exec -W .forgejo/workflows/e2e.yml --event=pull_request ``` -## Run sqlite e2e tests -Start tests -``` -make test-e2e-sqlite -``` +### Run e2e tests with another database + +This approach is not currently used, +neither in the CI/CD nor by core contributors on their lcoal machines. +It is still documented for the sake of completeness: +You can also perform e2e tests using MariaDB/MySQL or PostgreSQL if you want. -## Run MySQL e2e tests Setup a MySQL database inside docker ``` docker run -e "MYSQL_DATABASE=test" -e "MYSQL_ALLOW_EMPTY_PASSWORD=yes" -p 3306:3306 --rm --name mysql mysql:latest #(just ctrl-c to stop db and clean the container) @@ -68,7 +130,6 @@ Start tests based on the database container TEST_MYSQL_HOST=localhost:3306 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=root TEST_MYSQL_PASSWORD='' make test-e2e-mysql ``` -## Run pgsql e2e tests Setup a pgsql database inside docker ``` docker run -e "POSTGRES_DB=test" -p 5432:5432 --rm --name pgsql postgres:latest #(just ctrl-c to stop db and clean the container) @@ -78,11 +139,12 @@ Start tests based on the database container TEST_PGSQL_HOST=localhost:5432 TEST_PGSQL_DBNAME=test TEST_PGSQL_USERNAME=postgres TEST_PGSQL_PASSWORD=postgres make test-e2e-pgsql ``` -## Running individual tests +### Running individual tests Example command to run `example.test.e2e.js` test file: -_Note: unlike integration tests, this filtering is at the file level, not function_ +> **Note** +> Unlike integration tests, this filtering is at the file level, not function For SQLite: @@ -90,13 +152,11 @@ For SQLite: make test-e2e-sqlite#example ``` -For PostgreSQL databases(replace `mysql` to `pgsql`): +### Visual testing -``` -TEST_MYSQL_HOST=localhost:1433 TEST_MYSQL_DBNAME=test TEST_MYSQL_USERNAME=sa TEST_MYSQL_PASSWORD=MwantsaSecurePassword1 make test-e2e-mysql#example -``` - -## Visual testing +> **Warning** +> This is not currently used by most Forgejo contributors. +> Your help to improve the situation and allow for visual testing is appreciated. Although the main goal of e2e is assertion testing, we have added a framework for visual regress testing. If you are working on front-end features, please use the following: - Check out `main`, `make clean frontend`, and run e2e tests with `VISUAL_TEST=1` to generate outputs. This will initially fail, as no screenshots exist. You can run the e2e tests again to assert it passes. @@ -106,14 +166,55 @@ VISUAL_TEST=1 will create screenshots in tests/e2e/test-snapshots. The test will ACCEPT_VISUAL=1 will overwrite the snapshot images with new images. -## With VSCodium or VSCode -To debug a test, you can use "Playwright Test" for -[VScodium](https://open-vsx.org/extension/ms-playwright/playwright) -or [VSCode](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). -Before doing that you will need to manually start a Forgejo instance and populate it -with data from `models/fixtures` by running: +## Tips and tricks -```sh -make TAGS='sqlite sqlite_unlock_notify' 'test-e2e-debugserver' -``` +If you know noteworthy tests that can act as an inspiration for new tests, +please add some details here. + +### Run tests very selectively + +Browser testing can take some time. +If you want to iterate fast, +save your time and only run very selected tests. +Use only one browser. + +### Skip Safari if it doesn't work + +Many contributors have issues getting Safari (webkit) +and especially Safari Mobile to work. + +At the top of your test function, you can use: + +~~~javascript +test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile.'); +~~~ + +### Don't forget the formatting. + +When writing tests without modifying other frontend code, +it is easy to forget that the JavaScript test files also need formatting. + +Run `make lint-frontend-fix`. + +### Define new repos + +Take a look at `declare_repos_test.go` to see how to add your repositories. +Feel free to improve the logic used there if you need more advanced functionality +(it is a simplified version of the code used in the integration tests). + +### Accessibility testing + +If you can, perform automated accessibility testing using +[AxeCore](https://github.com/dequelabs/axe-core-npm/blob/develop/packages/playwright/README.md). + +Take a look at `shared/forms.js` and some other places for inspiration. + +### List related files coverage + +If you think your playwright tests covers an important aspect of some template, CSS or backend files, +consider adding the paths to `.forgejo/workflows/e2e.yml` in the path filter. + +It ensures that future modifications to this file will be tested as well. + +Currently, we do not run the e2e tests on all changes. diff --git a/tests/e2e/declare_repos_test.go b/tests/e2e/declare_repos_test.go index 44ee79f388..7057b26b6f 100644 --- a/tests/e2e/declare_repos_test.go +++ b/tests/e2e/declare_repos_test.go @@ -21,15 +21,19 @@ import ( "github.com/stretchr/testify/require" ) +// first entry represents filename +// the following entries define the full file content over time type FileChanges [][]string // put your Git repo declarations in here // feel free to amend the helper function below or use the raw variant directly func DeclareGitRepos(t *testing.T) func() { - var cleanupFunctions []func() - cleanupFunctions = append(cleanupFunctions, newRepo(t, 2, "diff-test", FileChanges{ - {"testfile", "hello", "hallo", "hola", "native", "ubuntu-latest", "- runs-on: ubuntu-latest", "- runs-on: debian-latest"}, - })) + cleanupFunctions := []func(){ + newRepo(t, 2, "diff-test", FileChanges{ + {"testfile", "hello", "hallo", "hola", "native", "ubuntu-latest", "- runs-on: ubuntu-latest", "- runs-on: debian-latest"}, + }), + // add your repo declarations here + } return func() { for _, cleanup := range cleanupFunctions {