From 4d9b563d22fd4f51a735e6f53ada9315c959d373 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 18 Jul 2024 17:22:02 +0200 Subject: test: run sharding tests in ci Signed-off-by: Robin Appelman --- .github/workflows/phpunit-mysql-sharding.yml | 184 +++++++++++++++++++++++++++ tests/preseed-config.php | 21 +++ 2 files changed, 205 insertions(+) create mode 100644 .github/workflows/phpunit-mysql-sharding.yml diff --git a/.github/workflows/phpunit-mysql-sharding.yml b/.github/workflows/phpunit-mysql-sharding.yml new file mode 100644 index 00000000000..40727b385c4 --- /dev/null +++ b/.github/workflows/phpunit-mysql-sharding.yml @@ -0,0 +1,184 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization +# +# SPDX-FileCopyrightText: 2022-2024 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: MIT + +name: PHPUnit sharding + +on: + pull_request: + schedule: + - cron: "5 2 * * *" + +permissions: + contents: read + +concurrency: + group: phpunit-mysql-sharding-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest-low + + outputs: + src: ${{ steps.changes.outputs.src }} + + steps: + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-mysql: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + matrix: + php-versions: ['8.1'] + mysql-versions: ['8.4'] + + name: Sharding - MySQL ${{ matrix.mysql-versions }} (PHP ${{ matrix.php-versions }}) - database tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + mysql: + image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: oc_autotest + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + shard1: + image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest + ports: + - 5001:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: nextcloud + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + shard2: + image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest + ports: + - 5002:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: nextcloud + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + shard3: + image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest + ports: + - 5003:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: nextcloud + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + shard4: + image: ghcr.io/nextcloud/continuous-integration-mysql-${{ matrix.mysql-versions }}:latest + ports: + - 5004:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: nextcloud + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + + steps: + - name: Checkout server + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@2e947f1f6932d141d076ca441d0e1e881775e95b #v2.31.0 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + ini-file: development + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Enable ONLY_FULL_GROUP_BY MySQL option + run: | + echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + + - name: Set up Nextcloud + env: + DB_PORT: 4444 + SHARDING: 1 + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php | grep -i -C9999 error && echo "Error during app setup" && exit 1 || exit 0 + + - name: PHPUnit + run: composer run test:db ${{ matrix.coverage && ' -- --coverage-clover ./clover.db.xml' || '' }} + + - name: Upload db code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v4.1.1 + with: + files: ./clover.db.xml + flags: phpunit-mysql + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest-low + needs: [changes, phpunit-mysql] + + if: always() + + name: phpunit-mysql-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mysql.result != 'success' }}; then exit 1; fi diff --git a/tests/preseed-config.php b/tests/preseed-config.php index 3739335676a..7c341278027 100644 --- a/tests/preseed-config.php +++ b/tests/preseed-config.php @@ -78,3 +78,24 @@ if (getenv('OBJECT_STORE') === 's3') { ] ]; } + +if (getenv('SHARDING') == '1') { + $CONFIG['dbsharding'] = [ + "filecache" => [ + "shards" => [ + [ + "port" => 5001, + ], + [ + "port" => 5002, + ], + [ + "port" => 5003, + ], + [ + "port" => 5004, + ], + ] + ] + ]; +} -- cgit v1.2.3