From 66e5baf4bc6270eb5132a17ba04ac3a95972205e Mon Sep 17 00:00:00 2001 From: Ferdinand Thiessen Date: Sat, 10 Feb 2024 22:40:06 +0100 Subject: [PATCH] chore(CI): Make `files_external` CI workflows consistent Signed-off-by: Ferdinand Thiessen --- .../{ftp.yml => files-external-ftp.yml} | 73 +++++++++--- .github/workflows/files-external-s3.yml | 1 - .github/workflows/files-external-sftp.yml | 109 ++++++++++++++++++ ...os.yml => files-external-smb-kerberos.yml} | 54 +++++++-- .github/workflows/files-external-webdav.yml | 1 - .github/workflows/sftp.yml | 73 ------------ 6 files changed, 209 insertions(+), 102 deletions(-) rename .github/workflows/{ftp.yml => files-external-ftp.yml} (58%) create mode 100644 .github/workflows/files-external-sftp.yml rename .github/workflows/{smb-kerberos.yml => files-external-smb-kerberos.yml} (58%) delete mode 100644 .github/workflows/sftp.yml diff --git a/.github/workflows/ftp.yml b/.github/workflows/files-external-ftp.yml similarity index 58% rename from .github/workflows/ftp.yml rename to .github/workflows/files-external-ftp.yml index cddb92ae94b..77ff4218e49 100644 --- a/.github/workflows/ftp.yml +++ b/.github/workflows/files-external-ftp.yml @@ -1,29 +1,51 @@ -name: FTP unit tests +name: PHPUnit files_external FTP on: pull_request: - paths: - - '.github/**' - - 'apps/files_external/**' + schedule: + - cron: "5 2 * * *" concurrency: - group: ftp-${{ github.head_ref || github.run_id }} + group: files-external-ftp-${{ github.head_ref || github.run_id }} cancel-in-progress: true -env: - APP_NAME: files_external - jobs: - ftp-tests: + changes: + runs-on: ubuntu-latest-low + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - 'apps/files_external/**' + - 'vendor/**' + - 'vendor-bin/**' + - 'composer.json' + - 'composer.lock' + - '**.php' + + files-external-ftp: runs-on: ubuntu-latest + needs: changes - if: ${{ github.repository_owner != 'nextcloud-gmbh' }} + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} strategy: # do not stop on another job's failure fail-fast: false matrix: - php-versions: ['8.0'] + php-versions: ['8.0', '8.3'] ftpd: ['proftpd', 'vsftpd', 'pure-ftpd'] + include: + - php-versions: '8.0' + coverage: ${{ github.event_name != 'pull_request' }} name: php${{ matrix.php-versions }}-${{ matrix.ftpd }} @@ -42,11 +64,14 @@ jobs: if [[ "${{ matrix.ftpd }}" == 'proftpd' ]]; then docker run --name ftp -d --net host -e PASV_ADDRESS=127.0.0.1 -e FTPUSER_NAME=test -v /tmp/secret.txt:/run/secrets/ftp-user-password-secret -v /tmp/ftp:/home/test instantlinux/proftpd; fi if [[ "${{ matrix.ftpd }}" == 'vsftpd' ]]; then docker run --name ftp -d --net host -e FTP_USER=test -e FTP_PASS=test -e PASV_ADDRESS=127.0.0.1 -v /tmp/ftp:/home/vsftpd/test fauria/vsftpd; fi if [[ "${{ matrix.ftpd }}" == 'pure-ftpd' ]]; then docker run --name ftp -d --net host -e "PUBLICHOST=localhost" -e FTP_USER_NAME=test -e FTP_USER_PASS=test -e FTP_USER_HOME=/home/test -v /tmp/ftp:/home/test -v /tmp/ftp:/etc/pure-ftpd/passwd stilliard/pure-ftpd; fi + - name: Set up php ${{ matrix.php-versions }} uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 with: php-version: ${{ matrix.php-versions }} - extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -55,8 +80,9 @@ jobs: composer install mkdir data ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password - ./occ app:enable --force ${{ env.APP_NAME }} - php -S localhost:8080 & + ./occ app:enable --force files_external + echo " true,'host' => 'localhost','user' => 'test','password' => 'test', 'root' => '${{ env.FTP_ROOT }}'];" > apps/files_external/tests/config.ftp.php + - name: smoketest ftp run: | php -r 'var_dump(file_put_contents("ftp://test:test@localhost${{ env.FTP_ROOT }}/ftp.txt", "asd"));' @@ -64,10 +90,19 @@ jobs: php -r 'var_dump(mkdir("ftp://test:test@localhost${{ env.FTP_ROOT }}/asdads"));' ls -l /tmp/ftp [ -f /tmp/ftp/ftp.txt ] + - name: PHPUnit - run: | - echo " true,'host' => 'localhost','user' => 'test','password' => 'test', 'root' => '${{ env.FTP_ROOT }}'];" > apps/${{ env.APP_NAME }}/tests/config.ftp.php - composer run test:files_external apps/files_external/tests/Storage/FtpTest.php + run: composer run test:files_external -- \ + apps/files_external/tests/Storage/FtpTest.php \ + ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-files-external-ftp + - name: ftpd logs if: always() run: | @@ -75,10 +110,10 @@ jobs: ftp-summary: runs-on: ubuntu-latest-low - needs: ftp-tests + needs: [changes, files-external-ftp] if: always() steps: - name: Summary status - run: if ${{ needs.ftp-tests.result != 'success' }}; then exit 1; fi + run: if ${{ needs.changes.outputs.src != 'false' && needs.files-external-ftp.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/files-external-s3.yml b/.github/workflows/files-external-s3.yml index cf60e87d000..5a0446c06b3 100644 --- a/.github/workflows/files-external-s3.yml +++ b/.github/workflows/files-external-s3.yml @@ -27,7 +27,6 @@ jobs: - 'apps/files_external/**' - 'vendor/**' - 'vendor-bin/**' - - '.php-cs-fixer.dist.php' - 'composer.json' - 'composer.lock' - '**.php' diff --git a/.github/workflows/files-external-sftp.yml b/.github/workflows/files-external-sftp.yml new file mode 100644 index 00000000000..666cec1da82 --- /dev/null +++ b/.github/workflows/files-external-sftp.yml @@ -0,0 +1,109 @@ +name: PHPUnit files_external sFTP +on: + pull_request: + schedule: + - cron: "5 2 * * *" + +concurrency: + group: files-external-sftp-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest-low + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - 'apps/files_external/**' + - 'vendor/**' + - 'vendor-bin/**' + - 'composer.json' + - 'composer.lock' + - '**.php' + + files-external-sftp: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} + + strategy: + # do not stop on another job's failure + fail-fast: false + matrix: + php-versions: ['8.0', '8.3'] + sftpd: ['openssh'] + include: + - php-versions: '8.0' + coverage: ${{ github.event_name != 'pull_request' }} + + name: php${{ matrix.php-versions }}-${{ matrix.sftpd }} + + steps: + - name: Checkout server + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + submodules: true + + - name: Set up sftpd + run: | + sudo mkdir /tmp/sftp + sudo chown -R 0777 /tmp/sftp + if [[ "${{ matrix.sftpd }}" == 'openssh' ]]; then docker run -p 2222:22 --name sftp -d -v /tmp/sftp:/home/test atmoz/sftp "test:test:::data"; fi + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, sqlite, pdo_sqlite + coverage: ${{ matrix.coverage && 'xdebug' || 'none' }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Nextcloud + run: | + composer install + mkdir data + ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password + ./occ app:enable --force files_external + echo " true, 'host' => 'localhost:2222','user' => 'test','password' => 'test', 'root' => 'data'];" > apps/files_external/tests/config.sftp.php + + - name: PHPUnit + run: composer run test:files_external -- \ + apps/files_external/tests/Storage/SftpTest.php \ + apps/files_external/tests/Storage/SFTP_KeyTest.php \ + ${{ matrix.coverage && ' --coverage-clover ./clover.xml' || '' }} + + - name: Upload code coverage + if: ${{ !cancelled() && matrix.coverage }} + uses: codecov/codecov-action@v3 + with: + files: ./clover.xml + flags: phpunit-files-external-sftp + + - name: sftpd logs + if: always() + run: | + ls -l /tmp/sftp + docker logs sftp + + sftp-summary: + runs-on: ubuntu-latest-low + needs: [changes, files-external-sftp] + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.files-external-sftp.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/smb-kerberos.yml b/.github/workflows/files-external-smb-kerberos.yml similarity index 58% rename from .github/workflows/smb-kerberos.yml rename to .github/workflows/files-external-smb-kerberos.yml index 41438988346..9b170820f85 100644 --- a/.github/workflows/smb-kerberos.yml +++ b/.github/workflows/files-external-smb-kerberos.yml @@ -1,19 +1,41 @@ name: Samba Kerberos SSO on: pull_request: - paths: - - 'apps/files_external/**' - - '.github/workflows/smb-kerberos.yml' + schedule: + - cron: "5 2 * * *" concurrency: - group: smb-kerberos-${{ github.head_ref || github.run_id }} + group: files-external-smb-kerberos-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: - smb-kerberos-tests: - runs-on: ubuntu-22.04 + changes: + runs-on: ubuntu-latest-low - if: ${{ github.repository_owner != 'nextcloud-gmbh' }} + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - 'apps/files_external/**' + - 'vendor/**' + - 'vendor-bin/**' + - 'composer.json' + - 'composer.lock' + - '**.php' + + files-external-smb-kerberos: + runs-on: ubuntu-latest + needs: changes + + if: ${{ github.repository_owner != 'nextcloud-gmbh' && needs.changes.outputs.src != 'false' }} name: smb-kerberos-sso @@ -22,11 +44,13 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: submodules: true + - name: Checkout user_saml uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: repository: nextcloud/user_saml path: apps/user_saml + - name: Pull images run: | docker pull ghcr.io/icewind1991/samba-krb-test-dc @@ -35,21 +59,35 @@ jobs: docker tag ghcr.io/icewind1991/samba-krb-test-dc icewind1991/samba-krb-test-dc docker tag ghcr.io/icewind1991/samba-krb-test-apache icewind1991/samba-krb-test-apache docker tag ghcr.io/icewind1991/samba-krb-test-client icewind1991/samba-krb-test-client + - name: Setup AD-DC run: | DC_IP=$(apps/files_external/tests/sso-setup/start-dc.sh) sleep 1 apps/files_external/tests/sso-setup/start-apache.sh $DC_IP $PWD echo "DC_IP=$DC_IP" >> $GITHUB_ENV + - name: Set up Nextcloud run: | apps/files_external/tests/sso-setup/setup-sso-nc.sh + - name: Test SSO run: | apps/files_external/tests/sso-setup/test-sso-smb.sh ${{ env.DC_IP }} + - name: Show logs - if: failure() + if: always() run: | FILEPATH=$(docker exec --user 33 apache ./occ log:file | grep "Log file:" | cut -d' ' -f3) echo "$FILEPATH:" docker exec --user 33 apache cat $FILEPATH + + sftp-summary: + runs-on: ubuntu-latest-low + needs: [changes, files-external-smb-kerberos] + + if: always() + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.files-external-smb-kerberos.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/files-external-webdav.yml b/.github/workflows/files-external-webdav.yml index 4effb64c20e..85f2ce6e8c2 100644 --- a/.github/workflows/files-external-webdav.yml +++ b/.github/workflows/files-external-webdav.yml @@ -27,7 +27,6 @@ jobs: - 'apps/files_external/**' - 'vendor/**' - 'vendor-bin/**' - - '.php-cs-fixer.dist.php' - 'composer.json' - 'composer.lock' - '**.php' diff --git a/.github/workflows/sftp.yml b/.github/workflows/sftp.yml deleted file mode 100644 index 1de9113bd8c..00000000000 --- a/.github/workflows/sftp.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: SFTP unit tests -on: - pull_request: - paths: - - 'apps/files_external/**' - -env: - APP_NAME: files_external - -concurrency: - group: sftp-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - sftp-tests: - runs-on: ubuntu-latest - - if: ${{ github.repository_owner != 'nextcloud-gmbh' }} - - strategy: - # do not stop on another job's failure - fail-fast: false - matrix: - php-versions: ['8.0'] - sftpd: ['openssh'] - - name: php${{ matrix.php-versions }}-${{ matrix.sftpd }} - - steps: - - name: Checkout server - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - with: - submodules: true - - - name: Set up sftpd - run: | - sudo mkdir /tmp/sftp - sudo chown -R 0777 /tmp/sftp - if [[ "${{ matrix.sftpd }}" == 'openssh' ]]; then docker run -p 2222:22 --name sftp -d -v /tmp/sftp:/home/test atmoz/sftp "test:test:::data"; fi - - name: Set up php ${{ matrix.php-versions }} - uses: shivammathur/setup-php@c5fc0d8281aba02c7fda07d3a70cc5371548067d #v2.25.2 - with: - php-version: ${{ matrix.php-versions }} - extensions: mbstring, fileinfo, intl, sqlite, pdo_sqlite, zip, gd - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Nextcloud - run: | - composer install - mkdir data - ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password - ./occ app:enable --force ${{ env.APP_NAME }} - php -S localhost:8080 & - - name: PHPUnit - run: | - echo " true, 'host' => 'localhost:2222','user' => 'test','password' => 'test', 'root' => 'data'];" > apps/${{ env.APP_NAME }}/tests/config.sftp.php - composer run test:files_external apps/files_external/tests/Storage/SftpTest.php - - name: sftpd logs - if: always() - run: | - ls -l /tmp/sftp - docker logs sftp - - sftp-summary: - runs-on: ubuntu-latest-low - needs: sftp-tests - - if: always() - - steps: - - name: Summary status - run: if ${{ needs.sftp-tests.result != 'success' }}; then exit 1; fi -- 2.39.5