[Backport] CI optimisation & add github token env var (#9875)

* ci: use docker image for golangci-lint (#9737)

* ci: re-ordering Drone CI for optimizing time (#9719)

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try re-ordering for optimizing time

* ci: try offloading mysql8 to arm64

* Revert "ci: try offloading mysql8 to arm64"

This reverts commit c60de5db1c.

* ci: try offloading pgsql to arm64

* ci: activate ldap on arm64

* ci: test mysql8 in place pgsql arm64

* chore: clean un-needed move

* typo

* ci: revert runnning mysql on arm64

* ci: run compliance on arm

* chore: limit change

* chore: readd maybe need for release fetch-tags

* ci: remove docker-linux-amd64-dry-run

* ci: remove docker-linux-amd64-dry-run

* Revert "ci: remove docker-linux-amd64-dry-run"

This reverts commit 0715f65b11.

Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

* ci: use new mssql image (#9720)

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

* ci: run notify on arm64 (#9762)

* ci: run notify on arm64

Free one jobs on amrd64

* Update .drone.yml

* Update .drone.yml

* Update .drone.yml

Based on: https://github.com/appleboy/drone-discord/blob/master/.drone.yml#L339

* improve trigger

Co-authored-by: techknowlogick <matti@mdranta.net>

* ci: move some integration tests on arm64 (#9747)

* tests: configure github remaining limit + read token (#9800)

* ci: configure remaining github limmit

* prepend with github since package is common to all migrations

* add RefreshRate

* Update github.go

* add missing space

* go fmt

* Read env variable GITHUB_READ_TOKEN for token

* Update .drone.yml

* ci: simplify tag/release by always running coverage (#9774)

* ci: simplify tag/release by always running coverage

* use mod and vendor for unit test coverage

* remove not needed lfs for unit test

* use arm drone agent for docs (#9776)

* run translations pipeline on arm server (#9865)

* add git-check to Makefile

Co-authored-by: Antoine GIRARD <sapk@users.noreply.github.com>
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: techknowlogick <matti@mdranta.net>

.drone.yml

@@ -1,59 +1,16 @@
 ---
 kind: pipeline
-name: testing
+name: compliance
 
 platform:
   os: linux
-  arch: amd64
+  arch: arm64
 
 workspace:
   base: /go
   path: src/code.gitea.io/gitea
 
-services:
-  - name: mysql
-    pull: default
-    image: mysql:5.7
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: yes
-      MYSQL_DATABASE: test
-
-  - name: mysql8
-    pull: default
-    image: mysql:8.0
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: yes
-      MYSQL_DATABASE: testgitea
-
-  - name: pgsql
-    pull: default
-    image: postgres:9.5
-    environment:
-      POSTGRES_DB: test
-
-  - name: mssql
-    pull: default
-    image: microsoft/mssql-server-linux:latest
-    environment:
-      ACCEPT_EULA: Y
-      MSSQL_PID: Standard
-      SA_PASSWORD: MwantsaSecurePassword1
-
-  - name: ldap
-    pull: default
-    image: gitea/test-openldap:latest
-
 steps:
-  - name: fetch-tags
-    pull: default
-    image: docker:git
-    commands:
-      - git fetch --tags --force
-    when:
-      event:
-        exclude:
-          - pull_request
-
   - name: pre-build
     pull: always
     image: node:10 # this step is kept at the lowest version of node that we support
@@ -68,7 +25,6 @@ steps:
       GO111MODULE: on
       GOPROXY: off
     commands:
-      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
       - go build -mod=vendor -o gitea_no_gcc # test if build succeeds without the sqlite tag
 
   - name: build-linux-386
@@ -80,98 +36,94 @@ steps:
       GOOS: linux
       GOARCH: 386
     commands:
-      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
       - go build -mod=vendor -o gitea_linux_386 # test if compatible with 32 bit
 
-  - name: build
+  - name: check
     pull: always
     image: golang:1.13
     commands:
-      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
       - make clean
       - make golangci-lint
       - make revive
       - make swagger-check
       - make swagger-validate
       - make test-vendor
-      - make build
     environment:
       GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
       GOSUMDB: sum.golang.org
       TAGS: bindata sqlite sqlite_unlock_notify
 
-  - name: unit-test
-    pull: always
-    image: golang:1.13
-    commands:
-      - make unit-test-coverage
+---
+kind: pipeline
+name: testing-amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+workspace:
+  base: /go
+  path: src/code.gitea.io/gitea
+
+services:
+  - name: mysql
+    pull: default
+    image: mysql:5.7
     environment:
-      GOPROXY: off
-      TAGS: bindata sqlite sqlite_unlock_notify
-    depends_on:
-      - build
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - pull_request
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: test
 
-  - name: release-test
-    pull: always
-    image: golang:1.13
-    commands:
-      - make test
+  - name: mysql8
+    pull: default
+    image: mysql:8.0
     environment:
-      GOPROXY: off
-      TAGS: bindata sqlite sqlite_unlock_notify
-    depends_on:
-      - build
-    when:
-      branch:
-        - "release/*"
-      event:
-        - push
-        - pull_request
+      MYSQL_ALLOW_EMPTY_PASSWORD: yes
+      MYSQL_DATABASE: testgitea
 
-  - name: tag-pre-condition
-    pull: always
-    image: alpine/git
+  - name: mssql
+    pull: default
+    image: mcr.microsoft.com/mssql/server:latest
+    environment:
+      ACCEPT_EULA: Y
+      MSSQL_PID: Standard
+      SA_PASSWORD: MwantsaSecurePassword1
+
+  - name: ldap
+    pull: default
+    image: gitea/test-openldap:latest
+
+steps:
+  - name: fetch-tags
+    pull: default
+    image: docker:git
     commands:
-      - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
-    depends_on:
-      - build
+      - git fetch --tags --force
     when:
       event:
-        - tag
+        exclude:
+          - pull_request
 
-  - name: tag-test
+  - name: build
     pull: always
     image: golang:1.13
     commands:
-      - make test
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
+      - make build
     environment:
-      GOPROXY: off
-      TAGS: bindata
-    depends_on:
-      - tag-pre-condition
-    when:
-      event:
-        - tag
+      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify
 
-  - name: test-sqlite
+  - name: unit-test
     pull: always
     image: golang:1.13
     commands:
-      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-      - apt-get install -y git-lfs
-      - timeout -s ABRT 20m make test-sqlite-migration
-      - timeout -s ABRT 20m make test-sqlite
+      - make unit-test-coverage
     environment:
       GOPROXY: off
-      TAGS: bindata
-    depends_on:
-      - build
+      TAGS: bindata sqlite sqlite_unlock_notify
+      GITHUB_READ_TOKEN:
+        from_secret: github_read_token
 
   - name: test-mysql
     pull: always
@@ -187,30 +139,6 @@ steps:
       TEST_LDAP: 1
     depends_on:
       - build
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - pull_request
-
-  - name: tag-test-mysql
-    pull: always
-    image: golang:1.13
-    commands:
-      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-      - apt-get install -y git-lfs
-      - timeout -s ABRT 20m make test-mysql-migration
-      - timeout -s ABRT 20m make test-mysql
-    environment:
-      GOPROXY: off
-      TAGS: bindata
-      TEST_LDAP: 1
-    depends_on:
-      - build
-    when:
-      event:
-        - tag
 
   - name: test-mysql8
     pull: always
@@ -227,21 +155,6 @@ steps:
     depends_on:
       - build
 
-  - name: test-pgsql
-    pull: always
-    image: golang:1.13
-    commands:
-      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
-      - apt-get install -y git-lfs
-      - timeout -s ABRT 20m make test-pgsql-migration
-      - timeout -s ABRT 20m make test-pgsql
-    environment:
-      GOPROXY: off
-      TAGS: bindata
-      TEST_LDAP: 1
-    depends_on:
-      - build
-
   - name: test-mssql
     pull: always
     image: golang:1.13
@@ -293,13 +206,89 @@ steps:
         - push
         - pull_request
 
+
+
+---
+kind: pipeline
+name: testing-arm64
+
+platform:
+  os: linux
+  arch: arm64
+
+workspace:
+  base: /go
+  path: src/code.gitea.io/gitea
+
+services:
+  - name: pgsql
+    pull: default
+    image: postgres:9.5
+    environment:
+      POSTGRES_DB: test
+
+  - name: ldap
+    pull: default
+    image: gitea/test-openldap:latest
+
+steps:
+  - name: fetch-tags
+    pull: default
+    image: docker:git
+    commands:
+      - git fetch --tags --force
+    when:
+      event:
+        exclude:
+          - pull_request
+
+  - name: build
+    pull: always
+    image: golang:1.13
+    commands:
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
+      - make build
+    environment:
+      GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
+      GOSUMDB: sum.golang.org
+      TAGS: bindata sqlite sqlite_unlock_notify
+
+  - name: test-sqlite
+    pull: always
+    image: golang:1.13
+    commands:
+      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
+      - apt-get install -y git-lfs
+      - timeout -s ABRT 20m make test-sqlite-migration
+      - timeout -s ABRT 20m make test-sqlite
+    environment:
+      GOPROXY: off
+      TAGS: bindata
+    depends_on:
+      - build
+
+  - name: test-pgsql
+    pull: always
+    image: golang:1.13
+    commands:
+      - "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash"
+      - apt-get install -y git-lfs
+      - timeout -s ABRT 20m make test-pgsql-migration
+      - timeout -s ABRT 20m make test-pgsql
+    environment:
+      GOPROXY: off
+      TAGS: bindata
+      TEST_LDAP: 1
+    depends_on:
+      - build
+
 ---
 kind: pipeline
 name: translations
 
 platform:
   os: linux
-  arch: amd64
+  arch: arm64
 
 workspace:
   base: /go
@@ -378,7 +367,8 @@ trigger:
     - push
 
 depends_on:
-  - testing
+  - testing-amd64
+  - testing-arm64
   - translations
 
 steps:
@@ -476,7 +466,8 @@ trigger:
     - tag
 
 depends_on:
-  - testing
+  - testing-arm64
+  - testing-amd64
 
 steps:
   - name: fetch-tags
@@ -545,17 +536,14 @@ name: docs
 
 platform:
   os: linux
-  arch: amd64
-
-workspace:
-  base: /go
-  path: src/code.gitea.io/gitea
+  arch: arm64
 
 steps:
   - name: build-docs
     pull: always
-    image: webhippie/hugo:latest
+    image: plugins/hugo:latest
     commands:
+      - apk add --no-cache make bash curl
       - cd docs
       - make trans-copy
       - make clean
@@ -563,7 +551,7 @@ steps:
 
   - name: publish-docs
     pull: always
-    image: lucap/drone-netlify:latest
+    image: techknowlogick/drone-netlify:latest
     settings:
       path: docs/public/
       site_id: d2260bae-7861-4c02-8646-8f6440b12672
@@ -578,7 +566,7 @@ steps:
 
 ---
 kind: pipeline
-name: docker-linux-amd64
+name: docker-linux-amd64-release
 
 platform:
   os: linux
@@ -589,13 +577,13 @@ workspace:
   path: src/code.gitea.io/gitea
 
 depends_on:
-  - testing
+  - testing-amd64
+  - testing-arm64
 
 trigger:
   ref:
   - refs/heads/master
   - "refs/tags/**"
-  - "refs/pull/**"
 
 steps:
   - name: fetch-tags
@@ -603,23 +591,6 @@ steps:
     image: docker:git
     commands:
       - git fetch --tags --force
-    when:
-      event:
-        exclude:
-          - pull_request
-
-  - name: dryrun
-    pull: always
-    image: plugins/docker:linux-amd64
-    settings:
-      dry_run: true
-      repo: gitea/gitea
-      tags: linux-amd64
-      build_args:
-        - GOPROXY=off
-    when:
-      event:
-        - pull_request
 
   - name: publish
     pull: always
@@ -641,7 +612,7 @@ steps:
 
 ---
 kind: pipeline
-name: docker-linux-arm64
+name: docker-linux-arm64-dry-run
 
 platform:
   os: linux
@@ -652,25 +623,13 @@ workspace:
   path: src/code.gitea.io/gitea
 
 depends_on:
-  - testing
+  - compliance
 
 trigger:
   ref:
-  - refs/heads/master
-  - "refs/tags/**"
   - "refs/pull/**"
 
 steps:
-  - name: fetch-tags
-    pull: default
-    image: docker:git
-    commands:
-      - git fetch --tags --force
-    when:
-      event:
-        exclude:
-          - pull_request
-
   - name: dryrun
     pull: always
     image: plugins/docker:linux-arm64
@@ -684,6 +643,33 @@ steps:
       event:
         - pull_request
 
+---
+kind: pipeline
+name: docker-linux-arm64-release
+
+platform:
+  os: linux
+  arch: arm64
+
+workspace:
+  base: /go
+  path: src/code.gitea.io/gitea
+
+depends_on:
+  - testing-amd64
+  - testing-arm64
+
+trigger:
+  ref:
+  - refs/heads/master
+  - "refs/tags/**"
+steps:
+  - name: fetch-tags
+    pull: default
+    image: docker:git
+    commands:
+      - git fetch --tags --force
+
   - name: publish
     pull: always
     image: plugins/docker:linux-arm64
@@ -729,45 +715,49 @@ trigger:
   - "refs/tags/**"
 
 depends_on:
-  - docker-linux-amd64
-  - docker-linux-arm64
+  - docker-linux-amd64-release
+  - docker-linux-arm64-release
 
 ---
 kind: pipeline
-name: notify
+name: notifications
 
 platform:
   os: linux
-  arch: amd64
-
-workspace:
-  base: /go
-  path: src/code.gitea.io/gitea
+  arch: arm64
 
 clone:
   disable: true
 
-when:
+trigger:
+  branch:
+    - master
+    - "release/*"
+  event:
+    - push
+    - tag
   status:
     - success
     - failure
 
 depends_on:
-  - testing
+  - testing-amd64
+  - testing-arm64
   - translations
   - release-version
   - release-master
-  - docker-linux-amd64
-  - docker-linux-arm64
+  - docker-linux-amd64-release
+  - docker-linux-arm64-release
   - docker-manifest
   - docs
 
 steps:
   - name: discord
     pull: always
-    image: appleboy/drone-discord:1.0.0
-    environment:
-      DISCORD_WEBHOOK_ID:
+    image: appleboy/drone-discord:1.2.4
+    settings:
+      message: "{{#success build.status}} ✅  Build #{{build.number}} of `{{repo.name}}` succeeded.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{else}} ❌  Build #{{build.number}} of `{{repo.name}}` failed.\n\n📝 Commit by {{commit.author}} on `{{commit.branch}}`:\n``` {{commit.message}} ```\n\n🌐 {{ build.link }} {{/success}}\n"
+      webhook_id:
         from_secret: discord_webhook_id
-      DISCORD_WEBHOOK_TOKEN:
+      webhook_token:
         from_secret: discord_webhook_token

Makefile

@@ -119,6 +119,13 @@ go-check:
 		exit 1; \
 	fi
 
+.PHONY: git-check
+git-check:
+	@if git lfs >/dev/null 2>&1 ; then : ; else \
+		echo "Gitea requires git with lfs support to run tests." ; \
+		exit 1; \
+	fi
+
 .PHONY: node-check
 node-check:
 	$(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | grep -Eo '[0-9]+\.?[0-9]+?\.?[0-9]?' | tr '.' ' ');))
@@ -233,7 +240,7 @@ coverage:
 
 .PHONY: unit-test-coverage
 unit-test-coverage:
-	$(GO) test -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
+	GO111MODULE=on $(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
 
 .PHONY: vendor
 vendor:
@@ -376,7 +383,7 @@ integrations.mssql.test: $(GO_SOURCES)
 integrations.sqlite.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
 
-integrations.cover.test: $(GO_SOURCES)
+integrations.cover.test: git-check $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test
 
 .PHONY: migrations.mysql.test

modules/migrations/github.go

@@ -24,6 +24,8 @@ import (
 var (
 	_ base.Downloader        = &GithubDownloaderV3{}
 	_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
+	// GithubLimitRateRemaining limit to wait for new rate to apply
+	GithubLimitRateRemaining = 0
 )
 
 func init() {
@@ -115,7 +117,7 @@ func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
 }
 
 func (g *GithubDownloaderV3) sleep() {
-	for g.rate != nil && g.rate.Remaining <= 0 {
+	for g.rate != nil && g.rate.Remaining <= GithubLimitRateRemaining {
 		timer := time.NewTimer(time.Until(g.rate.Reset.Time))
 		select {
 		case <-g.ctx.Done():
@@ -124,13 +126,22 @@ func (g *GithubDownloaderV3) sleep() {
 		case <-timer.C:
 		}
 
-		rates, _, err := g.client.RateLimits(g.ctx)
+		err := g.RefreshRate()
 		if err != nil {
 			log.Error("g.client.RateLimits: %s", err)
 		}
+	}
+}
 
-		g.rate = rates.GetCore()
+// RefreshRate update the current rate (doesn't count in rate limit)
+func (g *GithubDownloaderV3) RefreshRate() error {
+	rates, _, err := g.client.RateLimits(g.ctx)
+	if err != nil {
+		return err
 	}
+
+	g.rate = rates.GetCore()
+	return nil
 }
 
 // GetRepoInfo returns a repository information

modules/migrations/github_test.go

@@ -6,6 +6,7 @@
 package migrations
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -62,7 +63,11 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base
 }
 
 func TestGitHubDownloadRepo(t *testing.T) {
-	downloader := NewGithubDownloaderV3("", "", "go-gitea", "test_repo")
+	GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
+	downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo")
+	err := downloader.RefreshRate()
+	assert.NoError(t, err)
+
 	repo, err := downloader.GetRepoInfo()
 	assert.NoError(t, err)
 	assert.EqualValues(t, &base.Repository{