Stop packaging node_modules in release tarballs (#15273)

- Don't package node_modules in tarballs, they are not cross-platform
  anymore and npm cache should not be messed with directly. Instead,
  require an internet connection to rebuild the UI, which is not necessary
  in the general use case because prebuilt UI files are shipped in the
  public directory.
- Simplify the fomantic build and make the target phony. We don't need
  anything more for something that is rarely ran.
- Use regular tar again to build tarballs and add variable for excludes
- Disable annoying npm update notifications

Fixes: https://github.com/go-gitea/gitea/pull/14578
Fixes: https://github.com/go-gitea/gitea/pull/15256
Fixes: https://github.com/go-gitea/gitea/pull/15262
Co-authored-by: 6543 <6543@obermui.de>

diff --git a/.drone.yml b/.drone.yml
index fa468730fbaeddb9565e065e6b78be9b1f484aa6..d1cedb73cc5bc0fe512b5527ceb612f8bbd0467a 100644 (file)
--- a/.drone.yml
+++ b/.drone.yml
@@ -501,7 +501,7 @@ steps:
     pull: always
     image: techknowlogick/xgo:go-1.16.x
     commands:
-      - curl -sL https://deb.nodesource.com/setup_14.x | bash - && apt -y install nodejs bsdtar\|libarchive-tools
+      - curl -sL https://deb.nodesource.com/setup_14.x | bash - && apt-get install -y nodejs
       - export PATH=$PATH:$GOPATH/bin
       - make release
     environment:
@@ -597,7 +597,7 @@ steps:
     pull: always
     image: techknowlogick/xgo:go-1.16.x
     commands:
-      - curl -sL https://deb.nodesource.com/setup_14.x | bash - && apt-get install -y nodejs bsdtar\|libarchive-tools
+      - curl -sL https://deb.nodesource.com/setup_14.x | bash - && apt-get install -y nodejs
       - export PATH=$PATH:$GOPATH/bin
       - make release
     environment:

diff --git a/.gitignore b/.gitignore
index 6cf366df6c52a16e227f66d3275b33f6d3053226..22f50057248406804b38d247edef56b443bc45d5 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -81,7 +81,6 @@ coverage.all
 /public/fonts
 /public/img/webpack
 /web_src/fomantic/node_modules
-/web_src/fomantic/semantic.json
 /web_src/fomantic/build/*
 !/web_src/fomantic/build/semantic.js
 !/web_src/fomantic/build/semantic.css
@@ -95,7 +94,6 @@ coverage.all
 !/web_src/fomantic/build/themes/default/assets/fonts/outline-icons.woff2
 /VERSION
 /.air
-/.npm-cache
 
 # Snapcraft
 snap/.snapcraft/

diff --git a/.npmrc b/.npmrc
index 25ed63413bbc49668896af09b447b4e9fa78a685..8dcb4185bd4129f307e686e8bde5558b1f0e4a57 100644 (file)
--- a/.npmrc
+++ b/.npmrc
@@ -1,5 +1,5 @@
 audit=false
 fund=false
+update-notifier=false
 package-lock=true
 save-exact=true
-cache=.npm-cache

diff --git a/Makefile b/Makefile
index 28de5bb4f86fec7f07cf6a8b7a228186f9ae5646..e8738c92897b30b8415f0bfe3d4c01e419f26aa4 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -14,8 +14,6 @@ else
 
 # This is the "normal" part of the Makefile
 
-TAR := $(shell hash bsdtar > /dev/null 2>&1 && echo "bsdtar --no-xattrs" || echo "tar" )
-
 DIST := dist
 DIST_DIRS := $(DIST)/binaries $(DIST)/release
 IMPORT := code.gitea.io/gitea
@@ -93,8 +91,6 @@ LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
 
 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/models/migrations code.gitea.io/gitea/integrations/migration-test code.gitea.io/gitea/integrations,$(shell $(GO) list -mod=vendor ./... | grep -v /vendor/))
 
-FOMANTIC_CONFIGS := semantic.json web_src/fomantic/theme.config.less web_src/fomantic/_site/globals/site.variables
-FOMANTIC_DEST := web_src/fomantic/build/semantic.js web_src/fomantic/build/semantic.css
 FOMANTIC_WORK_DIR := web_src/fomantic
 
 WEBPACK_SOURCES := $(shell find web_src/js web_src/less -type f)
@@ -115,6 +111,8 @@ TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags
 
 TEST_TAGS ?= sqlite sqlite_unlock_notify
 
+TAR_EXCLUDES := .git data indexers queues log node_modules $(EXECUTABLE) $(FOMANTIC_WORK_DIR)/node_modules $(DIST) $(MAKE_EVIDENCE_DIR) $(AIR_TMP_DIR)
+
 GO_DIRS := cmd integrations models modules routers build services vendor tools
 
 GO_SOURCES := $(wildcard *.go)
@@ -589,7 +587,7 @@ install: $(wildcard *.go)
 build: frontend backend
 
 .PHONY: frontend
-frontend: node-check $(WEBPACK_DEST)
+frontend: $(WEBPACK_DEST)
 
 .PHONY: backend
 backend: go-check generate $(EXECUTABLE)
@@ -654,16 +652,16 @@ release-compress: | $(DIST_DIRS)
        cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && gxz -k -9 $${file}; done;
 
 .PHONY: release-sources
-release-sources: | $(DIST_DIRS) npm-cache
+release-sources: | $(DIST_DIRS)
        echo $(VERSION) > $(STORED_VERSION_FILE)
-       $(eval EXCL := --exclude=$(shell [ ! "$(TAR)" = "tar" ] && echo "^" )./)
-       $(eval EXCL_RECURSIVE := --exclude=)
-       $(TAR) $(EXCL)$(DIST) $(EXCL).git $(EXCL)$(MAKE_EVIDENCE_DIR) $(EXCL_RECURSIVE)node_modules $(EXCL)$(AIR_TMP_DIR) -czf $(DIST)/release/gitea-src-$(VERSION).tar.gz .
+# bsdtar needs a ^ to prevent matching subdirectories
+       $(eval EXCL := --exclude=$(shell tar --help | grep -q bsdtar && echo "^")./)
+       tar $(addprefix $(EXCL),$(TAR_EXCLUDES)) -czf $(DIST)/release/gitea-src-$(VERSION).tar.gz .
        rm -f $(STORED_VERSION_FILE)
 
 .PHONY: release-docs
 release-docs: | $(DIST_DIRS) docs
-       $(TAR) -czf $(DIST)/release/gitea-docs-$(VERSION).tar.gz -C ./docs/public .
+       tar -czf $(DIST)/release/gitea-docs-$(VERSION).tar.gz -C ./docs/public .
 
 .PHONY: docs
 docs:
@@ -676,25 +674,6 @@ node_modules: package-lock.json
        npm install --no-save
        @touch node_modules
 
-.PHONY: npm-cache
-npm-cache: .npm-cache $(FOMANTIC_WORK_DIR)/node_modules/fomantic-ui
-
-.npm-cache: package-lock.json
-       rm -rf .npm-cache
-       $(eval ESBUILD_VERSION := $(shell node -p "require('./package-lock.json').dependencies.esbuild.version"))
-       npm config --userconfig=.npmrc set cache=.npm-cache
-       rm -rf node_modules && npm install --no-save
-       npm config --userconfig=$(FOMANTIC_WORK_DIR)/.npmrc set cache=../../.npm-cache
-       echo $(foreach build, darwin-64 $(foreach arch,arm arm64 32 64,linux-${arch}) $(foreach arch,32 64,windows-${arch}), esbuild-${build}@$(ESBUILD_VERSION)) | tr " " "\n" | xargs -n 1 -P 4 npm cache add
-       rm -rf $(FOMANTIC_WORK_DIR)/node_modules
-       @touch .npm-cache
-
-.PHONY: npm-uncache
-npm-uncache:
-       rm -rf .npm-cache
-       npm config --userconfig=$(FOMANTIC_WORK_DIR)/.npmrc rm cache
-       npm config --userconfig=.npmrc rm cache
-
 .PHONY: npm-update
 npm-update: node-check | node_modules
        npx updates -cu
@@ -703,30 +682,18 @@ npm-update: node-check | node_modules
        @touch node_modules
 
 .PHONY: fomantic
-fomantic: $(FOMANTIC_DEST)
-
-$(FOMANTIC_WORK_DIR)/node_modules/fomantic-ui:
-       ln -sf ../../semantic.json $(FOMANTIC_WORK_DIR)
-       cd $(FOMANTIC_WORK_DIR); \
-               rm -rf node_modules && mkdir node_modules && \
-               npm install fomantic-ui; \
-               rm -f semantic.json
-       @touch $(FOMANTIC_WORK_DIR)/node_modules
-
-$(FOMANTIC_DEST): $(FOMANTIC_CONFIGS) $(FOMANTIC_WORK_DIR)/node_modules/fomantic-ui
-       ln -sf ../../semantic.json $(FOMANTIC_WORK_DIR)
+fomantic:
        rm -rf $(FOMANTIC_WORK_DIR)/build
-       cd $(FOMANTIC_WORK_DIR); \
-               cp -f theme.config.less node_modules/fomantic-ui/src/theme.config; \
-               cp -rf _site node_modules/fomantic-ui/src/; \
-               npx gulp -f node_modules/fomantic-ui/gulpfile.js build; \
-               rm -f semantic.json
-       @touch $(FOMANTIC_DEST)
+       cd $(FOMANTIC_WORK_DIR) && npm install --no-save
+       cp -f $(FOMANTIC_WORK_DIR)/theme.config.less $(FOMANTIC_WORK_DIR)/node_modules/fomantic-ui/src/theme.config
+       cp -rf $(FOMANTIC_WORK_DIR)/_site $(FOMANTIC_WORK_DIR)/node_modules/fomantic-ui/src/
+       cd $(FOMANTIC_WORK_DIR) && npx gulp -f node_modules/fomantic-ui/gulpfile.js build
 
 .PHONY: webpack
 webpack: $(WEBPACK_DEST)
 
-$(WEBPACK_DEST): $(WEBPACK_SOURCES) $(WEBPACK_CONFIGS) package-lock.json | node_modules
+$(WEBPACK_DEST): $(WEBPACK_SOURCES) $(WEBPACK_CONFIGS) package-lock.json
+       @$(MAKE) -s node-check node_modules
        rm -rf $(WEBPACK_DEST_ENTRIES)
        npx webpack
        @touch $(WEBPACK_DEST)

diff --git a/README.md b/README.md
index 04cd05ad3d59bc1dc49baa7cab8e8e5f0b48653d..3bf8115eb25458f3f7a0ee4a439d4beba9393081 100644 (file)
--- a/README.md
+++ b/README.md
@@ -77,13 +77,11 @@ or if sqlite support is required:
 The `build` target is split into two sub-targets:
 
 - `make backend` which requires [Go 1.13](https://golang.org/dl/) or greater.
-- `make frontend` which requires [Node.js 12.17](https://nodejs.org/en/download/) or greater.
+- `make frontend` which requires [Node.js 12.17](https://nodejs.org/en/download/) or greater and Internet connectivity to download npm dependencies.
 
-If pre-built frontend files are present it is possible to only build the backend:
+When building from the official source tarballs which include pre-built frontend files, the `frontend` target will not be triggered, making it possible to build without Node.js and Internet connectivity.
 
-    TAGS="bindata" make backend
-
-Parallelism is not supported for these targets, so please don't include `-j <num>`.
+Parallelism (`make -j <num>`) is not supported.
 
 More info: https://docs.gitea.io/en-us/install-from-source/
 

diff --git a/semantic.json b/semantic.json
deleted file mode 100644 (file)
index c0fcd79..0000000
--- a/semantic.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  "version": "2.8.7",
-  "base": "node_modules/fomantic-ui",
-  "paths": {
-    "source": {
-      "config": "src/theme.config",
-      "definitions": "src/definitions/",
-      "site": "src/site/",
-      "themes": "src/themes/"
-    },
-    "output": {
-      "packaged": "../../build/",
-      "uncompressed": "../../build/components/",
-      "compressed": "../../build/components/",
-      "themes": "../../build/themes/"
-    },
-    "clean": "../../build/"
-  },
-  "permission": false,
-  "autoInstall": false,
-  "rtl": false,
-  "admin": false,
-  "components": [
-    "accordion",
-    "api",
-    "breadcrumb",
-    "button",
-    "card",
-    "checkbox",
-    "comment",
-    "container",
-    "dimmer",
-    "divider",
-    "dropdown",
-    "form",
-    "grid",
-    "header",
-    "icon",
-    "image",
-    "input",
-    "item",
-    "label",
-    "list",
-    "loader",
-    "menu",
-    "message",
-    "modal",
-    "popup",
-    "progress",
-    "reset",
-    "search",
-    "segment",
-    "sidebar",
-    "site",
-    "tab",
-    "table",
-    "text",
-    "transition"
-  ]
-}

diff --git a/web_src/fomantic/.npmrc b/web_src/fomantic/.npmrc
index 5f8054bbb037c21b839637106f00e1f4f98aadb0..15ecd04e11e2cee94e32af3d2e6c356c55b0d446 100644 (file)
--- a/web_src/fomantic/.npmrc
+++ b/web_src/fomantic/.npmrc
@@ -1,3 +1,5 @@
-optional=false
+audit=false
+fund=false
+update-notifier=false
 package-lock=false
-cache=../../.npm-cache
+optional=false

diff --git a/web_src/fomantic/package.json b/web_src/fomantic/package.json
new file mode 100644 (file)
index 0000000..c031c07
--- /dev/null
+++ b/web_src/fomantic/package.json
@@ -0,0 +1,5 @@
+{
+  "dependencies": {
+    "fomantic-ui": "2.8.7"
+  }
+}

diff --git a/web_src/fomantic/semantic.json b/web_src/fomantic/semantic.json
new file mode 100644 (file)
index 0000000..c0fcd79
--- /dev/null
+++ b/web_src/fomantic/semantic.json
@@ -0,0 +1,60 @@
+{
+  "version": "2.8.7",
+  "base": "node_modules/fomantic-ui",
+  "paths": {
+    "source": {
+      "config": "src/theme.config",
+      "definitions": "src/definitions/",
+      "site": "src/site/",
+      "themes": "src/themes/"
+    },
+    "output": {
+      "packaged": "../../build/",
+      "uncompressed": "../../build/components/",
+      "compressed": "../../build/components/",
+      "themes": "../../build/themes/"
+    },
+    "clean": "../../build/"
+  },
+  "permission": false,
+  "autoInstall": false,
+  "rtl": false,
+  "admin": false,
+  "components": [
+    "accordion",
+    "api",
+    "breadcrumb",
+    "button",
+    "card",
+    "checkbox",
+    "comment",
+    "container",
+    "dimmer",
+    "divider",
+    "dropdown",
+    "form",
+    "grid",
+    "header",
+    "icon",
+    "image",
+    "input",
+    "item",
+    "label",
+    "list",
+    "loader",
+    "menu",
+    "message",
+    "modal",
+    "popup",
+    "progress",
+    "reset",
+    "search",
+    "segment",
+    "sidebar",
+    "site",
+    "tab",
+    "table",
+    "text",
+    "transition"
+  ]
+}