Add Node.js build dep, remove built js/css files

- Added Node.js as build dependency and removes build files from git.
- Added version checks for both Go and Node.js.
- Overhauled the js/css make target to only run when needed.
- Merged the `generate` make target into `build` as per suggestion.

Fixes: #6782
Fixes: #9216

Author: silverwind

.drone.yml

@@ -56,11 +56,10 @@ steps:
 
   - name: pre-build
     pull: always
-    image: webhippie/nodejs:latest
+    image: node:10 # this step is kept at the lowest version of node that we support
     commands:
       - make css
       - make js
-      - bash -c '[ -z "$(git status --porcelain public/js public/css)" ] || (echo "Generated js/css files do not match" && git status --porcelain public/js public/css && exit 1)'
 
   - name: build-without-gcc
     pull: always
@@ -69,6 +68,7 @@ steps:
       GO111MODULE: on
       GOPROXY: off
     commands:
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
       - go build -mod=vendor -o gitea_no_gcc # test if build succeeds without the sqlite tag
 
   - name: build-linux-386
@@ -80,12 +80,14 @@ steps:
       GOOS: linux
       GOARCH: 386
     commands:
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
       - go build -mod=vendor -o gitea_linux_386 # test if compatible with 32 bit
 
   - name: build
     pull: always
     image: golang:1.13
     commands:
+      - curl -sL https://deb.nodesource.com/setup_12.x | bash - && apt -y install nodejs
       - make clean
       - make generate
       - make golangci-lint

.gitignore

@@ -69,6 +69,8 @@ coverage.all
 /modules/indexer/issues/indexers
 routers/repo/authorized_keys
 /yarn.lock
+/public/js
+/public/css
 
 # Snapcraft
 snap/.snapcraft/
@@ -78,4 +80,4 @@ prime/
 *.snap
 *.snap-build
 *_source.tar.bz2
-.DS_Store
+.DS_Store

Dockerfile

@@ -11,7 +11,7 @@ ARG TAGS="sqlite sqlite_unlock_notify"
 ENV TAGS "bindata $TAGS"
 
 #Build deps
-RUN apk --no-cache add build-base git
+RUN apk --no-cache add build-base git nodejs npm
 
 #Setup repo
 COPY . ${GOPATH}/src/code.gitea.io/gitea

Makefile

@@ -18,7 +18,6 @@ else
 	endif
 endif
 
-BINDATA := modules/{options,public,templates}/bindata.go
 GOFILES := $(shell find . -name "*.go" -type f ! -path "./vendor/*" ! -path "*/bindata.go")
 GOFMT ?= gofmt -s
 
@@ -42,7 +41,17 @@ endif
 LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)"
 
 PACKAGES ?= $(filter-out code.gitea.io/gitea/integrations/migration-test,$(filter-out code.gitea.io/gitea/integrations,$(shell GO111MODULE=on $(GO) list -mod=vendor ./... | grep -v /vendor/)))
-SOURCES ?= $(shell find . -name "*.go" -type f)
+
+GO_SOURCES ?= $(shell find . -name "*.go" -type f)
+JS_SOURCES ?= $(shell find web_src/js web_src/css -type f)
+CSS_SOURCES ?= $(shell find web_src/less -type f)
+
+JS_DEST := public/js/index.js
+CSS_DEST := public/css/index.css
+BINDATA_DEST := modules/public/bindata.go modules/options/bindata.go modules/templates/bindata.go
+
+JS_DEST_DIR := public/js
+CSS_DEST_DIR := public/css
 
 TAGS ?=
 
@@ -80,10 +89,31 @@ all: build
 
 include docker/Makefile
 
+.PHONY: go-check
+go-check:
+	$(eval GO_VERSION := $(shell printf "%03d%03d%03d" $(shell go version | grep -Eo '[0-9]+\.?[0-9]+?\.?[0-9]?\s' | tr '.' ' ');))
+	@if [ "$(GO_VERSION)" -lt "001011000" ]; then \
+		echo "Gitea requires Go 1.11.0 or greater to build. You can get it at https://golang.org/dl/"; \
+		exit 1; \
+	fi
+
+.PHONY: node-check
+node-check:
+	$(eval NODE_VERSION := $(shell printf "%03d%03d%03d" $(shell node -v | grep -Eo '[0-9]+\.?[0-9]+?\.?[0-9]?' | tr '.' ' ');))
+	$(eval NPM_MISSING := $(shell hash npm > /dev/null 2>&1 || echo 1))
+	@if [ "$(NODE_VERSION)" -lt "010000000" -o "$(NPM_MISSING)" = "1" ]; then \
+		echo "Gitea requires Node.js 10.0.0 or greater and npm to build. You can get it at https://nodejs.org/en/download/"; \
+		exit 1; \
+	fi
+
+.PHONY: clean-all
+clean-all: clean
+	rm -rf $(JS_DEST_DIR) $(CSS_DEST_DIR)
+
 .PHONY: clean
 clean:
 	$(GO) clean -i ./...
-	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA) \
+	rm -rf $(EXECUTABLE) $(DIST) $(BINDATA_DEST) \
 		integrations*.test \
 		integrations/gitea-integration-pgsql/ integrations/gitea-integration-mysql/ integrations/gitea-integration-mysql8/ integrations/gitea-integration-sqlite/ \
 		integrations/gitea-integration-mssql/ integrations/indexers-mysql/ integrations/indexers-mysql8/ integrations/indexers-pgsql integrations/indexers-sqlite \
@@ -309,42 +339,42 @@ bench-pgsql: integrations.pgsql.test generate-ini-pgsql
 integration-test-coverage: integrations.cover.test generate-ini-mysql
 	GITEA_ROOT=${CURDIR} GITEA_CONF=integrations/mysql.ini ./integrations.cover.test -test.coverprofile=integration.coverage.out
 
-integrations.mysql.test: $(SOURCES)
+integrations.mysql.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mysql.test
 
-integrations.mysql8.test: $(SOURCES)
+integrations.mysql8.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mysql8.test
 
-integrations.pgsql.test: $(SOURCES)
+integrations.pgsql.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.pgsql.test
 
-integrations.mssql.test: $(SOURCES)
+integrations.mssql.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test
 
-integrations.sqlite.test: $(SOURCES)
+integrations.sqlite.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
 
-integrations.cover.test: $(SOURCES)
+integrations.cover.test: $(GO_SOURCES)
 	GO111MODULE=on $(GO) test -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(PACKAGES) | tr ' ' ',') -o integrations.cover.test
 
 .PHONY: migrations.mysql.test
-migrations.mysql.test: $(SOURCES)
+migrations.mysql.test: $(GO_SOURCES)
 	$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.mysql.test
 
 .PHONY: migrations.mysql8.test
-migrations.mysql8.test: $(SOURCES)
+migrations.mysql8.test: $(GO_SOURCES)
 	$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.mysql8.test
 
 .PHONY: migrations.pgsql.test
-migrations.pgsql.test: $(SOURCES)
+migrations.pgsql.test: $(GO_SOURCES)
 	$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.pgsql.test
 
 .PHONY: migrations.mssql.test
-migrations.mssql.test: $(SOURCES)
+migrations.mssql.test: $(GO_SOURCES)
 	$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.mssql.test
 
 .PHONY: migrations.sqlite.test
-migrations.sqlite.test: $(SOURCES)
+migrations.sqlite.test: $(GO_SOURCES)
 	$(GO) test -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
 
 .PHONY: check
@@ -354,10 +384,16 @@ check: test
 install: $(wildcard *.go)
 	$(GO) install -v -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)'
 
+.PHONY: go
+go: go-check $(EXECUTABLE)
+
+.PHONY: go-all
+go-all: go-check generate go
+
 .PHONY: build
-build: $(EXECUTABLE)
+build: js css go-all
 
-$(EXECUTABLE): $(SOURCES)
+$(EXECUTABLE): $(GO_SOURCES)
 	GO111MODULE=on $(GO) build -mod=vendor $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@
 
 .PHONY: release
@@ -412,33 +448,26 @@ release-compress:
 	fi
 	cd $(DIST)/release/; for file in `find . -type f -name "*"`; do echo "compressing $${file}" && gxz -k -9 $${file}; done;
 
-npm-check:
-	@hash npm > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
-		echo "Please install Node.js 8.x or greater with npm"; \
-		exit 1; \
-	fi;
-	@hash npx > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
-		echo "Please install Node.js 8.x or greater with npm"; \
-		exit 1; \
-	fi;
-
-.PHONY: npm
-npm: npm-check
+node_modules: package-lock.json
 	npm install --no-save
 
 .PHONY: npm-update
-npm-update: npm-check
+npm-update: node-check node_modules
 	npx updates -cu
 	rm -rf node_modules package-lock.json
 	npm install --package-lock
 
 .PHONY: js
-js: npm
+js: node-check $(JS_DEST)
+
+$(JS_DEST): node_modules $(JS_SOURCES)
 	npx eslint web_src/js webpack.config.js
 	npx webpack
 
 .PHONY: css
-css: npm
+css: node-check $(CSS_DEST)
+
+$(CSS_DEST): node_modules $(CSS_SOURCES)
 	npx stylelint web_src/less
 	npx lessc --clean-css="--s0 -b" web_src/less/index.less public/css/index.css
 	$(foreach file, $(filter-out web_src/less/themes/_base.less, $(wildcard web_src/less/themes/*)),npx lessc --clean-css="--s0 -b" web_src/less/themes/$(notdir $(file)) > public/css/theme-$(notdir $(call strip-suffix,$(file))).css;)

docs/content/doc/advanced/hacking-on-gitea.en-us.md

@@ -23,6 +23,10 @@ environment variable and to add the go bin directory or directories
 `${GOPATH//://bin:}/bin` to the `$PATH`. See the Go wiki entry for
 [GOPATH](https://github.com/golang/go/wiki/GOPATH).
 
+Next, [install Node.js with npm](https://nodejs.org/en/download/) which is
+required to build the JavaScript and CSS files. The minimum supported Node.js
+version is 10 and the latest LTS version is recommended.
+
 You will also need make.
 <a href='{{< relref "doc/advanced/make.en-us.md" >}}'>(See here how to get Make)</a>
 
@@ -98,7 +102,7 @@ from source</a>.
 The simplest recommended way to build from source is:
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make generate build
+TAGS="bindata sqlite sqlite_unlock_notify" make build
 ```
 
 However, there are a number of additional make tasks you should be aware of.
@@ -136,19 +140,17 @@ You should run revive, vet and spell-check on the code with:
 make revive vet misspell-check
 ```
 
-### Updating CSS
-
-To generate the CSS, you need [Node.js](https://nodejs.org/) 8.0 or greater with npm. We use [less](http://lesscss.org/) and [postcss](https://postcss.org) to generate our CSS. Do **not** edit the files in `public/css` directly, as they are generated from `lessc` from the files in `public/less`.
+### Working on CSS
 
-Edit files in `public/less`, and then run the linter and build the CSS files via:
+Edit files in `web_src/less` and run the linter and build the CSS files via:
 
 ```bash
 make css
 ```
 
-### Updating JS
+### Working on JS
 
-To generate the JS files, you need [Node.js](https://nodejs.org/) 8.0 or greater with npm. Edit files in `public/js`, run the linter and build the JS files via:
+Edit files in `web_src/js`, run the linter and build the JS files via:
 
 ```bash
 make js
@@ -235,7 +237,7 @@ Unit tests will not and cannot completely test Gitea alone. Therefore, we
 have written integration tests; however, these are database dependent.
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make generate build test-sqlite
+TAGS="bindata sqlite sqlite_unlock_notify" make build test-sqlite
 ```
 
 will run the integration tests in an sqlite environment. Other database tests

docs/content/doc/installation/from-source.en-us.md

@@ -21,6 +21,10 @@ environment variable and to add the go bin directory or directories
 `${GOPATH//://bin:}/bin` to the `$PATH`. See the Go wiki entry for
 [GOPATH](https://github.com/golang/go/wiki/GOPATH).
 
+Next, [install Node.js with npm](https://nodejs.org/en/download/) which is
+required to build the JavaScript and CSS files. The minimum supported Node.js
+version is 10 and the latest LTS version is recommended.
+
 **Note**: When executing make tasks that require external tools, like
 `make misspell-check`, Gitea will automatically download and build these as
 necessary. To be able to use these, you must have the `"$GOPATH/bin"` directory
@@ -75,9 +79,12 @@ git checkout v{{< version >}}  # or git checkout pr-xyz
 
 ## Build
 
-Since all required libraries are already bundled in the Gitea source, it's
-possible to build Gitea with no additional downloads apart from Make
-<a href='{{< relref "doc/advanced/make.en-us.md" >}}'>(See here how to get Make)</a>.
+To build from source, the following programs must be present on the system:
+
+- `go` 1.11.0 or higher, see [here](https://golang.org/dl/)
+- `node` 10.0.0 or higher with `npm`, see [here](https://nodejs.org/en/download/)
+- `make`, see <a href='{{< relref "doc/advanced/make.en-us.md" >}}'>here</a>
+
 Various [make tasks](https://github.com/go-gitea/gitea/blob/master/Makefile)
 are provided to keep the build process as simple as possible.
 
@@ -93,19 +100,18 @@ Depending on requirements, the following build tags can be included.
 
 Bundling assets into the binary using the `bindata` build tag can make
 development and testing easier, but is not ideal for a production deployment.
-To include assets, they must be built separately using the `generate` make
-task e.g.:
+To include assets, add the `bindata` tag:
 
 ```bash
-TAGS="bindata" make generate build
+TAGS="bindata" make build
 ```
 
 In the default release build of our continuous integration system, the build
 tags are: `TAGS="bindata sqlite sqlite_unlock_notify"`. The simplest
 recommended way to build from source is therefore:
 
 ```bash
-TAGS="bindata sqlite sqlite_unlock_notify" make generate build
+TAGS="bindata sqlite sqlite_unlock_notify" make build
 ```
 
 ## Test

package.json

@@ -2,7 +2,7 @@
   "license": "MIT",
   "private": true,
   "engines": {
-    "node": ">=8"
+    "node": ">=10"
   },
   "devDependencies": {
     "@babel/core": "7.7.2",