Support HAProxy protocol

This PR adds functionality to allow Gitea to sit behind an
HAProxy and HAProxy protocolled connections directly.

Fix go-gitea#7508

Signed-off-by: Andrew Thornton <[email protected]>

Author: zeripath

cmd/web.go

@@ -60,7 +60,7 @@ func runHTTPRedirector() {
 		http.Redirect(w, r, target, http.StatusTemporaryRedirect)
 	})
 
-	var err = runHTTP("tcp", source, context2.ClearHandler(handler))
+	var err = runHTTP("tcp", source, context2.ClearHandler(handler), setting.RedirectHAProxy)
 
 	if err != nil {
 		log.Fatal("Failed to start port redirection: %v", err)
@@ -77,12 +77,12 @@ func runLetsEncrypt(listenAddr, domain, directory, email string, m http.Handler)
 	go func() {
 		log.Info("Running Let's Encrypt handler on %s", setting.HTTPAddr+":"+setting.PortToRedirect)
 		// all traffic coming into HTTP will be redirect to HTTPS automatically (LE HTTP-01 validation happens here)
-		var err = runHTTP("tcp", setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)))
+		var err = runHTTP("tcp", setting.HTTPAddr+":"+setting.PortToRedirect, certManager.HTTPHandler(http.HandlerFunc(runLetsEncryptFallbackHandler)), setting.RedirectHAProxy)
 		if err != nil {
 			log.Fatal("Failed to start the Let's Encrypt handler on port %s: %v", setting.PortToRedirect, err)
 		}
 	}()
-	return runHTTPSWithTLSConfig("tcp", listenAddr, certManager.TLSConfig(), context2.ClearHandler(m))
+	return runHTTPSWithTLSConfig("tcp", listenAddr, certManager.TLSConfig(), context2.ClearHandler(m), setting.HAProxy, setting.HAProxyTLSBridging)
 }
 
 func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
@@ -177,7 +177,7 @@ func runWeb(ctx *cli.Context) error {
 	switch setting.Protocol {
 	case setting.HTTP:
 		NoHTTPRedirector()
-		err = runHTTP("tcp", listenAddr, context2.ClearHandler(m))
+		err = runHTTP("tcp", listenAddr, context2.ClearHandler(m), setting.HAProxy)
 	case setting.HTTPS:
 		if setting.EnableLetsEncrypt {
 			err = runLetsEncrypt(listenAddr, setting.Domain, setting.LetsEncryptDirectory, setting.LetsEncryptEmail, context2.ClearHandler(m))
@@ -188,16 +188,16 @@ func runWeb(ctx *cli.Context) error {
 		} else {
 			NoHTTPRedirector()
 		}
-		err = runHTTPS("tcp", listenAddr, setting.CertFile, setting.KeyFile, context2.ClearHandler(m))
+		err = runHTTPS("tcp", listenAddr, setting.CertFile, setting.KeyFile, context2.ClearHandler(m), setting.HAProxy, setting.HAProxyTLSBridging)
 	case setting.FCGI:
 		NoHTTPRedirector()
-		err = runFCGI("tcp", listenAddr, context2.ClearHandler(m))
+		err = runFCGI("tcp", listenAddr, context2.ClearHandler(m), setting.HAProxy)
 	case setting.UnixSocket:
 		NoHTTPRedirector()
-		err = runHTTP("unix", listenAddr, context2.ClearHandler(m))
+		err = runHTTP("unix", listenAddr, context2.ClearHandler(m), setting.HAProxy)
 	case setting.FCGIUnix:
 		NoHTTPRedirector()
-		err = runFCGI("unix", listenAddr, context2.ClearHandler(m))
+		err = runFCGI("unix", listenAddr, context2.ClearHandler(m), setting.HAProxy)
 	default:
 		log.Fatal("Invalid protocol: %s", setting.Protocol)
 	}

cmd/web_graceful.go

@@ -14,16 +14,16 @@ import (
 	"code.gitea.io/gitea/modules/log"
 )
 
-func runHTTP(network, listenAddr string, m http.Handler) error {
-	return graceful.HTTPListenAndServe(network, listenAddr, m)
+func runHTTP(network, listenAddr string, m http.Handler, haProxy bool) error {
+	return graceful.HTTPListenAndServe(network, listenAddr, m, haProxy)
 }
 
-func runHTTPS(network, listenAddr, certFile, keyFile string, m http.Handler) error {
-	return graceful.HTTPListenAndServeTLS(network, listenAddr, certFile, keyFile, m)
+func runHTTPS(network, listenAddr, certFile, keyFile string, m http.Handler, haProxy, haProxyTLSBridging bool) error {
+	return graceful.HTTPListenAndServeTLS(network, listenAddr, certFile, keyFile, m, haProxy, haProxyTLSBridging)
 }
 
-func runHTTPSWithTLSConfig(network, listenAddr string, tlsConfig *tls.Config, m http.Handler) error {
-	return graceful.HTTPListenAndServeTLSConfig(network, listenAddr, tlsConfig, m)
+func runHTTPSWithTLSConfig(network, listenAddr string, tlsConfig *tls.Config, m http.Handler, haProxy, haProxyTLSBridging bool) error {
+	return graceful.HTTPListenAndServeTLSConfig(network, listenAddr, tlsConfig, m, haProxy, haProxyTLSBridging)
 }
 
 // NoHTTPRedirector tells our cleanup routine that we will not be using a fallback http redirector
@@ -37,13 +37,13 @@ func NoMainListener() {
 	graceful.GetManager().InformCleanup()
 }
 
-func runFCGI(network, listenAddr string, m http.Handler) error {
+func runFCGI(network, listenAddr string, m http.Handler, haProxy bool) error {
 	// This needs to handle stdin as fcgi point
 	fcgiServer := graceful.NewServer(network, listenAddr)
 
 	err := fcgiServer.ListenAndServe(func(listener net.Listener) error {
 		return fcgi.Serve(listener, m)
-	})
+	}, haProxy)
 	if err != nil {
 		log.Fatal("Failed to start FCGI main server: %v", err)
 	}

custom/conf/app.example.ini

@@ -247,6 +247,14 @@ FILE_EXTENSIONS = .md,.markdown,.mdown,.mkd
 [server]
 ; The protocol the server listens on. One of 'http', 'https', 'unix' or 'fcgi'.
 PROTOCOL = http
+; Expect HAPROXY headers on connections
+HAPROXY = false
+; Use HAPROXY in TLS Bridging mode
+HAPROXY_TLS_BRIDGING = false
+; Timeout to wait for HAProxy header (set to 0 to have no timeout)
+HAPROXY_HEADER_TIMEOUT=5s
+; Accept Unknown HAProxy
+HAPROXY_ACCEPT_UNKNOWN=false
 DOMAIN = localhost
 ROOT_URL = %(PROTOCOL)s://%(DOMAIN)s:%(HTTP_PORT)s/
 ; when STATIC_URL_PREFIX is empty it will follow ROOT_URL
@@ -261,17 +269,23 @@ HTTP_PORT = 3000
 ; PORT_TO_REDIRECT.
 REDIRECT_OTHER_PORT = false
 PORT_TO_REDIRECT = 80
+; expect HAPROXY header on connections to https redirector.
+REDIRECT_HAPROXY = %(HAPROXY)
 ; Permission for unix socket
 UNIX_SOCKET_PERMISSION = 666
 ; Local (DMZ) URL for Gitea workers (such as SSH update) accessing web service.
 ; In most cases you do not need to change the default value.
 ; Alter it only if your SSH server node is not the same as HTTP node.
 ; Do not set this variable if PROTOCOL is set to 'unix'.
 LOCAL_ROOT_URL = %(PROTOCOL)s://%(HTTP_ADDR)s:%(HTTP_PORT)s/
+; When making local connections pass the HAPROXY header.
+LOCAL_HAPROXY = %(HAPROXY)
 ; Disable SSH feature when not available
 DISABLE_SSH = false
 ; Whether to use the builtin SSH server or not.
 START_SSH_SERVER = false
+; Expect HAPROXY header on connections to the built-in SSH server
+SSH_SERVER_HAPROXY = false
 ; Username to use for the builtin SSH server. If blank, then it is the value of RUN_USER.
 BUILTIN_SSH_SERVER_USER =
 ; Domain name to be exposed in clone URL

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -168,6 +168,10 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 ## Server (`server`)
 
 - `PROTOCOL`: **http**: \[http, https, fcgi, unix, fcgi+unix\]
+- `HAPROXY`: **false**: Expect HAPROXY headers on connections
+- `HAPROXY_TLS_BRIDGING`: **false**: When protocol is https, expect HAPROXY header after TLS negotiation.
+- `HAPROXY_HEADER_TIMEOUT`: **5s**: Timeout to wait for HAProxy header (set to 0 to have no timeout)
+- `HAPROXY_ACCEPT_UNKNOWN`: **false**: Accept Unknown HAProxy
 - `DOMAIN`: **localhost**: Domain name of this server.
 - `ROOT_URL`: **%(PROTOCOL)s://%(DOMAIN)s:%(HTTP\_PORT)s/**:
    Overwrite the automatically generated public URL.
@@ -192,8 +196,11 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
    most cases you do not need to change the default value. Alter it only if
    your SSH server node is not the same as HTTP node. Do not set this variable
    if `PROTOCOL` is set to `unix`.
+- `LOCAL_HAPROXY`: **%(HAPROXY)**: When making local connections pass the HAPROXY header.
+   This should be set to false if the local connection will go through the HAPROXY.
 - `DISABLE_SSH`: **false**: Disable SSH feature when it's not available.
 - `START_SSH_SERVER`: **false**: When enabled, use the built-in SSH server.
+- `SSH_SERVER_HAPROXY`: **false**: Expect HAPROXY header on connections to the built-in SSH Server.
 - `SSH_DOMAIN`: **%(DOMAIN)s**: Domain name of this server, used for displayed clone URL.
 - `SSH_PORT`: **22**: SSH port displayed in clone URL.
 - `SSH_LISTEN_HOST`: **0.0.0.0**: Listen address for the built-in SSH server.
@@ -213,6 +220,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
 - `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
 - `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
+- `REDIRECT_HAPROXY`: **%(HAPROXY)**: expect HAPROXY header on connections to https redirector.
 - `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
 - `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server).
    By using Lets Encrypt **you must consent** to their [terms of service](https://letsencrypt.org/documents/LE-SA-v1.2-November-15-2017.pdf).

modules/graceful/server.go

@@ -16,7 +16,9 @@ import (
 	"syscall"
 	"time"
 
+	"code.gitea.io/gitea/modules/haproxy"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
 )
 
 var (
@@ -71,16 +73,27 @@ func NewServer(network, address string) *Server {
 
 // ListenAndServe listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func (srv *Server) ListenAndServe(serve ServeFunction) error {
+func (srv *Server) ListenAndServe(serve ServeFunction, haProxy bool) error {
 	go srv.awaitShutdown()
 
-	l, err := GetListener(srv.network, srv.address)
+	listener, err := GetListener(srv.network, srv.address)
 	if err != nil {
 		log.Error("Unable to GetListener: %v", err)
 		return err
 	}
 
-	srv.listener = newWrappedListener(l, srv)
+	// we need to wrap the listener to take account of our lifecycle
+	listener = newWrappedListener(listener, srv)
+
+	// Now we need to take account of HAProxy settings...
+	if haProxy {
+		listener = &haproxy.Listener{
+			Listener:           listener,
+			ProxyHeaderTimeout: setting.HAProxyHeaderTimeout,
+			AcceptUnknown:      setting.HAProxyAcceptUnknown,
+		}
+	}
+	srv.listener = listener
 
 	srv.BeforeBegin(srv.network, srv.address)
 
@@ -94,7 +107,7 @@ func (srv *Server) ListenAndServe(serve ServeFunction) error {
 // be provided. If the certificate is signed by a certificate authority, the
 // certFile should be the concatenation of the server's certificate followed by the
 // CA's certificate.
-func (srv *Server) ListenAndServeTLS(certFile, keyFile string, serve ServeFunction) error {
+func (srv *Server) ListenAndServeTLS(certFile, keyFile string, serve ServeFunction, haProxy, haProxyTLSBridging bool) error {
 	config := &tls.Config{}
 	if config.NextProtos == nil {
 		config.NextProtos = []string{"http/1.1"}
@@ -120,23 +133,45 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string, serve ServeFuncti
 		return err
 	}
 
-	return srv.ListenAndServeTLSConfig(config, serve)
+	return srv.ListenAndServeTLSConfig(config, serve, haProxy, haProxyTLSBridging)
 }
 
 // ListenAndServeTLSConfig listens on the provided network address and then calls
 // Serve to handle requests on incoming TLS connections.
-func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFunction) error {
+func (srv *Server) ListenAndServeTLSConfig(tlsConfig *tls.Config, serve ServeFunction, haProxy, haProxyTLSBridging bool) error {
 	go srv.awaitShutdown()
 
-	l, err := GetListener(srv.network, srv.address)
+	listener, err := GetListener(srv.network, srv.address)
 	if err != nil {
 		log.Error("Unable to get Listener: %v", err)
 		return err
 	}
 
-	wl := newWrappedListener(l, srv)
-	srv.listener = tls.NewListener(wl, tlsConfig)
+	// we need to wrap the listener to take account of our lifecycle
+	listener = newWrappedListener(listener, srv)
+
+	// Now we need to take account of HAProxy settings... If we're not bridging then we expect that the proxy will forward the connection to us
+	if haProxy && !haProxyTLSBridging {
+		listener = &haproxy.Listener{
+			Listener:           listener,
+			ProxyHeaderTimeout: setting.HAProxyHeaderTimeout,
+			AcceptUnknown:      setting.HAProxyAcceptUnknown,
+		}
+	}
+
+	// Now handle the tls protocol
+	listener = tls.NewListener(listener, tlsConfig)
+
+	// Now if we're bridging then we need the proxy to tell us who we're bridging for...
+	if haProxy && haProxyTLSBridging {
+		listener = &haproxy.Listener{
+			Listener:           listener,
+			ProxyHeaderTimeout: setting.HAProxyHeaderTimeout,
+			AcceptUnknown:      setting.HAProxyAcceptUnknown,
+		}
+	}
 
+	srv.listener = listener
 	srv.BeforeBegin(srv.network, srv.address)
 
 	return srv.Serve(serve)

modules/graceful/server_http.go

@@ -25,21 +25,21 @@ func newHTTPServer(network, address string, handler http.Handler) (*Server, Serv
 
 // HTTPListenAndServe listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func HTTPListenAndServe(network, address string, handler http.Handler) error {
+func HTTPListenAndServe(network, address string, handler http.Handler, haProxy bool) error {
 	server, lHandler := newHTTPServer(network, address, handler)
-	return server.ListenAndServe(lHandler)
+	return server.ListenAndServe(lHandler, haProxy)
 }
 
 // HTTPListenAndServeTLS listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func HTTPListenAndServeTLS(network, address, certFile, keyFile string, handler http.Handler) error {
+func HTTPListenAndServeTLS(network, address, certFile, keyFile string, handler http.Handler, haProxy, haProxyTLSBridging bool) error {
 	server, lHandler := newHTTPServer(network, address, handler)
-	return server.ListenAndServeTLS(certFile, keyFile, lHandler)
+	return server.ListenAndServeTLS(certFile, keyFile, lHandler, haProxy, haProxyTLSBridging)
 }
 
 // HTTPListenAndServeTLSConfig listens on the provided network address and then calls Serve
 // to handle requests on incoming connections.
-func HTTPListenAndServeTLSConfig(network, address string, tlsConfig *tls.Config, handler http.Handler) error {
+func HTTPListenAndServeTLSConfig(network, address string, tlsConfig *tls.Config, handler http.Handler, haProxy, haProxyTLSBridging bool) error {
 	server, lHandler := newHTTPServer(network, address, handler)
-	return server.ListenAndServeTLSConfig(tlsConfig, lHandler)
+	return server.ListenAndServeTLSConfig(tlsConfig, lHandler, haProxy, haProxyTLSBridging)
 }