[release-branch.go1.20] syscall: restore original NOFILE rlimit in child process

If we increased the NOFILE rlimit when starting the program,
restore the original rlimit when forking a child process.

In CL 393354 the os package was changed to raise the open file rlimit
at program start. That code is not inherently tied to the os package.
This CL moves it into the syscall package.

This is a backport of CLs 476096 and 476097 from trunk.

For #46279
Fixes #59064

Change-Id: Ib813de896de0a5d28fa2b29afdf414a89fbe7b2a
Reviewed-on: https://go-review.googlesource.com/c/go/+/478659
Run-TryBot: Ian Lance Taylor <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: David Chase <[email protected]>
Reviewed-by: Michael Knyszek <[email protected]>
Reviewed-by: Tobias Klauser <[email protected]>

Author: ianlancetaylor

src/runtime/syscall2_solaris.go

@@ -17,6 +17,7 @@ import _ "unsafe" // for go:linkname
 //go:cgo_import_dynamic libc_ioctl ioctl "libc.so"
 //go:cgo_import_dynamic libc_setgid setgid "libc.so"
 //go:cgo_import_dynamic libc_setgroups setgroups "libc.so"
+//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so"
 //go:cgo_import_dynamic libc_setsid setsid "libc.so"
 //go:cgo_import_dynamic libc_setuid setuid "libc.so"
 //go:cgo_import_dynamic libc_setpgid setpgid "libc.so"
@@ -34,6 +35,7 @@ import _ "unsafe" // for go:linkname
 //go:linkname libc_ioctl libc_ioctl
 //go:linkname libc_setgid libc_setgid
 //go:linkname libc_setgroups libc_setgroups
+//go:linkname libc_setrlimit libc_setrlimit
 //go:linkname libc_setsid libc_setsid
 //go:linkname libc_setuid libc_setuid
 //go:linkname libc_setpgid libc_setpgid

src/runtime/syscall_aix.go

@@ -18,6 +18,7 @@ import "unsafe"
 //go:cgo_import_dynamic libc_ioctl ioctl "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setgid setgid "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setgroups setgroups "libc.a/shr_64.o"
+//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setsid setsid "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setuid setuid "libc.a/shr_64.o"
 //go:cgo_import_dynamic libc_setpgid setpgid "libc.a/shr_64.o"
@@ -31,6 +32,7 @@ import "unsafe"
 //go:linkname libc_ioctl libc_ioctl
 //go:linkname libc_setgid libc_setgid
 //go:linkname libc_setgroups libc_setgroups
+//go:linkname libc_setrlimit libc_setrlimit
 //go:linkname libc_setsid libc_setsid
 //go:linkname libc_setuid libc_setuid
 //go:linkname libc_setpgid libc_setpgid
@@ -45,6 +47,7 @@ var (
 	libc_ioctl,
 	libc_setgid,
 	libc_setgroups,
+	libc_setrlimit,
 	libc_setsid,
 	libc_setuid,
 	libc_setpgid libFunc
@@ -199,6 +202,13 @@ func syscall_setgroups1(ngid, gid uintptr) (err uintptr) {
 	return
 }
 
+//go:linkname syscall_setrlimit1 syscall.setrlimit1
+//go:nosplit
+func syscall_setrlimit1(which uintptr, lim unsafe.Pointer) (err uintptr) {
+	_, err = syscall2(&libc_setrlimit, which, uintptr(lim))
+	return
+}
+
 //go:linkname syscall_setsid syscall.setsid
 //go:nosplit
 func syscall_setsid() (pid, err uintptr) {

src/runtime/syscall_solaris.go

@@ -18,6 +18,7 @@ var (
 	libc_ioctl,
 	libc_setgid,
 	libc_setgroups,
+	libc_setrlimit,
 	libc_setsid,
 	libc_setuid,
 	libc_setpgid,
@@ -234,6 +235,19 @@ func syscall_setgroups(ngid, gid uintptr) (err uintptr) {
 	return call.err
 }
 
+//go:nosplit
+//go:linkname syscall_setrlimit
+//go:cgo_unsafe_args
+func syscall_setrlimit(which uintptr, lim unsafe.Pointer) (err uintptr) {
+	call := libcall{
+		fn:   uintptr(unsafe.Pointer(&libc_setrlimit)),
+		n:    2,
+		args: uintptr(unsafe.Pointer(&which)),
+	}
+	asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&call))
+	return call.err
+}
+
 //go:nosplit
 //go:linkname syscall_setsid
 func syscall_setsid() (pid, err uintptr) {

src/syscall/asm_solaris_amd64.s

@@ -60,6 +60,9 @@ TEXT ·setgid(SB),NOSPLIT,$0
 TEXT ·setgroups1(SB),NOSPLIT,$0
 	JMP	runtime·syscall_setgroups(SB)
 
+TEXT ·setrlimit1(SB),NOSPLIT,$0
+	JMP	runtime·syscall_setrlimit(SB)
+
 TEXT ·setsid(SB),NOSPLIT,$0
 	JMP	runtime·syscall_setsid(SB)
 

src/syscall/exec_bsd.go

@@ -61,6 +61,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		i      int
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// guard against side effects of shuffling fds below.
 	// Make sure that nextfd is beyond any currently open files so
 	// that we can't run the risk of overwriting any of them.
@@ -270,6 +272,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		RawSyscall(SYS_SETRLIMIT, uintptr(RLIMIT_NOFILE), uintptr(unsafe.Pointer(&rlim)), 0)
+	}
+
 	// Time to exec.
 	_, _, err1 = RawSyscall(SYS_EXECVE,
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_freebsd.go

@@ -66,6 +66,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		i      int
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// Record parent PID so child can test if it has died.
 	ppid, _, _ := RawSyscall(SYS_GETPID, 0, 0, 0)
 
@@ -283,6 +285,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		RawSyscall(SYS_SETRLIMIT, uintptr(RLIMIT_NOFILE), uintptr(unsafe.Pointer(&rlim)), 0)
+	}
+
 	// Time to exec.
 	_, _, err1 = RawSyscall(SYS_EXECVE,
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_libc.go

@@ -53,6 +53,7 @@ func getpid() (pid uintptr, err Errno)
 func ioctl(fd uintptr, req uintptr, arg uintptr) (err Errno)
 func setgid(gid uintptr) (err Errno)
 func setgroups1(ngid uintptr, gid uintptr) (err Errno)
+func setrlimit1(which uintptr, lim unsafe.Pointer) (err Errno)
 func setsid() (pid uintptr, err Errno)
 func setuid(uid uintptr) (err Errno)
 func setpgid(pid uintptr, pgid uintptr) (err Errno)
@@ -87,6 +88,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		i      int
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// guard against side effects of shuffling fds below.
 	// Make sure that nextfd is beyond any currently open files so
 	// that we can't run the risk of overwriting any of them.
@@ -289,6 +292,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		setrlimit1(RLIMIT_NOFILE, unsafe.Pointer(&rlim))
+	}
+
 	// Time to exec.
 	err1 = execve(
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_libc2.go

@@ -62,6 +62,8 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		i      int
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	// guard against side effects of shuffling fds below.
 	// Make sure that nextfd is beyond any currently open files so
 	// that we can't run the risk of overwriting any of them.
@@ -266,6 +268,11 @@ func forkAndExecInChild(argv0 *byte, argv, envv []*byte, chroot, dir *byte, attr
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		rawSyscall(abi.FuncPCABI0(libc_setrlimit_trampoline), uintptr(RLIMIT_NOFILE), uintptr(unsafe.Pointer(&rlim)), 0)
+	}
+
 	// Time to exec.
 	_, _, err1 = rawSyscall(abi.FuncPCABI0(libc_execve_trampoline),
 		uintptr(unsafe.Pointer(argv0)),

src/syscall/exec_linux.go

@@ -228,6 +228,8 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att
 		clone3                    *cloneArgs
 	)
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+
 	if sys.UidMappings != nil {
 		puid = []byte("/proc/self/uid_map\000")
 		uidmap = formatIDMappings(sys.UidMappings)
@@ -592,6 +594,11 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att
 		}
 	}
 
+	// Restore original rlimit.
+	if rlimOK && rlim.Cur != 0 {
+		rawSetrlimit(RLIMIT_NOFILE, &rlim)
+	}
+
 	// Enable tracing if requested.
 	// Do this right before exec so that we don't unnecessarily trace the runtime
 	// setting up after the fork. See issue #21428.

src/syscall/exec_unix.go

@@ -282,6 +282,11 @@ func Exec(argv0 string, argv []string, envv []string) (err error) {
 	}
 	runtime_BeforeExec()
 
+	rlim, rlimOK := origRlimitNofile.Load().(Rlimit)
+	if rlimOK && rlim.Cur != 0 {
+		Setrlimit(RLIMIT_NOFILE, &rlim)
+	}
+
 	var err1 error
 	if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" || runtime.GOOS == "aix" {
 		// RawSyscall should never be used on Solaris, illumos, or AIX.

src/syscall/exec_unix_test.go

@@ -7,12 +7,15 @@
 package syscall_test
 
 import (
+	"bytes"
+	"fmt"
 	"internal/testenv"
 	"io"
 	"math/rand"
 	"os"
 	"os/exec"
 	"os/signal"
+	"strconv"
 	"syscall"
 	"testing"
 	"time"
@@ -345,3 +348,42 @@ func TestExecHelper(t *testing.T) {
 
 	t.Error("syscall.Exec returned")
 }
+
+// Test that rlimit values are restored by exec.
+func TestRlimitRestored(t *testing.T) {
+	if os.Getenv("GO_WANT_HELPER_PROCESS") != "" {
+		fmt.Println(syscall.OrigRlimitNofile().Cur)
+		os.Exit(0)
+	}
+
+	orig := syscall.OrigRlimitNofile()
+	if orig.Cur == 0 {
+		t.Skip("skipping test because rlimit not adjusted at startup")
+	}
+
+	executable, err := os.Executable()
+	if err != nil {
+		executable = os.Args[0]
+	}
+
+	cmd := testenv.Command(t, executable, "-test.run=TestRlimitRestored")
+	cmd = testenv.CleanCmdEnv(cmd)
+	cmd.Env = append(cmd.Env, "GO_WANT_HELPER_PROCESS=1")
+
+	out, err := cmd.CombinedOutput()
+	if len(out) > 0 {
+		t.Logf("%s", out)
+	}
+	if err != nil {
+		t.Fatalf("subprocess failed: %v", err)
+	}
+	s := string(bytes.TrimSpace(out))
+	v, err := strconv.ParseUint(s, 10, 64)
+	if err != nil {
+		t.Fatalf("could not parse %q as number: %v", s, v)
+	}
+
+	if v != uint64(orig.Cur) {
+		t.Errorf("exec rlimit = %d, want %d", v, orig)
+	}
+}

src/syscall/export_rlimit_test.go

@@ -0,0 +1,14 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build unix
+
+package syscall
+
+func OrigRlimitNofile() Rlimit {
+	if rlim, ok := origRlimitNofile.Load().(Rlimit); ok {
+		return rlim
+	}
+	return Rlimit{0, 0}
+}

src/os/rlimit.go renamed to src/syscall/rlimit.go

@@ -4,9 +4,16 @@
 
 //go:build unix
 
-package os
+package syscall
 
-import "syscall"
+import (
+	"sync/atomic"
+)
+
+// origRlimitNofile, if not {0, 0}, is the original soft RLIMIT_NOFILE.
+// When we can assume that we are bootstrapping with Go 1.19,
+// this can be atomic.Pointer[Rlimit].
+var origRlimitNofile atomic.Value // of Rlimit
 
 // Some systems set an artificially low soft limit on open file count, for compatibility
 // with code that uses select and its hard-coded maximum file descriptor
@@ -23,10 +30,21 @@ import "syscall"
 // Code that really wants Go to leave the limit alone can set the hard limit,
 // which Go of course has no choice but to respect.
 func init() {
-	var lim syscall.Rlimit
-	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max {
+	var lim Rlimit
+	if err := Getrlimit(RLIMIT_NOFILE, &lim); err == nil && lim.Cur != lim.Max {
+		origRlimitNofile.Store(lim)
 		lim.Cur = lim.Max
 		adjustFileLimit(&lim)
-		syscall.Setrlimit(syscall.RLIMIT_NOFILE, &lim)
+		setrlimit(RLIMIT_NOFILE, &lim)
+	}
+}
+
+func Setrlimit(resource int, rlim *Rlimit) error {
+	err := setrlimit(resource, rlim)
+	if err == nil && resource == RLIMIT_NOFILE {
+		// Store zeroes in origRlimitNofile to tell StartProcess
+		// to not adjust the rlimit in the child process.
+		origRlimitNofile.Store(Rlimit{0, 0})
 	}
+	return err
 }

src/os/rlimit_darwin.go renamed to src/syscall/rlimit_darwin.go

@@ -4,15 +4,13 @@
 
 //go:build darwin
 
-package os
-
-import "syscall"
+package syscall
 
 // adjustFileLimit adds per-OS limitations on the Rlimit used for RLIMIT_NOFILE. See rlimit.go.
-func adjustFileLimit(lim *syscall.Rlimit) {
+func adjustFileLimit(lim *Rlimit) {
 	// On older macOS, setrlimit(RLIMIT_NOFILE, lim) with lim.Cur = infinity fails.
 	// Set to the value of kern.maxfilesperproc instead.
-	n, err := syscall.SysctlUint32("kern.maxfilesperproc")
+	n, err := SysctlUint32("kern.maxfilesperproc")
 	if err != nil {
 		return
 	}

src/os/rlimit_stub.go renamed to src/syscall/rlimit_stub.go

@@ -4,9 +4,7 @@
 
 //go:build aix || dragonfly || freebsd || linux || netbsd || openbsd || solaris
 
-package os
-
-import "syscall"
+package syscall
 
 // adjustFileLimit adds per-OS limitations on the Rlimit used for RLIMIT_NOFILE. See rlimit.go.
-func adjustFileLimit(lim *syscall.Rlimit) {}
+func adjustFileLimit(lim *Rlimit) {}

src/os/rlimit_test.go renamed to src/syscall/rlimit_test.go

@@ -2,10 +2,10 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package os_test
+package syscall_test
 
 import (
-	. "os"
+	"os"
 	"runtime"
 	"testing"
 )
@@ -24,9 +24,9 @@ func TestOpenFileLimit(t *testing.T) {
 		fileCount = 768
 	}
 
-	var files []*File
+	var files []*os.File
 	for i := 0; i < fileCount; i++ {
-		f, err := Open("rlimit.go")
+		f, err := os.Open("rlimit.go")
 		if err != nil {
 			t.Error(err)
 			break

src/syscall/syscall_aix.go

@@ -633,7 +633,7 @@ func PtraceDetach(pid int) (err error) { return ptrace64(PT_DETACH, int64(pid),
 //sys	Setpriority(which int, who int, prio int) (err error)
 //sysnb	Setregid(rgid int, egid int) (err error)
 //sysnb	Setreuid(ruid int, euid int) (err error)
-//sysnb	Setrlimit(which int, lim *Rlimit) (err error)
+//sysnb	setrlimit(which int, lim *Rlimit) (err error)
 //sys	Stat(path string, stat *Stat_t) (err error)
 //sys	Statfs(path string, buf *Statfs_t) (err error)
 //sys	Symlink(path string, link string) (err error)