os: increase the amount of data transfer for sendfile(2) to reduce syscalls

For the moment, Go calls sendfile(2) to transfer at most 4MB at a time
while sendfile(2) actually allows a larger amount of data on one call.
To reduce system calls of sendfile(2) during data copying, we should
specify the number of bytes to copy as large as possible.

This optimization is especially advantageous for bulky file-to-file copies,
it would lead to a performance boost, the magnitude of this performance
increase may not be very exciting, but it can also cut down the CPU overhead
by decreasing the number of system calls.

This is also how we've done in sendfile_windows.go with TransmitFile.

goos: linux
goarch: amd64
pkg: os
cpu: DO-Premium-AMD
           │    old     │                new                │
           │   sec/op   │   sec/op    vs base               │
SendFile-8   1.135 ± 4%   1.052 ± 3%  -7.24% (p=0.000 n=10)

           │     old      │                 new                 │
           │     B/s      │     B/s       vs base               │
SendFile-8   902.5Mi ± 4%   973.0Mi ± 3%  +7.81% (p=0.000 n=10)

           │    old     │              new               │
           │    B/op    │    B/op     vs base            │
SendFile-8   272.0 ± 0%   272.0 ± 0%  ~ (p=1.000 n=10) ¹
¹ all samples are equal

           │    old     │              new               │
           │ allocs/op  │ allocs/op   vs base            │
SendFile-8   20.00 ± 0%   20.00 ± 0%  ~ (p=1.000 n=10) ¹
¹ all samples are equal

Change-Id: Ib4d4c6bc693e23db24697363b29226f0c9776bb0
Reviewed-on: https://go-review.googlesource.com/c/go/+/605235
LUCI-TryBot-Result: Go LUCI <[email protected]>
Auto-Submit: Ian Lance Taylor <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Jorropo <[email protected]>
Run-TryBot: Andy Pan <[email protected]>
Reviewed-by: Ian Lance Taylor <[email protected]>
Reviewed-by: Carlos Amedee <[email protected]>

Author: panjf2000

src/internal/poll/sendfile_bsd.go

@@ -10,7 +10,10 @@ import "syscall"
 
 // maxSendfileSize is the largest chunk size we ask the kernel to copy
 // at a time.
-const maxSendfileSize int = 4 << 20
+// sendfile(2)s on *BSD and Darwin don't have a limit on the size of
+// data to copy at a time, we pick the typical SSIZE_MAX on 32-bit systems,
+// which ought to be sufficient for all practical purposes.
+const maxSendfileSize int = 1<<31 - 1
 
 // SendFile wraps the sendfile system call.
 func SendFile(dstFD *FD, src int, pos, remain int64) (written int64, err error, handled bool) {

src/internal/poll/sendfile_linux.go

@@ -8,7 +8,10 @@ import "syscall"
 
 // maxSendfileSize is the largest chunk size we ask the kernel to copy
 // at a time.
-const maxSendfileSize int = 4 << 20
+// sendfile(2) on Linux will transfer at most 0x7ffff000 (2,147,479,552)
+// bytes, which is true on both 32-bit and 64-bit systems.
+// See https://man7.org/linux/man-pages/man2/sendfile.2.html#NOTES for details.
+const maxSendfileSize int = 0x7ffff000
 
 // SendFile wraps the sendfile system call.
 func SendFile(dstFD *FD, src int, remain int64) (written int64, err error, handled bool) {

src/internal/poll/sendfile_solaris.go

@@ -15,7 +15,10 @@ import "syscall"
 
 // maxSendfileSize is the largest chunk size we ask the kernel to copy
 // at a time.
-const maxSendfileSize int = 4 << 20
+// sendfile(2)s on SunOS derivatives don't have a limit on the size of
+// data to copy at a time, we pick the typical SSIZE_MAX on 32-bit systems,
+// which ought to be sufficient for all practical purposes.
+const maxSendfileSize int = 1<<31 - 1
 
 // SendFile wraps the sendfile system call.
 func SendFile(dstFD *FD, src int, pos, remain int64) (written int64, err error, handled bool) {

src/os/readfrom_linux_test.go

@@ -345,18 +345,20 @@ func hookCopyFileRange(t *testing.T) (hook *copyFileHook, name string) {
 	return
 }
 
-func hookSendFileOverCopyFileRange(t *testing.T) (hook *copyFileHook, name string) {
-	name = "hookSendFileOverCopyFileRange"
+func hookSendFileOverCopyFileRange(t *testing.T) (*copyFileHook, string) {
+	return hookSendFileTB(t), "hookSendFileOverCopyFileRange"
+}
 
+func hookSendFileTB(tb testing.TB) *copyFileHook {
 	// Disable poll.CopyFileRange to force the fallback to poll.SendFile.
 	originalCopyFileRange := *PollCopyFileRangeP
 	*PollCopyFileRangeP = func(dst, src *poll.FD, remain int64) (written int64, handled bool, err error) {
 		return 0, false, nil
 	}
 
-	hook = new(copyFileHook)
+	hook := new(copyFileHook)
 	orig := poll.TestHookDidSendFile
-	t.Cleanup(func() {
+	tb.Cleanup(func() {
 		*PollCopyFileRangeP = originalCopyFileRange
 		poll.TestHookDidSendFile = orig
 	})
@@ -368,7 +370,7 @@ func hookSendFileOverCopyFileRange(t *testing.T) (hook *copyFileHook, name strin
 		hook.err = err
 		hook.handled = handled
 	}
-	return
+	return hook
 }
 
 func hookSpliceFile(t *testing.T) *spliceFileHook {

src/os/readfrom_sendfile_test.go

@@ -0,0 +1,55 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux || solaris
+
+package os_test
+
+import (
+	"io"
+	. "os"
+	"testing"
+)
+
+func BenchmarkSendFile(b *testing.B) {
+	hook := hookSendFileTB(b)
+
+	// 1 GiB file size for copy.
+	const fileSize = 1 << 30
+
+	src, _ := createTempFile(b, "benchmark-sendfile-src", int64(fileSize))
+	dst, err := CreateTemp(b.TempDir(), "benchmark-sendfile-dst")
+	if err != nil {
+		b.Fatalf("failed to create temporary file of destination: %v", err)
+	}
+	b.Cleanup(func() {
+		dst.Close()
+	})
+
+	b.ReportAllocs()
+	b.SetBytes(int64(fileSize))
+	b.ResetTimer()
+
+	for i := 0; i <= b.N; i++ {
+		sent, err := io.Copy(dst, src)
+
+		if err != nil {
+			b.Fatalf("failed to copy data: %v", err)
+		}
+		if !hook.called {
+			b.Fatalf("should have called the sendfile(2)")
+		}
+		if sent != int64(fileSize) {
+			b.Fatalf("sent %d bytes, want %d", sent, fileSize)
+		}
+
+		// Rewind the files for the next iteration.
+		if _, err := src.Seek(0, io.SeekStart); err != nil {
+			b.Fatalf("failed to rewind the source file: %v", err)
+		}
+		if _, err := dst.Seek(0, io.SeekStart); err != nil {
+			b.Fatalf("failed to rewind the destination file: %v", err)
+		}
+	}
+}

src/os/readfrom_solaris_test.go

@@ -38,12 +38,14 @@ func newSendfileTest(t *testing.T, size int64) (dst, src *File, data []byte, hoo
 	return
 }
 
-func hookSendFile(t *testing.T) (hook *copyFileHook, name string) {
-	name = "hookSendFile"
+func hookSendFile(t *testing.T) (*copyFileHook, string) {
+	return hookSendFileTB(t), "hookSendFile"
+}
 
-	hook = new(copyFileHook)
+func hookSendFileTB(tb testing.TB) *copyFileHook {
+	hook := new(copyFileHook)
 	orig := poll.TestHookDidSendFile
-	t.Cleanup(func() {
+	tb.Cleanup(func() {
 		poll.TestHookDidSendFile = orig
 	})
 	poll.TestHookDidSendFile = func(dstFD *poll.FD, src int, written int64, err error, handled bool) {
@@ -54,5 +56,5 @@ func hookSendFile(t *testing.T) (hook *copyFileHook, name string) {
 		hook.err = err
 		hook.handled = handled
 	}
-	return
+	return hook
 }

src/os/readfrom_unix_test.go

@@ -428,28 +428,28 @@ type copyFileHook struct {
 	err     error
 }
 
-func createTempFile(t *testing.T, name string, size int64) (*File, []byte) {
-	f, err := CreateTemp(t.TempDir(), name)
+func createTempFile(tb testing.TB, name string, size int64) (*File, []byte) {
+	f, err := CreateTemp(tb.TempDir(), name)
 	if err != nil {
-		t.Fatalf("failed to create temporary file: %v", err)
+		tb.Fatalf("failed to create temporary file: %v", err)
 	}
-	t.Cleanup(func() {
+	tb.Cleanup(func() {
 		f.Close()
 	})
 
 	randSeed := time.Now().Unix()
-	t.Logf("random data seed: %d\n", randSeed)
+	tb.Logf("random data seed: %d\n", randSeed)
 	prng := rand.New(rand.NewSource(randSeed))
 	data := make([]byte, size)
 	prng.Read(data)
 	if _, err := f.Write(data); err != nil {
-		t.Fatalf("failed to create and feed the file: %v", err)
+		tb.Fatalf("failed to create and feed the file: %v", err)
 	}
 	if err := f.Sync(); err != nil {
-		t.Fatalf("failed to save the file: %v", err)
+		tb.Fatalf("failed to save the file: %v", err)
 	}
 	if _, err := f.Seek(0, io.SeekStart); err != nil {
-		t.Fatalf("failed to rewind the file: %v", err)
+		tb.Fatalf("failed to rewind the file: %v", err)
 	}
 
 	return f, data