Skip to content

Commit 193088b

Browse files
aclementsaclements
committed
runtime: separate error result for mmap
Currently mmap returns an unsafe.Pointer that encodes OS errors as values less than 4096. In practice this is okay, but it borders on being really unsafe: for example, the value has to be checked immediately after return and if stack copying were ever to observe such a value, it would panic. It's also not remotely idiomatic. Fix this by making mmap return a separate pointer value and error, like a normal Go function. Updates #22218. Change-Id: Iefd965095ffc82cc91118872753a5d39d785c3a6 Reviewed-on: https://go-review.googlesource.com/71270 Run-TryBot: Austin Clements <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Ian Lance Taylor <[email protected]>
1 parent 3ba818c commit 193088b

35 files changed

+237
-103
lines changed

‎src/runtime/cgo/gcc_mmap.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
#include "libcgo.h"
1313

14-
void *
14+
uintptr_t
1515
x_cgo_mmap(void *addr, uintptr_t length, int32_t prot, int32_t flags, int32_t fd, uint32_t offset) {
1616
void *p;
1717

@@ -20,9 +20,9 @@ x_cgo_mmap(void *addr, uintptr_t length, int32_t prot, int32_t flags, int32_t fd
2020
_cgo_tsan_release();
2121
if (p == MAP_FAILED) {
2222
/* This is what the Go code expects on failure. */
23-
p = (void *) (uintptr_t) errno;
23+
return (uintptr_t)errno;
2424
}
25-
return p;
25+
return (uintptr_t)p;
2626
}
2727

2828
void

‎src/runtime/cgo_mmap.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,21 @@ var _cgo_mmap unsafe.Pointer
2020
//go:linkname _cgo_munmap _cgo_munmap
2121
var _cgo_munmap unsafe.Pointer
2222

23-
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer {
23+
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
2424
if _cgo_mmap != nil {
2525
// Make ret a uintptr so that writing to it in the
2626
// function literal does not trigger a write barrier.
2727
// A write barrier here could break because of the way
2828
// that mmap uses the same value both as a pointer and
2929
// an errno value.
30-
// TODO: Fix mmap to return two values.
3130
var ret uintptr
3231
systemstack(func() {
3332
ret = callCgoMmap(addr, n, prot, flags, fd, off)
3433
})
35-
return unsafe.Pointer(ret)
34+
if ret < 4096 {
35+
return nil, int(ret)
36+
}
37+
return unsafe.Pointer(ret), 0
3638
}
3739
return sysMmap(addr, n, prot, flags, fd, off)
3840
}
@@ -46,7 +48,7 @@ func munmap(addr unsafe.Pointer, n uintptr) {
4648
}
4749

4850
// sysMmap calls the mmap system call. It is implemented in assembly.
49-
func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer
51+
func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int)
5052

5153
// callCgoMmap calls the mmap function in the runtime/cgo package
5254
// using the GCC calling convention. It is implemented in assembly.

‎src/runtime/mem_bsd.go

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ import (
1515
// which prevents us from allocating more stack.
1616
//go:nosplit
1717
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
18-
v := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
19-
if uintptr(v) < 4096 {
18+
v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
19+
if err != 0 {
2020
return nil
2121
}
2222
mSysStatInc(sysStat, n)
@@ -51,8 +51,8 @@ func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
5151
return v
5252
}
5353

54-
p := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
55-
if uintptr(p) < 4096 {
54+
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
55+
if err != 0 {
5656
return nil
5757
}
5858
*reserved = true
@@ -76,22 +76,22 @@ func sysMap(v unsafe.Pointer, n uintptr, reserved bool, sysStat *uint64) {
7676
// to do this - we do not on other platforms.
7777
flags |= _MAP_FIXED
7878
}
79-
p := mmap(v, n, _PROT_READ|_PROT_WRITE, flags, -1, 0)
80-
if uintptr(p) == _ENOMEM || (GOOS == "solaris" && uintptr(p) == _sunosEAGAIN) {
79+
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, flags, -1, 0)
80+
if err == _ENOMEM || (GOOS == "solaris" && err == _sunosEAGAIN) {
8181
throw("runtime: out of memory")
8282
}
83-
if p != v {
84-
print("runtime: address space conflict: map(", v, ") = ", p, "\n")
83+
if p != v || err != 0 {
84+
print("runtime: address space conflict: map(", v, ") = ", p, "(err ", err, ")\n")
8585
throw("runtime: address space conflict")
8686
}
8787
return
8888
}
8989

90-
p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
91-
if uintptr(p) == _ENOMEM || (GOOS == "solaris" && uintptr(p) == _sunosEAGAIN) {
90+
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
91+
if err == _ENOMEM || (GOOS == "solaris" && err == _sunosEAGAIN) {
9292
throw("runtime: out of memory")
9393
}
94-
if p != v {
94+
if p != v || err != 0 {
9595
throw("runtime: cannot map pages in arena address space")
9696
}
9797
}

‎src/runtime/mem_darwin.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import "unsafe"
1010
// which prevents us from allocating more stack.
1111
//go:nosplit
1212
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
13-
v := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
14-
if uintptr(v) < 4096 {
13+
v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
14+
if err != 0 {
1515
return nil
1616
}
1717
mSysStatInc(sysStat, n)
@@ -40,8 +40,8 @@ func sysFault(v unsafe.Pointer, n uintptr) {
4040

4141
func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
4242
*reserved = true
43-
p := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
44-
if uintptr(p) < 4096 {
43+
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
44+
if err != 0 {
4545
return nil
4646
}
4747
return p
@@ -53,11 +53,11 @@ const (
5353

5454
func sysMap(v unsafe.Pointer, n uintptr, reserved bool, sysStat *uint64) {
5555
mSysStatInc(sysStat, n)
56-
p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
57-
if uintptr(p) == _ENOMEM {
56+
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
57+
if err == _ENOMEM {
5858
throw("runtime: out of memory")
5959
}
60-
if p != v {
60+
if p != v || err != 0 {
6161
throw("runtime: cannot map pages in arena address space")
6262
}
6363
}

‎src/runtime/mem_linux.go

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -41,30 +41,30 @@ func addrspace_free(v unsafe.Pointer, n uintptr) bool {
4141
return true
4242
}
4343

44-
func mmap_fixed(v unsafe.Pointer, n uintptr, prot, flags, fd int32, offset uint32) unsafe.Pointer {
45-
p := mmap(v, n, prot, flags, fd, offset)
44+
func mmap_fixed(v unsafe.Pointer, n uintptr, prot, flags, fd int32, offset uint32) (unsafe.Pointer, int) {
45+
p, err := mmap(v, n, prot, flags, fd, offset)
4646
// On some systems, mmap ignores v without
4747
// MAP_FIXED, so retry if the address space is free.
4848
if p != v && addrspace_free(v, n) {
49-
if uintptr(p) > 4096 {
49+
if err == 0 {
5050
munmap(p, n)
5151
}
52-
p = mmap(v, n, prot, flags|_MAP_FIXED, fd, offset)
52+
p, err = mmap(v, n, prot, flags|_MAP_FIXED, fd, offset)
5353
}
54-
return p
54+
return p, err
5555
}
5656

5757
// Don't split the stack as this method may be invoked without a valid G, which
5858
// prevents us from allocating more stack.
5959
//go:nosplit
6060
func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer {
61-
p := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
62-
if uintptr(p) < 4096 {
63-
if uintptr(p) == _EACCES {
61+
p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
62+
if err != 0 {
63+
if err == _EACCES {
6464
print("runtime: mmap: access denied\n")
6565
exit(2)
6666
}
67-
if uintptr(p) == _EAGAIN {
67+
if err == _EAGAIN {
6868
print("runtime: mmap: too much locked memory (check 'ulimit -l').\n")
6969
exit(2)
7070
}
@@ -186,9 +186,9 @@ func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
186186
// if we can reserve at least 64K and check the assumption in SysMap.
187187
// Only user-mode Linux (UML) rejects these requests.
188188
if sys.PtrSize == 8 && uint64(n) > 1<<32 {
189-
p := mmap_fixed(v, 64<<10, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
190-
if p != v {
191-
if uintptr(p) >= 4096 {
189+
p, err := mmap_fixed(v, 64<<10, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
190+
if p != v || err != 0 {
191+
if err == 0 {
192192
munmap(p, 64<<10)
193193
}
194194
return nil
@@ -198,8 +198,8 @@ func sysReserve(v unsafe.Pointer, n uintptr, reserved *bool) unsafe.Pointer {
198198
return v
199199
}
200200

201-
p := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
202-
if uintptr(p) < 4096 {
201+
p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
202+
if err != 0 {
203203
return nil
204204
}
205205
*reserved = true
@@ -211,22 +211,22 @@ func sysMap(v unsafe.Pointer, n uintptr, reserved bool, sysStat *uint64) {
211211

212212
// On 64-bit, we don't actually have v reserved, so tread carefully.
213213
if !reserved {
214-
p := mmap_fixed(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
215-
if uintptr(p) == _ENOMEM {
214+
p, err := mmap_fixed(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
215+
if err == _ENOMEM {
216216
throw("runtime: out of memory")
217217
}
218-
if p != v {
219-
print("runtime: address space conflict: map(", v, ") = ", p, "\n")
218+
if p != v || err != 0 {
219+
print("runtime: address space conflict: map(", v, ") = ", p, " (err ", err, ")\n")
220220
throw("runtime: address space conflict")
221221
}
222222
return
223223
}
224224

225-
p := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
226-
if uintptr(p) == _ENOMEM {
225+
p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0)
226+
if err == _ENOMEM {
227227
throw("runtime: out of memory")
228228
}
229-
if p != v {
229+
if p != v || err != 0 {
230230
throw("runtime: cannot map pages in arena address space")
231231
}
232232
}

‎src/runtime/mmap.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ import "unsafe"
1616
// We only pass the lower 32 bits of file offset to the
1717
// assembly routine; the higher bits (if required), should be provided
1818
// by the assembly routine as 0.
19-
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer
19+
// The err result is an OS error code such as ENOMEM.
20+
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int)
2021

2122
// munmap calls the munmap system call. It is implemented in assembly.
2223
func munmap(addr unsafe.Pointer, n uintptr)

‎src/runtime/os3_solaris.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -402,12 +402,12 @@ func madvise(addr unsafe.Pointer, n uintptr, flags int32) {
402402
}
403403

404404
//go:nosplit
405-
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer {
405+
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
406406
p, err := doMmap(uintptr(addr), n, uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off))
407407
if p == ^uintptr(0) {
408-
return unsafe.Pointer(err)
408+
return nil, int(err)
409409
}
410-
return unsafe.Pointer(p)
410+
return unsafe.Pointer(p), 0
411411
}
412412

413413
//go:nosplit

‎src/runtime/os_linux.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -220,8 +220,8 @@ func sysargs(argc int32, argv **byte) {
220220
// try using mincore to detect the physical page size.
221221
// mincore should return EINVAL when address is not a multiple of system page size.
222222
const size = 256 << 10 // size of memory region to allocate
223-
p := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
224-
if uintptr(p) < 4096 {
223+
p, err := mmap(nil, size, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0)
224+
if err != 0 {
225225
return
226226
}
227227
var n uintptr

‎src/runtime/os_nacl.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ func nacl_thread_create(fn uintptr, stk, tls, xx unsafe.Pointer) int32
3333
//go:noescape
3434
func nacl_nanosleep(ts, extra *timespec) int32
3535
func nanotime() int64
36-
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer
36+
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int)
3737
func exit(code int32)
3838
func osyield()
3939

‎src/runtime/runtime_mmap_test.go

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,10 @@ import (
1616
// what the code in mem_bsd.go, mem_darwin.go, and mem_linux.go expects.
1717
// See the uses of ENOMEM in sysMap in those files.
1818
func TestMmapErrorSign(t *testing.T) {
19-
p := runtime.Mmap(nil, ^uintptr(0)&^(runtime.GetPhysPageSize()-1), 0, runtime.MAP_ANON|runtime.MAP_PRIVATE, -1, 0)
19+
p, err := runtime.Mmap(nil, ^uintptr(0)&^(runtime.GetPhysPageSize()-1), 0, runtime.MAP_ANON|runtime.MAP_PRIVATE, -1, 0)
2020

21-
// The runtime.mmap function is nosplit, but t.Errorf is not.
22-
// Reset the pointer so that we don't get an "invalid stack
23-
// pointer" error from t.Errorf if we call it.
24-
v := uintptr(p)
25-
p = nil
26-
27-
if v != runtime.ENOMEM {
28-
t.Errorf("mmap = %v, want %v", v, runtime.ENOMEM)
21+
if p != nil || err != runtime.ENOMEM {
22+
t.Errorf("mmap = %v, %v, want nil, %v", p, err, runtime.ENOMEM)
2923
}
3024
}
3125

@@ -35,20 +29,20 @@ func TestPhysPageSize(t *testing.T) {
3529
ps := runtime.GetPhysPageSize()
3630

3731
// Get a region of memory to play with. This should be page-aligned.
38-
b := uintptr(runtime.Mmap(nil, 2*ps, 0, runtime.MAP_ANON|runtime.MAP_PRIVATE, -1, 0))
39-
if b < 4096 {
40-
t.Fatalf("Mmap: %v", b)
32+
b, err := runtime.Mmap(nil, 2*ps, 0, runtime.MAP_ANON|runtime.MAP_PRIVATE, -1, 0)
33+
if err != 0 {
34+
t.Fatalf("Mmap: %v", err)
4135
}
4236

4337
// Mmap should fail at a half page into the buffer.
44-
err := uintptr(runtime.Mmap(unsafe.Pointer(uintptr(b)+ps/2), ps, 0, runtime.MAP_ANON|runtime.MAP_PRIVATE|runtime.MAP_FIXED, -1, 0))
45-
if err >= 4096 {
38+
_, err = runtime.Mmap(unsafe.Pointer(uintptr(b)+ps/2), ps, 0, runtime.MAP_ANON|runtime.MAP_PRIVATE|runtime.MAP_FIXED, -1, 0)
39+
if err == 0 {
4640
t.Errorf("Mmap should have failed with half-page alignment %d, but succeeded: %v", ps/2, err)
4741
}
4842

4943
// Mmap should succeed at a full page into the buffer.
50-
err = uintptr(runtime.Mmap(unsafe.Pointer(uintptr(b)+ps), ps, 0, runtime.MAP_ANON|runtime.MAP_PRIVATE|runtime.MAP_FIXED, -1, 0))
51-
if err < 4096 {
44+
_, err = runtime.Mmap(unsafe.Pointer(uintptr(b)+ps), ps, 0, runtime.MAP_ANON|runtime.MAP_PRIVATE|runtime.MAP_FIXED, -1, 0)
45+
if err != 0 {
5246
t.Errorf("Mmap at full-page alignment %d failed: %v", ps, err)
5347
}
5448
}

‎src/runtime/sys_darwin_386.s

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,13 @@ TEXT runtime·raiseproc(SB),NOSPLIT,$16
103103
TEXT runtime·mmap(SB),NOSPLIT,$0
104104
MOVL $197, AX
105105
INT $0x80
106-
MOVL AX, ret+24(FP)
106+
JAE ok
107+
MOVL $0, p+24(FP)
108+
MOVL AX, err+28(FP)
109+
RET
110+
ok:
111+
MOVL AX, p+24(FP)
112+
MOVL $0, err+28(FP)
107113
RET
108114

109115
TEXT runtime·madvise(SB),NOSPLIT,$0

‎src/runtime/sys_darwin_amd64.s

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,13 @@ TEXT runtime·mmap(SB),NOSPLIT,$0
374374
MOVL off+28(FP), R9 // arg 6 offset
375375
MOVL $(0x2000000+197), AX // syscall entry
376376
SYSCALL
377-
MOVQ AX, ret+32(FP)
377+
JCC ok
378+
MOVQ $0, p+32(FP)
379+
MOVQ AX, err+40(FP)
380+
RET
381+
ok:
382+
MOVQ AX, p+32(FP)
383+
MOVQ $0, err+40(FP)
378384
RET
379385

380386
TEXT runtime·munmap(SB),NOSPLIT,$0

‎src/runtime/sys_darwin_arm.s

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,14 @@ TEXT runtime·mmap(SB),NOSPLIT,$0
140140
MOVW $0, R6 // off_t is uint64_t
141141
MOVW $SYS_mmap, R12
142142
SWI $0x80
143-
MOVW R0, ret+24(FP)
143+
MOVW $0, R1
144+
BCC ok
145+
MOVW R1, p+24(FP)
146+
MOVW R0, err+28(FP)
147+
RET
148+
ok:
149+
MOVW R0, p+24(FP)
150+
MOVW R1, err+28(FP)
144151
RET
145152

146153
TEXT runtime·munmap(SB),NOSPLIT,$0

‎src/runtime/sys_darwin_arm64.s

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,13 @@ TEXT runtime·mmap(SB),NOSPLIT,$0
135135
MOVW off+28(FP), R5
136136
MOVW $SYS_mmap, R16
137137
SVC $0x80
138-
MOVD R0, ret+32(FP)
138+
BCC ok
139+
MOVD $0, p+32(FP)
140+
MOVD R0, err+40(FP)
141+
RET
142+
ok:
143+
MOVD R0, p+32(FP)
144+
MOVD $0, err+40(FP)
139145
RET
140146

141147
TEXT runtime·munmap(SB),NOSPLIT,$0

0 commit comments

Comments
 (0)