Skip to content

bpo-35134: Add Include/cpython/pymem.h #12840

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 15, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions Include/cpython/pymem.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
#ifndef Py_CPYTHON_PYMEM_H
# error "this header file must not be included directly"
#endif

#ifdef __cplusplus
extern "C" {
#endif

PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);

/* Configure the Python memory allocators. Pass NULL to use default
allocators. */
PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt);

/* Try to get the allocators name set by _PyMem_SetupAllocators(). */
PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void);

PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);

/* strdup() using PyMem_RawMalloc() */
PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);

/* strdup() using PyMem_Malloc() */
PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);

/* wcsdup() using PyMem_RawMalloc() */
PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str);


typedef enum {
/* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
PYMEM_DOMAIN_RAW,

/* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */
PYMEM_DOMAIN_MEM,

/* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */
PYMEM_DOMAIN_OBJ
} PyMemAllocatorDomain;

typedef struct {
/* user context passed as the first argument to the 4 functions */
void *ctx;

/* allocate a memory block */
void* (*malloc) (void *ctx, size_t size);

/* allocate a memory block initialized by zeros */
void* (*calloc) (void *ctx, size_t nelem, size_t elsize);

/* allocate or resize a memory block */
void* (*realloc) (void *ctx, void *ptr, size_t new_size);

/* release a memory block */
void (*free) (void *ctx, void *ptr);
} PyMemAllocatorEx;

/* Get the memory block allocator of the specified domain. */
PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
PyMemAllocatorEx *allocator);

/* Set the memory block allocator of the specified domain.

The new allocator must return a distinct non-NULL pointer when requesting
zero bytes.

For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL
is not held when the allocator is called.

If the new allocator is not a hook (don't call the previous allocator), the
PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
on top on the new allocator. */
PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
PyMemAllocatorEx *allocator);

/* Setup hooks to detect bugs in the following Python memory allocator
functions:

- PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
- PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
- PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()

Newly allocated memory is filled with the byte 0xCB, freed memory is filled
with the byte 0xDB. Additional checks:

- detect API violations, ex: PyObject_Free() called on a buffer allocated
by PyMem_Malloc()
- detect write before the start of the buffer (buffer underflow)
- detect write after the end of the buffer (buffer overflow)

The function does nothing if Python is not compiled is debug mode. */
PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);

#ifdef __cplusplus
}
#endif
102 changes: 7 additions & 95 deletions Include/pymem.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,6 @@
extern "C" {
#endif

#ifndef Py_LIMITED_API
PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size);
PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize);
PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_RawFree(void *ptr);

/* Configure the Python memory allocators. Pass NULL to use default
allocators. */
PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt);

/* Try to get the allocators name set by _PyMem_SetupAllocators(). */
PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void);
#endif /* !defined(Py_LIMITED_API) */


/* BEWARE:

Each interface exports both functions and macros. Extension modules should
Expand Down Expand Up @@ -65,23 +50,9 @@ PyAPI_FUNC(const char*) _PyMem_GetAllocatorsName(void);
*/

PyAPI_FUNC(void *) PyMem_Malloc(size_t size);
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize);
#endif
PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
PyAPI_FUNC(void) PyMem_Free(void *ptr);

#ifndef Py_LIMITED_API
/* strdup() using PyMem_RawMalloc() */
PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);

/* strdup() using PyMem_Malloc() */
PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);

/* wcsdup() using PyMem_RawMalloc() */
PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str);
#endif

/* Macros. */

/* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL
Expand Down Expand Up @@ -130,72 +101,6 @@ PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str);
#define PyMem_Del PyMem_Free
#define PyMem_DEL PyMem_FREE

#ifndef Py_LIMITED_API
typedef enum {
/* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */
PYMEM_DOMAIN_RAW,

/* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */
PYMEM_DOMAIN_MEM,

/* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */
PYMEM_DOMAIN_OBJ
} PyMemAllocatorDomain;

typedef struct {
/* user context passed as the first argument to the 4 functions */
void *ctx;

/* allocate a memory block */
void* (*malloc) (void *ctx, size_t size);

/* allocate a memory block initialized by zeros */
void* (*calloc) (void *ctx, size_t nelem, size_t elsize);

/* allocate or resize a memory block */
void* (*realloc) (void *ctx, void *ptr, size_t new_size);

/* release a memory block */
void (*free) (void *ctx, void *ptr);
} PyMemAllocatorEx;

/* Get the memory block allocator of the specified domain. */
PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain,
PyMemAllocatorEx *allocator);

/* Set the memory block allocator of the specified domain.

The new allocator must return a distinct non-NULL pointer when requesting
zero bytes.

For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL
is not held when the allocator is called.

If the new allocator is not a hook (don't call the previous allocator), the
PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks
on top on the new allocator. */
PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
PyMemAllocatorEx *allocator);

/* Setup hooks to detect bugs in the following Python memory allocator
functions:

- PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree()
- PyMem_Malloc(), PyMem_Realloc(), PyMem_Free()
- PyObject_Malloc(), PyObject_Realloc() and PyObject_Free()

Newly allocated memory is filled with the byte 0xCB, freed memory is filled
with the byte 0xDB. Additional checks:

- detect API violations, ex: PyObject_Free() called on a buffer allocated
by PyMem_Malloc()
- detect write before the start of the buffer (buffer underflow)
- detect write after the end of the buffer (buffer overflow)

The function does nothing if Python is not compiled is debug mode. */
PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
#endif /* Py_LIMITED_API */

/* bpo-35053: expose _Py_tracemalloc_config for performance:
_Py_NewReference() needs an efficient check to test if tracemalloc is
tracing.
Expand Down Expand Up @@ -231,6 +136,13 @@ PyAPI_DATA(struct _PyTraceMalloc_Config) _Py_tracemalloc_config;
.max_nframe = 1, \
.use_domain = 0}


#ifndef Py_LIMITED_API
# define Py_CPYTHON_PYMEM_H
# include "cpython/pymem.h"
# undef Py_CPYTHON_PYMEM_H
#endif

#ifdef __cplusplus
}
#endif
Expand Down