Skip to content

WIP: convert MetadataDType to a heap type #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

WIP: convert MetadataDType to a heap type #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 92 additions & 45 deletions metadatadtype/metadatadtype/src/dtype.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,12 @@

#include "casts.h"

PyTypeObject *MetadataScalar_Type = NULL;

/*
* `get_value` and `get_unit` are small helpers to deal with the scalar.
*/

static double
get_value(PyObject *scalar)
get_value(PyObject *scalar, const PyTypeObject *MetadataScalar_Type)
{
PyTypeObject *scalar_type = Py_TYPE(scalar);
if (scalar_type != MetadataScalar_Type) {
Expand All @@ -36,7 +34,7 @@ get_value(PyObject *scalar)
}

static PyObject *
get_metadata(PyObject *scalar)
get_metadata(PyObject *scalar, const PyTypeObject *MetadataScalar_Type)
{
if (Py_TYPE(scalar) != MetadataScalar_Type) {
PyErr_SetString(
Expand Down Expand Up @@ -64,11 +62,13 @@ get_metadata(PyObject *scalar)
* Internal helper to create new instances
*/
MetadataDTypeObject *
new_metadatadtype_instance(PyObject *metadata)
new_metadatadtype_instance(metadatadtype_state *state, PyObject *metadata)
{
if (state == NULL) {
return NULL;
}
MetadataDTypeObject *new = (MetadataDTypeObject *)PyArrayDescr_Type.tp_new(
/* TODO: Using NULL for args here works, but seems not clean? */
(PyTypeObject *)&MetadataDType, NULL, NULL);
(PyTypeObject *)state->MetadataDType, NULL, NULL);
if (new == NULL) {
return NULL;
}
Expand Down Expand Up @@ -111,17 +111,18 @@ common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
}

static PyArray_Descr *
metadata_discover_descriptor_from_pyobject(PyArray_DTypeMeta *NPY_UNUSED(cls),
metadata_discover_descriptor_from_pyobject(PyArray_DTypeMeta *cls,
PyObject *obj)
{
if (Py_TYPE(obj) != MetadataScalar_Type) {
metadatadtype_state *state = PyType_GetModuleState((PyTypeObject *)cls);
if (Py_TYPE(obj) != state->MetadataScalar_Type) {
PyErr_SetString(
PyExc_TypeError,
"Can only store MetadataScalar in a MetadataDType array.");
return NULL;
}

PyObject *metadata = get_metadata(obj);
PyObject *metadata = get_metadata(obj, state->MetadataScalar_Type);
if (metadata == NULL) {
return NULL;
}
Expand All @@ -135,7 +136,8 @@ metadata_discover_descriptor_from_pyobject(PyArray_DTypeMeta *NPY_UNUSED(cls),
static int
metadatadtype_setitem(MetadataDTypeObject *descr, PyObject *obj, char *dataptr)
{
double value = get_value(obj);
metadatadtype_state *state = PyType_GetModuleState(Py_TYPE(descr));
double value = get_value(obj, state->MetadataScalar_Type);
if (value == -1 && PyErr_Occurred()) {
return -1;
}
Expand All @@ -148,6 +150,7 @@ metadatadtype_setitem(MetadataDTypeObject *descr, PyObject *obj, char *dataptr)
static PyObject *
metadatadtype_getitem(MetadataDTypeObject *descr, char *dataptr)
{
metadatadtype_state *state = PyType_GetModuleState(Py_TYPE(descr));
double val;
/* get the value */
memcpy(&val, dataptr, sizeof(double)); // NOLINT
Expand All @@ -158,7 +161,7 @@ metadatadtype_getitem(MetadataDTypeObject *descr, char *dataptr)
}

PyObject *res = PyObject_CallFunctionObjArgs(
(PyObject *)MetadataScalar_Type, val_obj, descr, NULL);
(PyObject *)state->MetadataScalar_Type, val_obj, descr, NULL);
if (res == NULL) {
return NULL;
}
Expand Down Expand Up @@ -186,9 +189,10 @@ static PyType_Slot MetadataDType_Slots[] = {
{0, NULL}};

static PyObject *
metadatadtype_new(PyTypeObject *NPY_UNUSED(cls), PyObject *args,
PyObject *kwds)
metadatadtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
metadatadtype_state *state = PyType_GetModuleState(type);

static char *kwargs_strs[] = {"metadata", NULL};

PyObject *metadata = NULL;
Expand All @@ -201,13 +205,41 @@ metadatadtype_new(PyTypeObject *NPY_UNUSED(cls), PyObject *args,
metadata = Py_None;
}

return (PyObject *)new_metadatadtype_instance(metadata);
return (PyObject *)new_metadatadtype_instance(state, metadata);
}

/* MetadataDType finalization */

static int
metadatadtype_traverse(PyObject *self_obj, visitproc visit, void *arg)
{
// Visit the type
Py_VISIT(Py_TYPE(self_obj));

// Visit the metadata attribute
Py_VISIT(((MetadataDTypeObject *)self_obj)->metadata);

return 0;
}

static int
metadatadtype_clear(MetadataDTypeObject *self)
{
Py_CLEAR(self->metadata);
return 0;
}

static void
metadatadtype_finalize(PyObject *self_obj)
{
Py_CLEAR(((MetadataDTypeObject *)self_obj)->metadata);
}

static void
metadatadtype_dealloc(MetadataDTypeObject *self)
{
Py_CLEAR(self->metadata);
PyObject_GC_UnTrack(self);
metadatadtype_finalize((PyObject *)self);
PyArrayDescr_Type.tp_dealloc((PyObject *)self);
}

Expand All @@ -221,32 +253,52 @@ metadatadtype_repr(MetadataDTypeObject *self)
static PyMemberDef MetadataDType_members[] = {
{"_metadata", T_OBJECT_EX, offsetof(MetadataDTypeObject, metadata),
READONLY, "some metadata"},
{NULL},
{NULL, 0, 0, 0, ""},
};

/*
* This is the basic things that you need to create a Python Type/Class in C.
* However, there is a slight difference here because we create a
* PyArray_DTypeMeta, which is a larger struct than a typical type.
* (This should get a bit nicer eventually with Python >3.11.)
*/
PyArray_DTypeMeta MetadataDType = {
{{
PyVarObject_HEAD_INIT(NULL, 0).tp_name =
"metadatadtype.MetadataDType",
.tp_basicsize = sizeof(MetadataDTypeObject),
.tp_new = metadatadtype_new,
.tp_dealloc = (destructor)metadatadtype_dealloc,
.tp_repr = (reprfunc)metadatadtype_repr,
.tp_str = (reprfunc)metadatadtype_repr,
.tp_members = MetadataDType_members,
}},
/* rest, filled in during DTypeMeta initialization */
static PyType_Slot MetadataDType_Type_slots[] = {
{Py_tp_traverse, metadatadtype_traverse},
{Py_tp_clear, metadatadtype_clear},
{Py_tp_finalize, metadatadtype_finalize},
{Py_tp_dealloc, metadatadtype_dealloc},
{Py_tp_str, (reprfunc)metadatadtype_repr},
{Py_tp_repr, (reprfunc)metadatadtype_repr},
{Py_tp_members, MetadataDType_members},
{Py_tp_new, metadatadtype_new},
{0, 0}, /* sentinel */
};

static PyType_Spec MetadataDType_Type_spec = {
.name = "metadatadtype.MetadataDType",
.basicsize = sizeof(MetadataDTypeObject),
.flags = Py_TPFLAGS_DEFAULT,
.slots = MetadataDType_Type_slots,
};

int
init_metadata_dtype(void)
init_metadata_dtype(PyObject *m)
{
metadatadtype_state *state = PyModule_GetState(m);

PyObject *bases = PyTuple_Pack(1, (PyObject *)&PyArrayDescr_Type);

state->MetadataDType = (PyArray_DTypeMeta *)PyType_FromModuleAndSpec(
m, &MetadataDType_Type_spec, bases);
if (state->MetadataDType == NULL) {
return -1;
}

// manually set type so it has the correct metaclass
((PyObject *)state->MetadataDType)->ob_type = &PyArrayDTypeMeta_Type;

// manually null remaining fields, is there a more forward compatible
// way to do this with e.g. memset?
state->MetadataDType->singleton = NULL;
state->MetadataDType->type_num = 0;
state->MetadataDType->scalar_type = NULL;
state->MetadataDType->flags = 0;
state->MetadataDType->dt_slots = 0;

/*
* To create our DType, we have to use a "Spec" that tells NumPy how to
* do it. You first have to create a static type, but see the note there!
Expand All @@ -256,22 +308,17 @@ init_metadata_dtype(void)
PyArrayDTypeMeta_Spec MetadataDType_DTypeSpec = {
.flags = NPY_DT_PARAMETRIC | NPY_DT_NUMERIC,
.casts = casts,
.typeobj = MetadataScalar_Type,
.typeobj = state->MetadataScalar_Type,
.slots = MetadataDType_Slots,
};
/* Loaded dynamically, so may need to be set here: */
((PyObject *)&MetadataDType)->ob_type = &PyArrayDTypeMeta_Type;
((PyTypeObject *)&MetadataDType)->tp_base = &PyArrayDescr_Type;
if (PyType_Ready((PyTypeObject *)&MetadataDType) < 0) {
return -1;
}

if (PyArrayInitDTypeMeta_FromSpec(&MetadataDType,
if (PyArrayInitDTypeMeta_FromSpec(state->MetadataDType,
&MetadataDType_DTypeSpec) < 0) {
return -1;
}

MetadataDType.singleton = PyArray_GetDefaultDescr(&MetadataDType);
state->MetadataDType->singleton =
PyArray_GetDefaultDescr(state->MetadataDType);

free(MetadataDType_DTypeSpec.casts[1]->dtypes);
free(MetadataDType_DTypeSpec.casts[1]);
Expand Down
18 changes: 13 additions & 5 deletions metadatadtype/metadatadtype/src/dtype.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,32 @@
#include "numpy/experimental_dtype_api.h"
#include "numpy/ndarraytypes.h"

// module state
typedef struct {
PyArray_DTypeMeta *MetadataDType;
PyTypeObject *MetadataScalar_Type;
} metadatadtype_state;

// MetadataDType objects

// Instance state
typedef struct {
PyArray_Descr base;
PyObject *metadata;
} MetadataDTypeObject;

extern PyArray_DTypeMeta MetadataDType;
extern PyTypeObject *MetadataScalar_Type;

MetadataDTypeObject *
new_metadatadtype_instance(PyObject *metadata);
new_metadatadtype_instance(metadatadtype_state *state, PyObject *metadata);

int
init_metadata_dtype(void);
init_metadata_dtype(PyObject *m);

PyArray_Descr *
common_instance(MetadataDTypeObject *dtype1,
MetadataDTypeObject *NPY_UNUSED(dtype2));

extern struct PyModuleDef metadatadtype_module;

// from numpy's dtypemeta.h, not publicly available
#define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr))

Expand Down
76 changes: 52 additions & 24 deletions metadatadtype/metadatadtype/src/metadatadtype_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,56 +8,84 @@
#include "dtype.h"
#include "umath.h"

static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
.m_name = "metadatadtype_main",
.m_size = -1,
};

/* Module initialization function */
PyMODINIT_FUNC
PyInit__metadatadtype_main(void)
static int
metadatadtypemodule_modexec(PyObject *m)
{
metadatadtype_state *state = PyModule_GetState(m);

if (_import_array() < 0) {
return NULL;
return -1;
}
if (import_experimental_dtype_api(11) < 0) {
return NULL;
}

PyObject *m = PyModule_Create(&moduledef);
if (m == NULL) {
return NULL;
return -1;
}

PyObject *mod = PyImport_ImportModule("metadatadtype");
if (mod == NULL) {
goto error;
}
MetadataScalar_Type =
state->MetadataScalar_Type =
(PyTypeObject *)PyObject_GetAttrString(mod, "MetadataScalar");
Py_DECREF(mod);

if (MetadataScalar_Type == NULL) {
if (state->MetadataScalar_Type == NULL) {
goto error;
}

if (init_metadata_dtype() < 0) {
if (init_metadata_dtype(m) < 0) {
goto error;
}

if (PyModule_AddObject(m, "MetadataDType", (PyObject *)&MetadataDType) <
0) {
if (PyModule_AddType(m, (PyTypeObject *)state->MetadataDType) < 0) {
goto error;
}

if (init_ufuncs() < 0) {
if (init_ufuncs(state->MetadataDType) < 0) {
goto error;
}

return m;
return 0;

error:
Py_DECREF(m);
return NULL;
return -1;
}

static int
metadatadtypemodule_traverse(PyObject *module, visitproc visit, void *arg)
{
metadatadtype_state *state = PyModule_GetState(module);
Py_VISIT(state->MetadataDType);
Py_VISIT(state->MetadataScalar_Type);
return 0;
}

static int
metadatadtypemodule_clear(PyObject *module)
{
metadatadtype_state *state = PyModule_GetState(module);
Py_CLEAR(state->MetadataDType);
Py_CLEAR(state->MetadataScalar_Type);
return 0;
}

static PyModuleDef_Slot metadatadtypemodule_slots[] = {
{Py_mod_exec, metadatadtypemodule_modexec},
{0, NULL},
};

struct PyModuleDef metadatadtype_module = {
PyModuleDef_HEAD_INIT,
.m_name = "_metadatadtype_main",
.m_size = sizeof(metadatadtype_state),
.m_slots = metadatadtypemodule_slots,
.m_traverse = metadatadtypemodule_traverse,
.m_clear = metadatadtypemodule_clear,
};

/* Module initialization function */
PyMODINIT_FUNC
PyInit__metadatadtype_main(void)
{
return PyModuleDef_Init(&metadatadtype_module);
}
Loading