diff --git a/.gitattributes b/.gitattributes index e88d6ea13e2a5e..15b6ccca459b30 100644 --- a/.gitattributes +++ b/.gitattributes @@ -81,6 +81,7 @@ Include/internal/pycore_uop_ids.h generated Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated +Include/slots_generated.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/idlelib/help.html generated @@ -94,7 +95,6 @@ Lib/test/test_stable_abi_ctypes.py generated Lib/test/test_zoneinfo/data/*.json generated Lib/token.py generated Misc/sbom.spdx.json generated -Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated Parser/token.c generated @@ -104,6 +104,7 @@ Python/executor_cases.c.h generated Python/generated_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated +Python/slots_generated.c generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated diff --git a/Include/Python.h b/Include/Python.h index 78083bbf31db75..1fa9c3731aa53e 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -83,7 +83,8 @@ __pragma(warning(disable: 4201)) #include "object.h" #include "refcount.h" #include "objimpl.h" -#include "typeslots.h" +#include "slots.h" +#include "slots_generated.h" #include "pyhash.h" #include "cpython/pydebug.h" #include "bytearrayobject.h" diff --git a/Include/exports.h b/Include/exports.h index 97a674ec2403a4..3db3cc45e88336 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -103,7 +103,7 @@ #define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* #endif #ifndef PyMODEXPORT_FUNC - #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* + #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PySlot* #endif #endif /* Py_EXPORTS_H */ diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index f60c5510d20075..9ed87a544234c5 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -124,7 +124,7 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); -typedef PyModuleDef_Slot *(*PyModExportFunction)(void); +typedef PySlot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, diff --git a/Include/internal/pycore_slots.h b/Include/internal/pycore_slots.h new file mode 100644 index 00000000000000..1b496ecea4853c --- /dev/null +++ b/Include/internal/pycore_slots.h @@ -0,0 +1,132 @@ +#ifndef _Py_PYCORE_SLOTS_H +#define _Py_PYCORE_SLOTS_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef enum _PySlot_DTYPE { + _PySlot_DTYPE_VOID, + _PySlot_DTYPE_FUNC, + _PySlot_DTYPE_PTR, + _PySlot_DTYPE_SIZE, + _PySlot_DTYPE_INT64, + _PySlot_DTYPE_UINT64, +}_PySlot_DTYPE; + +typedef enum _PySlot_KIND { + _PySlot_KIND_TYPE, + _PySlot_KIND_MOD, + _PySlot_KIND_COMPAT, + _PySlot_KIND_SLOT, +} _PySlot_KIND; + +typedef enum _PySlot_PROBLEM_HANDLING { + _PySlot_PROBLEM_ALLOW = 0, + _PySlot_PROBLEM_DEPRECATED, + _PySlot_PROBLEM_REJECT, +} _PySlot_PROBLEM_HANDLING; + +PyAPI_DATA(char *) _PySlot_names[]; + +#define _PySlot_MAX_NESTING 5 + +/* State for one nesting level of a slots iterator */ +typedef struct _PySlotIterator_state { + union { + // tagged by slot_struct_kind: + const PySlot *slot; // with _PySlot_KIND_SLOT + const PyType_Slot *tp_slot; // with _PySlot_KIND_TYPE + const PyModuleDef_Slot *mod_slot; // with _PySlot_KIND_MOD + const void *any_slot; + }; + _PySlot_KIND slot_struct_kind; + bool ignoring_fallbacks :1; +} _PySlotIterator_state; + +/* State for a slots iterator */ +typedef struct { + _PySlotIterator_state *state; + _PySlotIterator_state states[_PySlot_MAX_NESTING]; + _PySlot_KIND kind; + uint8_t recursion_level; + unsigned int seen[_Py_slot_COUNT / sizeof(unsigned int) + 1]; + bool is_at_end :1; + bool is_first_run :1; + + // Name of the object (type/module) being defined, NULL if unknown. + // Must be set by the callers as soon as it's known. + const char *name; + + /* Output information: */ + + // The slot. Always a copy; may be modified by caller of the iterator. + PySlot current; + +} _PySlotIterator; + +/* Initialize an iterator using a Py_Slot array */ +PyAPI_FUNC(void) +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind); + +/* Initialize an iterator using a legacy slot array */ +PyAPI_FUNC(void) +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind); + +/* Reset a *successfully exhausted* iterator to the beginning. + * The *slots* must be the same as for the previous + * `_PySlotIterator_InitWithKind` call. + * (Subsequent iterations skip some validation.) + */ +PyAPI_FUNC(void) _PySlotIterator_Rewind(_PySlotIterator *it, const void *slots); + +/* Iteration function. + * + * Return false at the end (when successfully exhausted). + * Otherwise (even on error), fill output information in `it` and return true. + * + * On error, set an exception and set `it->current.sl_id` to `Py_slot_invalid`. + */ +PyAPI_FUNC(bool) _PySlotIterator_Next(_PySlotIterator *it); + +/* Return 1 if given slot was "seen" by an earlier _PySlotIterator_Next call. + * (This state is not reset by rewinding.) + */ +PyAPI_FUNC(bool) _PySlotIterator_SawSlot(_PySlotIterator *, int); + +static inline const char * +_PySlot_GetName(uint16_t id) +{ + if (id >= _Py_slot_COUNT) { + return ""; + } + if (id == Py_slot_invalid) { + return "Py_slot_invalid"; + } + return _PySlot_names[id]; +} + +static inline void +_PySlot_err_bad_slot(char *kind, uint16_t id) +{ + if (id < _Py_slot_COUNT) { + PyErr_Format(PyExc_SystemError, "invalid %s slot %d (%s)", + kind, (unsigned int)id, _PySlot_names[id]); + } + else if (id == Py_slot_invalid) { + PyErr_Format(PyExc_SystemError, "invalid slot (Py_slot_invalid, %u)", + (unsigned int)id); + } + else { + PyErr_Format(PyExc_SystemError, "unknown %s slot ID %u", + kind, (unsigned int)id); + } +} + +#include "internal/pycore_slots_generated.h" + +#endif // _Py_PYCORE_SLOTS_H diff --git a/Include/internal/pycore_slots_generated.h b/Include/internal/pycore_slots_generated.h new file mode 100644 index 00000000000000..e0269b0d0fd514 --- /dev/null +++ b/Include/internal/pycore_slots_generated.h @@ -0,0 +1,454 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H +#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H + +static inline uint16_t +_PySlot_resolve_type_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return 88; + case 2: + return 89; + case 3: + return 90; + case 4: + return 91; + case 0: case 5: case 6: case 7: case 8: case 9: case 10: case 11: + case 12: case 13: case 14: case 15: case 16: case 17: case 18: case 19: + case 20: case 21: case 22: case 23: case 24: case 25: case 26: case 27: + case 28: case 29: case 30: case 31: case 32: case 33: case 34: case 35: + case 36: case 37: case 38: case 39: case 40: case 41: case 42: case 43: + case 44: case 45: case 46: case 47: case 48: case 49: case 50: case 51: + case 52: case 53: case 54: case 55: case 56: case 57: case 58: case 59: + case 60: case 61: case 62: case 63: case 64: case 65: case 66: case 67: + case 68: case 69: case 70: case 71: case 72: case 73: case 74: case 75: + case 76: case 77: case 78: case 79: case 80: case 81: case 82: case 83: + case 88: case 89: case 90: case 91: case 92: case 93: case 95: case 96: + case 97: case 98: case 99: case 107: case 108: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline uint16_t +_PySlot_resolve_mod_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return 84; + case 2: + return 85; + case 3: + return 86; + case 4: + return 87; + case 0: case 84: case 85: case 86: case 87: case 92: case 94: case 100: + case 101: case 102: case 103: case 104: case 105: case 106: case 109: + case 110: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline void* +_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id) +{ + switch (slot_id) { + case 5: return (tp->tp_as_mapping) + ? tp->tp_as_mapping->mp_subscript : NULL; + case 6: return (tp->tp_as_number) + ? tp->tp_as_number->nb_absolute : NULL; + case 7: return (tp->tp_as_number) + ? tp->tp_as_number->nb_add : NULL; + case 8: return (tp->tp_as_number) + ? tp->tp_as_number->nb_and : NULL; + case 9: return (tp->tp_as_number) + ? tp->tp_as_number->nb_bool : NULL; + case 10: return (tp->tp_as_number) + ? tp->tp_as_number->nb_divmod : NULL; + case 11: return (tp->tp_as_number) + ? tp->tp_as_number->nb_float : NULL; + case 12: return (tp->tp_as_number) + ? tp->tp_as_number->nb_floor_divide : NULL; + case 13: return (tp->tp_as_number) + ? tp->tp_as_number->nb_index : NULL; + case 14: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_add : NULL; + case 15: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_and : NULL; + case 16: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_floor_divide : NULL; + case 17: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_lshift : NULL; + case 18: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_multiply : NULL; + case 19: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_or : NULL; + case 20: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_power : NULL; + case 21: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_remainder : NULL; + case 22: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_rshift : NULL; + case 23: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_subtract : NULL; + case 24: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_true_divide : NULL; + case 25: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_xor : NULL; + case 26: return (tp->tp_as_number) + ? tp->tp_as_number->nb_int : NULL; + case 27: return (tp->tp_as_number) + ? tp->tp_as_number->nb_invert : NULL; + case 28: return (tp->tp_as_number) + ? tp->tp_as_number->nb_lshift : NULL; + case 29: return (tp->tp_as_number) + ? tp->tp_as_number->nb_multiply : NULL; + case 30: return (tp->tp_as_number) + ? tp->tp_as_number->nb_negative : NULL; + case 31: return (tp->tp_as_number) + ? tp->tp_as_number->nb_or : NULL; + case 32: return (tp->tp_as_number) + ? tp->tp_as_number->nb_positive : NULL; + case 33: return (tp->tp_as_number) + ? tp->tp_as_number->nb_power : NULL; + case 34: return (tp->tp_as_number) + ? tp->tp_as_number->nb_remainder : NULL; + case 35: return (tp->tp_as_number) + ? tp->tp_as_number->nb_rshift : NULL; + case 36: return (tp->tp_as_number) + ? tp->tp_as_number->nb_subtract : NULL; + case 37: return (tp->tp_as_number) + ? tp->tp_as_number->nb_true_divide : NULL; + case 38: return (tp->tp_as_number) + ? tp->tp_as_number->nb_xor : NULL; + case 39: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_ass_item : NULL; + case 40: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_concat : NULL; + case 41: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_contains : NULL; + case 42: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_inplace_concat : NULL; + case 43: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_inplace_repeat : NULL; + case 44: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_item : NULL; + case 45: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_length : NULL; + case 46: return (tp->tp_as_sequence) + ? tp->tp_as_sequence->sq_repeat : NULL; + case 47: return (void*)tp->tp_alloc; + case 48: return (void*)tp->tp_base; + case 49: return (void*)tp->tp_bases; + case 50: return (void*)tp->tp_call; + case 51: return (void*)tp->tp_clear; + case 52: return (void*)tp->tp_dealloc; + case 53: return (void*)tp->tp_del; + case 54: return (void*)tp->tp_descr_get; + case 55: return (void*)tp->tp_descr_set; + case 56: return (void*)tp->tp_doc; + case 57: return (void*)tp->tp_getattr; + case 58: return (void*)tp->tp_getattro; + case 59: return (void*)tp->tp_hash; + case 60: return (void*)tp->tp_init; + case 61: return (void*)tp->tp_is_gc; + case 62: return (void*)tp->tp_iter; + case 63: return (void*)tp->tp_iternext; + case 64: return (void*)tp->tp_methods; + case 65: return (void*)tp->tp_new; + case 66: return (void*)tp->tp_repr; + case 67: return (void*)tp->tp_richcompare; + case 68: return (void*)tp->tp_setattr; + case 69: return (void*)tp->tp_setattro; + case 70: return (void*)tp->tp_str; + case 71: return (void*)tp->tp_traverse; + case 72: return (void*)tp->tp_members; + case 73: return (void*)tp->tp_getset; + case 74: return (void*)tp->tp_free; + case 75: return (tp->tp_as_number) + ? tp->tp_as_number->nb_matrix_multiply : NULL; + case 76: return (tp->tp_as_number) + ? tp->tp_as_number->nb_inplace_matrix_multiply : NULL; + case 77: return (tp->tp_as_async) + ? tp->tp_as_async->am_await : NULL; + case 78: return (tp->tp_as_async) + ? tp->tp_as_async->am_aiter : NULL; + case 79: return (tp->tp_as_async) + ? tp->tp_as_async->am_anext : NULL; + case 80: return (void*)tp->tp_finalize; + case 81: return (tp->tp_as_async) + ? tp->tp_as_async->am_send : NULL; + case 82: return (void*)tp->tp_vectorcall; + case 83: return (tp->tp_flags & Py_TPFLAGS_HEAPTYPE) + ? ((PyHeapTypeObject*)tp)->ht_token : NULL; + case 88: return (tp->tp_as_buffer) + ? tp->tp_as_buffer->bf_getbuffer : NULL; + case 89: return (tp->tp_as_buffer) + ? tp->tp_as_buffer->bf_releasebuffer : NULL; + case 90: return (tp->tp_as_mapping) + ? tp->tp_as_mapping->mp_ass_subscript : NULL; + case 91: return (tp->tp_as_mapping) + ? tp->tp_as_mapping->mp_length : NULL; + } + _PySlot_err_bad_slot("PyType_GetSlot", slot_id); + return NULL; +} + +static inline void +_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht, PySlot slot) +{ + switch (slot.sl_id) { + case 5: ht->as_mapping.mp_subscript = (binaryfunc)slot.sl_func; break; + case 6: ht->as_number.nb_absolute = (unaryfunc)slot.sl_func; break; + case 7: ht->as_number.nb_add = (binaryfunc)slot.sl_func; break; + case 8: ht->as_number.nb_and = (binaryfunc)slot.sl_func; break; + case 9: ht->as_number.nb_bool = (inquiry)slot.sl_func; break; + case 10: ht->as_number.nb_divmod = (binaryfunc)slot.sl_func; break; + case 11: ht->as_number.nb_float = (unaryfunc)slot.sl_func; break; + case 12: ht->as_number.nb_floor_divide = (binaryfunc)slot.sl_func; break; + case 13: ht->as_number.nb_index = (unaryfunc)slot.sl_func; break; + case 14: ht->as_number.nb_inplace_add = (binaryfunc)slot.sl_func; break; + case 15: ht->as_number.nb_inplace_and = (binaryfunc)slot.sl_func; break; + case 16: ht->as_number.nb_inplace_floor_divide = (binaryfunc)slot.sl_func; break; + case 17: ht->as_number.nb_inplace_lshift = (binaryfunc)slot.sl_func; break; + case 18: ht->as_number.nb_inplace_multiply = (binaryfunc)slot.sl_func; break; + case 19: ht->as_number.nb_inplace_or = (binaryfunc)slot.sl_func; break; + case 20: ht->as_number.nb_inplace_power = (ternaryfunc)slot.sl_func; break; + case 21: ht->as_number.nb_inplace_remainder = (binaryfunc)slot.sl_func; break; + case 22: ht->as_number.nb_inplace_rshift = (binaryfunc)slot.sl_func; break; + case 23: ht->as_number.nb_inplace_subtract = (binaryfunc)slot.sl_func; break; + case 24: ht->as_number.nb_inplace_true_divide = (binaryfunc)slot.sl_func; break; + case 25: ht->as_number.nb_inplace_xor = (binaryfunc)slot.sl_func; break; + case 26: ht->as_number.nb_int = (unaryfunc)slot.sl_func; break; + case 27: ht->as_number.nb_invert = (unaryfunc)slot.sl_func; break; + case 28: ht->as_number.nb_lshift = (binaryfunc)slot.sl_func; break; + case 29: ht->as_number.nb_multiply = (binaryfunc)slot.sl_func; break; + case 30: ht->as_number.nb_negative = (unaryfunc)slot.sl_func; break; + case 31: ht->as_number.nb_or = (binaryfunc)slot.sl_func; break; + case 32: ht->as_number.nb_positive = (unaryfunc)slot.sl_func; break; + case 33: ht->as_number.nb_power = (ternaryfunc)slot.sl_func; break; + case 34: ht->as_number.nb_remainder = (binaryfunc)slot.sl_func; break; + case 35: ht->as_number.nb_rshift = (binaryfunc)slot.sl_func; break; + case 36: ht->as_number.nb_subtract = (binaryfunc)slot.sl_func; break; + case 37: ht->as_number.nb_true_divide = (binaryfunc)slot.sl_func; break; + case 38: ht->as_number.nb_xor = (binaryfunc)slot.sl_func; break; + case 39: ht->as_sequence.sq_ass_item = (ssizeobjargproc)slot.sl_func; break; + case 40: ht->as_sequence.sq_concat = (binaryfunc)slot.sl_func; break; + case 41: ht->as_sequence.sq_contains = (objobjproc)slot.sl_func; break; + case 42: ht->as_sequence.sq_inplace_concat = (binaryfunc)slot.sl_func; break; + case 43: ht->as_sequence.sq_inplace_repeat = (ssizeargfunc)slot.sl_func; break; + case 44: ht->as_sequence.sq_item = (ssizeargfunc)slot.sl_func; break; + case 45: ht->as_sequence.sq_length = (lenfunc)slot.sl_func; break; + case 46: ht->as_sequence.sq_repeat = (ssizeargfunc)slot.sl_func; break; + case 47: ht->ht_type.tp_alloc = (allocfunc)slot.sl_func; break; + case 48: ht->ht_type.tp_base = slot.sl_ptr; break; + case 49: ht->ht_type.tp_bases = slot.sl_ptr; break; + case 50: ht->ht_type.tp_call = (ternaryfunc)slot.sl_func; break; + case 51: ht->ht_type.tp_clear = (inquiry)slot.sl_func; break; + case 52: ht->ht_type.tp_dealloc = (destructor)slot.sl_func; break; + case 53: ht->ht_type.tp_del = (destructor)slot.sl_func; break; + case 54: ht->ht_type.tp_descr_get = (descrgetfunc)slot.sl_func; break; + case 55: ht->ht_type.tp_descr_set = (descrsetfunc)slot.sl_func; break; + case 56: ht->ht_type.tp_doc = slot.sl_ptr; break; + case 57: ht->ht_type.tp_getattr = (getattrfunc)slot.sl_func; break; + case 58: ht->ht_type.tp_getattro = (getattrofunc)slot.sl_func; break; + case 59: ht->ht_type.tp_hash = (hashfunc)slot.sl_func; break; + case 60: ht->ht_type.tp_init = (initproc)slot.sl_func; break; + case 61: ht->ht_type.tp_is_gc = (inquiry)slot.sl_func; break; + case 62: ht->ht_type.tp_iter = (getiterfunc)slot.sl_func; break; + case 63: ht->ht_type.tp_iternext = (iternextfunc)slot.sl_func; break; + case 64: ht->ht_type.tp_methods = slot.sl_ptr; break; + case 65: ht->ht_type.tp_new = (newfunc)slot.sl_func; break; + case 66: ht->ht_type.tp_repr = (reprfunc)slot.sl_func; break; + case 67: ht->ht_type.tp_richcompare = (richcmpfunc)slot.sl_func; break; + case 68: ht->ht_type.tp_setattr = (setattrfunc)slot.sl_func; break; + case 69: ht->ht_type.tp_setattro = (setattrofunc)slot.sl_func; break; + case 70: ht->ht_type.tp_str = (reprfunc)slot.sl_func; break; + case 71: ht->ht_type.tp_traverse = (traverseproc)slot.sl_func; break; + case 72: ht->ht_type.tp_members = slot.sl_ptr; break; + case 73: ht->ht_type.tp_getset = slot.sl_ptr; break; + case 74: ht->ht_type.tp_free = (freefunc)slot.sl_func; break; + case 75: ht->as_number.nb_matrix_multiply = (binaryfunc)slot.sl_func; break; + case 76: ht->as_number.nb_inplace_matrix_multiply = (binaryfunc)slot.sl_func; break; + case 77: ht->as_async.am_await = (unaryfunc)slot.sl_func; break; + case 78: ht->as_async.am_aiter = (unaryfunc)slot.sl_func; break; + case 79: ht->as_async.am_anext = (unaryfunc)slot.sl_func; break; + case 80: ht->ht_type.tp_finalize = (destructor)slot.sl_func; break; + case 81: ht->as_async.am_send = (sendfunc)slot.sl_func; break; + case 82: ht->ht_type.tp_vectorcall = (vectorcallfunc)slot.sl_func; break; + case 88: ht->as_buffer.bf_getbuffer = (getbufferproc)slot.sl_func; break; + case 89: ht->as_buffer.bf_releasebuffer = (releasebufferproc)slot.sl_func; break; + case 90: ht->as_mapping.mp_ass_subscript = (objobjargproc)slot.sl_func; break; + case 91: ht->as_mapping.mp_length = (lenfunc)slot.sl_func; break; + } +} + +static inline _PySlot_DTYPE +_PySlot_get_dtype(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: return _PySlot_DTYPE_VOID; + case Py_mp_subscript: return _PySlot_DTYPE_FUNC; + case Py_nb_absolute: return _PySlot_DTYPE_FUNC; + case Py_nb_add: return _PySlot_DTYPE_FUNC; + case Py_nb_and: return _PySlot_DTYPE_FUNC; + case Py_nb_bool: return _PySlot_DTYPE_FUNC; + case Py_nb_divmod: return _PySlot_DTYPE_FUNC; + case Py_nb_float: return _PySlot_DTYPE_FUNC; + case Py_nb_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_index: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_add: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_and: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_or: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_power: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_xor: return _PySlot_DTYPE_FUNC; + case Py_nb_int: return _PySlot_DTYPE_FUNC; + case Py_nb_invert: return _PySlot_DTYPE_FUNC; + case Py_nb_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_negative: return _PySlot_DTYPE_FUNC; + case Py_nb_or: return _PySlot_DTYPE_FUNC; + case Py_nb_positive: return _PySlot_DTYPE_FUNC; + case Py_nb_power: return _PySlot_DTYPE_FUNC; + case Py_nb_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_xor: return _PySlot_DTYPE_FUNC; + case Py_sq_ass_item: return _PySlot_DTYPE_FUNC; + case Py_sq_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_contains: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_repeat: return _PySlot_DTYPE_FUNC; + case Py_sq_item: return _PySlot_DTYPE_FUNC; + case Py_sq_length: return _PySlot_DTYPE_FUNC; + case Py_sq_repeat: return _PySlot_DTYPE_FUNC; + case Py_tp_alloc: return _PySlot_DTYPE_FUNC; + case Py_tp_base: return _PySlot_DTYPE_PTR; + case Py_tp_bases: return _PySlot_DTYPE_PTR; + case Py_tp_call: return _PySlot_DTYPE_FUNC; + case Py_tp_clear: return _PySlot_DTYPE_FUNC; + case Py_tp_dealloc: return _PySlot_DTYPE_FUNC; + case Py_tp_del: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_get: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_set: return _PySlot_DTYPE_FUNC; + case Py_tp_doc: return _PySlot_DTYPE_PTR; + case Py_tp_getattr: return _PySlot_DTYPE_FUNC; + case Py_tp_getattro: return _PySlot_DTYPE_FUNC; + case Py_tp_hash: return _PySlot_DTYPE_FUNC; + case Py_tp_init: return _PySlot_DTYPE_FUNC; + case Py_tp_is_gc: return _PySlot_DTYPE_FUNC; + case Py_tp_iter: return _PySlot_DTYPE_FUNC; + case Py_tp_iternext: return _PySlot_DTYPE_FUNC; + case Py_tp_methods: return _PySlot_DTYPE_PTR; + case Py_tp_new: return _PySlot_DTYPE_FUNC; + case Py_tp_repr: return _PySlot_DTYPE_FUNC; + case Py_tp_richcompare: return _PySlot_DTYPE_FUNC; + case Py_tp_setattr: return _PySlot_DTYPE_FUNC; + case Py_tp_setattro: return _PySlot_DTYPE_FUNC; + case Py_tp_str: return _PySlot_DTYPE_FUNC; + case Py_tp_traverse: return _PySlot_DTYPE_FUNC; + case Py_tp_members: return _PySlot_DTYPE_PTR; + case Py_tp_getset: return _PySlot_DTYPE_PTR; + case Py_tp_free: return _PySlot_DTYPE_FUNC; + case Py_nb_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_am_await: return _PySlot_DTYPE_FUNC; + case Py_am_aiter: return _PySlot_DTYPE_FUNC; + case Py_am_anext: return _PySlot_DTYPE_FUNC; + case Py_tp_finalize: return _PySlot_DTYPE_FUNC; + case Py_am_send: return _PySlot_DTYPE_FUNC; + case Py_tp_vectorcall: return _PySlot_DTYPE_FUNC; + case Py_tp_token: return _PySlot_DTYPE_PTR; + case Py_mod_create: return _PySlot_DTYPE_FUNC; + case Py_mod_exec: return _PySlot_DTYPE_FUNC; + case Py_mod_multiple_interpreters: return _PySlot_DTYPE_UINT64; + case Py_mod_gil: return _PySlot_DTYPE_UINT64; + case Py_bf_getbuffer: return _PySlot_DTYPE_FUNC; + case Py_bf_releasebuffer: return _PySlot_DTYPE_FUNC; + case Py_mp_ass_subscript: return _PySlot_DTYPE_FUNC; + case Py_mp_length: return _PySlot_DTYPE_FUNC; + case Py_slot_subslots: return _PySlot_DTYPE_PTR; + case Py_tp_slots: return _PySlot_DTYPE_PTR; + case Py_mod_slots: return _PySlot_DTYPE_PTR; + case Py_tp_name: return _PySlot_DTYPE_PTR; + case Py_tp_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_extra_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_itemsize: return _PySlot_DTYPE_SIZE; + case Py_tp_flags: return _PySlot_DTYPE_UINT64; + case Py_mod_name: return _PySlot_DTYPE_PTR; + case Py_mod_doc: return _PySlot_DTYPE_PTR; + case Py_mod_state_size: return _PySlot_DTYPE_SIZE; + case Py_mod_methods: return _PySlot_DTYPE_PTR; + case Py_mod_state_traverse: return _PySlot_DTYPE_FUNC; + case Py_mod_state_clear: return _PySlot_DTYPE_FUNC; + case Py_mod_state_free: return _PySlot_DTYPE_FUNC; + case Py_tp_metaclass: return _PySlot_DTYPE_PTR; + case Py_tp_module: return _PySlot_DTYPE_PTR; + case Py_mod_abi: return _PySlot_DTYPE_PTR; + case Py_mod_token: return _PySlot_DTYPE_PTR; + default: return _PySlot_DTYPE_VOID; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_duplicate_handling(uint16_t slot_id) +{ + switch (slot_id) { + case 5: case 6: case 7: case 8: case 9: case 10: case 11: case 12: + case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20: + case 21: case 22: case 23: case 24: case 25: case 26: case 27: case 28: + case 29: case 30: case 31: case 32: case 33: case 34: case 35: case 36: + case 37: case 38: case 39: case 40: case 41: case 42: case 43: case 44: + case 45: case 46: case 47: case 48: case 49: case 50: case 51: case 52: + case 53: case 54: case 55: case 57: case 58: case 59: case 60: case 61: + case 62: case 63: case 64: case 65: case 66: case 67: case 68: case 69: + case 70: case 71: case 73: case 74: case 75: case 76: case 77: case 78: + case 79: case 80: case 81: case 82: case 83: case 88: case 89: case 90: + case 91: + return _PySlot_PROBLEM_DEPRECATED; + case 85: + return _PySlot_PROBLEM_ALLOW; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_null_handling(uint16_t slot_id) +{ + switch (slot_id) { + case 0: case 1: case 2: case 3: case 4: case 56: case 83: case 86: + case 87: case 92: case 93: case 96: case 97: case 98: case 99: + case 102: + return _PySlot_PROBLEM_ALLOW; + case 5: case 6: case 7: case 8: case 9: case 10: case 11: case 12: + case 13: case 14: case 15: case 16: case 17: case 18: case 19: case 20: + case 21: case 22: case 23: case 24: case 25: case 26: case 27: case 28: + case 29: case 30: case 31: case 32: case 33: case 34: case 35: case 36: + case 37: case 38: case 39: case 40: case 41: case 42: case 43: case 44: + case 45: case 46: case 47: case 48: case 49: case 50: case 51: case 52: + case 53: case 54: case 55: case 57: case 58: case 59: case 60: case 61: + case 62: case 63: case 64: case 65: case 66: case 67: case 68: case 69: + case 70: case 71: case 72: case 73: case 74: case 75: case 76: case 77: + case 78: case 79: case 80: case 81: case 82: case 84: case 85: case 88: + case 89: case 90: case 91: + return _PySlot_PROBLEM_DEPRECATED; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */ diff --git a/Include/moduleobject.h b/Include/moduleobject.h index e83bc395aa4618..2766cfd59d9f82 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -73,31 +73,6 @@ struct PyModuleDef_Slot { void *value; }; -#define Py_mod_create 1 -#define Py_mod_exec 2 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 -# define Py_mod_multiple_interpreters 3 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -# define Py_mod_gil 4 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -# define Py_mod_abi 5 -# define Py_mod_name 6 -# define Py_mod_doc 7 -# define Py_mod_state_size 8 -# define Py_mod_methods 9 -# define Py_mod_state_traverse 10 -# define Py_mod_state_clear 11 -# define Py_mod_state_free 12 -# define Py_mod_token 13 -#endif - - -#ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 13 -#endif - #endif /* New in 3.5 */ /* for Py_mod_multiple_interpreters: */ @@ -118,7 +93,7 @@ PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *, +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PySlot *, PyObject *spec); PyAPI_FUNC(int) PyModule_Exec(PyObject *mod); PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *mod, Py_ssize_t *result); diff --git a/Include/object.h b/Include/object.h index ad452be8405671..291769cf301c16 100644 --- a/Include/object.h +++ b/Include/object.h @@ -438,6 +438,9 @@ PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); #define Py_TP_USE_SPEC NULL #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 14) +PyAPI_FUNC(PyObject *) PyType_FromSlots(struct PySlot *); +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); diff --git a/Include/pytypedefs.h b/Include/pytypedefs.h index e78ed56a3b67cd..1d0b4e3e85ef74 100644 --- a/Include/pytypedefs.h +++ b/Include/pytypedefs.h @@ -14,6 +14,7 @@ typedef struct PyModuleDef_Slot PyModuleDef_Slot; typedef struct PyMethodDef PyMethodDef; typedef struct PyGetSetDef PyGetSetDef; typedef struct PyMemberDef PyMemberDef; +typedef struct PySlot PySlot; typedef struct _object PyObject; typedef struct _longobject PyLongObject; diff --git a/Include/slots.h b/Include/slots.h new file mode 100644 index 00000000000000..8ff928f9bd4c03 --- /dev/null +++ b/Include/slots.h @@ -0,0 +1,55 @@ +#ifndef _Py_HAVE_SLOTS_H +#define _Py_HAVE_SLOTS_H + +typedef void (*_Py_funcptr_t)(void); + +struct PySlot { + uint16_t sl_id; + uint16_t sl_flags; + _Py_ANONYMOUS union { + uint32_t sl_reserved; + }; + _Py_ANONYMOUS union { + void *sl_ptr; + _Py_funcptr_t sl_func; + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; +}; + +#define PySlot_OPTIONAL 0x01 +#define PySlot_HAS_FALLBACK 0x02 +#define PySlot_STATIC 0x08 +#define PySlot_INTPTR 0x10 + +#define Py_slot_invalid 0xffff + +#define PySlot_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_INTPTR, .sl_ptr=(void*)(VALUE)} + +#define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=(NAME), .sl_func=(_Py_funcptr_t)(VALUE)} + +#define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=(NAME), .sl_size=(Py_ssize_t)(VALUE)} + +#define PySlot_INT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_int64=(int64_t)(VALUE)} + +#define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_uint64=(uint64_t)(VALUE)} + +#define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + +#define PySlot_END {0} + + +#define PySlot_PTR(NAME, VALUE) \ + {(NAME), PySlot_INTPTR, {0}, {(void*)(VALUE)}} + +#define PySlot_PTR_STATIC(NAME, VALUE) \ + {(NAME), PySlot_INTPTR | PySlot_STATIC, {0}, {(void*)(VALUE)}} + +#endif // _Py_HAVE_SLOTS_H diff --git a/Include/slots_generated.h b/Include/slots_generated.h new file mode 100644 index 00000000000000..42edd3ad4c69ff --- /dev/null +++ b/Include/slots_generated.h @@ -0,0 +1,121 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_SLOTS_GENERATED_H +#define _PY_HAVE_SLOTS_GENERATED_H + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW +#else +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD +#endif + +#define Py_slot_end 0 +#define Py_mp_subscript 5 +#define Py_nb_absolute 6 +#define Py_nb_add 7 +#define Py_nb_and 8 +#define Py_nb_bool 9 +#define Py_nb_divmod 10 +#define Py_nb_float 11 +#define Py_nb_floor_divide 12 +#define Py_nb_index 13 +#define Py_nb_inplace_add 14 +#define Py_nb_inplace_and 15 +#define Py_nb_inplace_floor_divide 16 +#define Py_nb_inplace_lshift 17 +#define Py_nb_inplace_multiply 18 +#define Py_nb_inplace_or 19 +#define Py_nb_inplace_power 20 +#define Py_nb_inplace_remainder 21 +#define Py_nb_inplace_rshift 22 +#define Py_nb_inplace_subtract 23 +#define Py_nb_inplace_true_divide 24 +#define Py_nb_inplace_xor 25 +#define Py_nb_int 26 +#define Py_nb_invert 27 +#define Py_nb_lshift 28 +#define Py_nb_multiply 29 +#define Py_nb_negative 30 +#define Py_nb_or 31 +#define Py_nb_positive 32 +#define Py_nb_power 33 +#define Py_nb_remainder 34 +#define Py_nb_rshift 35 +#define Py_nb_subtract 36 +#define Py_nb_true_divide 37 +#define Py_nb_xor 38 +#define Py_sq_ass_item 39 +#define Py_sq_concat 40 +#define Py_sq_contains 41 +#define Py_sq_inplace_concat 42 +#define Py_sq_inplace_repeat 43 +#define Py_sq_item 44 +#define Py_sq_length 45 +#define Py_sq_repeat 46 +#define Py_tp_alloc 47 +#define Py_tp_base 48 +#define Py_tp_bases 49 +#define Py_tp_call 50 +#define Py_tp_clear 51 +#define Py_tp_dealloc 52 +#define Py_tp_del 53 +#define Py_tp_descr_get 54 +#define Py_tp_descr_set 55 +#define Py_tp_doc 56 +#define Py_tp_getattr 57 +#define Py_tp_getattro 58 +#define Py_tp_hash 59 +#define Py_tp_init 60 +#define Py_tp_is_gc 61 +#define Py_tp_iter 62 +#define Py_tp_iternext 63 +#define Py_tp_methods 64 +#define Py_tp_new 65 +#define Py_tp_repr 66 +#define Py_tp_richcompare 67 +#define Py_tp_setattr 68 +#define Py_tp_setattro 69 +#define Py_tp_str 70 +#define Py_tp_traverse 71 +#define Py_tp_members 72 +#define Py_tp_getset 73 +#define Py_tp_free 74 +#define Py_nb_matrix_multiply 75 +#define Py_nb_inplace_matrix_multiply 76 +#define Py_am_await 77 +#define Py_am_aiter 78 +#define Py_am_anext 79 +#define Py_tp_finalize 80 +#define Py_am_send 81 +#define Py_tp_vectorcall 82 +#define Py_tp_token 83 +#define Py_mod_create _Py_SLOT_COMPAT_VALUE(1, 84) +#define Py_mod_exec _Py_SLOT_COMPAT_VALUE(2, 85) +#define Py_mod_multiple_interpreters _Py_SLOT_COMPAT_VALUE(3, 86) +#define Py_mod_gil _Py_SLOT_COMPAT_VALUE(4, 87) +#define Py_bf_getbuffer _Py_SLOT_COMPAT_VALUE(1, 88) +#define Py_bf_releasebuffer _Py_SLOT_COMPAT_VALUE(2, 89) +#define Py_mp_ass_subscript _Py_SLOT_COMPAT_VALUE(3, 90) +#define Py_mp_length _Py_SLOT_COMPAT_VALUE(4, 91) +#define Py_slot_subslots 92 +#define Py_tp_slots 93 +#define Py_mod_slots 94 +#define Py_tp_name 95 +#define Py_tp_basicsize 96 +#define Py_tp_extra_basicsize 97 +#define Py_tp_itemsize 98 +#define Py_tp_flags 99 +#define Py_mod_name 100 +#define Py_mod_doc 101 +#define Py_mod_state_size 102 +#define Py_mod_methods 103 +#define Py_mod_state_traverse 104 +#define Py_mod_state_clear 105 +#define Py_mod_state_free 106 +#define Py_tp_metaclass 107 +#define Py_tp_module 108 +#define Py_mod_abi 109 +#define Py_mod_token 110 + +#define _Py_slot_COUNT 111 +#endif /* _PY_HAVE_SLOTS_GENERATED_H */ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c9c757857a8a5d..f2e2a044c9e009 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -1043,13 +1043,13 @@ def test_heaptype_relative_members(self): def test_heaptype_relative_members_errors(self): with self.assertRaisesRegex( SystemError, - r"With Py_RELATIVE_OFFSET, basicsize must be negative"): + r"With Py_RELATIVE_OFFSET, basicsize must be extended"): _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) with self.assertRaisesRegex( - SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + SystemError, r"Member offset out of range \(0\.\.extra_basicsize\)"): _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) with self.assertRaisesRegex( - SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + SystemError, r"Member offset must not be negative"): _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) @@ -1066,7 +1066,7 @@ def test_heaptype_relative_special_members_errors(self): with self.subTest(member_name=member_name): with self.assertRaisesRegex( SystemError, - r"With Py_RELATIVE_OFFSET, basicsize must be negative."): + r"With Py_RELATIVE_OFFSET, basicsize must be extended"): _testlimitedcapi.make_heaptype_with_member( basicsize=sys.getsizeof(object()) + 100, add_relative_flag=True, @@ -1077,7 +1077,7 @@ def test_heaptype_relative_special_members_errors(self): ) with self.assertRaisesRegex( SystemError, - r"Member offset out of range \(0\.\.-basicsize\)"): + r"Member offset must not be negative"): _testlimitedcapi.make_heaptype_with_member( basicsize=-8, add_relative_flag=True, @@ -1086,6 +1086,17 @@ def test_heaptype_relative_special_members_errors(self): member_type=_testlimitedcapi.Py_T_PYSSIZET, member_flags=_testlimitedcapi.Py_READONLY, ) + with self.assertRaisesRegex( + SystemError, + r"Member offset out of range \(0\.\.extra_basicsize\)"): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-8, + add_relative_flag=True, + member_name=member_name, + member_offset=1234, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) with self.assertRaisesRegex( SystemError, r"type of %s must be Py_T_PYSSIZET" % member_name): @@ -2857,24 +2868,6 @@ def func(): names = ["func", "outer", "outer", "inner", "inner", "outer", "inner"] self.do_test(func, names) - def test_replaced_interpreter(self): - def inner(): - yield 'abc' - def outer(): - yield from inner() - def func(): - list(outer()) - _testinternalcapi.set_eval_frame_interp() - try: - func() - finally: - _testinternalcapi.set_eval_frame_default() - - stats = _testinternalcapi.get_eval_frame_stats() - - self.assertEqual(stats["resumes"], 5) - self.assertEqual(stats["loads"], 5) - @unittest.skipUnless(support.Py_GIL_DISABLED, 'need Py_GIL_DISABLED') class TestPyThreadId(unittest.TestCase): diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py index 823e2ab6b2ef0d..0ce4029504e7d8 100644 --- a/Lib/test/test_capi/test_module.py +++ b/Lib/test/test_capi/test_module.py @@ -142,11 +142,13 @@ def test_repeated_def_slot(self): with self.assertRaises(SystemError) as cm: _testcapi.module_from_slots_repeat_slot(spec) self.assertIn(name, str(cm.exception)) - self.assertIn("more than one", str(cm.exception)) + self.assertRegex( + str(cm.exception), + rf"^module( [_\w]+)? has multiple {name}( \(\d+\))? slots$") def test_null_def_slot(self): """Slots that replace PyModuleDef fields can't be NULL""" - for name in (*DEF_SLOTS, 'Py_mod_exec'): + for name in {*DEF_SLOTS, 'Py_mod_exec'} - {'Py_mod_state_size'}: with self.subTest(name): spec = FakeSpec() spec._test_slot_id = getattr(_testcapi, name) diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 0f668c1da32d6e..4198578614ea9a 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -96,12 +96,12 @@ _Py_COMP_DIAG_PUSH PyDoc_STRVAR(_testcext_doc, "C test extension."); -static PyModuleDef_Slot _testcext_slots[] = { - {Py_mod_name, STR(MODULE_NAME)}, - {Py_mod_doc, (void*)(char*)_testcext_doc}, - {Py_mod_exec, (void*)_testcext_exec}, - {Py_mod_methods, _testcext_methods}, - {0, NULL} +static PySlot _testcext_slots[] = { + PySlot_DATA(Py_mod_name, STR(MODULE_NAME)), + PySlot_DATA(Py_mod_doc, (void*)(char*)_testcext_doc), + PySlot_FUNC(Py_mod_exec, (void*)_testcext_exec), + PySlot_FUNC(Py_mod_methods, _testcext_methods), + PySlot_END, }; _Py_COMP_DIAG_POP diff --git a/Makefile.pre.in b/Makefile.pre.in index 8531162943ae35..abd70ac7502bff 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -489,6 +489,8 @@ PYTHON_OBJS= \ Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ + Python/slots.o \ + Python/slots_generated.o \ Python/stackrefs.o \ Python/structmember.o \ Python/symtable.o \ @@ -1232,12 +1234,13 @@ PYTHON_HEADERS= \ $(srcdir)/Include/refcount.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ + $(srcdir)/Include/slots.h \ + $(srcdir)/Include/slots_generated.h \ $(srcdir)/Include/structmember.h \ $(srcdir)/Include/structseq.h \ $(srcdir)/Include/sysmodule.h \ $(srcdir)/Include/traceback.h \ $(srcdir)/Include/tupleobject.h \ - $(srcdir)/Include/typeslots.h \ $(srcdir)/Include/unicodeobject.h \ $(srcdir)/Include/warnings.h \ $(srcdir)/Include/weakrefobject.h \ @@ -1418,6 +1421,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_setobject.h \ $(srcdir)/Include/internal/pycore_signal.h \ $(srcdir)/Include/internal/pycore_sliceobject.h \ + $(srcdir)/Include/internal/pycore_slots.h \ + $(srcdir)/Include/internal/pycore_slots_generated.h \ $(srcdir)/Include/internal/pycore_stats.h \ $(srcdir)/Include/internal/pycore_strhex.h \ $(srcdir)/Include/internal/pycore_stackref.h \ @@ -1902,7 +1907,7 @@ regen-unicodedata: # "clinic" is regenerated implicitly via "regen-global-objects". .PHONY: regen-all -regen-all: regen-cases regen-typeslots \ +regen-all: regen-cases regen-slots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -2268,16 +2273,17 @@ Python/import.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) -Objects/typeobject.o: Objects/typeslots.inc - .PHONY: regen-typeslots regen-typeslots: - # Regenerate Objects/typeslots.inc from Include/typeslotsh - # using Objects/typeslots.py - $(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \ - < $(srcdir)/Include/typeslots.h \ - $(srcdir)/Objects/typeslots.inc.new - $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new + echo 'NOTE: "regen-typeslots" was renamed to "regen-slots"' + $(MAKE) regen-slots + +.PHONY: regen-slots +regen-slots: + # Regenerate {Python,Include}/slots_generated.{c,h} + # from Python/slots.csv using Tools/build/generate_slots.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \ + --generate-all $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c index ef657842e77494..4dd87c304fa4f2 100644 --- a/Modules/_testcapi/module.c +++ b/Modules/_testcapi/module.c @@ -10,8 +10,8 @@ static PyObject * module_from_slots_empty(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {0}, + PySlot slots[] = { + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -25,11 +25,11 @@ module_from_slots_null(PyObject *self, PyObject *spec) static PyObject * module_from_slots_name(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_name, "currently ignored..."}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_name, "currently ignored..."), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -37,11 +37,11 @@ module_from_slots_name(PyObject *self, PyObject *spec) static PyObject * module_from_slots_doc(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_doc, "the docstring"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_doc, "the docstring"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -49,11 +49,11 @@ module_from_slots_doc(PyObject *self, PyObject *spec) static PyObject * module_from_slots_size(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_state_size, (void*)123}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_SIZE(Py_mod_state_size, (void*)123), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -76,11 +76,11 @@ static PyMethodDef a_methoddef_array[] = { static PyObject * module_from_slots_methods(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_methods, a_methoddef_array}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_methods, a_methoddef_array), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -94,13 +94,13 @@ static void noop_free(void *self) { } static PyObject * module_from_slots_gc(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_state_traverse, noop_traverse}, - {Py_mod_state_clear, noop_clear}, - {Py_mod_state_free, noop_free}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_FUNC(Py_mod_state_traverse, noop_traverse), + PySlot_FUNC(Py_mod_state_clear, noop_clear), + PySlot_FUNC(Py_mod_state_free, noop_free), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -126,11 +126,11 @@ static const char test_token; static PyObject * module_from_slots_token(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_token, (void*)&test_token}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_token, (void*)&test_token), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -154,11 +154,11 @@ simple_exec(PyObject *module) static PyObject * module_from_slots_exec(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_exec, simple_exec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_FUNC(Py_mod_exec, simple_exec), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -187,11 +187,11 @@ create_attr_from_spec(PyObject *spec, PyModuleDef *def) static PyObject * module_from_slots_create(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_create, create_attr_from_spec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_FUNC(Py_mod_create, create_attr_from_spec), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -218,12 +218,12 @@ module_from_slots_repeat_slot(PyObject *self, PyObject *spec) if (slot_id < 0) { return NULL; } - PyModuleDef_Slot slots[] = { - {slot_id, "anything"}, - {slot_id, "anything else"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_PTR(slot_id, "anything"), + PySlot_PTR(slot_id, "anything_else"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -235,11 +235,11 @@ module_from_slots_null_slot(PyObject *self, PyObject *spec) if (slot_id < 0) { return NULL; } - PyModuleDef_Slot slots[] = { - {slot_id, NULL}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_PTR(slot_id, NULL), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index e286eaae820b2b..50486a31e4dc3d 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -589,7 +589,7 @@ PyInit__testmultiphase_null_slots(void) /**** Problematic modules ****/ static PyModuleDef_Slot slots_bad_large[] = { - {_Py_mod_LAST_SLOT + 1, NULL}, + {Py_slot_invalid, NULL}, {0, NULL}, }; @@ -1022,11 +1022,11 @@ PyInit__test_no_multiple_interpreter_slot(void) PyMODEXPORT_FUNC PyModExport__test_from_modexport(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_name, "_test_from_modexport"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_name, "_test_from_modexport"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_USED), + PySlot_END, }; return slots; } @@ -1034,11 +1034,11 @@ PyModExport__test_from_modexport(void) PyMODEXPORT_FUNC PyModExport__test_from_modexport_gil_used(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_name, "_test_from_modexport_gil_used"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_name, "_test_from_modexport_gil_used"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_USED), + PySlot_END, }; return slots; } @@ -1084,12 +1084,12 @@ modexport_create_string(PyObject *spec, PyModuleDef *def) PyMODEXPORT_FUNC PyModExport__test_from_modexport_create_nonmodule(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_name, "_test_from_modexport_create_nonmodule"}, - {Py_mod_create, modexport_create_string}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_name, "_test_from_modexport_create_nonmodule"), + PySlot_FUNC(Py_mod_create, modexport_create_string), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return slots; } @@ -1097,18 +1097,18 @@ PyModExport__test_from_modexport_create_nonmodule(void) PyMODEXPORT_FUNC PyModExport__test_from_modexport_create_nonmodule_gil_used(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_name, "_test_from_modexport_create_nonmodule"}, - {Py_mod_create, modexport_create_string}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_name, "_test_from_modexport_create_nonmodule"), + PySlot_FUNC(Py_mod_create, modexport_create_string), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_USED), + PySlot_END, }; return slots; } -static PyModuleDef_Slot modexport_empty_slots[] = { - {0}, +static PySlot modexport_empty_slots[] = { + PySlot_END, }; PyMODEXPORT_FUNC @@ -1186,17 +1186,17 @@ PyModExport__test_from_modexport_smoke(void) {"get_modexport_empty_slots", modexport_get_empty_slots, METH_NOARGS}, {0}, }; - static PyModuleDef_Slot slots[] = { - {Py_mod_name, "_test_from_modexport_smoke"}, - {Py_mod_doc, "the expected docstring"}, - {Py_mod_exec, modexport_smoke_exec}, - {Py_mod_state_size, (void*)sizeof(int)}, - {Py_mod_methods, methods}, - {Py_mod_state_free, modexport_smoke_free}, - {Py_mod_token, (void*)&modexport_smoke_test_token}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_name, "_test_from_modexport_smoke"), + PySlot_DATA(Py_mod_doc, "the expected docstring"), + PySlot_FUNC(Py_mod_exec, modexport_smoke_exec), + PySlot_SIZE(Py_mod_state_size, (void*)sizeof(int)), + PySlot_FUNC(Py_mod_methods, methods), + PySlot_FUNC(Py_mod_state_free, modexport_smoke_free), + PySlot_DATA(Py_mod_token, (void*)&modexport_smoke_test_token), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return slots; } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 5a0b16ba57242d..ed275a7f1bf123 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -13,6 +13,7 @@ #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_slots.h" // _PySlotIterator_Init #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -367,22 +368,20 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return (PyObject*)m; } +typedef PyObject *(*createfunc_t)(PyObject *, PyModuleDef*); + static PyObject * -module_from_def_and_spec( - PyModuleDef* def_like, /* not necessarily a valid Python object */ +module_from_slots_and_spec( + const PySlot *slots, PyObject *spec, int module_api_version, PyModuleDef* original_def /* NULL if not defined by a def */) { - PyModuleDef_Slot* cur_slot; - PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; + createfunc_t create = NULL; PyObject *nameobj; PyObject *m = NULL; - int has_multiple_interpreters_slot = 0; - void *multiple_interpreters = (void *)0; - int has_gil_slot = 0; + uint64_t multiple_interpreters = (uint64_t)Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; bool requires_gil = true; - int has_execution_slots = 0; const char *name; int ret; void *token = NULL; @@ -402,123 +401,98 @@ module_from_def_and_spec( goto error; } - if (def_like->m_size < 0) { - PyErr_Format( - PyExc_SystemError, - "module %s: m_size may not be negative for multi-phase initialization", - name); - goto error; + _PySlotIterator it; + + PyModuleDef _dummy_def = {0}; + PyModuleDef* def_like; + if (slots) { + assert(!original_def); + def_like = &_dummy_def; + _PySlotIterator_Init(&it, slots, _PySlot_KIND_MOD); } + else { + assert(original_def); + def_like = original_def; + _PySlotIterator_InitLegacy(&it, def_like->m_slots, _PySlot_KIND_MOD); + } + it.name = name; - for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - // Macro to copy a non-NULL, non-repeatable slot that's unusable with - // PyModuleDef. The destination must be initially NULL. -#define COPY_COMMON_SLOT(SLOT, TYPE, DEST) \ + // Macro to copy a non-NULL, non-repeatable slot that's unusable with + // PyModuleDef. The destination must be initially NULL. + #define COPY_COMMON_SLOT(TYPE, SL_MEMBER, DEST) \ do { \ - if (!(TYPE)(cur_slot->value)) { \ - PyErr_Format( \ - PyExc_SystemError, \ - "module %s: " #SLOT " must not be NULL", \ - name); \ - goto error; \ - } \ if (original_def) { \ PyErr_Format( \ PyExc_SystemError, \ - "module %s: " #SLOT " used with PyModuleDef", \ - name); \ - goto error; \ - } \ - if (DEST) { \ - PyErr_Format( \ - PyExc_SystemError, \ - "module %s has more than one " #SLOT " slot", \ - name); \ + "module %s: %s used with PyModuleDef", \ + name, _PySlot_GetName(it.current.sl_id)); \ goto error; \ } \ - DEST = (TYPE)(cur_slot->value); \ + DEST = (TYPE)(it.current.SL_MEMBER); \ } while (0); \ ///////////////////////////////////////////////////////////////// - switch (cur_slot->slot) { + + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: + goto error; case Py_mod_create: - if (create) { - PyErr_Format( - PyExc_SystemError, - "module %s has multiple create slots", - name); - goto error; - } - create = cur_slot->value; + create = (createfunc_t)it.current.sl_func; break; case Py_mod_exec: - has_execution_slots = 1; if (!original_def) { - COPY_COMMON_SLOT(Py_mod_exec, _Py_modexecfunc, m_exec); + if (m_exec) { + PyErr_Format( + PyExc_SystemError, + "module %s has multiple Py_mod_exec slots", + name); + goto error; + } + if (!it.current.sl_func) { + PyErr_Format( + PyExc_SystemError, + "module %s: Py_mod_exec slot must not be NULL", + name); + goto error; + } + COPY_COMMON_SLOT(_Py_modexecfunc, sl_func, m_exec); } break; case Py_mod_multiple_interpreters: - if (has_multiple_interpreters_slot) { - PyErr_Format( - PyExc_SystemError, - "module %s has more than one 'multiple interpreters' slots", - name); - goto error; - } - multiple_interpreters = cur_slot->value; - has_multiple_interpreters_slot = 1; + multiple_interpreters = it.current.sl_uint64; break; case Py_mod_gil: - if (has_gil_slot) { - PyErr_Format( - PyExc_SystemError, - "module %s has more than one 'gil' slot", - name); - goto error; - } - requires_gil = (cur_slot->value != Py_MOD_GIL_NOT_USED); - has_gil_slot = 1; + requires_gil = (it.current.sl_uint64 != (uint64_t)Py_MOD_GIL_NOT_USED); break; case Py_mod_abi: - if (PyABIInfo_Check((PyABIInfo *)cur_slot->value, name) < 0) { + if (PyABIInfo_Check(it.current.sl_ptr, name) < 0) { goto error; } break; case Py_mod_name: - COPY_COMMON_SLOT(Py_mod_name, char*, def_like->m_name); + COPY_COMMON_SLOT(char*, sl_ptr, def_like->m_name); break; case Py_mod_doc: - COPY_COMMON_SLOT(Py_mod_doc, char*, def_like->m_doc); + COPY_COMMON_SLOT(char*, sl_ptr, def_like->m_doc); break; case Py_mod_state_size: - COPY_COMMON_SLOT(Py_mod_state_size, Py_ssize_t, - def_like->m_size); + COPY_COMMON_SLOT(Py_ssize_t, sl_size, def_like->m_size); break; case Py_mod_methods: - COPY_COMMON_SLOT(Py_mod_methods, PyMethodDef*, - def_like->m_methods); + COPY_COMMON_SLOT(PyMethodDef*, sl_ptr, def_like->m_methods); break; case Py_mod_state_traverse: - COPY_COMMON_SLOT(Py_mod_state_traverse, traverseproc, - def_like->m_traverse); + COPY_COMMON_SLOT(traverseproc, sl_func, def_like->m_traverse); break; case Py_mod_state_clear: - COPY_COMMON_SLOT(Py_mod_state_clear, inquiry, - def_like->m_clear); + COPY_COMMON_SLOT(inquiry, sl_func, def_like->m_clear); break; case Py_mod_state_free: - COPY_COMMON_SLOT(Py_mod_state_free, freefunc, - def_like->m_free); + COPY_COMMON_SLOT(freefunc, sl_func, def_like->m_free); break; case Py_mod_token: - COPY_COMMON_SLOT(Py_mod_token, void*, token); + COPY_COMMON_SLOT(void*, sl_ptr, token); break; - default: - assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); - PyErr_Format( - PyExc_SystemError, - "module %s uses unknown slot ID %i", - name, cur_slot->slot); - goto error; } #undef COPY_COMMON_SLOT } @@ -539,19 +513,24 @@ module_from_def_and_spec( } #endif + if (def_like->m_size < 0) { + PyErr_Format( + PyExc_SystemError, + "module %s: m_size may not be negative for multi-phase initialization", + name); + goto error; + } + /* By default, multi-phase init modules are expected to work under multiple interpreters. */ - if (!has_multiple_interpreters_slot) { - multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; - } - if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { + if (multiple_interpreters == (int64_t)(intptr_t)Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { if (!_Py_IsMainInterpreter(interp) && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) { goto error; } } - else if (multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + else if (multiple_interpreters != (int64_t)(intptr_t)Py_MOD_PER_INTERPRETER_GIL_SUPPORTED && interp->ceval.own_gil && !_Py_IsMainInterpreter(interp) && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) @@ -613,7 +592,7 @@ module_from_def_and_spec( name); goto error; } - if (has_execution_slots) { + if (_PySlotIterator_SawSlot(&it, Py_mod_exec)) { PyErr_Format( PyExc_SystemError, "module %s specifies execution slots, but did not create " @@ -658,11 +637,11 @@ PyObject * PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) { PyModuleDef_Init(def); - return module_from_def_and_spec(def, spec, module_api_version, def); + return module_from_slots_and_spec(NULL, spec, module_api_version, def); } PyObject * -PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec) { if (!slots) { PyErr_SetString( @@ -670,11 +649,9 @@ PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) "PyModule_FromSlotsAndSpec called with NULL slots"); return NULL; } - // Fill in enough of a PyModuleDef to pass to common machinery - PyModuleDef def_like = {.m_slots = (PyModuleDef_Slot *)slots}; - return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION, - NULL); + return module_from_slots_and_spec(slots, spec, PYTHON_API_VERSION, + NULL); } #ifdef Py_GIL_DISABLED @@ -760,8 +737,6 @@ PyModule_Exec(PyObject *module) int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { - PyModuleDef_Slot *cur_slot; - if (alloc_state(module) < 0) { return -1; } @@ -772,13 +747,17 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return 0; } - for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - if (cur_slot->slot == Py_mod_exec) { - int (*func)(PyObject *) = cur_slot->value; - if (run_exec_func(module, func) < 0) { + _PySlotIterator it; + _PySlotIterator_InitLegacy(&it, def->m_slots, _PySlot_KIND_MOD); + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: return -1; - } - continue; + case Py_mod_exec: + _Py_modexecfunc func = (_Py_modexecfunc)it.current.sl_func; + if (run_exec_func(module, func) < 0) { + return -1; + } } } return 0; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 54263fd603ed73..ea326491448347 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -18,6 +18,7 @@ #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_slots.h" // _PySlotIterator_Init #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_typeobject.h" // struct type_cache #include "pycore_unicodeobject.h" // _PyUnicode_Copy @@ -197,11 +198,6 @@ type_lock_allow_release(void) #define PyTypeObject_CAST(op) ((PyTypeObject *)(op)) -typedef struct PySlot_Offset { - short subslot_offset; - short slot_offset; -} PySlot_Offset; - static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer); @@ -5095,21 +5091,6 @@ type_vectorcall(PyObject *metatype, PyObject *const *args, return _PyObject_MakeTpCall(tstate, metatype, args, nargs, kwnames); } -/* An array of type slot offsets corresponding to Py_tp_* constants, - * for use in e.g. PyType_Spec and PyType_GetSlot. - * Each entry has two offsets: "slot_offset" and "subslot_offset". - * If is subslot_offset is -1, slot_offset is an offset within the - * PyTypeObject struct. - * Otherwise slot_offset is an offset to a pointer to a sub-slots struct - * (such as "tp_as_number"), and subslot_offset is the offset within - * that struct. - * The actual table is generated by a script. - */ -static const PySlot_Offset pyslot_offsets[] = { - {0, 0}, -#include "typeslots.inc" -}; - /* Align up to the nearest multiple of alignof(max_align_t) * (like _Py_ALIGN_UP, but for a size rather than pointer) */ @@ -5119,43 +5100,6 @@ _align_up(Py_ssize_t size) return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1); } -/* Given a PyType_FromMetaclass `bases` argument (NULL, type, or tuple of - * types), return a tuple of types. - */ -inline static PyObject * -get_bases_tuple(PyObject *bases_in, PyType_Spec *spec) -{ - if (!bases_in) { - /* Default: look in the spec, fall back to (type,). */ - PyTypeObject *base = &PyBaseObject_Type; // borrowed ref - PyObject *bases = NULL; // borrowed ref - const PyType_Slot *slot; - for (slot = spec->slots; slot->slot; slot++) { - switch (slot->slot) { - case Py_tp_base: - base = slot->pfunc; - break; - case Py_tp_bases: - bases = slot->pfunc; - break; - } - } - if (!bases) { - return PyTuple_Pack(1, base); - } - if (PyTuple_Check(bases)) { - return Py_NewRef(bases); - } - PyErr_SetString(PyExc_SystemError, "Py_tp_bases is not a tuple"); - return NULL; - } - if (PyTuple_Check(bases_in)) { - return Py_NewRef(bases_in); - } - // Not a tuple, should be a single type - return PyTuple_Pack(1, bases_in); -} - static inline int check_basicsize_includes_size_and_offsets(PyTypeObject* type) { @@ -5257,10 +5201,11 @@ special_offset_from_member( return -1; } -PyObject * -PyType_FromMetaclass( - PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) + +static PyObject * +type_from_slots_or_spec( + PySlot *slots, PyType_Spec *spec, + PyTypeObject *metaclass, PyObject *module, PyObject *bases_in) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -5275,47 +5220,127 @@ PyType_FromMetaclass( int r; - /* Prepare slots that need special handling. - * Keep in mind that a slot can be given multiple times: - * if that would cause trouble (leaks, UB, ...), raise an exception. - */ + /* First pass of slots */ - const PyType_Slot *slot; Py_ssize_t nmembers = 0; const PyMemberDef *weaklistoffset_member = NULL; const PyMemberDef *dictoffset_member = NULL; const PyMemberDef *vectorcalloffset_member = NULL; - char *res_start; + Py_ssize_t basicsize = 0; + Py_ssize_t extra_basicsize = 0; + Py_ssize_t itemsize = 0; + int flags = 0; + void *token = NULL; - for (slot = spec->slots; slot->slot; slot++) { - if (slot->slot < 0 - || (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) { - PyErr_SetString(PyExc_RuntimeError, "invalid slot offset"); - goto finally; + bool have_relative_members = false; + Py_ssize_t max_relative_offset = 0; + + PyObject *bases_slot = NULL; /* borrowed from the slots */ + + _PySlotIterator it; + + if (spec) { + assert(!slots); + if (spec->basicsize > 0) { + basicsize = spec->basicsize; } - switch (slot->slot) { - case Py_tp_members: - if (nmembers != 0) { + if (spec->basicsize < 0) { + extra_basicsize = -spec->basicsize; + } + itemsize = spec->itemsize; + flags = spec->flags; + _PySlotIterator_InitLegacy(&it, spec->slots, _PySlot_KIND_TYPE); + it.name = spec->name; + } + else { + assert(!spec); + assert(!metaclass); + assert(!module); + assert(!bases_in); + _PySlotIterator_Init(&it, slots, _PySlot_KIND_TYPE); + } + + #define NO_SPEC \ + if (spec) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "%s must not be used with PyType_Spec", \ + _PySlot_GetName(it.current.sl_id)); \ + goto finally; \ + } \ + ///////////////////////////////////////////////////// + + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: + goto finally; + case Py_tp_name: + NO_SPEC; + it.name = it.current.sl_ptr; + break; + case Py_tp_metaclass: + NO_SPEC; + metaclass = it.current.sl_ptr; + break; + case Py_tp_module: + NO_SPEC; + module = it.current.sl_ptr; + break; + case Py_tp_bases: + bases_slot = it.current.sl_ptr; + break; + case Py_tp_base: + if (!_PySlotIterator_SawSlot(&it, Py_tp_bases)) { + bases_slot = it.current.sl_ptr; + } + break; + case Py_tp_basicsize: + NO_SPEC; + basicsize = it.current.sl_size; + if (basicsize <= 0) { + PyErr_SetString( + PyExc_SystemError, + "Py_tp_basicsize must be positive"); + goto finally; + } + break; + case Py_tp_extra_basicsize: + NO_SPEC; + extra_basicsize = it.current.sl_size; + if (extra_basicsize <= 0) { + PyErr_SetString( + PyExc_SystemError, + "Py_tp_extra_basicsize must be positive"); + goto finally; + } + break; + case Py_tp_itemsize: + NO_SPEC; + itemsize = it.current.sl_size; + if (itemsize <= 0) { PyErr_SetString( PyExc_SystemError, - "Multiple Py_tp_members slots are not supported."); + "Py_tp_itemsize must be positive"); goto finally; } - for (const PyMemberDef *memb = slot->pfunc; memb->name != NULL; memb++) { + break; + case Py_tp_flags: + NO_SPEC; + flags = it.current.sl_uint64; + break; + case Py_tp_members: + for (const PyMemberDef *memb = it.current.sl_ptr; memb->name != NULL; memb++) { nmembers++; if (memb->flags & Py_RELATIVE_OFFSET) { - if (spec->basicsize > 0) { + if (memb->offset < 0) { PyErr_SetString( PyExc_SystemError, - "With Py_RELATIVE_OFFSET, basicsize must be negative."); - goto finally; - } - if (memb->offset < 0 || memb->offset >= -spec->basicsize) { - PyErr_SetString( - PyExc_SystemError, - "Member offset out of range (0..-basicsize)"); + "Member offset must not be negative"); goto finally; } + have_relative_members = true; + max_relative_offset = Py_MAX(max_relative_offset, + memb->offset); } if (strcmp(memb->name, "__weaklistoffset__") == 0) { weaklistoffset_member = memb; @@ -5328,43 +5353,86 @@ PyType_FromMetaclass( } } break; + case Py_tp_token: + token = it.current.sl_ptr; + if (token == Py_TP_USE_SPEC) { + if (!spec) { + PyErr_SetString( + PyExc_SystemError, + "Py_tp_token: Py_TP_USE_SPEC (NULL) can only be " + "used with PyType_Spec"); + goto finally; + } + token = spec; + } + break; case Py_tp_doc: /* For the docstring slot, which usually points to a static string literal, we need to make a copy */ - if (tp_doc != NULL) { - PyErr_SetString( - PyExc_SystemError, - "Multiple Py_tp_doc slots are not supported."); - goto finally; - } - if (slot->pfunc == NULL) { + if (it.current.sl_ptr == NULL) { PyMem_Free(tp_doc); tp_doc = NULL; } else { - size_t len = strlen(slot->pfunc)+1; + size_t len = strlen(it.current.sl_ptr)+1; tp_doc = PyMem_Malloc(len); if (tp_doc == NULL) { PyErr_NoMemory(); goto finally; } - memcpy(tp_doc, slot->pfunc, len); + memcpy(tp_doc, it.current.sl_ptr, len); } break; } } + #undef NO_SPEC - /* Prepare the type name and qualname */ + /* Required slots & bad combinations */ - if (spec->name == NULL) { - PyErr_SetString(PyExc_SystemError, - "Type spec does not define the name field."); + if (it.name == NULL) { + if (spec) { + PyErr_SetString(PyExc_SystemError, + "Type spec does not define the name field."); + } + else { + PyErr_SetString(PyExc_SystemError, + "Py_tp_name slot is required."); + } + goto finally; + } + + if (_PySlotIterator_SawSlot(&it, Py_tp_basicsize) + && _PySlotIterator_SawSlot(&it, Py_tp_extra_basicsize)) + { + PyErr_Format( + PyExc_SystemError, + "type %s: Py_tp_basicsize and Py_tp_extra_basicsize are " + "mutually exclusive", + it.name); goto finally; } - const char *s = strrchr(spec->name, '.'); + if (have_relative_members) { + if (!extra_basicsize) { + PyErr_SetString( + PyExc_SystemError, + "With Py_RELATIVE_OFFSET, basicsize must be extended"); + goto finally; + } + if (max_relative_offset >= extra_basicsize) { + PyErr_SetString( + PyExc_SystemError, + "Member offset out of range (0..extra_basicsize)"); + goto finally; + } + } + + /* Prepare the type name and qualname */ + + assert(it.name); + const char *s = strrchr(it.name, '.'); if (s == NULL) { - s = spec->name; + s = it.name; } else { s++; @@ -5375,7 +5443,7 @@ PyType_FromMetaclass( goto finally; } - /* Copy spec->name to a buffer we own. + /* Copy the name to a buffer we own. * * Unfortunately, we can't use tp_name directly (with some * flag saying that it should be deallocated with the type), @@ -5384,28 +5452,53 @@ PyType_FromMetaclass( * So, we use a separate buffer, _ht_tpname, that's always * deallocated with the type (if it's non-NULL). */ - Py_ssize_t name_buf_len = strlen(spec->name) + 1; + Py_ssize_t name_buf_len = strlen(it.name) + 1; _ht_tpname = PyMem_Malloc(name_buf_len); if (_ht_tpname == NULL) { goto finally; } - memcpy(_ht_tpname, spec->name, name_buf_len); + memcpy(_ht_tpname, it.name, name_buf_len); /* Get a tuple of bases. * bases is a strong reference (unlike bases_in). + * (This is convoluted for backwards compatibility -- preserving priority + * of the various ways to specify bases) */ - bases = get_bases_tuple(bases_in, spec); + if ((bases_in ? 1 : 0) + + (_PySlotIterator_SawSlot(&it, Py_tp_bases) ? 1 : 0) + + (_PySlotIterator_SawSlot(&it, Py_tp_base) ? 1 : 0) + > 1) + { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "type %s specifies multiple of: bases argument, Py_tp_bases, " + "and Py_tp_base. This will become an error in Python 3.20.", + it.name)) + goto finally; + } + if (!bases_in) { + bases_in = bases_slot; + } + if (bases_in) { + if (PyTuple_Check(bases_in)) { + bases = Py_NewRef(bases_in); + } + else { + bases = PyTuple_Pack(1, bases_in); + } + } + else { + bases = PyTuple_Pack(1, &PyBaseObject_Type); + } if (!bases) { goto finally; } - /* If this is an immutable type, check if all bases are also immutable, - * and (for now) fire a deprecation warning if not. + /* If this is an immutable type, check if all bases are also immutable. * (This isn't necessary for static types: those can't have heap bases, * and only heap types can be mutable.) */ - if (spec->flags & Py_TPFLAGS_IMMUTABLETYPE) { - if (check_immutable_bases(spec->name, bases, 0) < 0) { + if (flags & Py_TPFLAGS_IMMUTABLETYPE) { + if (check_immutable_bases(it.name, bases, 0) < 0) { goto finally; } } @@ -5443,20 +5536,16 @@ PyType_FromMetaclass( /* Calculate sizes */ - Py_ssize_t basicsize = spec->basicsize; - Py_ssize_t type_data_offset = spec->basicsize; - if (basicsize == 0) { - /* Inherit */ - basicsize = base->tp_basicsize; - } - else if (basicsize < 0) { + Py_ssize_t type_data_offset = basicsize; + if (extra_basicsize) { /* Extend */ + assert(basicsize == 0); type_data_offset = _align_up(base->tp_basicsize); - basicsize = type_data_offset + _align_up(-spec->basicsize); + basicsize = type_data_offset + _align_up(extra_basicsize); /* Inheriting variable-sized types is limited */ if (base->tp_itemsize - && !((base->tp_flags | spec->flags) & Py_TPFLAGS_ITEMS_AT_END)) + && !((base->tp_flags | flags) & Py_TPFLAGS_ITEMS_AT_END)) { PyErr_SetString( PyExc_SystemError, @@ -5464,8 +5553,10 @@ PyType_FromMetaclass( goto finally; } } - - Py_ssize_t itemsize = spec->itemsize; + if (basicsize == 0) { + /* Inherit */ + basicsize = base->tp_basicsize; + } /* Compute special offsets */ @@ -5497,11 +5588,10 @@ PyType_FromMetaclass( if (res == NULL) { goto finally; } - res_start = (char*)res; type = &res->ht_type; /* The flags must be initialized early, before the GC traverses us */ - type_set_flags(type, spec->flags | Py_TPFLAGS_HEAPTYPE); + type_set_flags(type, flags | Py_TPFLAGS_HEAPTYPE); res->ht_module = Py_XNewRef(module); @@ -5530,15 +5620,20 @@ PyType_FromMetaclass( res->_ht_tpname = _ht_tpname; _ht_tpname = NULL; // Give ownership to the type + res->ht_token = token; + /* Copy the sizes */ type->tp_basicsize = basicsize; type->tp_itemsize = itemsize; - /* Copy all the ordinary slots */ + /* Second pass of slots: copy most of them into the type */ - for (slot = spec->slots; slot->slot; slot++) { - switch (slot->slot) { + _PySlotIterator_Rewind(&it, spec ? (void*)spec->slots : (void*)slots); + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: + goto finally; case Py_tp_base: case Py_tp_bases: case Py_tp_doc: @@ -5548,7 +5643,7 @@ PyType_FromMetaclass( { /* Move the slots to the heap type itself */ size_t len = Py_TYPE(type)->tp_itemsize * nmembers; - memcpy(_PyHeapType_GET_MEMBERS(res), slot->pfunc, len); + memcpy(_PyHeapType_GET_MEMBERS(res), it.current.sl_ptr, len); type->tp_members = _PyHeapType_GET_MEMBERS(res); PyMemberDef *memb; Py_ssize_t i; @@ -5562,26 +5657,8 @@ PyType_FromMetaclass( } } break; - case Py_tp_token: - { - res->ht_token = slot->pfunc == Py_TP_USE_SPEC ? spec : slot->pfunc; - } - break; default: - { - /* Copy other slots directly */ - PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; - short slot_offset = slotoffsets.slot_offset; - if (slotoffsets.subslot_offset == -1) { - /* Set a slot in the main PyTypeObject */ - *(void**)((char*)res_start + slot_offset) = slot->pfunc; - } - else { - void *procs = *(void**)((char*)res_start + slot_offset); - short subslot_offset = slotoffsets.subslot_offset; - *(void**)((char*)procs + subslot_offset) = slot->pfunc; - } - } + _PySlot_heaptype_apply_field_slot(res, it.current); break; } } @@ -5647,10 +5724,10 @@ PyType_FromMetaclass( goto finally; } if (r == 0) { - s = strrchr(spec->name, '.'); + s = strrchr(it.name, '.'); if (s != NULL) { PyObject *modname = PyUnicode_FromStringAndSize( - spec->name, (Py_ssize_t)(s - spec->name)); + it.name, (Py_ssize_t)(s - it.name)); if (modname == NULL) { goto finally; } @@ -5663,7 +5740,7 @@ PyType_FromMetaclass( else { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "builtin type %.200s has no __module__ attribute", - spec->name)) + it.name)) goto finally; } } @@ -5685,22 +5762,36 @@ PyType_FromMetaclass( return (PyObject*)res; } +PyObject * +PyType_FromSlots(PySlot *slots) +{ + return type_from_slots_or_spec(slots, NULL, NULL, NULL, NULL); +} + +PyObject * +PyType_FromMetaclass( + PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases) +{ + return type_from_slots_or_spec(NULL, spec, metaclass, module, bases); +} + PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, module, spec, bases); + return type_from_slots_or_spec(NULL, spec, NULL, module, bases); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, NULL, spec, bases); + return type_from_slots_or_spec(NULL, spec, NULL, NULL, bases); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return PyType_FromMetaclass(NULL, NULL, spec, NULL); + return type_from_slots_or_spec(NULL, spec, NULL, NULL, NULL); } PyObject * @@ -5722,32 +5813,10 @@ PyType_GetModuleName(PyTypeObject *type) } void * -PyType_GetSlot(PyTypeObject *type, int slot) +PyType_GetSlot(PyTypeObject *type, int slot_in) { - void *parent_slot; - int slots_len = Py_ARRAY_LENGTH(pyslot_offsets); - - if (slot <= 0 || slot >= slots_len) { - PyErr_BadInternalCall(); - return NULL; - } - int slot_offset = pyslot_offsets[slot].slot_offset; - - if (slot_offset >= (int)sizeof(PyTypeObject)) { - if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - return NULL; - } - } - - parent_slot = *(void**)((char*)type + slot_offset); - if (parent_slot == NULL) { - return NULL; - } - /* Return slot directly if we have no sub slot. */ - if (pyslot_offsets[slot].subslot_offset == -1) { - return parent_slot; - } - return *(void**)((char*)parent_slot + pyslot_offsets[slot].subslot_offset); + uint16_t slot = _PySlot_resolve_type_slot(slot_in); + return _PySlot_type_getslot(type, slot); } PyObject * diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc deleted file mode 100644 index 642160fe0bd8bc..00000000000000 --- a/Objects/typeslots.inc +++ /dev/null @@ -1,84 +0,0 @@ -/* Generated by typeslots.py */ -{offsetof(PyBufferProcs, bf_getbuffer), offsetof(PyTypeObject, tp_as_buffer)}, -{offsetof(PyBufferProcs, bf_releasebuffer), offsetof(PyTypeObject, tp_as_buffer)}, -{offsetof(PyMappingMethods, mp_ass_subscript), offsetof(PyTypeObject, tp_as_mapping)}, -{offsetof(PyMappingMethods, mp_length), offsetof(PyTypeObject, tp_as_mapping)}, -{offsetof(PyMappingMethods, mp_subscript), offsetof(PyTypeObject, tp_as_mapping)}, -{offsetof(PyNumberMethods, nb_absolute), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_add), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_and), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_bool), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_divmod), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_float), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_floor_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_index), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_add), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_and), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_floor_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_lshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_or), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_power), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_remainder), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_rshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_subtract), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_true_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_xor), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_int), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_invert), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_lshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_negative), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_or), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_positive), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_power), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_remainder), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_rshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_subtract), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_true_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_xor), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PySequenceMethods, sq_ass_item), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_concat), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_contains), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_inplace_concat), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_inplace_repeat), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_item), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_length), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_repeat), offsetof(PyTypeObject, tp_as_sequence)}, -{-1, offsetof(PyTypeObject, tp_alloc)}, -{-1, offsetof(PyTypeObject, tp_base)}, -{-1, offsetof(PyTypeObject, tp_bases)}, -{-1, offsetof(PyTypeObject, tp_call)}, -{-1, offsetof(PyTypeObject, tp_clear)}, -{-1, offsetof(PyTypeObject, tp_dealloc)}, -{-1, offsetof(PyTypeObject, tp_del)}, -{-1, offsetof(PyTypeObject, tp_descr_get)}, -{-1, offsetof(PyTypeObject, tp_descr_set)}, -{-1, offsetof(PyTypeObject, tp_doc)}, -{-1, offsetof(PyTypeObject, tp_getattr)}, -{-1, offsetof(PyTypeObject, tp_getattro)}, -{-1, offsetof(PyTypeObject, tp_hash)}, -{-1, offsetof(PyTypeObject, tp_init)}, -{-1, offsetof(PyTypeObject, tp_is_gc)}, -{-1, offsetof(PyTypeObject, tp_iter)}, -{-1, offsetof(PyTypeObject, tp_iternext)}, -{-1, offsetof(PyTypeObject, tp_methods)}, -{-1, offsetof(PyTypeObject, tp_new)}, -{-1, offsetof(PyTypeObject, tp_repr)}, -{-1, offsetof(PyTypeObject, tp_richcompare)}, -{-1, offsetof(PyTypeObject, tp_setattr)}, -{-1, offsetof(PyTypeObject, tp_setattro)}, -{-1, offsetof(PyTypeObject, tp_str)}, -{-1, offsetof(PyTypeObject, tp_traverse)}, -{-1, offsetof(PyTypeObject, tp_members)}, -{-1, offsetof(PyTypeObject, tp_getset)}, -{-1, offsetof(PyTypeObject, tp_free)}, -{offsetof(PyNumberMethods, nb_matrix_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_matrix_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyAsyncMethods, am_await), offsetof(PyTypeObject, tp_as_async)}, -{offsetof(PyAsyncMethods, am_aiter), offsetof(PyTypeObject, tp_as_async)}, -{offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, -{-1, offsetof(PyTypeObject, tp_finalize)}, -{offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, -{-1, offsetof(PyTypeObject, tp_vectorcall)}, -{-1, offsetof(PyHeapTypeObject, ht_token)}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py deleted file mode 100755 index c7f8a33bb1e74e..00000000000000 --- a/Objects/typeslots.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python -# Usage: typeslots.py < Include/typeslots.h typeslots.inc - -import sys, re - - -def generate_typeslots(out=sys.stdout): - out.write("/* Generated by typeslots.py */\n") - res = {} - for line in sys.stdin: - m = re.match("#define Py_([a-z_]+) ([0-9]+)", line) - if not m: - continue - - member = m.group(1) - if member == "tp_token": - # The heap type structure (ht_*) is an implementation detail; - # the public slot for it has a familiar `tp_` prefix - member = '{-1, offsetof(PyHeapTypeObject, ht_token)}' - elif member.startswith("tp_"): - member = f'{{-1, offsetof(PyTypeObject, {member})}}' - elif member.startswith("am_"): - member = (f'{{offsetof(PyAsyncMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_async)}') - elif member.startswith("nb_"): - member = (f'{{offsetof(PyNumberMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_number)}') - elif member.startswith("mp_"): - member = (f'{{offsetof(PyMappingMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_mapping)}') - elif member.startswith("sq_"): - member = (f'{{offsetof(PySequenceMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_sequence)}') - elif member.startswith("bf_"): - member = (f'{{offsetof(PyBufferProcs, {member}),'+ - ' offsetof(PyTypeObject, tp_as_buffer)}') - res[int(m.group(2))] = member - - M = max(res.keys())+1 - for i in range(1,M): - if i in res: - out.write("%s,\n" % res[i]) - else: - out.write("{0, 0},\n") - - -def main(): - if len(sys.argv) == 2: - with open(sys.argv[1], "w") as f: - generate_typeslots(f) - else: - generate_typeslots() - -if __name__ == "__main__": - main() diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 5e7e739dc5f787..e28502ea8f88e0 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -311,6 +311,8 @@ + + @@ -381,6 +383,8 @@ + + @@ -674,6 +678,8 @@ + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 247f4b5a784f9c..056acbd29816f0 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -225,6 +225,12 @@ Include + + Include + + + Include + Include @@ -837,6 +843,12 @@ Include\internal + + Include\internal + + + Include\internal + Include\internal @@ -1559,6 +1571,12 @@ Python + + Objects + + + Objects + Python diff --git a/Python/import.c b/Python/import.c index 466c5868ab7ee8..7018a020411b98 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1998,7 +1998,7 @@ import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, /* This is like import_run_extension, but avoids interpreter switching * and code for for single-phase modules. */ - PyModuleDef_Slot *slots = ex0(); + PySlot *slots = ex0(); if (!slots) { if (!PyErr_Occurred()) { PyErr_Format( diff --git a/Python/slots.c b/Python/slots.c new file mode 100644 index 00000000000000..10e4d7c2dae8e4 --- /dev/null +++ b/Python/slots.c @@ -0,0 +1,400 @@ +/* Common handling of type/module slots + */ + +#include "Python.h" + +#include "pycore_slots.h" + +#include + +// Iterating through a recursive structure doesn't look great in a debugger. +// Define this to get a trace on stderr. +// (The messages can also serve as code comments.) +#if 0 +#define MSG(...) { \ + fprintf(stderr, "slotiter: " __VA_ARGS__); fprintf(stderr, "\n");} +#else +#define MSG(...) +#endif + +static char* +kind_name(_PySlot_KIND kind) +{ + switch (kind) { + case _PySlot_KIND_TYPE: return "type"; + case _PySlot_KIND_MOD: return "module"; + case _PySlot_KIND_COMPAT: return "compat"; + case _PySlot_KIND_SLOT: return "generic slot"; + } + Py_UNREACHABLE(); + return ""; +} + +static void +init_with_kind(_PySlotIterator *it, const void *slots, + _PySlot_KIND result_kind, + _PySlot_KIND slot_struct_kind) +{ + MSG(""); + MSG("init (%s slot iterator)", kind_name(result_kind)); + it->state = it->states; + it->state->any_slot = slots; + it->state->slot_struct_kind = slot_struct_kind; + it->state->ignoring_fallbacks = false; + it->kind = result_kind; + it->name = NULL; + it->recursion_level = 0; + it->is_at_end = false; + it->is_first_run = true; + it->current.sl_id = 0; + memset(it->seen, 0, sizeof(it->seen)); +} + +void +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind) +{ + init_with_kind(it, slots, result_kind, _PySlot_KIND_SLOT); +} + +void +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind) +{ + init_with_kind(it, slots, kind, kind); +} + +void +_PySlotIterator_Rewind(_PySlotIterator *it, const void *slots) +{ + MSG(""); + MSG("rewind (%s slot iterator)", kind_name(it->kind)); + assert (it->is_at_end); + assert (it->recursion_level == 0); + assert (it->state == it->states); + assert (!it->state->ignoring_fallbacks); + it->is_at_end = false; + it->state->any_slot = slots; + it->is_first_run = false; +} + +static Py_ssize_t +seen_index(uint16_t id) +{ + return id / sizeof(unsigned int); +} + +static unsigned int +seen_mask(uint16_t id) +{ + return ((unsigned int)1) << (id % sizeof(unsigned int)); +} + +bool +_PySlotIterator_SawSlot(_PySlotIterator *it, int id) +{ + assert (id > 0); + assert (id < _Py_slot_COUNT); + return it->seen[seen_index(id)] & seen_mask(id); +} + +// Advance `it` to the next entry. Currently cannot fail. +static void +advance(_PySlotIterator *it) +{ + MSG("advance (at level %d)", (int)it->recursion_level); + switch (it->state->slot_struct_kind) { + case _PySlot_KIND_SLOT: it->state->slot++; break; + case _PySlot_KIND_TYPE: it->state->tp_slot++; break; + case _PySlot_KIND_MOD: it->state->mod_slot++; break; + default: + Py_UNREACHABLE(); + } +} + +static int handle_first_run(_PySlotIterator *it); + +bool +_PySlotIterator_Next(_PySlotIterator *it) +{ + MSG("next"); + assert(it); + assert(!it->is_at_end); + assert(!PyErr_Occurred()); + + it->current.sl_id = -1; + + while (true) { + if (it->state->slot == NULL) { + if (it->recursion_level == 0) { + MSG("end (initial nesting level done)"); + it->is_at_end = true; + return 0; + } + MSG("pop nesting level %d", (int)it->recursion_level); + it->recursion_level--; + it->state = &it->states[it->recursion_level]; + advance(it); + continue; + } + + switch (it->state->slot_struct_kind) { + case _PySlot_KIND_SLOT: { + MSG("copying PySlot structure"); + it->current = *it->state->slot; /* struct copy */ + } break; + case _PySlot_KIND_TYPE: { + MSG("converting PyType_Slot structure"); + memset(&it->current, 0, sizeof(it->current)); + it->current.sl_id = (uint16_t)it->state->tp_slot->slot; + it->current.sl_flags = PySlot_INTPTR; + it->current.sl_ptr = (void*)it->state->tp_slot->pfunc; + } break; + case _PySlot_KIND_MOD: { + MSG("converting PyModuleDef_Slot structure"); + memset(&it->current, 0, sizeof(it->current)); + it->current.sl_id = (uint16_t)it->state->mod_slot->slot; + it->current.sl_flags = PySlot_INTPTR; + it->current.sl_ptr = (void*)it->state->mod_slot->value; + } break; + default: { + Py_UNREACHABLE(); + } break; + } + + /* shorter local names */ + PySlot *const result = &it->current; + uint16_t flags = result->sl_flags; + + if (it->state->ignoring_fallbacks) { + if (!(it->state->slot->sl_flags & PySlot_HAS_FALLBACK)) { + MSG("stopping to ignore fallbacks"); + it->state->ignoring_fallbacks = false; + } + MSG("skipped (ignoring fallbacks)"); + advance(it); + continue; + } + + MSG("slot %d, flags 0x%x, from %p", + (int)result->sl_id, (unsigned)flags, it->state->slot); + + uint16_t orig_id = result->sl_id; + switch (it->kind) { + case _PySlot_KIND_TYPE: + result->sl_id = _PySlot_resolve_type_slot(result->sl_id); + break; + case _PySlot_KIND_MOD: + result->sl_id = _PySlot_resolve_mod_slot(result->sl_id); + break; + default: + Py_UNREACHABLE(); + } + MSG("resolved to slot %s (%d)", + (int)result->sl_id, _PySlot_GetName(result->sl_id)); + + if (result->sl_id == Py_slot_invalid) { + if (flags & (PySlot_OPTIONAL | PySlot_HAS_FALLBACK)) { + MSG("skipped (unknown/invalid slot)"); + advance(it); + continue; + } + MSG("error (unknown/invalid slot)"); + _PySlot_err_bad_slot(kind_name(it->kind), orig_id); + goto error; + } + if (result->sl_id == Py_slot_end) { + MSG("sentinel slot, flags %x", (unsigned)flags); + if (flags & PySlot_OPTIONAL) { + MSG("skipped (optional sentinel)"); + advance(it); + continue; + } + const uint16_t bad_flags = PySlot_OPTIONAL | PySlot_HAS_FALLBACK; + if (flags & bad_flags) { + MSG("error (bad flags on sentinel)"); + PyErr_Format(PyExc_SystemError, + "invalid flags for Py_slot_end: 0x%x", + (unsigned int)flags); + goto error; + } + it->state->slot = NULL; + continue; + } + + if (result->sl_id == Py_slot_subslots + || result->sl_id == Py_tp_slots + || result->sl_id == Py_mod_slots + ) { + if (result->sl_ptr == NULL) { + MSG("NULL subslots; skipping"); + advance(it); + continue; + } + it->recursion_level++; + MSG("recursing into level %d", it->recursion_level); + if (it->recursion_level >= _PySlot_MAX_NESTING) { + MSG("error (too much nesting)"); + PyErr_Format(PyExc_SystemError, + "%s (slot %d): too many levels of nested slots", + _PySlot_GetName(result->sl_id), orig_id); + goto error; + } + it->state = &it->states[it->recursion_level]; + memset(it->state, 0, sizeof(_PySlotIterator_state)); + it->state->slot = result->sl_ptr; + switch (result->sl_id) { + case Py_slot_subslots: + it->state->slot_struct_kind = _PySlot_KIND_SLOT; break; + case Py_tp_slots: + it->state->slot_struct_kind = _PySlot_KIND_TYPE; break; + case Py_mod_slots: + it->state->slot_struct_kind = _PySlot_KIND_MOD; break; + } + continue; + } + + if (flags & PySlot_INTPTR) { + MSG("casting from intptr"); + /* this should compile to nothing on common architectures */ + switch (_PySlot_get_dtype(result->sl_id)) { + case _PySlot_DTYPE_SIZE: { + result->sl_size = (Py_ssize_t)(intptr_t)result->sl_ptr; + } break; + case _PySlot_DTYPE_INT64: { + result->sl_int64 = (int64_t)(intptr_t)result->sl_ptr; + } break; + case _PySlot_DTYPE_UINT64: { + result->sl_uint64 = (uint64_t)(intptr_t)result->sl_ptr; + } break; + case _PySlot_DTYPE_PTR: + case _PySlot_DTYPE_FUNC: + case _PySlot_DTYPE_VOID: + break; + } + } + + if (flags & PySlot_HAS_FALLBACK) { + MSG("starting to ignore fallbacks"); + it->state->ignoring_fallbacks = true; + } + + advance(it); + switch (_PySlot_get_dtype(result->sl_id)) { + case _PySlot_DTYPE_VOID: + case _PySlot_DTYPE_PTR: + MSG("result: %d (%s): %p", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (void*)result->sl_ptr); + break; + case _PySlot_DTYPE_FUNC: + MSG("result: %d (%s): %p", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (void*)result->sl_func); + break; + case _PySlot_DTYPE_SIZE: + MSG("result: %d (%s): %zd", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (Py_ssize_t)result->sl_size); + break; + case _PySlot_DTYPE_INT64: + MSG("result: %d (%s): %ld", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (long)result->sl_int64); + break; + case _PySlot_DTYPE_UINT64: + MSG("result: %d (%s): %lu (0x%lx)", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (unsigned long)result->sl_int64, + (unsigned long)result->sl_int64); + break; + } + assert (result->sl_id > 0); + assert (result->sl_id <= _Py_slot_COUNT); + assert (result->sl_id <= INT_MAX); + if (it->is_first_run && handle_first_run(it) < 0) { + goto error; + } + return result->sl_id != Py_slot_end; + } + Py_UNREACHABLE(); + +error: + it->current.sl_id = Py_slot_invalid; + return true; +} + +/* Validate current slot, and do bookkeeping */ +static int +handle_first_run(_PySlotIterator *it) +{ + int id = it->current.sl_id; + + _PySlot_PROBLEM_HANDLING null_handling = _PySlot_get_null_handling(id); + if (null_handling != _PySlot_PROBLEM_ALLOW) { + bool is_null = false; + switch (_PySlot_get_dtype(id)) { + case _PySlot_DTYPE_PTR: { + is_null = it->current.sl_ptr == NULL; + } break; + case _PySlot_DTYPE_FUNC: { + is_null = it->current.sl_func == NULL; + } break; + default: { + //Py_UNREACHABLE(); + } break; + } + if (is_null) { + MSG("slot is NULL but shouldn't"); + if (null_handling == _PySlot_PROBLEM_REJECT) { + MSG("error (NULL rejected)"); + PyErr_Format(PyExc_SystemError, + "NULL not allowed for slot %s", + _PySlot_GetName(id)); + return -1; + } + MSG("deprecated NULL"); + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, + 1, + "NULL value in slot %s is deprecated", + _PySlot_GetName(id)) < 0) + { + return -1; + } + } + } + + _PySlot_PROBLEM_HANDLING duplicate_handling = _PySlot_get_duplicate_handling(id); + if (duplicate_handling != _PySlot_PROBLEM_ALLOW) { + if (_PySlotIterator_SawSlot(it, id)) { + MSG("slot was seen before but shouldn't be duplicated"); + if (duplicate_handling == _PySlot_PROBLEM_REJECT) { + MSG("error (duplicate rejected)"); + PyErr_Format( + PyExc_SystemError, + "%s%s%s has multiple %s (%d) slots", + kind_name(it->kind), + it->name ? " " : "", + it->name ? it->name : "", + _PySlot_GetName(id), + (int)it->current.sl_id); + return -1; + } + MSG("deprecated duplicate"); + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, + 0, + "%s%s%s has multiple %s (%d) slots. This is deprecated.", + kind_name(it->kind), + it->name ? " " : "", + it->name ? it->name : "", + _PySlot_GetName(id), + (int)it->current.sl_id) < 0) { + return -1; + } + } + } + it->seen[seen_index(id)] |= seen_mask(id); + return 0; +} diff --git a/Python/slots.csv b/Python/slots.csv new file mode 100644 index 00000000000000..8d253e998dd811 --- /dev/null +++ b/Python/slots.csv @@ -0,0 +1,159 @@ +id, name, kind, dtype + +#, The syntax of this file is what currently works best for editing; +#, it can change at any time. Run `typeslots.py --jsonl` to get the info +#, in a more stable format. + +#, This file is a superset of CSV; readable with +#, csv.DictReader but also hopefully human-readable and -editable. +#, Perhaps CSV shouldn't be used this way; sue me. +#, +#, Comment lines start with '#' and a comma; that is they have '#' in the first +#, field. Completely blank lines should be ignored too. +#, Avoid commas in comments; you can use semicolons. +#, +#, Each non-comment line has these mandatory columns: +#, - id: the slot's unique number (or '#' for comments) +#, - name: the slot's name without "Py_" +#, - dtype: basic data type +#, - 'func'; 'ptr'; 'size'; 'uint64'; 'int64': the field of the union +#, - 'void': value is ignored +#, - kind: +#, - 'type' is for PyTypeObject slots +#, - 'mod' is for module slots +#, - 'slot' works for any kind; handled by the slot system itself +#, - 'compat' reserved for backwards compatibility +#, +#, After these there can be 0 or more "named" columns whose contents start with +#, with '='; everything after that is the value. No spaces around '='. +#, +#, For all types: +#, - 'compat': slot number before 3.14 +#, - 'subslots=true': contains an array of slots to be merged in +#, - 'duplicates': behavior if multiple slots of this type are given +#, - 'yes': duplicates allowed +#, - 'no': duplicates rejected +#, - default: warn +#, For pointers: +#, - 'null': Behavior for NULL values: +#, - 'reject': NULLs not allowed +#, - 'allow': NULLs allowed +#, - default is that NULL is deprecated +#, - 'is_name': this is the name slot (used automatically for error messages) +#, For type slots: +#, - 'table': Slot table like 'nb' or 'tp'. If not specified: use the first +#, part of the name (except virtual slots) +#, - 'field': Field name. If not specified: use the name +#, - 'virtual=true': Does not directly correspond to a PyTypeObject (sub)field + +0, "slot_end", slot, void +1, "bf_getbuffer/mod_create", compat, void +2, "bf_releasebuffer/mod_exec", compat, void +3, "mp_ass_subscript/mod_multiple_interpreters", compat, void +4, "mp_length/mod_gil", compat, void +5, "mp_subscript", type, func +6, "nb_absolute", type, func +7, "nb_add", type, func +8, "nb_and", type, func +9, "nb_bool", type, func +10, "nb_divmod", type, func +11, "nb_float", type, func +12, "nb_floor_divide", type, func +13, "nb_index", type, func +14, "nb_inplace_add", type, func +15, "nb_inplace_and", type, func +16, "nb_inplace_floor_divide", type, func +17, "nb_inplace_lshift", type, func +18, "nb_inplace_multiply", type, func +19, "nb_inplace_or", type, func +20, "nb_inplace_power", type, func +21, "nb_inplace_remainder", type, func +22, "nb_inplace_rshift", type, func +23, "nb_inplace_subtract", type, func +24, "nb_inplace_true_divide", type, func +25, "nb_inplace_xor", type, func +26, "nb_int", type, func +27, "nb_invert", type, func +28, "nb_lshift", type, func +29, "nb_multiply", type, func +30, "nb_negative", type, func +31, "nb_or", type, func +32, "nb_positive", type, func +33, "nb_power", type, func +34, "nb_remainder", type, func +35, "nb_rshift", type, func +36, "nb_subtract", type, func +37, "nb_true_divide", type, func +38, "nb_xor", type, func +39, "sq_ass_item", type, func +40, "sq_concat", type, func +41, "sq_contains", type, func +42, "sq_inplace_concat", type, func +43, "sq_inplace_repeat", type, func +44, "sq_item", type, func +45, "sq_length", type, func +46, "sq_repeat", type, func +47, "tp_alloc", type, func +48, "tp_base", type, ptr +49, "tp_bases", type, ptr +50, "tp_call", type, func +51, "tp_clear", type, func +52, "tp_dealloc", type, func +53, "tp_del", type, func +54, "tp_descr_get", type, func +55, "tp_descr_set", type, func +56, "tp_doc", type, ptr, null=allow, duplicates=no +57, "tp_getattr", type, func +58, "tp_getattro", type, func +59, "tp_hash", type, func +60, "tp_init", type, func +61, "tp_is_gc", type, func +62, "tp_iter", type, func +63, "tp_iternext", type, func +64, "tp_methods", type, ptr, duplicates=no +65, "tp_new", type, func +66, "tp_repr", type, func +67, "tp_richcompare", type, func +68, "tp_setattr", type, func +69, "tp_setattro", type, func +70, "tp_str", type, func +71, "tp_traverse", type, func +72, "tp_members", type, ptr, duplicates=no +73, "tp_getset", type, ptr, duplicates=no +74, "tp_free", type, func +75, "nb_matrix_multiply", type, func +76, "nb_inplace_matrix_multiply", type, func +77, "am_await", type, func +78, "am_aiter", type, func +79, "am_anext", type, func +80, "tp_finalize", type, func +81, "am_send", type, func +82, "tp_vectorcall", type, func +83, "tp_token", type, ptr, table=ht, field=ht_token, null=allow, duplicates=no +84, "mod_create", mod, func, compat=1, duplicates=no +85, "mod_exec", mod, func, compat=2, duplicates=yes +86, "mod_multiple_interpreters", mod, uint64, compat=3, duplicates=no +87, "mod_gil", mod, uint64, compat=4, duplicates=no +88, "bf_getbuffer", type, func, compat=1 +89, "bf_releasebuffer", type, func, compat=2 +90, "mp_ass_subscript", type, func, compat=3 +91, "mp_length", type, func, compat=4 +92, "slot_subslots", slot, ptr, subslots=true, null=allow +93, "tp_slots", type, ptr, virtual=true, subslots=true, null=allow +94, "mod_slots", mod, ptr, subslots=true, null=allow +95, "tp_name", type, ptr, virtual=true, null=reject, duplicates=no, is_name=true +96, "tp_basicsize", type, size, virtual=true, duplicates=no +97, "tp_extra_basicsize", type, size, virtual=true, duplicates=no +98, "tp_itemsize", type, size, virtual=true, duplicates=no +99, "tp_flags", type, uint64, virtual=true, duplicates=no +100, "mod_name", mod, ptr, null=reject, duplicates=no, is_name=true +101, "mod_doc", mod, ptr, null=reject, duplicates=no +102, "mod_state_size", mod, size, duplicates=no +103, "mod_methods", mod, ptr, null=reject, duplicates=no +104, "mod_state_traverse", mod, func, null=reject, duplicates=no +105, "mod_state_clear", mod, func, null=reject, duplicates=no +106, "mod_state_free", mod, func, null=reject, duplicates=no +107, "tp_metaclass", type, ptr, virtual=true, null=reject, duplicates=no +108, "tp_module", type, ptr, virtual=true, null=reject, duplicates=no +109, "mod_abi", mod, ptr, null=reject, duplicates=no +110, "mod_token", mod, ptr, null=reject, duplicates=no diff --git a/Python/slots.toml b/Python/slots.toml new file mode 100644 index 00000000000000..7076ec86af7d9f --- /dev/null +++ b/Python/slots.toml @@ -0,0 +1,829 @@ +# This file lists all PySlot values +# This should only be used as input to Tools/build/generate_slots.py, +# its format can change at any time (e.g. we can rename it to slots.csv) + +# Entries: +# name: name of the slot +# kind: +# - 'type', 'mod': slots to create a particular object +# - 'slot': special slots applicable to any kind of object +# - 'compat': old IDs that nneed to be resolved +# dtype: datat type (tag for the union of sl_ptr, sl_size, etc.) +# equivalents: for 'compat' slots; the slots to resolve to +# is_type_field: slot that corresponds to a field in the type object (or in +# an array like PyNumberMethods). +# functype: C function type, where needed +# duplicates, nulls: How to handle common "problems" -- duplicate slots with +# the same ID, and NULL pointers, respectively +# - 'allow': not a problem for this slot +# - 'deprecated': issue a deprecation warning. Don't use for new slots. +# (typically, the problem was disallowed in docs, but allowed in practice) +# - 'reject': raise error +# The default is 'reject' (except NULL for non-pointer slots) + + +[0] +name = 'Py_slot_end' +kind = 'slot' +dtype = 'void' + +[1] +kind = 'compat' +equivalents = {type='Py_bf_getbuffer', mod='Py_mod_create'} + +[2] +kind = 'compat' +equivalents = {type='Py_bf_releasebuffer', mod='Py_mod_exec'} + +[3] +kind = 'compat' +equivalents = {type='Py_mp_ass_subscript', mod='Py_mod_multiple_interpreters'} + +[4] +kind = 'compat' +equivalents = {type='Py_mp_length', mod='Py_mod_gil'} + +[5] +name = 'Py_mp_subscript' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[6] +name = 'Py_nb_absolute' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[7] +name = 'Py_nb_add' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[8] +name = 'Py_nb_and' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[9] +name = 'Py_nb_bool' +kind = 'type' +is_type_field = true +functype = 'inquiry' +duplicates = 'deprecated' +nulls = 'deprecated' + +[10] +name = 'Py_nb_divmod' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[11] +name = 'Py_nb_float' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[12] +name = 'Py_nb_floor_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[13] +name = 'Py_nb_index' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[14] +name = 'Py_nb_inplace_add' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[15] +name = 'Py_nb_inplace_and' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[16] +name = 'Py_nb_inplace_floor_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[17] +name = 'Py_nb_inplace_lshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[18] +name = 'Py_nb_inplace_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[19] +name = 'Py_nb_inplace_or' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[20] +name = 'Py_nb_inplace_power' +kind = 'type' +is_type_field = true +functype = 'ternaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[21] +name = 'Py_nb_inplace_remainder' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[22] +name = 'Py_nb_inplace_rshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[23] +name = 'Py_nb_inplace_subtract' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[24] +name = 'Py_nb_inplace_true_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[25] +name = 'Py_nb_inplace_xor' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[26] +name = 'Py_nb_int' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[27] +name = 'Py_nb_invert' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[28] +name = 'Py_nb_lshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[29] +name = 'Py_nb_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[30] +name = 'Py_nb_negative' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[31] +name = 'Py_nb_or' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[32] +name = 'Py_nb_positive' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[33] +name = 'Py_nb_power' +kind = 'type' +is_type_field = true +functype = 'ternaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[34] +name = 'Py_nb_remainder' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[35] +name = 'Py_nb_rshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[36] +name = 'Py_nb_subtract' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[37] +name = 'Py_nb_true_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[38] +name = 'Py_nb_xor' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[39] +name = 'Py_sq_ass_item' +kind = 'type' +is_type_field = true +functype = 'ssizeobjargproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[40] +name = 'Py_sq_concat' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[41] +name = 'Py_sq_contains' +kind = 'type' +is_type_field = true +functype = 'objobjproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[42] +name = 'Py_sq_inplace_concat' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[43] +name = 'Py_sq_inplace_repeat' +kind = 'type' +is_type_field = true +functype = 'ssizeargfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[44] +name = 'Py_sq_item' +kind = 'type' +is_type_field = true +functype = 'ssizeargfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[45] +name = 'Py_sq_length' +kind = 'type' +is_type_field = true +functype = 'lenfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[46] +name = 'Py_sq_repeat' +kind = 'type' +is_type_field = true +functype = 'ssizeargfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[47] +name = 'Py_tp_alloc' +kind = 'type' +is_type_field = true +functype = 'allocfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[48] +name = 'Py_tp_base' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' + +[49] +name = 'Py_tp_bases' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' + +[50] +name = 'Py_tp_call' +kind = 'type' +is_type_field = true +functype = 'ternaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[51] +name = 'Py_tp_clear' +kind = 'type' +is_type_field = true +functype = 'inquiry' +duplicates = 'deprecated' +nulls = 'deprecated' + +[52] +name = 'Py_tp_dealloc' +kind = 'type' +is_type_field = true +functype = 'destructor' +duplicates = 'deprecated' +nulls = 'deprecated' + +[53] +name = 'Py_tp_del' +kind = 'type' +is_type_field = true +functype = 'destructor' +duplicates = 'deprecated' +nulls = 'deprecated' + +[54] +name = 'Py_tp_descr_get' +kind = 'type' +is_type_field = true +functype = 'descrgetfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[55] +name = 'Py_tp_descr_set' +kind = 'type' +is_type_field = true +functype = 'descrsetfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[56] +name = 'Py_tp_doc' +kind = 'type' +is_type_field = true +dtype = 'ptr' +nulls = 'allow' + +[57] +name = 'Py_tp_getattr' +kind = 'type' +is_type_field = true +functype = 'getattrfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[58] +name = 'Py_tp_getattro' +kind = 'type' +is_type_field = true +functype = 'getattrofunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[59] +name = 'Py_tp_hash' +kind = 'type' +is_type_field = true +functype = 'hashfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[60] +name = 'Py_tp_init' +kind = 'type' +is_type_field = true +functype = 'initproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[61] +name = 'Py_tp_is_gc' +kind = 'type' +is_type_field = true +functype = 'inquiry' +duplicates = 'deprecated' +nulls = 'deprecated' + +[62] +name = 'Py_tp_iter' +kind = 'type' +is_type_field = true +functype = 'getiterfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[63] +name = 'Py_tp_iternext' +kind = 'type' +is_type_field = true +functype = 'iternextfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[64] +name = 'Py_tp_methods' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' + +[65] +name = 'Py_tp_new' +kind = 'type' +is_type_field = true +functype = 'newfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[66] +name = 'Py_tp_repr' +kind = 'type' +is_type_field = true +functype = 'reprfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[67] +name = 'Py_tp_richcompare' +kind = 'type' +is_type_field = true +functype = 'richcmpfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[68] +name = 'Py_tp_setattr' +kind = 'type' +is_type_field = true +functype = 'setattrfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[69] +name = 'Py_tp_setattro' +kind = 'type' +is_type_field = true +functype = 'setattrofunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[70] +name = 'Py_tp_str' +kind = 'type' +is_type_field = true +functype = 'reprfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[71] +name = 'Py_tp_traverse' +kind = 'type' +is_type_field = true +functype = 'traverseproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[72] +name = 'Py_tp_members' +kind = 'type' +is_type_field = true +dtype = 'ptr' +nulls = 'deprecated' + +[73] +name = 'Py_tp_getset' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' + +[74] +name = 'Py_tp_free' +kind = 'type' +is_type_field = true +functype = 'freefunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[75] +name = 'Py_nb_matrix_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[76] +name = 'Py_nb_inplace_matrix_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[77] +name = 'Py_am_await' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[78] +name = 'Py_am_aiter' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[79] +name = 'Py_am_anext' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[80] +name = 'Py_tp_finalize' +kind = 'type' +is_type_field = true +functype = 'destructor' +duplicates = 'deprecated' +nulls = 'deprecated' + +[81] +name = 'Py_am_send' +kind = 'type' +is_type_field = true +functype = 'sendfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[82] +name = 'Py_tp_vectorcall' +kind = 'type' +is_type_field = true +functype = 'vectorcallfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[83] +name = 'Py_tp_token' +kind = 'type' +is_type_field = true +dtype = 'ptr' +field = 'ht_token' +duplicates = 'deprecated' +nulls = 'allow' + +[84] +name = 'Py_mod_create' +kind = 'mod' +dtype = 'func' +nulls = 'deprecated' + +[85] +name = 'Py_mod_exec' +kind = 'mod' +dtype = 'func' +duplicates = 'allow' +nulls = 'deprecated' + +[86] +name = 'Py_mod_multiple_interpreters' +kind = 'mod' +dtype = 'uint64' + +[87] +name = 'Py_mod_gil' +kind = 'mod' +dtype = 'uint64' + +[88] +name = 'Py_bf_getbuffer' +kind = 'type' +is_type_field = true +functype = 'getbufferproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[89] +name = 'Py_bf_releasebuffer' +kind = 'type' +is_type_field = true +functype = 'releasebufferproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[90] +name = 'Py_mp_ass_subscript' +kind = 'type' +is_type_field = true +functype = 'objobjargproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[91] +name = 'Py_mp_length' +kind = 'type' +is_type_field = true +functype = 'lenfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[92] +name = 'Py_slot_subslots' +kind = 'slot' +dtype = 'ptr' +nulls = 'allow' + +[93] +name = 'Py_tp_slots' +kind = 'type' +dtype = 'ptr' +nulls = 'allow' + +[94] +name = 'Py_mod_slots' +kind = 'mod' +dtype = 'ptr' +n.ulls = 'allow' + +[95] +name = 'Py_tp_name' +kind = 'type' +dtype = 'ptr' + +[96] +name = 'Py_tp_basicsize' +kind = 'type' +dtype = 'size' + +[97] +name = 'Py_tp_extra_basicsize' +kind = 'type' +dtype = 'size' + +[98] +name = 'Py_tp_itemsize' +kind = 'type' +dtype = 'size' + +[99] +name = 'Py_tp_flags' +kind = 'type' +dtype = 'uint64' + +[100] +name = 'Py_mod_name' +kind = 'mod' +dtype = 'ptr' + +[101] +name = 'Py_mod_doc' +kind = 'mod' +dtype = 'ptr' + +[102] +name = 'Py_mod_state_size' +kind = 'mod' +dtype = 'size' + +[103] +name = 'Py_mod_methods' +kind = 'mod' +dtype = 'ptr' + +[104] +name = 'Py_mod_state_traverse' +kind = 'mod' +dtype = 'func' + +[105] +name = 'Py_mod_state_clear' +kind = 'mod' +dtype = 'func' + +[106] +name = 'Py_mod_state_free' +kind = 'mod' +dtype = 'func' + +[107] +name = 'Py_tp_metaclass' +kind = 'type' +dtype = 'ptr' + +[108] +name = 'Py_tp_module' +kind = 'type' +dtype = 'ptr' + +[109] +name = 'Py_mod_abi' +kind = 'mod' +dtype = 'ptr' + +[110] +name = 'Py_mod_token' +kind = 'mod' +dtype = 'ptr' + diff --git a/Python/slots_generated.c b/Python/slots_generated.c new file mode 100644 index 00000000000000..f1c369f3a22bba --- /dev/null +++ b/Python/slots_generated.c @@ -0,0 +1,40 @@ +/* Generated by Tools/build/generate_slots.py */ + +#include "Python.h" +#include "pycore_slots.h" // _PySlot_names + +char *_PySlot_names[] = { + "Py_slot_end", "Py_bf_getbuffer/Py_mod_create", + "Py_bf_releasebuffer/Py_mod_exec", + "Py_mp_ass_subscript/Py_mod_multiple_interpreters", + "Py_mp_length/Py_mod_gil", "Py_mp_subscript", "Py_nb_absolute", + "Py_nb_add", "Py_nb_and", "Py_nb_bool", "Py_nb_divmod", "Py_nb_float", + "Py_nb_floor_divide", "Py_nb_index", "Py_nb_inplace_add", + "Py_nb_inplace_and", "Py_nb_inplace_floor_divide", "Py_nb_inplace_lshift", + "Py_nb_inplace_multiply", "Py_nb_inplace_or", "Py_nb_inplace_power", + "Py_nb_inplace_remainder", "Py_nb_inplace_rshift", + "Py_nb_inplace_subtract", "Py_nb_inplace_true_divide", "Py_nb_inplace_xor", + "Py_nb_int", "Py_nb_invert", "Py_nb_lshift", "Py_nb_multiply", + "Py_nb_negative", "Py_nb_or", "Py_nb_positive", "Py_nb_power", + "Py_nb_remainder", "Py_nb_rshift", "Py_nb_subtract", "Py_nb_true_divide", + "Py_nb_xor", "Py_sq_ass_item", "Py_sq_concat", "Py_sq_contains", + "Py_sq_inplace_concat", "Py_sq_inplace_repeat", "Py_sq_item", + "Py_sq_length", "Py_sq_repeat", "Py_tp_alloc", "Py_tp_base", "Py_tp_bases", + "Py_tp_call", "Py_tp_clear", "Py_tp_dealloc", "Py_tp_del", + "Py_tp_descr_get", "Py_tp_descr_set", "Py_tp_doc", "Py_tp_getattr", + "Py_tp_getattro", "Py_tp_hash", "Py_tp_init", "Py_tp_is_gc", "Py_tp_iter", + "Py_tp_iternext", "Py_tp_methods", "Py_tp_new", "Py_tp_repr", + "Py_tp_richcompare", "Py_tp_setattr", "Py_tp_setattro", "Py_tp_str", + "Py_tp_traverse", "Py_tp_members", "Py_tp_getset", "Py_tp_free", + "Py_nb_matrix_multiply", "Py_nb_inplace_matrix_multiply", "Py_am_await", + "Py_am_aiter", "Py_am_anext", "Py_tp_finalize", "Py_am_send", + "Py_tp_vectorcall", "Py_tp_token", "Py_mod_create", "Py_mod_exec", + "Py_mod_multiple_interpreters", "Py_mod_gil", "Py_bf_getbuffer", + "Py_bf_releasebuffer", "Py_mp_ass_subscript", "Py_mp_length", + "Py_slot_subslots", "Py_tp_slots", "Py_mod_slots", "Py_tp_name", + "Py_tp_basicsize", "Py_tp_extra_basicsize", "Py_tp_itemsize", + "Py_tp_flags", "Py_mod_name", "Py_mod_doc", "Py_mod_state_size", + "Py_mod_methods", "Py_mod_state_traverse", "Py_mod_state_clear", + "Py_mod_state_free", "Py_tp_metaclass", "Py_tp_module", "Py_mod_abi", + "Py_mod_token", NULL +}; diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 996f725fdcb9b5..4a3dd618f6559f 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -38,3 +38,7 @@ ignore = [ "generate_{re_casefix,sre_constants,token}.py" = [ "UP031", # Use format specifiers instead of percent format ] +"generate_slots.py" = [ + "I001", # Import block is un-sorted + "ISC003", # Explicitly concatenated string +] diff --git a/Tools/build/generate_slots.py b/Tools/build/generate_slots.py new file mode 100755 index 00000000000000..61eaeb4e390285 --- /dev/null +++ b/Tools/build/generate_slots.py @@ -0,0 +1,380 @@ +#!/usr/bin/python +"""Generate type/module slot files. +""" + +import io +import sys +import json +import tomllib +import argparse +import functools +import contextlib +import collections +from pathlib import Path + +GENERATED_BY = 'Generated by Tools/build/generate_slots.py' + +REPO_ROOT = Path(__file__).parent.parent.parent +DEFAULT_INPUT_PATH = REPO_ROOT / 'Python/slots.toml' +INCLUDE_PATH = REPO_ROOT / 'Include' +DEFAULT_PUBLIC_HEADER_PATH = INCLUDE_PATH / 'slots_generated.h' +DEFAULT_PRIVATE_HEADER_PATH = INCLUDE_PATH / 'internal/pycore_slots_generated.h' +DEFAULT_C_PATH = REPO_ROOT / 'Python/slots_generated.c' + +FIRST_3_15_SLOT = 83 + +TABLES = { + 'tp': 'ht_type', + 'am': 'as_async', + 'nb': 'as_number', + 'mp': 'as_mapping', + 'sq': 'as_sequence', + 'bf': 'as_buffer', +} + + +class SlotInfo: + def __init__(self, id, data): + self.id = id + self.kind = data['kind'] + self._data = data + try: + self.name = data['name'] + except KeyError: + self.name = '/'.join(data["equivalents"].values()) + else: + assert self.name.isidentifier + + @functools.cached_property + def equivalents(self): + return self._data['equivalents'] + + @functools.cached_property + def dtype(self): + try: + return self._data['dtype'] + except KeyError: + if self.is_type_field: + return 'func' + raise + + @functools.cached_property + def functype(self): + return self._data['functype'] + + @functools.cached_property + def is_type_field(self): + return self._data.get('is_type_field') + + @functools.cached_property + def type_field(self): + return self._data.get('field', self.name.removeprefix('Py_')) + + @functools.cached_property + def type_table_ident(self): + return self._data.get('table', self.type_field[:2]) + + @functools.cached_property + def duplicate_handling(self): + return self._data.get('duplicates') + + @functools.cached_property + def null_handling(self): + return self._data.get('nulls') + + +def parse_slots(file): + toml_contents = tomllib.load(file) + result = [None] * len(toml_contents) + for key, data in toml_contents.items(): + slot_id = int(key) + try: + if result[slot_id]: + raise ValueError(f'slot ID {slot_id} repeated') + result[slot_id] = SlotInfo(slot_id, data) + except Exception as e: + e.add_note(f'handling slot {slot_id}') + raise + return result + + +class CWriter: + """Simple helper for generating C code""" + def __init__(self, file): + self.file = file + self.indent = '' + self(f'/* {GENERATED_BY} */') + self() + + def out(self, *args, **kwargs): + """print args to the file, with current indent at the start""" + self.file.write(self.indent) + print(*args, file=self.file, **kwargs) + + __call__ = out + + @contextlib.contextmanager + def block(self, header=None, end=''): + """Context for a {}-enclosed block of C""" + if header is None: + self.out('{') + else: + self.out(header, '{') + old_indent = self.indent + self.indent += ' ' + yield + self.indent = old_indent + self.out('}' + end) + + def spam(self, segments): + """Write *segments*, putting as many as will fit on a single line""" + maxlen = 79-len(self.indent) + segments = iter(segments) + current_line = next(segments) + for segment in segments: + if len(current_line) + 1 + len(segment) > maxlen: + self.out(current_line) + current_line = segment + else: + current_line += ' ' + segment + if current_line: + self.out(current_line) + + +def write_public_header(f, slots): + out = CWriter(f) + out(f'#ifndef _PY_HAVE_SLOTS_GENERATED_H') + out(f'#define _PY_HAVE_SLOTS_GENERATED_H') + out() + out(f'#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15)') + out(f'#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW') + out(f'#else') + out(f'#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD') + out(f'#endif') + out() + compat_ids = {} + for slot in slots: + if slot.kind == 'compat': + for new_name in slot.equivalents.values(): + compat_ids[new_name] = slot.id + for slot in slots: + if slot.kind == 'compat': + continue + slot_id = slot.id + if compat := compat_ids.get(slot.name): + slot_id = f'_Py_SLOT_COMPAT_VALUE({compat}, {slot_id})' + out(f'#define {slot.name} {slot_id}') + out() + out(f'#define _Py_slot_COUNT {len(slots)}') + out(f'#endif /* _PY_HAVE_SLOTS_GENERATED_H */') + + +def write_private_header(f, slots): + out = CWriter(f) + + def add_case(slot): + out(out(f' case {slot.id}:')) + + slots_by_name = {slot.name: slot for slot in slots} + + out(f'#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H') + out(f'#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H') + for kind in 'type', 'mod': + out() + out(f'static inline uint16_t') + out(f'_PySlot_resolve_{kind}_slot(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + good_slots = [] + for slot in slots: + if slot.kind == 'compat': + new_slot = slots_by_name[slot.equivalents[kind]] + out(f'case {slot.id}:') + out(f' return {new_slot.id};') + elif slot.kind in {kind, 'slot'}: + good_slots.append(f'case {slot.id}:') + out.spam(good_slots) + out(f' return slot_id;') + out(f'default:') + out(f' return Py_slot_invalid;') + out() + out(f'static inline void*') + out(f'_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + for slot in slots: + if slot.is_type_field: + field = slot.type_field + table_ident = slot.type_table_ident + if table_ident == 'tp': + out(f'case {slot.id}: return (void*)tp->{field};') + else: + if table_ident == 'ht': + cond = 'tp->tp_flags & Py_TPFLAGS_HEAPTYPE' + val = f'((PyHeapTypeObject*)tp)->{field}' + else: + table = TABLES[table_ident] + cond = f'tp->tp_{table}' + val = f'tp->tp_{table}->{field}' + out(f'case {slot.id}: return ({cond})') + out(f' ? {val} : NULL;') + out(f'_PySlot_err_bad_slot("PyType_GetSlot", slot_id);') + out(f'return NULL;') + out() + out(f'static inline void') + out(f'_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht,', + f'PySlot slot)') + with out.block(): + with out.block('switch (slot.sl_id)'): + for slot in slots: + if slot.is_type_field: + field = slot.type_field + table_ident = slot.type_table_ident + if table_ident == 'ht': + continue + table = TABLES[table_ident] + if slot.dtype == 'func': + functype = f'({slot.functype})' + else: + functype = '' + out( + f'case {slot.id}:', + f'ht->{table}.{field} = {functype}slot.sl_{slot.dtype};', + f'break;' + ) + out() + out(f'static inline _PySlot_DTYPE') + out(f'_PySlot_get_dtype(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + for slot in slots: + if slot.kind == 'compat': + continue + dtype = slot.dtype + name = slot.name + out(f'case {name}: return _PySlot_DTYPE_{dtype.upper()};') + out(f'default: return _PySlot_DTYPE_VOID;') + out() + out(f'static inline _PySlot_PROBLEM_HANDLING') + out(f'_PySlot_get_duplicate_handling(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + results = collections.defaultdict(list) + for slot in slots: + handling = slot.duplicate_handling or 'reject' + results[handling.upper()].append(f'case {slot.id}:') + results.pop('REJECT') + for handling, cases in results.items(): + out.spam(cases) + out(f' return _PySlot_PROBLEM_{handling};') + out(f'default:') + out(f' return _PySlot_PROBLEM_REJECT;') + out() + out(f'static inline _PySlot_PROBLEM_HANDLING') + out(f'_PySlot_get_null_handling(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + results = collections.defaultdict(list) + for slot in slots: + handling = slot.null_handling + if handling is None: + if slot.kind != 'compat' and slot.dtype in {'ptr', 'func'}: + handling = 'reject' + else: + handling = 'allow' + results[handling.upper()].append(f'case {slot.id}:') + results.pop('REJECT') + for handling, cases in results.items(): + out.spam(cases) + out(f' return _PySlot_PROBLEM_{handling};') + out(f'default:') + out(f' return _PySlot_PROBLEM_REJECT;') + out() + out(f'#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */') + + +def write_c(f, slots): + out = CWriter(f) + out('#include "Python.h"') + out('#include "pycore_slots.h" // _PySlot_names') + out() + with out.block(f'char *_PySlot_names[] =', end=';'): + out.spam([f'"{slot.name}",' for slot in slots] + ['NULL']) + + +@contextlib.contextmanager +def replace_file(filename): + file_path = Path(filename) + with io.StringIO() as sio: + yield sio + try: + old_text = file_path.read_text() + except FileNotFoundError: + old_text = None + new_text = sio.getvalue() + if old_text == new_text: + print(f'{filename}: not modified', file=sys.stderr) + else: + print(f'{filename}: writing new content', file=sys.stderr) + file_path.write_text(new_text) + + +def main(argv): + if len(argv) == 1: + # No sens calling this with no arguments. + argv.append('--help') + + parser = argparse.ArgumentParser(prog=argv[0], description=__doc__) + parser.add_argument( + '-i', '--input', default=DEFAULT_INPUT_PATH, + help=f'the input file (default: {DEFAULT_INPUT_PATH})') + parser.add_argument( + '--generate-all', action=argparse.BooleanOptionalAction, + help='write all output files to their default locations') + parser.add_argument( + '-j', '--jsonl', action=argparse.BooleanOptionalAction, + help='write info to stdout in "JSON Lines" format (one JSON per line)') + outfile_group = parser.add_argument_group( + 'output files', + description='By default, no files are generated. Use --generate-all ' + + 'or the options below to generate them.') + outfile_group.add_argument( + '-H', '--public-header', + help='file into which to write the public header') + outfile_group.add_argument( + '-I', '--private-header', + help='file into which to write the private header') + outfile_group.add_argument( + '-C', '--cfile', + help='file into which to write internal C code') + args = parser.parse_args(argv[1:]) + + if args.generate_all: + if args.public_header is None: + args.public_header = DEFAULT_PUBLIC_HEADER_PATH + if args.private_header is None: + args.private_header = DEFAULT_PRIVATE_HEADER_PATH + if args.cfile is None: + args.cfile = DEFAULT_C_PATH + + with open(args.input, 'rb') as f: + slots = parse_slots(f) + + if args.jsonl: + for slot in slots: + print(json.dumps(slot.to_dict())) + + if args.public_header: + with replace_file(args.public_header) as f: + write_public_header(f, slots) + + if args.private_header: + with replace_file(args.private_header) as f: + write_private_header(f, slots) + + if args.cfile: + with replace_file(args.cfile) as f: + write_c(f, slots) + +if __name__ == "__main__": + main(sys.argv) diff --git a/configure b/configure index c826a1bb85667b..525601f5dd9728 100755 --- a/configure +++ b/configure @@ -3581,7 +3581,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then # If we're building out-of-tree, we need to make sure the following # resources get picked up before their $srcdir counterparts. - # Objects/ -> typeslots.inc + # Objects/ -> slots_generated.c # Include/ -> Python.h # (A side effect of this is that these resources will automatically be # regenerated when building out-of-tree, regardless of whether or not diff --git a/configure.ac b/configure.ac index 322d33dd0e3c99..09ba4ad46cd5c5 100644 --- a/configure.ac +++ b/configure.ac @@ -99,7 +99,7 @@ AC_SUBST([BASECPPFLAGS]) if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then # If we're building out-of-tree, we need to make sure the following # resources get picked up before their $srcdir counterparts. - # Objects/ -> typeslots.inc + # Objects/ -> slots_generated.c # Include/ -> Python.h # (A side effect of this is that these resources will automatically be # regenerated when building out-of-tree, regardless of whether or not