diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index bf89700f54b7e9..1a7b456a8fc6ab 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -657,12 +657,13 @@ Nested structures can also be initialized in the constructor in several ways:: >>> r = RECT((1, 2), (3, 4)) Field :term:`descriptor`\s can be retrieved from the *class*, they are useful -for debugging because they can provide useful information:: +for debugging because they can provide useful information. +See :class:`CField`:: - >>> print(POINT.x) - - >>> print(POINT.y) - + >>> POINT.x + + >>> POINT.y + >>> @@ -2812,6 +2813,98 @@ fields, or any other data types containing pointer type fields. present in :attr:`_fields_`. +.. class:: CField(*args, **kw) + + Descriptor for fields of a :class:`Structure` and :class:`Union`. + For example:: + + >>> class Color(Structure): + ... _fields_ = ( + ... ('red', c_uint8), + ... ('green', c_uint8), + ... ('blue', c_uint8), + ... ('intense', c_bool, 1), + ... ('blinking', c_bool, 1), + ... ) + ... + >>> Color.red + + >>> Color.green.type + + >>> Color.blue.byte_offset + 2 + >>> Color.intense + + >>> Color.blinking.bit_offset + 1 + + All attributes are read-only. + + :class:`!CField` objects are created via :attr:`~Structure._fields_`; + do not instantiate the class directly. + + .. versionadded:: next + + Previously, descriptors only had ``offset`` and ``size`` attributes + and a readable string representation; the :class:`!CField` class was not + available directly. + + .. attribute:: name + + Name of the field, as a string. + + .. attribute:: type + + Type of the field, as a :ref:`ctypes class `. + + .. attribute:: offset + byte_offset + + Offset of the field, in bytes. + + For bitfields, this is the offset of the underlying byte-aligned + *storage unit*; see :attr:`~CField.bit_offset`. + + .. attribute:: byte_size + + Size of the field, in bytes. + + For bitfields, this is the size of the underlying *storage unit*. + Typically, it has the same size as the bitfield's type. + + .. attribute:: size + + For non-bitfields, equivalent to :attr:`~CField.byte_size`. + + For bitfields, this contains a backwards-compatible bit-packed + value that combines :attr:`~CField.bit_size` and + :attr:`~CField.bit_offset`. + Prefer using the explicit attributes instead. + + .. attribute:: is_bitfield + + True if this is a bitfield. + + .. attribute:: bit_offset + bit_size + + The location of a bitfield within its *storage unit*, that is, within + :attr:`~CField.byte_size` bytes of memory starting at + :attr:`~CField.byte_offset`. + + To get the field's value, read the storage unit as an integer, + :ref:`shift left ` by :attr:`!bit_offset` and + take the :attr:`!bit_size` least significant bits. + + For non-bitfields, :attr:`!bit_offset` is zero + and :attr:`!bit_size` is equal to ``byte_size * 8``. + + .. attribute:: is_anonymous + + True if this field is anonymous, that is, it contains nested sub-fields + that should be be merged into a containing structure or union. + + .. _ctypes-arrays-pointers: Arrays and pointers diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 65ab57eb821c6c..89871d84bdd188 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -487,6 +487,11 @@ ctypes to help match a non-default ABI. (Contributed by Petr Viktorin in :gh:`97702`.) +* The class of :class:`~ctypes.Structure`/:class:`~ctypes.Union` + field descriptors is now available as :class:`~ctypes.CField`, + and has new attributes to aid debugging and introspection. + (Contributed by Petr Viktorin in :gh:`128715`.) + * On Windows, the :exc:`~ctypes.COMError` exception is now public. (Contributed by Jun Komoda in :gh:`126686`.) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 90214a314031d1..0612184e10c70c 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -756,6 +756,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_handle_fromlist)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_internal_use)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_isatty_open_only)); @@ -806,6 +807,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(before)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(big)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(binary_form)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_offset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bit_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(block)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bound)); @@ -816,6 +818,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(buffers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bufsize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(builtins)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byte_offset)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byte_size)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(byteorder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(bytes_per_sep)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 97a75d0c46c867..22ffe80d9a5b11 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -245,6 +245,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_get_sourcefile) STRUCT_FOR_ID(_handle_fromlist) STRUCT_FOR_ID(_initializing) + STRUCT_FOR_ID(_internal_use) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) STRUCT_FOR_ID(_isatty_open_only) @@ -295,6 +296,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(before) STRUCT_FOR_ID(big) STRUCT_FOR_ID(binary_form) + STRUCT_FOR_ID(bit_offset) STRUCT_FOR_ID(bit_size) STRUCT_FOR_ID(block) STRUCT_FOR_ID(bound) @@ -305,6 +307,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(buffers) STRUCT_FOR_ID(bufsize) STRUCT_FOR_ID(builtins) + STRUCT_FOR_ID(byte_offset) + STRUCT_FOR_ID(byte_size) STRUCT_FOR_ID(byteorder) STRUCT_FOR_ID(bytes) STRUCT_FOR_ID(bytes_per_sep) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4f928cc050bf8e..4a7111a01bf00c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -754,6 +754,7 @@ extern "C" { INIT_ID(_get_sourcefile), \ INIT_ID(_handle_fromlist), \ INIT_ID(_initializing), \ + INIT_ID(_internal_use), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ INIT_ID(_isatty_open_only), \ @@ -804,6 +805,7 @@ extern "C" { INIT_ID(before), \ INIT_ID(big), \ INIT_ID(binary_form), \ + INIT_ID(bit_offset), \ INIT_ID(bit_size), \ INIT_ID(block), \ INIT_ID(bound), \ @@ -814,6 +816,8 @@ extern "C" { INIT_ID(buffers), \ INIT_ID(bufsize), \ INIT_ID(builtins), \ + INIT_ID(byte_offset), \ + INIT_ID(byte_size), \ INIT_ID(byteorder), \ INIT_ID(bytes), \ INIT_ID(bytes_per_sep), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 5b78d038fc1192..1ec99a1b5b3a5c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -776,6 +776,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_internal_use); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_io); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -976,6 +980,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(bit_offset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(bit_size); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1016,6 +1024,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(byte_offset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(byte_size); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(byteorder); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 8e2a2926f7a853..d9e55816211737 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -12,6 +12,7 @@ from _ctypes import RTLD_LOCAL, RTLD_GLOBAL from _ctypes import ArgumentError from _ctypes import SIZEOF_TIME_T +from _ctypes import CField from struct import calcsize as _calcsize diff --git a/Lib/ctypes/_layout.py b/Lib/ctypes/_layout.py index adda3f9a6f2acc..beb3b86414c010 100644 --- a/Lib/ctypes/_layout.py +++ b/Lib/ctypes/_layout.py @@ -19,29 +19,6 @@ def round_up(n, multiple): assert multiple > 0 return ((n + multiple - 1) // multiple) * multiple -def LOW_BIT(offset): - return offset & 0xFFFF - -def NUM_BITS(bitsize): - return bitsize >> 16 - -def BUILD_SIZE(bitsize, offset): - assert 0 <= offset, offset - assert offset <= 0xFFFF, offset - # We don't support zero length bitfields. - # And GET_BITFIELD uses NUM_BITS(size) == 0, - # to figure out whether we are handling a bitfield. - assert bitsize > 0, bitsize - result = (bitsize << 16) + offset - assert bitsize == NUM_BITS(result), (bitsize, result) - assert offset == LOW_BIT(result), (offset, result) - return result - -def build_size(bit_size, bit_offset, big_endian, type_size): - if big_endian: - return BUILD_SIZE(bit_size, 8 * type_size - bit_offset - bit_size) - return BUILD_SIZE(bit_size, bit_offset) - _INT_MAX = (1 << (ctypes.sizeof(ctypes.c_int) * 8) - 1) - 1 @@ -213,13 +190,10 @@ def get_layout(cls, input_fields, is_struct, base): offset = round_down(next_bit_offset, type_bit_align) // 8 if is_bitfield: - effective_bit_offset = next_bit_offset - 8 * offset - size = build_size(bit_size, effective_bit_offset, - big_endian, type_size) - assert effective_bit_offset <= type_bit_size + bit_offset = next_bit_offset - 8 * offset + assert bit_offset <= type_bit_size else: assert offset == next_bit_offset / 8 - size = type_size next_bit_offset += bit_size struct_size = round_up(next_bit_offset, 8) // 8 @@ -253,18 +227,17 @@ def get_layout(cls, input_fields, is_struct, base): offset = next_byte_offset - last_field_bit_size // 8 if is_bitfield: assert 0 <= (last_field_bit_size + next_bit_offset) - size = build_size(bit_size, - last_field_bit_size + next_bit_offset, - big_endian, type_size) - else: - size = type_size + bit_offset = last_field_bit_size + next_bit_offset if type_bit_size: assert (last_field_bit_size + next_bit_offset) < type_bit_size next_bit_offset += bit_size struct_size = next_byte_offset - assert (not is_bitfield) or (LOW_BIT(size) <= size * 8) + if is_bitfield and big_endian: + # On big-endian architectures, bit fields are also laid out + # starting with the big end. + bit_offset = type_bit_size - bit_size - bit_offset # Add the format spec parts if is_struct: @@ -286,16 +259,21 @@ def get_layout(cls, input_fields, is_struct, base): # a bytes name would be rejected later, but we check early # to avoid a BytesWarning with `python -bb` raise TypeError( - "field {name!r}: name must be a string, not bytes") + f"field {name!r}: name must be a string, not bytes") format_spec_parts.append(f"{fieldfmt}:{name}:") result_fields.append(CField( name=name, type=ctype, - size=size, - offset=offset, + byte_size=type_size, + byte_offset=offset, bit_size=bit_size if is_bitfield else None, + bit_offset=bit_offset if is_bitfield else None, index=i, + + # Do not use CField outside ctypes, yet. + # The constructor is internal API and may change without warning. + _internal_use=True, )) if is_bitfield and not gcc_layout: assert type_bit_size > 0 diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py index e4c2b33825ae8f..946d654a19aff8 100644 --- a/Lib/test/test_ctypes/_support.py +++ b/Lib/test/test_ctypes/_support.py @@ -2,15 +2,13 @@ import ctypes from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr +import sys +from test import support _CData = Structure.__base__ assert _CData.__name__ == "_CData" -class _X(Structure): - _fields_ = [("x", ctypes.c_int)] -CField = type(_X.x) - # metaclasses PyCStructType = type(Structure) UnionType = type(Union) @@ -22,3 +20,132 @@ class _X(Structure): # type flags Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7 Py_TPFLAGS_IMMUTABLETYPE = 1 << 8 + + +def is_underaligned(ctype): + """Return true when type's alignment is less than its size. + + A famous example is 64-bit int on 32-bit x86. + """ + return ctypes.alignment(ctype) < ctypes.sizeof(ctype) + + +class StructCheckMixin: + def check_struct(self, structure): + """Assert that a structure is well-formed""" + self._check_struct_or_union(structure, is_struct=True) + + def check_union(self, union): + """Assert that a union is well-formed""" + self._check_struct_or_union(union, is_struct=False) + + def check_struct_or_union(self, cls): + if issubclass(cls, Structure): + self._check_struct_or_union(cls, is_struct=True) + elif issubclass(cls, Union): + self._check_struct_or_union(cls, is_struct=False) + else: + raise TypeError(cls) + + def _check_struct_or_union(self, cls, is_struct): + + # Check that fields are not overlapping (for structs), + # and that their metadata is consistent. + + used_bits = 0 + + is_little_endian = ( + hasattr(cls, '_swappedbytes_') ^ (sys.byteorder == 'little')) + + anon_names = getattr(cls, '_anonymous_', ()) + cls_size = ctypes.sizeof(cls) + for name, requested_type, *rest_of_tuple in cls._fields_: + field = getattr(cls, name) + with self.subTest(name=name, field=field): + is_bitfield = len(rest_of_tuple) > 0 + + # name + self.assertEqual(field.name, name) + + # type + self.assertEqual(field.type, requested_type) + + # offset === byte_offset + self.assertEqual(field.byte_offset, field.offset) + if not is_struct: + self.assertEqual(field.byte_offset, 0) + + # byte_size + self.assertEqual(field.byte_size, ctypes.sizeof(field.type)) + self.assertGreaterEqual(field.byte_size, 0) + + # Check that the field is inside the struct. + # See gh-130410 for why this is skipped for bitfields of + # underaligned types. Later in this function (see `bit_end`) + # we assert that the value *bits* are inside the struct. + if not (field.is_bitfield and is_underaligned(field.type)): + self.assertLessEqual(field.byte_offset + field.byte_size, + cls_size) + + # size + self.assertGreaterEqual(field.size, 0) + if is_bitfield: + # size has backwards-compatible bit-packed info + expected_size = (field.bit_size << 16) + field.bit_offset + self.assertEqual(field.size, expected_size) + else: + # size == byte_size + self.assertEqual(field.size, field.byte_size) + + # is_bitfield (bool) + self.assertIs(field.is_bitfield, is_bitfield) + + # bit_offset + if is_bitfield: + self.assertGreaterEqual(field.bit_offset, 0) + self.assertLessEqual(field.bit_offset + field.bit_size, + field.byte_size * 8) + else: + self.assertEqual(field.bit_offset, 0) + if not is_struct: + if is_little_endian: + self.assertEqual(field.bit_offset, 0) + else: + self.assertEqual(field.bit_offset, + field.byte_size * 8 - field.bit_size) + + # bit_size + if is_bitfield: + self.assertGreaterEqual(field.bit_size, 0) + self.assertLessEqual(field.bit_size, field.byte_size * 8) + [requested_bit_size] = rest_of_tuple + self.assertEqual(field.bit_size, requested_bit_size) + else: + self.assertEqual(field.bit_size, field.byte_size * 8) + + # is_anonymous (bool) + self.assertIs(field.is_anonymous, name in anon_names) + + # In a struct, field should not overlap. + # (Test skipped if the structs is enormous.) + if is_struct and cls_size < 10_000: + # Get a mask indicating where the field is within the struct + if is_little_endian: + tp_shift = field.byte_offset * 8 + else: + tp_shift = (cls_size + - field.byte_offset + - field.byte_size) * 8 + mask = (1 << field.bit_size) - 1 + mask <<= (tp_shift + field.bit_offset) + assert mask.bit_count() == field.bit_size + # Check that these bits aren't shared with previous fields + self.assertEqual(used_bits & mask, 0) + # Mark the bits for future checks + used_bits |= mask + + # field is inside cls + bit_end = (field.byte_offset * 8 + + field.bit_offset + + field.bit_size) + self.assertLessEqual(bit_end, cls_size * 8) diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py index a208fb9a00966a..26d24f31b29f7b 100644 --- a/Lib/test/test_ctypes/test_aligned_structures.py +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -5,9 +5,9 @@ ) import struct import unittest +from ._support import StructCheckMixin - -class TestAlignedStructures(unittest.TestCase): +class TestAlignedStructures(unittest.TestCase, StructCheckMixin): def test_aligned_string(self): for base, e in ( (LittleEndianStructure, "<"), @@ -19,12 +19,14 @@ class Aligned(base): _fields_ = [ ('value', c_char * 12) ] + self.check_struct(Aligned) class Main(base): _fields_ = [ ('first', c_uint32), ('string', Aligned), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(main.first, 7) @@ -46,12 +48,14 @@ class SomeBools(base): ("bool1", c_ubyte), ("bool2", c_ubyte), ] + self.check_struct(SomeBools) class Main(base): _fields_ = [ ("x", c_ubyte), ("y", SomeBools), ("z", c_ubyte), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(alignment(SomeBools), 4) @@ -75,11 +79,13 @@ class SomeBoolsTooBig(base): ("bool2", c_ubyte), ("bool3", c_ubyte), ] + self.check_struct(SomeBoolsTooBig) class Main(base): _fields_ = [ ("y", SomeBoolsTooBig), ("z", c_uint32), ] + self.check_struct(Main) with self.assertRaises(ValueError) as ctx: Main.from_buffer(data) self.assertEqual( @@ -98,18 +104,21 @@ class UnalignedSub(base): _fields_ = [ ("x", c_uint32), ] + self.check_struct(UnalignedSub) class AlignedStruct(UnalignedSub): _align_ = 8 _fields_ = [ ("y", c_uint32), ] + self.check_struct(AlignedStruct) class Main(base): _fields_ = [ ("a", c_uint32), ("b", AlignedStruct) ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(alignment(main.b), 8) @@ -134,12 +143,14 @@ class AlignedUnion(ubase): ("a", c_uint32), ("b", c_ubyte * 7), ] + self.check_union(AlignedUnion) class Main(sbase): _fields_ = [ ("first", c_uint32), ("union", AlignedUnion), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(main.first, 1) @@ -162,18 +173,21 @@ class Sub(sbase): ("x", c_uint32), ("y", c_uint32), ] + self.check_struct(Sub) class MainUnion(ubase): _fields_ = [ ("a", c_uint32), ("b", Sub), ] + self.check_union(MainUnion) class Main(sbase): _fields_ = [ ("first", c_uint32), ("union", MainUnion), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(Main.first.size, 4) @@ -198,17 +212,20 @@ class SubUnion(ubase): ("unsigned", c_ubyte), ("signed", c_byte), ] + self.check_union(SubUnion) class MainUnion(SubUnion): _fields_ = [ ("num", c_uint32) ] + self.check_union(SubUnion) class Main(sbase): _fields_ = [ ("first", c_uint16), ("union", MainUnion), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(main.union.num, 0xD60102D7) @@ -232,11 +249,13 @@ class SubUnion(ubase): ("unsigned", c_ubyte), ("signed", c_byte), ] + self.check_union(SubUnion) class Main(SubUnion): _fields_ = [ ("num", c_uint32) ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(alignment(main), 8) @@ -258,6 +277,7 @@ class Inner(sbase): ("x", c_uint16), ("y", c_uint16), ] + self.check_struct(Inner) class Main(sbase): _pack_ = 1 @@ -266,6 +286,7 @@ class Main(sbase): ("b", Inner), ("c", c_ubyte), ] + self.check_struct(Main) main = Main.from_buffer(data) self.assertEqual(sizeof(main), 10) diff --git a/Lib/test/test_ctypes/test_anon.py b/Lib/test/test_ctypes/test_anon.py index b36397b510fefe..2e16e708635989 100644 --- a/Lib/test/test_ctypes/test_anon.py +++ b/Lib/test/test_ctypes/test_anon.py @@ -1,20 +1,23 @@ import unittest import test.support from ctypes import c_int, Union, Structure, sizeof +from ._support import StructCheckMixin -class AnonTest(unittest.TestCase): +class AnonTest(unittest.TestCase, StructCheckMixin): def test_anon(self): class ANON(Union): _fields_ = [("a", c_int), ("b", c_int)] + self.check_union(ANON) class Y(Structure): _fields_ = [("x", c_int), ("_", ANON), ("y", c_int)] _anonymous_ = ["_"] + self.check_struct(Y) self.assertEqual(Y.a.offset, sizeof(c_int)) self.assertEqual(Y.b.offset, sizeof(c_int)) @@ -52,17 +55,20 @@ class Name(Structure): def test_nested(self): class ANON_S(Structure): _fields_ = [("a", c_int)] + self.check_struct(ANON_S) class ANON_U(Union): _fields_ = [("_", ANON_S), ("b", c_int)] _anonymous_ = ["_"] + self.check_union(ANON_U) class Y(Structure): _fields_ = [("x", c_int), ("_", ANON_U), ("y", c_int)] _anonymous_ = ["_"] + self.check_struct(Y) self.assertEqual(Y.x.offset, 0) self.assertEqual(Y.a.offset, sizeof(c_int)) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 19ba2f4484e7da..dc81e752567c42 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -10,27 +10,33 @@ Union) from test import support from test.support import import_helper +from ._support import StructCheckMixin _ctypes_test = import_helper.import_module("_ctypes_test") +TEST_FIELDS = ( + ("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7), +) + + class BITS(Structure): - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] + _fields_ = TEST_FIELDS func = CDLL(_ctypes_test.__file__).unpack_bitfields func.argtypes = POINTER(BITS), c_char @@ -38,23 +44,12 @@ class BITS(Structure): class BITS_msvc(Structure): _layout_ = "ms" - _fields_ = [("A", c_int, 1), - ("B", c_int, 2), - ("C", c_int, 3), - ("D", c_int, 4), - ("E", c_int, 5), - ("F", c_int, 6), - ("G", c_int, 7), - ("H", c_int, 8), - ("I", c_int, 9), - - ("M", c_short, 1), - ("N", c_short, 2), - ("O", c_short, 3), - ("P", c_short, 4), - ("Q", c_short, 5), - ("R", c_short, 6), - ("S", c_short, 7)] + _fields_ = TEST_FIELDS + + +class BITS_gcc(Structure): + _layout_ = "gcc-sysv" + _fields_ = TEST_FIELDS try: @@ -124,13 +119,19 @@ def test_shorts_msvc_mode(self): unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) int_types = unsigned_int_types + signed_int_types -class BitFieldTest(unittest.TestCase): +class BitFieldTest(unittest.TestCase, StructCheckMixin): + + def test_generic_checks(self): + self.check_struct(BITS) + self.check_struct(BITS_msvc) + self.check_struct(BITS_gcc) def test_longlong(self): class X(Structure): _fields_ = [("a", c_longlong, 1), ("b", c_longlong, 62), ("c", c_longlong, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_longlong)) x = X() @@ -142,6 +143,7 @@ class X(Structure): _fields_ = [("a", c_ulonglong, 1), ("b", c_ulonglong, 62), ("c", c_ulonglong, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_longlong)) x = X() @@ -159,6 +161,7 @@ class X(Structure): ("a", c_typ, 3), ("b", c_typ, 3), ("c", c_typ, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_typ)*2) x = X() @@ -178,6 +181,7 @@ class X(Structure): _fields_ = [("a", c_typ, 3), ("b", c_typ, 3), ("c", c_typ, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_typ)) x = X() @@ -210,12 +214,14 @@ def test_nonint_types(self): class Empty(Structure): _fields_ = [] + self.check_struct(Empty) result = self.fail_fields(("a", Empty, 1)) self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) class Dummy(Structure): _fields_ = [("x", c_int)] + self.check_struct(Dummy) result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) @@ -240,10 +246,12 @@ def test_single_bitfield_size(self): class X(Structure): _fields_ = [("a", c_typ, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_typ)) class X(Structure): _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_typ)) result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) @@ -255,6 +263,7 @@ class X(Structure): _fields_ = [("a", c_short, 1), ("b", c_short, 14), ("c", c_short, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_short)) class X(Structure): @@ -262,6 +271,7 @@ class X(Structure): ("a1", c_short), ("b", c_short, 14), ("c", c_short, 1)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_short)*3) self.assertEqual(X.a.offset, 0) self.assertEqual(X.a1.offset, sizeof(c_short)) @@ -272,6 +282,7 @@ class X(Structure): _fields_ = [("a", c_short, 3), ("b", c_short, 14), ("c", c_short, 14)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_short)*3) self.assertEqual(X.a.offset, sizeof(c_short)*0) self.assertEqual(X.b.offset, sizeof(c_short)*1) @@ -287,6 +298,7 @@ def test_mixed_1(self): class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_int, 4)] + self.check_struct(X) if os.name == "nt": self.assertEqual(sizeof(X), sizeof(c_int)*2) else: @@ -296,12 +308,14 @@ def test_mixed_2(self): class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_int, 32)] + self.check_struct(X) self.assertEqual(sizeof(X), alignment(c_int)+sizeof(c_int)) def test_mixed_3(self): class X(Structure): _fields_ = [("a", c_byte, 4), ("b", c_ubyte, 4)] + self.check_struct(X) self.assertEqual(sizeof(X), sizeof(c_byte)) def test_mixed_4(self): @@ -312,6 +326,7 @@ class X(Structure): ("d", c_short, 4), ("e", c_short, 4), ("f", c_int, 24)] + self.check_struct(X) # MSVC does NOT combine c_short and c_int into one field, GCC # does (unless GCC is run with '-mms-bitfields' which # produces code compatible with MSVC). @@ -325,6 +340,7 @@ class X(Structure): _fields_ = [ ('A', c_uint, 1), ('B', c_ushort, 16)] + self.check_struct(X) a = X() a.A = 0 a.B = 1 @@ -335,6 +351,7 @@ class X(Structure): _fields_ = [ ('A', c_ulonglong, 1), ('B', c_uint, 32)] + self.check_struct(X) a = X() a.A = 0 a.B = 1 @@ -348,6 +365,7 @@ class X(Structure): ("A", c_uint32), ('B', c_uint32, 20), ('C', c_uint64, 24)] + self.check_struct(X) self.assertEqual(16, sizeof(X)) def test_mixed_8(self): @@ -357,6 +375,7 @@ class Foo(Structure): ("B", c_uint32, 32), ("C", c_ulonglong, 1), ] + self.check_struct(Foo) class Bar(Structure): _fields_ = [ @@ -364,6 +383,7 @@ class Bar(Structure): ("B", c_uint32), ("C", c_ulonglong, 1), ] + self.check_struct(Bar) self.assertEqual(sizeof(Foo), sizeof(Bar)) def test_mixed_9(self): @@ -372,6 +392,7 @@ class X(Structure): ("A", c_uint8), ("B", c_uint32, 1), ] + self.check_struct(X) if sys.platform == 'win32': self.assertEqual(8, sizeof(X)) else: @@ -385,6 +406,7 @@ class X(Structure): ("A", c_uint32, 1), ("B", c_uint64, 1), ] + self.check_struct(X) if sys.platform == 'win32': self.assertEqual(8, alignment(X)) self.assertEqual(16, sizeof(X)) @@ -399,6 +421,7 @@ class TestStruct(Structure): ("Field1", c_uint32, field_width), ("Field2", c_uint8, 8) ] + self.check_struct(TestStruct) cmd = TestStruct() cmd.Field2 = 1 @@ -442,6 +465,9 @@ class Good(Structure): ("b0", c_uint16, 4), ("b1", c_uint16, 12), ] + self.check_struct(Bad) + self.check_struct(GoodA) + self.check_struct(Good) self.assertEqual(3, sizeof(Bad)) self.assertEqual(3, sizeof(Good)) @@ -461,6 +487,7 @@ class MyStructure(Structure): ("C", c_uint32, 20), ("R2", c_uint32, 2) ] + self.check_struct(MyStructure) self.assertEqual(8, sizeof(MyStructure)) def test_gh_86098(self): @@ -470,6 +497,7 @@ class X(Structure): ("b", c_uint8, 8), ("c", c_uint32, 16) ] + self.check_struct(X) if sys.platform == 'win32': self.assertEqual(8, sizeof(X)) else: @@ -484,9 +512,13 @@ class Y(Structure): _anonymous_ = ["_"] _fields_ = [("_", X)] + self.check_struct(X) + self.check_struct(Y) + def test_uint32(self): class X(Structure): _fields_ = [("a", c_uint32, 32)] + self.check_struct(X) x = X() x.a = 10 self.assertEqual(x.a, 10) @@ -496,6 +528,7 @@ class X(Structure): def test_uint64(self): class X(Structure): _fields_ = [("a", c_uint64, 64)] + self.check_struct(X) x = X() x.a = 10 self.assertEqual(x.a, 10) @@ -508,6 +541,7 @@ class Little(LittleEndianStructure): _fields_ = [("a", c_uint32, 24), ("b", c_uint32, 4), ("c", c_uint32, 4)] + self.check_struct(Little) b = bytearray(4) x = Little.from_buffer(b) x.a = 0xabcdef @@ -521,6 +555,7 @@ class Big(BigEndianStructure): _fields_ = [("a", c_uint32, 24), ("b", c_uint32, 4), ("c", c_uint32, 4)] + self.check_struct(Big) b = bytearray(4) x = Big.from_buffer(b) x.a = 0xabcdef @@ -533,6 +568,7 @@ class BitfieldUnion(Union): _fields_ = [("a", c_uint32, 1), ("b", c_uint32, 2), ("c", c_uint32, 3)] + self.check_union(BitfieldUnion) self.assertEqual(sizeof(BitfieldUnion), 4) b = bytearray(4) x = BitfieldUnion.from_buffer(b) diff --git a/Lib/test/test_ctypes/test_bytes.py b/Lib/test/test_ctypes/test_bytes.py index fa11e1bbd49faf..0e7f81b9482e06 100644 --- a/Lib/test/test_ctypes/test_bytes.py +++ b/Lib/test/test_ctypes/test_bytes.py @@ -3,9 +3,10 @@ import unittest from _ctypes import _SimpleCData from ctypes import Structure, c_char, c_char_p, c_wchar, c_wchar_p +from ._support import StructCheckMixin -class BytesTest(unittest.TestCase): +class BytesTest(unittest.TestCase, StructCheckMixin): def test_c_char(self): x = c_char(b"x") self.assertRaises(TypeError, c_char, "x") @@ -40,6 +41,7 @@ def test_c_wchar_p(self): def test_struct(self): class X(Structure): _fields_ = [("a", c_char * 3)] + self.check_struct(X) x = X(b"abc") self.assertRaises(TypeError, X, "abc") @@ -49,6 +51,7 @@ class X(Structure): def test_struct_W(self): class X(Structure): _fields_ = [("a", c_wchar * 3)] + self.check_struct(X) x = X("abc") self.assertRaises(TypeError, X, b"abc") diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index 78eff0392c4548..072c60d53dd8cb 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -11,6 +11,7 @@ c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong, c_uint32, c_float, c_double) +from ._support import StructCheckMixin def bin(s): @@ -24,15 +25,17 @@ def bin(s): # # For Structures and Unions, these types are created on demand. -class Test(unittest.TestCase): +class Test(unittest.TestCase, StructCheckMixin): def test_slots(self): class BigPoint(BigEndianStructure): __slots__ = () _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(BigPoint) class LowPoint(LittleEndianStructure): __slots__ = () _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(LowPoint) big = BigPoint() little = LowPoint() @@ -200,6 +203,7 @@ def test_struct_fields_unsupported_byte_order(self): with self.assertRaises(TypeError): class T(BigEndianStructure if sys.byteorder == "little" else LittleEndianStructure): _fields_ = fields + [("x", typ)] + self.check_struct(T) def test_struct_struct(self): @@ -219,9 +223,11 @@ def test_struct_struct(self): class NestedStructure(nested): _fields_ = [("x", c_uint32), ("y", c_uint32)] + self.check_struct(NestedStructure) class TestStructure(parent): _fields_ = [("point", NestedStructure)] + self.check_struct(TestStructure) self.assertEqual(len(data), sizeof(TestStructure)) ptr = POINTER(TestStructure) @@ -248,6 +254,7 @@ class S(base): ("h", c_short), ("i", c_int), ("d", c_double)] + self.check_struct(S) s1 = S(0x12, 0x1234, 0x12345678, 3.14) s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14) @@ -271,6 +278,7 @@ class S(base): ("_2", c_byte), ("d", c_double)] + self.check_struct(S) s1 = S() s1.b = 0x12 @@ -298,6 +306,7 @@ class S(Structure): ("_2", c_byte), ("d", c_double)] + self.check_struct(S) s1 = S() s1.b = 0x12 @@ -334,6 +343,7 @@ def test_union_fields_unsupported_byte_order(self): with self.assertRaises(TypeError): class T(BigEndianUnion if sys.byteorder == "little" else LittleEndianUnion): _fields_ = fields + [("x", typ)] + self.check_union(T) def test_union_struct(self): # nested structures in unions with different byteorders @@ -352,9 +362,11 @@ def test_union_struct(self): class NestedStructure(nested): _fields_ = [("x", c_uint32), ("y", c_uint32)] + self.check_struct(NestedStructure) class TestUnion(parent): _fields_ = [("point", NestedStructure)] + self.check_union(TestUnion) self.assertEqual(len(data), sizeof(TestUnion)) ptr = POINTER(TestUnion) @@ -374,12 +386,15 @@ def test_build_struct_union_opposite_system_byteorder(self): class S1(_Structure): _fields_ = [("a", c_byte), ("b", c_byte)] + self.check_struct(S1) class U1(_Union): _fields_ = [("s1", S1), ("ab", c_short)] + self.check_union(U1) class S2(_Structure): _fields_ = [("u1", U1), ("c", c_byte)] + self.check_struct(S2) if __name__ == "__main__": diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index 8362fb16d94dcd..be641da30eadae 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -5,7 +5,7 @@ from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, - Py_TPFLAGS_IMMUTABLETYPE) + Py_TPFLAGS_IMMUTABLETYPE, StructCheckMixin) try: @@ -17,7 +17,7 @@ lib = CDLL(_ctypes_test.__file__) -class CFuncPtrTestCase(unittest.TestCase): +class CFuncPtrTestCase(unittest.TestCase, StructCheckMixin): def test_inheritance_hierarchy(self): self.assertEqual(_CFuncPtr.mro(), [_CFuncPtr, _CData, object]) @@ -88,6 +88,7 @@ class WNDCLASS(Structure): ("hCursor", HCURSOR), ("lpszMenuName", LPCTSTR), ("lpszClassName", LPCTSTR)] + self.check_struct(WNDCLASS) wndclass = WNDCLASS() wndclass.lpfnWndProc = WNDPROC(wndproc) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 1df9f0dc16368f..9a8102219d8769 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -10,16 +10,22 @@ """ import unittest -from test.support import import_helper +from test.support import import_helper, verbose import re from dataclasses import dataclass from functools import cached_property +import sys import ctypes from ctypes import Structure, Union from ctypes import sizeof, alignment, pointer, string_at _ctypes_test = import_helper.import_module("_ctypes_test") +from test.test_ctypes._support import StructCheckMixin + +# A 64-bit number where each nibble (hex digit) is different and +# has 2-3 bits set. +TEST_PATTERN = 0xae7596db # ctypes erases the difference between `c_int` and e.g.`c_int16`. # To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: @@ -426,7 +432,7 @@ class X(Structure): _fields_ = [("_", X), ('y', c_byte)] -class GeneratedTest(unittest.TestCase): +class GeneratedTest(unittest.TestCase, StructCheckMixin): def test_generated_data(self): """Check that a ctypes struct/union matches its C equivalent. @@ -448,6 +454,7 @@ def test_generated_data(self): """ for name, cls in TESTCASES.items(): with self.subTest(name=name): + self.check_struct_or_union(cls) if _maybe_skip := getattr(cls, '_maybe_skip', None): _maybe_skip() expected = iter(_ctypes_test.get_generated_test_data(name)) @@ -461,7 +468,7 @@ def test_generated_data(self): obj = cls() ptr = pointer(obj) for field in iterfields(cls): - for value in -1, 1, 0: + for value in -1, 1, TEST_PATTERN, 0: with self.subTest(field=field.full_name, value=value): field.set_to(obj, value) py_mem = string_at(ptr, sizeof(obj)) @@ -472,6 +479,17 @@ def test_generated_data(self): m = "\n".join([str(field), 'in:', *lines]) self.assertEqual(py_mem.hex(), c_mem.hex(), m) + descriptor = field.descriptor + field_mem = py_mem[ + field.byte_offset + : field.byte_offset + descriptor.byte_size] + field_int = int.from_bytes(field_mem, sys.byteorder) + mask = (1 << descriptor.bit_size) - 1 + self.assertEqual( + (field_int >> descriptor.bit_offset) & mask, + value & mask) + + # The rest of this file is generating C code from a ctypes type. # This is only meant for (and tested with) the known inputs in this file! @@ -569,6 +587,8 @@ class FieldInfo: bits: int | None # number if this is a bit field parent_type: type parent: 'FieldInfo' #| None + descriptor: object + byte_offset: int @cached_property def attr_path(self): @@ -600,10 +620,6 @@ def root(self): else: return self.parent - @cached_property - def descriptor(self): - return getattr(self.parent_type, self.name) - def __repr__(self): qname = f'{self.root.parent_type.__name__}.{self.full_name}' try: @@ -621,7 +637,11 @@ def iterfields(tp, parent=None): else: for fielddesc in fields: f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) - sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + descriptor = getattr(tp, f_name) + byte_offset = descriptor.byte_offset + if parent: + byte_offset += parent.byte_offset + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent, descriptor, byte_offset) yield from iterfields(f_tp, sub) @@ -629,10 +649,9 @@ def iterfields(tp, parent=None): # Dump C source to stdout def output(string): print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output("/* Generated by Lib/test/test_ctypes/test_generated_structs.py */") + output(f"#define TEST_PATTERN {TEST_PATTERN}") output(""" - /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ - - // Append VALUE to the result. #define APPEND(ITEM) { \\ PyObject *item = ITEM; \\ @@ -657,12 +676,13 @@ def output(string): (char*)&value, sizeof(value))); \\ } - // Set a field to -1, 1 and 0; append a snapshot of the memory + // Set a field to test values; append a snapshot of the memory // after each of the operations. - #define TEST_FIELD(TYPE, TARGET) { \\ - SET_AND_APPEND(TYPE, TARGET, -1) \\ - SET_AND_APPEND(TYPE, TARGET, 1) \\ - SET_AND_APPEND(TYPE, TARGET, 0) \\ + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, (TYPE)TEST_PATTERN) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ } #if defined(__GNUC__) || defined(__clang__) diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index aafdb3582f2a25..5c713247a0f418 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -1,12 +1,12 @@ import unittest import sys -from ctypes import Structure, Union, sizeof, c_char, c_int -from ._support import CField, Py_TPFLAGS_IMMUTABLETYPE +from ctypes import Structure, Union, sizeof, c_char, c_int, CField +from ._support import Py_TPFLAGS_IMMUTABLETYPE, StructCheckMixin NOTHING = object() -class FieldsTestBase: +class FieldsTestBase(StructCheckMixin): # Structure/Union classes must get 'finalized' sooner or # later, when one of these things happen: # @@ -78,25 +78,48 @@ class Subclass(BrokenStructure): ... def test_max_field_size_gh126937(self): # Classes for big structs should be created successfully. # (But they most likely can't be instantiated.) - # Here we test the exact limit: the number of *bits* must fit - # in Py_ssize_t. + # The size must fit in Py_ssize_t. - class X(self.cls): + max_field_size = sys.maxsize + + class X(Structure): _fields_ = [('char', c_char),] - max_field_size = sys.maxsize // 8 + self.check_struct(X) - class Y(self.cls): + class Y(Structure): _fields_ = [('largeField', X * max_field_size)] - class Z(self.cls): + self.check_struct(Y) + + class Z(Structure): _fields_ = [('largeField', c_char * max_field_size)] + self.check_struct(Z) - with self.assertRaises(ValueError): - class TooBig(self.cls): + # The *bit* size overflows Py_ssize_t. + self.assertEqual(Y.largeField.bit_size, max_field_size * 8) + self.assertEqual(Z.largeField.bit_size, max_field_size * 8) + + self.assertEqual(Y.largeField.byte_size, max_field_size) + self.assertEqual(Z.largeField.byte_size, max_field_size) + self.assertEqual(sizeof(Y), max_field_size) + self.assertEqual(sizeof(Z), max_field_size) + + with self.assertRaises(OverflowError): + class TooBig(Structure): _fields_ = [('largeField', X * (max_field_size + 1))] - with self.assertRaises(ValueError): - class TooBig(self.cls): + with self.assertRaises(OverflowError): + class TooBig(Structure): _fields_ = [('largeField', c_char * (max_field_size + 1))] + # Also test around edge case for the bit_size calculation + for size in (max_field_size // 8 - 1, + max_field_size // 8, + max_field_size // 8 + 1): + class S(Structure): + _fields_ = [('largeField', c_char * size),] + self.check_struct(S) + self.assertEqual(S.largeField.bit_size, size * 8) + + # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. def test___set__(self): diff --git a/Lib/test/test_ctypes/test_structunion.py b/Lib/test/test_ctypes/test_structunion.py index 973ac3b2f1919d..8d8b7e5e995132 100644 --- a/Lib/test/test_ctypes/test_structunion.py +++ b/Lib/test/test_ctypes/test_structunion.py @@ -1,10 +1,12 @@ """Common tests for ctypes.Structure and ctypes.Union""" import unittest +import sys from ctypes import (Structure, Union, POINTER, sizeof, alignment, c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, - c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) + c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double, + c_int8, c_int16, c_int32) from ._support import (_CData, PyCStructType, UnionType, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -175,6 +177,102 @@ class X(self.cls): # XXX Should we check nested data types also? # offset is always relative to the class... + def test_field_descriptor_attributes(self): + """Test information provided by the descriptors""" + class Inner(Structure): + _fields_ = [ + ("a", c_int16), + ("b", c_int8, 1), + ("c", c_int8, 2), + ] + class X(self.cls): + _fields_ = [ + ("x", c_int32), + ("y", c_int16, 1), + ("_", Inner), + ] + _anonymous_ = ["_"] + + field_names = "xy_abc" + + # name + + for name in field_names: + with self.subTest(name=name): + self.assertEqual(getattr(X, name).name, name) + + # type + + expected_types = dict( + x=c_int32, + y=c_int16, + _=Inner, + a=c_int16, + b=c_int8, + c=c_int8, + ) + assert set(expected_types) == set(field_names) + for name, tp in expected_types.items(): + with self.subTest(name=name): + self.assertEqual(getattr(X, name).type, tp) + self.assertEqual(getattr(X, name).byte_size, sizeof(tp)) + + # offset, byte_offset + + expected_offsets = dict( + x=(0, 0), + y=(0, 4), + _=(0, 6), + a=(0, 6), + b=(2, 8), + c=(2, 8), + ) + assert set(expected_offsets) == set(field_names) + for name, (union_offset, struct_offset) in expected_offsets.items(): + with self.subTest(name=name): + self.assertEqual(getattr(X, name).offset, + getattr(X, name).byte_offset) + if self.cls == Structure: + self.assertEqual(getattr(X, name).offset, struct_offset) + else: + self.assertEqual(getattr(X, name).offset, union_offset) + + # is_bitfield, bit_size, bit_offset + # size + + little_endian = (sys.byteorder == 'little') + expected_bitfield_info = dict( + # (bit_size, bit_offset) + b=(1, 0 if little_endian else 7), + c=(2, 1 if little_endian else 5), + y=(1, 0 if little_endian else 15), + ) + for name in field_names: + with self.subTest(name=name): + if info := expected_bitfield_info.get(name): + self.assertEqual(getattr(X, name).is_bitfield, True) + expected_bit_size, expected_bit_offset = info + self.assertEqual(getattr(X, name).bit_size, + expected_bit_size) + self.assertEqual(getattr(X, name).bit_offset, + expected_bit_offset) + self.assertEqual(getattr(X, name).size, + (expected_bit_size << 16) + | expected_bit_offset) + else: + self.assertEqual(getattr(X, name).is_bitfield, False) + type_size = sizeof(expected_types[name]) + self.assertEqual(getattr(X, name).bit_size, type_size * 8) + self.assertEqual(getattr(X, name).bit_offset, 0) + self.assertEqual(getattr(X, name).size, type_size) + + # is_anonymous + + for name in field_names: + with self.subTest(name=name): + self.assertEqual(getattr(X, name).is_anonymous, (name == '_')) + + def test_invalid_field_types(self): class POINT(self.cls): pass @@ -182,11 +280,19 @@ class POINT(self.cls): def test_invalid_name(self): # field name must be string - def declare_with_name(name): - class S(self.cls): - _fields_ = [(name, c_int)] - - self.assertRaises(TypeError, declare_with_name, b"x") + for name in b"x", 3, None: + with self.subTest(name=name): + with self.assertRaises(TypeError): + class S(self.cls): + _fields_ = [(name, c_int)] + + def test_str_name(self): + class WeirdString(str): + def __str__(self): + return "unwanted value" + class S(self.cls): + _fields_ = [(WeirdString("f"), c_int)] + self.assertEqual(S.f.name, "f") def test_intarray_fields(self): class SomeInts(self.cls): diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 0ec238e04b74cd..67616086b1907c 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -15,15 +15,17 @@ from collections import namedtuple from test import support from test.support import import_helper +from ._support import StructCheckMixin _ctypes_test = import_helper.import_module("_ctypes_test") -class StructureTestCase(unittest.TestCase): +class StructureTestCase(unittest.TestCase, StructCheckMixin): def test_packed(self): class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 1 + self.check_struct(X) self.assertEqual(sizeof(X), 9) self.assertEqual(X.b.offset, 1) @@ -32,6 +34,7 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 2 + self.check_struct(X) self.assertEqual(sizeof(X), 10) self.assertEqual(X.b.offset, 2) @@ -42,6 +45,7 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 4 + self.check_struct(X) self.assertEqual(sizeof(X), min(4, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(4, longlong_align)) @@ -49,6 +53,7 @@ class X(Structure): _fields_ = [("a", c_byte), ("b", c_longlong)] _pack_ = 8 + self.check_struct(X) self.assertEqual(sizeof(X), min(8, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(8, longlong_align)) @@ -89,6 +94,7 @@ class Person(Structure): def test_conflicting_initializers(self): class POINT(Structure): _fields_ = [("phi", c_float), ("rho", c_float)] + self.check_struct(POINT) # conflicting positional and keyword args self.assertRaisesRegex(TypeError, "phi", POINT, 2, 3, phi=4) self.assertRaisesRegex(TypeError, "rho", POINT, 2, 3, rho=4) @@ -99,6 +105,7 @@ class POINT(Structure): def test_keyword_initializers(self): class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + self.check_struct(POINT) pt = POINT(1, 2) self.assertEqual((pt.x, pt.y), (1, 2)) @@ -110,11 +117,13 @@ def test_nested_initializers(self): class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] + self.check_struct(Phone) class Person(Structure): _fields_ = [("name", c_char * 12), ("phone", Phone), ("age", c_int)] + self.check_struct(Person) p = Person(b"Someone", (b"1234", b"5678"), 5) @@ -127,6 +136,7 @@ def test_structures_with_wchar(self): class PersonW(Structure): _fields_ = [("name", c_wchar * 12), ("age", c_int)] + self.check_struct(PersonW) p = PersonW("Someone \xe9") self.assertEqual(p.name, "Someone \xe9") @@ -142,11 +152,13 @@ def test_init_errors(self): class Phone(Structure): _fields_ = [("areacode", c_char*6), ("number", c_char*12)] + self.check_struct(Phone) class Person(Structure): _fields_ = [("name", c_char * 12), ("phone", Phone), ("age", c_int)] + self.check_struct(Person) cls, msg = self.get_except(Person, b"Someone", (1, 2)) self.assertEqual(cls, RuntimeError) @@ -169,12 +181,19 @@ def test_positional_args(self): # see also http://bugs.python.org/issue5042 class W(Structure): _fields_ = [("a", c_int), ("b", c_int)] + self.check_struct(W) + class X(W): _fields_ = [("c", c_int)] + self.check_struct(X) + class Y(X): pass + self.check_struct(Y) + class Z(Y): _fields_ = [("d", c_int), ("e", c_int), ("f", c_int)] + self.check_struct(Z) z = Z(1, 2, 3, 4, 5, 6) self.assertEqual((z.a, z.b, z.c, z.d, z.e, z.f), @@ -193,6 +212,7 @@ class Test(Structure): ('second', c_ulong), ('third', c_ulong), ] + self.check_struct(Test) s = Test() s.first = 0xdeadbeef @@ -222,6 +242,7 @@ class Test(Structure): ] def __del__(self): finalizer_calls.append("called") + self.check_struct(Test) s = Test(1, 2, 3) # Test the StructUnionType_paramfunc() code path which copies the @@ -251,6 +272,7 @@ class X(Structure): ('first', c_uint), ('second', c_uint) ] + self.check_struct(X) s = X() s.first = 0xdeadbeef @@ -339,36 +361,43 @@ class Test2(Structure): _fields_ = [ ('data', c_ubyte * 16), ] + self.check_struct(Test2) class Test3AParent(Structure): _fields_ = [ ('data', c_float * 2), ] + self.check_struct(Test3AParent) class Test3A(Test3AParent): _fields_ = [ ('more_data', c_float * 2), ] + self.check_struct(Test3A) class Test3B(Structure): _fields_ = [ ('data', c_double * 2), ] + self.check_struct(Test3B) class Test3C(Structure): _fields_ = [ ("data", c_double * 4) ] + self.check_struct(Test3C) class Test3D(Structure): _fields_ = [ ("data", c_double * 8) ] + self.check_struct(Test3D) class Test3E(Structure): _fields_ = [ ("data", c_double * 9) ] + self.check_struct(Test3E) # Tests for struct Test2 @@ -467,6 +496,8 @@ class U(Union): ('f2', c_uint16 * 8), ('f3', c_uint32 * 4), ] + self.check_union(U) + u = U() u.f3[0] = 0x01234567 u.f3[1] = 0x89ABCDEF @@ -493,18 +524,21 @@ class Nested1(Structure): ('an_int', c_int), ('another_int', c_int), ] + self.check_struct(Nested1) class Test4(Union): _fields_ = [ ('a_long', c_long), ('a_struct', Nested1), ] + self.check_struct(Test4) class Nested2(Structure): _fields_ = [ ('an_int', c_int), ('a_union', Test4), ] + self.check_struct(Nested2) class Test5(Structure): _fields_ = [ @@ -512,6 +546,7 @@ class Test5(Structure): ('nested', Nested2), ('another_int', c_int), ] + self.check_struct(Test5) test4 = Test4() dll = CDLL(_ctypes_test.__file__) @@ -576,6 +611,7 @@ class Test6(Structure): ('C', c_int, 3), ('D', c_int, 2), ] + self.check_struct(Test6) test6 = Test6() # As these are signed int fields, all are logically -1 due to sign @@ -611,6 +647,8 @@ class Test7(Structure): ('C', c_uint, 3), ('D', c_uint, 2), ] + self.check_struct(Test7) + test7 = Test7() test7.A = 1 test7.B = 3 @@ -634,6 +672,7 @@ class Test8(Union): ('C', c_int, 3), ('D', c_int, 2), ] + self.check_union(Test8) test8 = Test8() with self.assertRaises(TypeError) as ctx: diff --git a/Misc/NEWS.d/next/Library/2025-01-17-17-35-16.gh-issue-128715.tQjo89.rst b/Misc/NEWS.d/next/Library/2025-01-17-17-35-16.gh-issue-128715.tQjo89.rst new file mode 100644 index 00000000000000..5ca6250795a44f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-17-17-35-16.gh-issue-128715.tQjo89.rst @@ -0,0 +1,3 @@ +The class of :class:`~ctypes.Structure`/:class:`~ctypes.Union` field +descriptors is now available as :class:`~ctypes.CField`, and has new +attributes to aid debugging and introspection. diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index d70b33eaa8b515..8bcbc770f21cdf 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -1,6 +1,5 @@ - /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ - - +/* Generated by Lib/test/test_ctypes/test_generated_structs.py */ +#define TEST_PATTERN 2926941915 // Append VALUE to the result. #define APPEND(ITEM) { \ PyObject *item = ITEM; \ @@ -25,12 +24,13 @@ (char*)&value, sizeof(value))); \ } - // Set a field to -1, 1 and 0; append a snapshot of the memory + // Set a field to test values; append a snapshot of the memory // after each of the operations. - #define TEST_FIELD(TYPE, TARGET) { \ - SET_AND_APPEND(TYPE, TARGET, -1) \ - SET_AND_APPEND(TYPE, TARGET, 1) \ - SET_AND_APPEND(TYPE, TARGET, 0) \ + #define TEST_FIELD(TYPE, TARGET) { \ + SET_AND_APPEND(TYPE, TARGET, -1) \ + SET_AND_APPEND(TYPE, TARGET, 1) \ + SET_AND_APPEND(TYPE, TARGET, (TYPE)TEST_PATTERN) \ + SET_AND_APPEND(TYPE, TARGET, 0) \ } #if defined(__GNUC__) || defined(__clang__) diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 6dd6f6ec56d008..ec113e41d16323 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -11,8 +11,6 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_runtime.h" // _Py_ID() -#include - #ifdef MS_WIN32 # include #endif diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index c6b6460126ca90..c652634a137431 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -66,8 +66,6 @@ module _ctypes #include "Python.h" -#include - #ifdef MS_WIN32 #include #include diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 9924d62c0881d1..7086ba2010607d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -10,7 +10,6 @@ #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_call.h" // _PyObject_CallNoArgs() -#include // bool #include #include "ctypes.h" @@ -56,71 +55,56 @@ Py_ssize_t LOW_BIT(Py_ssize_t offset); @classmethod _ctypes.CField.__new__ as PyCField_new + * name: object(subclass_of='&PyUnicode_Type') type as proto: object - size: Py_ssize_t - offset: Py_ssize_t + byte_size: Py_ssize_t + byte_offset: Py_ssize_t index: Py_ssize_t + _internal_use: bool bit_size as bit_size_obj: object = None + bit_offset as bit_offset_obj: object = None [clinic start generated code]*/ static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, - Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, - PyObject *bit_size_obj) -/*[clinic end generated code: output=43649ef9157c5f58 input=3d813f56373c4caa]*/ + Py_ssize_t byte_size, Py_ssize_t byte_offset, + Py_ssize_t index, int _internal_use, + PyObject *bit_size_obj, PyObject *bit_offset_obj) +/*[clinic end generated code: output=3f2885ee4108b6e2 input=b343436e33c0d782]*/ { CFieldObject* self = NULL; - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "size of field %R must not be negative, got %zd", - name, size); + + if (!_internal_use) { + // Do not instantiate outside ctypes, yet. + // The constructor is internal API and may change without warning. + PyErr_Format(PyExc_TypeError, "cannot create %T object", type); goto error; } - // assert: no overflow; - if ((unsigned long long int) size - >= (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8) { + if (byte_size < 0) { PyErr_Format(PyExc_ValueError, - "size of field %R is too big: %zd", name, size); + "byte size of field %R must not be negative, got %zd", + name, byte_size); goto error; } - PyTypeObject *tp = type; - ctypes_state *st = get_module_state_by_class(tp); - self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (!self) { - return NULL; - } - if (PyUnicode_CheckExact(name)) { - self->name = Py_NewRef(name); - } else { - self->name = PyObject_Str(name); - if (!self->name) { - goto error; - } - } - + ctypes_state *st = get_module_state_by_class(type); StgInfo *info; if (PyStgInfo_FromType(st, proto, &info) < 0) { goto error; } if (info == NULL) { PyErr_Format(PyExc_TypeError, - "type of field %R must be a C type", self->name); + "type of field %R must be a C type", name); goto error; } + assert(byte_size == info->size); + Py_ssize_t bitfield_size = 0; + Py_ssize_t bit_offset = 0; if (bit_size_obj != Py_None) { -#ifdef Py_DEBUG - Py_ssize_t bit_size = NUM_BITS(size); - assert(bit_size > 0); - assert(bit_size <= info->size * 8); - // Currently, the bit size is specified redundantly - // in NUM_BITS(size) and bit_size_obj. - // Verify that they match. - assert(PyLong_AsSsize_t(bit_size_obj) == bit_size); -#endif + // It's a bit field! switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: @@ -144,11 +128,67 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, ((PyTypeObject*)proto)->tp_name); goto error; } + + if (byte_size > 100) { + // Bitfields must "live" in a field defined by a ffi type, + // so they're limited to about 8 bytes. + // This check is here to avoid overflow in later checks. + PyErr_Format(PyExc_ValueError, + "bit field %R size too large, got %zd", + name, byte_size); + goto error; + } + bitfield_size = PyLong_AsSsize_t(bit_size_obj); + if ((bitfield_size <= 0) || (bitfield_size > 255)) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "bit size of field %R out of range, got %zd", + name, bitfield_size); + } + goto error; + } + bit_offset = PyLong_AsSsize_t(bit_offset_obj); + if ((bit_offset < 0) || (bit_offset > 255)) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "bit offset of field %R out of range, got %zd", + name, bit_offset); + } + goto error; + } + if ((bitfield_size + bit_offset) > byte_size * 8) { + PyErr_Format( + PyExc_ValueError, + "bit field %R overflows its type (%zd + %zd >= %zd)", + name, bit_offset, byte_size*8); + goto error; + } + } + else { + if (bit_offset_obj != Py_None) { + PyErr_Format( + PyExc_ValueError, + "field %R: bit_offset must be specified if bit_size is", + name); + goto error; + } + } + + self = _CFieldObject_CAST(type->tp_alloc(type, 0)); + if (!self) { + return NULL; + } + self->name = PyUnicode_FromObject(name); + if (!self->name) { + goto error; } + assert(PyUnicode_CheckExact(self->name)); self->proto = Py_NewRef(proto); - self->size = size; - self->offset = offset; + self->byte_size = byte_size; + self->byte_offset = byte_offset; + self->bitfield_size = (uint8_t)bitfield_size; + self->bit_offset = (uint8_t)bit_offset; self->index = index; @@ -192,6 +232,15 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, return NULL; } +static inline Py_ssize_t +_pack_legacy_size(CFieldObject *field) +{ + if (field->bitfield_size) { + Py_ssize_t bit_offset = field->bit_offset; + return (field->bitfield_size << 16) | bit_offset; + } + return field->byte_size; +} static int PyCField_set(PyObject *op, PyObject *inst, PyObject *value) @@ -206,14 +255,14 @@ PyCField_set(PyObject *op, PyObject *inst, PyObject *value) return -1; } dst = _CDataObject_CAST(inst); - ptr = dst->b_ptr + self->offset; + ptr = dst->b_ptr + self->byte_offset; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "can't delete attribute"); return -1; } return PyCData_set(st, inst, self->proto, self->setfunc, value, - self->index, self->size, ptr); + self->index, _pack_legacy_size(self), ptr); } static PyObject * @@ -232,25 +281,110 @@ PyCField_get(PyObject *op, PyObject *inst, PyTypeObject *type) } src = _CDataObject_CAST(inst); return PyCData_get(st, self->proto, self->getfunc, inst, - self->index, self->size, src->b_ptr + self->offset); + self->index, _pack_legacy_size(self), + src->b_ptr + self->byte_offset); } static PyObject * -PyCField_get_offset(PyObject *self, void *data) +PyCField_get_legacy_size(PyObject *self, void *Py_UNUSED(closure)) { - return PyLong_FromSsize_t(_CFieldObject_CAST(self)->offset); + CFieldObject *field = _CFieldObject_CAST(self); + return PyLong_FromSsize_t(_pack_legacy_size(field)); } static PyObject * -PyCField_get_size(PyObject *self, void *data) +PyCField_get_bit_size(PyObject *self, void *Py_UNUSED(closure)) { - return PyLong_FromSsize_t(_CFieldObject_CAST(self)->size); + CFieldObject *field = _CFieldObject_CAST(self); + if (field->bitfield_size) { + return PyLong_FromSsize_t(field->bitfield_size); + } + if (field->byte_size < PY_SSIZE_T_MAX / 8) { + return PyLong_FromSsize_t(field->byte_size * 8); + } + + // If the bit size overflows Py_ssize_t, we don't try fitting it in + // a bigger C type. Use Python ints. + PyObject *byte_size_obj = NULL; + PyObject *eight = NULL; + PyObject *result = NULL; + + byte_size_obj = PyLong_FromSsize_t(field->byte_size); + if (!byte_size_obj) { + goto finally; + } + eight = PyLong_FromLong(8); + if (!eight) { + goto finally; + } + result = PyNumber_Multiply(byte_size_obj, eight); +finally: + Py_XDECREF(byte_size_obj); + Py_XDECREF(eight); + return result; +} + +static PyObject * +PyCField_is_bitfield(PyObject *self, void *Py_UNUSED(closure)) +{ + return PyBool_FromLong(_CFieldObject_CAST(self)->bitfield_size); +} + +static PyObject * +PyCField_is_anonymous(PyObject *self, void *Py_UNUSED(closure)) +{ + return PyBool_FromLong(_CFieldObject_CAST(self)->anonymous); } static PyGetSetDef PyCField_getset[] = { - { "offset", PyCField_get_offset, NULL, PyDoc_STR("offset in bytes of this field") }, - { "size", PyCField_get_size, NULL, PyDoc_STR("size in bytes of this field") }, - { NULL, NULL, NULL, NULL }, + { "size", PyCField_get_legacy_size, NULL, + PyDoc_STR("size in bytes of this field. For bitfields, this is a " + "legacy packed value; use byte_size instead") }, + + { "bit_size", PyCField_get_bit_size, NULL, + PyDoc_STR("size of this field in bits") }, + { "is_bitfield", PyCField_is_bitfield, NULL, + PyDoc_STR("true if this is a bitfield") }, + { "is_anonymous", PyCField_is_anonymous, NULL, + PyDoc_STR("true if this field is anonymous") }, + { NULL }, +}; + +static PyMemberDef PyCField_members[] = { + { "name", + .type = Py_T_OBJECT_EX, + .offset = offsetof(CFieldObject, name), + .flags = Py_READONLY, + .doc = PyDoc_STR("name of this field") }, + { "type", + .type = Py_T_OBJECT_EX, + .offset = offsetof(CFieldObject, proto), + .flags = Py_READONLY, + .doc = PyDoc_STR("type of this field") }, + { "offset", + .type = Py_T_PYSSIZET, + .offset = offsetof(CFieldObject, byte_offset), + .flags = Py_READONLY, + .doc = PyDoc_STR( + "offset in bytes of this field (same as byte_offset)") }, + { "byte_offset", + .type = Py_T_PYSSIZET, + .offset = offsetof(CFieldObject, byte_offset), + .flags = Py_READONLY, + .doc = PyDoc_STR("offset in bytes of this field. " + "For bitfields: excludes bit_offset.") }, + { "byte_size", + .type = Py_T_PYSSIZET, + .offset = offsetof(CFieldObject, byte_size), + .flags = Py_READONLY, + .doc = PyDoc_STR("size of this field in bytes") }, + { "bit_offset", + .type = Py_T_UBYTE, + .offset = offsetof(CFieldObject, bit_offset), + .flags = Py_READONLY, + .doc = PyDoc_STR("additional offset in bits (relative to byte_offset);" + " zero for non-bitfields") }, + { NULL }, }; static int @@ -282,24 +416,27 @@ PyCField_dealloc(PyObject *self) } static PyObject * -PyCField_repr(PyObject *op) +PyCField_repr(PyObject *self) { + CFieldObject *field = _CFieldObject_CAST(self); PyObject *result; - CFieldObject *self = _CFieldObject_CAST(op); - Py_ssize_t bits = NUM_BITS(self->size); - Py_ssize_t size = LOW_BIT(self->size); - const char *name; - - name = ((PyTypeObject *)self->proto)->tp_name; + const char *tp_name = ((PyTypeObject *)field->proto)->tp_name; - if (bits) + if (field->bitfield_size) { result = PyUnicode_FromFormat( - "", - name, self->offset, size, bits); - else + "<%T %R type=%s, ofs=%zd, bit_size=%zd, bit_offset=%zd>", + self, + field->name, tp_name, field->byte_offset, + (Py_ssize_t)field->bitfield_size, + (Py_ssize_t)field->bit_offset); + } + else { result = PyUnicode_FromFormat( - "", - name, self->offset, size); + "<%T %R type=%s, ofs=%zd, size=%zd>", + self, + field->name, tp_name, field->byte_offset, + field->byte_size); + } return result; } @@ -311,13 +448,14 @@ static PyType_Slot cfield_slots[] = { {Py_tp_traverse, PyCField_traverse}, {Py_tp_clear, PyCField_clear}, {Py_tp_getset, PyCField_getset}, + {Py_tp_members, PyCField_members}, {Py_tp_descr_get, PyCField_get}, {Py_tp_descr_set, PyCField_set}, {0, NULL}, }; PyType_Spec cfield_spec = { - .name = "_ctypes.CField", + .name = "ctypes.CField", .basicsize = sizeof(CFieldObject), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), diff --git a/Modules/_ctypes/clinic/cfield.c.h b/Modules/_ctypes/clinic/cfield.c.h index bbf108a1a07b67..3d16139f2d614a 100644 --- a/Modules/_ctypes/clinic/cfield.c.h +++ b/Modules/_ctypes/clinic/cfield.c.h @@ -11,8 +11,9 @@ preserve static PyObject * PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, - Py_ssize_t size, Py_ssize_t offset, Py_ssize_t index, - PyObject *bit_size_obj); + Py_ssize_t byte_size, Py_ssize_t byte_offset, + Py_ssize_t index, int _internal_use, + PyObject *bit_size_obj, PyObject *bit_offset_obj); static PyObject * PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -20,14 +21,14 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 6 + #define NUM_KEYWORDS 8 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(size), &_Py_ID(offset), &_Py_ID(index), &_Py_ID(bit_size), }, + .ob_item = { &_Py_ID(name), &_Py_ID(type), &_Py_ID(byte_size), &_Py_ID(byte_offset), &_Py_ID(index), &_Py_ID(_internal_use), &_Py_ID(bit_size), &_Py_ID(bit_offset), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -36,26 +37,28 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "type", "size", "offset", "index", "bit_size", NULL}; + static const char * const _keywords[] = {"name", "type", "byte_size", "byte_offset", "index", "_internal_use", "bit_size", "bit_offset", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "CField", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[8]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 5; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 6; PyObject *name; PyObject *proto; - Py_ssize_t size; - Py_ssize_t offset; + Py_ssize_t byte_size; + Py_ssize_t byte_offset; Py_ssize_t index; + int _internal_use; PyObject *bit_size_obj = Py_None; + PyObject *bit_offset_obj = Py_None; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 5, /*maxpos*/ 6, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 0, /*maxpos*/ 0, /*minkw*/ 6, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } @@ -75,7 +78,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (ival == -1 && PyErr_Occurred()) { goto exit; } - size = ival; + byte_size = ival; } { Py_ssize_t ival = -1; @@ -87,7 +90,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) if (ival == -1 && PyErr_Occurred()) { goto exit; } - offset = ival; + byte_offset = ival; } { Py_ssize_t ival = -1; @@ -101,14 +104,24 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } index = ival; } + _internal_use = PyObject_IsTrue(fastargs[5]); + if (_internal_use < 0) { + goto exit; + } if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; + } + if (fastargs[6]) { + bit_size_obj = fastargs[6]; + if (!--noptargs) { + goto skip_optional_kwonly; + } } - bit_size_obj = fastargs[5]; -skip_optional_pos: - return_value = PyCField_new_impl(type, name, proto, size, offset, index, bit_size_obj); + bit_offset_obj = fastargs[7]; +skip_optional_kwonly: + return_value = PyCField_new_impl(type, name, proto, byte_size, byte_offset, index, _internal_use, bit_size_obj, bit_offset_obj); exit: return return_value; } -/*[clinic end generated code: output=6b450bdd861571e7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7160ded221fb00ff input=a9049054013a1b77]*/ diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 07049d0968c790..2b8192059a0dc2 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -2,6 +2,8 @@ # include #endif +#include + #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_typeobject.h" // _PyType_GetModuleState() @@ -280,14 +282,36 @@ extern char *_ctypes_get_simple_type_chars(void); typedef struct CFieldObject { PyObject_HEAD - Py_ssize_t offset; - Py_ssize_t size; - Py_ssize_t index; /* Index into CDataObject's - object array */ + + /* byte size & offset + * For bit fields, this identifies a chunk of memory that the bits are + * extracted from. The entire chunk needs to be contained in the enclosing + * struct/union. + * byte_size is the same as the underlying ctype size (and thus it is + * redundant and could be eliminated). + * Note that byte_offset might not be aligned to proto's alignment. + */ + Py_ssize_t byte_offset; + Py_ssize_t byte_size; + + Py_ssize_t index; /* Index into CDataObject's object array */ PyObject *proto; /* underlying ctype; must have StgInfo */ GETFUNC getfunc; /* getter function if proto is NULL */ SETFUNC setfunc; /* setter function if proto is NULL */ - int anonymous; + bool anonymous: 1; + + /* If this is a bit field, bitfield_size must be positive. + * bitfield_size and bit_offset specify the field inside the chunk of + * memory identified by byte_offset & byte_size. + * Otherwise, these are both zero. + * + * Note that for NON-bitfields: + * - `bit_size` (user-facing Python attribute) `is byte_size*8` + * - `bitfield_size` (this) is zero + * Hence the different name. + */ + uint8_t bitfield_size; + uint8_t bit_offset; PyObject *name; /* exact PyUnicode */ } CFieldObject; diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 05239d85c44d2c..65646f0a8f8978 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -120,7 +120,7 @@ MakeFields(PyObject *type, CFieldObject *descr, if (fdescr->anonymous) { int rc = MakeFields(type, fdescr, index + fdescr->index, - offset + fdescr->offset); + offset + fdescr->byte_offset); Py_DECREF(fdescr); if (rc == -1) { Py_DECREF(fieldlist); @@ -135,12 +135,16 @@ MakeFields(PyObject *type, CFieldObject *descr, return -1; } assert(Py_IS_TYPE(new_descr, cfield_tp)); - new_descr->size = fdescr->size; - new_descr->offset = fdescr->offset + offset; + new_descr->byte_size = fdescr->byte_size; + new_descr->byte_offset = fdescr->byte_offset + offset; + new_descr->bitfield_size = fdescr->bitfield_size; + new_descr->bit_offset = fdescr->bit_offset; new_descr->index = fdescr->index + index; new_descr->proto = Py_XNewRef(fdescr->proto); new_descr->getfunc = fdescr->getfunc; new_descr->setfunc = fdescr->setfunc; + new_descr->name = Py_NewRef(fdescr->name); + new_descr->anonymous = fdescr->anonymous; Py_DECREF(fdescr); @@ -198,7 +202,7 @@ MakeAnonFields(PyObject *type) /* descr is in the field descriptor. */ if (-1 == MakeFields(type, (CFieldObject *)descr, ((CFieldObject *)descr)->index, - ((CFieldObject *)descr)->offset)) { + ((CFieldObject *)descr)->byte_offset)) { Py_DECREF(descr); Py_DECREF(anon_names); return -1;