From c9a7699b9f56807060b342417a2a462a5368b51f Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Mon, 28 Oct 2024 15:07:57 +0100 Subject: [PATCH 01/20] feat: extend the struct parser a bit more --- libdestruct/c/ctypes_generic_field.py | 10 +++- libdestruct/c/struct_parser.py | 49 +++++++++++++++++-- libdestruct/common/array/array_field.py | 5 +- libdestruct/common/array/array_impl.py | 17 +++++-- libdestruct/common/enum/enum_field.py | 5 +- libdestruct/common/field.py | 2 + libdestruct/common/ptr/__init__.py | 0 libdestruct/common/{ => ptr}/ptr.py | 0 .../common/{struct => ptr}/ptr_factory.py | 2 +- .../{struct => ptr}/ptr_struct_field.py | 4 +- libdestruct/common/struct/__init__.py | 2 +- .../common/struct/struct_field_inflater.py | 2 +- libdestruct/common/struct/struct_impl.py | 4 ++ 13 files changed, 87 insertions(+), 15 deletions(-) create mode 100644 libdestruct/common/ptr/__init__.py rename libdestruct/common/{ => ptr}/ptr.py (100%) rename libdestruct/common/{struct => ptr}/ptr_factory.py (91%) rename libdestruct/common/{struct => ptr}/ptr_struct_field.py (94%) diff --git a/libdestruct/c/ctypes_generic_field.py b/libdestruct/c/ctypes_generic_field.py index 26ba849..1ff6291 100644 --- a/libdestruct/c/ctypes_generic_field.py +++ b/libdestruct/c/ctypes_generic_field.py @@ -11,6 +11,8 @@ from libdestruct.c.ctypes_generic import _ctypes_generic from libdestruct.common.type_registry import TypeRegistry +registry = TypeRegistry() + def ctypes_type_handler(obj_type: type) -> type[_ctypes_generic]: """Return the ctypes type handler for the given object type. @@ -21,11 +23,15 @@ def ctypes_type_handler(obj_type: type) -> type[_ctypes_generic]: if not issubclass(obj_type, _SimpleCData): raise TypeError(f"Unsupported object type: {obj_type}.") - return type( + typ = type( f"ctypes_{obj_type.__name__}", (_ctypes_generic,), {"backing_type": obj_type, "size": sizeof(obj_type)}, ) + registry.register_mapping(typ, typ) + + return typ + -TypeRegistry().register_type_handler(_SimpleCData, ctypes_type_handler) +registry.register_type_handler(_SimpleCData, ctypes_type_handler) diff --git a/libdestruct/c/struct_parser.py b/libdestruct/c/struct_parser.py index a8bb4b4..aa77e8e 100644 --- a/libdestruct/c/struct_parser.py +++ b/libdestruct/c/struct_parser.py @@ -14,6 +14,8 @@ from pycparser import c_ast, c_parser +from libdestruct.common.array.array_of import array_of +from libdestruct.common.ptr.ptr_factory import ptr_to, ptr_to_self from libdestruct.common.struct import struct if TYPE_CHECKING: @@ -60,7 +62,7 @@ def struct_to_type(struct_node: c_ast.Struct) -> type[struct]: for decl in struct_node.decls: name = decl.name - typ = type_decl_to_type(decl.type) + typ = type_decl_to_type(decl.type, struct_node) fields[name] = typ type_name = struct_node.name if struct_node.name else "anon_struct" @@ -68,11 +70,49 @@ def struct_to_type(struct_node: c_ast.Struct) -> type[struct]: return type(type_name, (struct,), {"__annotations__": fields}) -def type_decl_to_type(decl: c_ast.TypeDecl) -> type[obj]: +def ptr_to_type(ptr: c_ast.PtrDecl, parent: c_ast.Struct | None = None) -> type[obj]: + """Converts a C pointer to a type.""" + if not isinstance(ptr, c_ast.PtrDecl): + raise TypeError("Definition must be a pointer.") + + if not isinstance(ptr.type, c_ast.TypeDecl): + raise TypeError("Definition must be a type declaration.") + + # Special case: this is a pointer to self + # Note that ptr can either be a struct or an identifier. + ptr_name = ptr.type.type.name if isinstance(ptr.type.type, c_ast.Struct) else ptr.type.type.names[0] + if parent and ptr_name == parent.name: + return ptr_to_self() + + typ = type_decl_to_type(ptr.type) + + return ptr_to(typ) + + +def arr_to_type(arr: c_ast.ArrayDecl) -> type[obj]: + """Converts a C array to a type.""" + if not isinstance(arr, c_ast.ArrayDecl): + raise TypeError("Definition must be an array.") + + if not isinstance(arr.type, c_ast.TypeDecl) and not isinstance(arr.type, c_ast.PtrDecl): + raise TypeError("Definition must be a type declaration.") + + typ = ptr_to_type(arr.type) if isinstance(arr.type, c_ast.PtrDecl) else type_decl_to_type(arr.type) + + return array_of(typ, int(arr.dim.value)) + + +def type_decl_to_type(decl: c_ast.TypeDecl, parent: c_ast.Struct | None = None) -> type[obj]: """Converts a C type declaration to a type.""" - if not isinstance(decl, c_ast.TypeDecl): + if not isinstance(decl, c_ast.TypeDecl) and not isinstance(decl, c_ast.PtrDecl) and not isinstance(decl, c_ast.ArrayDecl): raise TypeError("Definition must be a type declaration.") + if isinstance(decl, c_ast.PtrDecl): + return ptr_to_type(decl, parent) + + if isinstance(decl, c_ast.ArrayDecl): + return arr_to_type(decl) + if isinstance(decl.type, c_ast.Struct): return struct_to_type(decl.type) @@ -95,6 +135,9 @@ def to_uniform_name(name: str) -> str: name = name.replace("int32_t", "int") name = name.replace("int64_t", "longlong") + # We have to convert uintptr_t + name = name.replace("uintptr_t", "ulonglong") + # Only size_t, ssize_t and time_t can end with _t if not any(x in name for x in ["size", "ssize", "time"]): name = name.replace("_t", "") diff --git a/libdestruct/common/array/array_field.py b/libdestruct/common/array/array_field.py index 22818f3..2265546 100644 --- a/libdestruct/common/array/array_field.py +++ b/libdestruct/common/array/array_field.py @@ -9,16 +9,19 @@ from abc import abstractmethod from typing import TYPE_CHECKING +from libdestruct.common.array import array from libdestruct.common.field import Field if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver - from libdestruct.common.array import array + from libdestruct.common.obj import obj class ArrayField(Field): """A generator for an array of items.""" + base_type: type[obj] = array + @abstractmethod def inflate(self: ArrayField, resolver: Resolver) -> array: """Inflate the field. diff --git a/libdestruct/common/array/array_impl.py b/libdestruct/common/array/array_impl.py index e69a85c..a5ec192 100644 --- a/libdestruct/common/array/array_impl.py +++ b/libdestruct/common/array/array_impl.py @@ -8,7 +8,9 @@ from typing import TYPE_CHECKING +from libdestruct.c.ctypes_generic import _ctypes_generic from libdestruct.common.array.array import array +from libdestruct.common.field import Field from libdestruct.common.obj import obj from libdestruct.common.struct.struct import struct @@ -16,7 +18,6 @@ from collections.abc import Generator from libdestruct.backing.resolver import Resolver - from libdestruct.common.obj import obj class array_impl(array): @@ -36,7 +37,15 @@ def __init__( self.backing_type = backing_type self._count = count - self.size = self.backing_type.size * self._count + + if hasattr(backing_type, "size"): + self.size = self.backing_type.size * self._count + self.item_size = self.backing_type.size + elif callable(backing_type): + self.size = 8 * self._count + self.item_size = 8 + else: + raise NotImplementedError("Unsupported backing type.") def count(self: array_impl) -> int: """Get the size of the array.""" @@ -44,7 +53,7 @@ def count(self: array_impl) -> int: def get(self: array, index: int) -> object: """Return the element at the given index.""" - return self.backing_type(self.resolver.relative_from_own(index * self.backing_type.size, 0)) + return self.backing_type(self.resolver.relative_from_own(index * self.item_size, 0)) def _set(self: array_impl, _: list[obj]) -> None: """Set the array from a list.""" @@ -64,7 +73,7 @@ def to_str(self: array_impl, indent: int = 0) -> str: return "[]" # If the backing type is a struct, we need to indent the output differently - if issubclass(self.backing_type, struct): + if isinstance(self.backing_type, type) and issubclass(self.backing_type, struct): padding = ",\n" + " " * (indent + 4) spacing = " " * (indent + 4) return "[\n" + spacing + padding.join(x.to_str(indent + 4) for x in self) + "\n" + " " * (indent) + "]" diff --git a/libdestruct/common/enum/enum_field.py b/libdestruct/common/enum/enum_field.py index c9da103..9f057bb 100644 --- a/libdestruct/common/enum/enum_field.py +++ b/libdestruct/common/enum/enum_field.py @@ -9,16 +9,19 @@ from abc import abstractmethod from typing import TYPE_CHECKING +from libdestruct.common.enum import enum from libdestruct.common.field import Field if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver - from libdestruct.common.enum import enum + from libdestruct.common.obj import obj class EnumField(Field): """A generator for an enum.""" + base_type: type[obj] = enum + @abstractmethod def inflate(self: EnumField, resolver: Resolver) -> enum: """Inflate the field. diff --git a/libdestruct/common/field.py b/libdestruct/common/field.py index 94c0ce2..5d0c3e5 100644 --- a/libdestruct/common/field.py +++ b/libdestruct/common/field.py @@ -17,6 +17,8 @@ class Field(ABC): """A generator for a generic field.""" + base_type: type[obj] + @abstractmethod def inflate(self: Field, resolver: Resolver) -> obj: """Inflate the field. diff --git a/libdestruct/common/ptr/__init__.py b/libdestruct/common/ptr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libdestruct/common/ptr.py b/libdestruct/common/ptr/ptr.py similarity index 100% rename from libdestruct/common/ptr.py rename to libdestruct/common/ptr/ptr.py diff --git a/libdestruct/common/struct/ptr_factory.py b/libdestruct/common/ptr/ptr_factory.py similarity index 91% rename from libdestruct/common/struct/ptr_factory.py rename to libdestruct/common/ptr/ptr_factory.py index 7c4189f..6423852 100644 --- a/libdestruct/common/struct/ptr_factory.py +++ b/libdestruct/common/ptr/ptr_factory.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING -from libdestruct.common.struct.ptr_struct_field import PtrStructField +from libdestruct.common.ptr.ptr_struct_field import PtrStructField if TYPE_CHECKING: # pragma: no cover from libdestruct.common.field import Field diff --git a/libdestruct/common/struct/ptr_struct_field.py b/libdestruct/common/ptr/ptr_struct_field.py similarity index 94% rename from libdestruct/common/struct/ptr_struct_field.py rename to libdestruct/common/ptr/ptr_struct_field.py index 36e2cda..f5596ae 100644 --- a/libdestruct/common/struct/ptr_struct_field.py +++ b/libdestruct/common/ptr/ptr_struct_field.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING from libdestruct.common.field import Field -from libdestruct.common.ptr import ptr +from libdestruct.common.ptr.ptr import ptr from libdestruct.common.struct.struct_field import StructField if TYPE_CHECKING: # pragma: no cover @@ -20,6 +20,8 @@ class PtrStructField(StructField): """A generator for a field of a struct.""" + base_type: type[obj] = ptr + def __init__(self: PtrStructField, backing_type: type | Field) -> None: """Initialize a pointer field. diff --git a/libdestruct/common/struct/__init__.py b/libdestruct/common/struct/__init__.py index 721a508..3e53d05 100644 --- a/libdestruct/common/struct/__init__.py +++ b/libdestruct/common/struct/__init__.py @@ -4,7 +4,7 @@ # Licensed under the MIT license. See LICENSE file in the project root for details. # -from libdestruct.common.struct.ptr_factory import ptr_to, ptr_to_self +from libdestruct.common.ptr.ptr_factory import ptr_to, ptr_to_self from libdestruct.common.struct.struct import struct from libdestruct.common.struct.struct_impl import struct_impl diff --git a/libdestruct/common/struct/struct_field_inflater.py b/libdestruct/common/struct/struct_field_inflater.py index d912050..4ee09cc 100644 --- a/libdestruct/common/struct/struct_field_inflater.py +++ b/libdestruct/common/struct/struct_field_inflater.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING -from libdestruct.common.struct.ptr_struct_field import PtrStructField +from libdestruct.common.ptr.ptr_struct_field import PtrStructField from libdestruct.common.type_registry import TypeRegistry if TYPE_CHECKING: # pragma: no cover diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index 6a9c2c3..08f5a62 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING +from libdestruct.common.field import Field from libdestruct.common.obj import obj from libdestruct.common.struct import struct from libdestruct.common.type_registry import TypeRegistry @@ -74,6 +75,9 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: field = getattr(reference_type, name) attribute = cls._inflater.inflater_for((field, annotation))(None) size += attribute.size + elif isinstance(annotation, Field): + attribute = cls._inflater.inflater_for((annotation, annotation.base_type))(None) + size += attribute.size else: attribute = cls._inflater.inflater_for(annotation) size += attribute.size From f8f340823d5409a9b93816515343eeab4bc62c51 Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 28 Oct 2024 16:43:14 +0100 Subject: [PATCH 02/20] feat: implement size_of to make nested field chains work --- libdestruct/c/__init__.py | 2 +- libdestruct/common/array/array_impl.py | 19 ++++------- .../common/array/linear_array_field.py | 7 +++- libdestruct/common/enum/int_enum_field.py | 4 +++ libdestruct/common/field.py | 6 +++- libdestruct/common/ptr/ptr_factory.py | 10 +++--- .../ptr/{ptr_struct_field.py => ptr_field.py} | 11 ++++--- .../ptr_field_inflater.py} | 6 ++-- libdestruct/common/struct/__init__.py | 2 +- libdestruct/common/struct/struct_field.py | 28 ---------------- libdestruct/common/utils.py | 33 +++++++++++++++++++ 11 files changed, 72 insertions(+), 56 deletions(-) rename libdestruct/common/ptr/{ptr_struct_field.py => ptr_field.py} (78%) rename libdestruct/common/{struct/struct_field_inflater.py => ptr/ptr_field_inflater.py} (86%) delete mode 100644 libdestruct/common/struct/struct_field.py create mode 100644 libdestruct/common/utils.py diff --git a/libdestruct/c/__init__.py b/libdestruct/c/__init__.py index bf84b5d..9f81aee 100644 --- a/libdestruct/c/__init__.py +++ b/libdestruct/c/__init__.py @@ -9,5 +9,5 @@ __all__ = ["c_char", "c_uchar", "c_short", "c_ushort", "c_int", "c_uint", "c_long", "c_ulong", "c_str"] -import libdestruct.c.base_type_inflater # noqa: F401 +import libdestruct.c.base_type_inflater import libdestruct.c.ctypes_generic_field # noqa: F401 diff --git a/libdestruct/common/array/array_impl.py b/libdestruct/common/array/array_impl.py index a5ec192..9f17249 100644 --- a/libdestruct/common/array/array_impl.py +++ b/libdestruct/common/array/array_impl.py @@ -8,16 +8,15 @@ from typing import TYPE_CHECKING -from libdestruct.c.ctypes_generic import _ctypes_generic from libdestruct.common.array.array import array -from libdestruct.common.field import Field -from libdestruct.common.obj import obj from libdestruct.common.struct.struct import struct +from libdestruct.common.utils import size_of if TYPE_CHECKING: # pragma: no cover from collections.abc import Generator from libdestruct.backing.resolver import Resolver + from libdestruct.common.obj import obj class array_impl(array): @@ -26,6 +25,9 @@ class array_impl(array): size: int """The size of the array.""" + item_size: int + """The size of each item in the array.""" + def __init__( self: array_impl, resolver: Resolver, @@ -37,15 +39,8 @@ def __init__( self.backing_type = backing_type self._count = count - - if hasattr(backing_type, "size"): - self.size = self.backing_type.size * self._count - self.item_size = self.backing_type.size - elif callable(backing_type): - self.size = 8 * self._count - self.item_size = 8 - else: - raise NotImplementedError("Unsupported backing type.") + self.item_size = size_of(self.backing_type) + self.size = self.item_size * self._count def count(self: array_impl) -> int: """Get the size of the array.""" diff --git a/libdestruct/common/array/linear_array_field.py b/libdestruct/common/array/linear_array_field.py index 0d2f340..4fa4501 100644 --- a/libdestruct/common/array/linear_array_field.py +++ b/libdestruct/common/array/linear_array_field.py @@ -10,8 +10,9 @@ from libdestruct.common.array.array_field import ArrayField from libdestruct.common.array.array_impl import array_impl +from libdestruct.common.utils import size_of -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver from libdestruct.common.array.array import array from libdestruct.common.obj import obj @@ -32,3 +33,7 @@ def inflate(self: LinearArrayField, resolver: Resolver) -> array: resolver: The backing resolver for the object. """ return array_impl(resolver, self.item, self.size) + + def get_size(self: LinearArrayField) -> int: + """Returns the size of the object inflated by this field.""" + return size_of(self.item) * self.size diff --git a/libdestruct/common/enum/int_enum_field.py b/libdestruct/common/enum/int_enum_field.py index 8958443..f19a5db 100644 --- a/libdestruct/common/enum/int_enum_field.py +++ b/libdestruct/common/enum/int_enum_field.py @@ -54,3 +54,7 @@ def inflate(self: IntEnumField, resolver: Resolver) -> int: resolver: The backing resolver for the object. """ return enum(resolver, self.enum, self.backing_type, self.lenient) + + def get_size(self: IntEnumField) -> int: + """Returns the size of the object inflated by this field.""" + return self.backing_type.size diff --git a/libdestruct/common/field.py b/libdestruct/common/field.py index 5d0c3e5..54bb0c5 100644 --- a/libdestruct/common/field.py +++ b/libdestruct/common/field.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver from libdestruct.common.obj import obj @@ -26,3 +26,7 @@ def inflate(self: Field, resolver: Resolver) -> obj: Args: resolver: The backing resolver for the object. """ + + @abstractmethod + def get_size(self: Field) -> int: + """Returns the size of the object inflated by this field.""" diff --git a/libdestruct/common/ptr/ptr_factory.py b/libdestruct/common/ptr/ptr_factory.py index 6423852..ca18c59 100644 --- a/libdestruct/common/ptr/ptr_factory.py +++ b/libdestruct/common/ptr/ptr_factory.py @@ -8,18 +8,18 @@ from typing import TYPE_CHECKING -from libdestruct.common.ptr.ptr_struct_field import PtrStructField +from libdestruct.common.ptr.ptr_field import PtrField if TYPE_CHECKING: # pragma: no cover from libdestruct.common.field import Field from libdestruct.common.obj import obj -def ptr_to(item: obj | Field) -> PtrStructField: +def ptr_to(item: obj | Field) -> PtrField: """Crafts a struct member which is a pointer to an object.""" - return PtrStructField(item) + return PtrField(item) -def ptr_to_self() -> PtrStructField: +def ptr_to_self() -> PtrField: """Crafts a struct member which is a pointer to a struct of the same type.""" - return PtrStructField(None) + return PtrField(None) diff --git a/libdestruct/common/ptr/ptr_struct_field.py b/libdestruct/common/ptr/ptr_field.py similarity index 78% rename from libdestruct/common/ptr/ptr_struct_field.py rename to libdestruct/common/ptr/ptr_field.py index f5596ae..041021a 100644 --- a/libdestruct/common/ptr/ptr_struct_field.py +++ b/libdestruct/common/ptr/ptr_field.py @@ -10,19 +10,18 @@ from libdestruct.common.field import Field from libdestruct.common.ptr.ptr import ptr -from libdestruct.common.struct.struct_field import StructField if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver from libdestruct.common.obj import obj -class PtrStructField(StructField): +class PtrField(Field): """A generator for a field of a struct.""" base_type: type[obj] = ptr - def __init__(self: PtrStructField, backing_type: type | Field) -> None: + def __init__(self: PtrField, backing_type: type | Field) -> None: """Initialize a pointer field. Args: @@ -30,7 +29,7 @@ def __init__(self: PtrStructField, backing_type: type | Field) -> None: """ self.backing_type = backing_type - def inflate(self: PtrStructField, resolver: Resolver) -> obj: + def inflate(self: PtrField, resolver: Resolver) -> obj: """Inflate the field. Args: @@ -40,3 +39,7 @@ def inflate(self: PtrStructField, resolver: Resolver) -> obj: return ptr(resolver, self.backing_type.inflate) return ptr(resolver, self.backing_type) + + def get_size(self: PtrField) -> int: + """Returns the size of the object inflated by this field.""" + return ptr.size diff --git a/libdestruct/common/struct/struct_field_inflater.py b/libdestruct/common/ptr/ptr_field_inflater.py similarity index 86% rename from libdestruct/common/struct/struct_field_inflater.py rename to libdestruct/common/ptr/ptr_field_inflater.py index 4ee09cc..2241741 100644 --- a/libdestruct/common/struct/struct_field_inflater.py +++ b/libdestruct/common/ptr/ptr_field_inflater.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING -from libdestruct.common.ptr.ptr_struct_field import PtrStructField +from libdestruct.common.ptr.ptr_field import PtrField from libdestruct.common.type_registry import TypeRegistry if TYPE_CHECKING: # pragma: no cover @@ -21,7 +21,7 @@ def ptr_field_inflater( - field: PtrStructField, + field: PtrField, _: type[obj], owner: tuple[obj, type[obj]] | None, ) -> Callable[[Resolver], obj]: @@ -37,4 +37,4 @@ def ptr_field_inflater( return field.inflate -registry.register_instance_handler(PtrStructField, ptr_field_inflater) +registry.register_instance_handler(PtrField, ptr_field_inflater) diff --git a/libdestruct/common/struct/__init__.py b/libdestruct/common/struct/__init__.py index 3e53d05..e7733b6 100644 --- a/libdestruct/common/struct/__init__.py +++ b/libdestruct/common/struct/__init__.py @@ -10,5 +10,5 @@ __all__ = ["struct", "struct_impl", "ptr_to", "ptr_to_self"] -import libdestruct.common.struct.struct_field_inflater +import libdestruct.common.ptr.ptr_field_inflater import libdestruct.common.struct.struct_inflater # noqa: F401 diff --git a/libdestruct/common/struct/struct_field.py b/libdestruct/common/struct/struct_field.py deleted file mode 100644 index 007f42f..0000000 --- a/libdestruct/common/struct/struct_field.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). -# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for details. -# - -from __future__ import annotations - -from abc import abstractmethod -from typing import TYPE_CHECKING - -from libdestruct.common.field import Field - -if TYPE_CHECKING: # pragma: no cover - from libdestruct.backing.resolver import Resolver - from libdestruct.common.obj import obj - - -class StructField(Field): - """A generator for a field of a struct.""" - - @abstractmethod - def inflate(self: StructField, resolver: Resolver) -> obj: - """Inflate the field. - - Args: - resolver: The backing resolver for the object. - """ diff --git a/libdestruct/common/utils.py b/libdestruct/common/utils.py new file mode 100644 index 0000000..f8b4a42 --- /dev/null +++ b/libdestruct/common/utils.py @@ -0,0 +1,33 @@ +# +# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). +# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for details. +# + +from __future__ import annotations + +from types import MethodType +from typing import TYPE_CHECKING + +from libdestruct.common.field import Field + +if TYPE_CHECKING: # pragma: no cover + from libdestruct.backing.resolver import Resolver + from libdestruct.common.obj import obj + + +def size_of(item_or_inflater: obj | callable[[Resolver], obj]) -> int: + """Return the size of an object, from an obj or it's inflater.""" + if hasattr(item_or_inflater, "size"): + return item_or_inflater.size + + # Check if item is the bound method of a Field + if not isinstance(item_or_inflater, MethodType): + raise TypeError("Provided inflater is not the bound method of a Field object") + + field_object = item_or_inflater.__self__ + + if not isinstance(field_object, Field): + raise TypeError("Provided inflater is not the bound method of a Field object") + + return field_object.get_size() From dd253dad9b4a88b76d64536eda44a654b0c2f73a Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 28 Oct 2024 16:45:00 +0100 Subject: [PATCH 03/20] ci: upgrade test runner to Python 3.13 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 08121ed..d8ae703 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - python-version: ["3.10", "3.12"] + python-version: ["3.10", "3.13"] steps: - uses: actions/checkout@v4 From 108848ff63cdce7e732b3034826a40dc0c8fe897 Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 28 Oct 2024 17:50:29 +0100 Subject: [PATCH 04/20] feat: allow typedefs and multiple structs in definition_to_type --- libdestruct/c/struct_parser.py | 56 +++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/libdestruct/c/struct_parser.py b/libdestruct/c/struct_parser.py index aa77e8e..92dc12b 100644 --- a/libdestruct/c/struct_parser.py +++ b/libdestruct/c/struct_parser.py @@ -22,6 +22,12 @@ from libdestruct.common.obj import obj +PARSED_STRUCT = {} +"""A cache for parsed struct definitions, indexed by name.""" + +TYPEDEFS = {} +"""A cache for parsed type definitions, indexed by name.""" + def definition_to_type(definition: str) -> type[obj]: """Converts a C struct definition to a struct object.""" parser = c_parser.CParser() @@ -29,27 +35,31 @@ def definition_to_type(definition: str) -> type[obj]: # If the definition contains includes, we must expand them. if "#include" in definition: definition = cleanup_attributes(expand_includes(definition)) - force_more_tops = True - elif "typedef" in definition: - force_more_tops = True - else: - force_more_tops = False try: ast = parser.parse(definition) except c_parser.ParseError as e: raise ValueError("Invalid definition. Please add the necessary includes if using non-standard type definitions.") from e - if not force_more_tops and len(ast.ext) != 1: - raise ValueError("Definition must contain exactly one top object.") - - # If force_more_tops is True, we take the last top object. - # This is useful when a struct definition is preceded by typedefs. - root = ast.ext[-1].type if force_more_tops else ast.ext[0].type + # We assume that the root declaration is the last one. + root = ast.ext[-1].type if not isinstance(root, c_ast.Struct): raise TypeError("Definition must be a struct.") + # We parse each declaration in the definition, except the last one, if it is a struct. + for decl in ast.ext[:-1]: + if isinstance(decl.type, c_ast.Struct): + struct_node = decl.type + + if struct_node.name: + PARSED_STRUCT[struct_node.name] = struct_to_type(struct_node) + elif isinstance(decl, c_ast.Typedef): + name, definition = typedef_to_pair(decl) + TYPEDEFS[name] = definition + + print(TYPEDEFS) + return struct_to_type(root) @@ -60,6 +70,12 @@ def struct_to_type(struct_node: c_ast.Struct) -> type[struct]: fields = {} + if not struct_node.decls and struct_node.name in PARSED_STRUCT: + # We can check if the struct is already parsed. + return PARSED_STRUCT[struct_node.name] + elif not struct_node.decls: + raise ValueError("Struct must have fields.") + for decl in struct_node.decls: name = decl.name typ = type_decl_to_type(decl.type, struct_node) @@ -122,6 +138,20 @@ def type_decl_to_type(decl: c_ast.TypeDecl, parent: c_ast.Struct | None = None) raise TypeError("Unsupported type.") +def typedef_to_pair(typedef: c_ast.Typedef) -> tuple[str, type[obj]]: + """Converts a C typedef to a pair of name and definition.""" + if not isinstance(typedef, c_ast.Typedef): + raise TypeError("Definition must be a typedef.") + + if not isinstance(typedef.type, c_ast.TypeDecl): + raise TypeError("Definition must be a type declaration.") + + name = "".join(typedef.name) + definition = type_decl_to_type(typedef.type) + + return name, definition + + def to_uniform_name(name: str) -> str: """Converts a name to a uniform name.""" name = name.replace("unsigned", "u") @@ -182,4 +212,8 @@ def identifier_to_type(identifier: c_ast.IdentifierType) -> type[obj]: if hasattr(ctypes, ctypes_name): return getattr(ctypes, ctypes_name) + # Check if we have a typedef to resolve this + if identifier_name in TYPEDEFS: + return TYPEDEFS[identifier_name] + raise ValueError(f"Unsupported identifier: {identifier_name}.") From 6468abac3654078159342ddff215b8182dcba928 Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 28 Oct 2024 17:54:15 +0100 Subject: [PATCH 05/20] fix: remove indentation when inline-printing ptrs --- libdestruct/common/obj.py | 2 +- libdestruct/common/ptr/ptr.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libdestruct/common/obj.py b/libdestruct/common/obj.py index 75ca2d0..82c03ba 100644 --- a/libdestruct/common/obj.py +++ b/libdestruct/common/obj.py @@ -108,7 +108,7 @@ def value(self: obj, value: object) -> None: self._set(value) - def to_str(self: obj, indent: int = 0) -> str: + def to_str(self: obj, _: int = 0) -> str: """Return a string representation of the object.""" return f"{self.get()}" diff --git a/libdestruct/common/ptr/ptr.py b/libdestruct/common/ptr/ptr.py index 4d68c7e..b857a44 100644 --- a/libdestruct/common/ptr/ptr.py +++ b/libdestruct/common/ptr/ptr.py @@ -82,10 +82,10 @@ def try_unwrap(self: ptr, length: int | None = None) -> obj | None: return self.unwrap(length) - def to_str(self: ptr, indent: int = 0) -> str: + def to_str(self: ptr, _: int = 0) -> str: """Return a string representation of the pointer.""" if not self.wrapper: - return f"{' ' * indent}ptr@0x{self.get():x}" + return f"ptr@0x{self.get():x}" # Pretty print inflaters: if callable(self.wrapper) and hasattr(self.wrapper, "__self__") and isinstance(self.wrapper.__self__, Field): @@ -93,7 +93,7 @@ def to_str(self: ptr, indent: int = 0) -> str: else: name = self.wrapper.__name__ - return f"{' ' * indent}{name}@0x{self.get():x}" + return f"{name}@0x{self.get():x}" def __str__(self: ptr) -> str: """Return a string representation of the pointer.""" From 3811b9b1c63c37493b6b2ed1f9035ee24a726015 Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Tue, 29 Oct 2024 15:02:43 +0100 Subject: [PATCH 06/20] fix: remove erroneous print and provide owner when inflating struct members --- libdestruct/c/struct_parser.py | 2 -- libdestruct/common/struct/struct_impl.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libdestruct/c/struct_parser.py b/libdestruct/c/struct_parser.py index 92dc12b..33a2330 100644 --- a/libdestruct/c/struct_parser.py +++ b/libdestruct/c/struct_parser.py @@ -58,8 +58,6 @@ def definition_to_type(definition: str) -> type[obj]: name, definition = typedef_to_pair(decl) TYPEDEFS[name] = definition - print(TYPEDEFS) - return struct_to_type(root) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index 08f5a62..39fb824 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -57,7 +57,7 @@ def _inflate_struct_attributes( field = getattr(reference_type, name) resolved_type = inflater.inflater_for((field, annotation), owner=(self, reference_type._type_impl)) else: - resolved_type = inflater.inflater_for(annotation) + resolved_type = inflater.inflater_for(annotation, owner=(self, reference_type._type_impl)) result = resolved_type(resolver.relative_from_own(current_offset, 0)) setattr(self, name, result) From 686135d389f56fb36c3dc8756d9738cd7f28fec1 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 29 Oct 2024 21:45:37 +0100 Subject: [PATCH 07/20] style: small refactor in struct_impl.py --- libdestruct/common/struct/struct_impl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index 39fb824..a689062 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -74,13 +74,12 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: # Field associated with the annotation field = getattr(reference_type, name) attribute = cls._inflater.inflater_for((field, annotation))(None) - size += attribute.size elif isinstance(annotation, Field): attribute = cls._inflater.inflater_for((annotation, annotation.base_type))(None) - size += attribute.size else: attribute = cls._inflater.inflater_for(annotation) - size += attribute.size + + size += attribute.size cls.size = size From fa987c9271fc9bb264a3e0e6bb95b44516dca402 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 29 Oct 2024 22:29:19 +0100 Subject: [PATCH 08/20] style: fix typing in struct_impl --- libdestruct/common/struct/struct_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index a689062..0a47890 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -95,7 +95,7 @@ def _set(self: struct_impl, _: str) -> None: """Set the value of the struct to the given value.""" raise RuntimeError("Cannot set the value of a struct.") - def freeze(self: obj) -> None: + def freeze(self: struct_impl) -> None: """Freeze the struct.""" # The struct has no implicit value, but it must freeze its members for member in self._members.values(): From 304044a78a2ba52c1abbfa7104105993f2500e82 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 29 Oct 2024 23:02:14 +0100 Subject: [PATCH 09/20] feat: implement a way to specify offsets in a struct --- libdestruct/__init__.py | 4 +- libdestruct/common/__init__.py | 4 -- libdestruct/common/attribute.py | 11 +++ libdestruct/common/attributes/__init__.py | 9 +++ libdestruct/common/attributes/offset.py | 14 ++++ .../common/attributes/offset_attribute.py | 19 ++++++ libdestruct/common/struct/struct_impl.py | 67 ++++++++++++++++--- libdestruct/common/utils.py | 17 ++--- 8 files changed, 123 insertions(+), 22 deletions(-) create mode 100644 libdestruct/common/attribute.py create mode 100644 libdestruct/common/attributes/__init__.py create mode 100644 libdestruct/common/attributes/offset.py create mode 100644 libdestruct/common/attributes/offset_attribute.py diff --git a/libdestruct/__init__.py b/libdestruct/__init__.py index 52c8dc9..8c1b630 100644 --- a/libdestruct/__init__.py +++ b/libdestruct/__init__.py @@ -12,15 +12,17 @@ pass from libdestruct.c import c_int, c_long, c_str, c_uint, c_ulong -from libdestruct.common import ptr from libdestruct.common.array import array, array_of +from libdestruct.common.attributes import offset from libdestruct.common.enum import enum, enum_of +from libdestruct.common.ptr import ptr from libdestruct.common.struct import ptr_to, ptr_to_self, struct from libdestruct.libdestruct import inflater __all__ = [ "array", "array_of", + "offset", "c_int", "c_long", "c_str", diff --git a/libdestruct/common/__init__.py b/libdestruct/common/__init__.py index f07c024..6a0982b 100644 --- a/libdestruct/common/__init__.py +++ b/libdestruct/common/__init__.py @@ -3,7 +3,3 @@ # Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. # Licensed under the MIT license. See LICENSE file in the project root for details. # - -from libdestruct.common.ptr import ptr - -__all__ = ["ptr"] diff --git a/libdestruct/common/attribute.py b/libdestruct/common/attribute.py new file mode 100644 index 0000000..6ff2edc --- /dev/null +++ b/libdestruct/common/attribute.py @@ -0,0 +1,11 @@ +# +# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). +# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for details. +# + +from abc import ABC + + +class Attribute(ABC): + """Base class for all attributes, which are used to describe fields in a struct.""" diff --git a/libdestruct/common/attributes/__init__.py b/libdestruct/common/attributes/__init__.py new file mode 100644 index 0000000..6b72ace --- /dev/null +++ b/libdestruct/common/attributes/__init__.py @@ -0,0 +1,9 @@ +# +# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). +# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for details. +# + +from libdestruct.common.attributes.offset import offset + +__all__ = ["offset"] diff --git a/libdestruct/common/attributes/offset.py b/libdestruct/common/attributes/offset.py new file mode 100644 index 0000000..a36a164 --- /dev/null +++ b/libdestruct/common/attributes/offset.py @@ -0,0 +1,14 @@ +# +# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). +# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for details. +# + +from __future__ import annotations + +from libdestruct.common.attributes.offset_attribute import OffsetAttribute + + +def offset(offset: int) -> OffsetAttribute: + """Create an offset field.""" + return OffsetAttribute(offset) diff --git a/libdestruct/common/attributes/offset_attribute.py b/libdestruct/common/attributes/offset_attribute.py new file mode 100644 index 0000000..3aa23ae --- /dev/null +++ b/libdestruct/common/attributes/offset_attribute.py @@ -0,0 +1,19 @@ +# +# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). +# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for details. +# + +from __future__ import annotations + +from libdestruct.common.attribute import Attribute + + +class OffsetAttribute(Attribute): + """A field that represents an offset in a struct.""" + + offset: int + + def __init__(self: OffsetAttribute, offset: int) -> None: + """Initialize the offset field.""" + self.offset = offset diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index 0a47890..dd58988 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -8,12 +8,13 @@ from typing import TYPE_CHECKING +from libdestruct.common.attributes.offset_attribute import OffsetAttribute from libdestruct.common.field import Field from libdestruct.common.obj import obj from libdestruct.common.struct import struct from libdestruct.common.type_registry import TypeRegistry -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver @@ -54,8 +55,35 @@ def _inflate_struct_attributes( for name, annotation in reference_type.__annotations__.items(): if name in reference_type.__dict__: # Field associated with the annotation - field = getattr(reference_type, name) - resolved_type = inflater.inflater_for((field, annotation), owner=(self, reference_type._type_impl)) + attrs = getattr(reference_type, name) + + # If attrs is not a tuple, we need to convert it to a tuple + if not isinstance(attrs, tuple): + attrs = (attrs,) + + # Assert that in all attributes, there is only one Field + if sum(isinstance(attr, Field) for attr in attrs) > 1: + raise ValueError("Only one Field is allowed per attribute.") + + resolved_type = None + + for attr in attrs: + if isinstance(attr, Field): + resolved_type = inflater.inflater_for( + (attr, annotation), + owner=(self, reference_type._type_impl), + ) + elif isinstance(attr, OffsetAttribute): + offset = attr.offset + if offset < current_offset: + raise ValueError("Offset must be greater than the current size.") + current_offset = offset + else: + raise TypeError("Only Field and OffsetAttribute are allowed in attributes.") + + # If we don't have a Field, we need to inflate the type as if we have no attributes + if not resolved_type: + resolved_type = inflater.inflater_for(annotation, owner=(self, reference_type._type_impl)) else: resolved_type = inflater.inflater_for(annotation, owner=(self, reference_type._type_impl)) @@ -72,8 +100,32 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: for name, annotation in reference_type.__annotations__.items(): if name in reference_type.__dict__: # Field associated with the annotation - field = getattr(reference_type, name) - attribute = cls._inflater.inflater_for((field, annotation))(None) + attrs = getattr(reference_type, name) + + # If attrs is not a tuple, we need to convert it to a tuple + if not isinstance(attrs, tuple): + attrs = (attrs,) + + # Assert that in all attributes, there is only one Field + if sum(isinstance(attr, Field) for attr in attrs) > 1: + raise ValueError("Only one Field is allowed per attribute.") + + attribute = None + + for attr in attrs: + if isinstance(attr, Field): + attribute = cls._inflater.inflater_for((attr, annotation))(None) + elif isinstance(attr, OffsetAttribute): + offset = attr.offset + if offset < size: + raise ValueError("Offset must be greater than the current size.") + size = offset + else: + raise TypeError("Only Field and OffsetAttribute are allowed in attributes.") + + # If we don't have a Field, we need to inflate the attribute as if we have no attributes + if not attribute: + attribute = cls._inflater.inflater_for(annotation) elif isinstance(annotation, Field): attribute = cls._inflater.inflater_for((annotation, annotation.base_type))(None) else: @@ -106,10 +158,7 @@ def freeze(self: struct_impl) -> None: def to_str(self: struct_impl, indent: int = 0) -> str: """Return a string representation of the struct.""" members = ",\n".join( - [ - f"{' ' * (indent + 4)}{name}: {member.to_str(indent + 4)}" - for name, member in self._members.items() - ], + [f"{' ' * (indent + 4)}{name}: {member.to_str(indent + 4)}" for name, member in self._members.items()], ) return f"""{self.name} {{ {members} diff --git a/libdestruct/common/utils.py b/libdestruct/common/utils.py index f8b4a42..586468c 100644 --- a/libdestruct/common/utils.py +++ b/libdestruct/common/utils.py @@ -16,18 +16,19 @@ from libdestruct.common.obj import obj +def is_field_bound_method(item: obj) -> bool: + """Check if the provided item is the bound method of a Field object.""" + return isinstance(item, MethodType) and isinstance(item.__self__, Field) + + def size_of(item_or_inflater: obj | callable[[Resolver], obj]) -> int: """Return the size of an object, from an obj or it's inflater.""" if hasattr(item_or_inflater, "size"): return item_or_inflater.size # Check if item is the bound method of a Field - if not isinstance(item_or_inflater, MethodType): - raise TypeError("Provided inflater is not the bound method of a Field object") - - field_object = item_or_inflater.__self__ - - if not isinstance(field_object, Field): - raise TypeError("Provided inflater is not the bound method of a Field object") + if is_field_bound_method(item_or_inflater): + field_object = item_or_inflater.__self__ + return field_object.get_size() - return field_object.get_size() + raise ValueError(f"Cannot determine the size of {item_or_inflater}") From 226886221c2f7ab018dd9bcf9c0ad364ac0200ea Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 29 Oct 2024 23:25:16 +0100 Subject: [PATCH 10/20] fix: save last result in PARSED_STRUCTS --- libdestruct/c/struct_parser.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libdestruct/c/struct_parser.py b/libdestruct/c/struct_parser.py index 33a2330..a854f26 100644 --- a/libdestruct/c/struct_parser.py +++ b/libdestruct/c/struct_parser.py @@ -22,7 +22,7 @@ from libdestruct.common.obj import obj -PARSED_STRUCT = {} +PARSED_STRUCTS = {} """A cache for parsed struct definitions, indexed by name.""" TYPEDEFS = {} @@ -53,12 +53,16 @@ def definition_to_type(definition: str) -> type[obj]: struct_node = decl.type if struct_node.name: - PARSED_STRUCT[struct_node.name] = struct_to_type(struct_node) + PARSED_STRUCTS[struct_node.name] = struct_to_type(struct_node) elif isinstance(decl, c_ast.Typedef): name, definition = typedef_to_pair(decl) TYPEDEFS[name] = definition - return struct_to_type(root) + result = struct_to_type(root) + + PARSED_STRUCTS[root.name] = result + + return result def struct_to_type(struct_node: c_ast.Struct) -> type[struct]: @@ -68,9 +72,9 @@ def struct_to_type(struct_node: c_ast.Struct) -> type[struct]: fields = {} - if not struct_node.decls and struct_node.name in PARSED_STRUCT: + if not struct_node.decls and struct_node.name in PARSED_STRUCTS: # We can check if the struct is already parsed. - return PARSED_STRUCT[struct_node.name] + return PARSED_STRUCTS[struct_node.name] elif not struct_node.decls: raise ValueError("Struct must have fields.") From 00bb181e052f5997ad3777824bbd0ae06770ba53 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Wed, 30 Oct 2024 00:05:43 +0100 Subject: [PATCH 11/20] feat: implement iterate_annotation_chain to allow subclassing in type structs --- libdestruct/common/struct/struct_impl.py | 5 +++-- libdestruct/common/struct/struct_inflater.py | 2 +- libdestruct/common/type_registry.py | 14 ++++++++------ libdestruct/common/utils.py | 12 +++++++++++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index dd58988..a6fb281 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -13,6 +13,7 @@ from libdestruct.common.obj import obj from libdestruct.common.struct import struct from libdestruct.common.type_registry import TypeRegistry +from libdestruct.common.utils import iterate_annotation_chain if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver @@ -52,7 +53,7 @@ def _inflate_struct_attributes( ) -> None: current_offset = 0 - for name, annotation in reference_type.__annotations__.items(): + for name, annotation in iterate_annotation_chain(reference_type, terminate_at=struct): if name in reference_type.__dict__: # Field associated with the annotation attrs = getattr(reference_type, name) @@ -97,7 +98,7 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: """Compute the size of the struct.""" size = 0 - for name, annotation in reference_type.__annotations__.items(): + for name, annotation in iterate_annotation_chain(reference_type, terminate_at=struct): if name in reference_type.__dict__: # Field associated with the annotation attrs = getattr(reference_type, name) diff --git a/libdestruct/common/struct/struct_inflater.py b/libdestruct/common/struct/struct_inflater.py index de7a106..70508bb 100644 --- a/libdestruct/common/struct/struct_inflater.py +++ b/libdestruct/common/struct/struct_inflater.py @@ -18,7 +18,7 @@ def inflate_struct_type(reference_type: type[struct]) -> type[struct_impl]: if issubclass(reference_type, struct_impl): return reference_type - type_impl = type(reference_type.__name__, (struct_impl,), {"_members": {}}) + type_impl = type(reference_type.__name__, (struct_impl, reference_type), {"_members": {}}) type_impl._reference_struct = reference_type reference_type._type_impl = type_impl diff --git a/libdestruct/common/type_registry.py b/libdestruct/common/type_registry.py index 7e4b245..71516d5 100644 --- a/libdestruct/common/type_registry.py +++ b/libdestruct/common/type_registry.py @@ -75,12 +75,14 @@ def inflater_for( def _inflater_for_type(self: TypeRegistry, item: type[obj]) -> type[obj]: parent = item.__base__ - for handler in self.type_handlers.get(parent, []): - result = handler(item) - - if result is not None: - self.mapping[item] = result - return result + while parent: + for handler in self.type_handlers.get(parent, []): + result = handler(item) + + if result is not None: + self.mapping[item] = result + return result + parent = parent.__base__ raise ValueError(f"No applicable inflater found for {item}") diff --git a/libdestruct/common/utils.py b/libdestruct/common/utils.py index 586468c..4e86777 100644 --- a/libdestruct/common/utils.py +++ b/libdestruct/common/utils.py @@ -7,11 +7,13 @@ from __future__ import annotations from types import MethodType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from libdestruct.common.field import Field if TYPE_CHECKING: # pragma: no cover + from collections.abc import Generator + from libdestruct.backing.resolver import Resolver from libdestruct.common.obj import obj @@ -32,3 +34,11 @@ def size_of(item_or_inflater: obj | callable[[Resolver], obj]) -> int: return field_object.get_size() raise ValueError(f"Cannot determine the size of {item_or_inflater}") + + +def iterate_annotation_chain(item: obj, terminate_at: object | None = None) -> Generator[tuple[str, Any]]: + """Iterate over the annotation chain of the provided item.""" + current_item = item + while current_item is not terminate_at: + yield from current_item.__annotations__.items() + current_item = current_item.__base__ if hasattr(current_item, "__base__") else None From 697c29c5a0a2254063c28f040b6347f8b3598b18 Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 4 Nov 2024 15:42:55 +0100 Subject: [PATCH 12/20] feat: make structs instantiatable, to generate their in-memory representation from the initializers --- libdestruct/backing/fake_resolver.py | 76 ++++++++++++++++++++++++ libdestruct/common/struct/struct.py | 7 +++ libdestruct/common/struct/struct_impl.py | 27 +++++++-- 3 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 libdestruct/backing/fake_resolver.py diff --git a/libdestruct/backing/fake_resolver.py b/libdestruct/backing/fake_resolver.py new file mode 100644 index 0000000..02f8e00 --- /dev/null +++ b/libdestruct/backing/fake_resolver.py @@ -0,0 +1,76 @@ +# +# This file is part of libdestruct (https://github.com/mrindeciso/libdestruct). +# Copyright (c) 2024 Roberto Alessandro Bertolini. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for details. +# + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from libdestruct.backing.resolver import Resolver + + +class FakeResolver(Resolver): + """A class that can resolve elements in a simulated memory storage.""" + + def __init__(self: FakeResolver, memory: dict | None = None, address: int | None = 0) -> FakeResolver: + """Initializes a basic fake resolver.""" + self.memory = memory if memory is not None else {} + self.address = address + self.parent = None + self.offset = None + + def resolve_address(self: FakeResolver) -> int: + """Resolves self's address, mainly used by children to determine their own address.""" + if self.address is not None: + return self.address + + return self.parent.resolve_address() + self.offset + + def relative_from_own(self: FakeResolver, address_offset: int, _: int) -> FakeResolver: + """Creates a resolver that references a parent, such that a change in the parent is propagated on the child.""" + new_resolver = FakeResolver(self.memory, None) + new_resolver.parent = self + new_resolver.offset = address_offset + return new_resolver + + def absolute_from_own(self: FakeResolver, address: int) -> FakeResolver: + """Creates a resolver that has an absolute reference to an object, from the parent's view.""" + return FakeResolver(self.memory, address) + + def resolve(self: FakeResolver, size: int, _: int) -> bytes: + """Resolves itself, providing the bytes it references for the specified size and index.""" + address = self.resolve_address() + # We store data in the dictionary as 4K pages + page_address = address & ~0xFFF + page_offset = address & 0xFFF + + result = b"" + + while size: + page = self.memory.get(page_address, b"\x00" * (0x1000 - page_offset)) + page_size = min(size, 0x1000 - page_offset) + result += page[page_offset : page_offset + page_size] + size -= page_size + page_address += 0x1000 + page_offset = 0 + + return result + + def modify(self: FakeResolver, size: int, _: int, value: bytes) -> None: + """Modifies itself in memory.""" + address = self.resolve_address() + # We store data in the dictionary as 4K pages + page_address = address & ~0xFFF + page_offset = address & 0xFFF + + while size: + page = self.memory.get(page_address, b"\x00" * 0x1000) + page_size = min(size, 0x1000 - page_offset) + page = page[:page_offset] + value[:page_size] + page[page_offset + page_size :] + self.memory[page_address] = page + size -= page_size + value = value[page_size:] + page_address += 0x1000 + page_offset = 0 diff --git a/libdestruct/common/struct/struct.py b/libdestruct/common/struct/struct.py index af3a995..2664018 100644 --- a/libdestruct/common/struct/struct.py +++ b/libdestruct/common/struct/struct.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING from libdestruct.common.obj import obj +from libdestruct.common.type_registry import TypeRegistry from libdestruct.libdestruct import inflater if TYPE_CHECKING: # pragma: no cover @@ -22,6 +23,12 @@ def __init__(self: struct) -> None: """Initialize the struct.""" raise RuntimeError("This type should not be directly instantiated.") + def __new__(cls: type[struct], *args: ..., **kwargs: ...) -> struct: # noqa: PYI034 + """Create a new struct.""" + # Look for an inflater for this struct + inflater = TypeRegistry().inflater_for(cls) + return inflater(*args, **kwargs) + @classmethod def from_bytes(cls: type[struct], data: bytes) -> struct_impl: """Create a struct from a serialized representation.""" diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index a6fb281..8a3a653 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -6,8 +6,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import Self +from libdestruct.backing.fake_resolver import FakeResolver +from libdestruct.backing.resolver import Resolver from libdestruct.common.attributes.offset_attribute import OffsetAttribute from libdestruct.common.field import Field from libdestruct.common.obj import obj @@ -15,9 +17,6 @@ from libdestruct.common.type_registry import TypeRegistry from libdestruct.common.utils import iterate_annotation_chain -if TYPE_CHECKING: # pragma: no cover - from libdestruct.backing.resolver import Resolver - class struct_impl(struct): """The implementation for the C struct type.""" @@ -34,9 +33,16 @@ class struct_impl(struct): _inflater: TypeRegistry = TypeRegistry() """The type registry, used for inflating the attributes.""" - def __init__(self: struct_impl, resolver: Resolver) -> None: + def __init__(self: struct_impl, resolver: Resolver | None = None, **kwargs: ...) -> None: """Initialize the struct implementation.""" - # array overrides the __init__ method, so we need to call the parent class __init__ method + # If we have kwargs and the resolver is None, we provide a fake resolver + if kwargs and resolver is None: + resolver = FakeResolver() + + if not isinstance(resolver, Resolver): + raise TypeError("The resolver must be a Resolver instance.") + + # struct overrides the __init__ method, so we need to call the parent class __init__ method obj.__init__(self, resolver) self.name = self.__class__.__name__ @@ -45,6 +51,15 @@ def __init__(self: struct_impl, resolver: Resolver) -> None: reference_type = self._reference_struct self._inflate_struct_attributes(self._inflater, resolver, reference_type) + for name, value in kwargs.items(): + getattr(self, name).value = value + + def __new__(cls: struct_impl, *args: ..., **kwargs: ...) -> Self: + """Create a new struct.""" + # Skip the __new__ method of the parent class + # struct_impl -> struct -> obj becomes struct_impl -> obj + return obj.__new__(cls) + def _inflate_struct_attributes( self: struct_impl, inflater: TypeRegistry, From a2d2a9a0f8ecdc8882c82a722e7f59df5b39e470 Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 4 Nov 2024 16:31:40 +0100 Subject: [PATCH 13/20] style: remove unused import --- libdestruct/backing/fake_resolver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libdestruct/backing/fake_resolver.py b/libdestruct/backing/fake_resolver.py index 02f8e00..30ed986 100644 --- a/libdestruct/backing/fake_resolver.py +++ b/libdestruct/backing/fake_resolver.py @@ -6,8 +6,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING - from libdestruct.backing.resolver import Resolver From 470ac9fe2bbe6118a4bb38f70d70739fdd37c25b Mon Sep 17 00:00:00 2001 From: MrIndeciso Date: Mon, 4 Nov 2024 16:34:42 +0100 Subject: [PATCH 14/20] fix: import Self from typing_extensions --- libdestruct/common/struct/struct_impl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index 8a3a653..c936630 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import Self +from typing_extensions import Self from libdestruct.backing.fake_resolver import FakeResolver from libdestruct.backing.resolver import Resolver From 8037e7e3b1820553466900f5366e2f04711f87df Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 5 Nov 2024 19:55:52 +0100 Subject: [PATCH 15/20] feat: add `inflate` shortcut --- libdestruct/__init__.py | 3 ++- libdestruct/libdestruct.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/libdestruct/__init__.py b/libdestruct/__init__.py index 8c1b630..14acbf3 100644 --- a/libdestruct/__init__.py +++ b/libdestruct/__init__.py @@ -17,7 +17,7 @@ from libdestruct.common.enum import enum, enum_of from libdestruct.common.ptr import ptr from libdestruct.common.struct import ptr_to, ptr_to_self, struct -from libdestruct.libdestruct import inflater +from libdestruct.libdestruct import inflate, inflater __all__ = [ "array", @@ -30,6 +30,7 @@ "c_ulong", "enum", "enum_of", + "inflate", "inflater", "struct", "ptr", diff --git a/libdestruct/libdestruct.py b/libdestruct/libdestruct.py index bc94890..de0ecd5 100644 --- a/libdestruct/libdestruct.py +++ b/libdestruct/libdestruct.py @@ -6,14 +6,37 @@ from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING +from libdestruct.backing.resolver import Resolver from libdestruct.common.inflater import Inflater -if TYPE_CHECKING: # pragma: no cover - from collections.abc import MutableSequence +if TYPE_CHECKING: # pragma: no cover + from libdestruct.common.obj import obj -def inflater(memory: MutableSequence) -> Inflater: + +def inflater(memory: Sequence) -> Inflater: """Return a TypeInflater instance.""" + if not isinstance(memory, Sequence): + raise TypeError(f"memory must be a MutableSequence, not {type(memory).__name__}") + return Inflater(memory) + + +def inflate(item: type, memory: Sequence, address: int | Resolver) -> obj: + """Inflate a memory-referencing type. + + Args: + item: The type to inflate. + memory: The memory view, which can be mutable or immutable. + address: The address of the object in the memory view. + + Returns: + The inflated object. + """ + if not isinstance(address, int) and not isinstance(address, Resolver): + raise TypeError(f"address must be an int or a Resolver, not {type(address).__name__}") + + return inflater(memory).inflate(item, address) From 6c62a5732422af9b1ceace2be7d7b9708b78ddd5 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 5 Nov 2024 23:22:39 +0100 Subject: [PATCH 16/20] fix: iterate over the annotation chain in correct order --- libdestruct/common/struct/struct_impl.py | 13 +++++++------ libdestruct/common/utils.py | 11 +++++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index c936630..c68bd0c 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -68,10 +68,10 @@ def _inflate_struct_attributes( ) -> None: current_offset = 0 - for name, annotation in iterate_annotation_chain(reference_type, terminate_at=struct): - if name in reference_type.__dict__: + for name, annotation, reference in iterate_annotation_chain(reference_type, terminate_at=struct): + if name in reference.__dict__: # Field associated with the annotation - attrs = getattr(reference_type, name) + attrs = getattr(reference, name) # If attrs is not a tuple, we need to convert it to a tuple if not isinstance(attrs, tuple): @@ -113,10 +113,11 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: """Compute the size of the struct.""" size = 0 - for name, annotation in iterate_annotation_chain(reference_type, terminate_at=struct): - if name in reference_type.__dict__: + for name, annotation, reference in iterate_annotation_chain(reference_type, terminate_at=struct): + print(name, annotation, reference) + if name in reference.__dict__: # Field associated with the annotation - attrs = getattr(reference_type, name) + attrs = getattr(reference, name) # If attrs is not a tuple, we need to convert it to a tuple if not isinstance(attrs, tuple): diff --git a/libdestruct/common/utils.py b/libdestruct/common/utils.py index 4e86777..ddfd341 100644 --- a/libdestruct/common/utils.py +++ b/libdestruct/common/utils.py @@ -36,9 +36,16 @@ def size_of(item_or_inflater: obj | callable[[Resolver], obj]) -> int: raise ValueError(f"Cannot determine the size of {item_or_inflater}") -def iterate_annotation_chain(item: obj, terminate_at: object | None = None) -> Generator[tuple[str, Any]]: +def iterate_annotation_chain(item: obj, terminate_at: object | None = None) -> Generator[tuple[str, Any, type[obj]]]: """Iterate over the annotation chain of the provided item.""" current_item = item + + chain = [] + while current_item is not terminate_at: - yield from current_item.__annotations__.items() + chain.insert(0, current_item) current_item = current_item.__base__ if hasattr(current_item, "__base__") else None + + for reference_item in chain: + for name, annotation in reference_item.__annotations__.items(): + yield name, annotation, reference_item From ae9242afbb2000b4723fcec5c80557897c905e76 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 5 Nov 2024 23:23:42 +0100 Subject: [PATCH 17/20] style: autorefactor whole project --- libdestruct/__init__.py | 4 ++-- libdestruct/backing/memory_resolver.py | 2 +- libdestruct/c/struct_parser.py | 17 ++++++++++++----- libdestruct/common/array/array_field.py | 2 +- .../common/array/array_field_inflater.py | 2 +- libdestruct/common/array/array_impl.py | 2 +- libdestruct/common/array/array_of.py | 2 +- libdestruct/common/enum/enum.py | 2 +- libdestruct/common/enum/enum_field.py | 2 +- libdestruct/common/enum/enum_field_inflater.py | 2 +- libdestruct/common/enum/enum_of.py | 2 +- libdestruct/common/enum/int_enum_field.py | 2 +- libdestruct/common/inflater.py | 2 +- libdestruct/common/obj.py | 2 +- libdestruct/common/ptr/ptr.py | 2 +- libdestruct/common/ptr/ptr_factory.py | 2 +- libdestruct/common/ptr/ptr_field.py | 2 +- libdestruct/common/ptr/ptr_field_inflater.py | 2 +- libdestruct/common/struct/struct.py | 4 ++-- libdestruct/libdestruct.py | 1 - 20 files changed, 32 insertions(+), 26 deletions(-) diff --git a/libdestruct/__init__.py b/libdestruct/__init__.py index 14acbf3..856d246 100644 --- a/libdestruct/__init__.py +++ b/libdestruct/__init__.py @@ -4,11 +4,11 @@ # Licensed under the MIT license. See LICENSE file in the project root for details. # -try: # pragma: no cover +try: # pragma: no cover from rich.traceback import install install() -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover pass from libdestruct.c import c_int, c_long, c_str, c_uint, c_ulong diff --git a/libdestruct/backing/memory_resolver.py b/libdestruct/backing/memory_resolver.py index ea8c059..291d8a1 100644 --- a/libdestruct/backing/memory_resolver.py +++ b/libdestruct/backing/memory_resolver.py @@ -10,7 +10,7 @@ from libdestruct.backing.resolver import Resolver -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from collections.abc import MutableSequence diff --git a/libdestruct/c/struct_parser.py b/libdestruct/c/struct_parser.py index a854f26..0b5e88f 100644 --- a/libdestruct/c/struct_parser.py +++ b/libdestruct/c/struct_parser.py @@ -28,6 +28,7 @@ TYPEDEFS = {} """A cache for parsed type definitions, indexed by name.""" + def definition_to_type(definition: str) -> type[obj]: """Converts a C struct definition to a struct object.""" parser = c_parser.CParser() @@ -39,7 +40,9 @@ def definition_to_type(definition: str) -> type[obj]: try: ast = parser.parse(definition) except c_parser.ParseError as e: - raise ValueError("Invalid definition. Please add the necessary includes if using non-standard type definitions.") from e + raise ValueError( + "Invalid definition. Please add the necessary includes if using non-standard type definitions." + ) from e # We assume that the root declaration is the last one. root = ast.ext[-1].type @@ -122,7 +125,11 @@ def arr_to_type(arr: c_ast.ArrayDecl) -> type[obj]: def type_decl_to_type(decl: c_ast.TypeDecl, parent: c_ast.Struct | None = None) -> type[obj]: """Converts a C type declaration to a type.""" - if not isinstance(decl, c_ast.TypeDecl) and not isinstance(decl, c_ast.PtrDecl) and not isinstance(decl, c_ast.ArrayDecl): + if ( + not isinstance(decl, c_ast.TypeDecl) + and not isinstance(decl, c_ast.PtrDecl) + and not isinstance(decl, c_ast.ArrayDecl) + ): raise TypeError("Definition must be a type declaration.") if isinstance(decl, c_ast.PtrDecl): @@ -158,7 +165,7 @@ def to_uniform_name(name: str) -> str: """Converts a name to a uniform name.""" name = name.replace("unsigned", "u") name = name.replace("_Bool", "bool") - name = name.replace("uchar", "ubyte") # uchar is not a valid ctypes type + name = name.replace("uchar", "ubyte") # uchar is not a valid ctypes type # We have to convert each intX, uintX, intX_t, uintX_t to the original char, short etc. name = name.replace("uint8_t", "ubyte") @@ -184,7 +191,7 @@ def expand_includes(definition: str) -> str: f.write(definition) f.flush() - result = subprocess.run(["cc", "-std=c99", "-E", f.name], capture_output=True, text=True, check=True) # noqa: S607 + result = subprocess.run(["cc", "-std=c99", "-E", f.name], capture_output=True, text=True, check=True) # noqa: S607 return result.stdout @@ -192,7 +199,7 @@ def expand_includes(definition: str) -> str: def cleanup_attributes(definition: str) -> str: """Cleans up attributes in a C definition.""" # Remove __attribute__ ((...)) from the definition. - pattern = r"__attribute__\s*\(\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\)\)" # ChatGPT provided this, don't ask me + pattern = r"__attribute__\s*\(\((?:[^()]+|\((?:[^()]+|\([^()]*\))*\))*\)\)" # ChatGPT provided this, don't ask me return re.sub(pattern, "", definition) diff --git a/libdestruct/common/array/array_field.py b/libdestruct/common/array/array_field.py index 2265546..13cd4c9 100644 --- a/libdestruct/common/array/array_field.py +++ b/libdestruct/common/array/array_field.py @@ -12,7 +12,7 @@ from libdestruct.common.array import array from libdestruct.common.field import Field -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver from libdestruct.common.obj import obj diff --git a/libdestruct/common/array/array_field_inflater.py b/libdestruct/common/array/array_field_inflater.py index 0dc0846..51738d9 100644 --- a/libdestruct/common/array/array_field_inflater.py +++ b/libdestruct/common/array/array_field_inflater.py @@ -12,7 +12,7 @@ from libdestruct.common.array.linear_array_field import LinearArrayField from libdestruct.common.type_registry import TypeRegistry -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from collections.abc import Callable from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/array/array_impl.py b/libdestruct/common/array/array_impl.py index 9f17249..41520ae 100644 --- a/libdestruct/common/array/array_impl.py +++ b/libdestruct/common/array/array_impl.py @@ -12,7 +12,7 @@ from libdestruct.common.struct.struct import struct from libdestruct.common.utils import size_of -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from collections.abc import Generator from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/array/array_of.py b/libdestruct/common/array/array_of.py index 4a4092d..316a9c1 100644 --- a/libdestruct/common/array/array_of.py +++ b/libdestruct/common/array/array_of.py @@ -10,7 +10,7 @@ from libdestruct.common.array.linear_array_field import LinearArrayField -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.common.array.array_field import ArrayField from libdestruct.common.obj import obj diff --git a/libdestruct/common/enum/enum.py b/libdestruct/common/enum/enum.py index daaed28..162a07a 100644 --- a/libdestruct/common/enum/enum.py +++ b/libdestruct/common/enum/enum.py @@ -11,7 +11,7 @@ from libdestruct.common.obj import obj from libdestruct.common.type_registry import TypeRegistry -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from enum import Enum from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/enum/enum_field.py b/libdestruct/common/enum/enum_field.py index 9f057bb..5098e91 100644 --- a/libdestruct/common/enum/enum_field.py +++ b/libdestruct/common/enum/enum_field.py @@ -12,7 +12,7 @@ from libdestruct.common.enum import enum from libdestruct.common.field import Field -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver from libdestruct.common.obj import obj diff --git a/libdestruct/common/enum/enum_field_inflater.py b/libdestruct/common/enum/enum_field_inflater.py index c562539..4724a3b 100644 --- a/libdestruct/common/enum/enum_field_inflater.py +++ b/libdestruct/common/enum/enum_field_inflater.py @@ -11,7 +11,7 @@ from libdestruct.common.enum.int_enum_field import IntEnumField from libdestruct.common.type_registry import TypeRegistry -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from collections.abc import Callable from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/enum/enum_of.py b/libdestruct/common/enum/enum_of.py index b5d4906..e546724 100644 --- a/libdestruct/common/enum/enum_of.py +++ b/libdestruct/common/enum/enum_of.py @@ -11,7 +11,7 @@ from libdestruct.common.enum.int_enum_field import IntEnumField -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.common.enum.enum_field import EnumField diff --git a/libdestruct/common/enum/int_enum_field.py b/libdestruct/common/enum/int_enum_field.py index f19a5db..291df0a 100644 --- a/libdestruct/common/enum/int_enum_field.py +++ b/libdestruct/common/enum/int_enum_field.py @@ -12,7 +12,7 @@ from libdestruct.common.enum.enum import enum from libdestruct.common.enum.enum_field import EnumField -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from enum import IntEnum from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/inflater.py b/libdestruct/common/inflater.py index 35e541b..fddb6f6 100644 --- a/libdestruct/common/inflater.py +++ b/libdestruct/common/inflater.py @@ -11,7 +11,7 @@ from libdestruct.backing.memory_resolver import MemoryResolver from libdestruct.common.type_registry import TypeRegistry -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from collections.abc import MutableSequence from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/obj.py b/libdestruct/common/obj.py index 82c03ba..84001ee 100644 --- a/libdestruct/common/obj.py +++ b/libdestruct/common/obj.py @@ -9,7 +9,7 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/ptr/ptr.py b/libdestruct/common/ptr/ptr.py index b857a44..8b3ebbd 100644 --- a/libdestruct/common/ptr/ptr.py +++ b/libdestruct/common/ptr/ptr.py @@ -11,7 +11,7 @@ from libdestruct.common.field import Field from libdestruct.common.obj import obj -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/ptr/ptr_factory.py b/libdestruct/common/ptr/ptr_factory.py index ca18c59..c106a87 100644 --- a/libdestruct/common/ptr/ptr_factory.py +++ b/libdestruct/common/ptr/ptr_factory.py @@ -10,7 +10,7 @@ from libdestruct.common.ptr.ptr_field import PtrField -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.common.field import Field from libdestruct.common.obj import obj diff --git a/libdestruct/common/ptr/ptr_field.py b/libdestruct/common/ptr/ptr_field.py index 041021a..38603f2 100644 --- a/libdestruct/common/ptr/ptr_field.py +++ b/libdestruct/common/ptr/ptr_field.py @@ -11,7 +11,7 @@ from libdestruct.common.field import Field from libdestruct.common.ptr.ptr import ptr -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.backing.resolver import Resolver from libdestruct.common.obj import obj diff --git a/libdestruct/common/ptr/ptr_field_inflater.py b/libdestruct/common/ptr/ptr_field_inflater.py index 2241741..749a5ef 100644 --- a/libdestruct/common/ptr/ptr_field_inflater.py +++ b/libdestruct/common/ptr/ptr_field_inflater.py @@ -11,7 +11,7 @@ from libdestruct.common.ptr.ptr_field import PtrField from libdestruct.common.type_registry import TypeRegistry -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from collections.abc import Callable from libdestruct.backing.resolver import Resolver diff --git a/libdestruct/common/struct/struct.py b/libdestruct/common/struct/struct.py index 2664018..894302a 100644 --- a/libdestruct/common/struct/struct.py +++ b/libdestruct/common/struct/struct.py @@ -12,7 +12,7 @@ from libdestruct.common.type_registry import TypeRegistry from libdestruct.libdestruct import inflater -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: # pragma: no cover from libdestruct.common.struct.struct_impl import struct_impl @@ -23,7 +23,7 @@ def __init__(self: struct) -> None: """Initialize the struct.""" raise RuntimeError("This type should not be directly instantiated.") - def __new__(cls: type[struct], *args: ..., **kwargs: ...) -> struct: # noqa: PYI034 + def __new__(cls: type[struct], *args: ..., **kwargs: ...) -> struct: # noqa: PYI034 """Create a new struct.""" # Look for an inflater for this struct inflater = TypeRegistry().inflater_for(cls) diff --git a/libdestruct/libdestruct.py b/libdestruct/libdestruct.py index de0ecd5..81e8235 100644 --- a/libdestruct/libdestruct.py +++ b/libdestruct/libdestruct.py @@ -13,7 +13,6 @@ from libdestruct.common.inflater import Inflater if TYPE_CHECKING: # pragma: no cover - from libdestruct.common.obj import obj From 80a4b82fa1d9f9bd6135ca104b979b666e01bfc5 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Tue, 5 Nov 2024 23:26:28 +0100 Subject: [PATCH 18/20] fix: remove wrongly-committed print --- libdestruct/common/struct/struct_impl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index c68bd0c..45ca2e8 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -114,7 +114,6 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: size = 0 for name, annotation, reference in iterate_annotation_chain(reference_type, terminate_at=struct): - print(name, annotation, reference) if name in reference.__dict__: # Field associated with the annotation attrs = getattr(reference, name) From 7dc804d47da9e9d2b0ec5c4f08a66155d62c8fc0 Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Sun, 18 May 2025 21:07:50 +0200 Subject: [PATCH 19/20] fix: don't use .size of struct_impl itself, retrieve size from backing type Structs may define their own .size attribute, breaking everything in the process --- libdestruct/common/struct/struct_impl.py | 15 ++++++--------- libdestruct/common/utils.py | 5 +++++ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/libdestruct/common/struct/struct_impl.py b/libdestruct/common/struct/struct_impl.py index 45ca2e8..dc40b6a 100644 --- a/libdestruct/common/struct/struct_impl.py +++ b/libdestruct/common/struct/struct_impl.py @@ -15,15 +15,12 @@ from libdestruct.common.obj import obj from libdestruct.common.struct import struct from libdestruct.common.type_registry import TypeRegistry -from libdestruct.common.utils import iterate_annotation_chain +from libdestruct.common.utils import iterate_annotation_chain, size_of class struct_impl(struct): """The implementation for the C struct type.""" - size: int - """The size of the struct in bytes.""" - _members: dict[str, obj] """The members of the struct.""" @@ -106,7 +103,7 @@ def _inflate_struct_attributes( result = resolved_type(resolver.relative_from_own(current_offset, 0)) setattr(self, name, result) self._members[name] = result - current_offset += result.size + current_offset += size_of(result) @classmethod def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: @@ -147,13 +144,13 @@ def compute_own_size(cls: type[struct_impl], reference_type: type) -> None: else: attribute = cls._inflater.inflater_for(annotation) - size += attribute.size + size += size_of(attribute) cls.size = size def get(self: struct_impl) -> str: """Return the value of the struct.""" - return f"{self.name}(address={self.address}, size={self.size})" + return f"{self.name}(address={self.address}, size={size_of(self)})" def to_bytes(self: struct_impl) -> bytes: """Return the serialized representation of the struct.""" @@ -185,7 +182,7 @@ def __repr__(self: struct_impl) -> str: members = ",\n".join([f"{name}: {member}" for name, member in self._members.items()]) return f"""{self.name} {{ address: 0x{self.address:x}, - size: 0x{self.size:x}, + size: 0x{size_of(self):x}, members: {{ {members} }} @@ -196,7 +193,7 @@ def __eq__(self: struct_impl, value: object) -> bool: if not isinstance(value, struct_impl): return False - if self.size != value.size: + if size_of(self) != size_of(value): return False if not self._members.keys() == value._members.keys(): diff --git a/libdestruct/common/utils.py b/libdestruct/common/utils.py index ddfd341..4d8aeeb 100644 --- a/libdestruct/common/utils.py +++ b/libdestruct/common/utils.py @@ -25,6 +25,11 @@ def is_field_bound_method(item: obj) -> bool: def size_of(item_or_inflater: obj | callable[[Resolver], obj]) -> int: """Return the size of an object, from an obj or it's inflater.""" + if hasattr(item_or_inflater.__class__, "size"): + # This has the priority over the size of the object itself + # as we might be dealing with a struct object + # that defines an attribute named "size" + return item_or_inflater.__class__.size if hasattr(item_or_inflater, "size"): return item_or_inflater.size From 461c336f7d394ee5a2a3890ea7b2641b02af72cb Mon Sep 17 00:00:00 2001 From: Roberto Bertolini Date: Sun, 18 May 2025 21:08:17 +0200 Subject: [PATCH 20/20] test: ensure size is a valid attribute name in structs and nested structs --- test/scripts/basic_struct_test.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/scripts/basic_struct_test.py b/test/scripts/basic_struct_test.py index 054a202..75a21d0 100644 --- a/test/scripts/basic_struct_test.py +++ b/test/scripts/basic_struct_test.py @@ -333,3 +333,38 @@ class test_t3(struct): test2 = test_t3.from_bytes(memory) self.assertNotEqual(test1, test2) + + def test_struct_member_name_collision(self): + # Ensure that we can inflate structs with an attribute named "size" + class test_t(struct): + size: c_int + a: c_long + b: ptr = ptr_to_self() + + memory = b"" + memory += (1337).to_bytes(4, "little") + memory += (13371337).to_bytes(8, "little") + memory += (4 + 8 + 8).to_bytes(8, "little") + + test = test_t.from_bytes(memory) + + self.assertEqual(test.size.value, 1337) + self.assertEqual(test.a.value, 13371337) + self.assertEqual(test.b.unwrap().address, 4 + 8 + 8) + self.assertEqual(test.address, 0x0) + + # Ensure that we can inflate nested structs with an attribute named "size" + class test_t2(struct): + size: test_t + a: c_long + + memory += (0xdeadbeef).to_bytes(8, "little") + + test2 = test_t2.from_bytes(memory) + + self.assertEqual(test2.size.size.value, 1337) + self.assertEqual(test2.size.a.value, 13371337) + self.assertEqual(test2.size.b.unwrap().address, 4 + 8 + 8) + self.assertEqual(test2.size.address, 0x0) + self.assertEqual(test2.a.value, 0xdeadbeef) + self.assertEqual(test2.address, 0x0)