From db9b35a0c2fd3840ef9f9ac297d70e7e748a1812 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 16:04:27 +0100 Subject: [PATCH 001/102] added `Constructable` Protocol --- construct_typed/generic_wrapper.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index aa4af4c..75c0ac2 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -39,3 +39,20 @@ else: ConstantOrContextLambda = t.Union[ValueType, t.Callable[[Context], t.Any]] PathType = str + + +@t.runtime_checkable +class Constructable(t.Protocol[ParsedType, BuildTypes]): + def __construct__(self) -> "Construct[ParsedType, BuildTypes]": + raise NotImplementedError + + +def construct( + constr: t.Union[ + Constructable[ParsedType, BuildTypes], "Construct[ParsedType, BuildTypes]" + ], +) -> Construct[ParsedType, BuildTypes]: + """Get construct instance of `Constructable` or `Construct`""" + if isinstance(constr, Constructable): + constr = constr.__construct__() + return constr From 753e4282ee48259d82c0944a8a728abb33412802 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 16:13:11 +0100 Subject: [PATCH 002/102] renamed generic_wrapper.py to generic.py and removed relative imports --- construct_typed/__init__.py | 6 +++--- construct_typed/dataclass_struct.py | 4 ++-- construct_typed/{generic_wrapper.py => generic.py} | 0 construct_typed/tenum.py | 2 +- construct_typed/version.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) rename construct_typed/{generic_wrapper.py => generic.py} (100%) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 00d5093..a11e26d 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -1,4 +1,4 @@ -from .dataclass_struct import ( +from construct_typed.dataclass_struct import ( DataclassBitStruct, DataclassMixin, DataclassStruct, @@ -10,7 +10,7 @@ from .dataclass_struct import ( csfield, sfield, ) -from .generic_wrapper import ( +from construct_typed.generic import ( Adapter, ConstantOrContextLambda, Construct, @@ -18,7 +18,7 @@ from .generic_wrapper import ( ListContainer, PathType, ) -from .tenum import EnumBase, FlagsEnumBase, TEnum, TFlagsEnum +from construct_typed.tenum import EnumBase, FlagsEnumBase, TEnum, TFlagsEnum __all__ = [ "DataclassBitStruct", diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 78d2e59..5980a82 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -12,7 +12,7 @@ from construct.lib.containers import ( ) from construct.lib.py3compat import bytestringtype, reprstring, unicodestringtype -from .generic_wrapper import Adapter, Construct, Context, ParsedType, PathType +from construct_typed.generic import Adapter, Construct, Context, ParsedType, PathType class DataclassMixin: @@ -269,4 +269,4 @@ TBitStruct = DataclassBitStruct TContainerMixin = DataclassMixin TContainerBase = DataclassMixin TStructField = csfield -sfield = csfield \ No newline at end of file +sfield = csfield diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic.py similarity index 100% rename from construct_typed/generic_wrapper.py rename to construct_typed/generic.py diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 7c93e33..91c4173 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -1,7 +1,7 @@ import enum import typing as t -from .generic_wrapper import * +from construct_typed.generic import * # ## TEnum ############################################################################################################ diff --git a/construct_typed/version.py b/construct_typed/version.py index 1a555cf..9498baa 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ version = (0, 5, 2) -version_string = "0.5.2" \ No newline at end of file +version_string = "0.5.2" From b896457f907dd4600402e73d938cd37a9eb96fcb Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 16:32:30 +0100 Subject: [PATCH 003/102] Changed implementation of DataclassStruct. It is now only nessesary to sublcass DataclassStruct and not to combine it with @dataclasses.dataclass. Also now the DataclassConstruct is included in the DataclassStruct class type itself. --- construct_typed/dataclass_struct.py | 437 ++++++++++++++++------------ 1 file changed, 253 insertions(+), 184 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 5980a82..23a00eb 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -14,19 +14,247 @@ from construct.lib.py3compat import bytestringtype, reprstring, unicodestringtyp from construct_typed.generic import Adapter, Construct, Context, ParsedType, PathType +T = t.TypeVar("T") -class DataclassMixin: + +# Static type inference support via __dataclass_transform__ implemented as per: +# https://github.com/microsoft/pyright/blob/1.1.135/specs/dataclass_transforms.md +def __dataclass_transform__( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + field_descriptors: t.Tuple[t.Union[type, t.Callable[..., t.Any]], ...] = (()), +) -> t.Callable[[T], T]: + return lambda a: a + + +DATACLASS_METADATA_KEY = "__construct_typed_subcon" + +if t.TYPE_CHECKING: + # specialisation for constructs, that builds from none and dont have to be declared in the __init__ method + @t.overload + def csfield( + subcon: cs.Construct[ParsedType, None], + doc: t.Optional[str] = None, + parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + init: t.Literal[False] = False, + ) -> ParsedType: + ... + + @t.overload + def csfield( + subcon: Construct[ParsedType, t.Any], + doc: t.Optional[str] = None, + parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + init: bool = True, + ) -> ParsedType: + ... + + +def csfield( + subcon: Construct[ParsedType, t.Any], + doc: t.Optional[str] = None, + parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + init: bool = True, +) -> ParsedType: """ - Mixin for the dataclasses which are passed to "DataclassStruct" and "DataclassBitStruct". + Helper method for "DataclassStruct" and "DataclassBitStruct" to create the dataclass fields. - Note: This implementation is different to the 'cs.Container' of the original 'construct' - library. In the original 'cs.Container' some names like "update", "keys", "items", ... can - only accessed via key access (square brackets) and not via attribute access (dot operator), - because they are also method names. This implementation is based on "dataclasses.dataclass" - which only uses modul-level instead of instance-level helper methods.So no instance-level - methods exists and every name can be used. + This method also processes Const and Default, to pass these values als default values to the dataclass. + """ + orig_subcon = subcon + + # Rename subcon, if doc or parsed are available + if (doc is not None) or (parsed is not None): + if doc is not None: + doc = textwrap.dedent(doc).strip("\n") + subcon = cs.Renamed(subcon, newdocs=doc, newparsed=parsed) + + if orig_subcon.flagbuildnone is True: + init = False + default = None + else: + init = True + default = dataclasses.MISSING + + # Set default values in case of special sucons + if isinstance(orig_subcon, cs.Const): + const_subcon: "cs.Const[t.Any, t.Any, t.Any, t.Any]" = orig_subcon + default = const_subcon.value + elif isinstance(orig_subcon, cs.Default): + default_subcon: "cs.Default[t.Any, t.Any, t.Any, t.Any]" = orig_subcon + if callable(default_subcon.value): + default = None # context lambda is only defined at parsing/building + else: + default = default_subcon.value + + return t.cast( + ParsedType, + dataclasses.field( + default=default, + init=init, + metadata={DATACLASS_METADATA_KEY: subcon}, + ), + ) + + +class DataclassConstruct(Adapter[t.Any, t.Any, T, T]): + r""" + TODO: Add Documentation + """ + subcon: "cs.Struct[t.Any, t.Any]" + if t.TYPE_CHECKING: + + def __new__( + cls, + dc_type: t.Type[T], + reverse: bool = False, + ) -> "DataclassConstruct[T]": + ... + + def __init__( + self, + dc_type: t.Type[T], + reverse: bool = False, + ) -> None: + if not isinstance(dc_type, DataclassStruct): + raise TypeError(f"'{repr(dc_type)}' has to be a 'DataclassStruct'") + if not dataclasses.is_dataclass(dc_type): + raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") + self.dc_type = dc_type + self.reverse = reverse + + # get all fields from the dataclass + fields = dataclasses.fields(self.dc_type) + if self.reverse: + fields = tuple(reversed(fields)) + + # extract the construct formats from the struct_type + subcon_fields = {} + for field in fields: + subcon_fields[field.name] = field.metadata[DATACLASS_METADATA_KEY] + + # init adatper + super().__init__(cs.Struct(**subcon_fields)) # type: ignore + + def __getattr__(self, name: str) -> t.Any: + return getattr(self.subcon, name) + + def _decode( + self, obj: "cs.Container[t.Any]", context: Context, path: PathType + ) -> T: + # get all fields from the dataclass + fields = dataclasses.fields(self.dc_type) + + # extract all fields from the container, that are used for create the dataclass object + dc_init = {} + for field in fields: + if field.init: + value = obj[field.name] + dc_init[field.name] = value + + # create object of dataclass + dc = self.dc_type(**dc_init) # type: ignore + + # extract all other values from the container, an pass it to the dataclass + for field in fields: + if not field.init: + value = obj[field.name] + setattr(dc, field.name, value) + + return dc + + def _encode(self, obj: T, context: Context, path: PathType) -> t.Dict[str, t.Any]: + if not isinstance(obj, self.dc_type): + raise TypeError(f"'{repr(obj)}' has to be of type {repr(self.dc_type)}") + + # get all fields from the dataclass + fields = dataclasses.fields(self.dc_type) + + # extract all fields from the container, that are used for create the dataclass object + ret_dict: t.Dict[str, t.Any] = {} + for field in fields: + value = getattr(obj, field.name) + ret_dict[field.name] = value + + return ret_dict + + +# Helper object for defining the `constr` of a `struct`. Will be replaced with the proper construct, when class is created. +this_struct: Construct[t.Any, t.Any] = Construct() + + +def _replace_this_struct(constr: "Construct[t.Any, t.Any]", replacement: t.Any): + """Recursive search for `this_struct` in all SubConstructs and replace it with AttrsStruct""" + subcon = getattr(constr, "subcon", None) + if subcon is this_struct: + setattr(constr, "subcon", replacement) + elif subcon is not None: + _replace_this_struct(subcon, replacement) + else: + raise ValueError( + "Could not find `this_struct`. Only SubConstructs are supported" + ) + + +@__dataclass_transform__(field_descriptors=(csfield,)) +class DataclassStruct: + """ + Adapter for a dataclasses for optimised type hints / static autocompletion in comparision to the original Struct. + + + Before this construct can be created a dataclasses.dataclass type must be created, which must also derive from DataclassMixin. In this dataclass all fields must be assigned to a construct type using csfield. + + Internally, all fields are converted to a normal Struct, which does the actual parsing/building. + + Parses to a dataclasses.dataclass instance, and builds from such instance. Size is the sum of all subcon sizes, unless any subcon raises SizeofError. + + :param constr: This can be used if the structure is nested inside a Subconstruct. To represent this struct use the constant `this_struct`. + :param reverse: Flag if the fields of the dataclass should be reversed + + Example:: + + >>> from construct import Bytes, Int8ub, this + >>> from construct_typed import DataclassMixin, DataclassStruct, csfield, construct + ... class Image(DataclassStruct): + ... width: int = csfield(Int8ub) + ... height: int = csfield(Int8ub) + ... pixels: bytes = csfield(Bytes(this.height * this.width)) + >>> d = construct(Image) + >>> d.parse(b"\x01\x0212") + Image(width=1, height=2, pixels=b'12') """ + @classmethod + def __init_subclass__( + cls, + constr: "cs.Construct[t.Any, t.Any]" = this_struct, + reverse_fields: bool = False, + ): + # validate types + if not isinstance(constr, cs.Construct): # type: ignore + raise ValueError("`constr` parameter has to be an `Construct` object") + if not isinstance(reverse_fields, bool): # type: ignore + raise ValueError("`reverse_fields` parameter has to be an `bool` object") + + # create dataclass + cls = dataclasses.dataclass(cls) + + # create construct format + dc_constr = DataclassConstruct(cls, reverse_fields) + if constr is this_struct: + constr = dc_constr + else: + _replace_this_struct(constr, dc_constr) + + # save construct format and make the class compatible to `Constructable` protocol + setattr(cls, "__construct__", lambda: constr) + + return cls + + # the `construct` library is using the [] access internally, so struct objects + # should also make this possible and not only via the dot access. def __getitem__(self, key: str) -> t.Any: return getattr(self, key) @@ -72,201 +300,42 @@ class DataclassMixin: text.append(indentation.join(str(v).split("\n"))) return "".join(text) - -def csfield( - subcon: Construct[ParsedType, t.Any], - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, -) -> ParsedType: - """ - Helper method for "DataclassStruct" and "DataclassBitStruct" to create the dataclass fields. - - This method also processes Const and Default, to pass these values als default values to the dataclass. - """ - orig_subcon = subcon - - # Rename subcon, if doc or parsed are available - if (doc is not None) or (parsed is not None): - if doc is not None: - doc = textwrap.dedent(doc).strip("\n") - subcon = cs.Renamed(subcon, newdocs=doc, newparsed=parsed) - - if orig_subcon.flagbuildnone is True: - init = False - default = None - else: - init = True - default = dataclasses.MISSING - - # Set default values in case of special sucons - if isinstance(orig_subcon, cs.Const): - const_subcon: "cs.Const[t.Any, t.Any, t.Any, t.Any]" = orig_subcon - default = const_subcon.value - elif isinstance(orig_subcon, cs.Default): - default_subcon: "cs.Default[t.Any, t.Any, t.Any, t.Any]" = orig_subcon - if callable(default_subcon.value): - default = None # context lambda is only defined at parsing/building - else: - default = default_subcon.value - - return t.cast( - ParsedType, - dataclasses.field( - default=default, - init=init, - metadata={"subcon": subcon}, - ), - ) - - -DataclassType = t.TypeVar("DataclassType", bound=DataclassMixin) - - -class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): - """ - Adapter for a dataclasses for optimised type hints / static autocompletion in comparision to the original Struct. - - Before this construct can be created a dataclasses.dataclass type must be created, which must also derive from DataclassMixin. In this dataclass all fields must be assigned to a construct type using csfield. - - Internally, all fields are converted to a Struct, which does the actual parsing/building. - - Parses to a dataclasses.dataclass instance, and builds from such instance. Size is the sum of all subcon sizes, unless any subcon raises SizeofError. - - :param dc_type: Type of the dataclass, which also inherits from DataclassMixin - :param reverse: Flag if the fields of the dataclass should be reversed - - Example:: - - >>> import dataclasses - >>> from construct import Bytes, Int8ub, this - >>> from construct_typed import DataclassMixin, DataclassStruct, csfield - >>> @dataclasses.dataclass - ... class Image(DataclassMixin): - ... width: int = csfield(Int8ub) - ... height: int = csfield(Int8ub) - ... pixels: bytes = csfield(Bytes(this.height * this.width)) - >>> d = DataclassStruct(Image) - >>> d.parse(b"\x01\x0212") - Image(width=1, height=2, pixels=b'12') - """ - - subcon: "cs.Struct[t.Any, t.Any]" if t.TYPE_CHECKING: - def __new__( - cls, - dc_type: t.Type[DataclassType], - reverse: bool = False, - ) -> "DataclassStruct[DataclassType]": + @classmethod + def __construct__(cls: t.Type[T]) -> "DataclassConstruct[T]": ... - def __init__( - self, - dc_type: t.Type[DataclassType], - reverse: bool = False, - ) -> None: - if not issubclass(dc_type, DataclassMixin): - raise TypeError(f"'{repr(dc_type)}' has to be a '{repr(DataclassMixin)}'") - if not dataclasses.is_dataclass(dc_type): - raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") - self.dc_type = dc_type - self.reverse = reverse - # get all fields from the dataclass - fields = dataclasses.fields(self.dc_type) - if self.reverse: - fields = tuple(reversed(fields)) - - # extract the construct formats from the struct_type - subcon_fields = {} - for field in fields: - subcon_fields[field.name] = field.metadata["subcon"] - - # init adatper - super().__init__(cs.Struct(**subcon_fields)) # type: ignore - - def __getattr__(self, name: str) -> t.Any: - return getattr(self.subcon, name) - - def _decode( - self, obj: "cs.Container[t.Any]", context: Context, path: PathType - ) -> DataclassType: - # get all fields from the dataclass - fields = dataclasses.fields(self.dc_type) - - # extract all fields from the container, that are used for create the dataclass object - dc_init = {} - for field in fields: - if field.init: - value = obj[field.name] - dc_init[field.name] = value - - # create object of dataclass - dc = self.dc_type(**dc_init) # type: ignore - - # extract all other values from the container, an pass it to the dataclass - for field in fields: - if not field.init: - value = obj[field.name] - setattr(dc, field.name, value) - - return dc - - def _encode( - self, obj: DataclassType, context: Context, path: PathType - ) -> t.Dict[str, t.Any]: - if not isinstance(obj, self.dc_type): - raise TypeError(f"'{repr(obj)}' has to be of type {repr(self.dc_type)}") - - # get all fields from the dataclass - fields = dataclasses.fields(self.dc_type) - - # extract all fields from the container, that are used for create the dataclass object - ret_dict: t.Dict[str, t.Any] = {} - for field in fields: - value = getattr(obj, field.name) - ret_dict[field.name] = value - - return ret_dict - - -def DataclassBitStruct( - dc_type: t.Type[DataclassType], reverse: bool = False -) -> t.Union[ - "cs.Transformed[DataclassType, DataclassType]", - "cs.Restreamed[DataclassType, DataclassType]", -]: +class DataclassBitStruct(DataclassStruct): r""" Makes a DataclassStruct inside a Bitwise. See :class:`~construct.core.Bitwise` and :class:`~construct_typed.dataclass_struct.DatclassStruct` for semantics and raisable exceptions. - :param dc_type: Type of the dataclass, which also inherits from DataclassMixin - :param reverse: Flag if the fields of the dataclass should be reversed + :param constr: TODO + :param reverse_fields: Flag if the fields of the dataclass should be reversed Example:: - DataclassBitStruct <--> Bitwise(DataclassStruct(...)) - >>> import dataclasses + TODO: >>> from construct import BitsInteger, Flag, Nibble, Padding - >>> from construct_typed import DataclassBitStruct, DataclassMixin, csfield - >>> @dataclasses.dataclass - ... class TestDataclass(DataclassMixin): + >>> from construct_typed import DataclassBitStruct, csfield, construct + ... class TestDataclass(DataclassBitStruct): ... a: int = csfield(Flag) ... b: int = csfield(Nibble) ... c: int = csfield(BitsInteger(10)) ... d: None = csfield(Padding(1)) - >>> d = DataclassBitStruct(TestDataclass) + >>> d = construct(TestDataclass) >>> d.parse(b"\x01\x02") TestDataclass(a=False, b=0, c=129, d=None) """ - return cs.Bitwise(DataclassStruct(dc_type, reverse)) - -# support legacy names -TStruct = DataclassStruct -TBitStruct = DataclassBitStruct -TContainerMixin = DataclassMixin -TContainerBase = DataclassMixin -TStructField = csfield -sfield = csfield + @classmethod + def __init_subclass__( + cls, + constr: "cs.Construct[t.Any, t.Any]" = this_struct, + reverse_fields: bool = False, + ): + cls = DataclassStruct.__init_subclass__.__func__(cls, cs.Bitwise(constr), reverse_fields) # type: ignore + return cls From 8169f0ed318a497f5d9c884d153ced16e056ed9d Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 16:51:20 +0100 Subject: [PATCH 004/102] Changed implementation of TEnum and TEnumFlags. --- construct_typed/__init__.py | 22 ++------ construct_typed/tenum.py | 101 ++++++++++++++++++++++++++++-------- 2 files changed, 84 insertions(+), 39 deletions(-) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index a11e26d..10e5b12 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -1,14 +1,7 @@ from construct_typed.dataclass_struct import ( DataclassBitStruct, - DataclassMixin, DataclassStruct, - TBitStruct, - TContainerBase, - TContainerMixin, - TStruct, - TStructField, csfield, - sfield, ) from construct_typed.generic import ( Adapter, @@ -18,23 +11,16 @@ from construct_typed.generic import ( ListContainer, PathType, ) -from construct_typed.tenum import EnumBase, FlagsEnumBase, TEnum, TFlagsEnum +from construct_typed.tenum import TEnum, TFlags, TEnumConstruct, TFlagsConstruct __all__ = [ "DataclassBitStruct", - "DataclassMixin", "DataclassStruct", - "TBitStruct", - "TContainerBase", - "TContainerMixin", - "TStruct", - "TStructField", "csfield", - "sfield", - "EnumBase", - "FlagsEnumBase", "TEnum", - "TFlagsEnum", + "TEnumConstruct", + "TFlags", + "TFlagsConstruct", "Adapter", "ConstantOrContextLambda", "Construct", diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 91c4173..4a84584 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -1,28 +1,54 @@ import enum import typing as t -from construct_typed.generic import * +import construct as cs + +from .generic import * + +T = t.TypeVar("T") # ## TEnum ############################################################################################################ -class EnumBase(enum.IntEnum): +class TEnum(enum.IntEnum): """ - Base class for an Enum used in `construct_typed.TEnum`. + Base class for an Enum used in `construct_typed.TEnumConstruct`. This class extends the standard `enum.IntEnum`, so that missing values are automatically generated. """ + @classmethod + def __init_subclass__( + cls, + subcon: "cs.Construct[t.Any, t.Any]", + **kwargs: t.Any, + ): + super().__init_subclass__(**kwargs) + + # validate types + if not isinstance(subcon, cs.Construct): # type: ignore + raise ValueError( + f"`subcon` parameter has to be an `Construct` object but is {type(subcon)}" + ) + + # create construct format + enum_constr = TEnumConstruct(subcon, cls) + + # save construct format and make the class compatible to `Constructable` protocol + setattr(cls, "__construct__", lambda: enum_constr) + + return cls + # Extend the enum type with __missing__ method. So if a enum value # not found in the enum, a new pseudo member is created. # The idea is taken from: https://stackoverflow.com/a/57179436 @classmethod - def _missing_(cls, value: t.Any) -> t.Optional["EnumBase"]: + def _missing_(cls, value: t.Any) -> t.Optional["TEnum"]: if isinstance(value, int): return cls._create_pseudo_member_(value) return None # will raise the ValueError in Enum.__new__ @classmethod - def _create_pseudo_member_(cls, value: int) -> "EnumBase": + def _create_pseudo_member_(cls, value: int) -> "TEnum": pseudo_member = cls._value2member_map_.get(value, None) # type: ignore if pseudo_member is None: new_member = int.__new__(cls, value) @@ -33,11 +59,17 @@ class EnumBase(enum.IntEnum): pseudo_member = cls._value2member_map_.setdefault(value, new_member) # type: ignore return pseudo_member # type: ignore + if t.TYPE_CHECKING: -EnumType = t.TypeVar("EnumType", bound=EnumBase) + @classmethod + def __construct__(cls: "t.Type[EnumType]") -> "TEnumConstruct[EnumType]": + ... -class TEnum(Adapter[int, int, EnumType, EnumType]): +EnumType = t.TypeVar("EnumType", bound=TEnum) + + +class TEnumConstruct(Adapter[int, int, EnumType, EnumType]): """ Typed enum. """ @@ -46,20 +78,20 @@ class TEnum(Adapter[int, int, EnumType, EnumType]): def __new__( cls, subcon: Construct[int, int], enum_type: t.Type[EnumType] - ) -> "TEnum[EnumType]": + ) -> "TEnumConstruct[EnumType]": ... def __init__(self, subcon: Construct[int, int], enum_type: t.Type[EnumType]): - if not issubclass(enum_type, EnumBase): + if not issubclass(enum_type, TEnum): raise TypeError( - "'{}' has to be a '{}'".format(repr(enum_type), repr(EnumBase)) + "'{}' has to be a '{}'".format(repr(enum_type), repr(TEnum)) ) # save enum type self.enum_type = t.cast(t.Type[EnumType], enum_type) # type: ignore # init adatper - super(TEnum, self).__init__(subcon) # type: ignore + super(TEnumConstruct, self).__init__(subcon) # type: ignore def _decode(self, obj: int, context: Context, path: PathType) -> EnumType: return self.enum_type(obj) @@ -76,38 +108,65 @@ class TEnum(Adapter[int, int, EnumType, EnumType]): "'{}' has to be of type {}".format(repr(obj), repr(self.enum_type)) ) +# ## TFlags ####################################################################################################### +class TFlags(enum.IntFlag): + @classmethod + def __init_subclass__( + cls, + subcon: "cs.Construct[t.Any, t.Any]", + **kwargs: t.Any, + ): + super().__init_subclass__(**kwargs) -# ## TFlagsEnum ####################################################################################################### -class FlagsEnumBase(enum.IntFlag): - pass + # validate types + if not isinstance(subcon, cs.Construct): # type: ignore + raise ValueError( + f"`subcon` parameter has to be an `Construct` object but is {type(subcon)}" + ) + + # create construct format + enum_constr = TFlagsConstruct(subcon, cls) + + # save construct format and make the class compatible to `Constructable` protocol + setattr(cls, "__construct__", lambda: enum_constr) + + return cls + + if t.TYPE_CHECKING: + + @classmethod + def __construct__( + cls: "t.Type[FlagsEnumType]", + ) -> "TFlagsConstruct[FlagsEnumType]": + ... -FlagsEnumType = t.TypeVar("FlagsEnumType", bound=FlagsEnumBase) +FlagsEnumType = t.TypeVar("FlagsEnumType", bound=TFlags) -class TFlagsEnum(Adapter[int, int, FlagsEnumType, FlagsEnumType]): +class TFlagsConstruct(Adapter[int, int, FlagsEnumType, FlagsEnumType]): """ - Typed enum. + Typed flags. """ if t.TYPE_CHECKING: def __new__( cls, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType] - ) -> "TFlagsEnum[FlagsEnumType]": + ) -> "TFlagsConstruct[FlagsEnumType]": ... def __init__(self, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType]): - if not issubclass(enum_type, FlagsEnumBase): + if not issubclass(enum_type, TFlags): raise TypeError( - "'{}' has to be a '{}'".format(repr(enum_type), repr(FlagsEnumBase)) + "'{}' has to be a '{}'".format(repr(enum_type), repr(TFlags)) ) # save enum type self.enum_type = t.cast(t.Type[FlagsEnumType], enum_type) # type: ignore # init adatper - super(TFlagsEnum, self).__init__(subcon) # type: ignore + super(TFlagsConstruct, self).__init__(subcon) # type: ignore def _decode(self, obj: int, context: Context, path: PathType) -> FlagsEnumType: return self.enum_type(obj) From 9a9b4ab95a2df28683294993bbe7992c97609a2a Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 16:59:01 +0100 Subject: [PATCH 005/102] fixed type error --- construct_typed/dataclass_struct.py | 4 ++-- tests/test_typed.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 23a00eb..f9f9705 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -118,8 +118,8 @@ class DataclassConstruct(Adapter[t.Any, t.Any, T, T]): dc_type: t.Type[T], reverse: bool = False, ) -> None: - if not isinstance(dc_type, DataclassStruct): - raise TypeError(f"'{repr(dc_type)}' has to be a 'DataclassStruct'") + if not issubclass(dc_type, DataclassStruct): + raise TypeError(f"'{repr(dc_type)}' has to be a subclass of 'DataclassStruct'") if not dataclasses.is_dataclass(dc_type): raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") self.dc_type = dc_type diff --git a/tests/test_typed.py b/tests/test_typed.py index d74e025..b76ff91 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -6,7 +6,7 @@ import typing as t import construct as cs import construct_typed as cst -from construct_typed import DataclassBitStruct, DataclassMixin, DataclassStruct, csfield +from construct_typed import DataclassBitStruct, DataclassStruct, csfield from .declarativeunittest import common, raises, setattrs From de76415ac58bb022243226f64bccb06c75191dc5 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 17:02:46 +0100 Subject: [PATCH 006/102] changed vscode test configuration to use the native testing api instead of the Test-Explorer --- .vscode/launch.json | 6 ------ .vscode/settings.json | 21 +++++++++++---------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d94289d..8655026 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,12 +9,6 @@ "type": "python", "request": "launch", "program": "${file}", - "console": "integratedTerminal" - }, - { - "name": "Debug Tests", - "type": "python", - "request": "test", "console": "integratedTerminal", "justMyCode": false } diff --git a/.vscode/settings.json b/.vscode/settings.json index 76bdfe9..b265945 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,24 +1,25 @@ { - "python.pythonPath": "python", "python.languageServer": "Pylance", - // "python.testing.unittestEnabled": false, - // "python.testing.nosetestsEnabled": false, - // "python.testing.pytestEnabled": true, - "pythonTestExplorer.testFramework": "pytest", + + // configure code formating "python.formatting.provider": "black", "python.sortImports.path": "isort", "python.sortImports.args": [ "--profile=black", ], - // "[python]": { - // "editor.codeActionsOnSave": { - // "source.organizeImports": true - // } - // } + + // configure pylance "python.analysis.typeCheckingMode": "strict", "python.analysis.autoImportCompletions": false, "python.analysis.diagnosticSeverityOverrides": { "reportPrivateUsage": "information", "reportUntypedNamedTuple": "information", }, + + // configure pytest + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, } \ No newline at end of file From 7c68aeecd486aaaff5cb189273658eb5f32dd6db Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 17:19:13 +0100 Subject: [PATCH 007/102] adapted all tests to the new api --- construct_typed/__init__.py | 2 + tests/test_typed.py | 177 ++++++++++++------------------------ 2 files changed, 61 insertions(+), 118 deletions(-) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 10e5b12..86e7d7a 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -1,3 +1,4 @@ +from construct_typed.generic import construct from construct_typed.dataclass_struct import ( DataclassBitStruct, DataclassStruct, @@ -16,6 +17,7 @@ from construct_typed.tenum import TEnum, TFlags, TEnumConstruct, TFlagsConstruct __all__ = [ "DataclassBitStruct", "DataclassStruct", + "construct", "csfield", "TEnum", "TEnumConstruct", diff --git a/tests/test_typed.py b/tests/test_typed.py index b76ff91..b69e50c 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -1,19 +1,22 @@ # -*- coding: utf-8 -*- # pyright: strict -import dataclasses -import enum import typing as t import construct as cs -import construct_typed as cst -from construct_typed import DataclassBitStruct, DataclassStruct, csfield +from construct_typed import ( + DataclassBitStruct, + DataclassStruct, + csfield, + construct, + TEnum, + TFlags, +) from .declarativeunittest import common, raises, setattrs def test_dataclass_const_default() -> None: - @dataclasses.dataclass - class ConstDefaultTest(DataclassMixin): + class ConstDefaultTest(DataclassStruct): const_bytes: bytes = csfield(cs.Const(b"BMP")) const_int: int = csfield(cs.Const(5, cs.Int8ub)) default_int: int = csfield(cs.Default(cs.Int8ub, 28)) @@ -29,8 +32,7 @@ def test_dataclass_const_default() -> None: def test_dataclass_access() -> None: - @dataclasses.dataclass - class TestTContainer(DataclassMixin): + class TestTContainer(DataclassStruct): a: t.Optional[int] = csfield(cs.Const(1, cs.Byte)) b: int = csfield(cs.Int8ub) @@ -50,17 +52,16 @@ def test_dataclass_access() -> None: assert tcontainer["a"] == 6 # wrong creation - assert raises(lambda: TestTContainer(a=0, b=1)) == TypeError + assert raises(lambda: TestTContainer(a=0, b=1)) == TypeError # type: ignore def test_dataclass_str_repr() -> None: - @dataclasses.dataclass - class Image(DataclassMixin): + class Image(DataclassStruct): signature: t.Optional[bytes] = csfield(cs.Const(b"BMP")) width: int = csfield(cs.Int8ub) height: int = csfield(cs.Int8ub) - format = DataclassStruct(Image) + format = construct(Image) obj = Image(width=3, height=2) assert ( str(obj) @@ -74,20 +75,19 @@ def test_dataclass_str_repr() -> None: def test_dataclass_struct() -> None: - @dataclasses.dataclass - class Image(DataclassMixin): + class Image(DataclassStruct): width: int = csfield(cs.Int8ub) height: int = csfield(cs.Int8ub) pixels: bytes = csfield(cs.Bytes(cs.this.height * cs.this.width)) common( - cst.DataclassStruct(Image), + construct(Image), b"\x01\x0212", Image(width=1, height=2, pixels=b"12"), ) # check __getattr__ - c = cst.DataclassStruct(Image) + c = Image.__construct__() assert c.width.name == "width" assert c.height.name == "height" assert c.width.subcon is cs.Int8ub @@ -95,43 +95,36 @@ def test_dataclass_struct() -> None: def test_dataclass_struct_reverse() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct, reverse_fields=True): a: int = csfield(cs.Int16ub) b: int = csfield(cs.Int8ub) common( - DataclassStruct(TestContainer, reverse=True), + construct(TestContainer), b"\x02\x00\x01", TestContainer(a=1, b=2), 3, ) - normal = DataclassStruct(TestContainer) - reverse = DataclassStruct(TestContainer, reverse=True) - assert str(normal.parse(b"\x00\x01\x02")) == str(reverse.parse(b"\x02\x00\x01")) def test_dataclass_struct_nested() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): - @dataclasses.dataclass - class InnerDataclass(DataclassMixin): + class TestContainer(DataclassStruct): + class InnerDataclass(DataclassStruct): b: int = csfield(cs.Byte) c: bytes = csfield(cs.Bytes(cs.this._.length)) length: int = csfield(cs.Byte) - a: InnerDataclass = csfield(DataclassStruct(InnerDataclass)) + a: InnerDataclass = csfield(construct(InnerDataclass)) common( - DataclassStruct(TestContainer), + construct(TestContainer), b"\x02\x01\xF1\xF2", TestContainer(length=2, a=TestContainer.InnerDataclass(b=1, c=b"\xF1\xF2")), ) def test_dataclass_struct_default_field() -> None: - @dataclasses.dataclass - class Image(DataclassMixin): + class Image(DataclassStruct): width: int = csfield(cs.Int8ub) height: int = csfield(cs.Int8ub) pixels: t.Optional[bytes] = csfield( @@ -142,7 +135,7 @@ def test_dataclass_struct_default_field() -> None: ) common( - DataclassStruct(Image), + construct(Image), b"\x02\x03\x00\x00\x00\x00\x00\x00", setattrs(Image(2, 3), pixels=bytes(6)), sample_building=Image(2, 3), @@ -150,12 +143,11 @@ def test_dataclass_struct_default_field() -> None: def test_dataclass_struct_const_field() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct): const_field: t.Optional[bytes] = csfield(cs.Const(b"\x00")) common( - DataclassStruct(TestContainer), + construct(TestContainer), bytes(1), setattrs(TestContainer(), const_field=b"\x00"), 1, @@ -163,7 +155,7 @@ def test_dataclass_struct_const_field() -> None: assert ( raises( - DataclassStruct(TestContainer).build, + construct(TestContainer).build, setattrs(TestContainer(), const_field=b"\x01"), ) == cs.ConstError @@ -171,12 +163,11 @@ def test_dataclass_struct_const_field() -> None: def test_dataclass_struct_array_field() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct): array_field: t.List[int] = csfield(cs.Array(5, cs.Int8ub)) common( - DataclassStruct(TestContainer), + construct(TestContainer), bytes(5), TestContainer(array_field=[0, 0, 0, 0, 0]), 5, @@ -184,15 +175,14 @@ def test_dataclass_struct_array_field() -> None: def test_dataclass_struct_anonymus_fields_1() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct): _1: t.Optional[bytes] = csfield(cs.Const(b"\x00")) _2: None = csfield(cs.Padding(1)) _3: None = csfield(cs.Pass) _4: None = csfield(cs.Terminated) common( - DataclassStruct(TestContainer), + construct(TestContainer), bytes(2), setattrs(TestContainer(), _1=b"\x00"), cs.SizeofError, @@ -200,22 +190,20 @@ def test_dataclass_struct_anonymus_fields_1() -> None: def test_dataclass_struct_anonymus_fields_2() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct): _1: int = csfield(cs.Computed(7)) _2: t.Optional[bytes] = csfield(cs.Const(b"JPEG")) _3: None = csfield(cs.Pass) _4: None = csfield(cs.Terminated) - d = DataclassStruct(TestContainer) + d = construct(TestContainer) assert d.build(TestContainer()) == d.build(TestContainer()) def test_dataclass_struct_overloaded_method() -> None: # Test dot access to some names that are not accessable via dot # in the original 'cs.Container'. - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct): clear: int = csfield(cs.Int8ul) copy: int = csfield(cs.Int8ul) fromkeys: int = csfield(cs.Int8ul) @@ -231,7 +219,7 @@ def test_dataclass_struct_overloaded_method() -> None: update: int = csfield(cs.Int8ul) values: int = csfield(cs.Int8ul) - d = DataclassStruct(TestContainer) + d = construct(TestContainer) obj = d.parse( d.build( TestContainer( @@ -268,44 +256,22 @@ def test_dataclass_struct_overloaded_method() -> None: assert obj.values == 14 -def test_dataclass_struct_no_dataclass() -> None: - class TestContainer(DataclassMixin): - a: int = csfield(cs.Int16ub) - b: int = csfield(cs.Int8ub) - - assert raises(lambda: DataclassStruct(TestContainer)) == TypeError - - -def test_dataclass_struct_no_DataclassMixin() -> None: - @dataclasses.dataclass - class TestContainer: - a: int = csfield(cs.Int16ub) - b: int = csfield(cs.Int8ub) - - cls = t.cast(t.Type[DataclassMixin], TestContainer) - assert raises(lambda: DataclassStruct(cls)) == TypeError - - def test_dataclass_struct_wrong_container() -> None: - @dataclasses.dataclass - class TestContainer1(DataclassMixin): + class TestContainer1(DataclassStruct): a: int = csfield(cs.Int16ub) b: int = csfield(cs.Int8ub) - @dataclasses.dataclass - class TestContainer2(DataclassMixin): + class TestContainer2(DataclassStruct): a: int = csfield(cs.Int16ub) b: int = csfield(cs.Int8ub) assert ( - raises(DataclassStruct(TestContainer1).build, TestContainer2(a=1, b=2)) - == TypeError + raises(construct(TestContainer1).build, TestContainer2(a=1, b=2)) == TypeError ) def test_dataclass_struct_doc() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassStruct): a: int = csfield(cs.Int16ub, "This is the documentation of a") b: int = csfield( cs.Int8ub, doc="This is the documentation of b\nwhich is multiline" @@ -318,7 +284,7 @@ def test_dataclass_struct_doc() -> None: """, ) - format = DataclassStruct(TestContainer) + format = TestContainer.__construct__() common(format, b"\x00\x01\x02\x03", TestContainer(a=1, b=2, c=3), 4) assert format.subcon.a.docs == "This is the documentation of a" @@ -330,39 +296,36 @@ def test_dataclass_struct_doc() -> None: def test_dataclass_bitstruct() -> None: - @dataclasses.dataclass - class TestContainer(DataclassMixin): + class TestContainer(DataclassBitStruct): a: int = csfield(cs.BitsInteger(7)) b: int = csfield(cs.Bit) c: int = csfield(cs.BitsInteger(8)) - print("") - common( - DataclassBitStruct(TestContainer), + construct(TestContainer), b"\xFD\x12", TestContainer(a=0x7E, b=1, c=0x12), 2, ) # check __getattr__ - c = DataclassStruct(TestContainer) - assert c.a.name == "a" - assert c.b.name == "b" - assert c.c.name == "c" - assert isinstance(c.a.subcon, cs.BitsInteger) - assert c.b.subcon is cs.Bit - assert isinstance(c.c.subcon, cs.BitsInteger) + c = TestContainer.__construct__() + assert c.subcon.a.name == "a" + assert c.subcon.b.name == "b" + assert c.subcon.c.name == "c" + assert isinstance(c.subcon.a.subcon, cs.BitsInteger) + assert c.subcon.b.subcon is cs.Bit + assert isinstance(c.subcon.c.subcon, cs.BitsInteger) def test_tenum() -> None: - class TestEnum(cst.EnumBase): + class TestEnum(TEnum, subcon=cs.Byte): one = 1 two = 2 four = 4 eight = 8 - d = cst.TEnum(cs.Byte, TestEnum) + d = construct(TestEnum) common(d, b"\x01", TestEnum.one, 1) common(d, b"\xff", TestEnum(255), 1) @@ -375,57 +338,35 @@ def test_tenum() -> None: assert raises(d.build, 8) == TypeError -def test_tenum_no_enumbase() -> None: - class E(enum.Enum): +def test_tenum_in_dataclass_struct() -> None: + class TestEnum(TEnum, subcon=cs.Int8ub): a = 1 b = 2 - cls = t.cast(t.Type[cst.EnumBase], E) - assert raises(lambda: cst.TEnum(cs.Byte, cls)) == TypeError - - -def test_dataclass_struct_wrong_enumbase() -> None: - class E1(cst.EnumBase): - a = 1 - b = 2 - - class E2(cst.EnumBase): - a = 1 - b = 2 - - assert raises(cst.TEnum(cs.Byte, E1).build, E2.a) == TypeError - - -def test_tenum_in_tstruct() -> None: - class TestEnum(cst.EnumBase): - a = 1 - b = 2 - - @dataclasses.dataclass - class TestContainer(DataclassMixin): - a: TestEnum = csfield(cst.TEnum(cs.Int8ub, TestEnum)) + class TestContainer(DataclassStruct): + a: TestEnum = csfield(construct(TestEnum)) b: int = csfield(cs.Int8ub) common( - DataclassStruct(TestContainer), + construct(TestContainer), b"\x01\x02", TestContainer(a=TestEnum.a, b=2), 2, ) assert ( - raises(cst.TEnum(cs.Byte, TestEnum).build, TestContainer(a=1, b=2)) == TypeError # type: ignore + raises(construct(TestEnum).build, TestContainer(a=1, b=2)) == TypeError # type: ignore ) def test_tenum_flags() -> None: - class TestEnum(cst.FlagsEnumBase): + class TestEnum(TFlags, subcon=cs.Byte): one = 1 two = 2 four = 4 eight = 8 - d = cst.TFlagsEnum(cs.Byte, TestEnum) + d = construct(TestEnum) common(d, b"\x03", TestEnum.one | TestEnum.two, 1) assert d.build(TestEnum(0)) == b"\x00" assert d.build(TestEnum.one | TestEnum.two) == b"\x03" From 8525c04165e617b4b0cd326800d858505597728b Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 19:29:56 +0100 Subject: [PATCH 008/102] added custom _EnumMeta, because __init_subclass__ is not working correctly together with the standard enum.EnumMeta... --- construct_typed/tenum.py | 128 +++++++++++++++++++++++---------------- 1 file changed, 77 insertions(+), 51 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 4a84584..47b089c 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -8,35 +8,78 @@ from .generic import * T = t.TypeVar("T") -# ## TEnum ############################################################################################################ -class TEnum(enum.IntEnum): +class _EnumMeta(enum.EnumMeta): + @classmethod + def __prepare__( + metacls, # type: ignore + name: str, + bases: t.Tuple[type, ...], + **kwargs: t.Any, + ) -> t.Mapping[str, object]: + # This method is needed, because the original __prepare__ method does not accept kwargs. + return super().__prepare__(name, bases) + + def __new__( + metacls: t.Type[T], # type: ignore + name: str, + bases: t.Tuple[type, ...], + namespace: t.Dict[str, t.Any], + **kwargs: t.Any, + ) -> T: + # create new enum object + cls = super().__new__(metacls, name, bases, namespace) # type: ignore + + # if the `TEnum` class is created, there are no parameters + if len(kwargs) == 0: + return cls + + # extract parameters from kwargs + subcon: "cs.Construct[t.Any, t.Any]" = kwargs.pop("subcon", None) + if not isinstance(subcon, cs.Construct): # type: ignore + raise ValueError( + f"`subcon` parameter has to be an `Construct` object but is {type(subcon)}" + ) + if len(kwargs) > 0: # check remaining parameters + unsupp_parm = ", ".join([f"'{k}'" for k in kwargs.keys()]) + raise ValueError(f"unsupported parameter(s) detected: {unsupp_parm}") + + # create construct format + if TEnum in bases: + enum_constr = TEnumConstruct(subcon, cls) # type: ignore + elif TFlags in bases: + enum_constr = TFlagsConstruct(subcon, cls) # type: ignore + else: + enum_constr = None + + # save construct format and make the class compatible to `Constructable` protocol + setattr(cls, "__construct__", lambda: enum_constr) # type: ignore + + return cls + + +# ## TEnumConstruct ############################################################################################################ +class TEnum(enum.IntEnum, metaclass=_EnumMeta): """ Base class for an Enum used in `construct_typed.TEnumConstruct`. This class extends the standard `enum.IntEnum`, so that missing values are automatically generated. """ - @classmethod - def __init_subclass__( - cls, - subcon: "cs.Construct[t.Any, t.Any]", - **kwargs: t.Any, - ): - super().__init_subclass__(**kwargs) + if t.TYPE_CHECKING: + # unfortunately the metaclass `enum.EnumMeta` does not forward the parameters to __init_subclass__, so that + # we have to make our own metaclass `ConstructEnumMeta`. + # But pylance/pyright is checking the type parameters passed to the class via __init_subclass__, so that we + # have to fake one. + @classmethod + def __init_subclass__( + cls, + subcon: "cs.Construct[t.Any, t.Any]", + ): + ... - # validate types - if not isinstance(subcon, cs.Construct): # type: ignore - raise ValueError( - f"`subcon` parameter has to be an `Construct` object but is {type(subcon)}" - ) - - # create construct format - enum_constr = TEnumConstruct(subcon, cls) - - # save construct format and make the class compatible to `Constructable` protocol - setattr(cls, "__construct__", lambda: enum_constr) - - return cls + @classmethod + def __construct__(cls: "t.Type[EnumType]") -> "TEnumConstruct[EnumType]": + ... # Extend the enum type with __missing__ method. So if a enum value # not found in the enum, a new pseudo member is created. @@ -59,12 +102,6 @@ class TEnum(enum.IntEnum): pseudo_member = cls._value2member_map_.setdefault(value, new_member) # type: ignore return pseudo_member # type: ignore - if t.TYPE_CHECKING: - - @classmethod - def __construct__(cls: "t.Type[EnumType]") -> "TEnumConstruct[EnumType]": - ... - EnumType = t.TypeVar("EnumType", bound=TEnum) @@ -108,31 +145,20 @@ class TEnumConstruct(Adapter[int, int, EnumType, EnumType]): "'{}' has to be of type {}".format(repr(obj), repr(self.enum_type)) ) + # ## TFlags ####################################################################################################### -class TFlags(enum.IntFlag): - @classmethod - def __init_subclass__( - cls, - subcon: "cs.Construct[t.Any, t.Any]", - **kwargs: t.Any, - ): - super().__init_subclass__(**kwargs) - - # validate types - if not isinstance(subcon, cs.Construct): # type: ignore - raise ValueError( - f"`subcon` parameter has to be an `Construct` object but is {type(subcon)}" - ) - - # create construct format - enum_constr = TFlagsConstruct(subcon, cls) - - # save construct format and make the class compatible to `Constructable` protocol - setattr(cls, "__construct__", lambda: enum_constr) - - return cls - +class TFlags(enum.IntFlag, metaclass=_EnumMeta): if t.TYPE_CHECKING: + # unfortunately the metaclass `enum.EnumMeta` does not forward the parameters to __init_subclass__, so that + # we have to make our own metaclass `ConstructEnumMeta`. + # But pylance/pyright is checking the type parameters passed to the class via __init_subclass__, so that we + # have to fake one. + @classmethod + def __init_subclass__( + cls, + subcon: "cs.Construct[t.Any, t.Any]", + ): + ... @classmethod def __construct__( From e99dd5d752e7413a8f58919b620afa4026703fde Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 19:30:35 +0100 Subject: [PATCH 009/102] some renaming --- construct_typed/tenum.py | 20 ++++++++++---------- tests/test_typed.py | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 47b089c..b146549 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -162,15 +162,15 @@ class TFlags(enum.IntFlag, metaclass=_EnumMeta): @classmethod def __construct__( - cls: "t.Type[FlagsEnumType]", - ) -> "TFlagsConstruct[FlagsEnumType]": + cls: "t.Type[FlagsType]", + ) -> "TFlagsConstruct[FlagsType]": ... -FlagsEnumType = t.TypeVar("FlagsEnumType", bound=TFlags) +FlagsType = t.TypeVar("FlagsType", bound=TFlags) -class TFlagsConstruct(Adapter[int, int, FlagsEnumType, FlagsEnumType]): +class TFlagsConstruct(Adapter[int, int, FlagsType, FlagsType]): """ Typed flags. """ @@ -178,28 +178,28 @@ class TFlagsConstruct(Adapter[int, int, FlagsEnumType, FlagsEnumType]): if t.TYPE_CHECKING: def __new__( - cls, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType] - ) -> "TFlagsConstruct[FlagsEnumType]": + cls, subcon: Construct[int, int], enum_type: t.Type[FlagsType] + ) -> "TFlagsConstruct[FlagsType]": ... - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType]): + def __init__(self, subcon: Construct[int, int], enum_type: t.Type[FlagsType]): if not issubclass(enum_type, TFlags): raise TypeError( "'{}' has to be a '{}'".format(repr(enum_type), repr(TFlags)) ) # save enum type - self.enum_type = t.cast(t.Type[FlagsEnumType], enum_type) # type: ignore + self.enum_type = t.cast(t.Type[FlagsType], enum_type) # type: ignore # init adatper super(TFlagsConstruct, self).__init__(subcon) # type: ignore - def _decode(self, obj: int, context: Context, path: PathType) -> FlagsEnumType: + def _decode(self, obj: int, context: Context, path: PathType) -> FlagsType: return self.enum_type(obj) def _encode( self, - obj: FlagsEnumType, + obj: FlagsType, context: Context, path: PathType, ) -> int: diff --git a/tests/test_typed.py b/tests/test_typed.py index b69e50c..8efdbb5 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -359,19 +359,19 @@ def test_tenum_in_dataclass_struct() -> None: ) -def test_tenum_flags() -> None: - class TestEnum(TFlags, subcon=cs.Byte): +def test_tflags() -> None: + class TestFlags(TFlags, subcon=cs.Byte): one = 1 two = 2 four = 4 eight = 8 - d = construct(TestEnum) - common(d, b"\x03", TestEnum.one | TestEnum.two, 1) - assert d.build(TestEnum(0)) == b"\x00" - assert d.build(TestEnum.one | TestEnum.two) == b"\x03" - assert d.build(TestEnum(8)) == b"\x08" - assert d.build(TestEnum(1 | 2)) == b"\x03" - assert d.build(TestEnum(255)) == b"\xff" - assert d.build(TestEnum.eight) == b"\x08" + d = construct(TestFlags) + common(d, b"\x03", TestFlags.one | TestFlags.two, 1) + assert d.build(TestFlags(0)) == b"\x00" + assert d.build(TestFlags.one | TestFlags.two) == b"\x03" + assert d.build(TestFlags(8)) == b"\x08" + assert d.build(TestFlags(1 | 2)) == b"\x03" + assert d.build(TestFlags(255)) == b"\xff" + assert d.build(TestFlags.eight) == b"\x08" assert raises(d.build, 2) == TypeError From 94a8097bec7212c820ef76e3e23257342cd12ae6 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 13 Feb 2022 19:48:00 +0100 Subject: [PATCH 010/102] added this_struct to public interface --- construct_typed/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 86e7d7a..1c11365 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -3,6 +3,7 @@ from construct_typed.dataclass_struct import ( DataclassBitStruct, DataclassStruct, csfield, + this_struct ) from construct_typed.generic import ( Adapter, @@ -23,6 +24,7 @@ __all__ = [ "TEnumConstruct", "TFlags", "TFlagsConstruct", + "this_struct", "Adapter", "ConstantOrContextLambda", "Construct", From be5ae240fb65c3662331fc1dee3ed0c5aac88469 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 12:46:28 +0100 Subject: [PATCH 011/102] renamed `construct` to `constr` so that there are no naming collisions with the construct package name --- construct_typed/__init__.py | 4 +-- construct_typed/dataclass_struct.py | 6 ++-- construct_typed/generic.py | 6 ++-- construct_typed/tenum.py | 6 ++-- tests/test_typed.py | 46 ++++++++++++++--------------- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 1c11365..56c0e57 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -1,4 +1,4 @@ -from construct_typed.generic import construct +from construct_typed.generic import constr from construct_typed.dataclass_struct import ( DataclassBitStruct, DataclassStruct, @@ -18,7 +18,7 @@ from construct_typed.tenum import TEnum, TFlags, TEnumConstruct, TFlagsConstruct __all__ = [ "DataclassBitStruct", "DataclassStruct", - "construct", + "constr", "csfield", "TEnum", "TEnumConstruct", diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index f9f9705..d6a8566 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -78,7 +78,7 @@ def csfield( init = True default = dataclasses.MISSING - # Set default values in case of special sucons + # Set default values in case of special subcons if isinstance(orig_subcon, cs.Const): const_subcon: "cs.Const[t.Any, t.Any, t.Any, t.Any]" = orig_subcon default = const_subcon.value @@ -249,7 +249,7 @@ class DataclassStruct: _replace_this_struct(constr, dc_constr) # save construct format and make the class compatible to `Constructable` protocol - setattr(cls, "__construct__", lambda: constr) + setattr(cls, "__constr__", lambda: constr) return cls @@ -303,7 +303,7 @@ class DataclassStruct: if t.TYPE_CHECKING: @classmethod - def __construct__(cls: t.Type[T]) -> "DataclassConstruct[T]": + def __constr__(cls: t.Type[T]) -> "DataclassConstruct[T]": ... diff --git a/construct_typed/generic.py b/construct_typed/generic.py index 75c0ac2..b988444 100644 --- a/construct_typed/generic.py +++ b/construct_typed/generic.py @@ -43,16 +43,16 @@ else: @t.runtime_checkable class Constructable(t.Protocol[ParsedType, BuildTypes]): - def __construct__(self) -> "Construct[ParsedType, BuildTypes]": + def __constr__(self) -> "Construct[ParsedType, BuildTypes]": raise NotImplementedError -def construct( +def constr( constr: t.Union[ Constructable[ParsedType, BuildTypes], "Construct[ParsedType, BuildTypes]" ], ) -> Construct[ParsedType, BuildTypes]: """Get construct instance of `Constructable` or `Construct`""" if isinstance(constr, Constructable): - constr = constr.__construct__() + constr = constr.__constr__() return constr diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index b146549..2cf5316 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -52,7 +52,7 @@ class _EnumMeta(enum.EnumMeta): enum_constr = None # save construct format and make the class compatible to `Constructable` protocol - setattr(cls, "__construct__", lambda: enum_constr) # type: ignore + setattr(cls, "__constr__", lambda: enum_constr) # type: ignore return cls @@ -78,7 +78,7 @@ class TEnum(enum.IntEnum, metaclass=_EnumMeta): ... @classmethod - def __construct__(cls: "t.Type[EnumType]") -> "TEnumConstruct[EnumType]": + def __constr__(cls: "t.Type[EnumType]") -> "TEnumConstruct[EnumType]": ... # Extend the enum type with __missing__ method. So if a enum value @@ -161,7 +161,7 @@ class TFlags(enum.IntFlag, metaclass=_EnumMeta): ... @classmethod - def __construct__( + def __constr__( cls: "t.Type[FlagsType]", ) -> "TFlagsConstruct[FlagsType]": ... diff --git a/tests/test_typed.py b/tests/test_typed.py index 8efdbb5..ef0226b 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -7,7 +7,7 @@ from construct_typed import ( DataclassBitStruct, DataclassStruct, csfield, - construct, + constr, TEnum, TFlags, ) @@ -61,7 +61,7 @@ def test_dataclass_str_repr() -> None: width: int = csfield(cs.Int8ub) height: int = csfield(cs.Int8ub) - format = construct(Image) + format = constr(Image) obj = Image(width=3, height=2) assert ( str(obj) @@ -81,13 +81,13 @@ def test_dataclass_struct() -> None: pixels: bytes = csfield(cs.Bytes(cs.this.height * cs.this.width)) common( - construct(Image), + constr(Image), b"\x01\x0212", Image(width=1, height=2, pixels=b"12"), ) # check __getattr__ - c = Image.__construct__() + c = Image.__constr__() assert c.width.name == "width" assert c.height.name == "height" assert c.width.subcon is cs.Int8ub @@ -100,7 +100,7 @@ def test_dataclass_struct_reverse() -> None: b: int = csfield(cs.Int8ub) common( - construct(TestContainer), + constr(TestContainer), b"\x02\x00\x01", TestContainer(a=1, b=2), 3, @@ -114,10 +114,10 @@ def test_dataclass_struct_nested() -> None: c: bytes = csfield(cs.Bytes(cs.this._.length)) length: int = csfield(cs.Byte) - a: InnerDataclass = csfield(construct(InnerDataclass)) + a: InnerDataclass = csfield(constr(InnerDataclass)) common( - construct(TestContainer), + constr(TestContainer), b"\x02\x01\xF1\xF2", TestContainer(length=2, a=TestContainer.InnerDataclass(b=1, c=b"\xF1\xF2")), ) @@ -135,7 +135,7 @@ def test_dataclass_struct_default_field() -> None: ) common( - construct(Image), + constr(Image), b"\x02\x03\x00\x00\x00\x00\x00\x00", setattrs(Image(2, 3), pixels=bytes(6)), sample_building=Image(2, 3), @@ -147,7 +147,7 @@ def test_dataclass_struct_const_field() -> None: const_field: t.Optional[bytes] = csfield(cs.Const(b"\x00")) common( - construct(TestContainer), + constr(TestContainer), bytes(1), setattrs(TestContainer(), const_field=b"\x00"), 1, @@ -155,7 +155,7 @@ def test_dataclass_struct_const_field() -> None: assert ( raises( - construct(TestContainer).build, + constr(TestContainer).build, setattrs(TestContainer(), const_field=b"\x01"), ) == cs.ConstError @@ -167,7 +167,7 @@ def test_dataclass_struct_array_field() -> None: array_field: t.List[int] = csfield(cs.Array(5, cs.Int8ub)) common( - construct(TestContainer), + constr(TestContainer), bytes(5), TestContainer(array_field=[0, 0, 0, 0, 0]), 5, @@ -182,7 +182,7 @@ def test_dataclass_struct_anonymus_fields_1() -> None: _4: None = csfield(cs.Terminated) common( - construct(TestContainer), + constr(TestContainer), bytes(2), setattrs(TestContainer(), _1=b"\x00"), cs.SizeofError, @@ -196,7 +196,7 @@ def test_dataclass_struct_anonymus_fields_2() -> None: _3: None = csfield(cs.Pass) _4: None = csfield(cs.Terminated) - d = construct(TestContainer) + d = constr(TestContainer) assert d.build(TestContainer()) == d.build(TestContainer()) @@ -219,7 +219,7 @@ def test_dataclass_struct_overloaded_method() -> None: update: int = csfield(cs.Int8ul) values: int = csfield(cs.Int8ul) - d = construct(TestContainer) + d = constr(TestContainer) obj = d.parse( d.build( TestContainer( @@ -266,7 +266,7 @@ def test_dataclass_struct_wrong_container() -> None: b: int = csfield(cs.Int8ub) assert ( - raises(construct(TestContainer1).build, TestContainer2(a=1, b=2)) == TypeError + raises(constr(TestContainer1).build, TestContainer2(a=1, b=2)) == TypeError ) @@ -284,7 +284,7 @@ def test_dataclass_struct_doc() -> None: """, ) - format = TestContainer.__construct__() + format = TestContainer.__constr__() common(format, b"\x00\x01\x02\x03", TestContainer(a=1, b=2, c=3), 4) assert format.subcon.a.docs == "This is the documentation of a" @@ -302,14 +302,14 @@ def test_dataclass_bitstruct() -> None: c: int = csfield(cs.BitsInteger(8)) common( - construct(TestContainer), + constr(TestContainer), b"\xFD\x12", TestContainer(a=0x7E, b=1, c=0x12), 2, ) # check __getattr__ - c = TestContainer.__construct__() + c = TestContainer.__constr__() assert c.subcon.a.name == "a" assert c.subcon.b.name == "b" assert c.subcon.c.name == "c" @@ -325,7 +325,7 @@ def test_tenum() -> None: four = 4 eight = 8 - d = construct(TestEnum) + d = constr(TestEnum) common(d, b"\x01", TestEnum.one, 1) common(d, b"\xff", TestEnum(255), 1) @@ -344,18 +344,18 @@ def test_tenum_in_dataclass_struct() -> None: b = 2 class TestContainer(DataclassStruct): - a: TestEnum = csfield(construct(TestEnum)) + a: TestEnum = csfield(constr(TestEnum)) b: int = csfield(cs.Int8ub) common( - construct(TestContainer), + constr(TestContainer), b"\x01\x02", TestContainer(a=TestEnum.a, b=2), 2, ) assert ( - raises(construct(TestEnum).build, TestContainer(a=1, b=2)) == TypeError # type: ignore + raises(constr(TestEnum).build, TestContainer(a=1, b=2)) == TypeError # type: ignore ) @@ -366,7 +366,7 @@ def test_tflags() -> None: four = 4 eight = 8 - d = construct(TestFlags) + d = constr(TestFlags) common(d, b"\x03", TestFlags.one | TestFlags.two, 1) assert d.build(TestFlags(0)) == b"\x00" assert d.build(TestFlags.one | TestFlags.two) == b"\x03" From 75dbd8d8228aa6111fa4c5b22f01230c8359f62c Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 14:10:18 +0100 Subject: [PATCH 012/102] fixed some mypy issues --- construct_typed/dataclass_struct.py | 53 ++++++++++++++--------------- construct_typed/tenum.py | 22 ++++++------ tests/declarativeunittest.pyi | 4 +-- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index d6a8566..ea89f5a 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -31,29 +31,29 @@ def __dataclass_transform__( DATACLASS_METADATA_KEY = "__construct_typed_subcon" -if t.TYPE_CHECKING: - # specialisation for constructs, that builds from none and dont have to be declared in the __init__ method - @t.overload - def csfield( - subcon: cs.Construct[ParsedType, None], - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, - init: t.Literal[False] = False, - ) -> ParsedType: - ... +# specialisation for constructs, that builds from none and dont have to be declared in the __init__ method +@t.overload +def csfield( + subcon: "cs.Construct[ParsedType, None]", + doc: t.Optional[str] = None, + parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + init: t.Literal[False] = False, +) -> ParsedType: + ... - @t.overload - def csfield( - subcon: Construct[ParsedType, t.Any], - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, - init: bool = True, - ) -> ParsedType: - ... + +@t.overload +def csfield( + subcon: "Construct[ParsedType, t.Any]", + doc: t.Optional[str] = None, + parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + init: bool = True, +) -> ParsedType: + ... def csfield( - subcon: Construct[ParsedType, t.Any], + subcon: "Construct[ParsedType, t.Any]", doc: t.Optional[str] = None, parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, init: bool = True, @@ -119,7 +119,9 @@ class DataclassConstruct(Adapter[t.Any, t.Any, T, T]): reverse: bool = False, ) -> None: if not issubclass(dc_type, DataclassStruct): - raise TypeError(f"'{repr(dc_type)}' has to be a subclass of 'DataclassStruct'") + raise TypeError( + f"'{repr(dc_type)}' has to be a subclass of 'DataclassStruct'" + ) if not dataclasses.is_dataclass(dc_type): raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") self.dc_type = dc_type @@ -155,7 +157,7 @@ class DataclassConstruct(Adapter[t.Any, t.Any, T, T]): dc_init[field.name] = value # create object of dataclass - dc = self.dc_type(**dc_init) # type: ignore + dc: T = self.dc_type(**dc_init) # type: ignore # extract all other values from the container, an pass it to the dataclass for field in fields: @@ -185,7 +187,7 @@ class DataclassConstruct(Adapter[t.Any, t.Any, T, T]): this_struct: Construct[t.Any, t.Any] = Construct() -def _replace_this_struct(constr: "Construct[t.Any, t.Any]", replacement: t.Any): +def _replace_this_struct(constr: "Construct[t.Any, t.Any]", replacement: t.Any) -> None: """Recursive search for `this_struct` in all SubConstructs and replace it with AttrsStruct""" subcon = getattr(constr, "subcon", None) if subcon is this_struct: @@ -231,7 +233,7 @@ class DataclassStruct: cls, constr: "cs.Construct[t.Any, t.Any]" = this_struct, reverse_fields: bool = False, - ): + ) -> None: # validate types if not isinstance(constr, cs.Construct): # type: ignore raise ValueError("`constr` parameter has to be an `Construct` object") @@ -251,8 +253,6 @@ class DataclassStruct: # save construct format and make the class compatible to `Constructable` protocol setattr(cls, "__constr__", lambda: constr) - return cls - # the `construct` library is using the [] access internally, so struct objects # should also make this possible and not only via the dot access. def __getitem__(self, key: str) -> t.Any: @@ -336,6 +336,5 @@ class DataclassBitStruct(DataclassStruct): cls, constr: "cs.Construct[t.Any, t.Any]" = this_struct, reverse_fields: bool = False, - ): + ) -> None: cls = DataclassStruct.__init_subclass__.__func__(cls, cs.Bitwise(constr), reverse_fields) # type: ignore - return cls diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 2cf5316..83f1d8c 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -12,22 +12,22 @@ class _EnumMeta(enum.EnumMeta): @classmethod def __prepare__( metacls, # type: ignore - name: str, - bases: t.Tuple[type, ...], + __name: str, + __bases: t.Tuple[type, ...], **kwargs: t.Any, ) -> t.Mapping[str, object]: # This method is needed, because the original __prepare__ method does not accept kwargs. - return super().__prepare__(name, bases) + return super().__prepare__(__name, __bases) def __new__( metacls: t.Type[T], # type: ignore - name: str, - bases: t.Tuple[type, ...], - namespace: t.Dict[str, t.Any], + __name: str, + __bases: t.Tuple[type, ...], + __namespace: t.Dict[str, t.Any], **kwargs: t.Any, ) -> T: # create new enum object - cls = super().__new__(metacls, name, bases, namespace) # type: ignore + cls: T = super().__new__(metacls, __name, __bases, __namespace) # type: ignore # if the `TEnum` class is created, there are no parameters if len(kwargs) == 0: @@ -44,9 +44,9 @@ class _EnumMeta(enum.EnumMeta): raise ValueError(f"unsupported parameter(s) detected: {unsupp_parm}") # create construct format - if TEnum in bases: + if TEnum in __bases: enum_constr = TEnumConstruct(subcon, cls) # type: ignore - elif TFlags in bases: + elif TFlags in __bases: enum_constr = TFlagsConstruct(subcon, cls) # type: ignore else: enum_constr = None @@ -74,7 +74,7 @@ class TEnum(enum.IntEnum, metaclass=_EnumMeta): def __init_subclass__( cls, subcon: "cs.Construct[t.Any, t.Any]", - ): + ) -> None: ... @classmethod @@ -157,7 +157,7 @@ class TFlags(enum.IntFlag, metaclass=_EnumMeta): def __init_subclass__( cls, subcon: "cs.Construct[t.Any, t.Any]", - ): + ) -> None: ... @classmethod diff --git a/tests/declarativeunittest.pyi b/tests/declarativeunittest.pyi index e2f8cab..5afd044 100644 --- a/tests/declarativeunittest.pyi +++ b/tests/declarativeunittest.pyi @@ -6,7 +6,7 @@ import construct_typed as cst Buffer = t.Union[bytes, memoryview, bytearray] ParsedType = t.TypeVar("ParsedType") BuildTypes = t.TypeVar("BuildTypes") -ContainerType = t.TypeVar("ContainerType", bound=cst.TContainerMixin) +ContainerType = t.TypeVar("ContainerType", bound=cst.DataclassStruct) T = t.TypeVar("T") IdentType = t.TypeVar("IdentType") @@ -20,7 +20,7 @@ def raises( ) -> t.Union[t.Any, Exception]: ... @t.overload def common( - format: cst.TStruct[ContainerType], + format: ContainerType, datasample: Buffer, objsample: t.Union[ContainerType, t.Dict[str, t.Any]], sizesample: t.Union[int, t.Type[Exception]] = ..., From b21c43ca65475f3045dc2d92a74c516a9dffb99a Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 21:27:52 +0100 Subject: [PATCH 013/102] added dataclass module from python 3.10 --- construct_typed/dataclass_py310.py | 1454 ++++++++++++++++++++++++++++ 1 file changed, 1454 insertions(+) create mode 100644 construct_typed/dataclass_py310.py diff --git a/construct_typed/dataclass_py310.py b/construct_typed/dataclass_py310.py new file mode 100644 index 0000000..fb3c7d5 --- /dev/null +++ b/construct_typed/dataclass_py310.py @@ -0,0 +1,1454 @@ +import re +import sys +import copy +import types +import inspect +import keyword +import builtins +import functools +import abc +import _thread +from types import FunctionType, GenericAlias + + +__all__ = ['dataclass', + 'field', + 'Field', + 'FrozenInstanceError', + 'InitVar', + 'KW_ONLY', + 'MISSING', + + # Helper functions. + 'fields', + 'asdict', + 'astuple', + 'make_dataclass', + 'replace', + 'is_dataclass', + ] + +# Conditions for adding methods. The boxes indicate what action the +# dataclass decorator takes. For all of these tables, when I talk +# about init=, repr=, eq=, order=, unsafe_hash=, or frozen=, I'm +# referring to the arguments to the @dataclass decorator. When +# checking if a dunder method already exists, I mean check for an +# entry in the class's __dict__. I never check to see if an attribute +# is defined in a base class. + +# Key: +# +=========+=========================================+ +# + Value | Meaning | +# +=========+=========================================+ +# | | No action: no method is added. | +# +---------+-----------------------------------------+ +# | add | Generated method is added. | +# +---------+-----------------------------------------+ +# | raise | TypeError is raised. | +# +---------+-----------------------------------------+ +# | None | Attribute is set to None. | +# +=========+=========================================+ + +# __init__ +# +# +--- init= parameter +# | +# v | | | +# | no | yes | <--- class has __init__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __repr__ +# +# +--- repr= parameter +# | +# v | | | +# | no | yes | <--- class has __repr__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + + +# __setattr__ +# __delattr__ +# +# +--- frozen= parameter +# | +# v | | | +# | no | yes | <--- class has __setattr__ or __delattr__ in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because not adding these methods would break the "frozen-ness" +# of the class. + +# __eq__ +# +# +--- eq= parameter +# | +# v | | | +# | no | yes | <--- class has __eq__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __lt__ +# __le__ +# __gt__ +# __ge__ +# +# +--- order= parameter +# | +# v | | | +# | no | yes | <--- class has any comparison method in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because to allow this case would interfere with using +# functools.total_ordering. + +# __hash__ + +# +------------------- unsafe_hash= parameter +# | +----------- eq= parameter +# | | +--- frozen= parameter +# | | | +# v v v | | | +# | no | yes | <--- class has explicitly defined __hash__ +# +=======+=======+=======+========+========+ +# | False | False | False | | | No __eq__, use the base class __hash__ +# +-------+-------+-------+--------+--------+ +# | False | False | True | | | No __eq__, use the base class __hash__ +# +-------+-------+-------+--------+--------+ +# | False | True | False | None | | <-- the default, not hashable +# +-------+-------+-------+--------+--------+ +# | False | True | True | add | | Frozen, so hashable, allows override +# +-------+-------+-------+--------+--------+ +# | True | False | False | add | raise | Has no __eq__, but hashable +# +-------+-------+-------+--------+--------+ +# | True | False | True | add | raise | Has no __eq__, but hashable +# +-------+-------+-------+--------+--------+ +# | True | True | False | add | raise | Not frozen, but hashable +# +-------+-------+-------+--------+--------+ +# | True | True | True | add | raise | Frozen, so hashable +# +=======+=======+=======+========+========+ +# For boxes that are blank, __hash__ is untouched and therefore +# inherited from the base class. If the base is object, then +# id-based hashing is used. +# +# Note that a class may already have __hash__=None if it specified an +# __eq__ method in the class body (not one that was created by +# @dataclass). +# +# See _hash_action (below) for a coded version of this table. + +# __match_args__ +# +# +--- match_args= parameter +# | +# v | | | +# | no | yes | <--- class has __match_args__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ +# __match_args__ is always added unless the class already defines it. It is a +# tuple of __init__ parameter names; non-init fields must be matched by keyword. + + +# Raised when an attempt is made to modify a frozen class. +class FrozenInstanceError(AttributeError): pass + +# A sentinel object for default values to signal that a default +# factory will be used. This is given a nice repr() which will appear +# in the function signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return '' +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() + +# A sentinel object to detect if a parameter is supplied or not. Use +# a class to give it a better repr. +class _MISSING_TYPE: + pass +MISSING = _MISSING_TYPE() + +# A sentinel object to indicate that following fields are keyword-only by +# default. Use a class to give it a better repr. +class _KW_ONLY_TYPE: + pass +KW_ONLY = _KW_ONLY_TYPE() + +# Since most per-field metadata will be unused, create an empty +# read-only proxy that can be shared among all fields. +_EMPTY_METADATA = types.MappingProxyType({}) + +# Markers for the various kinds of fields and pseudo-fields. +class _FIELD_BASE: + def __init__(self, name): + self.name = name + def __repr__(self): + return self.name +_FIELD = _FIELD_BASE('_FIELD') +_FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') +_FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') + +# The name of an attribute on the class where we store the Field +# objects. Also used to check if a class is a Data Class. +_FIELDS = '__dataclass_fields__' + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = '__dataclass_params__' + +# The name of the function, that if it exists, is called at the end of +# __init__. +_POST_INIT_NAME = '__post_init__' + +# String regex that string annotations for ClassVar or InitVar must match. +# Allows "identifier.identifier[" or "identifier[". +# https://bugs.python.org/issue33453 for details. +_MODULE_IDENTIFIER_RE = re.compile(r'^(?:\s*(\w+)\s*\.)?\s*(\w+)') + +class InitVar: + __slots__ = ('type', ) + + def __init__(self, type): + self.type = type + + def __repr__(self): + if isinstance(self.type, type) and not isinstance(self.type, GenericAlias): + type_name = self.type.__name__ + else: + # typing objects, e.g. List[int] + type_name = repr(self.type) + return f'dataclasses.InitVar[{type_name}]' + + def __class_getitem__(cls, type): + return InitVar(type) + +# Instances of Field are only ever created from within this module, +# and only from the field() function, although Field instances are +# exposed externally as (conceptually) read-only objects. +# +# name and type are filled in after the fact, not in __init__. +# They're not known at the time this class is instantiated, but it's +# convenient if they're available later. +# +# When cls._FIELDS is filled in with a list of Field objects, the name +# and type fields will have been populated. +class Field: + __slots__ = ('name', + 'type', + 'default', + 'default_factory', + 'repr', + 'hash', + 'init', + 'compare', + 'metadata', + 'kw_only', + '_field_type', # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, + metadata, kw_only): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + self.metadata = (_EMPTY_METADATA + if metadata is None else + types.MappingProxyType(metadata)) + self.kw_only = kw_only + self._field_type = None + + def __repr__(self): + return ('Field(' + f'name={self.name!r},' + f'type={self.type!r},' + f'default={self.default!r},' + f'default_factory={self.default_factory!r},' + f'init={self.init!r},' + f'repr={self.repr!r},' + f'hash={self.hash!r},' + f'compare={self.compare!r},' + f'metadata={self.metadata!r},' + f'kw_only={self.kw_only!r},' + f'_field_type={self._field_type}' + ')') + + # This is used to support the PEP 487 __set_name__ protocol in the + # case where we're using a field that contains a descriptor as a + # default value. For details on __set_name__, see + # https://www.python.org/dev/peps/pep-0487/#implementation-details. + # + # Note that in _process_class, this Field object is overwritten + # with the default value, so the end result is a descriptor that + # had __set_name__ called on it at the right time. + def __set_name__(self, owner, name): + func = getattr(type(self.default), '__set_name__', None) + if func: + # There is a __set_name__ method on the descriptor, call + # it. + func(self.default, owner, name) + + __class_getitem__ = classmethod(GenericAlias) + + +class _DataclassParams: + __slots__ = ('init', + 'repr', + 'eq', + 'order', + 'unsafe_hash', + 'frozen', + ) + + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + + def __repr__(self): + return ('_DataclassParams(' + f'init={self.init!r},' + f'repr={self.repr!r},' + f'eq={self.eq!r},' + f'order={self.order!r},' + f'unsafe_hash={self.unsafe_hash!r},' + f'frozen={self.frozen!r}' + ')') + + +# This function is used instead of exposing Field creation directly, +# so that a type checker can be told (via overloads) that this is a +# function whose type depends on its parameters. +def field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, + hash=None, compare=True, metadata=None, kw_only=MISSING): + """Return an object to identify dataclass fields. + + default is the default value of the field. default_factory is a + 0-argument function called to initialize a field's value. If init + is true, the field will be a parameter to the class's __init__() + function. If repr is true, the field will be included in the + object's repr(). If hash is true, the field will be included in the + object's hash(). If compare is true, the field will be used in + comparison functions. metadata, if specified, must be a mapping + which is stored but not otherwise examined by dataclass. If kw_only + is true, the field will become a keyword-only parameter to + __init__(). + + It is an error to specify both default and default_factory. + """ + + if default is not MISSING and default_factory is not MISSING: + raise ValueError('cannot specify both default and default_factory') + return Field(default, default_factory, init, repr, hash, compare, + metadata, kw_only) + + +def _fields_in_init_order(fields): + # Returns the fields as __init__ will output them. It returns 2 tuples: + # the first for normal args, and the second for keyword args. + + return (tuple(f for f in fields if f.init and not f.kw_only), + tuple(f for f in fields if f.init and f.kw_only) + ) + + +def _tuple_str(obj_name, fields): + # Return a string representing each field of obj_name as a tuple + # member. So, if fields is ['x', 'y'] and obj_name is "self", + # return "(self.x,self.y)". + + # Special case for the 0-tuple. + if not fields: + return '()' + # Note the trailing comma, needed if this turns out to be a 1-tuple. + return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' + + +# This function's logic is copied from "recursive_repr" function in +# reprlib module to avoid dependency. +def _recursive_repr(user_function): + # Decorator to make a repr function return "..." for a recursive + # call. + repr_running = set() + + @functools.wraps(user_function) + def wrapper(self): + key = id(self), _thread.get_ident() + if key in repr_running: + return '...' + repr_running.add(key) + try: + result = user_function(self) + finally: + repr_running.discard(key) + return result + return wrapper + + +def _create_fn(name, args, body, *, globals=None, locals=None, + return_type=MISSING): + # Note that we mutate locals when exec() is called. Caller + # beware! The only callers are internal to this module, so no + # worries about external callers. + if locals is None: + locals = {} + if 'BUILTINS' not in locals: + locals['BUILTINS'] = builtins + return_annotation = '' + if return_type is not MISSING: + locals['_return_type'] = return_type + return_annotation = '->_return_type' + args = ','.join(args) + body = '\n'.join(f' {b}' for b in body) + + # Compute the text of the entire function. + txt = f' def {name}({args}){return_annotation}:\n{body}' + + local_vars = ', '.join(locals.keys()) + txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" + ns = {} + exec(txt, globals, ns) + return ns['__create_fn__'](**locals) + + +def _field_assign(frozen, name, value, self_name): + # If we're a frozen class, then assign to our fields in __init__ + # via object.__setattr__. Otherwise, just use a simple + # assignment. + # + # self_name is what "self" is called in this function: don't + # hard-code "self", since that might be a field name. + if frozen: + return f'BUILTINS.object.__setattr__({self_name},{name!r},{value})' + return f'{self_name}.{name}={value}' + + +def _field_init(f, frozen, globals, self_name, slots): + # Return the text of the line in the body of __init__ that will + # initialize this field. + + default_name = f'_dflt_{f.name}' + if f.default_factory is not MISSING: + if f.init: + # This field has a default factory. If a parameter is + # given, use it. If not, call the factory. + globals[default_name] = f.default_factory + value = (f'{default_name}() ' + f'if {f.name} is _HAS_DEFAULT_FACTORY ' + f'else {f.name}') + else: + # This is a field that's not in the __init__ params, but + # has a default factory function. It needs to be + # initialized here by calling the factory function, + # because there's no other way to initialize it. + + # For a field initialized with a default=defaultvalue, the + # class dict just has the default value + # (cls.fieldname=defaultvalue). But that won't work for a + # default factory, the factory must be called in __init__ + # and we must assign that to self.fieldname. We can't + # fall back to the class dict's value, both because it's + # not set, and because it might be different per-class + # (which, after all, is why we have a factory function!). + + globals[default_name] = f.default_factory + value = f'{default_name}()' + else: + # No default factory. + if f.init: + if f.default is MISSING: + # There's no default, just do an assignment. + value = f.name + elif f.default is not MISSING: + globals[default_name] = f.default + value = f.name + else: + # If the class has slots, then initialize this field. + if slots and f.default is not MISSING: + globals[default_name] = f.default + value = default_name + else: + # This field does not need initialization: reading from it will + # just use the class attribute that contains the default. + # Signify that to the caller by returning None. + return None + + # Only test this now, so that we can create variables for the + # default. However, return None to signify that we're not going + # to actually do the assignment statement for InitVars. + if f._field_type is _FIELD_INITVAR: + return None + + # Now, actually generate the field assignment. + return _field_assign(frozen, f.name, value, self_name) + + +def _init_param(f): + # Return the __init__ parameter string for this field. For + # example, the equivalent of 'x:int=3' (except instead of 'int', + # reference a variable set to int, and instead of '3', reference a + # variable set to 3). + if f.default is MISSING and f.default_factory is MISSING: + # There's no default, and no default_factory, just output the + # variable name and type. + default = '' + elif f.default is not MISSING: + # There's a default, this will be the name that's used to look + # it up. + default = f'=_dflt_{f.name}' + elif f.default_factory is not MISSING: + # There's a factory function. Set a marker. + default = '=_HAS_DEFAULT_FACTORY' + return f'{f.name}:_type_{f.name}{default}' + + +def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, + self_name, globals, slots): + # fields contains both real fields and InitVar pseudo-fields. + + # Make sure we don't have fields without defaults following fields + # with defaults. This actually would be caught when exec-ing the + # function source code, but catching it here gives a better error + # message, and future-proofs us in case we build up the function + # using ast. + + seen_default = False + for f in std_fields: + # Only consider the non-kw-only fields in the __init__ call. + if f.init: + if not (f.default is MISSING and f.default_factory is MISSING): + seen_default = True + elif seen_default: + raise TypeError(f'non-default argument {f.name!r} ' + 'follows default argument') + + locals = {f'_type_{f.name}': f.type for f in fields} + locals.update({ + 'MISSING': MISSING, + '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, + }) + + body_lines = [] + for f in fields: + line = _field_init(f, frozen, locals, self_name, slots) + # line is None means that this field doesn't require + # initialization (it's a pseudo-field). Just skip it. + if line: + body_lines.append(line) + + # Does this class have a post-init function? + if has_post_init: + params_str = ','.join(f.name for f in fields + if f._field_type is _FIELD_INITVAR) + body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') + + # If no body lines, use 'pass'. + if not body_lines: + body_lines = ['pass'] + + _init_params = [_init_param(f) for f in std_fields] + if kw_only_fields: + # Add the keyword-only args. Because the * can only be added if + # there's at least one keyword-only arg, there needs to be a test here + # (instead of just concatenting the lists together). + _init_params += ['*'] + _init_params += [_init_param(f) for f in kw_only_fields] + return _create_fn('__init__', + [self_name] + _init_params, + body_lines, + locals=locals, + globals=globals, + return_type=None) + + +def _repr_fn(fields, globals): + fn = _create_fn('__repr__', + ('self',), + ['return self.__class__.__qualname__ + f"(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in fields]) + + ')"'], + globals=globals) + return _recursive_repr(fn) + + +def _frozen_get_del_attr(cls, fields, globals): + locals = {'cls': cls, + 'FrozenInstanceError': FrozenInstanceError} + if fields: + fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' + else: + # Special case for the zero-length tuple. + fields_str = '()' + return (_create_fn('__setattr__', + ('self', 'name', 'value'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f'super(cls, self).__setattr__(name, value)'), + locals=locals, + globals=globals), + _create_fn('__delattr__', + ('self', 'name'), + (f'if type(self) is cls or name in {fields_str}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f'super(cls, self).__delattr__(name)'), + locals=locals, + globals=globals), + ) + + +def _cmp_fn(name, op, self_tuple, other_tuple, globals): + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + + return _create_fn(name, + ('self', 'other'), + [ 'if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + 'return NotImplemented'], + globals=globals) + + +def _hash_fn(fields, globals): + self_tuple = _tuple_str('self', fields) + return _create_fn('__hash__', + ('self',), + [f'return hash({self_tuple})'], + globals=globals) + + +def _is_classvar(a_type, typing): + # This test uses a typing internal class, but it's the best way to + # test if this is a ClassVar. + return (a_type is typing.ClassVar + or (type(a_type) is typing._GenericAlias + and a_type.__origin__ is typing.ClassVar)) + + +def _is_initvar(a_type, dataclasses): + # The module we're checking against is the module we're + # currently in (dataclasses.py). + return (a_type is dataclasses.InitVar + or type(a_type) is dataclasses.InitVar) + +def _is_kw_only(a_type, dataclasses): + return a_type is dataclasses.KW_ONLY + + +def _is_type(annotation, cls, a_module, a_type, is_type_predicate): + # Given a type annotation string, does it refer to a_type in + # a_module? For example, when checking that annotation denotes a + # ClassVar, then a_module is typing, and a_type is + # typing.ClassVar. + + # It's possible to look up a_module given a_type, but it involves + # looking in sys.modules (again!), and seems like a waste since + # the caller already knows a_module. + + # - annotation is a string type annotation + # - cls is the class that this annotation was found in + # - a_module is the module we want to match + # - a_type is the type in that module we want to match + # - is_type_predicate is a function called with (obj, a_module) + # that determines if obj is of the desired type. + + # Since this test does not do a local namespace lookup (and + # instead only a module (global) lookup), there are some things it + # gets wrong. + + # With string annotations, cv0 will be detected as a ClassVar: + # CV = ClassVar + # @dataclass + # class C0: + # cv0: CV + + # But in this example cv1 will not be detected as a ClassVar: + # @dataclass + # class C1: + # CV = ClassVar + # cv1: CV + + # In C1, the code in this function (_is_type) will look up "CV" in + # the module and not find it, so it will not consider cv1 as a + # ClassVar. This is a fairly obscure corner case, and the best + # way to fix it would be to eval() the string "CV" with the + # correct global and local namespaces. However that would involve + # a eval() penalty for every single field of every dataclass + # that's defined. It was judged not worth it. + + match = _MODULE_IDENTIFIER_RE.match(annotation) + if match: + ns = None + module_name = match.group(1) + if not module_name: + # No module name, assume the class's module did + # "from dataclasses import InitVar". + ns = sys.modules.get(cls.__module__).__dict__ + else: + # Look up module_name in the class's module. + module = sys.modules.get(cls.__module__) + if module and module.__dict__.get(module_name) is a_module: + ns = sys.modules.get(a_type.__module__).__dict__ + if ns and is_type_predicate(ns.get(match.group(2)), a_module): + return True + return False + + +def _get_field(cls, a_name, a_type, default_kw_only): + # Return a Field object for this field name and type. ClassVars and + # InitVars are also returned, but marked as such (see f._field_type). + # default_kw_only is the value of kw_only to use if there isn't a field() + # that defines it. + + # If the default value isn't derived from Field, then it's only a + # normal default value. Convert it to a Field(). + default = getattr(cls, a_name, MISSING) + if isinstance(default, Field): + f = default + else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING + f = field(default=default) + + # Only at this point do we know the name and the type. Set them. + f.name = a_name + f.type = a_type + + # Assume it's a normal field until proven otherwise. We're next + # going to decide if it's a ClassVar or InitVar, everything else + # is just a normal field. + f._field_type = _FIELD + + # In addition to checking for actual types here, also check for + # string annotations. get_type_hints() won't always work for us + # (see https://github.com/python/typing/issues/508 for example), + # plus it's expensive and would require an eval for every string + # annotation. So, make a best effort to see if this is a ClassVar + # or InitVar using regex's and checking that the thing referenced + # is actually of the correct type. + + # For the complete discussion, see https://bugs.python.org/issue33453 + + # If typing has not been imported, then it's impossible for any + # annotation to be a ClassVar. So, only look for ClassVar if + # typing has been imported by any module (not necessarily cls's + # module). + typing = sys.modules.get('typing') + if typing: + if (_is_classvar(a_type, typing) + or (isinstance(f.type, str) + and _is_type(f.type, cls, typing, typing.ClassVar, + _is_classvar))): + f._field_type = _FIELD_CLASSVAR + + # If the type is InitVar, or if it's a matching string annotation, + # then it's an InitVar. + if f._field_type is _FIELD: + # The module we're checking against is the module we're + # currently in (dataclasses.py). + dataclasses = sys.modules[__name__] + if (_is_initvar(a_type, dataclasses) + or (isinstance(f.type, str) + and _is_type(f.type, cls, dataclasses, dataclasses.InitVar, + _is_initvar))): + f._field_type = _FIELD_INITVAR + + # Validations for individual fields. This is delayed until now, + # instead of in the Field() constructor, since only here do we + # know the field name, which allows for better error reporting. + + # Special restrictions for ClassVar and InitVar. + if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): + if f.default_factory is not MISSING: + raise TypeError(f'field {f.name} cannot have a ' + 'default factory') + # Should I check for other field settings? default_factory + # seems the most serious to check for. Maybe add others. For + # example, how about init=False (or really, + # init=)? It makes no sense for + # ClassVar and InitVar to specify init=. + + # kw_only validation and assignment. + if f._field_type in (_FIELD, _FIELD_INITVAR): + # For real and InitVar fields, if kw_only wasn't specified use the + # default value. + if f.kw_only is MISSING: + f.kw_only = default_kw_only + else: + # Make sure kw_only isn't set for ClassVars + assert f._field_type is _FIELD_CLASSVAR + if f.kw_only is not MISSING: + raise TypeError(f'field {f.name} is a ClassVar but specifies ' + 'kw_only') + + # For real fields, disallow mutable defaults for known types. + if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): + raise ValueError(f'mutable default {type(f.default)} for field ' + f'{f.name} is not allowed: use default_factory') + + return f + +def _set_qualname(cls, value): + # Ensure that the functions returned from _create_fn uses the proper + # __qualname__ (the class they belong to). + if isinstance(value, FunctionType): + value.__qualname__ = f"{cls.__qualname__}.{value.__name__}" + return value + +def _set_new_attribute(cls, name, value): + # Never overwrites an existing attribute. Returns True if the + # attribute already exists. + if name in cls.__dict__: + return True + _set_qualname(cls, value) + setattr(cls, name, value) + return False + + +# Decide if/how we're going to create a hash function. Key is +# (unsafe_hash, eq, frozen, does-hash-exist). Value is the action to +# take. The common case is to do nothing, so instead of providing a +# function that is a no-op, use None to signify that. + +def _hash_set_none(cls, fields, globals): + return None + +def _hash_add(cls, fields, globals): + flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] + return _set_qualname(cls, _hash_fn(flds, globals)) + +def _hash_exception(cls, fields, globals): + # Raise an exception. + raise TypeError(f'Cannot overwrite attribute __hash__ ' + f'in class {cls.__name__}') + +# +# +-------------------------------------- unsafe_hash? +# | +------------------------------- eq? +# | | +------------------------ frozen? +# | | | +---------------- has-explicit-hash? +# | | | | +# | | | | +------- action +# | | | | | +# v v v v v +_hash_action = {(False, False, False, False): None, + (False, False, False, True ): None, + (False, False, True, False): None, + (False, False, True, True ): None, + (False, True, False, False): _hash_set_none, + (False, True, False, True ): None, + (False, True, True, False): _hash_add, + (False, True, True, True ): None, + (True, False, False, False): _hash_add, + (True, False, False, True ): _hash_exception, + (True, False, True, False): _hash_add, + (True, False, True, True ): _hash_exception, + (True, True, False, False): _hash_add, + (True, True, False, True ): _hash_exception, + (True, True, True, False): _hash_add, + (True, True, True, True ): _hash_exception, + } +# See https://bugs.python.org/issue32929#msg312829 for an if-statement +# version of this table. + + +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, + match_args, kw_only, slots): + # Now that dicts retain insertion order, there's no reason to use + # an ordered dict. I am leveraging that ordering here, because + # derived class fields overwrite base class fields, but the order + # is defined by the base class, which is found first. + fields = {} + + if cls.__module__ in sys.modules: + globals = sys.modules[cls.__module__].__dict__ + else: + # Theoretically this can happen if someone writes + # a custom string to cls.__module__. In which case + # such dataclass won't be fully introspectable + # (w.r.t. typing.get_type_hints) but will still function + # correctly. + globals = {} + + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, + unsafe_hash, frozen)) + + # Find our base classes in reverse MRO order, and exclude + # ourselves. In reversed order so that more derived classes + # override earlier field definitions in base classes. As long as + # we're iterating over them, see if any are frozen. + any_frozen_base = False + has_dataclass_bases = False + for b in cls.__mro__[-1:0:-1]: + # Only process classes that have been processed by our + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) + if base_fields is not None: + has_dataclass_bases = True + for f in base_fields.values(): + fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True + + # Annotations that are defined in this class (not in base + # classes). If __annotations__ isn't present, then this class + # adds no new annotations. We use this to compute fields that are + # added by this class. + # + # Fields are found from cls_annotations, which is guaranteed to be + # ordered. Default values are from class attributes, if a field + # has a default. If the default value is a Field(), then it + # contains additional info beyond (and possibly including) the + # actual default value. Pseudo-fields ClassVars and InitVars are + # included, despite the fact that they're not real fields. That's + # dealt with later. + cls_annotations = cls.__dict__.get('__annotations__', {}) + + # Now find fields in our class. While doing so, validate some + # things, and set the default values (as class attributes) where + # we can. + cls_fields = [] + # Get a reference to this module for the _is_kw_only() test. + KW_ONLY_seen = False + dataclasses = sys.modules[__name__] + for name, type in cls_annotations.items(): + # See if this is a marker to change the value of kw_only. + if (_is_kw_only(type, dataclasses) + or (isinstance(type, str) + and _is_type(type, cls, dataclasses, dataclasses.KW_ONLY, + _is_kw_only))): + # Switch the default to kw_only=True, and ignore this + # annotation: it's not a real field. + if KW_ONLY_seen: + raise TypeError(f'{name!r} is KW_ONLY, but KW_ONLY ' + 'has already been specified') + KW_ONLY_seen = True + kw_only = True + else: + # Otherwise it's a field of some type. + cls_fields.append(_get_field(cls, name, type, kw_only)) + + for f in cls_fields: + fields[f.name] = f + + # If the class attribute (which is the default value for this + # field) exists and is of type 'Field', replace it with the + # real default. This is so that normal class introspection + # sees a real default value, not a Field. + if isinstance(getattr(cls, f.name, None), Field): + if f.default is MISSING: + # If there's no default, delete the class attribute. + # This happens if we specify field(repr=False), for + # example (that is, we specified a field object, but + # no default value). Also if we're using a default + # factory. The class attribute should not be set at + # all in the post-processed class. + delattr(cls, f.name) + else: + setattr(cls, f.name, f.default) + + # Do we have any Field members that don't also have annotations? + for name, value in cls.__dict__.items(): + if isinstance(value, Field) and not name in cls_annotations: + raise TypeError(f'{name!r} is a field but has no type annotation') + + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') + + # Raise an exception if we're frozen, but none of our bases are. + if not any_frozen_base and frozen: + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') + + # Remember all of the fields on our class (including bases). This + # also marks this class as being a dataclass. + setattr(cls, _FIELDS, fields) + + # Was this class defined with an explicit __hash__? Note that if + # __eq__ is defined in this class, then python will automatically + # set __hash__ to None. This is a heuristic, as it's possible + # that such a __hash__ == None was not auto-generated, but it + # close enough. + class_hash = cls.__dict__.get('__hash__', MISSING) + has_explicit_hash = not (class_hash is MISSING or + (class_hash is None and '__eq__' in cls.__dict__)) + + # If we're generating ordering methods, we must be generating the + # eq methods. + if order and not eq: + raise ValueError('eq must be true if order is true') + + # Include InitVars and regular fields (so, not ClassVars). This is + # initialized here, outside of the "if init:" test, because std_init_fields + # is used with match_args, below. + all_init_fields = [f for f in fields.values() + if f._field_type in (_FIELD, _FIELD_INITVAR)] + (std_init_fields, + kw_only_init_fields) = _fields_in_init_order(all_init_fields) + + if init: + # Does this class have a post-init function? + has_post_init = hasattr(cls, _POST_INIT_NAME) + + _set_new_attribute(cls, '__init__', + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + globals, + slots, + )) + + # Get the fields as a list, and include only real fields. This is + # used in all of the following methods. + field_list = [f for f in fields.values() if f._field_type is _FIELD] + + if repr: + flds = [f for f in field_list if f.repr] + _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) + + if eq: + # Create __eq__ method. There's no need for a __ne__ method, + # since python will call __eq__ and negate it. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str('self', flds) + other_tuple = _tuple_str('other', flds) + _set_new_attribute(cls, '__eq__', + _cmp_fn('__eq__', '==', + self_tuple, other_tuple, + globals=globals)) + + if order: + # Create and set the ordering methods. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str('self', flds) + other_tuple = _tuple_str('other', flds) + for name, op in [('__lt__', '<'), + ('__le__', '<='), + ('__gt__', '>'), + ('__ge__', '>='), + ]: + if _set_new_attribute(cls, name, + _cmp_fn(name, op, self_tuple, other_tuple, + globals=globals)): + raise TypeError(f'Cannot overwrite attribute {name} ' + f'in class {cls.__name__}. Consider using ' + 'functools.total_ordering') + + if frozen: + for fn in _frozen_get_del_attr(cls, field_list, globals): + if _set_new_attribute(cls, fn.__name__, fn): + raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + + # Decide if/how we're going to create a hash function. + hash_action = _hash_action[bool(unsafe_hash), + bool(eq), + bool(frozen), + has_explicit_hash] + if hash_action: + # No need to call _set_new_attribute here, since by the time + # we're here the overwriting is unconditional. + cls.__hash__ = hash_action(cls, field_list, globals) + + if not getattr(cls, '__doc__'): + # Create a class doc-string. + cls.__doc__ = (cls.__name__ + + str(inspect.signature(cls)).replace(' -> None', '')) + + if match_args: + # I could probably compute this once + _set_new_attribute(cls, '__match_args__', + tuple(f.name for f in std_init_fields)) + + if slots: + cls = _add_slots(cls, frozen) + + abc.update_abstractmethods(cls) + + return cls + + +# _dataclass_getstate and _dataclass_setstate are needed for pickling frozen +# classes with slots. These could be slighly more performant if we generated +# the code instead of iterating over fields. But that can be a project for +# another day, if performance becomes an issue. +def _dataclass_getstate(self): + return [getattr(self, f.name) for f in fields(self)] + + +def _dataclass_setstate(self, state): + for field, value in zip(fields(self), state): + # use setattr because dataclass may be frozen + object.__setattr__(self, field.name, value) + + +def _add_slots(cls, is_frozen): + # Need to create a new class, since we can't set __slots__ + # after a class has been created. + + # Make sure __slots__ isn't already set. + if '__slots__' in cls.__dict__: + raise TypeError(f'{cls.__name__} already specifies __slots__') + + # Create a new dict for our new class. + cls_dict = dict(cls.__dict__) + field_names = tuple(f.name for f in fields(cls)) + cls_dict['__slots__'] = field_names + for field_name in field_names: + # Remove our attributes, if present. They'll still be + # available in _MARKER. + cls_dict.pop(field_name, None) + + # Remove __dict__ itself. + cls_dict.pop('__dict__', None) + + # And finally create the class. + qualname = getattr(cls, '__qualname__', None) + cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + if qualname is not None: + cls.__qualname__ = qualname + + if is_frozen: + # Need this for pickling frozen classes with slots. + cls.__getstate__ = _dataclass_getstate + cls.__setstate__ = _dataclass_setstate + + return cls + + +def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, + unsafe_hash=False, frozen=False, match_args=True, + kw_only=False, slots=False): + """Returns the same class as was passed in, with dunder methods + added based on the fields defined in the class. + + Examines PEP 526 __annotations__ to determine fields. + + If init is true, an __init__() method is added to the class. If + repr is true, a __repr__() method is added. If order is true, rich + comparison dunder methods are added. If unsafe_hash is true, a + __hash__() method function is added. If frozen is true, fields may + not be assigned to after instance creation. If match_args is true, + the __match_args__ tuple is added. If kw_only is true, then by + default all fields are keyword-only. If slots is true, an + __slots__ attribute is added. + """ + + def wrap(cls): + return _process_class(cls, init, repr, eq, order, unsafe_hash, + frozen, match_args, kw_only, slots) + + # See if we're being called as @dataclass or @dataclass(). + if cls is None: + # We're called with parens. + return wrap + + # We're called as @dataclass without parens. + return wrap(cls) + + +def fields(class_or_instance): + """Return a tuple describing the fields of this dataclass. + + Accepts a dataclass or an instance of one. Tuple elements are of + type Field. + """ + + # Might it be worth caching this, per class? + try: + fields = getattr(class_or_instance, _FIELDS) + except AttributeError: + raise TypeError('must be called with a dataclass type or instance') + + # Exclude pseudo-fields. Note that fields is sorted by insertion + # order, so the order of the tuple is as the fields were defined. + return tuple(f for f in fields.values() if f._field_type is _FIELD) + + +def _is_dataclass_instance(obj): + """Returns True if obj is an instance of a dataclass.""" + return hasattr(type(obj), _FIELDS) + + +def is_dataclass(obj): + """Returns True if obj is a dataclass or an instance of a + dataclass.""" + cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj) + return hasattr(cls, _FIELDS) + + +def asdict(obj, *, dict_factory=dict): + """Return the fields of a dataclass instance as a new dictionary mapping + field names to field values. + + Example usage: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert asdict(c) == {'x': 1, 'y': 2} + + If given, 'dict_factory' will be used instead of built-in dict. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + if not _is_dataclass_instance(obj): + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + + +def _asdict_inner(obj, dict_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name), dict_factory) + result.append((f.name, value)) + return dict_factory(result) + elif isinstance(obj, tuple) and hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + + return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) + elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + else: + return copy.deepcopy(obj) + + +def astuple(obj, *, tuple_factory=tuple): + """Return the fields of a dataclass instance as a new tuple of field values. + + Example usage:: + + @dataclass + class C: + x: int + y: int + + c = C(1, 2) + assert astuple(c) == (1, 2) + + If given, 'tuple_factory' will be used instead of built-in tuple. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + + if not _is_dataclass_instance(obj): + raise TypeError("astuple() should be called on dataclass instances") + return _astuple_inner(obj, tuple_factory) + + +def _astuple_inner(obj, tuple_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _astuple_inner(getattr(obj, f.name), tuple_factory) + result.append(value) + return tuple_factory(result) + elif isinstance(obj, tuple) and hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + return type(obj)(*[_astuple_inner(v, tuple_factory) for v in obj]) + elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). + return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + for k, v in obj.items()) + else: + return copy.deepcopy(obj) + + +def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, + repr=True, eq=True, order=False, unsafe_hash=False, + frozen=False, match_args=True, kw_only=False, slots=False): + """Return a new dynamically created dataclass. + + The dataclass name will be 'cls_name'. 'fields' is an iterable + of either (name), (name, type) or (name, type, Field) objects. If type is + omitted, use the string 'typing.Any'. Field objects are created by + the equivalent of calling 'field(name, type [, Field-info])'. + + C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) + + is equivalent to: + + @dataclass + class C(Base): + x: 'typing.Any' + y: int + z: int = field(init=False) + + For the bases and namespace parameters, see the builtin type() function. + + The parameters init, repr, eq, order, unsafe_hash, and frozen are passed to + dataclass(). + """ + + if namespace is None: + namespace = {} + + # While we're looking through the field names, validate that they + # are identifiers, are not keywords, and not duplicates. + seen = set() + annotations = {} + defaults = {} + for item in fields: + if isinstance(item, str): + name = item + tp = 'typing.Any' + elif len(item) == 2: + name, tp, = item + elif len(item) == 3: + name, tp, spec = item + defaults[name] = spec + else: + raise TypeError(f'Invalid field: {item!r}') + + if not isinstance(name, str) or not name.isidentifier(): + raise TypeError(f'Field names must be valid identifiers: {name!r}') + if keyword.iskeyword(name): + raise TypeError(f'Field names must not be keywords: {name!r}') + if name in seen: + raise TypeError(f'Field name duplicated: {name!r}') + + seen.add(name) + annotations[name] = tp + + # Update 'ns' with the user-supplied namespace plus our calculated values. + def exec_body_callback(ns): + ns.update(namespace) + ns.update(defaults) + ns['__annotations__'] = annotations + + # We use `types.new_class()` instead of simply `type()` to allow dynamic creation + # of generic dataclasses. + cls = types.new_class(cls_name, bases, {}, exec_body_callback) + + # Apply the normal decorator. + return dataclass(cls, init=init, repr=repr, eq=eq, order=order, + unsafe_hash=unsafe_hash, frozen=frozen, + match_args=match_args, kw_only=kw_only, slots=slots) + + +def replace(obj, /, **changes): + """Return a new object replacing specified fields with new values. + + This is especially useful for frozen classes. Example usage: + + @dataclass(frozen=True) + class C: + x: int + y: int + + c = C(1, 2) + c1 = replace(c, x=3) + assert c1.x == 3 and c1.y == 2 + """ + + # We're going to mutate 'changes', but that's okay because it's a + # new dict, even if called with 'replace(obj, **my_changes)'. + + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + + # It's an error to have init=False fields in 'changes'. + # If a field is not in 'changes', read its value from the provided obj. + + for f in getattr(obj, _FIELDS).values(): + # Only consider normal fields or InitVars. + if f._field_type is _FIELD_CLASSVAR: + continue + + if not f.init: + # Error if this field is specified in changes. + if f.name in changes: + raise ValueError(f'field {f.name} is declared with ' + 'init=False, it cannot be specified with ' + 'replace()') + continue + + if f.name not in changes: + if f._field_type is _FIELD_INITVAR and f.default is MISSING: + raise ValueError(f"InitVar {f.name!r} " + 'must be specified with replace()') + changes[f.name] = getattr(obj, f.name) + + # Create the new object, which calls __init__() and + # __post_init__() (if defined), using all of the init fields we've + # added and/or left in 'changes'. If there are values supplied in + # changes that aren't fields, this will correctly raise a + # TypeError. + return obj.__class__(**changes) \ No newline at end of file From a0cadcce854dc1c83ecd3aa45882a183904dedc2 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 21:54:04 +0100 Subject: [PATCH 014/102] removed things that not work in Python 3.8 --- construct_typed/dataclass_py310.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/construct_typed/dataclass_py310.py b/construct_typed/dataclass_py310.py index fb3c7d5..9d6d9bb 100644 --- a/construct_typed/dataclass_py310.py +++ b/construct_typed/dataclass_py310.py @@ -8,7 +8,7 @@ import builtins import functools import abc import _thread -from types import FunctionType, GenericAlias +from types import FunctionType __all__ = ['dataclass', @@ -229,7 +229,7 @@ class InitVar: self.type = type def __repr__(self): - if isinstance(self.type, type) and not isinstance(self.type, GenericAlias): + if isinstance(self.type, type): type_name = self.type.__name__ else: # typing objects, e.g. List[int] @@ -309,8 +309,6 @@ class Field: # it. func(self.default, owner, name) - __class_getitem__ = classmethod(GenericAlias) - class _DataclassParams: __slots__ = ('init', @@ -1101,8 +1099,6 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if slots: cls = _add_slots(cls, frozen) - abc.update_abstractmethods(cls) - return cls @@ -1211,7 +1207,7 @@ def _is_dataclass_instance(obj): def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj) + cls = obj if isinstance(obj, type) else type(obj) return hasattr(cls, _FIELDS) From f9e2d24d714ef092f398d127cc5ba2dd8f4c5479 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 21:55:16 +0100 Subject: [PATCH 015/102] Use marker instances from the original dataclass module so that hopefully also the original dataclass functions are working --- construct_typed/dataclass_py310.py | 52 ++++++++++++++++++------------ 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/construct_typed/dataclass_py310.py b/construct_typed/dataclass_py310.py index 9d6d9bb..0a8c4d6 100644 --- a/construct_typed/dataclass_py310.py +++ b/construct_typed/dataclass_py310.py @@ -9,6 +9,18 @@ import functools import abc import _thread from types import FunctionType +from dataclasses import ( + _FIELD, + _FIELD_CLASSVAR, + _FIELD_INITVAR, + _FIELDS, + _PARAMS, + _POST_INIT_NAME, + _EMPTY_METADATA, + MISSING, + _HAS_DEFAULT_FACTORY, + FrozenInstanceError +) __all__ = ['dataclass', @@ -169,21 +181,21 @@ __all__ = ['dataclass', # Raised when an attempt is made to modify a frozen class. -class FrozenInstanceError(AttributeError): pass +# class FrozenInstanceError(AttributeError): pass # A sentinel object for default values to signal that a default # factory will be used. This is given a nice repr() which will appear # in the function signature of dataclasses' constructors. -class _HAS_DEFAULT_FACTORY_CLASS: - def __repr__(self): - return '' -_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() +# class _HAS_DEFAULT_FACTORY_CLASS: +# def __repr__(self): +# return '' +# _HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() # A sentinel object to detect if a parameter is supplied or not. Use # a class to give it a better repr. -class _MISSING_TYPE: - pass -MISSING = _MISSING_TYPE() +# class _MISSING_TYPE: +# pass +# MISSING = _MISSING_TYPE() # A sentinel object to indicate that following fields are keyword-only by # default. Use a class to give it a better repr. @@ -193,29 +205,29 @@ KW_ONLY = _KW_ONLY_TYPE() # Since most per-field metadata will be unused, create an empty # read-only proxy that can be shared among all fields. -_EMPTY_METADATA = types.MappingProxyType({}) +# _EMPTY_METADATA = types.MappingProxyType({}) # Markers for the various kinds of fields and pseudo-fields. -class _FIELD_BASE: - def __init__(self, name): - self.name = name - def __repr__(self): - return self.name -_FIELD = _FIELD_BASE('_FIELD') -_FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') -_FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') +# class _FIELD_BASE: +# def __init__(self, name): +# self.name = name +# def __repr__(self): +# return self.name +# _FIELD = _FIELD_BASE('_FIELD') +# _FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') +# _FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') # The name of an attribute on the class where we store the Field # objects. Also used to check if a class is a Data Class. -_FIELDS = '__dataclass_fields__' +# _FIELDS = '__dataclass_fields__' # The name of an attribute on the class that stores the parameters to # @dataclass. -_PARAMS = '__dataclass_params__' +# _PARAMS = '__dataclass_params__' # The name of the function, that if it exists, is called at the end of # __init__. -_POST_INIT_NAME = '__post_init__' +# _POST_INIT_NAME = '__post_init__' # String regex that string annotations for ClassVar or InitVar must match. # Allows "identifier.identifier[" or "identifier[". From 46fc3dd48c161aab8982a0e14fb721561ee8c96a Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 21:55:31 +0100 Subject: [PATCH 016/102] renamed file --- construct_typed/{dataclass_py310.py => dataclasses_py310.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename construct_typed/{dataclass_py310.py => dataclasses_py310.py} (100%) diff --git a/construct_typed/dataclass_py310.py b/construct_typed/dataclasses_py310.py similarity index 100% rename from construct_typed/dataclass_py310.py rename to construct_typed/dataclasses_py310.py From 8529bd287897fd61dd6fa9b66e534ee050767482 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 19 Feb 2022 21:56:50 +0100 Subject: [PATCH 017/102] ignore typing issues and removed unnessesary imports --- construct_typed/dataclasses_py310.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct_typed/dataclasses_py310.py b/construct_typed/dataclasses_py310.py index 0a8c4d6..a2f76d0 100644 --- a/construct_typed/dataclasses_py310.py +++ b/construct_typed/dataclasses_py310.py @@ -1,3 +1,4 @@ +# type: ignore import re import sys import copy @@ -6,7 +7,6 @@ import inspect import keyword import builtins import functools -import abc import _thread from types import FunctionType from dataclasses import ( From cab9a07c4f204af9fa2df4a9bda7003b75fdbeac Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 12:32:49 +0100 Subject: [PATCH 018/102] Use standard dataclasses modul for python >= 3.10 or use the provided dataclasses module from this library fol python < 3.10. So we can use the `kw_only` option. --- construct_typed/dataclass_struct.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index ea89f5a..8b24286 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # pyright: strict import dataclasses +import sys import textwrap import typing as t @@ -14,6 +15,13 @@ from construct.lib.py3compat import bytestringtype, reprstring, unicodestringtyp from construct_typed.generic import Adapter, Construct, Context, ParsedType, PathType +# The `key_only` keyword is possible since python 3.10. To support it in python 3.8 & 3.9 +# the dataclasses module from python 3.10 is copied to this package. +if sys.version_info >= (3, 10) or t.TYPE_CHECKING: + import dataclasses +else: + import construct_typed.dataclasses_py310 as dataclasses + T = t.TypeVar("T") @@ -200,7 +208,7 @@ def _replace_this_struct(constr: "Construct[t.Any, t.Any]", replacement: t.Any) ) -@__dataclass_transform__(field_descriptors=(csfield,)) +@__dataclass_transform__(field_descriptors=(csfield,), kw_only_default=True) class DataclassStruct: """ Adapter for a dataclasses for optimised type hints / static autocompletion in comparision to the original Struct. @@ -241,7 +249,7 @@ class DataclassStruct: raise ValueError("`reverse_fields` parameter has to be an `bool` object") # create dataclass - cls = dataclasses.dataclass(cls) + dataclasses.dataclass(cls, kw_only=True) # type: ignore # create construct format dc_constr = DataclassConstruct(cls, reverse_fields) From a46c180e7a810e878f9d2888a856e3c0a495711a Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 12:47:10 +0100 Subject: [PATCH 019/102] Dont parse the provided subcon in csfield. Only check if it builds from none. But now you can use the `const` or `default` parameters instead to provied default or constant values. --- construct_typed/dataclass_struct.py | 87 ++++++++++++++++++++--------- tests/test_typed.py | 38 +++++++------ 2 files changed, 82 insertions(+), 43 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 8b24286..4fc3f65 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -39,64 +39,97 @@ def __dataclass_transform__( DATACLASS_METADATA_KEY = "__construct_typed_subcon" -# specialisation for constructs, that builds from none and dont have to be declared in the __init__ method +# specialisation for constructs, that builds from none -> this field does not appear in the __init__ method and has a default of None @t.overload def csfield( subcon: "cs.Construct[ParsedType, None]", - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + *, + doc: t.Optional[str] = ..., init: t.Literal[False] = False, -) -> ParsedType: + default: t.Literal[None] = None, + const: t.Literal[dataclasses.MISSING] = ..., +) -> t.Optional[ParsedType]: ... +# normal mode, when neither default nor const is defined -> this field does appear in the __init__ method but has no default value @t.overload def csfield( subcon: "Construct[ParsedType, t.Any]", - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, - init: bool = True, + *, + doc: t.Optional[str] = ..., + init: t.Literal[True] = ..., + default: t.Literal[dataclasses.MISSING] = ..., + const: t.Literal[dataclasses.MISSING] = ..., +) -> ParsedType: + ... + + +# specialisation when const parameter is set -> this field does not appear in the __init__ method but has a default value +@t.overload +def csfield( + subcon: "Construct[ParsedType, t.Any]", + *, + doc: t.Optional[str] = ..., + init: t.Literal[False] = ..., + default: t.Literal[dataclasses.MISSING] = ..., + const: t.Optional[ParsedType] = ..., +) -> ParsedType: + ... + + +# specialisation when default parameter is set -> this field does appear in the __init__ method and has a default value +@t.overload +def csfield( + subcon: "Construct[ParsedType, t.Any]", + *, + doc: t.Optional[str] = ..., + init: t.Literal[True] = ..., + default: t.Optional[ParsedType] = ..., + const: t.Literal[dataclasses.MISSING] = ..., ) -> ParsedType: ... def csfield( subcon: "Construct[ParsedType, t.Any]", + *, doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, - init: bool = True, + init: bool = True, # dont use this parameter, this is only used for `dataclass_transform` + default: t.Optional[t.Any] = dataclasses.MISSING, + const: t.Optional[t.Any] = dataclasses.MISSING, ) -> ParsedType: """ Helper method for "DataclassStruct" and "DataclassBitStruct" to create the dataclass fields. This method also processes Const and Default, to pass these values als default values to the dataclass. + + Only one of the parameters `default` or `const` can be vaild. They are mutually exclusive. """ - orig_subcon = subcon - # Rename subcon, if doc or parsed are available - if (doc is not None) or (parsed is not None): - if doc is not None: - doc = textwrap.dedent(doc).strip("\n") - subcon = cs.Renamed(subcon, newdocs=doc, newparsed=parsed) + if (default is not dataclasses.MISSING) and (const is not dataclasses.MISSING): + raise ValueError("default and const are mutally exclusive") - if orig_subcon.flagbuildnone is True: + # Rename subcon, if doc is available + if doc is not None: + doc = textwrap.dedent(doc).strip("\n") + subcon = cs.Renamed(subcon, newdocs=doc) + + if default is not dataclasses.MISSING: + init = True + default = default + subcon = cs.Default(subcon, default) + elif const is not dataclasses.MISSING: + init = False + default = const + subcon = cs.Const(const, subcon) + elif subcon.flagbuildnone is True: init = False default = None else: init = True default = dataclasses.MISSING - # Set default values in case of special subcons - if isinstance(orig_subcon, cs.Const): - const_subcon: "cs.Const[t.Any, t.Any, t.Any, t.Any]" = orig_subcon - default = const_subcon.value - elif isinstance(orig_subcon, cs.Default): - default_subcon: "cs.Default[t.Any, t.Any, t.Any, t.Any]" = orig_subcon - if callable(default_subcon.value): - default = None # context lambda is only defined at parsing/building - else: - default = default_subcon.value - return t.cast( ParsedType, dataclasses.field( diff --git a/tests/test_typed.py b/tests/test_typed.py index ef0226b..9cf4fc5 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -12,28 +12,36 @@ from construct_typed import ( TFlags, ) -from .declarativeunittest import common, raises, setattrs +from tests.declarativeunittest import common, raises, setattrs def test_dataclass_const_default() -> None: class ConstDefaultTest(DataclassStruct): - const_bytes: bytes = csfield(cs.Const(b"BMP")) - const_int: int = csfield(cs.Const(5, cs.Int8ub)) - default_int: int = csfield(cs.Default(cs.Int8ub, 28)) - default_lambda: bytes = csfield( + const_bytes: bytes = csfield(cs.Bytes(3), const=b"BMP") + const_int: int = csfield(cs.Int8ub, const=5) + default_int: int = csfield(cs.Int8ub, default=26) + default_lambda: t.Optional[bytes] = csfield( cs.Default(cs.Bytes(cs.this.const_int), lambda ctx: bytes(ctx.const_int)) ) a = ConstDefaultTest() assert a.const_bytes == b"BMP" assert a.const_int == 5 - assert a.default_int == 28 + assert a.default_int == 26 assert a.default_lambda == None + a = ConstDefaultTest(default_int=1) + assert a.default_int == 1 + + format = ConstDefaultTest.__constr__() + assert isinstance(format.const_bytes.subcon, cs.Const) + assert isinstance(format.const_int.subcon, cs.Const) + assert isinstance(format.default_int.subcon, cs.Default) + assert isinstance(format.default_lambda.subcon, cs.Default) def test_dataclass_access() -> None: class TestTContainer(DataclassStruct): - a: t.Optional[int] = csfield(cs.Const(1, cs.Byte)) + a: int = csfield(cs.Byte, const=1) b: int = csfield(cs.Int8ub) tcontainer = TestTContainer(b=2) @@ -57,7 +65,7 @@ def test_dataclass_access() -> None: def test_dataclass_str_repr() -> None: class Image(DataclassStruct): - signature: t.Optional[bytes] = csfield(cs.Const(b"BMP")) + signature: bytes = csfield(cs.Bytes(3), const=b"BMP") width: int = csfield(cs.Int8ub) height: int = csfield(cs.Int8ub) @@ -137,8 +145,8 @@ def test_dataclass_struct_default_field() -> None: common( constr(Image), b"\x02\x03\x00\x00\x00\x00\x00\x00", - setattrs(Image(2, 3), pixels=bytes(6)), - sample_building=Image(2, 3), + setattrs(Image(width=2, height=3), pixels=bytes(6)), + sample_building=Image(width=2, height=3), ) @@ -191,7 +199,7 @@ def test_dataclass_struct_anonymus_fields_1() -> None: def test_dataclass_struct_anonymus_fields_2() -> None: class TestContainer(DataclassStruct): - _1: int = csfield(cs.Computed(7)) + _1: t.Optional[int] = csfield(cs.Computed(7)) _2: t.Optional[bytes] = csfield(cs.Const(b"JPEG")) _3: None = csfield(cs.Pass) _4: None = csfield(cs.Terminated) @@ -265,20 +273,18 @@ def test_dataclass_struct_wrong_container() -> None: a: int = csfield(cs.Int16ub) b: int = csfield(cs.Int8ub) - assert ( - raises(constr(TestContainer1).build, TestContainer2(a=1, b=2)) == TypeError - ) + assert raises(constr(TestContainer1).build, TestContainer2(a=1, b=2)) == TypeError def test_dataclass_struct_doc() -> None: class TestContainer(DataclassStruct): - a: int = csfield(cs.Int16ub, "This is the documentation of a") + a: int = csfield(cs.Int16ub, doc="This is the documentation of a") b: int = csfield( cs.Int8ub, doc="This is the documentation of b\nwhich is multiline" ) c: int = csfield( cs.Int8ub, - """ + doc=""" This is the documentation of c which is also multiline """, From 935f021a4fb466394634069db75963ef59c34dfe Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 16:53:12 +0100 Subject: [PATCH 020/102] added a litte bit of documentation --- construct_typed/dataclass_struct.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 4fc3f65..1ed95e9 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -253,6 +253,11 @@ class DataclassStruct: Parses to a dataclasses.dataclass instance, and builds from such instance. Size is the sum of all subcon sizes, unless any subcon raises SizeofError. + Every construct that builds from None (eg. Const, Default, Index, Rebuild, Check, Checksum, ...) will automatically initialised with None. + If a default or const value should be used in a DataclassStruct the best is to use the `default` or `const` parameters of `csfield`. These + are internally used for creating a `Const` or `Default` construct but also adds the default/const value to the DataclassStruct while creating + it via __init__. + :param constr: This can be used if the structure is nested inside a Subconstruct. To represent this struct use the constant `this_struct`. :param reverse: Flag if the fields of the dataclass should be reversed From acc9f6a9e4a441fe75223d41a753f0ad798188ba Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 17:07:41 +0100 Subject: [PATCH 021/102] removed this_struct and added a lambda instead --- construct_typed/__init__.py | 4 +-- construct_typed/dataclass_struct.py | 47 ++++++++++------------------- tests/test_typed.py | 23 ++++++++++++++ 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 56c0e57..537de1a 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -2,8 +2,7 @@ from construct_typed.generic import constr from construct_typed.dataclass_struct import ( DataclassBitStruct, DataclassStruct, - csfield, - this_struct + csfield ) from construct_typed.generic import ( Adapter, @@ -24,7 +23,6 @@ __all__ = [ "TEnumConstruct", "TFlags", "TFlagsConstruct", - "this_struct", "Adapter", "ConstantOrContextLambda", "Construct", diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 1ed95e9..cee8c89 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -224,23 +224,6 @@ class DataclassConstruct(Adapter[t.Any, t.Any, T, T]): return ret_dict -# Helper object for defining the `constr` of a `struct`. Will be replaced with the proper construct, when class is created. -this_struct: Construct[t.Any, t.Any] = Construct() - - -def _replace_this_struct(constr: "Construct[t.Any, t.Any]", replacement: t.Any) -> None: - """Recursive search for `this_struct` in all SubConstructs and replace it with AttrsStruct""" - subcon = getattr(constr, "subcon", None) - if subcon is this_struct: - setattr(constr, "subcon", replacement) - elif subcon is not None: - _replace_this_struct(subcon, replacement) - else: - raise ValueError( - "Could not find `this_struct`. Only SubConstructs are supported" - ) - - @__dataclass_transform__(field_descriptors=(csfield,), kw_only_default=True) class DataclassStruct: """ @@ -258,7 +241,7 @@ class DataclassStruct: are internally used for creating a `Const` or `Default` construct but also adds the default/const value to the DataclassStruct while creating it via __init__. - :param constr: This can be used if the structure is nested inside a Subconstruct. To represent this struct use the constant `this_struct`. + :param constr: Lambda for creating the construct object. This can be used if the DataclassStruct is nested inside a Subconstruct. Eg. `lambda cls: cs.Bitwise(cls)`. :param reverse: Flag if the fields of the dataclass should be reversed Example:: @@ -276,13 +259,15 @@ class DataclassStruct: @classmethod def __init_subclass__( - cls, - constr: "cs.Construct[t.Any, t.Any]" = this_struct, + cls: t.Type[T], + constr: t.Callable[ + [DataclassConstruct[T]], Construct[t.Any, t.Any] + ] = lambda cls: cls, reverse_fields: bool = False, ) -> None: # validate types - if not isinstance(constr, cs.Construct): # type: ignore - raise ValueError("`constr` parameter has to be an `Construct` object") + if not callable(constr): + raise ValueError("`constr` parameter has to be a function or lambda") if not isinstance(reverse_fields, bool): # type: ignore raise ValueError("`reverse_fields` parameter has to be an `bool` object") @@ -290,14 +275,12 @@ class DataclassStruct: dataclasses.dataclass(cls, kw_only=True) # type: ignore # create construct format - dc_constr = DataclassConstruct(cls, reverse_fields) - if constr is this_struct: - constr = dc_constr - else: - _replace_this_struct(constr, dc_constr) + dc_constr = constr(DataclassConstruct(cls, reverse_fields)) + if not isinstance(dc_constr, cs.Construct): # type: ignore + raise ValueError("`constr` sould return a `Construct` object") # save construct format and make the class compatible to `Constructable` protocol - setattr(cls, "__constr__", lambda: constr) + setattr(cls, "__constr__", lambda: dc_constr) # the `construct` library is using the [] access internally, so struct objects # should also make this possible and not only via the dot access. @@ -379,8 +362,10 @@ class DataclassBitStruct(DataclassStruct): @classmethod def __init_subclass__( - cls, - constr: "cs.Construct[t.Any, t.Any]" = this_struct, + cls: t.Type[T], + constr: t.Callable[ + [DataclassConstruct[T]], Construct[t.Any, t.Any] + ] = lambda cls: cls, reverse_fields: bool = False, ) -> None: - cls = DataclassStruct.__init_subclass__.__func__(cls, cs.Bitwise(constr), reverse_fields) # type: ignore + DataclassStruct.__init_subclass__.__func__(cls, lambda cls: cs.Bitwise(constr(cls)), reverse_fields) # type: ignore diff --git a/tests/test_typed.py b/tests/test_typed.py index 9cf4fc5..24e29a0 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -301,6 +301,29 @@ def test_dataclass_struct_doc() -> None: ) +def test_dataclass_bitwise() -> None: + class TestContainer(DataclassStruct, constr=lambda cls: cs.Bitwise(cls)): + a: int = csfield(cs.BitsInteger(7)) + b: int = csfield(cs.Bit) + c: int = csfield(cs.BitsInteger(8)) + + common( + constr(TestContainer), + b"\xFD\x12", + TestContainer(a=0x7E, b=1, c=0x12), + 2, + ) + + # check __getattr__ + c = TestContainer.__constr__() + assert c.subcon.a.name == "a" + assert c.subcon.b.name == "b" + assert c.subcon.c.name == "c" + assert isinstance(c.subcon.a.subcon, cs.BitsInteger) + assert c.subcon.b.subcon is cs.Bit + assert isinstance(c.subcon.c.subcon, cs.BitsInteger) + + def test_dataclass_bitstruct() -> None: class TestContainer(DataclassBitStruct): a: int = csfield(cs.BitsInteger(7)) From c5d08f9b87d63c0a5f09d092a68481f283f884e8 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 20:33:28 +0100 Subject: [PATCH 022/102] add docstring to construct `docs` --- construct_typed/dataclass_struct.py | 8 ++++ construct_typed/tenum.py | 13 ++++++- tests/test_typed.py | 57 ++++++++++++++++++++++------- 3 files changed, 62 insertions(+), 16 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index cee8c89..0b799ee 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -271,6 +271,11 @@ class DataclassStruct: if not isinstance(reverse_fields, bool): # type: ignore raise ValueError("`reverse_fields` parameter has to be an `bool` object") + # get documentation before creating the dataclass + docs = "" + if cls.__doc__ is not None: + docs = textwrap.dedent(cls.__doc__).strip("\n") + # create dataclass dataclasses.dataclass(cls, kw_only=True) # type: ignore @@ -279,6 +284,9 @@ class DataclassStruct: if not isinstance(dc_constr, cs.Construct): # type: ignore raise ValueError("`constr` sould return a `Construct` object") + # save docs + dc_constr.docs = docs + # save construct format and make the class compatible to `Constructable` protocol setattr(cls, "__constr__", lambda: dc_constr) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 83f1d8c..5cb9116 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -1,9 +1,10 @@ import enum +import textwrap import typing as t import construct as cs -from .generic import * +from construct_typed.generic import * T = t.TypeVar("T") @@ -26,6 +27,11 @@ class _EnumMeta(enum.EnumMeta): __namespace: t.Dict[str, t.Any], **kwargs: t.Any, ) -> T: + # get documentation before creating the enum + docs = "" + if "__doc__" in __namespace: + docs = textwrap.dedent(__namespace["__doc__"]).strip("\n") + # create new enum object cls: T = super().__new__(metacls, __name, __bases, __namespace) # type: ignore @@ -49,7 +55,10 @@ class _EnumMeta(enum.EnumMeta): elif TFlags in __bases: enum_constr = TFlagsConstruct(subcon, cls) # type: ignore else: - enum_constr = None + raise TypeError("neither `TEnum` nor `TFlags` in bases") + + # save documentation + enum_constr.docs = docs # save construct format and make the class compatible to `Constructable` protocol setattr(cls, "__constr__", lambda: enum_constr) # type: ignore diff --git a/tests/test_typed.py b/tests/test_typed.py index 24e29a0..f3d51d6 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -277,28 +277,39 @@ def test_dataclass_struct_wrong_container() -> None: def test_dataclass_struct_doc() -> None: - class TestContainer(DataclassStruct): - a: int = csfield(cs.Int16ub, doc="This is the documentation of a") - b: int = csfield( - cs.Int8ub, doc="This is the documentation of b\nwhich is multiline" - ) + class TestContainer1(DataclassStruct): + """ + Documentation of TestContainer + """ + + a: int = csfield(cs.Int16ub, doc="This is the doc of a") + b: int = csfield(cs.Int8ub, doc="This is the doc of b\nwhich is multiline") c: int = csfield( cs.Int8ub, doc=""" - This is the documentation of c + This is the doc of c which is also multiline """, ) - format = TestContainer.__constr__() - common(format, b"\x00\x01\x02\x03", TestContainer(a=1, b=2, c=3), 4) + format1 = TestContainer1.__constr__() + common(format1, b"\x00\x01\x02\x03", TestContainer1(a=1, b=2, c=3), 4) - assert format.subcon.a.docs == "This is the documentation of a" - assert format.subcon.b.docs == "This is the documentation of b\nwhich is multiline" - assert ( - format.subcon.c.docs - == "This is the documentation of c\nwhich is also multiline" - ) + assert format1.docs == "Documentation of TestContainer" + assert format1.subcon.a.docs == "This is the doc of a" + assert format1.subcon.b.docs == "This is the doc of b\nwhich is multiline" + assert format1.subcon.c.docs == "This is the doc of c\nwhich is also multiline" + + class TestContainer2(DataclassStruct): + a: int = csfield(cs.Int16ub) + b: int = csfield(cs.Int8ub) + c: int = csfield(cs.Int8ub) + + format2 = TestContainer2.__constr__() + assert format2.docs == "" + assert format2.subcon.a.docs == "" + assert format2.subcon.b.docs == "" + assert format2.subcon.c.docs == "" def test_dataclass_bitwise() -> None: @@ -367,6 +378,24 @@ def test_tenum() -> None: assert raises(d.build, 8) == TypeError +def test_tenum_doc() -> None: + class TestEnum1(TEnum, subcon=cs.Byte): + """ + TestEnum documentation + """ + + one = 1 + + d1 = constr(TestEnum1) + assert d1.docs == "TestEnum documentation" + + class TestEnum2(TEnum, subcon=cs.Byte): + two = 2 + + d2 = constr(TestEnum2) + assert d2.docs == "" + + def test_tenum_in_dataclass_struct() -> None: class TestEnum(TEnum, subcon=cs.Int8ub): a = 1 From af1a7799e9701eb2bb2c5a4ed3442dce7029b4cb Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 20:42:36 +0100 Subject: [PATCH 023/102] make consistent namings in tests --- tests/test_typed.py | 231 ++++++++++++++++++++++---------------------- 1 file changed, 115 insertions(+), 116 deletions(-) diff --git a/tests/test_typed.py b/tests/test_typed.py index f3d51d6..ed578c8 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -16,7 +16,7 @@ from tests.declarativeunittest import common, raises, setattrs def test_dataclass_const_default() -> None: - class ConstDefaultTest(DataclassStruct): + class TestDataclass(DataclassStruct): const_bytes: bytes = csfield(cs.Bytes(3), const=b"BMP") const_int: int = csfield(cs.Int8ub, const=5) default_int: int = csfield(cs.Int8ub, default=26) @@ -24,43 +24,42 @@ def test_dataclass_const_default() -> None: cs.Default(cs.Bytes(cs.this.const_int), lambda ctx: bytes(ctx.const_int)) ) - a = ConstDefaultTest() - assert a.const_bytes == b"BMP" - assert a.const_int == 5 - assert a.default_int == 26 - assert a.default_lambda == None - a = ConstDefaultTest(default_int=1) - assert a.default_int == 1 + obj = TestDataclass() + assert obj.const_bytes == b"BMP" + assert obj.const_int == 5 + assert obj.default_int == 26 + assert obj.default_lambda == None + obj = TestDataclass(default_int=1) + assert obj.default_int == 1 - format = ConstDefaultTest.__constr__() - assert isinstance(format.const_bytes.subcon, cs.Const) - assert isinstance(format.const_int.subcon, cs.Const) - assert isinstance(format.default_int.subcon, cs.Default) - assert isinstance(format.default_lambda.subcon, cs.Default) + fmt = TestDataclass.__constr__() + assert isinstance(fmt.const_bytes.subcon, cs.Const) + assert isinstance(fmt.const_int.subcon, cs.Const) + assert isinstance(fmt.default_int.subcon, cs.Default) + assert isinstance(fmt.default_lambda.subcon, cs.Default) def test_dataclass_access() -> None: - class TestTContainer(DataclassStruct): + class TestDataclass(DataclassStruct): a: int = csfield(cs.Byte, const=1) b: int = csfield(cs.Int8ub) - tcontainer = TestTContainer(b=2) + obj = TestDataclass(b=2) - # tcontainer - assert tcontainer.a == 1 - assert tcontainer["a"] == 1 - assert tcontainer.b == 2 - assert tcontainer["b"] == 2 + assert obj.a == 1 + assert obj["a"] == 1 + assert obj.b == 2 + assert obj["b"] == 2 - tcontainer.a = 5 - assert tcontainer.a == 5 - assert tcontainer["a"] == 5 - tcontainer["a"] = 6 - assert tcontainer.a == 6 - assert tcontainer["a"] == 6 + obj.a = 5 + assert obj.a == 5 + assert obj["a"] == 5 + obj["a"] = 6 + assert obj.a == 6 + assert obj["a"] == 6 # wrong creation - assert raises(lambda: TestTContainer(a=0, b=1)) == TypeError # type: ignore + assert raises(lambda: TestDataclass(a=0, b=1)) == TypeError # type: ignore def test_dataclass_str_repr() -> None: @@ -69,13 +68,13 @@ def test_dataclass_str_repr() -> None: width: int = csfield(cs.Int8ub) height: int = csfield(cs.Int8ub) - format = constr(Image) + fmt = constr(Image) obj = Image(width=3, height=2) assert ( str(obj) == "Image: \n signature = b'BMP' (total 3)\n width = 3\n height = 2" ) - obj = format.parse(format.build(obj)) + obj = fmt.parse(fmt.build(obj)) assert ( str(obj) == "Image: \n signature = b'BMP' (total 3)\n width = 3\n height = 2" @@ -95,28 +94,28 @@ def test_dataclass_struct() -> None: ) # check __getattr__ - c = Image.__constr__() - assert c.width.name == "width" - assert c.height.name == "height" - assert c.width.subcon is cs.Int8ub - assert c.height.subcon is cs.Int8ub + fmt = Image.__constr__() + assert fmt.width.name == "width" + assert fmt.height.name == "height" + assert fmt.width.subcon is cs.Int8ub + assert fmt.height.subcon is cs.Int8ub def test_dataclass_struct_reverse() -> None: - class TestContainer(DataclassStruct, reverse_fields=True): + class TestDataclass(DataclassStruct, reverse_fields=True): a: int = csfield(cs.Int16ub) b: int = csfield(cs.Int8ub) common( - constr(TestContainer), + constr(TestDataclass), b"\x02\x00\x01", - TestContainer(a=1, b=2), + TestDataclass(a=1, b=2), 3, ) def test_dataclass_struct_nested() -> None: - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): class InnerDataclass(DataclassStruct): b: int = csfield(cs.Byte) c: bytes = csfield(cs.Bytes(cs.this._.length)) @@ -125,9 +124,9 @@ def test_dataclass_struct_nested() -> None: a: InnerDataclass = csfield(constr(InnerDataclass)) common( - constr(TestContainer), + constr(TestDataclass), b"\x02\x01\xF1\xF2", - TestContainer(length=2, a=TestContainer.InnerDataclass(b=1, c=b"\xF1\xF2")), + TestDataclass(length=2, a=TestDataclass.InnerDataclass(b=1, c=b"\xF1\xF2")), ) @@ -151,67 +150,67 @@ def test_dataclass_struct_default_field() -> None: def test_dataclass_struct_const_field() -> None: - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): const_field: t.Optional[bytes] = csfield(cs.Const(b"\x00")) common( - constr(TestContainer), + constr(TestDataclass), bytes(1), - setattrs(TestContainer(), const_field=b"\x00"), + setattrs(TestDataclass(), const_field=b"\x00"), 1, ) assert ( raises( - constr(TestContainer).build, - setattrs(TestContainer(), const_field=b"\x01"), + constr(TestDataclass).build, + setattrs(TestDataclass(), const_field=b"\x01"), ) == cs.ConstError ) def test_dataclass_struct_array_field() -> None: - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): array_field: t.List[int] = csfield(cs.Array(5, cs.Int8ub)) common( - constr(TestContainer), + constr(TestDataclass), bytes(5), - TestContainer(array_field=[0, 0, 0, 0, 0]), + TestDataclass(array_field=[0, 0, 0, 0, 0]), 5, ) def test_dataclass_struct_anonymus_fields_1() -> None: - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): _1: t.Optional[bytes] = csfield(cs.Const(b"\x00")) _2: None = csfield(cs.Padding(1)) _3: None = csfield(cs.Pass) _4: None = csfield(cs.Terminated) common( - constr(TestContainer), + constr(TestDataclass), bytes(2), - setattrs(TestContainer(), _1=b"\x00"), + setattrs(TestDataclass(), _1=b"\x00"), cs.SizeofError, ) def test_dataclass_struct_anonymus_fields_2() -> None: - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): _1: t.Optional[int] = csfield(cs.Computed(7)) _2: t.Optional[bytes] = csfield(cs.Const(b"JPEG")) _3: None = csfield(cs.Pass) _4: None = csfield(cs.Terminated) - d = constr(TestContainer) - assert d.build(TestContainer()) == d.build(TestContainer()) + fmt = constr(TestDataclass) + assert fmt.build(TestDataclass()) == fmt.build(TestDataclass()) def test_dataclass_struct_overloaded_method() -> None: # Test dot access to some names that are not accessable via dot # in the original 'cs.Container'. - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): clear: int = csfield(cs.Int8ul) copy: int = csfield(cs.Int8ul) fromkeys: int = csfield(cs.Int8ul) @@ -227,10 +226,10 @@ def test_dataclass_struct_overloaded_method() -> None: update: int = csfield(cs.Int8ul) values: int = csfield(cs.Int8ul) - d = constr(TestContainer) - obj = d.parse( - d.build( - TestContainer( + fmt = constr(TestDataclass) + obj = fmt.parse( + fmt.build( + TestDataclass( clear=1, copy=2, fromkeys=3, @@ -277,9 +276,9 @@ def test_dataclass_struct_wrong_container() -> None: def test_dataclass_struct_doc() -> None: - class TestContainer1(DataclassStruct): + class TestDataclass1(DataclassStruct): """ - Documentation of TestContainer + Documentation of TestDataclass1 """ a: int = csfield(cs.Int16ub, doc="This is the doc of a") @@ -292,70 +291,70 @@ def test_dataclass_struct_doc() -> None: """, ) - format1 = TestContainer1.__constr__() - common(format1, b"\x00\x01\x02\x03", TestContainer1(a=1, b=2, c=3), 4) + fmt1 = TestDataclass1.__constr__() + common(fmt1, b"\x00\x01\x02\x03", TestDataclass1(a=1, b=2, c=3), 4) - assert format1.docs == "Documentation of TestContainer" - assert format1.subcon.a.docs == "This is the doc of a" - assert format1.subcon.b.docs == "This is the doc of b\nwhich is multiline" - assert format1.subcon.c.docs == "This is the doc of c\nwhich is also multiline" + assert fmt1.docs == "Documentation of TestDataclass1" + assert fmt1.subcon.a.docs == "This is the doc of a" + assert fmt1.subcon.b.docs == "This is the doc of b\nwhich is multiline" + assert fmt1.subcon.c.docs == "This is the doc of c\nwhich is also multiline" - class TestContainer2(DataclassStruct): + class TestDataclass2(DataclassStruct): a: int = csfield(cs.Int16ub) b: int = csfield(cs.Int8ub) c: int = csfield(cs.Int8ub) - format2 = TestContainer2.__constr__() - assert format2.docs == "" - assert format2.subcon.a.docs == "" - assert format2.subcon.b.docs == "" - assert format2.subcon.c.docs == "" + fmt2 = TestDataclass2.__constr__() + assert fmt2.docs == "" + assert fmt2.subcon.a.docs == "" + assert fmt2.subcon.b.docs == "" + assert fmt2.subcon.c.docs == "" def test_dataclass_bitwise() -> None: - class TestContainer(DataclassStruct, constr=lambda cls: cs.Bitwise(cls)): + class TestDataclass(DataclassStruct, constr=lambda cls: cs.Bitwise(cls)): a: int = csfield(cs.BitsInteger(7)) b: int = csfield(cs.Bit) c: int = csfield(cs.BitsInteger(8)) common( - constr(TestContainer), + constr(TestDataclass), b"\xFD\x12", - TestContainer(a=0x7E, b=1, c=0x12), + TestDataclass(a=0x7E, b=1, c=0x12), 2, ) # check __getattr__ - c = TestContainer.__constr__() - assert c.subcon.a.name == "a" - assert c.subcon.b.name == "b" - assert c.subcon.c.name == "c" - assert isinstance(c.subcon.a.subcon, cs.BitsInteger) - assert c.subcon.b.subcon is cs.Bit - assert isinstance(c.subcon.c.subcon, cs.BitsInteger) + fmt = TestDataclass.__constr__() + assert fmt.subcon.a.name == "a" + assert fmt.subcon.b.name == "b" + assert fmt.subcon.c.name == "c" + assert isinstance(fmt.subcon.a.subcon, cs.BitsInteger) + assert fmt.subcon.b.subcon is cs.Bit + assert isinstance(fmt.subcon.c.subcon, cs.BitsInteger) def test_dataclass_bitstruct() -> None: - class TestContainer(DataclassBitStruct): + class TestDataclass(DataclassBitStruct): a: int = csfield(cs.BitsInteger(7)) b: int = csfield(cs.Bit) c: int = csfield(cs.BitsInteger(8)) common( - constr(TestContainer), + constr(TestDataclass), b"\xFD\x12", - TestContainer(a=0x7E, b=1, c=0x12), + TestDataclass(a=0x7E, b=1, c=0x12), 2, ) # check __getattr__ - c = TestContainer.__constr__() - assert c.subcon.a.name == "a" - assert c.subcon.b.name == "b" - assert c.subcon.c.name == "c" - assert isinstance(c.subcon.a.subcon, cs.BitsInteger) - assert c.subcon.b.subcon is cs.Bit - assert isinstance(c.subcon.c.subcon, cs.BitsInteger) + fmt = TestDataclass.__constr__() + assert fmt.subcon.a.name == "a" + assert fmt.subcon.b.name == "b" + assert fmt.subcon.c.name == "c" + assert isinstance(fmt.subcon.a.subcon, cs.BitsInteger) + assert fmt.subcon.b.subcon is cs.Bit + assert isinstance(fmt.subcon.c.subcon, cs.BitsInteger) def test_tenum() -> None: @@ -365,17 +364,17 @@ def test_tenum() -> None: four = 4 eight = 8 - d = constr(TestEnum) + fmt = constr(TestEnum) - common(d, b"\x01", TestEnum.one, 1) - common(d, b"\xff", TestEnum(255), 1) - assert d.parse(b"\x01") == TestEnum.one - assert d.parse(b"\x01") == 1 - assert int(d.parse(b"\x01")) == 1 - assert d.parse(b"\xff") == TestEnum(255) - assert d.parse(b"\xff") == 255 - assert int(d.parse(b"\xff")) == 255 - assert raises(d.build, 8) == TypeError + common(fmt, b"\x01", TestEnum.one, 1) + common(fmt, b"\xff", TestEnum(255), 1) + assert fmt.parse(b"\x01") == TestEnum.one + assert fmt.parse(b"\x01") == 1 + assert int(fmt.parse(b"\x01")) == 1 + assert fmt.parse(b"\xff") == TestEnum(255) + assert fmt.parse(b"\xff") == 255 + assert int(fmt.parse(b"\xff")) == 255 + assert raises(fmt.build, 8) == TypeError def test_tenum_doc() -> None: @@ -401,19 +400,19 @@ def test_tenum_in_dataclass_struct() -> None: a = 1 b = 2 - class TestContainer(DataclassStruct): + class TestDataclass(DataclassStruct): a: TestEnum = csfield(constr(TestEnum)) b: int = csfield(cs.Int8ub) common( - constr(TestContainer), + constr(TestDataclass), b"\x01\x02", - TestContainer(a=TestEnum.a, b=2), + TestDataclass(a=TestEnum.a, b=2), 2, ) assert ( - raises(constr(TestEnum).build, TestContainer(a=1, b=2)) == TypeError # type: ignore + raises(constr(TestEnum).build, TestDataclass(a=1, b=2)) == TypeError # type: ignore ) @@ -424,12 +423,12 @@ def test_tflags() -> None: four = 4 eight = 8 - d = constr(TestFlags) - common(d, b"\x03", TestFlags.one | TestFlags.two, 1) - assert d.build(TestFlags(0)) == b"\x00" - assert d.build(TestFlags.one | TestFlags.two) == b"\x03" - assert d.build(TestFlags(8)) == b"\x08" - assert d.build(TestFlags(1 | 2)) == b"\x03" - assert d.build(TestFlags(255)) == b"\xff" - assert d.build(TestFlags.eight) == b"\x08" - assert raises(d.build, 2) == TypeError + fmt = constr(TestFlags) + common(fmt, b"\x03", TestFlags.one | TestFlags.two, 1) + assert fmt.build(TestFlags(0)) == b"\x00" + assert fmt.build(TestFlags.one | TestFlags.two) == b"\x03" + assert fmt.build(TestFlags(8)) == b"\x08" + assert fmt.build(TestFlags(1 | 2)) == b"\x03" + assert fmt.build(TestFlags(255)) == b"\xff" + assert fmt.build(TestFlags.eight) == b"\x08" + assert raises(fmt.build, 2) == TypeError From e730fd8d879466ecc7aecedccb5c11c89b8064a8 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 20 Feb 2022 21:14:05 +0100 Subject: [PATCH 024/102] use a special MISSING enum, because typing.Literal only accepts enums --- construct_typed/dataclass_struct.py | 40 +++++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 0b799ee..2e23dbc 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -4,6 +4,7 @@ import dataclasses import sys import textwrap import typing as t +import enum import construct as cs from construct.lib.containers import ( @@ -37,17 +38,22 @@ def __dataclass_transform__( return lambda a: a +# this is nessesary, because typing.Literal type has to be an enum +class Flag(enum.Enum): + MISSING = dataclasses.MISSING + + DATACLASS_METADATA_KEY = "__construct_typed_subcon" # specialisation for constructs, that builds from none -> this field does not appear in the __init__ method and has a default of None @t.overload -def csfield( - subcon: "cs.Construct[ParsedType, None]", +def csfield( # type: ignore + subcon: "Construct[ParsedType, None]", *, doc: t.Optional[str] = ..., - init: t.Literal[False] = False, - default: t.Literal[None] = None, - const: t.Literal[dataclasses.MISSING] = ..., + init: t.Literal[False] = ..., + default: t.Literal[None] = ..., + const: t.Literal[Flag.MISSING] = ..., ) -> t.Optional[ParsedType]: ... @@ -59,8 +65,8 @@ def csfield( *, doc: t.Optional[str] = ..., init: t.Literal[True] = ..., - default: t.Literal[dataclasses.MISSING] = ..., - const: t.Literal[dataclasses.MISSING] = ..., + default: t.Literal[Flag.MISSING] = ..., + const: t.Literal[Flag.MISSING] = ..., ) -> ParsedType: ... @@ -72,7 +78,7 @@ def csfield( *, doc: t.Optional[str] = ..., init: t.Literal[False] = ..., - default: t.Literal[dataclasses.MISSING] = ..., + default: t.Literal[Flag.MISSING] = ..., const: t.Optional[ParsedType] = ..., ) -> ParsedType: ... @@ -86,7 +92,7 @@ def csfield( doc: t.Optional[str] = ..., init: t.Literal[True] = ..., default: t.Optional[ParsedType] = ..., - const: t.Literal[dataclasses.MISSING] = ..., + const: t.Literal[Flag.MISSING] = ..., ) -> ParsedType: ... @@ -95,10 +101,10 @@ def csfield( subcon: "Construct[ParsedType, t.Any]", *, doc: t.Optional[str] = None, - init: bool = True, # dont use this parameter, this is only used for `dataclass_transform` - default: t.Optional[t.Any] = dataclasses.MISSING, - const: t.Optional[t.Any] = dataclasses.MISSING, -) -> ParsedType: + init: bool = True, # dont use `init`, this is only used for `dataclass_transform` + default: t.Optional[t.Any] = Flag.MISSING, + const: t.Optional[t.Any] = Flag.MISSING, +) -> t.Optional[ParsedType]: """ Helper method for "DataclassStruct" and "DataclassBitStruct" to create the dataclass fields. @@ -107,7 +113,7 @@ def csfield( Only one of the parameters `default` or `const` can be vaild. They are mutually exclusive. """ - if (default is not dataclasses.MISSING) and (const is not dataclasses.MISSING): + if (default is not Flag.MISSING) and (const is not Flag.MISSING): raise ValueError("default and const are mutally exclusive") # Rename subcon, if doc is available @@ -115,11 +121,11 @@ def csfield( doc = textwrap.dedent(doc).strip("\n") subcon = cs.Renamed(subcon, newdocs=doc) - if default is not dataclasses.MISSING: + if default is not Flag.MISSING: init = True default = default subcon = cs.Default(subcon, default) - elif const is not dataclasses.MISSING: + elif const is not Flag.MISSING: init = False default = const subcon = cs.Const(const, subcon) @@ -131,7 +137,7 @@ def csfield( default = dataclasses.MISSING return t.cast( - ParsedType, + t.Optional[ParsedType], dataclasses.field( default=default, init=init, From 47fd2838f7e059951f253675297fa27de16f88b2 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:12:49 +0200 Subject: [PATCH 025/102] changed requirements for Python 3.11 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2d70b89..2c2ee3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ construct==2.10.67 pytest>=6.2.0 -numpy==1.21.* +numpy==1.23.* arrow ruamel.yaml cloudpickle From 19fd0bf7f354c414c03b066265c30006e8af4518 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:12:55 +0200 Subject: [PATCH 026/102] Fixed Error in Python 3.11: "_sunder_ names, such as '_create_pseudo_member_', are reserved for future Enum use" --- construct_typed/tenum.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 7c93e33..e66c11e 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -18,11 +18,11 @@ class EnumBase(enum.IntEnum): @classmethod def _missing_(cls, value: t.Any) -> t.Optional["EnumBase"]: if isinstance(value, int): - return cls._create_pseudo_member_(value) + return cls._create_pseudo_member(value) return None # will raise the ValueError in Enum.__new__ @classmethod - def _create_pseudo_member_(cls, value: int) -> "EnumBase": + def _create_pseudo_member(cls, value: int) -> "EnumBase": pseudo_member = cls._value2member_map_.get(value, None) # type: ignore if pseudo_member is None: new_member = int.__new__(cls, value) From 18597a05db90f70e9b4c5d31a6bb4e954dce713a Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:19:07 +0200 Subject: [PATCH 027/102] changed requirements to use the latest numpy, because numpy 1.23.* is not supporting Python 3.7. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2c2ee3f..87d1d1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ construct==2.10.67 pytest>=6.2.0 -numpy==1.23.* +numpy arrow ruamel.yaml cloudpickle From 94896208b985fda53cc0fc2d1c634e91423bf850 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:24:08 +0200 Subject: [PATCH 028/102] fixed mypy error: `error: Returning Any from function declared to return "int"` --- tests/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 6f65772..759dbe8 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1213,7 +1213,7 @@ def test_checksum() -> None: def test_checksum_nonbytes_issue_323() -> None: d = Struct( "vals" / Byte[2], - "checksum" / Checksum(Byte, lambda vals: sum(vals) & 0xFF, this.vals), + "checksum" / Checksum(Byte, lambda vals: int(sum(vals)) & 0xFF, this.vals), ) assert d.parse(b"\x00\x00\x00") == Container(vals=[0, 0], checksum=0) assert raises(d.parse, b"\x00\x00\x01") == ChecksumError @@ -1704,7 +1704,7 @@ def test_from_issue_324() -> None: )), "checksum" / Checksum( Byte, - lambda data: sum(data) & 0xFF, + lambda data: int(sum(data)) & 0xFF, this.vals.data ), ) From a03190d816121a7e92f58cbd7a98b65f9277e19e Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 22:48:42 +0200 Subject: [PATCH 029/102] fixed some pyright/pylance issues --- construct-stubs/core.pyi | 19 ++++++++++++------- construct-stubs/expr.pyi | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 010f471..324f010 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -482,8 +482,8 @@ class RepeatUntil( Subconstruct[ SubconParsedType, SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], + ParsedType, + BuildTypes, ] ): predicate: t.Union[ @@ -491,8 +491,8 @@ class RepeatUntil( t.Callable[[SubconParsedType, ListContainer[SubconParsedType], Context], bool], ] discard: bool - def __init__( - self, + def __new__( + cls, predicate: t.Union[ bool, t.Callable[ @@ -501,7 +501,12 @@ class RepeatUntil( ], subcon: Construct[SubconParsedType, SubconBuildTypes], discard: bool = ..., - ) -> None: ... + ) -> RepeatUntil[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ]: ... # =============================================================================== # specials @@ -994,7 +999,7 @@ class Restreamed( ) -> None: ... class ProcessXor( - Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconParsedType] + Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): padfunc: ConstantOrContextLambda2[t.Union[int, bytes]] def __new__( @@ -1004,7 +1009,7 @@ class ProcessXor( ) -> ProcessXor[SubconParsedType, SubconBuildTypes]: ... class ProcessRotateLeft( - Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconParsedType] + Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): amount: ConstantOrContextLambda2[int] group: ConstantOrContextLambda2[int] diff --git a/construct-stubs/expr.pyi b/construct-stubs/expr.pyi index 8450a44..56fae52 100644 --- a/construct-stubs/expr.pyi +++ b/construct-stubs/expr.pyi @@ -543,7 +543,7 @@ class Path2(ExprMixin[ReturnType]): class FuncPath(ExprMixin[ReturnType]): - def __init__(self, func: t.Callable[[t.Any], t.Any], operand: t.Optional[t.Any] = ...) -> None: ... + def __init__(self, func: t.Callable[[t.Any], ReturnType], operand: t.Optional[t.Any] = ...) -> None: ... def __call__(self, operand: t.Any, *args: t.Any) -> ReturnType: ... From aeff7c6b7cc02813ac43038933230ac2e6cada56 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 23:03:40 +0200 Subject: [PATCH 030/102] remove github actions deprecation warnings. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4a3e3db..48d1816 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: steps: # Checks out a copy of your repository on the machine - name: Checkout code - uses: actions/checkout@v1 + uses: actions/checkout@v3 # Setup python - name: Setup python From 2b780ae7b969182f190fd6311cd024bc7914c044 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 Oct 2022 23:49:58 +0200 Subject: [PATCH 031/102] changed `Adapter` `__new__` to `__init__` to fix #13. But this change on `Adapter` also reqires to change other `Adapter` subclasses. --- construct-stubs/core.pyi | 79 +++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 18 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 324f010..22fc43d 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -182,9 +182,9 @@ class Subconstruct( class Adapter( Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], ): - def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes] - ) -> Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... + def __init__( + self, subcon: Construct[SubconParsedType, SubconBuildTypes] + ) -> None: ... def _decode( self, obj: SubconBuildTypes, context: Context, path: PathType ) -> ParsedType: ... @@ -391,6 +391,12 @@ class Enum(Adapter[int, int, ParsedType, BuildTypes]): *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **mapping: int ) -> Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]]: ... + def __init__( + self, + subcon: Construct[int, int], + *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], + **mapping: int + ) -> None: ... def __getattr__(self, name: str) -> EnumIntegerString: ... class BitwisableString(str): @@ -405,6 +411,12 @@ class FlagsEnum(Adapter[int, int, ParsedType, BuildTypes]): *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **flags: int ) -> FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]]: ... + def __init__( + self, + subcon: Construct[int, int], + *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], + **flags: int + ) -> None: ... def __getattr__(self, name: str) -> BitwisableString: ... class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): @@ -415,6 +427,11 @@ class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): subcon: Construct[SubconParsedType, SubconBuildTypes], mapping: t.Dict[t.Any, t.Any], ) -> Mapping[t.Any, t.Any]: ... + def __init__( + self, + subcon: Construct[SubconParsedType, SubconBuildTypes], + mapping: t.Dict[t.Any, t.Any], + ) -> None: ... # =============================================================================== # structures and sequences @@ -492,7 +509,7 @@ class RepeatUntil( ] discard: bool def __new__( - cls, + cls, predicate: t.Union[ bool, t.Callable[ @@ -615,6 +632,12 @@ class NamedTuple( t.Tuple[t.Any, ...], t.Union[t.Tuple[t.Any, ...], t.List[t.Any], t.Dict[str, t.Any]], ]: ... + def __init__( + self, + tuplename: str, + tuplefields: str, + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... if sys.version_info >= (3, 8): MSDOS = t.Literal["msdos"] @@ -1121,30 +1144,28 @@ class LazyBound(Construct[ParsedType, BuildTypes]): # adapters and validators # =============================================================================== class ExprAdapter(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - def __new__( - cls, + def __init__( + self, subcon: Construct[SubconParsedType, SubconBuildTypes], decoder: t.Callable[[SubconParsedType, Context], ParsedType], encoder: t.Callable[[BuildTypes, Context], SubconBuildTypes], - ) -> ExprAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... + ) -> None: ... class ExprSymmetricAdapter( ExprAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes] ): - def __new__( - cls, + def __init__( + self, subcon: Construct[SubconParsedType, SubconBuildTypes], encoder: t.Callable[[BuildTypes, Context], SubconBuildTypes], - ) -> ExprSymmetricAdapter[ - SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes - ]: ... + ) -> None: ... class ExprValidator(Validator[SubconParsedType, SubconBuildTypes]): - def __new__( - cls, + def __init__( + self, subcon: Construct[SubconParsedType, SubconBuildTypes], validator: t.Callable[[SubconParsedType, Context], bool], - ) -> ExprValidator[SubconParsedType, SubconBuildTypes]: ... + ) -> None: ... def OneOf( subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1186,12 +1207,34 @@ class Slicing( step: int = ..., empty: t.Optional[SubconParsedType] = ..., ) -> Slicing[ListContainer[SubconParsedType], t.List[SubconBuildTypes]]: ... + def __init__( + self, + subcon: t.Union[ + Array[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + GreedyRange[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + ], + count: int, + start: t.Optional[int], + stop: t.Optional[int], + step: int = ..., + empty: t.Optional[SubconParsedType] = ..., + ) -> None: ... class Indexing( Adapter[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): - def __new__( - cls, + def __init__( + self, subcon: t.Union[ Array[ SubconParsedType, @@ -1209,4 +1252,4 @@ class Indexing( count: int, index: int, empty: t.Optional[SubconParsedType] = ..., - ) -> Indexing[SubconParsedType, SubconBuildTypes]: ... + ) -> None: ... From e10938f30838a13f27cae3c0ff5865ee4606b847 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Mon, 24 Oct 2022 00:07:57 +0200 Subject: [PATCH 032/102] Added type hint to self type, for mypy support. --- construct-stubs/core.pyi | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 22fc43d..7e6dfd1 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -25,6 +25,11 @@ from construct.lib import ( # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python +# The type checkers mypy and pyright/pylance unfortunately work a little bit different with __init__ and __new__. +# For supporting some constructs (eg. Enum, NamedTuple, Slicing) in mypy the __init__ self parameter has to have a +# type hint. But for supporting pyright/pylance, the same type hint has to be used as the return type of __new__. +# (see discussion here: https://github.com/python/typeshed/issues/4846). + StreamType = t.BinaryIO FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str @@ -392,7 +397,7 @@ class Enum(Adapter[int, int, ParsedType, BuildTypes]): **mapping: int ) -> Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]]: ... def __init__( - self, + self: Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]], subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **mapping: int @@ -412,7 +417,7 @@ class FlagsEnum(Adapter[int, int, ParsedType, BuildTypes]): **flags: int ) -> FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]]: ... def __init__( - self, + self: FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]], subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **flags: int @@ -428,7 +433,7 @@ class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): mapping: t.Dict[t.Any, t.Any], ) -> Mapping[t.Any, t.Any]: ... def __init__( - self, + self: Mapping[t.Any, t.Any], subcon: Construct[SubconParsedType, SubconBuildTypes], mapping: t.Dict[t.Any, t.Any], ) -> None: ... @@ -633,7 +638,12 @@ class NamedTuple( t.Union[t.Tuple[t.Any, ...], t.List[t.Any], t.Dict[str, t.Any]], ]: ... def __init__( - self, + self: NamedTuple[ + SubconParsedType, + SubconBuildTypes, + t.Tuple[t.Any, ...], + t.Union[t.Tuple[t.Any, ...], t.List[t.Any], t.Dict[str, t.Any]], + ], tuplename: str, tuplefields: str, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1208,7 +1218,7 @@ class Slicing( empty: t.Optional[SubconParsedType] = ..., ) -> Slicing[ListContainer[SubconParsedType], t.List[SubconBuildTypes]]: ... def __init__( - self, + self: Slicing[ListContainer[SubconParsedType], t.List[SubconBuildTypes]], subcon: t.Union[ Array[ SubconParsedType, From 92b3555c508d32eaaf81ebd40cab5d8385f78094 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Tue, 25 Oct 2022 19:26:37 +0200 Subject: [PATCH 033/102] removed `_create_pseudo_member` completely and integrated it into `_missing_` --- construct_typed/tenum.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index e66c11e..6855f3d 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -12,27 +12,23 @@ class EnumBase(enum.IntEnum): This class extends the standard `enum.IntEnum`, so that missing values are automatically generated. """ - # Extend the enum type with __missing__ method. So if a enum value + # Extend the enum type with _missing_ method. So if a enum value # not found in the enum, a new pseudo member is created. # The idea is taken from: https://stackoverflow.com/a/57179436 @classmethod - def _missing_(cls, value: t.Any) -> t.Optional["EnumBase"]: + def _missing_(cls, value: t.Any) -> t.Optional[enum.Enum]: if isinstance(value, int): - return cls._create_pseudo_member(value) + pseudo_member = cls._value2member_map_.get(value, None) + if pseudo_member is None: + new_member = int.__new__(cls, value) + # I expect a name attribute to hold a string, hence str(value) + # However, new_member._name_ = value works, too + new_member._name_ = str(value) + new_member._value_ = value + pseudo_member = cls._value2member_map_.setdefault(value, new_member) + return pseudo_member return None # will raise the ValueError in Enum.__new__ - @classmethod - def _create_pseudo_member(cls, value: int) -> "EnumBase": - pseudo_member = cls._value2member_map_.get(value, None) # type: ignore - if pseudo_member is None: - new_member = int.__new__(cls, value) - # I expect a name attribute to hold a string, hence str(value) - # However, new_member._name_ = value works, too - new_member._name_ = str(value) - new_member._value_ = value - pseudo_member = cls._value2member_map_.setdefault(value, new_member) # type: ignore - return pseudo_member # type: ignore - EnumType = t.TypeVar("EnumType", bound=EnumBase) From ce871936d9e885c84f2f507c723fc5a7e99c9096 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Tue, 25 Oct 2022 20:49:07 +0200 Subject: [PATCH 034/102] Added Python 3.10 and 3.11 to the Test-Matrix and removed github warning "Node.js 12 actions are deprecated." --- .github/workflows/main.yml | 8 ++++---- setup.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48d1816..3bdc963 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'windows-latest'] - python-version: [ '3.7', '3.8', '3.9' ] + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] runs-on: ${{ matrix.os }} name: OS ${{ matrix.os }}, Python ${{ matrix.python-version }} @@ -20,16 +20,16 @@ jobs: # Setup python - name: Setup python - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 # Setup node.js (for pyright) - name: Setup node.js (for pyright) - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: - node-version: '14' + node-version: 16 # Install pyright - name: Install pyright diff --git a/setup.py b/setup.py index 6cdeb7a..ca87762 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,8 @@ setup( "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: Implementation :: CPython", "Typing :: Typed", ], From 25883cb7c8eae9f8ddfd241b84504d859e59710e Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:54:15 +0100 Subject: [PATCH 035/102] Updated settings.json --- .vscode/settings.json | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 76bdfe9..75b5040 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,24 +1,26 @@ { - "python.pythonPath": "python", + // static analysis "python.languageServer": "Pylance", - // "python.testing.unittestEnabled": false, - // "python.testing.nosetestsEnabled": false, - // "python.testing.pytestEnabled": true, - "pythonTestExplorer.testFramework": "pytest", - "python.formatting.provider": "black", - "python.sortImports.path": "isort", - "python.sortImports.args": [ - "--profile=black", - ], - // "[python]": { - // "editor.codeActionsOnSave": { - // "source.organizeImports": true - // } - // } "python.analysis.typeCheckingMode": "strict", "python.analysis.autoImportCompletions": false, "python.analysis.diagnosticSeverityOverrides": { "reportPrivateUsage": "information", "reportUntypedNamedTuple": "information", }, + + // formating + "python.formatting.provider": "black", + + // sorting + "python.sortImports.path": "isort", + "python.sortImports.args": [ + "--profile=black", + ], + + // tests + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, } \ No newline at end of file From f3051a1116d0f3767ff27dc9e6a036e637c92fe2 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Mon, 31 Oct 2022 09:54:50 +0100 Subject: [PATCH 036/102] upgrade requirement to construct==2.10.68 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ca87762..54dd52d 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( url="https://github.com/timrid/construct-typing", author="Tim Riddermann", python_requires=">=3.7", - install_requires=["construct==2.10.67"], + install_requires=["construct==2.10.68"], keywords=[ "construct", "kaitai", From ceb4b67ff8a1a3800b06c6468798c0c72b4f9edf Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Mon, 31 Oct 2022 10:06:15 +0100 Subject: [PATCH 037/102] incremented version to 0.5.3 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index 1a555cf..ce77de7 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 5, 2) -version_string = "0.5.2" \ No newline at end of file +version = (0, 5, 3) +version_string = "0.5.3" \ No newline at end of file From 349c8e5dd214c4e1bbdf027a3bf995d7b483dde3 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Fri, 23 Dec 2022 20:37:05 +0100 Subject: [PATCH 038/102] Updated IfThenElse so that it represents the real implementation --- construct-stubs/core.pyi | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 7e6dfd1..32b4b77 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -767,23 +767,23 @@ ThenBuildTypes = t.TypeVar("ThenBuildTypes") ElseParsedType = t.TypeVar("ElseParsedType") ElseBuildTypes = t.TypeVar("ElseBuildTypes") -# This does not represent the original code, but it is the only solution that works good with pyright -class _IfThenElse(Construct[ParsedType, BuildTypes]): +class IfThenElse(Construct[ParsedType, BuildTypes]): condfunc: ConstantOrContextLambda[bool] thensubcon: Construct[ParsedType, BuildTypes] elsesubcon: Construct[ParsedType, BuildTypes] + def __new__( + cls, + condfunc: ConstantOrContextLambda[bool], + thensubcon: Construct[ThenParsedType, ThenBuildTypes], + elsesubcon: Construct[ElseParsedType, ElseBuildTypes], + ) -> IfThenElse[ + t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] + ]: ... -def IfThenElse( - condfunc: ConstantOrContextLambda[bool], - thensubcon: Construct[ThenParsedType, ThenBuildTypes], - elsesubcon: Construct[ElseParsedType, ElseBuildTypes], -) -> _IfThenElse[ - t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] -]: ... def If( condfunc: ConstantOrContextLambda[bool], subcon: Construct[ThenParsedType, ThenBuildTypes], -) -> _IfThenElse[t.Union[ThenParsedType, None], t.Union[ThenBuildTypes, None]]: ... +) -> IfThenElse[t.Union[ThenParsedType, None], t.Union[ThenBuildTypes, None]]: ... SwitchType = t.TypeVar("SwitchType") From b7e92c3c7d7fa3c41cc5043c1c2cc915f0686b9a Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Fri, 23 Dec 2022 20:37:38 +0100 Subject: [PATCH 039/102] removed mypy error 'Unused "type: ignore" comment' --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index 759dbe8..95e5e59 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +# mypy: no-warn-unused-ignores from .declarativeunittest import raises, common, commonhex, commondumpdeprecated, commondump, commonbytes, ident, devzero from construct.core import * from construct import * From 916349f876c23e191973eb17bbb969e478858ed7 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 24 Dec 2022 12:00:23 +0100 Subject: [PATCH 040/102] enhanced `EnumBase` and `FlagsEnumBase` to support induvidual documentation for each enum value via `EnumValue` --- construct_typed/__init__.py | 3 +- construct_typed/tenum.py | 91 ++++++++++++++++++++++++++++++++++++- tests/test_typed.py | 72 +++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 00d5093..e052ee5 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -18,7 +18,7 @@ from .generic_wrapper import ( ListContainer, PathType, ) -from .tenum import EnumBase, FlagsEnumBase, TEnum, TFlagsEnum +from .tenum import EnumBase, EnumValue, FlagsEnumBase, TEnum, TFlagsEnum __all__ = [ "DataclassBitStruct", @@ -32,6 +32,7 @@ __all__ = [ "csfield", "sfield", "EnumBase", + "EnumValue", "FlagsEnumBase", "TEnum", "TFlagsEnum", diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 6855f3d..3ead0dd 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -5,13 +5,55 @@ from .generic_wrapper import * # ## TEnum ############################################################################################################ +class EnumValue: + """ + This is a helper class for adding documentation to an enum value. + """ + + def __init__(self, value: int, doc: t.Optional[str] = None) -> None: + self.value = value + self.__doc__ = doc if doc else "" + + def __int__(self) -> int: + return self.value + + class EnumBase(enum.IntEnum): """ Base class for an Enum used in `construct_typed.TEnum`. - This class extends the standard `enum.IntEnum`, so that missing values are automatically generated. + This class extends the standard `enum.IntEnum` by. + - missing values are automatically generated + - possibility to add documentation for each enum value (see `EnumValue`) + + Example:: + + >>> class State(EnumBase): + ... Idle = 1 + ... Running = EnumValue(2, "This is the running state.") + + >>> State(1) + + + >>> State["Idle"] + + + >>> State.Idle + + + >>> State(3) # missing value + + + >>> State.Running.__doc__ # documentation + 'This is the running state.' """ + def __init__(self, val: t.Union[EnumValue, int]): + if isinstance(val, EnumValue): + self.__doc__ = val.__doc__ + else: + self.__doc__ = "" + # Extend the enum type with _missing_ method. So if a enum value # not found in the enum, a new pseudo member is created. # The idea is taken from: https://stackoverflow.com/a/57179436 @@ -25,6 +67,7 @@ class EnumBase(enum.IntEnum): # However, new_member._name_ = value works, too new_member._name_ = str(value) new_member._value_ = value + new_member.__doc__ = "missing value" pseudo_member = cls._value2member_map_.setdefault(value, new_member) return pseudo_member return None # will raise the ValueError in Enum.__new__ @@ -75,7 +118,51 @@ class TEnum(Adapter[int, int, EnumType, EnumType]): # ## TFlagsEnum ####################################################################################################### class FlagsEnumBase(enum.IntFlag): - pass + """ + Base class for an Enum used in `construct_typed.TFlagsEnum`. + + This class extends the standard `enum.IntFlag` by. + - possibility to add documentation for each enum value (see `EnumValue`) + + Example:: + + >>> class Option(FlagsEnumBase): + ... OptOne = 1 + ... OptTwo = EnumValue(2, "This is option two.") + + >>> Option(1) + + + >>> Option["OptOne"] + + + >>> Option.OptOne + + + >>> Option(3) + + + >>> Option(4) + + + >>> Option.OptTwo.__doc__ # documentation + 'This is option two.' + """ + + def __init__(self, val: t.Union[EnumValue, int]): + if isinstance(val, EnumValue): + self.__doc__ = val.__doc__ + else: + self.__doc__ = "" + + @classmethod + def _missing_(cls, value: t.Any) -> t.Any: + """ + Returns member (possibly creating it) if one can be found for value. + """ + new_member = super()._missing_(value) + new_member.__doc__ = "missing value" + return new_member FlagsEnumType = t.TypeVar("FlagsEnumType", bound=FlagsEnumBase) diff --git a/tests/test_typed.py b/tests/test_typed.py index d74e025..756b0f6 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -384,6 +384,42 @@ def test_tenum_no_enumbase() -> None: assert raises(lambda: cst.TEnum(cs.Byte, cls)) == TypeError +def test_tenum_docstring() -> None: + class TestEnum(cst.EnumBase): + """ + This is an test enum. + """ + + Value_WithDoc = cst.EnumValue(0, doc="an enum with a documentation") + Value_WithMultilineDoc = cst.EnumValue( + 1, + """ + An enum with a multiline documentation... + ...next line... + """, + ) + Value_NoDoc = cst.EnumValue(2) + Value_NoDoc2 = 3 + + assert ( + TestEnum.__doc__ + == """ + This is an test enum. + """ + ) + assert TestEnum.Value_WithDoc.__doc__ == "an enum with a documentation" + assert ( + TestEnum.Value_WithMultilineDoc.__doc__ + == """ + An enum with a multiline documentation... + ...next line... + """ + ) + assert TestEnum.Value_NoDoc.__doc__ == "" + assert TestEnum.Value_NoDoc2.__doc__ == "" + assert TestEnum(5).__doc__ == "missing value" + + def test_dataclass_struct_wrong_enumbase() -> None: class E1(cst.EnumBase): a = 1 @@ -434,3 +470,39 @@ def test_tenum_flags() -> None: assert d.build(TestEnum(255)) == b"\xff" assert d.build(TestEnum.eight) == b"\x08" assert raises(d.build, 2) == TypeError + + +def test_tenum_flags_docstring() -> None: + class TestEnum(cst.FlagsEnumBase): + """ + This is an test flags enum. + """ + + Value_WithDoc = cst.EnumValue(0, doc="an enum with a documentation") + Value_WithMultilineDoc = cst.EnumValue( + 1, + """ + An enum with a multiline documentation... + ...next line... + """, + ) + Value_NoDoc = cst.EnumValue(2) + Value_NoDoc2 = 4 + + assert ( + TestEnum.__doc__ + == """ + This is an test flags enum. + """ + ) + assert TestEnum.Value_WithDoc.__doc__ == "an enum with a documentation" + assert ( + TestEnum.Value_WithMultilineDoc.__doc__ + == """ + An enum with a multiline documentation... + ...next line... + """ + ) + assert TestEnum.Value_NoDoc.__doc__ == "" + assert TestEnum.Value_NoDoc2.__doc__ == "" + assert TestEnum(8).__doc__ == "missing value" From ed5d0fe826a229a31fc70ad9fa587accd1c91291 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sat, 24 Dec 2022 12:20:03 +0100 Subject: [PATCH 041/102] incremented version to 0.5.4 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index ce77de7..e9cf6fc 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 5, 3) -version_string = "0.5.3" \ No newline at end of file +version = (0, 5, 4) +version_string = "0.5.4" \ No newline at end of file From 31a10dcc88e2a757107edf9530120bb7e2864a71 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 10:32:19 +0100 Subject: [PATCH 042/102] added type hints for `Struct._subcons`, `Sequence._subcons`, `FocusedSeq._subcons`, `Union._subcons`, `LazyStruct._subcons`, `LazyStruct._subconsindexes` --- construct-stubs/core.pyi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 32b4b77..0ff8888 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -444,6 +444,7 @@ class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): # this can maybe made better when variadic generics are available class Struct(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] + _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __new__( cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] ) -> Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... @@ -452,6 +453,7 @@ class Struct(Construct[ParsedType, BuildTypes]): # this can maybe made better when variadic generics are available class Sequence(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] + _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __new__( cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] ) -> Sequence[ListContainer[t.Any], t.Optional[t.List[t.Any]]]: ... @@ -603,6 +605,7 @@ Error: Construct[None, None] class FocusedSeq(Construct[t.Any, t.Any]): subcons: t.List[Construct[t.Any, t.Any]] + _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __init__( self, parsebuildfrom: ConstantOrContextLambda[str], @@ -743,6 +746,7 @@ class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes class Union(Construct[Container[t.Any], t.Dict[str, t.Any]]): parsefrom: t.Optional[ConstantOrContextLambda[t.Union[int, str]]] subcons: t.List[Construct[t.Any, t.Any]] + _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __init__( self, parsefrom: t.Optional[ConstantOrContextLambda[t.Union[int, str]]], @@ -1117,6 +1121,8 @@ class LazyContainer(t.Generic[ContainerType], t.Dict[str, ContainerType]): class LazyStruct(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] + _subcons: t.Dict[str, Construct[t.Any, t.Any]] + _subconsindexes: t.Dict[str, Construct[t.Any, t.Any]] def __new__( cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] ) -> LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... From 747bfaebd132d025001f914a82012d7cdcee5099 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 10:32:44 +0100 Subject: [PATCH 043/102] added generic wrapper for cs.Array --- construct_typed/generic_wrapper.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index aa4af4c..99d8026 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -37,5 +37,11 @@ else: class Context: pass + class Array( + t.Generic[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], + cs.Array, + ): + pass + ConstantOrContextLambda = t.Union[ValueType, t.Callable[[Context], t.Any]] PathType = str From f98532a382fba26b1aa41b22da83ea52f6d06fba Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 10:35:07 +0100 Subject: [PATCH 044/102] fixed _subconsindexes --- construct-stubs/core.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 0ff8888..bf280c0 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -1122,7 +1122,7 @@ class LazyContainer(t.Generic[ContainerType], t.Dict[str, ContainerType]): class LazyStruct(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] - _subconsindexes: t.Dict[str, Construct[t.Any, t.Any]] + _subconsindexes: t.Dict[str, int] def __new__( cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] ) -> LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... From b5add648ed3aedbcd09330594b79c16315466a06 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 10:45:55 +0100 Subject: [PATCH 045/102] fixed pyright 1.1.287 error --- tests/test_core.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 95e5e59..a50b7c2 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1033,13 +1033,14 @@ def test_prefixed() -> None: common(d5, b"\x0a"+bytes(10), u"\x00"*10, SizeofError) def test_prefixedarray() -> None: - common(PrefixedArray(Byte,Byte), b"\x02\x0a\x0b", [10,11], SizeofError) - assert PrefixedArray(Byte, Byte).parse(b"\x03\x01\x02\x03") == [1,2,3] - assert PrefixedArray(Byte, Byte).parse(b"\x00") == [] - assert PrefixedArray(Byte, Byte).build([1,2,3]) == b"\x03\x01\x02\x03" - assert raises(PrefixedArray(Byte, Byte).parse, b"") == StreamError - assert raises(PrefixedArray(Byte, Byte).parse, b"\x03\x01") == StreamError - assert raises(PrefixedArray(Byte, Byte).sizeof) == SizeofError + d = PrefixedArray(Byte, Byte) + common(d, b"\x02\x0a\x0b", [10,11], SizeofError) + assert d.parse(b"\x03\x01\x02\x03") == [1,2,3] + assert d.parse(b"\x00") == [] + assert d.build([1,2,3]) == b"\x03\x01\x02\x03" + assert raises(d.parse, b"") == StreamError + assert raises(d.parse, b"\x03\x01") == StreamError + assert raises(d.sizeof) == SizeofError def test_fixedsized() -> None: d1 = FixedSized(10, Byte) From 7f6577af470221c8a65bdf5a0e45006705f75de4 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 12:00:42 +0100 Subject: [PATCH 046/102] added `__init__` methods for all that currently only defined an `__new__` method --- construct-stubs/core.pyi | 378 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 353 insertions(+), 25 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index bf280c0..d62f962 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -183,6 +183,19 @@ class Subconstruct( def __new__( cls, *args: t.Any, **kwargs: t.Any ) -> Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... + @t.overload + def __init__( + self: Subconstruct[ + SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], + *args: t.Any, + **kwargs: t.Any, + ) -> None: ... class Adapter( Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], @@ -224,8 +237,13 @@ class Tunnel( class Bytes(Construct[ParsedType, BuildTypes]): length: ConstantOrContextLambda[int] def __new__( - cls, length: ConstantOrContextLambda[int] + cls, + length: ConstantOrContextLambda[int], ) -> Bytes[bytes, t.Union[bytes, int]]: ... + def __init__( + self: Bytes[bytes, t.Union[bytes, int]], + length: ConstantOrContextLambda[int], + ) -> None: ... GreedyBytes: Construct[bytes, bytes] @@ -255,20 +273,63 @@ class FormatField(Construct[ParsedType, BuildTypes]): FORMAT_BOOL = t.Literal["?"] @t.overload def __new__( - cls, endianity: str, format: FORMAT_INT + cls, + endianity: str, + format: FORMAT_INT, ) -> FormatField[int, int]: ... @t.overload def __new__( - cls, endianity: str, format: FORMAT_FLOAT + cls, + endianity: str, + format: FORMAT_FLOAT, ) -> FormatField[float, float]: ... @t.overload def __new__( - cls, endianity: str, format: FORMAT_BOOL + cls, + endianity: str, + format: FORMAT_BOOL, ) -> FormatField[bool, bool]: ... @t.overload - def __new__(cls, endianity: str, format: str) -> FormatField[t.Any, t.Any]: ... + def __new__( + cls, + endianity: str, + format: str, + ) -> FormatField[t.Any, t.Any]: ... + @t.overload + def __init__( + self: FormatField[int, int], + endianity: str, + format: FORMAT_INT, + ) -> None: ... + @t.overload + def __init__( + self: FormatField[float, float], + endianity: str, + format: FORMAT_FLOAT, + ) -> None: ... + @t.overload + def __init__( + self: FormatField[bool, bool], + endianity: str, + format: FORMAT_BOOL, + ) -> None: ... + @t.overload + def __init__( + self: FormatField[t.Any, t.Any], + endianity: str, + format: str, + ) -> None: ... else: - def __new__(cls, endianity: str, format: str) -> FormatField[t.Any, t.Any]: ... + def __new__( + cls, + endianity: str, + format: str, + ) -> FormatField[t.Any, t.Any]: ... + def __init__( + self: FormatField[t.Any, t.Any], + endianity: str, + format: str, + ) -> None: ... class BytesInteger(Construct[ParsedType, BuildTypes]): length: ConstantOrContextLambda[int] @@ -280,6 +341,12 @@ class BytesInteger(Construct[ParsedType, BuildTypes]): signed: bool = ..., swapped: ConstantOrContextLambda[bool] = ..., ) -> BytesInteger[int, int]: ... + def __init__( + self: BytesInteger[int, int], + length: ConstantOrContextLambda[int], + signed: bool = ..., + swapped: ConstantOrContextLambda[bool] = ..., + ) -> None: ... class BitsInteger(Construct[ParsedType, BuildTypes]): length: ConstantOrContextLambda[int] @@ -291,6 +358,12 @@ class BitsInteger(Construct[ParsedType, BuildTypes]): signed: bool = ..., swapped: ConstantOrContextLambda[bool] = ..., ) -> BitsInteger[int, int]: ... + def __init__( + self: BitsInteger[int, int], + length: ConstantOrContextLambda[int], + signed: bool = ..., + swapped: ConstantOrContextLambda[bool] = ..., + ) -> None: ... Bit: BitsInteger[int, int] Nibble: BitsInteger[int, int] @@ -363,8 +436,15 @@ class StringEncoded(Construct[ParsedType, BuildTypes]): ENCODING = str encoding: ENCODING def __new__( - cls, subcon: Construct[ParsedType, BuildTypes], encoding: ENCODING + cls, + subcon: Construct[ParsedType, BuildTypes], + encoding: ENCODING, ) -> StringEncoded[str, str]: ... + def __init__( + self: StringEncoded[str, str], + subcon: Construct[ParsedType, BuildTypes], + encoding: ENCODING, + ) -> None: ... def PaddedString( length: ConstantOrContextLambda[int], encoding: StringEncoded.ENCODING @@ -394,13 +474,13 @@ class Enum(Adapter[int, int, ParsedType, BuildTypes]): cls, subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], - **mapping: int + **mapping: int, ) -> Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]]: ... def __init__( self: Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]], subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], - **mapping: int + **mapping: int, ) -> None: ... def __getattr__(self, name: str) -> EnumIntegerString: ... @@ -414,13 +494,13 @@ class FlagsEnum(Adapter[int, int, ParsedType, BuildTypes]): cls, subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], - **flags: int + **flags: int, ) -> FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]]: ... def __init__( self: FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]], subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], - **flags: int + **flags: int, ) -> None: ... def __getattr__(self, name: str) -> BitwisableString: ... @@ -446,8 +526,15 @@ class Struct(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __new__( - cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] + cls, + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], ) -> Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... + def __init__( + self: Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]], + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], + ) -> None: ... def __getattr__(self, name: str) -> t.Any: ... # this can maybe made better when variadic generics are available @@ -455,8 +542,15 @@ class Sequence(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __new__( - cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] + cls, + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], ) -> Sequence[ListContainer[t.Any], t.Optional[t.List[t.Any]]]: ... + def __init__( + self: Sequence[ListContainer[t.Any], t.Optional[t.List[t.Any]]], + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], + ) -> None: ... def __getattr__(self, name: str) -> t.Any: ... # =============================================================================== @@ -483,6 +577,17 @@ class Array( ListContainer[SubconParsedType], t.List[SubconBuildTypes], ]: ... + def __init__( + self: Array[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + count: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + discard: bool = ..., + ) -> None: ... class GreedyRange( Subconstruct[ @@ -494,13 +599,25 @@ class GreedyRange( ): discard: bool def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes], discard: bool = ... + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + discard: bool = ..., ) -> GreedyRange[ SubconParsedType, SubconBuildTypes, ListContainer[SubconParsedType], t.List[SubconBuildTypes], ]: ... + def __init__( + self: GreedyRange[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + discard: bool = ..., + ) -> None: ... class RepeatUntil( Subconstruct[ @@ -531,6 +648,22 @@ class RepeatUntil( ListContainer[SubconParsedType], t.List[SubconBuildTypes], ]: ... + def __init__( + self: RepeatUntil[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + predicate: t.Union[ + bool, + t.Callable[ + [SubconParsedType, ListContainer[SubconParsedType], Context], bool + ], + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + discard: bool = ..., + ) -> None: ... # =============================================================================== # specials @@ -567,12 +700,24 @@ class Computed(Construct[ParsedType, BuildTypes]): func: ConstantOrContextLambda2[ParsedType] @t.overload def __new__( - cls, func: ConstantOrContextLambda2[ParsedType] + cls, + func: ConstantOrContextLambda2[ParsedType], ) -> Computed[ParsedType, None]: ... @t.overload def __new__( - cls, func: ConstantOrContextLambda2[t.Any] + cls, + func: ConstantOrContextLambda2[t.Any], ) -> Computed[t.Any, None]: ... + @t.overload + def __init__( + self: Computed[ParsedType, None], + func: ConstantOrContextLambda2[ParsedType], + ) -> None: ... + @t.overload + def __init__( + self: Computed[t.Any, None], + func: ConstantOrContextLambda2[t.Any], + ) -> None: ... Index: Construct[int, t.Any] @@ -583,6 +728,11 @@ class Rebuild(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, Build subcon: Construct[SubconParsedType, SubconBuildTypes], func: ConstantOrContextLambda[SubconBuildTypes], ) -> Rebuild[SubconParsedType, SubconBuildTypes, SubconParsedType, None]: ... + def __init__( + self: Rebuild[SubconParsedType, SubconBuildTypes, SubconParsedType, None], + subcon: Construct[SubconParsedType, SubconBuildTypes], + func: ConstantOrContextLambda[SubconBuildTypes], + ) -> None: ... class Default(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): value: ConstantOrContextLambda[SubconBuildTypes] @@ -596,10 +746,27 @@ class Default(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, Build SubconParsedType, t.Optional[SubconBuildTypes], ]: ... + def __init__( + self: Default[ + SubconParsedType, + SubconBuildTypes, + SubconParsedType, + t.Optional[SubconBuildTypes], + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + value: ConstantOrContextLambda[SubconBuildTypes], + ) -> None: ... class Check(Construct[ParsedType, BuildTypes]): func: ConstantOrContextLambda[bool] - def __new__(cls, func: ConstantOrContextLambda[bool]) -> Check[None, None]: ... + def __new__( + cls, + func: ConstantOrContextLambda[bool], + ) -> Check[None, None]: ... + def __init__( + self: Check[None, None], + func: ConstantOrContextLambda[bool], + ) -> None: ... Error: Construct[None, None] @@ -610,7 +777,7 @@ class FocusedSeq(Construct[t.Any, t.Any]): self, parsebuildfrom: ConstantOrContextLambda[str], *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any] + **subconskw: Construct[t.Any, t.Any], ) -> None: ... def __getattr__(self, name: str) -> t.Any: ... @@ -711,6 +878,40 @@ class Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): ) -> Hex[ SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes ]: ... + @t.overload + def __init__( + self: Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes], + subcon: Construct[int, BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes], + subcon: Construct[bytes, BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: Hex[ + RawCopyObj[SubconParsedType], + BuildTypes, + HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], + BuildTypes, + ], + subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: Hex[ + Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes + ], + subcon: Construct[Container[t.Any], BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: Hex[ + SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): @t.overload @@ -738,6 +939,35 @@ class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes ) -> HexDump[ SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes ]: ... + @t.overload + def __init__( + self: HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes], + subcon: Construct[bytes, BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: HexDump[ + RawCopyObj[SubconParsedType], + BuildTypes, + HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], + BuildTypes, + ], + subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: HexDump[ + Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes + ], + subcon: Construct[Container[t.Any], BuildTypes], + ) -> None: ... + @t.overload + def __init__( + self: HexDump[ + SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... # =============================================================================== # conditional @@ -751,7 +981,7 @@ class Union(Construct[Container[t.Any], t.Dict[str, t.Any]]): self, parsefrom: t.Optional[ConstantOrContextLambda[t.Union[int, str]]], *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any] + **subconskw: Construct[t.Any, t.Any], ) -> None: ... def __getattr__(self, name: str) -> t.Any: ... @@ -759,8 +989,15 @@ class Union(Construct[Container[t.Any], t.Dict[str, t.Any]]): class Select(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] def __new__( - cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] + cls, + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], ) -> Select[t.Any, t.Any]: ... + def __init__( + self: Select[t.Any, t.Any], + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], + ) -> None: ... def Optional( subcon: Construct[SubconParsedType, SubconBuildTypes] @@ -783,6 +1020,15 @@ class IfThenElse(Construct[ParsedType, BuildTypes]): ) -> IfThenElse[ t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] ]: ... + def __init__( + self: IfThenElse[ + t.Union[ThenParsedType, ElseParsedType], + t.Union[ThenBuildTypes, ElseBuildTypes], + ], + condfunc: ConstantOrContextLambda[bool], + thensubcon: Construct[ThenParsedType, ThenBuildTypes], + elsesubcon: Construct[ElseParsedType, ElseBuildTypes], + ) -> None: ... def If( condfunc: ConstantOrContextLambda[bool], @@ -809,10 +1055,31 @@ class Switch(Construct[ParsedType, BuildTypes]): cases: t.Dict[t.Any, Construct[t.Any, t.Any]], default: t.Optional[Construct[t.Any, t.Any]] = ..., ) -> Switch[t.Any, t.Any]: ... + @t.overload + def __init__( + self: Switch[int, t.Optional[int]], + keyfunc: ConstantOrContextLambda[SwitchType], + cases: t.Dict[SwitchType, Construct[int, int]], + default: t.Optional[Construct[int, int]] = ..., + ) -> None: ... + @t.overload + def __init__( + self: Switch[t.Any, t.Any], + keyfunc: ConstantOrContextLambda[t.Any], + cases: t.Dict[t.Any, Construct[t.Any, t.Any]], + default: t.Optional[Construct[t.Any, t.Any]] = ..., + ) -> None: ... class StopIf(Construct[ParsedType, BuildTypes]): condfunc: ConstantOrContextLambda[bool] - def __new__(cls, condfunc: ConstantOrContextLambda[bool]) -> StopIf[None, None]: ... + def __new__( + cls, + condfunc: ConstantOrContextLambda[bool], + ) -> StopIf[None, None]: ... + def __init__( + self: StopIf[None, None], + condfunc: ConstantOrContextLambda[bool], + ) -> None: ... # =============================================================================== # alignment and padding @@ -848,7 +1115,7 @@ class Aligned( def AlignedStruct( modulus: ConstantOrContextLambda[int], *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any] + **subconskw: Construct[t.Any, t.Any], ) -> Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... def BitStruct( *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] @@ -882,6 +1149,15 @@ class Peek(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTyp SubconParsedType, t.Union[SubconBuildTypes, None], ]: ... + def __init__( + self: Peek[ + SubconParsedType, + SubconBuildTypes, + SubconParsedType, + t.Union[SubconBuildTypes, None], + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... class Seek(Construct[int, None]): at: ConstantOrContextLambda[int] @@ -913,13 +1189,23 @@ class RawCopyObj(t.Generic[ParsedType], Container[t.Any]): class RawCopy(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes] + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> RawCopy[ SubconParsedType, SubconBuildTypes, RawCopyObj[SubconParsedType], t.Optional[t.Dict[str, t.Union[SubconBuildTypes, bytes]]], ]: ... + def __init__( + self: RawCopy[ + SubconParsedType, + SubconBuildTypes, + RawCopyObj[SubconParsedType], + t.Optional[t.Dict[str, t.Union[SubconBuildTypes, bytes]]], + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... def ByteSwapped( subcon: Construct[SubconParsedType, SubconBuildTypes] @@ -1044,6 +1330,11 @@ class ProcessXor( padfunc: ConstantOrContextLambda2[t.Union[int, bytes]], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> ProcessXor[SubconParsedType, SubconBuildTypes]: ... + def __init__( + self: ProcessXor[SubconParsedType, SubconBuildTypes], + padfunc: ConstantOrContextLambda2[t.Union[int, bytes]], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... class ProcessRotateLeft( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] @@ -1056,6 +1347,12 @@ class ProcessRotateLeft( group: ConstantOrContextLambda2[int], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> ProcessRotateLeft[SubconParsedType, SubconBuildTypes]: ... + def __init__( + self: ProcessRotateLeft[SubconParsedType, SubconBuildTypes], + amount: ConstantOrContextLambda2[int], + group: ConstantOrContextLambda2[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... T = t.TypeVar("T") @@ -1111,6 +1408,15 @@ class Lazy(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTyp t.Callable[[], SubconParsedType], t.Union[t.Callable[[], SubconParsedType], SubconParsedType], ]: ... + def __init__( + self: Lazy[ + SubconParsedType, + SubconBuildTypes, + t.Callable[[], SubconParsedType], + t.Union[t.Callable[[], SubconParsedType], SubconParsedType], + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... class LazyContainer(t.Generic[ContainerType], t.Dict[str, ContainerType]): def __getattr__(self, name: str) -> ContainerType: ... @@ -1124,8 +1430,15 @@ class LazyStruct(Construct[ParsedType, BuildTypes]): _subcons: t.Dict[str, Construct[t.Any, t.Any]] _subconsindexes: t.Dict[str, int] def __new__( - cls, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] + cls, + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], ) -> LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... + def __init__( + self: LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]], + *subcons: Construct[t.Any, t.Any], + **subconskw: Construct[t.Any, t.Any], + ) -> None: ... def __getattr__(self, name: str) -> t.Any: ... class LazyListContainer(t.List[ListType]): ... @@ -1149,12 +1462,27 @@ class LazyArray( ListContainer[SubconParsedType], t.List[SubconBuildTypes], ]: ... + def __init__( + self: LazyArray[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + count: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... class LazyBound(Construct[ParsedType, BuildTypes]): subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]] def __new__( - cls, subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]] + cls, + subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]], ) -> LazyBound[ParsedType, BuildTypes]: ... + def __init__( + self: LazyBound[ParsedType, BuildTypes], + subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]], + ) -> None: ... # =============================================================================== # adapters and validators From 232fcbd725416d44ec486aa3b01a543402b57cdb Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 12:00:56 +0100 Subject: [PATCH 047/102] inserted missing asserts --- tests/test_core.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index a50b7c2..4b1e4d0 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -224,8 +224,8 @@ def test_paddedstring() -> None: common(PaddedString(100, e), data, s, 100) for e in ["ascii","utf8","utf16","utf-16-le","utf32","utf-32-le"]: - PaddedString(10, e).sizeof() == 10 - PaddedString(this.n, e).sizeof(n=10) == 10 + assert PaddedString(10, e).sizeof() == 10 + assert PaddedString(this.n, e).sizeof(n=10) == 10 def test_pascalstring() -> None: for e,_ in [("utf8",1),("utf16",2),("utf_16_le",2),("utf32",4),("utf_32_le",4)]: @@ -236,8 +236,8 @@ def test_pascalstring() -> None: common(PascalString(sc, e), sc.build(0), u"") for e in ["utf8","utf16","utf-16-le","utf32","utf-32-le","ascii"]: - raises(PascalString(Byte, e).sizeof) == SizeofError - raises(PascalString(VarInt, e).sizeof) == SizeofError + assert raises(PascalString(Byte, e).sizeof) == SizeofError + assert raises(PascalString(VarInt, e).sizeof) == SizeofError def test_cstring() -> None: s = u"" @@ -246,12 +246,12 @@ def test_cstring() -> None: common(CString(e), s.encode(e)+bytes(us), s) common(CString(e), bytes(us), u"") - CString("utf8").build(s) == b'\xd0\x90\xd1\x84\xd0\xbe\xd0\xbd'+b"\x00" - CString("utf16").build(s) == b'\xff\xfe\x10\x04D\x04>\x04=\x04'+b"\x00\x00" - CString("utf32").build(s) == b'\xff\xfe\x00\x00\x10\x04\x00\x00D\x04\x00\x00>\x04\x00\x00=\x04\x00\x00'+b"\x00\x00\x00\x00" + assert CString("utf8").build(s) == b'\xd0\x90\xd1\x84\xd0\xbe\xd0\xbd'+b"\x00" + assert CString("utf16").build(s) == b'\xff\xfe\x10\x04D\x04>\x04=\x04'+b"\x00\x00" + assert CString("utf32").build(s) == b'\xff\xfe\x00\x00\x10\x04\x00\x00D\x04\x00\x00>\x04\x00\x00=\x04\x00\x00'+b"\x00\x00\x00\x00" for e in ["utf8","utf16","utf-16-le","utf32","utf-32-le","ascii"]: - raises(CString(e).sizeof) == SizeofError + assert raises(CString(e).sizeof) == SizeofError def test_greedystring() -> None: for e,_ in [("utf8",1),("utf16",2),("utf_16_le",2),("utf32",4),("utf_32_le",4)]: @@ -260,7 +260,7 @@ def test_greedystring() -> None: common(GreedyString(e), b"", u"") for e in ["utf8","utf16","utf-16-le","utf32","utf-32-le","ascii"]: - raises(GreedyString(e).sizeof) == SizeofError + assert raises(GreedyString(e).sizeof) == SizeofError def test_string_encodings() -> None: # checks that "-" is replaced with "_" @@ -271,7 +271,7 @@ def test_flag() -> None: d = Flag common(d, b"\x00", False, 1) common(d, b"\x01", True, 1) - d.parse(b"\xff") == True + assert d.parse(b"\xff") == True def test_enum() -> None: d = Enum(Byte, one=1, two=2, four=4, eight=8) @@ -591,7 +591,7 @@ def test_rebuild_issue_664() -> None: def test_default() -> None: d = Default(Byte, 0) common(d, b"\xff", 255, 1) - d.build(None) == b"\x00" + assert d.build(None) == b"\x00" def test_check() -> None: common(Check(True), b"", None, 0) From 238ca5d2dfeff18d471f10d82829aa8b9789c2be Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 6 Jan 2023 12:05:14 +0100 Subject: [PATCH 048/102] removed not working __init__ --- construct-stubs/core.pyi | 9 --------- 1 file changed, 9 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index d62f962..83a7525 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -1020,15 +1020,6 @@ class IfThenElse(Construct[ParsedType, BuildTypes]): ) -> IfThenElse[ t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] ]: ... - def __init__( - self: IfThenElse[ - t.Union[ThenParsedType, ElseParsedType], - t.Union[ThenBuildTypes, ElseBuildTypes], - ], - condfunc: ConstantOrContextLambda[bool], - thensubcon: Construct[ThenParsedType, ThenBuildTypes], - elsesubcon: Construct[ElseParsedType, ElseBuildTypes], - ) -> None: ... def If( condfunc: ConstantOrContextLambda[bool], From 80b11b2ef9f4b942699ec324435adcdf19d54630 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 9 Jan 2023 08:18:16 +0100 Subject: [PATCH 049/102] added missing parts for cst.Array --- construct_typed/__init__.py | 2 ++ construct_typed/generic_wrapper.py | 1 + 2 files changed, 3 insertions(+) diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index e052ee5..9ea0ccf 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -17,6 +17,7 @@ from .generic_wrapper import ( Context, ListContainer, PathType, + Array ) from .tenum import EnumBase, EnumValue, FlagsEnumBase, TEnum, TFlagsEnum @@ -42,4 +43,5 @@ __all__ = [ "Context", "ListContainer", "PathType", + "Array" ] diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index 99d8026..f570f70 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -16,6 +16,7 @@ if t.TYPE_CHECKING: from construct import Context as Context from construct import ListContainer as ListContainer from construct import PathType as PathType + from construct import Array as Array else: From bea99456d7c2288c5b5cb2db32af79302fa0ce39 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 9 Jan 2023 08:58:11 +0100 Subject: [PATCH 050/102] using __new__ instead of __init__ for `EnumBase` to create enum member objects (fixes #18) --- construct_typed/tenum.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 3ead0dd..3712957 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -14,9 +14,6 @@ class EnumValue: self.value = value self.__doc__ = doc if doc else "" - def __int__(self) -> int: - return self.value - class EnumBase(enum.IntEnum): """ @@ -48,11 +45,16 @@ class EnumBase(enum.IntEnum): 'This is the running state.' """ - def __init__(self, val: t.Union[EnumValue, int]): + def __new__(cls, val: t.Union[EnumValue, int]) -> "EnumBase": if isinstance(val, EnumValue): - self.__doc__ = val.__doc__ + obj = int.__new__(cls, val.value) + obj._value_ = val.value + obj.__doc__ = val.__doc__ else: - self.__doc__ = "" + obj = int.__new__(cls, val) + obj._value_ = val + obj.__doc__ = "" + return obj # Extend the enum type with _missing_ method. So if a enum value # not found in the enum, a new pseudo member is created. @@ -149,11 +151,16 @@ class FlagsEnumBase(enum.IntFlag): 'This is option two.' """ - def __init__(self, val: t.Union[EnumValue, int]): + def __new__(cls, val: t.Union[EnumValue, int]) -> "FlagsEnumBase": if isinstance(val, EnumValue): - self.__doc__ = val.__doc__ + obj = int.__new__(cls, val.value) + obj._value_ = val.value + obj.__doc__ = val.__doc__ else: - self.__doc__ = "" + obj = int.__new__(cls, val) + obj._value_ = val + obj.__doc__ = "" + return obj @classmethod def _missing_(cls, value: t.Any) -> t.Any: From acc3fa344396eeb986859e94a1e761eff67c9a1c Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 9 Jan 2023 13:23:37 +0100 Subject: [PATCH 051/102] incremented version to 0.5.5 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index e9cf6fc..ee25123 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 5, 4) -version_string = "0.5.4" \ No newline at end of file +version = (0, 5, 5) +version_string = "0.5.5" From ab33490dec3644f5d123994e56088b345f1bfd3b Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 10:46:36 +0200 Subject: [PATCH 052/102] Pickle enums by value instead of name (restores pre-3.11 behavior) to support `dataclasses.asdict` --- construct_typed/tenum.py | 14 +++++++++++ tests/test_typed.py | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 3712957..08e4a4b 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -74,6 +74,13 @@ class EnumBase(enum.IntEnum): return pseudo_member return None # will raise the ValueError in Enum.__new__ + def __reduce_ex__(self, proto: t.Any): + """ + Pickle enums by value instead of name (restores pre-3.11 behavior). + See https://github.com/python/cpython/pull/26658 for why this exists. + """ + return self.__class__, (self._value_,) + EnumType = t.TypeVar("EnumType", bound=EnumBase) @@ -171,6 +178,13 @@ class FlagsEnumBase(enum.IntFlag): new_member.__doc__ = "missing value" return new_member + def __reduce_ex__(self, proto: t.Any): + """ + Pickle enums by value instead of name (restores pre-3.11 behavior). + See https://github.com/python/cpython/pull/26658 for why this exists. + """ + return self.__class__, (self._value_,) + FlagsEnumType = t.TypeVar("FlagsEnumType", bound=FlagsEnumBase) diff --git a/tests/test_typed.py b/tests/test_typed.py index 756b0f6..4dd9528 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -384,6 +384,32 @@ def test_tenum_no_enumbase() -> None: assert raises(lambda: cst.TEnum(cs.Byte, cls)) == TypeError +def test_tenum_asdict(): + # see: https://github.com/timrid/construct-typing/issues/21 + import construct_typed as cst + import dataclasses + + class TestEnum(cst.EnumBase): + one = 1 + two = 2 + four = 4 + eight = 8 + + @dataclasses.dataclass + class SomeDataclass: + a: TestEnum + + dc = SomeDataclass(TestEnum.one) + dc_dict = dataclasses.asdict(dc) + assert dc_dict["a"] == dc.a + assert dc_dict["a"] is dc.a + + dc = SomeDataclass(TestEnum(5)) + dc_dict = dataclasses.asdict(dc) + assert dc_dict["a"] == dc.a + assert dc_dict["a"] is dc.a + + def test_tenum_docstring() -> None: class TestEnum(cst.EnumBase): """ @@ -472,6 +498,31 @@ def test_tenum_flags() -> None: assert raises(d.build, 2) == TypeError +def test_tenum_flags_asdict(): + import construct_typed as cst + import dataclasses + + class TestEnum(cst.FlagsEnumBase): + one = 1 + two = 2 + four = 4 + eight = 8 + + @dataclasses.dataclass + class SomeDataclass: + a: TestEnum + + dc = SomeDataclass(TestEnum.one) + dc_dict = dataclasses.asdict(dc) + assert dc_dict["a"] == dc.a + assert dc_dict["a"] is dc.a + + dc = SomeDataclass(TestEnum(5)) + dc_dict = dataclasses.asdict(dc) + assert dc_dict["a"] == dc.a + assert dc_dict["a"] is dc.a + + def test_tenum_flags_docstring() -> None: class TestEnum(cst.FlagsEnumBase): """ From 50a2f34fc68d3f81262814ba6d8980e5b056a0ec Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 11:12:42 +0200 Subject: [PATCH 053/102] fixed mypy issue --- construct_typed/tenum.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 08e4a4b..a71fb7f 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -74,7 +74,7 @@ class EnumBase(enum.IntEnum): return pseudo_member return None # will raise the ValueError in Enum.__new__ - def __reduce_ex__(self, proto: t.Any): + def __reduce_ex__(self, proto: t.Any) -> t.Tuple[t.Any, ...]: """ Pickle enums by value instead of name (restores pre-3.11 behavior). See https://github.com/python/cpython/pull/26658 for why this exists. @@ -178,7 +178,7 @@ class FlagsEnumBase(enum.IntFlag): new_member.__doc__ = "missing value" return new_member - def __reduce_ex__(self, proto: t.Any): + def __reduce_ex__(self, proto: t.Any) -> t.Tuple[t.Any, ...]: """ Pickle enums by value instead of name (restores pre-3.11 behavior). See https://github.com/python/cpython/pull/26658 for why this exists. From 3a676478ee56349b6fa80e5ecef12752a6bd1fd4 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 11:27:19 +0200 Subject: [PATCH 054/102] fixed mypy issues --- construct_typed/dataclass_struct.py | 4 +++- tests/test_typed.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 78d2e59..d72ce5c 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -27,6 +27,8 @@ class DataclassMixin: methods exists and every name can be used. """ + __dataclass_fields__: t.ClassVar[dict[str, dataclasses.Field[t.Any]]] + def __getitem__(self, key: str) -> t.Any: return getattr(self, key) @@ -269,4 +271,4 @@ TBitStruct = DataclassBitStruct TContainerMixin = DataclassMixin TContainerBase = DataclassMixin TStructField = csfield -sfield = csfield \ No newline at end of file +sfield = csfield diff --git a/tests/test_typed.py b/tests/test_typed.py index 4dd9528..7d726a3 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -384,7 +384,7 @@ def test_tenum_no_enumbase() -> None: assert raises(lambda: cst.TEnum(cs.Byte, cls)) == TypeError -def test_tenum_asdict(): +def test_tenum_asdict() -> None: # see: https://github.com/timrid/construct-typing/issues/21 import construct_typed as cst import dataclasses @@ -498,7 +498,7 @@ def test_tenum_flags() -> None: assert raises(d.build, 2) == TypeError -def test_tenum_flags_asdict(): +def test_tenum_flags_asdict() -> None: import construct_typed as cst import dataclasses From d969fab1d3d6e3fa6e136beb70d25101d629f9e3 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 11:31:18 +0200 Subject: [PATCH 055/102] fixed python 3.7 "TypeError: 'type' object is not subscriptable" --- construct_typed/dataclass_struct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index d72ce5c..6e6c2d7 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -27,7 +27,7 @@ class DataclassMixin: methods exists and every name can be used. """ - __dataclass_fields__: t.ClassVar[dict[str, dataclasses.Field[t.Any]]] + __dataclass_fields__: "t.ClassVar[dict[str, dataclasses.Field[t.Any]]]" def __getitem__(self, key: str) -> t.Any: return getattr(self, key) From 6185a95e74da8ebe03171f2b5c9a13b7da4800b5 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 11:44:09 +0200 Subject: [PATCH 056/102] ignored typing error --- construct_typed/dataclass_struct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 6e6c2d7..b7dbe14 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -212,7 +212,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): value = obj[field.name] setattr(dc, field.name, value) - return dc + return dc # type: ignore def _encode( self, obj: DataclassType, context: Context, path: PathType From a44ea8429984505f79318ac1d147c13ed300b2c1 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 11:47:10 +0200 Subject: [PATCH 057/102] fixed mypy error --- construct_typed/dataclass_struct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index b7dbe14..6ecae05 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -27,7 +27,7 @@ class DataclassMixin: methods exists and every name can be used. """ - __dataclass_fields__: "t.ClassVar[dict[str, dataclasses.Field[t.Any]]]" + __dataclass_fields__: "t.ClassVar[t.Dict[str, dataclasses.Field[t.Any]]]" def __getitem__(self, key: str) -> t.Any: return getattr(self, key) From bb3935e7ebb9426f54f4bd4a6f0a243dbb7184ff Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 12:27:46 +0200 Subject: [PATCH 058/102] fixes #19 --- construct-stubs/core.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 83a7525..093199f 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -30,7 +30,7 @@ from construct.lib import ( # type hint. But for supporting pyright/pylance, the same type hint has to be used as the return type of __new__. # (see discussion here: https://github.com/python/typeshed/issues/4846). -StreamType = t.BinaryIO +StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str ContextKWType = t.Any From 2200a0a8d0ee4f965b9462b6001bd1272de95670 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 12:31:38 +0200 Subject: [PATCH 059/102] fixed mypy errors --- construct-stubs/core.pyi | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 093199f..392c8fc 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -75,18 +75,18 @@ class CancelParsing(ConstructError): ... # used internally # =============================================================================== def stream_read( - stream: t.BinaryIO, length: int, path: t.Optional[PathType] + stream: StreamType, length: int, path: t.Optional[PathType] ) -> bytes: ... -def stream_read_entire(stream: t.BinaryIO, path: t.Optional[PathType]) -> bytes: ... +def stream_read_entire(stream: StreamType, path: t.Optional[PathType]) -> bytes: ... def stream_write( - stream: t.BinaryIO, data: bytes, length: int, path: t.Optional[PathType] + stream: StreamType, data: bytes, length: int, path: t.Optional[PathType] ) -> None: ... def stream_seek( - stream: t.BinaryIO, offset: int, whence: int, path: t.Optional[PathType] + stream: StreamType, offset: int, whence: int, path: t.Optional[PathType] ) -> int: ... -def stream_tell(stream: t.BinaryIO, path: t.Optional[PathType]) -> int: ... -def stream_size(stream: t.BinaryIO) -> int: ... -def stream_iseof(stream: t.BinaryIO) -> bool: ... +def stream_tell(stream: StreamType, path: t.Optional[PathType]) -> int: ... +def stream_size(stream: StreamType) -> int: ... +def stream_iseof(stream: StreamType) -> bool: ... def evaluate(param: ConstantOrContextLambda2[T], context: Context) -> T: ... # =============================================================================== From 3b51654b1e2a61a57b8e0639f89dc2908bad0509 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 9 May 2023 13:03:26 +0200 Subject: [PATCH 060/102] incremented version to 0.5.6 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index ee25123..740da2f 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 5, 5) -version_string = "0.5.5" +version = (0, 5, 6) +version_string = "0.5.6" From 6550e59dd3d9ad51659c3ffcbffbc45e8d65b57f Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 30 Jun 2023 13:06:29 +0200 Subject: [PATCH 061/102] added missing __new__ methods --- construct-stubs/core.pyi | 125 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 392c8fc..adcc1d8 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -200,6 +200,9 @@ class Subconstruct( class Adapter( Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], ): + def __new__( + cls, subcon: Construct[SubconParsedType, SubconBuildTypes] + ) -> Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes] ) -> None: ... @@ -671,6 +674,13 @@ class RepeatUntil( class Renamed( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + newname: t.Optional[str] = ..., + newdocs: t.Optional[str] = ..., + newparsed: t.Optional[t.Callable[[t.Any, Context], None]] = ..., + ) -> Renamed[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1084,6 +1094,12 @@ class Padded( ): length: ConstantOrContextLambda[int] pattern: bytes + def __new__( + cls, + length: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + pattern: bytes = ..., + ) -> Padded[SubconParsedType, SubconBuildTypes]: ... def __init__( self, length: ConstantOrContextLambda[int], @@ -1096,6 +1112,12 @@ class Aligned( ): modulus: ConstantOrContextLambda[int] pattern: bytes + def __new__( + cls, + modulus: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + pattern: bytes = ..., + ) -> Aligned[SubconParsedType, SubconBuildTypes]: ... def __init__( self, modulus: ConstantOrContextLambda[int], @@ -1123,6 +1145,12 @@ class Pointer( ): offset: ConstantOrContextLambda[int] stream: t.Optional[t.Callable[[Context], StreamType]] + def __new__( + cls, + offset: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + stream: t.Optional[t.Callable[[Context], StreamType]] = ..., + ) -> Pointer[SubconParsedType, SubconBuildTypes]: ... def __init__( self, offset: ConstantOrContextLambda[int], @@ -1213,6 +1241,12 @@ class Prefixed( ): lengthfield: Construct[SubconParsedType, SubconBuildTypes] includelength: t.Optional[bool] + def __new__( + cls, + lengthfield: Construct[int, int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + includelength: t.Optional[bool] = ..., + ) -> Prefixed[SubconParsedType, SubconBuildTypes]: ... def __init__( self, lengthfield: Construct[int, int], @@ -1234,6 +1268,11 @@ class FixedSized( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): length: ConstantOrContextLambda[int] + def __new__( + cls, + length: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> FixedSized[SubconParsedType, SubconBuildTypes]: ... def __init__( self, length: ConstantOrContextLambda[int], @@ -1247,6 +1286,14 @@ class NullTerminated( include: t.Optional[bool] consume: t.Optional[bool] require: t.Optional[bool] + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + term: bytes = ..., + include: t.Optional[bool] = ..., + consume: t.Optional[bool] = ..., + require: t.Optional[bool] = ..., + ) -> NullTerminated[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1260,6 +1307,9 @@ class NullStripped( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): pad: bytes + def __new__( + cls, subcon: Construct[SubconParsedType, SubconBuildTypes], pad: bytes = ... + ) -> NullStripped[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], pad: bytes = ... ) -> None: ... @@ -1270,6 +1320,13 @@ class RestreamData( datafunc: t.Union[ bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] ] + def __new__( + cls, + datafunc: t.Union[ + bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] + ], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> RestreamData[SubconParsedType, SubconBuildTypes]: ... def __init__( self, datafunc: t.Union[ @@ -1285,6 +1342,14 @@ class Transformed( decodeamount: t.Optional[int] encodefunc: t.Callable[[bytes], bytes] encodeamount: t.Optional[int] + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + decodefunc: t.Callable[[bytes], bytes], + decodeamount: t.Optional[int], + encodefunc: t.Callable[[bytes], bytes], + encodeamount: t.Optional[int], + ) -> Transformed[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1302,6 +1367,15 @@ class Restreamed( encoder: t.Callable[[bytes], bytes] encoderunit: int sizecomputer: t.Callable[[int], int] + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + decoder: t.Callable[[bytes], bytes], + decoderunit: int, + encoder: t.Callable[[bytes], bytes], + encoderunit: int, + sizecomputer: t.Callable[[int], int], + ) -> Restreamed[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1362,6 +1436,12 @@ class Compressed(Tunnel[SubconParsedType, SubconBuildTypes]): encoding: str level: t.Optional[int] lib: t.Any + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + encoding: str, + level: t.Optional[int] = ..., + ) -> Compressed[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1371,6 +1451,10 @@ class Compressed(Tunnel[SubconParsedType, SubconBuildTypes]): class CompressedLZ4(Tunnel[SubconParsedType, SubconBuildTypes]): lib: t.Any + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> CompressedLZ4[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1380,6 +1464,11 @@ class Rebuffered( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): stream2: RebufferedBytesIO + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + tailcutoff: t.Optional[int] = ..., + ) -> Rebuffered[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1479,6 +1568,12 @@ class LazyBound(Construct[ParsedType, BuildTypes]): # adapters and validators # =============================================================================== class ExprAdapter(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + decoder: t.Callable[[SubconParsedType, Context], ParsedType], + encoder: t.Callable[[BuildTypes, Context], SubconBuildTypes], + ) -> ExprAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1489,6 +1584,11 @@ class ExprAdapter(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildT class ExprSymmetricAdapter( ExprAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes] ): + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + encoder: t.Callable[[BuildTypes, Context], SubconBuildTypes], + ) -> ExprSymmetricAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1496,6 +1596,11 @@ class ExprSymmetricAdapter( ) -> None: ... class ExprValidator(Validator[SubconParsedType, SubconBuildTypes]): + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + validator: t.Callable[[SubconParsedType, Context], bool], + ) -> ExprValidator[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1568,6 +1673,26 @@ class Slicing( class Indexing( Adapter[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): + def __new__( + cls, + subcon: t.Union[ + Array[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + GreedyRange[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], + t.List[SubconBuildTypes], + ], + ], + count: int, + index: int, + empty: t.Optional[SubconParsedType] = ..., + ) -> Indexing[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: t.Union[ From dcdeeca39fb7880d5eac323d1888828394fd2e1d Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Fri, 30 Jun 2023 14:13:01 +0200 Subject: [PATCH 062/102] removed all __new__ methods. mypy v1.4.1 is working. --- construct-stubs/core.pyi | 550 +++------------------------- construct_typed/dataclass_struct.py | 2 +- 2 files changed, 47 insertions(+), 505 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index adcc1d8..d3b46fa 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -25,11 +25,6 @@ from construct.lib import ( # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python -# The type checkers mypy and pyright/pylance unfortunately work a little bit different with __init__ and __new__. -# For supporting some constructs (eg. Enum, NamedTuple, Slicing) in mypy the __init__ self parameter has to have a -# type hint. But for supporting pyright/pylance, the same type hint has to be used as the return type of __new__. -# (see discussion here: https://github.com/python/typeshed/issues/4846). - StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str @@ -174,25 +169,13 @@ class Subconstruct( ): subcon: Construct[SubconParsedType, SubconBuildTypes] @t.overload - def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes] - ) -> Subconstruct[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ]: ... - @t.overload - def __new__( - cls, *args: t.Any, **kwargs: t.Any - ) -> Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... - @t.overload def __init__( - self: Subconstruct[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ], + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @t.overload def __init__( - self: Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], + self: t.Self, *args: t.Any, **kwargs: t.Any, ) -> None: ... @@ -200,11 +183,8 @@ class Subconstruct( class Adapter( Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], ): - def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes] - ) -> Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... def __init__( - self, subcon: Construct[SubconParsedType, SubconBuildTypes] + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes] ) -> None: ... def _decode( self, obj: SubconBuildTypes, context: Context, path: PathType @@ -239,10 +219,6 @@ class Tunnel( # =============================================================================== class Bytes(Construct[ParsedType, BuildTypes]): length: ConstantOrContextLambda[int] - def __new__( - cls, - length: ConstantOrContextLambda[int], - ) -> Bytes[bytes, t.Union[bytes, int]]: ... def __init__( self: Bytes[bytes, t.Union[bytes, int]], length: ConstantOrContextLambda[int], @@ -275,30 +251,6 @@ class FormatField(Construct[ParsedType, BuildTypes]): FORMAT_FLOAT = t.Literal["f", "d", "e"] FORMAT_BOOL = t.Literal["?"] @t.overload - def __new__( - cls, - endianity: str, - format: FORMAT_INT, - ) -> FormatField[int, int]: ... - @t.overload - def __new__( - cls, - endianity: str, - format: FORMAT_FLOAT, - ) -> FormatField[float, float]: ... - @t.overload - def __new__( - cls, - endianity: str, - format: FORMAT_BOOL, - ) -> FormatField[bool, bool]: ... - @t.overload - def __new__( - cls, - endianity: str, - format: str, - ) -> FormatField[t.Any, t.Any]: ... - @t.overload def __init__( self: FormatField[int, int], endianity: str, @@ -323,11 +275,6 @@ class FormatField(Construct[ParsedType, BuildTypes]): format: str, ) -> None: ... else: - def __new__( - cls, - endianity: str, - format: str, - ) -> FormatField[t.Any, t.Any]: ... def __init__( self: FormatField[t.Any, t.Any], endianity: str, @@ -338,12 +285,6 @@ class BytesInteger(Construct[ParsedType, BuildTypes]): length: ConstantOrContextLambda[int] signed: bool swapped: ConstantOrContextLambda[bool] - def __new__( - cls, - length: ConstantOrContextLambda[int], - signed: bool = ..., - swapped: ConstantOrContextLambda[bool] = ..., - ) -> BytesInteger[int, int]: ... def __init__( self: BytesInteger[int, int], length: ConstantOrContextLambda[int], @@ -355,12 +296,6 @@ class BitsInteger(Construct[ParsedType, BuildTypes]): length: ConstantOrContextLambda[int] signed: bool swapped: ConstantOrContextLambda[bool] - def __new__( - cls, - length: ConstantOrContextLambda[int], - signed: bool = ..., - swapped: ConstantOrContextLambda[bool] = ..., - ) -> BitsInteger[int, int]: ... def __init__( self: BitsInteger[int, int], length: ConstantOrContextLambda[int], @@ -438,11 +373,6 @@ class StringEncoded(Construct[ParsedType, BuildTypes]): else: ENCODING = str encoding: ENCODING - def __new__( - cls, - subcon: Construct[ParsedType, BuildTypes], - encoding: ENCODING, - ) -> StringEncoded[str, str]: ... def __init__( self: StringEncoded[str, str], subcon: Construct[ParsedType, BuildTypes], @@ -469,18 +399,12 @@ class EnumIntegerString(str): @staticmethod def new(intvalue: int, stringvalue: str) -> EnumIntegerString: ... -class Enum(Adapter[int, int, ParsedType, BuildTypes]): +class Enum(Adapter[int, int, t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]]): encmapping: t.Dict[str, int] decmapping: t.Dict[int, EnumIntegerString] ksymapping: t.Dict[int, str] - def __new__( - cls, - subcon: Construct[int, int], - *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], - **mapping: int, - ) -> Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]]: ... def __init__( - self: Enum[t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]], + self: t.Self, subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **mapping: int, @@ -490,17 +414,11 @@ class Enum(Adapter[int, int, ParsedType, BuildTypes]): class BitwisableString(str): def __or__(self, other: BitwisableString) -> BitwisableString: ... -class FlagsEnum(Adapter[int, int, ParsedType, BuildTypes]): +class FlagsEnum(Adapter[int, int, Container[bool], t.Union[int, str, t.Dict[str, bool]]]): flags: t.Dict[str, int] reverseflags: t.Dict[int, str] - def __new__( - cls, - subcon: Construct[int, int], - *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], - **flags: int, - ) -> FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]]: ... def __init__( - self: FlagsEnum[Container[bool], t.Union[int, str, t.Dict[str, bool]]], + self: t.Self, subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **flags: int, @@ -510,13 +428,8 @@ class FlagsEnum(Adapter[int, int, ParsedType, BuildTypes]): class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): decmapping: t.Dict[int, str] encmapping: t.Dict[str, int] - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - mapping: t.Dict[t.Any, t.Any], - ) -> Mapping[t.Any, t.Any]: ... def __init__( - self: Mapping[t.Any, t.Any], + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], mapping: t.Dict[t.Any, t.Any], ) -> None: ... @@ -525,32 +438,22 @@ class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): # structures and sequences # =============================================================================== # this can maybe made better when variadic generics are available -class Struct(Construct[ParsedType, BuildTypes]): +class Struct(Construct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] - def __new__( - cls, - *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any], - ) -> Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... def __init__( - self: Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]], + self: t.Self, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], ) -> None: ... def __getattr__(self, name: str) -> t.Any: ... # this can maybe made better when variadic generics are available -class Sequence(Construct[ParsedType, BuildTypes]): +class Sequence(Construct[ListContainer[t.Any], t.Optional[t.List[t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] - def __new__( - cls, - *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any], - ) -> Sequence[ListContainer[t.Any], t.Optional[t.List[t.Any]]]: ... def __init__( - self: Sequence[ListContainer[t.Any], t.Optional[t.List[t.Any]]], + self: t.Self, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], ) -> None: ... @@ -569,17 +472,6 @@ class Array( ): count: ConstantOrContextLambda[int] discard: bool - def __new__( - cls, - count: ConstantOrContextLambda[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - discard: bool = ..., - ) -> Array[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ]: ... def __init__( self: Array[ SubconParsedType, @@ -601,16 +493,6 @@ class GreedyRange( ] ): discard: bool - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - discard: bool = ..., - ) -> GreedyRange[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ]: ... def __init__( self: GreedyRange[ SubconParsedType, @@ -635,22 +517,6 @@ class RepeatUntil( t.Callable[[SubconParsedType, ListContainer[SubconParsedType], Context], bool], ] discard: bool - def __new__( - cls, - predicate: t.Union[ - bool, - t.Callable[ - [SubconParsedType, ListContainer[SubconParsedType], Context], bool - ], - ], - subcon: Construct[SubconParsedType, SubconBuildTypes], - discard: bool = ..., - ) -> RepeatUntil[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ]: ... def __init__( self: RepeatUntil[ SubconParsedType, @@ -674,15 +540,8 @@ class RepeatUntil( class Renamed( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - newname: t.Optional[str] = ..., - newdocs: t.Optional[str] = ..., - newparsed: t.Optional[t.Callable[[t.Any, Context], None]] = ..., - ) -> Renamed[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], newname: t.Optional[str] = ..., newdocs: t.Optional[str] = ..., @@ -695,30 +554,21 @@ class Renamed( class Const(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): value: SubconBuildTypes @t.overload - def __new__( - cls, + def __init__( + self: Const[None, None, bytes, t.Optional[bytes]], value: bytes, - ) -> Const[None, None, bytes, t.Optional[bytes]]: ... + ) -> None: ... @t.overload - def __new__( - cls, + def __init__( + self: Const[None, None, SubconParsedType, t.Optional[SubconBuildTypes]], value: SubconBuildTypes, subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> Const[None, None, SubconParsedType, t.Optional[SubconBuildTypes]]: ... + ) -> None: ... + class Computed(Construct[ParsedType, BuildTypes]): func: ConstantOrContextLambda2[ParsedType] @t.overload - def __new__( - cls, - func: ConstantOrContextLambda2[ParsedType], - ) -> Computed[ParsedType, None]: ... - @t.overload - def __new__( - cls, - func: ConstantOrContextLambda2[t.Any], - ) -> Computed[t.Any, None]: ... - @t.overload def __init__( self: Computed[ParsedType, None], func: ConstantOrContextLambda2[ParsedType], @@ -733,11 +583,6 @@ Index: Construct[int, t.Any] class Rebuild(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): func: ConstantOrContextLambda[SubconBuildTypes] - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - func: ConstantOrContextLambda[SubconBuildTypes], - ) -> Rebuild[SubconParsedType, SubconBuildTypes, SubconParsedType, None]: ... def __init__( self: Rebuild[SubconParsedType, SubconBuildTypes, SubconParsedType, None], subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -746,16 +591,6 @@ class Rebuild(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, Build class Default(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): value: ConstantOrContextLambda[SubconBuildTypes] - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - value: ConstantOrContextLambda[SubconBuildTypes], - ) -> Default[ - SubconParsedType, - SubconBuildTypes, - SubconParsedType, - t.Optional[SubconBuildTypes], - ]: ... def __init__( self: Default[ SubconParsedType, @@ -769,10 +604,6 @@ class Default(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, Build class Check(Construct[ParsedType, BuildTypes]): func: ConstantOrContextLambda[bool] - def __new__( - cls, - func: ConstantOrContextLambda[bool], - ) -> Check[None, None]: ... def __init__( self: Check[None, None], func: ConstantOrContextLambda[bool], @@ -806,17 +637,6 @@ class NamedTuple( tuplename: str tuplefields: str factory: Construct[SubconParsedType, SubconBuildTypes] - def __new__( - cls, - tuplename: str, - tuplefields: str, - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> NamedTuple[ - SubconParsedType, - SubconBuildTypes, - t.Tuple[t.Any, ...], - t.Union[t.Tuple[t.Any, ...], t.List[t.Any], t.Dict[str, t.Any]], - ]: ... def __init__( self: NamedTuple[ SubconParsedType, @@ -859,35 +679,6 @@ K = t.TypeVar("K") V = t.TypeVar("V") class Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - @t.overload - def __new__( - cls, subcon: Construct[int, BuildTypes] - ) -> Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes]: ... - @t.overload - def __new__( - cls, subcon: Construct[bytes, BuildTypes] - ) -> Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes]: ... - @t.overload - def __new__( - cls, subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes] - ) -> Hex[ - RawCopyObj[SubconParsedType], - BuildTypes, - HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], - BuildTypes, - ]: ... - @t.overload - def __new__( - cls, subcon: Construct[Container[t.Any], BuildTypes] - ) -> Hex[ - Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes - ]: ... - @t.overload - def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes] - ) -> Hex[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ]: ... @t.overload def __init__( self: Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes], @@ -924,31 +715,6 @@ class Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): ) -> None: ... class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - @t.overload - def __new__( - cls, subcon: Construct[bytes, BuildTypes] - ) -> HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes]: ... - @t.overload - def __new__( - cls, subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes] - ) -> HexDump[ - RawCopyObj[SubconParsedType], - BuildTypes, - HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], - BuildTypes, - ]: ... - @t.overload - def __new__( - cls, subcon: Construct[Container[t.Any], BuildTypes] - ) -> HexDump[ - Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes - ]: ... - @t.overload - def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes] - ) -> HexDump[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ]: ... @t.overload def __init__( self: HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes], @@ -998,11 +764,6 @@ class Union(Construct[Container[t.Any], t.Dict[str, t.Any]]): # this can maybe made better when variadic generics are available class Select(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] - def __new__( - cls, - *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any], - ) -> Select[t.Any, t.Any]: ... def __init__( self: Select[t.Any, t.Any], *subcons: Construct[t.Any, t.Any], @@ -1018,23 +779,21 @@ ThenBuildTypes = t.TypeVar("ThenBuildTypes") ElseParsedType = t.TypeVar("ElseParsedType") ElseBuildTypes = t.TypeVar("ElseBuildTypes") -class IfThenElse(Construct[ParsedType, BuildTypes]): +class IfThenElse(Construct[t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes]]): condfunc: ConstantOrContextLambda[bool] - thensubcon: Construct[ParsedType, BuildTypes] - elsesubcon: Construct[ParsedType, BuildTypes] - def __new__( - cls, + thensubcon: Construct[ThenParsedType, ThenBuildTypes] + elsesubcon: Construct[ElseParsedType, ElseBuildTypes] + def __init__( + self: t.Self, condfunc: ConstantOrContextLambda[bool], thensubcon: Construct[ThenParsedType, ThenBuildTypes], elsesubcon: Construct[ElseParsedType, ElseBuildTypes], - ) -> IfThenElse[ - t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] - ]: ... + ) -> None: ... def If( condfunc: ConstantOrContextLambda[bool], subcon: Construct[ThenParsedType, ThenBuildTypes], -) -> IfThenElse[t.Union[ThenParsedType, None], t.Union[ThenBuildTypes, None]]: ... +) -> IfThenElse[ThenParsedType, None, ThenBuildTypes, None]: ... SwitchType = t.TypeVar("SwitchType") @@ -1043,20 +802,6 @@ class Switch(Construct[ParsedType, BuildTypes]): cases: t.Dict[t.Any, Construct[t.Any, t.Any]] default: Construct[t.Any, t.Any] @t.overload - def __new__( - cls, - keyfunc: ConstantOrContextLambda[SwitchType], - cases: t.Dict[SwitchType, Construct[int, int]], - default: t.Optional[Construct[int, int]] = ..., - ) -> Switch[int, t.Optional[int]]: ... - @t.overload - def __new__( - cls, - keyfunc: ConstantOrContextLambda[t.Any], - cases: t.Dict[t.Any, Construct[t.Any, t.Any]], - default: t.Optional[Construct[t.Any, t.Any]] = ..., - ) -> Switch[t.Any, t.Any]: ... - @t.overload def __init__( self: Switch[int, t.Optional[int]], keyfunc: ConstantOrContextLambda[SwitchType], @@ -1071,14 +816,10 @@ class Switch(Construct[ParsedType, BuildTypes]): default: t.Optional[Construct[t.Any, t.Any]] = ..., ) -> None: ... -class StopIf(Construct[ParsedType, BuildTypes]): +class StopIf(Construct[None, None]): condfunc: ConstantOrContextLambda[bool] - def __new__( - cls, - condfunc: ConstantOrContextLambda[bool], - ) -> StopIf[None, None]: ... def __init__( - self: StopIf[None, None], + self: t.Self, condfunc: ConstantOrContextLambda[bool], ) -> None: ... @@ -1094,14 +835,8 @@ class Padded( ): length: ConstantOrContextLambda[int] pattern: bytes - def __new__( - cls, - length: ConstantOrContextLambda[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - pattern: bytes = ..., - ) -> Padded[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, length: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], pattern: bytes = ..., @@ -1112,14 +847,8 @@ class Aligned( ): modulus: ConstantOrContextLambda[int] pattern: bytes - def __new__( - cls, - modulus: ConstantOrContextLambda[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - pattern: bytes = ..., - ) -> Aligned[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, modulus: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], pattern: bytes = ..., @@ -1145,36 +874,16 @@ class Pointer( ): offset: ConstantOrContextLambda[int] stream: t.Optional[t.Callable[[Context], StreamType]] - def __new__( - cls, - offset: ConstantOrContextLambda[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - stream: t.Optional[t.Callable[[Context], StreamType]] = ..., - ) -> Pointer[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, offset: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], stream: t.Optional[t.Callable[[Context], StreamType]] = ..., ) -> None: ... -class Peek(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> Peek[ - SubconParsedType, - SubconBuildTypes, - SubconParsedType, - t.Union[SubconBuildTypes, None], - ]: ... +class Peek(Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, t.Union[SubconBuildTypes, None]]): def __init__( - self: Peek[ - SubconParsedType, - SubconBuildTypes, - SubconParsedType, - t.Union[SubconBuildTypes, None], - ], + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1207,15 +916,6 @@ class RawCopyObj(t.Generic[ParsedType], Container[t.Any]): length: int class RawCopy(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> RawCopy[ - SubconParsedType, - SubconBuildTypes, - RawCopyObj[SubconParsedType], - t.Optional[t.Dict[str, t.Union[SubconBuildTypes, bytes]]], - ]: ... def __init__( self: RawCopy[ SubconParsedType, @@ -1241,14 +941,8 @@ class Prefixed( ): lengthfield: Construct[SubconParsedType, SubconBuildTypes] includelength: t.Optional[bool] - def __new__( - cls, - lengthfield: Construct[int, int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - includelength: t.Optional[bool] = ..., - ) -> Prefixed[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, lengthfield: Construct[int, int], subcon: Construct[SubconParsedType, SubconBuildTypes], includelength: t.Optional[bool] = ..., @@ -1268,13 +962,8 @@ class FixedSized( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): length: ConstantOrContextLambda[int] - def __new__( - cls, - length: ConstantOrContextLambda[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> FixedSized[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, length: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1286,16 +975,8 @@ class NullTerminated( include: t.Optional[bool] consume: t.Optional[bool] require: t.Optional[bool] - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - term: bytes = ..., - include: t.Optional[bool] = ..., - consume: t.Optional[bool] = ..., - require: t.Optional[bool] = ..., - ) -> NullTerminated[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], term: bytes = ..., include: t.Optional[bool] = ..., @@ -1307,11 +988,8 @@ class NullStripped( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): pad: bytes - def __new__( - cls, subcon: Construct[SubconParsedType, SubconBuildTypes], pad: bytes = ... - ) -> NullStripped[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, subcon: Construct[SubconParsedType, SubconBuildTypes], pad: bytes = ... + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], pad: bytes = ... ) -> None: ... class RestreamData( @@ -1320,15 +998,8 @@ class RestreamData( datafunc: t.Union[ bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] ] - def __new__( - cls, - datafunc: t.Union[ - bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] - ], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> RestreamData[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, datafunc: t.Union[ bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] ], @@ -1342,16 +1013,8 @@ class Transformed( decodeamount: t.Optional[int] encodefunc: t.Callable[[bytes], bytes] encodeamount: t.Optional[int] - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - decodefunc: t.Callable[[bytes], bytes], - decodeamount: t.Optional[int], - encodefunc: t.Callable[[bytes], bytes], - encodeamount: t.Optional[int], - ) -> Transformed[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], decodefunc: t.Callable[[bytes], bytes], decodeamount: t.Optional[int], @@ -1367,17 +1030,8 @@ class Restreamed( encoder: t.Callable[[bytes], bytes] encoderunit: int sizecomputer: t.Callable[[int], int] - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - decoder: t.Callable[[bytes], bytes], - decoderunit: int, - encoder: t.Callable[[bytes], bytes], - encoderunit: int, - sizecomputer: t.Callable[[int], int], - ) -> Restreamed[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], decoder: t.Callable[[bytes], bytes], decoderunit: int, @@ -1390,13 +1044,8 @@ class ProcessXor( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): padfunc: ConstantOrContextLambda2[t.Union[int, bytes]] - def __new__( - cls, - padfunc: ConstantOrContextLambda2[t.Union[int, bytes]], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> ProcessXor[SubconParsedType, SubconBuildTypes]: ... def __init__( - self: ProcessXor[SubconParsedType, SubconBuildTypes], + self: t.Self, padfunc: ConstantOrContextLambda2[t.Union[int, bytes]], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1406,14 +1055,8 @@ class ProcessRotateLeft( ): amount: ConstantOrContextLambda2[int] group: ConstantOrContextLambda2[int] - def __new__( - cls, - amount: ConstantOrContextLambda2[int], - group: ConstantOrContextLambda2[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> ProcessRotateLeft[SubconParsedType, SubconBuildTypes]: ... def __init__( - self: ProcessRotateLeft[SubconParsedType, SubconBuildTypes], + self: t.Self, amount: ConstantOrContextLambda2[int], group: ConstantOrContextLambda2[int], subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1436,14 +1079,8 @@ class Compressed(Tunnel[SubconParsedType, SubconBuildTypes]): encoding: str level: t.Optional[int] lib: t.Any - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - encoding: str, - level: t.Optional[int] = ..., - ) -> Compressed[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], encoding: str, level: t.Optional[int] = ..., @@ -1451,12 +1088,8 @@ class Compressed(Tunnel[SubconParsedType, SubconBuildTypes]): class CompressedLZ4(Tunnel[SubconParsedType, SubconBuildTypes]): lib: t.Any - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> CompressedLZ4[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1464,13 +1097,8 @@ class Rebuffered( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): stream2: RebufferedBytesIO - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - tailcutoff: t.Optional[int] = ..., - ) -> Rebuffered[SubconParsedType, SubconBuildTypes]: ... def __init__( - self, + self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], tailcutoff: t.Optional[int] = ..., ) -> None: ... @@ -1479,15 +1107,6 @@ class Rebuffered( # lazy equivalents # =============================================================================== class Lazy(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> Lazy[ - SubconParsedType, - SubconBuildTypes, - t.Callable[[], SubconParsedType], - t.Union[t.Callable[[], SubconParsedType], SubconParsedType], - ]: ... def __init__( self: Lazy[ SubconParsedType, @@ -1509,11 +1128,6 @@ class LazyStruct(Construct[ParsedType, BuildTypes]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] _subconsindexes: t.Dict[str, int] - def __new__( - cls, - *subcons: Construct[t.Any, t.Any], - **subconskw: Construct[t.Any, t.Any], - ) -> LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... def __init__( self: LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]], *subcons: Construct[t.Any, t.Any], @@ -1532,16 +1146,6 @@ class LazyArray( ] ): count: ConstantOrContextLambda[int] - def __new__( - cls, - count: ConstantOrContextLambda[int], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> LazyArray[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ]: ... def __init__( self: LazyArray[ SubconParsedType, @@ -1555,10 +1159,6 @@ class LazyArray( class LazyBound(Construct[ParsedType, BuildTypes]): subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]] - def __new__( - cls, - subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]], - ) -> LazyBound[ParsedType, BuildTypes]: ... def __init__( self: LazyBound[ParsedType, BuildTypes], subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]], @@ -1568,12 +1168,6 @@ class LazyBound(Construct[ParsedType, BuildTypes]): # adapters and validators # =============================================================================== class ExprAdapter(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - decoder: t.Callable[[SubconParsedType, Context], ParsedType], - encoder: t.Callable[[BuildTypes, Context], SubconBuildTypes], - ) -> ExprAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1584,11 +1178,6 @@ class ExprAdapter(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildT class ExprSymmetricAdapter( ExprAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes] ): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - encoder: t.Callable[[BuildTypes, Context], SubconBuildTypes], - ) -> ExprSymmetricAdapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1596,11 +1185,6 @@ class ExprSymmetricAdapter( ) -> None: ... class ExprValidator(Validator[SubconParsedType, SubconBuildTypes]): - def __new__( - cls, - subcon: Construct[SubconParsedType, SubconBuildTypes], - validator: t.Callable[[SubconParsedType, Context], bool], - ) -> ExprValidator[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1625,28 +1209,6 @@ def Filter( class Slicing( Adapter[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): - def __new__( - cls, - subcon: t.Union[ - Array[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], - GreedyRange[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], - ], - count: int, - start: t.Optional[int], - stop: t.Optional[int], - step: int = ..., - empty: t.Optional[SubconParsedType] = ..., - ) -> Slicing[ListContainer[SubconParsedType], t.List[SubconBuildTypes]]: ... def __init__( self: Slicing[ListContainer[SubconParsedType], t.List[SubconBuildTypes]], subcon: t.Union[ @@ -1673,26 +1235,6 @@ class Slicing( class Indexing( Adapter[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): - def __new__( - cls, - subcon: t.Union[ - Array[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], - GreedyRange[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], - ], - count: int, - index: int, - empty: t.Optional[SubconParsedType] = ..., - ) -> Indexing[SubconParsedType, SubconBuildTypes]: ... def __init__( self, subcon: t.Union[ diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 6ecae05..f79fc7e 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -152,7 +152,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): Image(width=1, height=2, pixels=b'12') """ - subcon: "cs.Struct[t.Any, t.Any]" + subcon: "cs.Struct" if t.TYPE_CHECKING: def __new__( From 1e7ced992d6c4e97622f0ec31b75a4e845f47fae Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 15:38:01 +0200 Subject: [PATCH 063/102] removed all self type annotations of __init__ methods --- construct-stubs/core.pyi | 625 ++++++++++++++++++--------------------- 1 file changed, 294 insertions(+), 331 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index d3b46fa..a19686b 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -25,6 +25,10 @@ from construct.lib import ( # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python +# unfortunalty the static type checkers "pyright" and "mypy" are slight different. pyright is not fully analysing the type hint of the +# self type in the __init__ (eg. self: Construct[int, int] is not working). but pyright would support such type hints of the return type +# of __new__. indeed mypy doens not support the type inference for the method __new__, but fully supports the annotation of self in __init__... + StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str @@ -129,20 +133,11 @@ class Construct(t.Generic[ParsedType, BuildTypes]): self, other: t.Union[str, bytes, t.Callable[[ParsedType, Context], None]], ) -> Renamed[ParsedType, BuildTypes]: ... - def __add__( - self, other: Construct[t.Any, t.Any] - ) -> Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... - def __rshift__( - self, other: Construct[t.Any, t.Any] - ) -> Sequence[ListContainer[t.Any], t.Optional[t.List[t.Any]]]: ... + def __add__(self, other: Construct[t.Any, t.Any]) -> Struct: ... + def __rshift__(self, other: Construct[t.Any, t.Any]) -> Sequence: ... def __getitem__( self, count: t.Union[int, t.Callable[[Context], int]] - ) -> Array[ - ParsedType, - BuildTypes, - ListContainer[ParsedType], - t.List[BuildTypes], - ]: ... + ) -> Array[ParsedType, BuildTypes,]: ... @t.type_check_only class Context(Container[t.Any]): @@ -170,12 +165,12 @@ class Subconstruct( subcon: Construct[SubconParsedType, SubconBuildTypes] @t.overload def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @t.overload def __init__( - self: t.Self, + self, *args: t.Any, **kwargs: t.Any, ) -> None: ... @@ -184,7 +179,7 @@ class Adapter( Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], ): def __init__( - self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes] + self, subcon: Construct[SubconParsedType, SubconBuildTypes] ) -> None: ... def _decode( self, obj: SubconBuildTypes, context: Context, path: PathType @@ -217,10 +212,10 @@ class Tunnel( # =============================================================================== # bytes and bits # =============================================================================== -class Bytes(Construct[ParsedType, BuildTypes]): +class Bytes(Construct[bytes, t.Union[bytes, int]]): length: ConstantOrContextLambda[int] def __init__( - self: Bytes[bytes, t.Union[bytes, int]], + self, length: ConstantOrContextLambda[int], ) -> None: ... @@ -242,121 +237,118 @@ def Bytewise( # =============================================================================== # integers and floats # =============================================================================== -class FormatField(Construct[ParsedType, BuildTypes]): +class _FormatField(Construct[ParsedType, BuildTypes]): fmtstr: str length: int - if sys.version_info >= (3, 8): - ENDIANITY = t.Union[t.Literal["=", "<", ">"], str] - FORMAT_INT = t.Literal["B", "H", "L", "Q", "b", "h", "l", "q"] - FORMAT_FLOAT = t.Literal["f", "d", "e"] - FORMAT_BOOL = t.Literal["?"] - @t.overload - def __init__( - self: FormatField[int, int], - endianity: str, - format: FORMAT_INT, - ) -> None: ... - @t.overload - def __init__( - self: FormatField[float, float], - endianity: str, - format: FORMAT_FLOAT, - ) -> None: ... - @t.overload - def __init__( - self: FormatField[bool, bool], - endianity: str, - format: FORMAT_BOOL, - ) -> None: ... - @t.overload - def __init__( - self: FormatField[t.Any, t.Any], - endianity: str, - format: str, - ) -> None: ... - else: - def __init__( - self: FormatField[t.Any, t.Any], - endianity: str, - format: str, - ) -> None: ... -class BytesInteger(Construct[ParsedType, BuildTypes]): +if sys.version_info >= (3, 8): + ENDIANITY = t.Union[t.Literal["=", "<", ">"], str] + FORMAT_INT = t.Literal["B", "H", "L", "Q", "b", "h", "l", "q"] + FORMAT_FLOAT = t.Literal["f", "d", "e"] + FORMAT_BOOL = t.Literal["?"] + @t.overload + def FormatField( + endianity: str, + format: FORMAT_INT, + ) -> _FormatField[int, int]: ... + @t.overload + def FormatField( + endianity: str, + format: FORMAT_FLOAT, + ) -> _FormatField[float, float]: ... + @t.overload + def FormatField( + endianity: str, + format: FORMAT_BOOL, + ) -> _FormatField[bool, bool]: ... + @t.overload + def FormatField( + endianity: str, + format: str, + ) -> _FormatField[t.Any, t.Any]: ... + +else: + def FormatField( + endianity: str, + format: str, + ) -> _FormatField[t.Any, t.Any]: ... + +class BytesInteger(Construct[int, int]): length: ConstantOrContextLambda[int] signed: bool swapped: ConstantOrContextLambda[bool] def __init__( - self: BytesInteger[int, int], + self, length: ConstantOrContextLambda[int], signed: bool = ..., swapped: ConstantOrContextLambda[bool] = ..., ) -> None: ... -class BitsInteger(Construct[ParsedType, BuildTypes]): +class BitsInteger(Construct[int, int]): length: ConstantOrContextLambda[int] signed: bool swapped: ConstantOrContextLambda[bool] def __init__( - self: BitsInteger[int, int], + self, length: ConstantOrContextLambda[int], signed: bool = ..., swapped: ConstantOrContextLambda[bool] = ..., ) -> None: ... -Bit: BitsInteger[int, int] -Nibble: BitsInteger[int, int] -Octet: BitsInteger[int, int] +Bit: BitsInteger +Nibble: BitsInteger +Octet: BitsInteger -Int8ub: FormatField[int, int] -Int16ub: FormatField[int, int] -Int32ub: FormatField[int, int] -Int64ub: FormatField[int, int] -Int8sb: FormatField[int, int] -Int16sb: FormatField[int, int] -Int32sb: FormatField[int, int] -Int64sb: FormatField[int, int] -Int8ul: FormatField[int, int] -Int16ul: FormatField[int, int] -Int32ul: FormatField[int, int] -Int64ul: FormatField[int, int] -Int8sl: FormatField[int, int] -Int16sl: FormatField[int, int] -Int32sl: FormatField[int, int] -Int64sl: FormatField[int, int] -Int8un: FormatField[int, int] -Int16un: FormatField[int, int] -Int32un: FormatField[int, int] -Int64un: FormatField[int, int] -Int8sn: FormatField[int, int] -Int16sn: FormatField[int, int] -Int32sn: FormatField[int, int] -Int64sn: FormatField[int, int] +Int8ub: _FormatField[int, int] +Int16ub: _FormatField[int, int] +Int32ub: _FormatField[int, int] +Int64ub: _FormatField[int, int] +Int8sb: _FormatField[int, int] +Int16sb: _FormatField[int, int] +Int32sb: _FormatField[int, int] +Int64sb: _FormatField[int, int] +Int8ul: _FormatField[int, int] +Int16ul: _FormatField[int, int] +Int32ul: _FormatField[int, int] +Int64ul: _FormatField[int, int] +Int8sl: _FormatField[int, int] +Int16sl: _FormatField[int, int] +Int32sl: _FormatField[int, int] +Int64sl: _FormatField[int, int] +Int8un: _FormatField[int, int] +Int16un: _FormatField[int, int] +Int32un: _FormatField[int, int] +Int64un: _FormatField[int, int] +Int8sn: _FormatField[int, int] +Int16sn: _FormatField[int, int] +Int32sn: _FormatField[int, int] +Int64sn: _FormatField[int, int] -Byte: FormatField[int, int] -Short: FormatField[int, int] -Int: FormatField[int, int] -Long: FormatField[int, int] +Byte: _FormatField[int, int] +Short: _FormatField[int, int] +Int: _FormatField[int, int] +Long: _FormatField[int, int] -Float16b: FormatField[float, float] -Float16l: FormatField[float, float] -Float16n: FormatField[float, float] -Float32b: FormatField[float, float] -Float32l: FormatField[float, float] -Float32n: FormatField[float, float] -Float64b: FormatField[float, float] -Float64l: FormatField[float, float] -Float64n: FormatField[float, float] +Float16b: _FormatField[float, float] +Float16l: _FormatField[float, float] +Float16n: _FormatField[float, float] +Float32b: _FormatField[float, float] +Float32l: _FormatField[float, float] +Float32n: _FormatField[float, float] +Float64b: _FormatField[float, float] +Float64l: _FormatField[float, float] +Float64n: _FormatField[float, float] -Half: FormatField[float, float] -Single: FormatField[float, float] -Double: FormatField[float, float] +Half: _FormatField[float, float] +Single: _FormatField[float, float] +Double: _FormatField[float, float] -Int24ub: BytesInteger[int, int] -Int24ul: BytesInteger[int, int] -Int24un: BytesInteger[int, int] -Int24sb: BytesInteger[int, int] -Int24sl: BytesInteger[int, int] -Int24sn: BytesInteger[int, int] +Int24ub: BytesInteger +Int24ul: BytesInteger +Int24un: BytesInteger +Int24sb: BytesInteger +Int24sl: BytesInteger +Int24sn: BytesInteger VarInt: Construct[int, int] ZigZag: Construct[int, int] @@ -364,7 +356,7 @@ ZigZag: Construct[int, int] # =============================================================================== # strings # =============================================================================== -class StringEncoded(Construct[ParsedType, BuildTypes]): +class StringEncoded(Construct[str, str]): if sys.version_info >= (3, 8): ENCODING_1 = t.Literal["ascii", "utf8", "utf_8", "u8"] ENCODING_2 = t.Literal["utf16", "utf_16", "u16", "utf_16_be", "utf_16_le"] @@ -374,19 +366,19 @@ class StringEncoded(Construct[ParsedType, BuildTypes]): ENCODING = str encoding: ENCODING def __init__( - self: StringEncoded[str, str], - subcon: Construct[ParsedType, BuildTypes], + self, + subcon: Construct[bytes, bytes], encoding: ENCODING, ) -> None: ... def PaddedString( length: ConstantOrContextLambda[int], encoding: StringEncoded.ENCODING -) -> StringEncoded[str, str]: ... +) -> StringEncoded: ... def PascalString( lengthfield: Construct[int, int], encoding: StringEncoded.ENCODING -) -> StringEncoded[str, str]: ... -def CString(encoding: StringEncoded.ENCODING) -> StringEncoded[str, str]: ... -def GreedyString(encoding: StringEncoded.ENCODING) -> StringEncoded[str, str]: ... +) -> StringEncoded: ... +def CString(encoding: StringEncoded.ENCODING) -> StringEncoded: ... +def GreedyString(encoding: StringEncoded.ENCODING) -> StringEncoded: ... # =============================================================================== # mappings @@ -399,12 +391,14 @@ class EnumIntegerString(str): @staticmethod def new(intvalue: int, stringvalue: str) -> EnumIntegerString: ... -class Enum(Adapter[int, int, t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]]): +class Enum( + Adapter[int, int, t.Union[EnumInteger, EnumIntegerString], t.Union[int, str]] +): encmapping: t.Dict[str, int] decmapping: t.Dict[int, EnumIntegerString] ksymapping: t.Dict[int, str] def __init__( - self: t.Self, + self, subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **mapping: int, @@ -414,11 +408,13 @@ class Enum(Adapter[int, int, t.Union[EnumInteger, EnumIntegerString], t.Union[in class BitwisableString(str): def __or__(self, other: BitwisableString) -> BitwisableString: ... -class FlagsEnum(Adapter[int, int, Container[bool], t.Union[int, str, t.Dict[str, bool]]]): +class FlagsEnum( + Adapter[int, int, Container[bool], t.Union[int, str, t.Dict[str, bool]]] +): flags: t.Dict[str, int] reverseflags: t.Dict[int, str] def __init__( - self: t.Self, + self, subcon: Construct[int, int], *merge: t.Union[t.Type[enum.IntEnum], t.Type[enum.IntFlag]], **flags: int, @@ -429,7 +425,7 @@ class Mapping(Adapter[SubconParsedType, SubconBuildTypes, t.Any, t.Any]): decmapping: t.Dict[int, str] encmapping: t.Dict[str, int] def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], mapping: t.Dict[t.Any, t.Any], ) -> None: ... @@ -442,7 +438,7 @@ class Struct(Construct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __init__( - self: t.Self, + self, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], ) -> None: ... @@ -453,7 +449,7 @@ class Sequence(Construct[ListContainer[t.Any], t.Optional[t.List[t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] def __init__( - self: t.Self, + self, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], ) -> None: ... @@ -466,19 +462,14 @@ class Array( Subconstruct[ SubconParsedType, SubconBuildTypes, - ParsedType, - BuildTypes, + ListContainer[SubconParsedType], # type: ignore + t.List[SubconBuildTypes], # type: ignore ] ): count: ConstantOrContextLambda[int] discard: bool def __init__( - self: Array[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], + self, count: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], discard: bool = ..., @@ -488,18 +479,13 @@ class GreedyRange( Subconstruct[ SubconParsedType, SubconBuildTypes, - ParsedType, - BuildTypes, + ListContainer[SubconParsedType], # type: ignore + t.List[SubconBuildTypes], # type: ignore ] ): discard: bool def __init__( - self: GreedyRange[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], + self, subcon: Construct[SubconParsedType, SubconBuildTypes], discard: bool = ..., ) -> None: ... @@ -508,8 +494,8 @@ class RepeatUntil( Subconstruct[ SubconParsedType, SubconBuildTypes, - ParsedType, - BuildTypes, + ListContainer[SubconParsedType], # type: ignore + t.List[SubconBuildTypes], # type: ignore ] ): predicate: t.Union[ @@ -518,12 +504,7 @@ class RepeatUntil( ] discard: bool def __init__( - self: RepeatUntil[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], + self, predicate: t.Union[ bool, t.Callable[ @@ -541,7 +522,7 @@ class Renamed( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], newname: t.Optional[str] = ..., newdocs: t.Optional[str] = ..., @@ -551,61 +532,54 @@ class Renamed( # =============================================================================== # miscellaneous # =============================================================================== -class Const(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - value: SubconBuildTypes - @t.overload - def __init__( - self: Const[None, None, bytes, t.Optional[bytes]], - value: bytes, - ) -> None: ... - @t.overload - def __init__( - self: Const[None, None, SubconParsedType, t.Optional[SubconBuildTypes]], - value: SubconBuildTypes, - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> None: ... - +class _Const(Subconstruct[None, None, SubconParsedType, SubconBuildTypes]): ... -class Computed(Construct[ParsedType, BuildTypes]): +@t.overload +def Const( + value: bytes, +) -> _Const[bytes, t.Optional[bytes]]: ... +@t.overload +def Const( + value: SubconBuildTypes, + subcon: Construct[SubconParsedType, SubconBuildTypes], +) -> _Const[SubconParsedType, t.Optional[SubconBuildTypes]]: ... + +class Computed(Construct[ParsedType, None]): func: ConstantOrContextLambda2[ParsedType] - @t.overload def __init__( - self: Computed[ParsedType, None], + self, func: ConstantOrContextLambda2[ParsedType], ) -> None: ... - @t.overload - def __init__( - self: Computed[t.Any, None], - func: ConstantOrContextLambda2[t.Any], - ) -> None: ... Index: Construct[int, t.Any] -class Rebuild(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): +class Rebuild(Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, None]): func: ConstantOrContextLambda[SubconBuildTypes] def __init__( - self: Rebuild[SubconParsedType, SubconBuildTypes, SubconParsedType, None], + self, subcon: Construct[SubconParsedType, SubconBuildTypes], func: ConstantOrContextLambda[SubconBuildTypes], ) -> None: ... -class Default(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): +class Default( + Subconstruct[ + SubconParsedType, + SubconBuildTypes, + SubconParsedType, + t.Optional[SubconBuildTypes], + ] +): value: ConstantOrContextLambda[SubconBuildTypes] def __init__( - self: Default[ - SubconParsedType, - SubconBuildTypes, - SubconParsedType, - t.Optional[SubconBuildTypes], - ], + self, subcon: Construct[SubconParsedType, SubconBuildTypes], value: ConstantOrContextLambda[SubconBuildTypes], ) -> None: ... -class Check(Construct[ParsedType, BuildTypes]): +class Check(Construct[None, None]): func: ConstantOrContextLambda[bool] def __init__( - self: Check[None, None], + self, func: ConstantOrContextLambda[bool], ) -> None: ... @@ -630,20 +604,15 @@ class NamedTuple( Adapter[ SubconParsedType, SubconBuildTypes, - ParsedType, - BuildTypes, + t.Tuple[t.Any, ...], + t.Union[t.Tuple[t.Any, ...], t.List[t.Any], t.Dict[str, t.Any]], ] ): tuplename: str tuplefields: str factory: Construct[SubconParsedType, SubconBuildTypes] def __init__( - self: NamedTuple[ - SubconParsedType, - SubconBuildTypes, - t.Tuple[t.Any, ...], - t.Union[t.Tuple[t.Any, ...], t.List[t.Any], t.Dict[str, t.Any]], - ], + self, tuplename: str, tuplefields: str, subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -678,72 +647,63 @@ def Timestamp( K = t.TypeVar("K") V = t.TypeVar("V") -class Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - @t.overload - def __init__( - self: Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes], - subcon: Construct[int, BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes], - subcon: Construct[bytes, BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: Hex[ - RawCopyObj[SubconParsedType], - BuildTypes, - HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], - BuildTypes, - ], - subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: Hex[ - Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes - ], - subcon: Construct[Container[t.Any], BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: Hex[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> None: ... +class _Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): + pass -class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - @t.overload - def __init__( - self: HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes], - subcon: Construct[bytes, BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: HexDump[ - RawCopyObj[SubconParsedType], - BuildTypes, - HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], - BuildTypes, - ], - subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: HexDump[ - Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes - ], - subcon: Construct[Container[t.Any], BuildTypes], - ) -> None: ... - @t.overload - def __init__( - self: HexDump[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ], - subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> None: ... +@t.overload +def Hex( + subcon: Construct[int, BuildTypes], +) -> _Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes]: ... +@t.overload +def Hex( + subcon: Construct[bytes, BuildTypes], +) -> _Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes]: ... +@t.overload +def Hex( + subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], +) -> _Hex[ + RawCopyObj[SubconParsedType], + BuildTypes, + HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], + BuildTypes, +]: ... +@t.overload +def Hex( + subcon: Construct[Container[t.Any], BuildTypes], +) -> _Hex[Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes]: ... +@t.overload +def Hex( + subcon: Construct[SubconParsedType, SubconBuildTypes], +) -> _Hex[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes]: ... + +class _HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): + pass + +@t.overload +def HexDump( + subcon: Construct[bytes, BuildTypes], +) -> _HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes]: ... +@t.overload +def HexDump( + subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], +) -> _HexDump[ + RawCopyObj[SubconParsedType], + BuildTypes, + HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], + BuildTypes, +]: ... +@t.overload +def HexDump( + subcon: Construct[Container[t.Any], BuildTypes], +) -> _HexDump[ + Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes +]: ... +@t.overload +def HexDump( + subcon: Construct[SubconParsedType, SubconBuildTypes], +) -> _HexDump[ + SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes +]: ... # =============================================================================== # conditional @@ -762,29 +722,33 @@ class Union(Construct[Container[t.Any], t.Dict[str, t.Any]]): def __getattr__(self, name: str) -> t.Any: ... # this can maybe made better when variadic generics are available -class Select(Construct[ParsedType, BuildTypes]): +class Select(Construct[t.Any, t.Any]): subcons: t.List[Construct[t.Any, t.Any]] def __init__( - self: Select[t.Any, t.Any], + self, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], ) -> None: ... def Optional( subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> Select[t.Union[SubconParsedType, None], t.Union[SubconBuildTypes, None]]: ... +) -> Construct[t.Union[SubconParsedType, None], t.Union[SubconBuildTypes, None]]: ... ThenParsedType = t.TypeVar("ThenParsedType") ThenBuildTypes = t.TypeVar("ThenBuildTypes") ElseParsedType = t.TypeVar("ElseParsedType") ElseBuildTypes = t.TypeVar("ElseBuildTypes") -class IfThenElse(Construct[t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes]]): +class IfThenElse( + Construct[ + t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] + ] +): condfunc: ConstantOrContextLambda[bool] thensubcon: Construct[ThenParsedType, ThenBuildTypes] elsesubcon: Construct[ElseParsedType, ElseBuildTypes] def __init__( - self: t.Self, + self, condfunc: ConstantOrContextLambda[bool], thensubcon: Construct[ThenParsedType, ThenBuildTypes], elsesubcon: Construct[ElseParsedType, ElseBuildTypes], @@ -797,29 +761,28 @@ def If( SwitchType = t.TypeVar("SwitchType") -class Switch(Construct[ParsedType, BuildTypes]): +class _Switch(Construct[ParsedType, BuildTypes]): keyfunc: ConstantOrContextLambda[t.Any] cases: t.Dict[t.Any, Construct[t.Any, t.Any]] default: Construct[t.Any, t.Any] - @t.overload - def __init__( - self: Switch[int, t.Optional[int]], - keyfunc: ConstantOrContextLambda[SwitchType], - cases: t.Dict[SwitchType, Construct[int, int]], - default: t.Optional[Construct[int, int]] = ..., - ) -> None: ... - @t.overload - def __init__( - self: Switch[t.Any, t.Any], - keyfunc: ConstantOrContextLambda[t.Any], - cases: t.Dict[t.Any, Construct[t.Any, t.Any]], - default: t.Optional[Construct[t.Any, t.Any]] = ..., - ) -> None: ... + +@t.overload +def Switch( + keyfunc: ConstantOrContextLambda[SwitchType], + cases: t.Dict[SwitchType, Construct[int, int]], + default: t.Optional[Construct[int, int]] = ..., +) -> _Switch[int, t.Optional[int]]: ... +@t.overload +def Switch( + keyfunc: ConstantOrContextLambda[t.Any], + cases: t.Dict[t.Any, Construct[t.Any, t.Any]], + default: t.Optional[Construct[t.Any, t.Any]] = ..., +) -> _Switch[t.Any, t.Any]: ... class StopIf(Construct[None, None]): condfunc: ConstantOrContextLambda[bool] def __init__( - self: t.Self, + self, condfunc: ConstantOrContextLambda[bool], ) -> None: ... @@ -836,7 +799,7 @@ class Padded( length: ConstantOrContextLambda[int] pattern: bytes def __init__( - self: t.Self, + self, length: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], pattern: bytes = ..., @@ -848,7 +811,7 @@ class Aligned( modulus: ConstantOrContextLambda[int] pattern: bytes def __init__( - self: t.Self, + self, modulus: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], pattern: bytes = ..., @@ -858,7 +821,7 @@ def AlignedStruct( modulus: ConstantOrContextLambda[int], *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], -) -> Struct[Container[t.Any], t.Optional[t.Dict[str, t.Any]]]: ... +) -> Struct: ... def BitStruct( *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any] ) -> t.Union[ @@ -875,15 +838,22 @@ class Pointer( offset: ConstantOrContextLambda[int] stream: t.Optional[t.Callable[[Context], StreamType]] def __init__( - self: t.Self, + self, offset: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], stream: t.Optional[t.Callable[[Context], StreamType]] = ..., ) -> None: ... -class Peek(Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, t.Union[SubconBuildTypes, None]]): +class Peek( + Subconstruct[ + SubconParsedType, + SubconBuildTypes, + SubconParsedType, + t.Union[SubconBuildTypes, None], + ] +): def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -915,14 +885,16 @@ class RawCopyObj(t.Generic[ParsedType], Container[t.Any]): offset2: int length: int -class RawCopy(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): +class RawCopy( + Subconstruct[ + SubconParsedType, + SubconBuildTypes, + RawCopyObj[SubconParsedType], + t.Optional[t.Dict[str, t.Union[SubconBuildTypes, bytes]]], + ] +): def __init__( - self: RawCopy[ - SubconParsedType, - SubconBuildTypes, - RawCopyObj[SubconParsedType], - t.Optional[t.Dict[str, t.Union[SubconBuildTypes, bytes]]], - ], + self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -942,7 +914,7 @@ class Prefixed( lengthfield: Construct[SubconParsedType, SubconBuildTypes] includelength: t.Optional[bool] def __init__( - self: t.Self, + self, lengthfield: Construct[int, int], subcon: Construct[SubconParsedType, SubconBuildTypes], includelength: t.Optional[bool] = ..., @@ -951,19 +923,14 @@ class Prefixed( def PrefixedArray( countfield: Construct[int, int], subcon: Construct[SubconParsedType, SubconBuildTypes], -) -> Array[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], -]: ... +) -> Array[SubconParsedType, SubconBuildTypes,]: ... class FixedSized( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] ): length: ConstantOrContextLambda[int] def __init__( - self: t.Self, + self, length: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -976,7 +943,7 @@ class NullTerminated( consume: t.Optional[bool] require: t.Optional[bool] def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], term: bytes = ..., include: t.Optional[bool] = ..., @@ -989,7 +956,9 @@ class NullStripped( ): pad: bytes def __init__( - self: t.Self, subcon: Construct[SubconParsedType, SubconBuildTypes], pad: bytes = ... + self, + subcon: Construct[SubconParsedType, SubconBuildTypes], + pad: bytes = ..., ) -> None: ... class RestreamData( @@ -999,7 +968,7 @@ class RestreamData( bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] ] def __init__( - self: t.Self, + self, datafunc: t.Union[ bytes, io.BytesIO, Construct[bytes, t.Any], t.Callable[[Context], bytes] ], @@ -1014,7 +983,7 @@ class Transformed( encodefunc: t.Callable[[bytes], bytes] encodeamount: t.Optional[int] def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], decodefunc: t.Callable[[bytes], bytes], decodeamount: t.Optional[int], @@ -1031,7 +1000,7 @@ class Restreamed( encoderunit: int sizecomputer: t.Callable[[int], int] def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], decoder: t.Callable[[bytes], bytes], decoderunit: int, @@ -1045,7 +1014,7 @@ class ProcessXor( ): padfunc: ConstantOrContextLambda2[t.Union[int, bytes]] def __init__( - self: t.Self, + self, padfunc: ConstantOrContextLambda2[t.Union[int, bytes]], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1056,7 +1025,7 @@ class ProcessRotateLeft( amount: ConstantOrContextLambda2[int] group: ConstantOrContextLambda2[int] def __init__( - self: t.Self, + self, amount: ConstantOrContextLambda2[int], group: ConstantOrContextLambda2[int], subcon: Construct[SubconParsedType, SubconBuildTypes], @@ -1080,7 +1049,7 @@ class Compressed(Tunnel[SubconParsedType, SubconBuildTypes]): level: t.Optional[int] lib: t.Any def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], encoding: str, level: t.Optional[int] = ..., @@ -1089,7 +1058,7 @@ class Compressed(Tunnel[SubconParsedType, SubconBuildTypes]): class CompressedLZ4(Tunnel[SubconParsedType, SubconBuildTypes]): lib: t.Any def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1098,7 +1067,7 @@ class Rebuffered( ): stream2: RebufferedBytesIO def __init__( - self: t.Self, + self, subcon: Construct[SubconParsedType, SubconBuildTypes], tailcutoff: t.Optional[int] = ..., ) -> None: ... @@ -1106,14 +1075,16 @@ class Rebuffered( # =============================================================================== # lazy equivalents # =============================================================================== -class Lazy(Subconstruct[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): +class Lazy( + Subconstruct[ + SubconParsedType, + SubconBuildTypes, + t.Callable[[], SubconParsedType], + t.Union[t.Callable[[], SubconParsedType], SubconParsedType], + ] +): def __init__( - self: Lazy[ - SubconParsedType, - SubconBuildTypes, - t.Callable[[], SubconParsedType], - t.Union[t.Callable[[], SubconParsedType], SubconParsedType], - ], + self, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1124,12 +1095,12 @@ class LazyContainer(t.Generic[ContainerType], t.Dict[str, ContainerType]): def values(self) -> t.List[ContainerType]: ... def items(self) -> t.List[t.Tuple[str, ContainerType]]: ... -class LazyStruct(Construct[ParsedType, BuildTypes]): +class LazyStruct(Construct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] _subcons: t.Dict[str, Construct[t.Any, t.Any]] _subconsindexes: t.Dict[str, int] def __init__( - self: LazyStruct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]], + self, *subcons: Construct[t.Any, t.Any], **subconskw: Construct[t.Any, t.Any], ) -> None: ... @@ -1141,18 +1112,13 @@ class LazyArray( Subconstruct[ SubconParsedType, SubconBuildTypes, - ParsedType, - BuildTypes, + ListContainer[SubconParsedType], # type: ignore + t.List[SubconBuildTypes], # type: ignore ] ): count: ConstantOrContextLambda[int] def __init__( - self: LazyArray[ - SubconParsedType, - SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], - ], + self, count: ConstantOrContextLambda[int], subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @@ -1160,7 +1126,7 @@ class LazyArray( class LazyBound(Construct[ParsedType, BuildTypes]): subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]] def __init__( - self: LazyBound[ParsedType, BuildTypes], + self, subconfunc: t.Callable[[], Construct[ParsedType, BuildTypes]], ) -> None: ... @@ -1207,22 +1173,23 @@ def Filter( ]: ... class Slicing( - Adapter[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] + Adapter[ + SubconParsedType, + SubconBuildTypes, + ListContainer[SubconParsedType], # type: ignore + t.List[SubconBuildTypes], # type: ignore + ] ): def __init__( - self: Slicing[ListContainer[SubconParsedType], t.List[SubconBuildTypes]], + self, subcon: t.Union[ Array[ SubconParsedType, SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], ], GreedyRange[ SubconParsedType, SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], ], ], count: int, @@ -1241,14 +1208,10 @@ class Indexing( Array[ SubconParsedType, SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], ], GreedyRange[ SubconParsedType, SubconBuildTypes, - ListContainer[SubconParsedType], - t.List[SubconBuildTypes], ], ], count: int, From 6b4a52e73a039560a3c8b7ca0b3893f8e331a582 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 15:38:40 +0200 Subject: [PATCH 064/102] merged .py and .pyi --- tests/declarativeunittest.py | 164 ++++++++++++++++++++++++++++++---- tests/declarativeunittest.pyi | 109 ---------------------- 2 files changed, 148 insertions(+), 125 deletions(-) delete mode 100644 tests/declarativeunittest.pyi diff --git a/tests/declarativeunittest.py b/tests/declarativeunittest.py index 1d1be0c..f3c8266 100644 --- a/tests/declarativeunittest.py +++ b/tests/declarativeunittest.py @@ -1,38 +1,170 @@ +import binascii +import io +import typing as t + import pytest +from construct import * +from construct.lib import * + +import construct_typed as cst xfail = pytest.mark.xfail skip = pytest.mark.skip skipif = pytest.mark.skipif -import os, math, random, collections, itertools, io, hashlib, binascii +Buffer = t.Union[bytes, memoryview, bytearray] +ParsedType = t.TypeVar("ParsedType") +BuildTypes = t.TypeVar("BuildTypes") +ContainerType = t.TypeVar("ContainerType", bound=cst.TContainerMixin) +T = t.TypeVar("T") -from construct import * -from construct.lib import * +IdentType = t.TypeVar("IdentType") class ZeroIO(io.BufferedIOBase): - def read(self, __size=None): + def read(self, __size: t.Optional[int] = None): if __size is not None: return bytes(__size) else: return bytes(0) - def read1(self, __size=0): + def read1(self, __size: int = 0): return bytes(__size) -ident = lambda x: x -devzero = ZeroIO() +def ident(x: IdentType) -> IdentType: + return x -def raises(func, *args, **kw): +devzero: t.BinaryIO = ZeroIO() # type: ignore + + +def raises( + func: t.Callable[..., t.Any], *args: t.Any, **kw: t.Any +) -> t.Union[t.Any, Exception]: try: return func(*args, **kw) except Exception as e: return e.__class__ -def common(format, datasample, objsample, sizesample=SizeofError, **kw): +@t.overload +def common( + format: cst.TStruct[ContainerType], + datasample: Buffer, + objsample: t.Union[ContainerType, t.Dict[str, t.Any]], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[ListContainer[ParsedType], t.Any]", + datasample: Buffer, + objsample: t.List[ParsedType], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[Container[t.Any], t.Any]", + datasample: Buffer, + objsample: t.Dict[str, t.Any], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[t.Union[EnumInteger, EnumIntegerString], t.Any]", + datasample: Buffer, + objsample: t.Union[int, str], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[HexDisplayedInteger, t.Any]", + datasample: Buffer, + objsample: t.Union[HexDisplayedInteger, int], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[HexDisplayedBytes, t.Any]", + datasample: Buffer, + objsample: t.Union[HexDisplayedBytes, bytes], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[HexDisplayedDict[str, t.Any], t.Any]", + datasample: Buffer, + objsample: t.Dict[str, t.Any], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[HexDumpDisplayedBytes, t.Any]", + datasample: Buffer, + objsample: t.Union[HexDumpDisplayedBytes, bytes], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[HexDumpDisplayedDict[str, t.Any], t.Any]", + datasample: Buffer, + objsample: t.Dict[str, t.Any], + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +@t.overload +def common( + format: "Construct[ParsedType, t.Any]", + datasample: Buffer, + objsample: ParsedType, + sizesample: t.Union[int, t.Type[Exception]] = ..., + **kw: t.Any +) -> None: + ... + + +def common( + format: "Construct[t.Any, t.Any]", + datasample: Buffer, + objsample: t.Any, + sizesample: t.Union[int, t.Type[Exception]] = SizeofError, + **kw: t.Any +) -> None: obj = format.parse(datasample, **kw) assert obj == objsample data = format.build(objsample, **kw) @@ -48,31 +180,31 @@ def common(format, datasample, objsample, sizesample=SizeofError, **kw): assert size == sizesample -def setattrs(obj, **kwargs): - """ Set multiple named values of an object """ +def setattrs(obj: T, **kwargs: t.Any) -> T: + """Set multiple named values of an object""" for name, value in kwargs.items(): setattr(obj, name, value) return obj -def commonhex(format, hexdata): +def commonhex(format: "Construct[t.Any, t.Any]", hexdata: str): commonbytes(format, binascii.unhexlify(hexdata)) -def commondumpdeprecated(format, filename): +def commondumpdeprecated(format: "Construct[t.Any, t.Any]", filename: str): filename = "tests/deprecated_gallery/blobs/" + filename with open(filename, "rb") as f: data = f.read() commonbytes(format, data) -def commondump(format, filename): +def commondump(format: "Construct[t.Any, t.Any]", filename: str): filename = "tests/gallery/blobs/" + filename with open(filename, "rb") as f: data = f.read() commonbytes(format, data) -def commonbytes(format, data): +def commonbytes(format: "Construct[t.Any, t.Any]", data: bytes): obj = format.parse(data) - data2 = format.build(obj) + format.build(obj) diff --git a/tests/declarativeunittest.pyi b/tests/declarativeunittest.pyi deleted file mode 100644 index e2f8cab..0000000 --- a/tests/declarativeunittest.pyi +++ /dev/null @@ -1,109 +0,0 @@ -import typing as t -from construct import * -from construct.lib import * -import construct_typed as cst - -Buffer = t.Union[bytes, memoryview, bytearray] -ParsedType = t.TypeVar("ParsedType") -BuildTypes = t.TypeVar("BuildTypes") -ContainerType = t.TypeVar("ContainerType", bound=cst.TContainerMixin) -T = t.TypeVar("T") - -IdentType = t.TypeVar("IdentType") - -def ident(p1: IdentType) -> IdentType: ... - -devzero: t.BinaryIO - -def raises( - func: t.Callable[..., t.Any], *args: t.Any, **kw: t.Any -) -> t.Union[t.Any, Exception]: ... -@t.overload -def common( - format: cst.TStruct[ContainerType], - datasample: Buffer, - objsample: t.Union[ContainerType, t.Dict[str, t.Any]], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[ListContainer[ParsedType], t.Any], - datasample: Buffer, - objsample: t.List[ParsedType], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[Container[t.Any], t.Any], - datasample: Buffer, - objsample: t.Dict[str, t.Any], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[t.Union[EnumInteger, EnumIntegerString], t.Any], - datasample: Buffer, - objsample: t.Union[int, str], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[HexDisplayedInteger, t.Any], - datasample: Buffer, - objsample: t.Union[HexDisplayedInteger, int], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[HexDisplayedBytes, t.Any], - datasample: Buffer, - objsample: t.Union[HexDisplayedBytes, bytes], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[HexDisplayedDict[str, t.Any], t.Any], - datasample: Buffer, - objsample: t.Dict[str, t.Any], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[HexDumpDisplayedBytes, t.Any], - datasample: Buffer, - objsample: t.Union[HexDumpDisplayedBytes, bytes], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[HexDumpDisplayedDict[str, t.Any], t.Any], - datasample: Buffer, - objsample: t.Dict[str, t.Any], - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -@t.overload -def common( - format: Construct[ParsedType, t.Any], - datasample: Buffer, - objsample: ParsedType, - sizesample: t.Union[int, t.Type[Exception]] = ..., - **kw: t.Any -) -> None: ... -def setattrs(obj: T, **kwargs: t.Any) -> T: ... -def commonhex(format: Construct[t.Any, t.Any], hexdata: str) -> None: ... -def commondumpdeprecated( - format: Construct[t.Any, t.Any], filename: str -) -> None: ... -def commondump(format: Construct[t.Any, t.Any], filename: str) -> None: ... -def commonbytes( - format: Construct[ParsedType, t.Any], data: ParsedType -) -> None: ... From dd1691528f85928e8887d1302079e798cf9e5c57 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 15:44:33 +0200 Subject: [PATCH 065/102] fixed pyright errors --- construct-stubs/expr.pyi | 6 +++--- tests/test_core.py | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/construct-stubs/expr.pyi b/construct-stubs/expr.pyi index 56fae52..5708077 100644 --- a/construct-stubs/expr.pyi +++ b/construct-stubs/expr.pyi @@ -498,7 +498,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __neg__(self: ExprMixin[float]) -> BinExpr[float]: ... @t.overload - def __neg__(self) -> UniExpr[t.Any]: ... + def __neg__(self) -> BinExpr[t.Any]: ... # __pos__ ########################################################################################################## @t.overload @@ -508,7 +508,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __pos__(self: ExprMixin[float]) -> BinExpr[float]: ... @t.overload - def __pos__(self) -> UniExpr[t.Any]: ... + def __pos__(self) -> BinExpr[t.Any]: ... # __invert__ ####################################################################################################### @t.overload @@ -516,7 +516,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __invert__(self: ExprMixin[bool]) -> BinExpr[int]: ... @t.overload - def __invert__(self) -> UniExpr[t.Any]: ... + def __invert__(self) -> BinExpr[t.Any]: ... # __inv__ ########################################################################################################## def __inv__(self) -> UniExpr[t.Any]: ... diff --git a/tests/test_core.py b/tests/test_core.py index 4b1e4d0..d5bfb37 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -637,8 +637,7 @@ def test_numpy_error() -> None: numpy.load(io.BytesIO(b"")) # type: ignore def test_namedtuple() -> None: - import collections - coord = collections.namedtuple("coord", "x y z") + coord = t.NamedTuple("coord", [("x", int), ("y", int), ("z", int)]) d1 = NamedTuple("coord", "x y z", Array(3, Byte)) common(d1, b"123", coord(49,50,51), 3) d2 = NamedTuple("coord", "x y z", GreedyRange(Byte)) @@ -808,8 +807,10 @@ def test_select_buildfromnone_issue_747() -> None: assert d.build(dict()) == b"" def test_if() -> None: - common(If(True, Byte), b"\x01", 1, 1) - common(If(False, Byte), b"", None, 0) + d = If(True, Byte) + common(d, b"\x01", 1, 1) + d = If(False, Byte) + common(d, b"", None, 0) def test_ifthenelse() -> None: common(IfThenElse(True, Int8ub, Int16ub), b"\x01", 1, 1) @@ -1545,7 +1546,7 @@ def test_operators() -> None: assert d.docs == "description" d = "description" * Byte assert d.docs == "description" - """ + _ = """ description """ * \ Byte @@ -1796,11 +1797,11 @@ def test_pickling_constructs() -> None: ) data = bytes(100) - du = cloudpickle.loads(cloudpickle.dumps(d, protocol=-1)) + du = cloudpickle.loads(cloudpickle.dumps(d, protocol=-1)) # type: ignore assert du.parse(data) == d.parse(data) def test_pickling_constructs_issue_894() -> None: - import cloudpickle + import cloudpickle # type: ignore fundus_header = Struct( 'width' / Int32un, @@ -1812,7 +1813,7 @@ def test_pickling_constructs_issue_894() -> None: 'img' / Int8un, ) - cloudpickle.dumps(fundus_header) + cloudpickle.dumps(fundus_header) # type: ignore def test_exposing_members_attributes() -> None: d1 = Struct( @@ -2023,7 +2024,7 @@ def test_struct_root_topmost() -> None: assert d.parse(b"", z=2) == Container(x=1, inner=Container(inner2=Container(x=1,z=2,zz=2))) def test_parsedhook_repeatersdiscard() -> None: - outputs = [] + outputs: t.List[int] = [] def printobj1(obj: int, ctx: "Context") -> None: outputs.append(obj) d1 = GreedyRange(Byte * printobj1, discard=True) From 7f08ab2f28a1a1fb4a2d261d9aca00fe6cf5598d Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:25:21 +0200 Subject: [PATCH 066/102] added intermediate variable, so that pyright v1.1.316 passes --- tests/test_core.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index d5bfb37..450007a 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # mypy: no-warn-unused-ignores -from .declarativeunittest import raises, common, commonhex, commondumpdeprecated, commondump, commonbytes, ident, devzero +from .declarativeunittest import raises, common, ident, devzero from construct.core import * from construct import * from construct.lib import * @@ -707,10 +707,13 @@ def test_hexdump() -> None: def test_hexdump_regression_issue_188() -> None: # Hex HexDump were not inheriting subcon flags - d = Struct(Hex(Const(b"MZ"))) + a = Hex(Const(b"MZ")) + d = Struct(a) assert d.parse(b"MZ") == Container() assert d.build(dict()) == b"MZ" - d = Struct(HexDump(Const(b"MZ"))) + + a = HexDump(Const(b"MZ")) + d = Struct(a) assert d.parse(b"MZ") == Container() assert d.build(dict()) == b"MZ" @@ -1688,9 +1691,11 @@ def test_from_issue_244() -> None: assert d.parse(b"abcd") == [Container(num=97, index=0),Container(num=98, index=1),Container(num=99, index=2),Container(num=100, index=3),] def test_from_issue_269() -> None: - d = Struct("enabled" / Byte, If(this.enabled, Padding(2))) + a = If(this.enabled, Padding(2)) + d = Struct("enabled" / Byte, a) assert d.build(dict(enabled=1)) == b"\x01\x00\x00" assert d.build(dict(enabled=0)) == b"\x00" + d = Struct("enabled" / Byte, "pad" / If(this.enabled, Padding(2))) assert d.build(dict(enabled=1)) == b"\x01\x00\x00" assert d.build(dict(enabled=0)) == b"\x00" From f75029247f3554fe7b7829fe64bef57173c9d375 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:26:03 +0200 Subject: [PATCH 067/102] added missing definitions --- construct-stubs/__init__.pyi | 4 ++++ construct-stubs/core.pyi | 13 ++++++++++++- construct-stubs/expr.pyi | 1 - construct-stubs/lib/hex.pyi | 4 +++- construct-stubs/lib/py3compat.pyi | 1 + 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/construct-stubs/__init__.pyi b/construct-stubs/__init__.pyi index 858384d..2cad48a 100644 --- a/construct-stubs/__init__.pyi +++ b/construct-stubs/__init__.pyi @@ -3,6 +3,10 @@ from construct.debug import * from construct.expr import * from construct.lib import * from construct.version import * +from construct import lib + +__author__: str +__version__: str #=============================================================================== # exposed names diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index a19686b..796624d 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -207,7 +207,16 @@ class Tunnel( def _decode(self, data: bytes, context: Context, path: PathType) -> bytes: ... def _encode(self, data: bytes, context: Context, path: PathType) -> bytes: ... -# TODO: Compiled +class Compiled(Construct[t.Any, t.Any]): + source: t.Optional[str] + defersubcon: t.Optional[Construct[t.Any, t.Any]] + parsefunc: t.Callable[[StreamType, Context], t.Any] + buildfunc: t.Callable[[t.Any, StreamType, Context], t.Any] + def __init__( + self, + parsefunc: t.Callable[[StreamType, Context], t.Any], + buildfunc: t.Callable[[t.Any, StreamType, Context], t.Any], + ) -> None: ... # =============================================================================== # bytes and bits @@ -356,6 +365,8 @@ ZigZag: Construct[int, int] # =============================================================================== # strings # =============================================================================== +possiblestringencodings: t.Dict[str, int] + class StringEncoded(Construct[str, str]): if sys.version_info >= (3, 8): ENCODING_1 = t.Literal["ascii", "utf8", "utf_8", "u8"] diff --git a/construct-stubs/expr.pyi b/construct-stubs/expr.pyi index 5708077..3d1b032 100644 --- a/construct-stubs/expr.pyi +++ b/construct-stubs/expr.pyi @@ -1,4 +1,3 @@ -import operator import typing as t from construct.core import * diff --git a/construct-stubs/lib/hex.pyi b/construct-stubs/lib/hex.pyi index afa985f..a39d918 100644 --- a/construct-stubs/lib/hex.pyi +++ b/construct-stubs/lib/hex.pyi @@ -1,6 +1,5 @@ import typing as t - class HexDisplayedInteger(int): ... class HexDisplayedBytes(bytes): ... @@ -10,3 +9,6 @@ V = t.TypeVar("V") class HexDisplayedDict(t.Dict[K, V]): ... class HexDumpDisplayedBytes(bytes): ... class HexDumpDisplayedDict(t.Dict[K, V]): ... + +def hexdump(data: bytes, linesize: int) -> str: ... +def hexundump(data: str, linesize: int) -> bytes: ... diff --git a/construct-stubs/lib/py3compat.pyi b/construct-stubs/lib/py3compat.pyi index f105096..c86f2f5 100644 --- a/construct-stubs/lib/py3compat.pyi +++ b/construct-stubs/lib/py3compat.pyi @@ -1,5 +1,6 @@ import typing as t +PY: t.Tuple[int, int] PY2: bool PY3: bool PYPY: bool From 3c81d99c12410cfc57566b8da16718472c235203 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:36:27 +0200 Subject: [PATCH 068/102] Added __new__ only where it is an absolute must have. --- construct-stubs/core.pyi | 318 ++++++++++++++++++++------------------- 1 file changed, 165 insertions(+), 153 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 796624d..6323b4f 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -246,41 +246,45 @@ def Bytewise( # =============================================================================== # integers and floats # =============================================================================== -class _FormatField(Construct[ParsedType, BuildTypes]): +class FormatField(Construct[ParsedType, BuildTypes]): fmtstr: str length: int + if sys.version_info >= (3, 8): + ENDIANITY = t.Union[t.Literal["=", "<", ">"], str] + FORMAT_INT = t.Literal["B", "H", "L", "Q", "b", "h", "l", "q"] + FORMAT_FLOAT = t.Literal["f", "d", "e"] + FORMAT_BOOL = t.Literal["?"] + @t.overload + def __new__( + cls, + endianity: str, + format: FORMAT_INT, + ) -> FormatField[int, int]: ... + @t.overload + def __new__( + cls, + endianity: str, + format: FORMAT_FLOAT, + ) -> FormatField[float, float]: ... + @t.overload + def __new__( + cls, + endianity: str, + format: FORMAT_BOOL, + ) -> FormatField[bool, bool]: ... + @t.overload + def __new__( + cls, + endianity: str, + format: str, + ) -> FormatField[t.Any, t.Any]: ... -if sys.version_info >= (3, 8): - ENDIANITY = t.Union[t.Literal["=", "<", ">"], str] - FORMAT_INT = t.Literal["B", "H", "L", "Q", "b", "h", "l", "q"] - FORMAT_FLOAT = t.Literal["f", "d", "e"] - FORMAT_BOOL = t.Literal["?"] - @t.overload - def FormatField( - endianity: str, - format: FORMAT_INT, - ) -> _FormatField[int, int]: ... - @t.overload - def FormatField( - endianity: str, - format: FORMAT_FLOAT, - ) -> _FormatField[float, float]: ... - @t.overload - def FormatField( - endianity: str, - format: FORMAT_BOOL, - ) -> _FormatField[bool, bool]: ... - @t.overload - def FormatField( - endianity: str, - format: str, - ) -> _FormatField[t.Any, t.Any]: ... - -else: - def FormatField( - endianity: str, - format: str, - ) -> _FormatField[t.Any, t.Any]: ... + else: + def __new__( + cls, + endianity: str, + format: str, + ) -> FormatField[t.Any, t.Any]: ... class BytesInteger(Construct[int, int]): length: ConstantOrContextLambda[int] @@ -308,49 +312,49 @@ Bit: BitsInteger Nibble: BitsInteger Octet: BitsInteger -Int8ub: _FormatField[int, int] -Int16ub: _FormatField[int, int] -Int32ub: _FormatField[int, int] -Int64ub: _FormatField[int, int] -Int8sb: _FormatField[int, int] -Int16sb: _FormatField[int, int] -Int32sb: _FormatField[int, int] -Int64sb: _FormatField[int, int] -Int8ul: _FormatField[int, int] -Int16ul: _FormatField[int, int] -Int32ul: _FormatField[int, int] -Int64ul: _FormatField[int, int] -Int8sl: _FormatField[int, int] -Int16sl: _FormatField[int, int] -Int32sl: _FormatField[int, int] -Int64sl: _FormatField[int, int] -Int8un: _FormatField[int, int] -Int16un: _FormatField[int, int] -Int32un: _FormatField[int, int] -Int64un: _FormatField[int, int] -Int8sn: _FormatField[int, int] -Int16sn: _FormatField[int, int] -Int32sn: _FormatField[int, int] -Int64sn: _FormatField[int, int] +Int8ub: FormatField[int, int] +Int16ub: FormatField[int, int] +Int32ub: FormatField[int, int] +Int64ub: FormatField[int, int] +Int8sb: FormatField[int, int] +Int16sb: FormatField[int, int] +Int32sb: FormatField[int, int] +Int64sb: FormatField[int, int] +Int8ul: FormatField[int, int] +Int16ul: FormatField[int, int] +Int32ul: FormatField[int, int] +Int64ul: FormatField[int, int] +Int8sl: FormatField[int, int] +Int16sl: FormatField[int, int] +Int32sl: FormatField[int, int] +Int64sl: FormatField[int, int] +Int8un: FormatField[int, int] +Int16un: FormatField[int, int] +Int32un: FormatField[int, int] +Int64un: FormatField[int, int] +Int8sn: FormatField[int, int] +Int16sn: FormatField[int, int] +Int32sn: FormatField[int, int] +Int64sn: FormatField[int, int] -Byte: _FormatField[int, int] -Short: _FormatField[int, int] -Int: _FormatField[int, int] -Long: _FormatField[int, int] +Byte: FormatField[int, int] +Short: FormatField[int, int] +Int: FormatField[int, int] +Long: FormatField[int, int] -Float16b: _FormatField[float, float] -Float16l: _FormatField[float, float] -Float16n: _FormatField[float, float] -Float32b: _FormatField[float, float] -Float32l: _FormatField[float, float] -Float32n: _FormatField[float, float] -Float64b: _FormatField[float, float] -Float64l: _FormatField[float, float] -Float64n: _FormatField[float, float] +Float16b: FormatField[float, float] +Float16l: FormatField[float, float] +Float16n: FormatField[float, float] +Float32b: FormatField[float, float] +Float32l: FormatField[float, float] +Float32n: FormatField[float, float] +Float64b: FormatField[float, float] +Float64l: FormatField[float, float] +Float64n: FormatField[float, float] -Half: _FormatField[float, float] -Single: _FormatField[float, float] -Double: _FormatField[float, float] +Half: FormatField[float, float] +Single: FormatField[float, float] +Double: FormatField[float, float] Int24ub: BytesInteger Int24ul: BytesInteger @@ -543,17 +547,19 @@ class Renamed( # =============================================================================== # miscellaneous # =============================================================================== -class _Const(Subconstruct[None, None, SubconParsedType, SubconBuildTypes]): ... - -@t.overload -def Const( - value: bytes, -) -> _Const[bytes, t.Optional[bytes]]: ... -@t.overload -def Const( - value: SubconBuildTypes, - subcon: Construct[SubconParsedType, SubconBuildTypes], -) -> _Const[SubconParsedType, t.Optional[SubconBuildTypes]]: ... +class Const(Subconstruct[None, None, SubconParsedType, SubconBuildTypes]): + value: SubconBuildTypes + @t.overload + def __new__( + cls, + value: bytes, + ) -> Const[bytes, t.Optional[bytes]]: ... + @t.overload + def __new__( + cls, + value: SubconBuildTypes, + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> Const[SubconParsedType, t.Optional[SubconBuildTypes]]: ... class Computed(Construct[ParsedType, None]): func: ConstantOrContextLambda2[ParsedType] @@ -658,63 +664,68 @@ def Timestamp( K = t.TypeVar("K") V = t.TypeVar("V") -class _Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - pass +class Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): + @t.overload + def __new__( + cls, + subcon: Construct[int, BuildTypes], + ) -> Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes]: ... + @t.overload + def __new__( + cls, + subcon: Construct[bytes, BuildTypes], + ) -> Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes]: ... + @t.overload + def __new__( + cls, + subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], + ) -> Hex[ + RawCopyObj[SubconParsedType], + BuildTypes, + HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], + BuildTypes, + ]: ... + @t.overload + def __new__( + cls, + subcon: Construct[Container[t.Any], BuildTypes], + ) -> Hex[Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes]: ... + @t.overload + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> Hex[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes]: ... -@t.overload -def Hex( - subcon: Construct[int, BuildTypes], -) -> _Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes]: ... -@t.overload -def Hex( - subcon: Construct[bytes, BuildTypes], -) -> _Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes]: ... -@t.overload -def Hex( - subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], -) -> _Hex[ - RawCopyObj[SubconParsedType], - BuildTypes, - HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], - BuildTypes, -]: ... -@t.overload -def Hex( - subcon: Construct[Container[t.Any], BuildTypes], -) -> _Hex[Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes]: ... -@t.overload -def Hex( - subcon: Construct[SubconParsedType, SubconBuildTypes], -) -> _Hex[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes]: ... - -class _HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): - pass - -@t.overload -def HexDump( - subcon: Construct[bytes, BuildTypes], -) -> _HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes]: ... -@t.overload -def HexDump( - subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], -) -> _HexDump[ - RawCopyObj[SubconParsedType], - BuildTypes, - HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], - BuildTypes, -]: ... -@t.overload -def HexDump( - subcon: Construct[Container[t.Any], BuildTypes], -) -> _HexDump[ - Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes -]: ... -@t.overload -def HexDump( - subcon: Construct[SubconParsedType, SubconBuildTypes], -) -> _HexDump[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes -]: ... +class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): + @t.overload + def __new__( + cls, + subcon: Construct[bytes, BuildTypes], + ) -> HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes]: ... + @t.overload + def __new__( + cls, + subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], + ) -> HexDump[ + RawCopyObj[SubconParsedType], + BuildTypes, + HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], + BuildTypes, + ]: ... + @t.overload + def __new__( + cls, + subcon: Construct[Container[t.Any], BuildTypes], + ) -> HexDump[ + Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes + ]: ... + @t.overload + def __new__( + cls, + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> HexDump[ + SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes + ]: ... # =============================================================================== # conditional @@ -772,23 +783,24 @@ def If( SwitchType = t.TypeVar("SwitchType") -class _Switch(Construct[ParsedType, BuildTypes]): +class Switch(Construct[ParsedType, BuildTypes]): keyfunc: ConstantOrContextLambda[t.Any] cases: t.Dict[t.Any, Construct[t.Any, t.Any]] default: Construct[t.Any, t.Any] - -@t.overload -def Switch( - keyfunc: ConstantOrContextLambda[SwitchType], - cases: t.Dict[SwitchType, Construct[int, int]], - default: t.Optional[Construct[int, int]] = ..., -) -> _Switch[int, t.Optional[int]]: ... -@t.overload -def Switch( - keyfunc: ConstantOrContextLambda[t.Any], - cases: t.Dict[t.Any, Construct[t.Any, t.Any]], - default: t.Optional[Construct[t.Any, t.Any]] = ..., -) -> _Switch[t.Any, t.Any]: ... + @t.overload + def __new__( + cls, + keyfunc: ConstantOrContextLambda[SwitchType], + cases: t.Dict[SwitchType, Construct[int, int]], + default: t.Optional[Construct[int, int]] = ..., + ) -> Switch[int, t.Optional[int]]: ... + @t.overload + def __new__( + cls, + keyfunc: ConstantOrContextLambda[t.Any], + cases: t.Dict[t.Any, Construct[t.Any, t.Any]], + default: t.Optional[Construct[t.Any, t.Any]] = ..., + ) -> Switch[t.Any, t.Any]: ... class StopIf(Construct[None, None]): condfunc: ConstantOrContextLambda[bool] From e39fc5b19d855a1c974bb02c16c07a1551a7a837 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:36:54 +0200 Subject: [PATCH 069/102] corrected version requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 87d1d1b..d1c8be1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -construct==2.10.67 +construct==2.10.68 pytest>=6.2.0 numpy arrow From 825783db574f506c25e86ecc490b62b51dab9561 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:37:29 +0200 Subject: [PATCH 070/102] fixed mypy issues --- construct_typed/dataclass_struct.py | 4 ++-- tests/declarativeunittest.py | 16 ++++++++-------- tests/test_core.py | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index f79fc7e..8a4c562 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -102,10 +102,10 @@ def csfield( # Set default values in case of special sucons if isinstance(orig_subcon, cs.Const): - const_subcon: "cs.Const[t.Any, t.Any, t.Any, t.Any]" = orig_subcon + const_subcon: "cs.Const[t.Any, t.Any]" = orig_subcon default = const_subcon.value elif isinstance(orig_subcon, cs.Default): - default_subcon: "cs.Default[t.Any, t.Any, t.Any, t.Any]" = orig_subcon + default_subcon: "cs.Default[t.Any, t.Any]" = orig_subcon if callable(default_subcon.value): default = None # context lambda is only defined at parsing/building else: diff --git a/tests/declarativeunittest.py b/tests/declarativeunittest.py index f3c8266..ed7fb59 100644 --- a/tests/declarativeunittest.py +++ b/tests/declarativeunittest.py @@ -22,13 +22,13 @@ IdentType = t.TypeVar("IdentType") class ZeroIO(io.BufferedIOBase): - def read(self, __size: t.Optional[int] = None): + def read(self, __size: t.Optional[int] = None) -> bytes: if __size is not None: return bytes(__size) else: return bytes(0) - def read1(self, __size: int = 0): + def read1(self, __size: int = 0) -> bytes: return bytes(__size) @@ -176,8 +176,8 @@ def common( size = format.sizeof(**kw) assert size == sizesample else: - size = raises(format.sizeof, **kw) - assert size == sizesample + size_ex = raises(format.sizeof, **kw) + assert size_ex == sizesample def setattrs(obj: T, **kwargs: t.Any) -> T: @@ -187,24 +187,24 @@ def setattrs(obj: T, **kwargs: t.Any) -> T: return obj -def commonhex(format: "Construct[t.Any, t.Any]", hexdata: str): +def commonhex(format: "Construct[t.Any, t.Any]", hexdata: str) -> None: commonbytes(format, binascii.unhexlify(hexdata)) -def commondumpdeprecated(format: "Construct[t.Any, t.Any]", filename: str): +def commondumpdeprecated(format: "Construct[t.Any, t.Any]", filename: str) -> None: filename = "tests/deprecated_gallery/blobs/" + filename with open(filename, "rb") as f: data = f.read() commonbytes(format, data) -def commondump(format: "Construct[t.Any, t.Any]", filename: str): +def commondump(format: "Construct[t.Any, t.Any]", filename: str) -> None: filename = "tests/gallery/blobs/" + filename with open(filename, "rb") as f: data = f.read() commonbytes(format, data) -def commonbytes(format: "Construct[t.Any, t.Any]", data: bytes): +def commonbytes(format: "Construct[t.Any, t.Any]", data: bytes) -> None: obj = format.parse(data) format.build(obj) diff --git a/tests/test_core.py b/tests/test_core.py index 450007a..6cfd8ba 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -712,8 +712,8 @@ def test_hexdump_regression_issue_188() -> None: assert d.parse(b"MZ") == Container() assert d.build(dict()) == b"MZ" - a = HexDump(Const(b"MZ")) - d = Struct(a) + b = HexDump(Const(b"MZ")) + d = Struct(b) assert d.parse(b"MZ") == Container() assert d.build(dict()) == b"MZ" From d9026772a0f9d95a8896a78b5c28359ecc624667 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:39:10 +0200 Subject: [PATCH 071/102] fixed further mypy issues --- tests/test_core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 6cfd8ba..0531715 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -420,11 +420,11 @@ def test_struct_proper_context() -> None: "x"/Byte, "inner"/Struct( "y"/Byte, - "a"/Computed(this._.x+1), - "b"/Computed(this.y+2), + "a"/Computed[int](this._.x+1), + "b"/Computed[int](this.y+2), ), - "c"/Computed(this.x+3), - "d"/Computed(this.inner.y+4), + "c"/Computed[int](this.x+3), + "d"/Computed[int](this.inner.y+4), ) assert d.parse(b"\x01\x0f") == Container(x=1, inner=Container(y=15, a=2, b=17), c=4, d=19) @@ -511,7 +511,7 @@ def test_const() -> None: def test_computed() -> None: common(Computed(255), b"", 255, 0) - common(Computed(lambda ctx: 255), b"", 255, 0) + common(Computed[int](lambda ctx: 255), b"", 255, 0) assert Computed(255).build(None) == b"" assert Struct(Computed(255)).build({}) == b"" assert raises(Computed(this.missing).parse, b"") == KeyError From ede1a72c70585ad4e6114388456cc23a9389075d Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Tue, 18 Jul 2023 17:42:57 +0200 Subject: [PATCH 072/102] added ignores, because 'Computed' is not subscriptable at runtime --- tests/test_core.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 0531715..b31cd6b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -420,11 +420,11 @@ def test_struct_proper_context() -> None: "x"/Byte, "inner"/Struct( "y"/Byte, - "a"/Computed[int](this._.x+1), - "b"/Computed[int](this.y+2), + "a"/Computed(this._.x+1), # type: ignore + "b"/Computed(this.y+2), # type: ignore ), - "c"/Computed[int](this.x+3), - "d"/Computed[int](this.inner.y+4), + "c"/Computed(this.x+3), # type: ignore + "d"/Computed(this.inner.y+4), # type: ignore ) assert d.parse(b"\x01\x0f") == Container(x=1, inner=Container(y=15, a=2, b=17), c=4, d=19) @@ -511,7 +511,7 @@ def test_const() -> None: def test_computed() -> None: common(Computed(255), b"", 255, 0) - common(Computed[int](lambda ctx: 255), b"", 255, 0) + common(Computed(lambda ctx: 255), b"", 255, 0) # type: ignore assert Computed(255).build(None) == b"" assert Struct(Computed(255)).build({}) == b"" assert raises(Computed(this.missing).parse, b"") == KeyError From 86f27dc1fbbade2c7469e9d49a95eec7e8886320 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 09:40:50 +0200 Subject: [PATCH 073/102] use method-scoped TypeVars for __new__ (see here https://github.com/microsoft/pyright/issues/5404#issuecomment-1645764913) --- construct-stubs/core.pyi | 66 ++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 6323b4f..d138539 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -256,32 +256,32 @@ class FormatField(Construct[ParsedType, BuildTypes]): FORMAT_BOOL = t.Literal["?"] @t.overload def __new__( - cls, + cls: "type[FormatField[int, int]]", endianity: str, format: FORMAT_INT, ) -> FormatField[int, int]: ... @t.overload def __new__( - cls, + cls: "type[FormatField[float, float]]", endianity: str, format: FORMAT_FLOAT, ) -> FormatField[float, float]: ... @t.overload def __new__( - cls, + cls: "type[FormatField[bool, bool]]", endianity: str, format: FORMAT_BOOL, ) -> FormatField[bool, bool]: ... @t.overload def __new__( - cls, + cls: "type[FormatField[t.Any, t.Any]]", endianity: str, format: str, ) -> FormatField[t.Any, t.Any]: ... else: def __new__( - cls, + cls: "type[FormatField[t.Any, t.Any]]", endianity: str, format: str, ) -> FormatField[t.Any, t.Any]: ... @@ -547,16 +547,16 @@ class Renamed( # =============================================================================== # miscellaneous # =============================================================================== -class Const(Subconstruct[None, None, SubconParsedType, SubconBuildTypes]): - value: SubconBuildTypes +class Const(Subconstruct[t.Any, t.Any, ParsedType, BuildTypes]): + value: BuildTypes @t.overload def __new__( - cls, + cls: "type[Const[bytes, t.Optional[bytes]]]", value: bytes, ) -> Const[bytes, t.Optional[bytes]]: ... @t.overload def __new__( - cls, + cls: "type[Const[SubconParsedType, t.Optional[SubconBuildTypes]]]", value: SubconBuildTypes, subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> Const[SubconParsedType, t.Optional[SubconBuildTypes]]: ... @@ -664,68 +664,60 @@ def Timestamp( K = t.TypeVar("K") V = t.TypeVar("V") -class Hex(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): +class Hex(Adapter[t.Any, t.Any, ParsedType, BuildTypes]): @t.overload def __new__( - cls, + cls: "type[Hex[HexDisplayedInteger, BuildTypes]]", subcon: Construct[int, BuildTypes], - ) -> Hex[int, BuildTypes, HexDisplayedInteger, BuildTypes]: ... + ) -> Hex[HexDisplayedInteger, BuildTypes]: ... @t.overload def __new__( - cls, + cls: "type[Hex[HexDisplayedBytes, BuildTypes]]", subcon: Construct[bytes, BuildTypes], - ) -> Hex[bytes, BuildTypes, HexDisplayedBytes, BuildTypes]: ... + ) -> Hex[HexDisplayedBytes, BuildTypes]: ... @t.overload def __new__( - cls, + cls: "type[Hex[HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], BuildTypes,]]", subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], ) -> Hex[ - RawCopyObj[SubconParsedType], - BuildTypes, HexDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], BuildTypes, ]: ... @t.overload def __new__( - cls, + cls: "type[Hex[HexDisplayedDict[str, t.Any], BuildTypes]]", subcon: Construct[Container[t.Any], BuildTypes], - ) -> Hex[Container[t.Any], BuildTypes, HexDisplayedDict[str, t.Any], BuildTypes]: ... + ) -> Hex[HexDisplayedDict[str, t.Any], BuildTypes]: ... @t.overload def __new__( - cls, + cls: "type[Hex[SubconParsedType, SubconBuildTypes]]", subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> Hex[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes]: ... + ) -> Hex[SubconParsedType, SubconBuildTypes]: ... -class HexDump(Adapter[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes]): +class HexDump(Adapter[t.Any, t.Any, ParsedType, BuildTypes]): @t.overload def __new__( - cls, + cls: "type[HexDump[HexDumpDisplayedBytes, BuildTypes]]", subcon: Construct[bytes, BuildTypes], - ) -> HexDump[bytes, BuildTypes, HexDumpDisplayedBytes, BuildTypes]: ... + ) -> HexDump[HexDumpDisplayedBytes, BuildTypes]: ... @t.overload def __new__( - cls, + cls: "type[HexDump[HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]],BuildTypes,]]", subcon: Construct[RawCopyObj[SubconParsedType], BuildTypes], ) -> HexDump[ - RawCopyObj[SubconParsedType], - BuildTypes, HexDumpDisplayedDict[str, t.Union[int, bytes, SubconParsedType]], BuildTypes, ]: ... @t.overload def __new__( - cls, + cls: "type[HexDump[HexDumpDisplayedDict[str, t.Any], BuildTypes]]", subcon: Construct[Container[t.Any], BuildTypes], - ) -> HexDump[ - Container[t.Any], BuildTypes, HexDumpDisplayedDict[str, t.Any], BuildTypes - ]: ... + ) -> HexDump[HexDumpDisplayedDict[str, t.Any], BuildTypes]: ... @t.overload def __new__( - cls, + cls: "type[HexDump[SubconParsedType, SubconBuildTypes]]", subcon: Construct[SubconParsedType, SubconBuildTypes], - ) -> HexDump[ - SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes - ]: ... + ) -> HexDump[SubconParsedType, SubconBuildTypes]: ... # =============================================================================== # conditional @@ -789,14 +781,14 @@ class Switch(Construct[ParsedType, BuildTypes]): default: Construct[t.Any, t.Any] @t.overload def __new__( - cls, + cls: "type[Switch[int, t.Optional[int]]]", keyfunc: ConstantOrContextLambda[SwitchType], cases: t.Dict[SwitchType, Construct[int, int]], default: t.Optional[Construct[int, int]] = ..., ) -> Switch[int, t.Optional[int]]: ... @t.overload def __new__( - cls, + cls: "type[Switch[t.Any, t.Any]]", keyfunc: ConstantOrContextLambda[t.Any], cases: t.Dict[t.Any, Construct[t.Any, t.Any]], default: t.Optional[Construct[t.Any, t.Any]] = ..., From 26f7fd82dc05a4774e9a575f41a1bfb37ed2a815 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 10:14:33 +0200 Subject: [PATCH 074/102] removed unnessesary comment --- construct-stubs/core.pyi | 4 ---- 1 file changed, 4 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index d138539..aa1c4c7 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -25,10 +25,6 @@ from construct.lib import ( # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python -# unfortunalty the static type checkers "pyright" and "mypy" are slight different. pyright is not fully analysing the type hint of the -# self type in the __init__ (eg. self: Construct[int, int] is not working). but pyright would support such type hints of the return type -# of __new__. indeed mypy doens not support the type inference for the method __new__, but fully supports the annotation of self in __init__... - StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str From 86fddbe2ace94e3d12e3455709aa06cbbc4ee6ed Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 10:14:52 +0200 Subject: [PATCH 075/102] simplified "IfThenElse" --- construct-stubs/core.pyi | 26 +++++++++++++++----------- tests/test_typed.py | 9 +++++++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index aa1c4c7..23d9c43 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -749,25 +749,29 @@ ThenBuildTypes = t.TypeVar("ThenBuildTypes") ElseParsedType = t.TypeVar("ElseParsedType") ElseBuildTypes = t.TypeVar("ElseBuildTypes") -class IfThenElse( - Construct[ - t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes] - ] -): +class IfThenElse(Construct[ParsedType, BuildTypes]): condfunc: ConstantOrContextLambda[bool] - thensubcon: Construct[ThenParsedType, ThenBuildTypes] - elsesubcon: Construct[ElseParsedType, ElseBuildTypes] - def __init__( - self, + thensubcon: Construct[t.Any, t.Any] + elsesubcon: Construct[t.Any, t.Any] + @t.overload + def __new__( + cls: "type[IfThenElse[t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes]]]", condfunc: ConstantOrContextLambda[bool], thensubcon: Construct[ThenParsedType, ThenBuildTypes], elsesubcon: Construct[ElseParsedType, ElseBuildTypes], - ) -> None: ... + ) -> "IfThenElse[t.Union[ThenParsedType, ElseParsedType], t.Union[ThenBuildTypes, ElseBuildTypes]]": ... + @t.overload + def __new__( + cls: "type[IfThenElse[t.Any, t.Any]]", + condfunc: ConstantOrContextLambda[bool], + thensubcon: Construct[t.Any, t.Any], + elsesubcon: Construct[t.Any, t.Any], + ) -> "IfThenElse[t.Any, t.Any]": ... def If( condfunc: ConstantOrContextLambda[bool], subcon: Construct[ThenParsedType, ThenBuildTypes], -) -> IfThenElse[ThenParsedType, None, ThenBuildTypes, None]: ... +) -> IfThenElse[t.Optional[ThenParsedType], t.Optional[ThenBuildTypes]]: ... SwitchType = t.TypeVar("SwitchType") diff --git a/tests/test_typed.py b/tests/test_typed.py index 7d726a3..b85058a 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -72,6 +72,15 @@ def test_dataclass_str_repr() -> None: == "Image: \n signature = b'BMP' (total 3)\n width = 3\n height = 2" ) +def test_dataclass_ifthenelse() -> None: + @dataclasses.dataclass + class IfThenElseTest(DataclassMixin): + test_if: t.Optional[int] = csfield(cs.If(False, cs.Int8ub)) + test_ifthenelse: t.Optional[int] = csfield(cs.IfThenElse(True, cs.Int8ub, cs.Pass)) + + a = IfThenElseTest(test_if=None, test_ifthenelse=None) + assert a.test_if == None + assert a.test_ifthenelse == None def test_dataclass_struct() -> None: @dataclasses.dataclass From c6bded59f4a53a8b220707bbddf6b2055dd7dffa Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 10:56:37 +0200 Subject: [PATCH 076/102] fixes #25 --- construct-stubs/core.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 23d9c43..34aaf79 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -105,10 +105,10 @@ class Construct(t.Generic[ParsedType, BuildTypes]): def build(self, obj: BuildTypes, **contextkw: ContextKWType) -> bytes: ... def build_stream( self, obj: BuildTypes, stream: StreamType, **contextkw: ContextKWType - ) -> bytes: ... + ) -> None: ... def build_file( self, obj: BuildTypes, filename: FilenameType, **contextkw: ContextKWType - ) -> bytes: ... + ) -> None: ... def sizeof(self, **contextkw: ContextKWType) -> int: ... def compile( self, filename: FilenameType = ... From c04a90575d5e68097ed455f918c01e89f1b3d744 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 09:58:17 +0200 Subject: [PATCH 077/102] use PEP688 buffer protocol for "parse" Method (fixes #24) --- construct-stubs/core.pyi | 8 ++++++-- setup.py | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 34aaf79..4a28191 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -17,6 +17,7 @@ from construct.lib import ( ListType, RebufferedBytesIO, ) +from typing_extensions import Buffer # unfortunately, there are a few duplications with "typing", e.g. Union and Optional, which is why the t. prefix must be used everywhere @@ -25,6 +26,7 @@ from construct.lib import ( # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python +ReadableBuffer: t.TypeAlias = Buffer StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str @@ -95,7 +97,7 @@ class Construct(t.Generic[ParsedType, BuildTypes]): docs: str flagbuildnone: bool parsed: t.Optional[t.Callable[[ParsedType, Context], None]] - def parse(self, data: bytes, **contextkw: ContextKWType) -> ParsedType: ... + def parse(self, data: ReadableBuffer, **contextkw: ContextKWType) -> ParsedType: ... def parse_stream( self, stream: StreamType, **contextkw: ContextKWType ) -> ParsedType: ... @@ -113,7 +115,9 @@ class Construct(t.Generic[ParsedType, BuildTypes]): def compile( self, filename: FilenameType = ... ) -> Construct[ParsedType, BuildTypes]: ... - def benchmark(self, sampledata: bytes, filename: FilenameType = ...) -> str: ... + def benchmark( + self, sampledata: ReadableBuffer, filename: FilenameType = ... + ) -> str: ... def export_ksy( self, schemaname: str = ..., filename: FilenameType = ... ) -> str: ... diff --git a/setup.py b/setup.py index 54dd52d..aa7b612 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,10 @@ setup( url="https://github.com/timrid/construct-typing", author="Tim Riddermann", python_requires=">=3.7", - install_requires=["construct==2.10.68"], + install_requires=[ + "construct==2.10.68", + "typing_extensions>=4.6.0" + ], keywords=[ "construct", "kaitai", From af1a93c630e5c977699c02c56f34f6ed16ca321a Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 11:02:39 +0200 Subject: [PATCH 078/102] incremented version to 0.6.0 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index 740da2f..f097ab1 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 5, 6) -version_string = "0.5.6" +version = (0, 6, 0) +version_string = "0.6.0" From 74593404b1edf6e2d4a7ee2684f13366c3c5f282 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 11:55:05 +0200 Subject: [PATCH 079/102] fixed missmatch between stub and runtime type for `Array` --- construct_typed/generic_wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index f570f70..742c267 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -39,7 +39,7 @@ else: pass class Array( - t.Generic[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], + t.Generic[SubconParsedType, SubconBuildTypes], cs.Array, ): pass From af98d2004d0f151e1f4e1e741fe98d5ade1cd9fb Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Mon, 24 Jul 2023 11:56:08 +0200 Subject: [PATCH 080/102] incremented version to 0.6.1 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index f097ab1..721c684 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 6, 0) -version_string = "0.6.0" +version = (0, 6, 1) +version_string = "0.6.1" From c405d09d5f9b696aee80fe88032e05d92b780798 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Thu, 3 Aug 2023 09:21:30 +0200 Subject: [PATCH 081/102] fixed error message from `EnumBase` and `FlagsEnumBase` that occures since pyright v1.1.320 --- construct_typed/tenum.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index a71fb7f..b602f23 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -1,6 +1,8 @@ import enum import typing as t +from typing_extensions import Self + from .generic_wrapper import * @@ -45,7 +47,7 @@ class EnumBase(enum.IntEnum): 'This is the running state.' """ - def __new__(cls, val: t.Union[EnumValue, int]) -> "EnumBase": + def __new__(cls, val: t.Union[EnumValue, int]) -> "Self": if isinstance(val, EnumValue): obj = int.__new__(cls, val.value) obj._value_ = val.value @@ -158,7 +160,7 @@ class FlagsEnumBase(enum.IntFlag): 'This is option two.' """ - def __new__(cls, val: t.Union[EnumValue, int]) -> "FlagsEnumBase": + def __new__(cls, val: t.Union[EnumValue, int]) -> "Self": if isinstance(val, EnumValue): obj = int.__new__(cls, val.value) obj._value_ = val.value From c7e3fc705770afea6c39c17c0265e5481a21ee1b Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Thu, 3 Aug 2023 09:22:33 +0200 Subject: [PATCH 082/102] removed unnessasary `__new__` methods in `DataclassStruct`, `TEnum` and `TFlagsEnum` --- construct_typed/dataclass_struct.py | 9 --------- construct_typed/tenum.py | 16 ---------------- 2 files changed, 25 deletions(-) diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 8a4c562..f276c99 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -153,15 +153,6 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): """ subcon: "cs.Struct" - if t.TYPE_CHECKING: - - def __new__( - cls, - dc_type: t.Type[DataclassType], - reverse: bool = False, - ) -> "DataclassStruct[DataclassType]": - ... - def __init__( self, dc_type: t.Type[DataclassType], diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index b602f23..4ae6c03 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -91,14 +91,6 @@ class TEnum(Adapter[int, int, EnumType, EnumType]): """ Typed enum. """ - - if t.TYPE_CHECKING: - - def __new__( - cls, subcon: Construct[int, int], enum_type: t.Type[EnumType] - ) -> "TEnum[EnumType]": - ... - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[EnumType]): if not issubclass(enum_type, EnumBase): raise TypeError( @@ -195,14 +187,6 @@ class TFlagsEnum(Adapter[int, int, FlagsEnumType, FlagsEnumType]): """ Typed enum. """ - - if t.TYPE_CHECKING: - - def __new__( - cls, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType] - ) -> "TFlagsEnum[FlagsEnumType]": - ... - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType]): if not issubclass(enum_type, FlagsEnumBase): raise TypeError( From 2ffa785ce60800d58523c394603bcd1bb409fde1 Mon Sep 17 00:00:00 2001 From: Tim Riddermann Date: Thu, 3 Aug 2023 09:27:59 +0200 Subject: [PATCH 083/102] incremented version to 0.6.2 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index 721c684..b1bde74 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 6, 1) -version_string = "0.6.1" +version = (0, 6, 2) +version_string = "0.6.2" From 9957b0e6f127132ef8341c1d69733d183a80303b Mon Sep 17 00:00:00 2001 From: Prilkop Date: Mon, 17 Jun 2024 15:46:04 +0300 Subject: [PATCH 084/102] added internal Construct methods, used when inheriting the class --- construct-stubs/core.pyi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 4a28191..70c7876 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -138,6 +138,10 @@ class Construct(t.Generic[ParsedType, BuildTypes]): def __getitem__( self, count: t.Union[int, t.Callable[[Context], int]] ) -> Array[ParsedType, BuildTypes,]: ... + def _parse(self, stream: StreamType, context: Context, path: PathType) -> ParsedType: ... + def _parsereport(self, stream: StreamType, context: Context, path: PathType) -> ParsedType: ... + def _build(self, obj: BuildTypes, stream: StreamType, context: Context, path: PathType) -> int: ... + def _sizeof(self, context: Context, path: PathType) -> int: ... @t.type_check_only class Context(Container[t.Any]): From 647c273bff48d439fa405dfab9fa19217760dd5c Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 11:52:00 +0100 Subject: [PATCH 085/102] Added Python 3.12 and 3.13 to the CI --- .github/workflows/main.yml | 4 ++-- setup.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3bdc963..1424f0e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,6 @@ name: CI -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: build: @@ -8,7 +8,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'windows-latest'] - python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} name: OS ${{ matrix.os }}, Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index aa7b612..0d25363 100644 --- a/setup.py +++ b/setup.py @@ -63,6 +63,8 @@ setup( "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Typing :: Typed", ], From bbeb5b185f442e319ae4678a126007b9bd4465db Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 11:58:33 +0100 Subject: [PATCH 086/102] removed Python Versions with EOL (3.7, 3.8) --- .github/workflows/main.yml | 2 +- setup.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1424f0e..46d379a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'windows-latest'] - python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] + python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} name: OS ${{ matrix.os }}, Python ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 0d25363..cf5ca66 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,6 @@ setup( "Topic :: Software Development :: Build Tools", "Topic :: Software Development :: Code Generators", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 457857389adc8d1bb78aff29cad40a3e21c3d975 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 12:22:45 +0100 Subject: [PATCH 087/102] According to PEP688 (https://peps.python.org/pep-0688/#removal-of-the-bytes-special-case) `bytes` has not any special meaning any more. So `bytearray` is not included in `bytes` any more and we have to declare it explicitly. `collections.abc.Buffer` cant be used, because `memoryview` is not supported by `construct` --- construct-stubs/core.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 4a28191..88173c2 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -221,14 +221,14 @@ class Compiled(Construct[t.Any, t.Any]): # =============================================================================== # bytes and bits # =============================================================================== -class Bytes(Construct[bytes, t.Union[bytes, int]]): +class Bytes(Construct[bytes, t.Union[bytes, bytearray, int]]): length: ConstantOrContextLambda[int] def __init__( self, length: ConstantOrContextLambda[int], ) -> None: ... -GreedyBytes: Construct[bytes, bytes] +GreedyBytes: Construct[bytes, t.Union[bytes, bytearray]] def Bitwise( subcon: Construct[SubconParsedType, SubconBuildTypes] From a2a6be536f737ede3164a6bbf00509152d671ef2 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 12:33:51 +0100 Subject: [PATCH 088/102] satisfy pyright 1.1.391 --- construct-stubs/core.pyi | 8 ++++---- construct-stubs/expr.pyi | 4 ++-- construct-stubs/lib/containers.pyi | 2 +- construct_typed/dataclass_struct.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 88173c2..36863e6 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -169,7 +169,7 @@ class Subconstruct( subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @t.overload - def __init__( + def __init__( # type: ignore self, *args: t.Any, **kwargs: t.Any, @@ -1110,9 +1110,9 @@ class Lazy( class LazyContainer(t.Generic[ContainerType], t.Dict[str, ContainerType]): def __getattr__(self, name: str) -> ContainerType: ... def __getitem__(self, index: t.Union[str, int]) -> ContainerType: ... - def keys(self) -> t.Iterator[str]: ... - def values(self) -> t.List[ContainerType]: ... - def items(self) -> t.List[t.Tuple[str, ContainerType]]: ... + def keys(self) -> t.Iterator[str]: ... # type: ignore + def values(self) -> t.List[ContainerType]: ... # type: ignore + def items(self) -> t.List[t.Tuple[str, ContainerType]]: ... # type: ignore class LazyStruct(Construct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] diff --git a/construct-stubs/expr.pyi b/construct-stubs/expr.pyi index 3d1b032..a7c1a1a 100644 --- a/construct-stubs/expr.pyi +++ b/construct-stubs/expr.pyi @@ -469,7 +469,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __eq__(self: ExprMixin[float], other: ConstOrCallable[float]) -> BinExpr[bool]: ... @t.overload - def __eq__(self, other: t.Any) -> BinExpr[t.Any]: ... + def __eq__(self, other: ConstOrCallable[t.Any]) -> BinExpr[t.Any]: ... # type: ignore # __ne__ ########################################################################################################### @t.overload @@ -487,7 +487,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __ne__(self: ExprMixin[float], other: ConstOrCallable[float]) -> BinExpr[bool]: ... @t.overload - def __ne__(self, other: t.Any) -> BinExpr[t.Any]: ... + def __ne__(self, other: t.Any) -> BinExpr[t.Any]: ... # type: ignore # __neg__ ########################################################################################################## @t.overload diff --git a/construct-stubs/lib/containers.pyi b/construct-stubs/lib/containers.pyi index a50033a..37efc75 100644 --- a/construct-stubs/lib/containers.pyi +++ b/construct-stubs/lib/containers.pyi @@ -19,7 +19,7 @@ def recursion_lock( class Container(t.Generic[ContainerType], t.Dict[str, ContainerType]): def __getattr__(self, name: str) -> ContainerType: ... - def update( + def update( # type: ignore self, seqordict: t.Union[t.Dict[str, ContainerType], t.Tuple[str, ContainerType]], ) -> None: ... diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index f276c99..e6ca2e8 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -152,13 +152,13 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): Image(width=1, height=2, pixels=b'12') """ - subcon: "cs.Struct" + subcon: "cs.Struct" # type: ignore def __init__( self, dc_type: t.Type[DataclassType], reverse: bool = False, ) -> None: - if not issubclass(dc_type, DataclassMixin): + if not issubclass(dc_type, DataclassMixin): # type: ignore raise TypeError(f"'{repr(dc_type)}' has to be a '{repr(DataclassMixin)}'") if not dataclasses.is_dataclass(dc_type): raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") From f01246ae823fa1a2490e4a992d5d2bb9a1ba3c90 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 12:43:04 +0100 Subject: [PATCH 089/102] Since Python 3.13 the compiler now strips common leading whitespace from every line in a docstring. So this have to be fixed in the pytests. --- tests/test_typed.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/test_typed.py b/tests/test_typed.py index b85058a..df8cc84 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -2,9 +2,11 @@ # pyright: strict import dataclasses import enum +import textwrap import typing as t import construct as cs + import construct_typed as cst from construct_typed import DataclassBitStruct, DataclassMixin, DataclassStruct, csfield @@ -72,16 +74,20 @@ def test_dataclass_str_repr() -> None: == "Image: \n signature = b'BMP' (total 3)\n width = 3\n height = 2" ) + def test_dataclass_ifthenelse() -> None: @dataclasses.dataclass class IfThenElseTest(DataclassMixin): test_if: t.Optional[int] = csfield(cs.If(False, cs.Int8ub)) - test_ifthenelse: t.Optional[int] = csfield(cs.IfThenElse(True, cs.Int8ub, cs.Pass)) + test_ifthenelse: t.Optional[int] = csfield( + cs.IfThenElse(True, cs.Int8ub, cs.Pass) + ) a = IfThenElseTest(test_if=None, test_ifthenelse=None) assert a.test_if == None assert a.test_ifthenelse == None + def test_dataclass_struct() -> None: @dataclasses.dataclass class Image(DataclassMixin): @@ -395,9 +401,10 @@ def test_tenum_no_enumbase() -> None: def test_tenum_asdict() -> None: # see: https://github.com/timrid/construct-typing/issues/21 - import construct_typed as cst import dataclasses + import construct_typed as cst + class TestEnum(cst.EnumBase): one = 1 two = 2 @@ -436,9 +443,9 @@ def test_tenum_docstring() -> None: Value_NoDoc = cst.EnumValue(2) Value_NoDoc2 = 3 - assert ( - TestEnum.__doc__ - == """ + assert TestEnum.__doc__ is not None + assert textwrap.dedent(TestEnum.__doc__) == textwrap.dedent( + """ This is an test enum. """ ) @@ -508,9 +515,10 @@ def test_tenum_flags() -> None: def test_tenum_flags_asdict() -> None: - import construct_typed as cst import dataclasses + import construct_typed as cst + class TestEnum(cst.FlagsEnumBase): one = 1 two = 2 @@ -549,9 +557,9 @@ def test_tenum_flags_docstring() -> None: Value_NoDoc = cst.EnumValue(2) Value_NoDoc2 = 4 - assert ( - TestEnum.__doc__ - == """ + assert TestEnum.__doc__ is not None + assert textwrap.dedent(TestEnum.__doc__) == textwrap.dedent( + """ This is an test flags enum. """ ) From a222fe769537828b5b2a37f0c7ba5098c49cfb2b Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 13:01:00 +0100 Subject: [PATCH 090/102] print pyright version in CI --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 46d379a..daadc0d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,6 +35,7 @@ jobs: - name: Install pyright run: | npm install -g pyright + pyright --version # Install this package - name: Install this package From 04e1bc0c612428d324c6897fcf46d47b9797ffd8 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 15:35:54 +0100 Subject: [PATCH 091/102] updated to construct==2.10.70 --- construct-stubs/core.pyi | 81 +++++++++++++++++----- requirements.txt | 5 +- setup.py | 2 +- tests/test_core.py | 142 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 205 insertions(+), 25 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 78609a4..ef6e20e 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -17,6 +17,9 @@ from construct.lib import ( ListType, RebufferedBytesIO, ) +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.aead import AESCCM, AESGCM, ChaCha20Poly1305 +from cryptography.hazmat.primitives.ciphers.modes import Mode from typing_extensions import Buffer # unfortunately, there are a few duplications with "typing", e.g. Union and Optional, which is why the t. prefix must be used everywhere @@ -67,6 +70,7 @@ class RawCopyError(ConstructError): ... class RotationError(ConstructError): ... class ChecksumError(ConstructError): ... class CancelParsing(ConstructError): ... +class CipherError(ConstructError): ... # =============================================================================== # used internally @@ -86,6 +90,17 @@ def stream_size(stream: StreamType) -> int: ... def stream_iseof(stream: StreamType) -> bool: ... def evaluate(param: ConstantOrContextLambda2[T], context: Context) -> T: ... +class BytesIOWithOffsets(io.BytesIO): + @staticmethod + def from_reading( + stream: StreamType, length: int, path: PathType + ) -> BytesIOWithOffsets: ... + def __init__( + self, contents: bytes, parent_stream: StreamType, offset: int + ) -> None: ... + def tell(self) -> int: ... + def seek(self, offset: int, whence: int = ...) -> int: ... + # =============================================================================== # abstract constructs # =============================================================================== @@ -135,12 +150,19 @@ class Construct(t.Generic[ParsedType, BuildTypes]): ) -> Renamed[ParsedType, BuildTypes]: ... def __add__(self, other: Construct[t.Any, t.Any]) -> Struct: ... def __rshift__(self, other: Construct[t.Any, t.Any]) -> Sequence: ... - def __getitem__( - self, count: t.Union[int, t.Callable[[Context], int]] - ) -> Array[ParsedType, BuildTypes,]: ... - def _parse(self, stream: StreamType, context: Context, path: PathType) -> ParsedType: ... - def _parsereport(self, stream: StreamType, context: Context, path: PathType) -> ParsedType: ... - def _build(self, obj: BuildTypes, stream: StreamType, context: Context, path: PathType) -> int: ... + def __getitem__(self, count: t.Union[int, t.Callable[[Context], int]]) -> Array[ + ParsedType, + BuildTypes, + ]: ... + def _parse( + self, stream: StreamType, context: Context, path: PathType + ) -> ParsedType: ... + def _parsereport( + self, stream: StreamType, context: Context, path: PathType + ) -> ParsedType: ... + def _build( + self, obj: BuildTypes, stream: StreamType, context: Context, path: PathType + ) -> int: ... def _sizeof(self, context: Context, path: PathType) -> int: ... @t.type_check_only @@ -234,15 +256,11 @@ class Bytes(Construct[bytes, t.Union[bytes, bytearray, int]]): GreedyBytes: Construct[bytes, t.Union[bytes, bytearray]] -def Bitwise( - subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> t.Union[ +def Bitwise(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[ Transformed[SubconParsedType, SubconBuildTypes], Restreamed[SubconParsedType, SubconBuildTypes], ]: ... -def Bytewise( - subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> t.Union[ +def Bytewise(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[ Transformed[SubconParsedType, SubconBuildTypes], Restreamed[SubconParsedType, SubconBuildTypes], ]: ... @@ -880,6 +898,16 @@ class Peek( subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... +class OffsettedEnd( + Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] +): + endoffset: ConstantOrContextLambda[int] + def __init__( + self, + endoffset: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... + class Seek(Construct[int, None]): at: ConstantOrContextLambda[int] if sys.version_info >= (3, 8): @@ -924,9 +952,7 @@ class RawCopy( def ByteSwapped( subcon: Construct[SubconParsedType, SubconBuildTypes] ) -> Transformed[SubconParsedType, SubconBuildTypes]: ... -def BitsSwapped( - subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> t.Union[ +def BitsSwapped(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[ Transformed[SubconParsedType, SubconBuildTypes], Restreamed[SubconParsedType, SubconBuildTypes], ]: ... @@ -946,7 +972,10 @@ class Prefixed( def PrefixedArray( countfield: Construct[int, int], subcon: Construct[SubconParsedType, SubconBuildTypes], -) -> Array[SubconParsedType, SubconBuildTypes,]: ... +) -> Array[ + SubconParsedType, + SubconBuildTypes, +]: ... class FixedSized( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] @@ -1095,6 +1124,26 @@ class Rebuffered( tailcutoff: t.Optional[int] = ..., ) -> None: ... +class EncryptedSym(Tunnel[SubconParsedType, SubconBuildTypes]): + cipher: ConstantOrContextLambda2[Cipher[Mode]] + def __init__( + self, + subcon: Construct[SubconParsedType, SubconBuildTypes], + cipher: ConstantOrContextLambda2[Cipher[Mode]], + ) -> None: ... + +class EncryptedSymAead(Tunnel[SubconParsedType, SubconBuildTypes]): + cipher: ConstantOrContextLambda2[t.Union[AESGCM, AESCCM, ChaCha20Poly1305]] + nonce: ConstantOrContextLambda2[bytes] + associated_data: ConstantOrContextLambda2[bytes] + def __init__( + self, + subcon: Construct[SubconParsedType, SubconBuildTypes], + cipher: ConstantOrContextLambda2[t.Union[AESGCM, AESCCM, ChaCha20Poly1305]], + nonce: ConstantOrContextLambda2[bytes], + associated_data: ConstantOrContextLambda2[bytes] = ..., + ) -> None: ... + # =============================================================================== # lazy equivalents # =============================================================================== diff --git a/requirements.txt b/requirements.txt index d1c8be1..a50644c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -construct==2.10.68 +construct==2.10.70 pytest>=6.2.0 numpy arrow @@ -7,4 +7,5 @@ cloudpickle lz4 black isort -mypy \ No newline at end of file +mypy +cryptography diff --git a/setup.py b/setup.py index cf5ca66..00a4b6d 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ setup( author="Tim Riddermann", python_requires=">=3.7", install_requires=[ - "construct==2.10.68", + "construct==2.10.70", "typing_extensions>=4.6.0" ], keywords=[ diff --git a/tests/test_core.py b/tests/test_core.py index b31cd6b..e697f36 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -150,18 +150,30 @@ def test_formatfield_bool_issue_901() -> None: assert d.build(False) == b"\x00" assert d.sizeof() == 1 -def test_bytesinteger() -> None: +def test_bytesinteger(): + d = BytesInteger(0) + assert raises(d.parse, b"") == IntegerError + assert raises(d.build, 0) == IntegerError d = BytesInteger(4, signed=True, swapped=False) common(d, b"\x01\x02\x03\x04", 0x01020304, 4) common(d, b"\xff\xff\xff\xff", -1, 4) d = BytesInteger(4, signed=False, swapped=this.swapped) common(d, b"\x01\x02\x03\x04", 0x01020304, 4, swapped=False) common(d, b"\x04\x03\x02\x01", 0x01020304, 4, swapped=True) + assert raises(BytesInteger(-1).parse, b"") == IntegerError + assert raises(BytesInteger(-1).build, 0) == IntegerError + assert raises(BytesInteger(8).build, None) == IntegerError + assert raises(BytesInteger(8, signed=False).build, -1) == IntegerError + assert raises(BytesInteger(8, True).build, -2**64) == IntegerError + assert raises(BytesInteger(8, True).build, 2**64) == IntegerError + assert raises(BytesInteger(8, False).build, -2**64) == IntegerError + assert raises(BytesInteger(8, False).build, 2**64) == IntegerError assert raises(BytesInteger(this.missing).sizeof) == SizeofError - assert raises(BytesInteger(4, signed=False).build, -1) == IntegerError - common(BytesInteger(0), b"", 0, 0) -def test_bitsinteger() -> None: +def test_bitsinteger(): + d = BitsInteger(0) + assert raises(d.parse, b"") == IntegerError + assert raises(d.build, 0) == IntegerError d = BitsInteger(8) common(d, b"\x01\x01\x01\x01\x01\x01\x01\x01", 255, 8) d = BitsInteger(8, signed=True) @@ -171,9 +183,17 @@ def test_bitsinteger() -> None: d = BitsInteger(16, swapped=this.swapped) common(d, b"\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00", 0xff00, 16, swapped=False) common(d, b"\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01", 0xff00, 16, swapped=True) - assert raises(BitsInteger(this.missing).sizeof) == SizeofError + assert raises(BitsInteger(-1).parse, b"") == IntegerError + assert raises(BitsInteger(-1).build, 0) == IntegerError + assert raises(BitsInteger(5, swapped=True).parse, bytes(5)) == IntegerError + assert raises(BitsInteger(5, swapped=True).build, 0) == IntegerError + assert raises(BitsInteger(8).build, None) == IntegerError assert raises(BitsInteger(8, signed=False).build, -1) == IntegerError - common(BitsInteger(0), b"", 0, 0) + assert raises(BitsInteger(8, True).build, -2**64) == IntegerError + assert raises(BitsInteger(8, True).build, 2**64) == IntegerError + assert raises(BitsInteger(8, False).build, -2**64) == IntegerError + assert raises(BitsInteger(8, False).build, 2**64) == IntegerError + assert raises(BitsInteger(this.missing).sizeof) == SizeofError def test_varint() -> None: d = VarInt @@ -926,6 +946,17 @@ def test_peek() -> None: assert d4.build(Container(a=0x01, b=0x0102)) == b"" assert d4.sizeof() == 0 +def test_offsettedend(): + d = Struct( + "header" / Bytes(2), + "data" / OffsettedEnd(-2, GreedyBytes), + "footer" / Bytes(2), + ) + common(d, b"\x01\x02\x03\x04\x05\x06\x07", Container(header=b'\x01\x02', data=b'\x03\x04\x05', footer=b'\x06\x07')) + + d = OffsettedEnd(0, Byte) + assert raises(d.sizeof) == SizeofError + def test_seek() -> None: d = Seek(5) assert d.parse(b"") == 5 @@ -1334,6 +1365,105 @@ def test_compressed_prefixed() -> None: assert st.parse(st.build(Container(one=zeros,two=zeros))) == Container(one=zeros,two=zeros) assert raises(d.sizeof) == SizeofError +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsym(): + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + iv = b"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + nonce = iv + + # AES 128/256 bit - ECB + d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.ECB())) + common(d, b"\xf4\x0f\x54\xb7\x6a\x7a\xf1\xdb\x92\x73\x14\xde\x2f\xa0\x3e\x2d", b'Secret Message..', key=key128, iv=iv) + common(d, b"\x82\x6b\x01\x82\x90\x02\xa1\x9e\x35\x0a\xe2\xc3\xee\x1a\x42\xf5", b'Secret Message..', key=key256, iv=iv) + + # AES 128/256 bit - CBC + d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.CBC(ctx.iv))) + common(d, b"\xba\x79\xc2\x62\x22\x08\x29\xb9\xfb\xd3\x90\xc4\x04\xb7\x55\x87", b'Secret Message..', key=key128, iv=iv) + common(d, b"\x60\xc2\x45\x0d\x7e\x41\xd4\xf8\x85\xd4\x8a\x64\xd1\x45\x49\xe3", b'Secret Message..', key=key256, iv=iv) + + # AES 128/256 bit - CTR + d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.CTR(ctx.nonce))) + common(d, b"\x80\x78\xb6\x0c\x07\xf5\x0c\x90\xce\xa2\xbf\xcb\x5b\x22\xb9\xb5", b'Secret Message..', key=key128, nonce=nonce) + common(d, b"\x6a\xae\x7b\x86\x1a\xa6\xe0\x6a\x49\x02\x02\x1b\xf2\x3c\xd8\x0d", b'Secret Message..', key=key256, nonce=nonce) + + assert raises(EncryptedSym(GreedyBytes, "AES").build, b"") == CipherError # type: ignore + assert raises(EncryptedSym(GreedyBytes, "AES").parse, b"") == CipherError # type: ignore + +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsym_cbc_example(): + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + d = Struct( + "iv" / Default(Bytes(16), os.urandom(16)), + "enc_data" / EncryptedSym( + Aligned(16, + Struct( + "width" / Int16ul, + "height" / Int16ul + ) + ), + lambda ctx: Cipher(algorithms.AES(ctx._.key), modes.CBC(ctx.iv)) + ) + ) + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + byts = d.build({"enc_data": {"width": 5, "height": 4}}, key=key128) + obj = d.parse(byts, key=key128) + assert obj.enc_data == Container(width=5, height=4) + +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsymaead(): + from cryptography.hazmat.primitives.ciphers import aead + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + nonce = b"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + + # AES 128/256 bit - GCM + d = Struct( + "associated_data" / Bytes(21), + "data" / EncryptedSymAead( + GreedyBytes, + lambda ctx: aead.AESGCM(ctx._.key), + this._.nonce, + this.associated_data + ) + ) + common( + d, + b"This is authenticated\xb6\xd3\x64\x0c\x7a\x31\xaa\x16\xa3\x58\xec\x17\x39\x99\x2e\xf8\x4e\x41\x17\x76\x3f\xd1\x06\x47\x04\x9f\x42\x1c\xf4\xa9\xfd\x99\x9c\xe9", + Container(associated_data=b"This is authenticated", data=b"The secret message"), + key=key128, + nonce=nonce + ) + common( + d, + b"This is authenticated\xde\xb4\x41\x79\xc8\x7f\xea\x8d\x0e\x41\xf6\x44\x2f\x93\x21\xe6\x37\xd1\xd3\x29\xa4\x97\xc3\xb5\xf4\x81\x72\xa1\x7f\x3b\x9b\x53\x24\xe4", + Container(associated_data=b"This is authenticated", data=b"The secret message"), + key=key256, + nonce=nonce + ) + assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).build, b"") == CipherError # type: ignore + assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).parse, b"") == CipherError # type: ignore + +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsymaead_gcm_example(): + from cryptography.hazmat.primitives.ciphers import aead + d = Struct( + "nonce" / Default(Bytes(16), os.urandom(16)), + "associated_data" / Bytes(21), + "enc_data" / EncryptedSymAead( + GreedyBytes, + lambda ctx: aead.AESGCM(ctx._.key), + this.nonce, + this.associated_data + ) + ) + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + byts = d.build({"associated_data": b"This is authenticated", "enc_data": b"The secret message"}, key=key128) + obj = d.parse(byts, key=key128) + assert obj.enc_data == b"The secret message" + assert obj.associated_data == b"This is authenticated" + def test_rebuffered() -> None: data = b"0" * 1000 assert Rebuffered(Array(1000,Byte)).parse_stream(io.BytesIO(data)) == [48]*1000 From a9f4b448010848136c7ee42541114932da21349d Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 12 Jan 2025 15:39:57 +0100 Subject: [PATCH 092/102] fixed mypy errors --- tests/test_core.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index e697f36..602a899 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -150,7 +150,7 @@ def test_formatfield_bool_issue_901() -> None: assert d.build(False) == b"\x00" assert d.sizeof() == 1 -def test_bytesinteger(): +def test_bytesinteger() -> None: d = BytesInteger(0) assert raises(d.parse, b"") == IntegerError assert raises(d.build, 0) == IntegerError @@ -170,7 +170,7 @@ def test_bytesinteger(): assert raises(BytesInteger(8, False).build, 2**64) == IntegerError assert raises(BytesInteger(this.missing).sizeof) == SizeofError -def test_bitsinteger(): +def test_bitsinteger() -> None: d = BitsInteger(0) assert raises(d.parse, b"") == IntegerError assert raises(d.build, 0) == IntegerError @@ -946,16 +946,16 @@ def test_peek() -> None: assert d4.build(Container(a=0x01, b=0x0102)) == b"" assert d4.sizeof() == 0 -def test_offsettedend(): - d = Struct( +def test_offsettedend() -> None: + d1 = Struct( "header" / Bytes(2), "data" / OffsettedEnd(-2, GreedyBytes), "footer" / Bytes(2), ) - common(d, b"\x01\x02\x03\x04\x05\x06\x07", Container(header=b'\x01\x02', data=b'\x03\x04\x05', footer=b'\x06\x07')) + common(d1, b"\x01\x02\x03\x04\x05\x06\x07", Container(header=b'\x01\x02', data=b'\x03\x04\x05', footer=b'\x06\x07')) - d = OffsettedEnd(0, Byte) - assert raises(d.sizeof) == SizeofError + d2 = OffsettedEnd(0, Byte) + assert raises(d2.sizeof) == SizeofError def test_seek() -> None: d = Seek(5) @@ -1366,7 +1366,7 @@ def test_compressed_prefixed() -> None: assert raises(d.sizeof) == SizeofError @pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") -def test_encryptedsym(): +def test_encryptedsym() -> None: from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" @@ -1392,7 +1392,7 @@ def test_encryptedsym(): assert raises(EncryptedSym(GreedyBytes, "AES").parse, b"") == CipherError # type: ignore @pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") -def test_encryptedsym_cbc_example(): +def test_encryptedsym_cbc_example() -> None: from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes d = Struct( "iv" / Default(Bytes(16), os.urandom(16)), @@ -1412,7 +1412,7 @@ def test_encryptedsym_cbc_example(): assert obj.enc_data == Container(width=5, height=4) @pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") -def test_encryptedsymaead(): +def test_encryptedsymaead() -> None: from cryptography.hazmat.primitives.ciphers import aead key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" @@ -1446,7 +1446,7 @@ def test_encryptedsymaead(): assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).parse, b"") == CipherError # type: ignore @pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") -def test_encryptedsymaead_gcm_example(): +def test_encryptedsymaead_gcm_example() -> None: from cryptography.hazmat.primitives.ciphers import aead d = Struct( "nonce" / Default(Bytes(16), os.urandom(16)), From c5ffc142bef32fbd45b867f23e57977c6f2dfc9c Mon Sep 17 00:00:00 2001 From: Olivier Morelle Date: Sun, 26 Oct 2025 20:40:54 +0100 Subject: [PATCH 093/102] fix(core.pyi): replaces typing.TypeAlias by typing_extensions.TypeAlias for compatibility with python3.9 --- construct-stubs/core.pyi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index ef6e20e..456ab4a 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -20,7 +20,7 @@ from construct.lib import ( from cryptography.hazmat.primitives.ciphers import Cipher from cryptography.hazmat.primitives.ciphers.aead import AESCCM, AESGCM, ChaCha20Poly1305 from cryptography.hazmat.primitives.ciphers.modes import Mode -from typing_extensions import Buffer +from typing_extensions import Buffer, TypeAlias # unfortunately, there are a few duplications with "typing", e.g. Union and Optional, which is why the t. prefix must be used everywhere @@ -29,7 +29,7 @@ from typing_extensions import Buffer # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python -ReadableBuffer: t.TypeAlias = Buffer +ReadableBuffer: TypeAlias = Buffer StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str From a3287676891c02838ff845336e735e54ece262fa Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Mon, 27 Oct 2025 19:33:50 +0100 Subject: [PATCH 094/102] replaced setup.py with pyproject.toml --- .github/workflows/python-publish.yml | 4 +- mypy.ini | 3 -- pyproject.toml | 76 ++++++++++++++++++++++++++++ requirements.txt | 3 ++ setup.py | 69 ------------------------- 5 files changed, 81 insertions(+), 74 deletions(-) delete mode 100644 mypy.ini create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 4e1ef42..05cfb95 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -21,11 +21,11 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install setuptools wheel build twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist bdist_wheel + python -m build --wheel --sdist twine upload dist/* diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 3412486..0000000 --- a/mypy.ini +++ /dev/null @@ -1,3 +0,0 @@ -[mypy] -strict = True -warn_unused_ignores = False \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8a2689b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,76 @@ + +[build-system] +requires = ["setuptools >= 75.8.0"] +build-backend = "setuptools.build_meta" + +[project] +name="construct-typing" +dynamic = ["version"] +license = { file = "LICENSE" } +description="Extension for the python package 'construct' that adds typing features" +readme = "README.md" +authors=[{ name = "Tim Riddermann" }] +requires-python = ">=3.9" +dependencies = [ + "construct==2.10.70", + "typing_extensions>=4.6.0" +] +keywords = [ + "construct", + "kaitai", + "declarative", + "data structure", + "struct", + "binary", + "symmetric", + "parser", + "builder", + "parsing", + "building", + "pack", + "unpack", + "packer", + "unpacker", + "bitstring", + "bytestring", + "annotation", + "type hint", + "typing", + "typed", + "bitstruct", + "PEP 561", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Code Generators", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Typing :: Typed", +] + +[project.urls] +"Homepage" = "https://github.com/timrid/construct-typing" +"Bug Reports" = "https://github.com/timrid/construct-typing/issues" + +[tool.setuptools] +packages=[ + "construct-stubs", + "construct-stubs.lib", + "construct_typed" +] + +[tool.setuptools.dynamic] +version = {attr = "construct_typed.version.version_string"} + +[tool.mypy] +strict = true +warn_unused_ignores = false \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a50644c..2514c06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,6 @@ black isort mypy cryptography +build +setuptools +wheel diff --git a/setup.py b/setup.py deleted file mode 100644 index 00a4b6d..0000000 --- a/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - -version_string = "?.?.?" -exec(open("./construct_typed/version.py").read()) - -setup( - name="construct-typing", - version=version_string, - packages=["construct-stubs", "construct_typed"], - package_data={ - "construct-stubs": ["*.pyi", "lib/*.pyi"], - "construct_typed": ["py.typed"], - }, - license="MIT", - license_files=("LICENSE",), - description="Extension for the python package 'construct' that adds typing features", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - platforms=["POSIX", "Windows"], - url="https://github.com/timrid/construct-typing", - author="Tim Riddermann", - python_requires=">=3.7", - install_requires=[ - "construct==2.10.70", - "typing_extensions>=4.6.0" - ], - keywords=[ - "construct", - "kaitai", - "declarative", - "data structure", - "struct", - "binary", - "symmetric", - "parser", - "builder", - "parsing", - "building", - "pack", - "unpack", - "packer", - "unpacker", - "bitstring", - "bytestring", - "annotation", - "type hint", - "typing", - "typed", - "bitstruct", - "PEP 561", - ], - classifiers=[ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: MIT License", - "Intended Audience :: Developers", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development :: Build Tools", - "Topic :: Software Development :: Code Generators", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: Implementation :: CPython", - "Typing :: Typed", - ], -) From 071038405cc1d14420513f1393a5c2a33693a0e2 Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Mon, 27 Oct 2025 19:52:19 +0100 Subject: [PATCH 095/102] added trusted publishing infos --- .github/workflows/main.yml | 27 +++++++++++++++++++ .github/workflows/python-publish.yml | 39 ++++++++++++++-------------- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index daadc0d..44dc00c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,3 +62,30 @@ jobs: - name: Run pyright run: | pyright + + create_wheel_and_sdist: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.13' + architecture: x64 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install wheel build + + - name: Build wheel and sdist + run: | + python -m build + + - name: Upload wheel and sdist as artifact + uses: actions/upload-artifact@v4 + with: + name: Package-Distributions-construct-typing + path: dist/ \ No newline at end of file diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 05cfb95..6cdb1dd 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,6 +1,3 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - name: Upload Python Package on: @@ -8,24 +5,28 @@ on: types: [created] jobs: - deploy: + create_wheel_and_sdist: + name: create_wheel_and_sdist + uses: ./.github/workflows/main.yml + with: + attest-package: "true" + deploy: + depends-on: create_wheel_and_sdist runs-on: ubuntu-latest + + environment: pypi + permissions: + id-token: write. # IMPORTANT: this permission is mandatory for Trusted Publishing steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v4 with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel build twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python -m build --wheel --sdist - twine upload dist/* + name: Package-Distributions-construct-typing + path: ./dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From fddd438ac8f6fcc724f1c06d2f007ee5a4a20c53 Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:05:12 +0100 Subject: [PATCH 096/102] fix publish workflow --- .github/workflows/main.yml | 6 +++++- .github/workflows/python-publish.yml | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 44dc00c..e09f947 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,10 @@ name: CI -on: [push, pull_request, workflow_dispatch] +on: + push: + pull_request: + workflow_dispatch: + workflow_call: jobs: build: diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 6cdb1dd..ea14263 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -8,16 +8,14 @@ jobs: create_wheel_and_sdist: name: create_wheel_and_sdist uses: ./.github/workflows/main.yml - with: - attest-package: "true" deploy: - depends-on: create_wheel_and_sdist + needs: [ create_wheel_and_sdist ] runs-on: ubuntu-latest environment: pypi permissions: - id-token: write. # IMPORTANT: this permission is mandatory for Trusted Publishing + id-token: write # IMPORTANT: this permission is mandatory for Trusted Publishing steps: - uses: actions/checkout@v3 From f3b7bc342ee6ecc1aee4d072972cdfd626666a81 Mon Sep 17 00:00:00 2001 From: timrid <6593626+timrid@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:18:53 +0100 Subject: [PATCH 097/102] incremented version to 0.7.0 --- construct_typed/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/construct_typed/version.py b/construct_typed/version.py index b1bde74..04fbf4c 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 6, 2) -version_string = "0.6.2" +version = (0, 7, 0) +version_string = "0.7.0" From 0c93e4d551079227d560dc71515998d8653711c0 Mon Sep 17 00:00:00 2001 From: wrapper Date: Tue, 7 Apr 2026 18:48:27 +0700 Subject: [PATCH 098/102] mod --- .gitignore | 3 ++ README.md | 8 ++++ construct-stubs/core.pyi | 27 +++++++++----- construct_typed/__init__.py | 12 +++++- construct_typed/dataclass_struct.py | 57 +++++++++++++++++++---------- construct_typed/generic_wrapper.py | 11 +++++- construct_typed/tenum.py | 45 +++++++++++------------ construct_typed/version.py | 2 +- 8 files changed, 110 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index b3d4398..1d9e0fe 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ dmypy.json example_737 example_888 example_ksy.ksy + +# Test stuff +devtest/ \ No newline at end of file diff --git a/README.md b/README.md index b11989c..d95463b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,11 @@ +## Modified version of "construct-typing" module used in my projects. +This modification features: +- **[EnhancedDataclassMixin](https://github.com/waszil/construct-typing/commit/479b51344bfd95149596a75ee574ac2e63c032df)** +- **ConstantOrContextLambda2 type** +- **Typing for Subconstruct** +- **Type hint for Computed** + +The original README.md file was described down below: # construct-typing [![PyPI](https://img.shields.io/pypi/v/construct-typing)](https://pypi.org/project/construct-typing/) ![PyPI - Implementation](https://img.shields.io/pypi/implementation/construct-typing) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 456ab4a..59f482e 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -800,6 +800,8 @@ def If( ) -> IfThenElse[t.Optional[ThenParsedType], t.Optional[ThenBuildTypes]]: ... SwitchType = t.TypeVar("SwitchType") +SwitchParsedType = t.TypeVar("SwitchParsedType") +SwitchBuildTypes = t.TypeVar("SwitchBuildTypes") class Switch(Construct[ParsedType, BuildTypes]): keyfunc: ConstantOrContextLambda[t.Any] @@ -807,18 +809,25 @@ class Switch(Construct[ParsedType, BuildTypes]): default: Construct[t.Any, t.Any] @t.overload def __new__( - cls: "type[Switch[int, t.Optional[int]]]", + cls: "type[Switch[SwitchParsedType | None, SwitchBuildTypes | None]]", keyfunc: ConstantOrContextLambda[SwitchType], - cases: t.Dict[SwitchType, Construct[int, int]], - default: t.Optional[Construct[int, int]] = ..., - ) -> Switch[int, t.Optional[int]]: ... + cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], + default: None = ..., + ) -> Switch[SwitchParsedType | None, SwitchBuildTypes | None]: ... @t.overload def __new__( - cls: "type[Switch[t.Any, t.Any]]", - keyfunc: ConstantOrContextLambda[t.Any], - cases: t.Dict[t.Any, Construct[t.Any, t.Any]], - default: t.Optional[Construct[t.Any, t.Any]] = ..., - ) -> Switch[t.Any, t.Any]: ... + cls: "type[Switch[SwitchParsedType, SwitchBuildTypes]]", + keyfunc: ConstantOrContextLambda[SwitchType], + cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], + default: Construct[SwitchParsedType, SwitchBuildTypes], + ) -> Switch[SwitchParsedType, SwitchBuildTypes]: ... + # @t.overload + # def __new__( + # cls: "type[Switch[t.Any, t.Any]]", + # keyfunc: ConstantOrContextLambda[t.Any], + # cases: t.Dict[t.Any, Construct[t.Any, t.Any]], + # default: t.Optional[Construct[t.Any, t.Any]] = ..., + # ) -> Switch[t.Any, t.Any]: ... class StopIf(Construct[None, None]): condfunc: ConstantOrContextLambda[bool] diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 9ea0ccf..f594f2b 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -9,15 +9,19 @@ from .dataclass_struct import ( TStructField, csfield, sfield, + EnhancedDataclassMixin ) from .generic_wrapper import ( Adapter, ConstantOrContextLambda, + ConstantOrContextLambda2, Construct, Context, ListContainer, PathType, - Array + Array, + Subconstruct, + Computed, ) from .tenum import EnumBase, EnumValue, FlagsEnumBase, TEnum, TFlagsEnum @@ -32,6 +36,7 @@ __all__ = [ "TStructField", "csfield", "sfield", + "EnhancedDataclassMixin", "EnumBase", "EnumValue", "FlagsEnumBase", @@ -39,9 +44,12 @@ __all__ = [ "TFlagsEnum", "Adapter", "ConstantOrContextLambda", + "ConstantOrContextLambda2", "Construct", "Context", "ListContainer", "PathType", - "Array" + "Array", + "Subconstruct", + "Computed" ] diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index e6ca2e8..e626085 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # pyright: strict +# pyright: reportIncompatibleVariableOverride=false, reportAny=false import dataclasses import textwrap import typing as t @@ -11,6 +12,7 @@ from construct.lib.containers import ( recursion_lock, ) from construct.lib.py3compat import bytestringtype, reprstring, unicodestringtype +from typing_extensions import override from .generic_wrapper import Adapter, Construct, Context, ParsedType, PathType @@ -27,7 +29,7 @@ class DataclassMixin: methods exists and every name can be used. """ - __dataclass_fields__: "t.ClassVar[t.Dict[str, dataclasses.Field[t.Any]]]" + __dataclass_fields__: "t.ClassVar[dict[str, dataclasses.Field[t.Any]]]" def __getitem__(self, key: str) -> t.Any: return getattr(self, key) @@ -77,8 +79,8 @@ class DataclassMixin: def csfield( subcon: Construct[ParsedType, t.Any], - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + doc: str | None = None, + parsed: t.Callable[[t.Any, Context], None] | None = None, ) -> ParsedType: """ Helper method for "DataclassStruct" and "DataclassBitStruct" to create the dataclass fields. @@ -155,15 +157,11 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): subcon: "cs.Struct" # type: ignore def __init__( self, - dc_type: t.Type[DataclassType], + dc_type: type[DataclassType], reverse: bool = False, ) -> None: - if not issubclass(dc_type, DataclassMixin): # type: ignore - raise TypeError(f"'{repr(dc_type)}' has to be a '{repr(DataclassMixin)}'") - if not dataclasses.is_dataclass(dc_type): - raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") - self.dc_type = dc_type - self.reverse = reverse + self.dc_type: type[DataclassType] = dc_type + self.reverse: bool = reverse # get all fields from the dataclass fields = dataclasses.fields(self.dc_type) @@ -171,7 +169,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): fields = tuple(reversed(fields)) # extract the construct formats from the struct_type - subcon_fields = {} + subcon_fields: dict[str, t.Any] = {} for field in fields: subcon_fields[field.name] = field.metadata["subcon"] @@ -181,6 +179,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): def __getattr__(self, name: str) -> t.Any: return getattr(self.subcon, name) + @override def _decode( self, obj: "cs.Container[t.Any]", context: Context, path: PathType ) -> DataclassType: @@ -205,9 +204,10 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): return dc # type: ignore + @override def _encode( self, obj: DataclassType, context: Context, path: PathType - ) -> t.Dict[str, t.Any]: + ) -> dict[str, t.Any]: if not isinstance(obj, self.dc_type): raise TypeError(f"'{repr(obj)}' has to be of type {repr(self.dc_type)}") @@ -215,20 +215,16 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): fields = dataclasses.fields(self.dc_type) # extract all fields from the container, that are used for create the dataclass object - ret_dict: t.Dict[str, t.Any] = {} + ret_dict: dict[str, t.Any] = {} for field in fields: value = getattr(obj, field.name) ret_dict[field.name] = value return ret_dict - def DataclassBitStruct( - dc_type: t.Type[DataclassType], reverse: bool = False -) -> t.Union[ - "cs.Transformed[DataclassType, DataclassType]", - "cs.Restreamed[DataclassType, DataclassType]", -]: + dc_type: type[DataclassType], reverse: bool = False +) -> "cs.Transformed[DataclassType, DataclassType] | cs.Restreamed[DataclassType, DataclassType]": r""" Makes a DataclassStruct inside a Bitwise. @@ -255,6 +251,29 @@ def DataclassBitStruct( """ return cs.Bitwise(DataclassStruct(dc_type, reverse)) +class EnhancedDataclassMixin(DataclassMixin): + @classmethod + def format(cls): + return DataclassStruct(cls) + + @classmethod + def build(cls, obj: t.Self, **kw: dict[str, t.Any]): + return cls.format().build(obj, **kw) + + @classmethod + def parse(cls, data: bytes | bytearray, **kw: dict[str, t.Any]): + return cls.format().parse(data, **kw) + + @classmethod + def parse_file(cls, file: str, **kw: dict[str, t.Any]): + return cls.format().parse_file(file, **kw) + + @classmethod + def parse_stream(cls, stream: t.IO[bytes], **kw: dict[str, t.Any]): + return cls.format().parse_stream(stream, **kw) + + def build_self(self) -> bytes: + return self.build(self) # support legacy names TStruct = DataclassStruct diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index 742c267..cd4788b 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -12,12 +12,14 @@ if t.TYPE_CHECKING: # while type checking, the original classes are already generics, because they are defined like this in the stubs. from construct import Adapter as Adapter from construct import ConstantOrContextLambda as ConstantOrContextLambda + from construct import ConstantOrContextLambda2 as ConstantOrContextLambda2 from construct import Construct as Construct from construct import Context as Context from construct import ListContainer as ListContainer from construct import PathType as PathType from construct import Array as Array - + from construct import Subconstruct as Subconstruct + from construct import Computed as Computed else: import construct as cs @@ -44,5 +46,12 @@ else: ): pass + class Subconstruct(t.Generic[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], cs.Subconstruct): + pass + + class Computed(t.Generic[ParsedType], cs.Computed): + pass + ConstantOrContextLambda = t.Union[ValueType, t.Callable[[Context], t.Any]] + ConstantOrContextLambda2 = t.Union[ValueType, t.Callable[[Context], ValueType]] PathType = str diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index 4ae6c03..4417c6b 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -1,9 +1,10 @@ +# pyright: reportAny=false import enum import typing as t -from typing_extensions import Self +from typing_extensions import Self, override -from .generic_wrapper import * +from .generic_wrapper import Construct, Adapter, Context, PathType # ## TEnum ############################################################################################################ @@ -12,8 +13,8 @@ class EnumValue: This is a helper class for adding documentation to an enum value. """ - def __init__(self, value: int, doc: t.Optional[str] = None) -> None: - self.value = value + def __init__(self, value: int, doc: str | None = None) -> None: + self.value: int = value self.__doc__ = doc if doc else "" @@ -47,7 +48,7 @@ class EnumBase(enum.IntEnum): 'This is the running state.' """ - def __new__(cls, val: t.Union[EnumValue, int]) -> "Self": + def __new__(cls, val: EnumValue | int) -> "Self": if isinstance(val, EnumValue): obj = int.__new__(cls, val.value) obj._value_ = val.value @@ -62,7 +63,8 @@ class EnumBase(enum.IntEnum): # not found in the enum, a new pseudo member is created. # The idea is taken from: https://stackoverflow.com/a/57179436 @classmethod - def _missing_(cls, value: t.Any) -> t.Optional[enum.Enum]: + @override + def _missing_(cls, value: t.Any) -> enum.Enum | None: if isinstance(value, int): pseudo_member = cls._value2member_map_.get(value, None) if pseudo_member is None: @@ -76,7 +78,8 @@ class EnumBase(enum.IntEnum): return pseudo_member return None # will raise the ValueError in Enum.__new__ - def __reduce_ex__(self, proto: t.Any) -> t.Tuple[t.Any, ...]: + @override + def __reduce_ex__(self, proto: t.Any) -> tuple[t.Any, ...]: """ Pickle enums by value instead of name (restores pre-3.11 behavior). See https://github.com/python/cpython/pull/26658 for why this exists. @@ -91,21 +94,18 @@ class TEnum(Adapter[int, int, EnumType, EnumType]): """ Typed enum. """ - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[EnumType]): - if not issubclass(enum_type, EnumBase): - raise TypeError( - "'{}' has to be a '{}'".format(repr(enum_type), repr(EnumBase)) - ) - + def __init__(self, subcon: Construct[int, int], enum_type: type[EnumType]): # save enum type - self.enum_type = t.cast(t.Type[EnumType], enum_type) # type: ignore + self.enum_type: type[EnumType] = enum_type # init adatper super(TEnum, self).__init__(subcon) # type: ignore + @override def _decode(self, obj: int, context: Context, path: PathType) -> EnumType: return self.enum_type(obj) + @override def _encode( self, obj: EnumType, @@ -152,7 +152,7 @@ class FlagsEnumBase(enum.IntFlag): 'This is option two.' """ - def __new__(cls, val: t.Union[EnumValue, int]) -> "Self": + def __new__(cls, val: EnumValue | int) -> "Self": if isinstance(val, EnumValue): obj = int.__new__(cls, val.value) obj._value_ = val.value @@ -164,6 +164,7 @@ class FlagsEnumBase(enum.IntFlag): return obj @classmethod + @override def _missing_(cls, value: t.Any) -> t.Any: """ Returns member (possibly creating it) if one can be found for value. @@ -172,7 +173,8 @@ class FlagsEnumBase(enum.IntFlag): new_member.__doc__ = "missing value" return new_member - def __reduce_ex__(self, proto: t.Any) -> t.Tuple[t.Any, ...]: + @override + def __reduce_ex__(self, proto: t.Any) -> tuple[t.Any, ...]: """ Pickle enums by value instead of name (restores pre-3.11 behavior). See https://github.com/python/cpython/pull/26658 for why this exists. @@ -187,21 +189,18 @@ class TFlagsEnum(Adapter[int, int, FlagsEnumType, FlagsEnumType]): """ Typed enum. """ - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType]): - if not issubclass(enum_type, FlagsEnumBase): - raise TypeError( - "'{}' has to be a '{}'".format(repr(enum_type), repr(FlagsEnumBase)) - ) - + def __init__(self, subcon: Construct[int, int], enum_type: type[FlagsEnumType]): # save enum type - self.enum_type = t.cast(t.Type[FlagsEnumType], enum_type) # type: ignore + self.enum_type: type[FlagsEnumType] = enum_type # init adatper super(TFlagsEnum, self).__init__(subcon) # type: ignore + @override def _decode(self, obj: int, context: Context, path: PathType) -> FlagsEnumType: return self.enum_type(obj) + @override def _encode( self, obj: FlagsEnumType, diff --git a/construct_typed/version.py b/construct_typed/version.py index 04fbf4c..38a2845 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ version = (0, 7, 0) -version_string = "0.7.0" +version_string = "0.7.0+wrapper" From 2f078b340bbb510d647ce69bec5b5ce13e181df8 Mon Sep 17 00:00:00 2001 From: wrapper Date: Tue, 7 Apr 2026 19:02:24 +0700 Subject: [PATCH 099/102] more switch fixes --- README.md | 1 + construct-stubs/core.pyi | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d95463b..73d2f1d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ This modification features: - **ConstantOrContextLambda2 type** - **Typing for Subconstruct** - **Type hint for Computed** +- **Switch typing fixes** The original README.md file was described down below: # construct-typing diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 59f482e..3792b96 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -802,6 +802,8 @@ def If( SwitchType = t.TypeVar("SwitchType") SwitchParsedType = t.TypeVar("SwitchParsedType") SwitchBuildTypes = t.TypeVar("SwitchBuildTypes") +SwitchDefaultParsedType = t.TypeVar("SwitchDefaultParsedType") +SwitchDefaultBuildTypes = t.TypeVar("SwitchDefaultBuildTypes") class Switch(Construct[ParsedType, BuildTypes]): keyfunc: ConstantOrContextLambda[t.Any] @@ -816,11 +818,11 @@ class Switch(Construct[ParsedType, BuildTypes]): ) -> Switch[SwitchParsedType | None, SwitchBuildTypes | None]: ... @t.overload def __new__( - cls: "type[Switch[SwitchParsedType, SwitchBuildTypes]]", + cls: "type[Switch[SwitchParsedType | SwitchDefaultParsedType, SwitchBuildTypes | SwitchDefaultBuildTypes]]", keyfunc: ConstantOrContextLambda[SwitchType], cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], - default: Construct[SwitchParsedType, SwitchBuildTypes], - ) -> Switch[SwitchParsedType, SwitchBuildTypes]: ... + default: Construct[SwitchDefaultParsedType, SwitchDefaultBuildTypes], + ) -> Switch[SwitchParsedType | SwitchDefaultParsedType, SwitchBuildTypes | SwitchDefaultBuildTypes]: ... # @t.overload # def __new__( # cls: "type[Switch[t.Any, t.Any]]", From c1896ab8dcb96bab8900269a8b706ed0e94591b2 Mon Sep 17 00:00:00 2001 From: wrapper Date: Tue, 7 Apr 2026 19:25:45 +0700 Subject: [PATCH 100/102] Construct Error class does not return --- construct-stubs/core.pyi | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 3792b96..4bee6b5 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -622,7 +622,7 @@ class Check(Construct[None, None]): func: ConstantOrContextLambda[bool], ) -> None: ... -Error: Construct[None, None] +Error: Construct[t.NoReturn, t.NoReturn] class FocusedSeq(Construct[t.Any, t.Any]): subcons: t.List[Construct[t.Any, t.Any]] @@ -817,6 +817,13 @@ class Switch(Construct[ParsedType, BuildTypes]): default: None = ..., ) -> Switch[SwitchParsedType | None, SwitchBuildTypes | None]: ... @t.overload + def __new__( + cls: "type[Switch[SwitchParsedType, SwitchBuildTypes]]", + keyfunc: ConstantOrContextLambda[SwitchType], + cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], + default: Construct[t.NoReturn, t.NoReturn], + ) -> Switch[SwitchParsedType, SwitchBuildTypes]: ... + @t.overload def __new__( cls: "type[Switch[SwitchParsedType | SwitchDefaultParsedType, SwitchBuildTypes | SwitchDefaultBuildTypes]]", keyfunc: ConstantOrContextLambda[SwitchType], From 0a4628935b6d279649114cfc1c632a90a785907a Mon Sep 17 00:00:00 2001 From: wrapper Date: Tue, 7 Apr 2026 20:19:38 +0700 Subject: [PATCH 101/102] add --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73d2f1d..c4bac18 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Modified version of "construct-typing" module used in my projects. This modification features: -- **[EnhancedDataclassMixin](https://github.com/waszil/construct-typing/commit/479b51344bfd95149596a75ee574ac2e63c032df)** +- **[EnhancedDataclassMixin](https://github.com/waszil/construct-typing/commit/479b51344bfd95149596a75ee574ac2e63c032df) with additional features** - **ConstantOrContextLambda2 type** - **Typing for Subconstruct** - **Type hint for Computed** From 486c553d6576ee38200f4b32df387c00e9d05db7 Mon Sep 17 00:00:00 2001 From: wrapper Date: Tue, 7 Apr 2026 23:28:33 +0700 Subject: [PATCH 102/102] fallback switch --- construct-stubs/core.pyi | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 4bee6b5..7ed1af6 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -830,13 +830,13 @@ class Switch(Construct[ParsedType, BuildTypes]): cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], default: Construct[SwitchDefaultParsedType, SwitchDefaultBuildTypes], ) -> Switch[SwitchParsedType | SwitchDefaultParsedType, SwitchBuildTypes | SwitchDefaultBuildTypes]: ... - # @t.overload - # def __new__( - # cls: "type[Switch[t.Any, t.Any]]", - # keyfunc: ConstantOrContextLambda[t.Any], - # cases: t.Dict[t.Any, Construct[t.Any, t.Any]], - # default: t.Optional[Construct[t.Any, t.Any]] = ..., - # ) -> Switch[t.Any, t.Any]: ... + @t.overload + def __new__( + cls: "type[Switch[t.Any, t.Any]]", + keyfunc: ConstantOrContextLambda[SwitchType], + cases: dict[t.Any, Construct[t.Any, t.Any]], + default: Construct[t.Any, t.Any] | None = ..., + ) -> Switch[t.Any, t.Any]: ... class StopIf(Construct[None, None]): condfunc: ConstantOrContextLambda[bool]