From 22a0ed7f24f00bb5aad0a4454ba401a2cd333fd6 Mon Sep 17 00:00:00 2001 From: Tim Rid <6593626+timrid@users.noreply.github.com> Date: Sun, 23 May 2021 15:28:27 +0200 Subject: [PATCH] - updated documentation - removed tunion completely --- .gitignore | 3 ++ README.md | 49 ++++++++++++++++------------- construct_typed/__init__.py | 46 +++++++++++++++------------ construct_typed/dataclass_struct.py | 25 +++++++-------- construct_typed/generic_wrapper.py | 3 -- construct_typed/tunion.py | 48 ---------------------------- 6 files changed, 68 insertions(+), 106 deletions(-) delete mode 100644 construct_typed/tunion.py diff --git a/.gitignore b/.gitignore index 7114a35..b3d4398 100644 --- a/.gitignore +++ b/.gitignore @@ -126,3 +126,6 @@ dmypy.json # Pyre type checker .pyre/ +example_737 +example_888 +example_ksy.ksy diff --git a/README.md b/README.md index 235ea31..a39e5e0 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,14 @@ The **construct-stubs** package is used for creating type hints for the orignial - `Construct.build`: converts an object of one of the types defined by `BuildTypes` to a `bytes` object. - `Construct.parse`: converts a `bytes` object to an object of type `ParsedType`. -For each of the `Construct`s in the stubs it is defined which type it parses to and from which it can be build. For example: +For each `Construct` the stub file defines to which type it parses to and from which it can be build. For example: -| Construct | parses to (ParsedType) | builds from (BuildTypes) | -| -------------------- | ------------------------------ | ------------------------------------ | -| `Int16ub` | `int` | `int` | -| `Bytes` | `bytes` | `bytes`, `bytearray` or `memoryview` | -| `Array(5, Int16ub)` | `ListContainer[int]` | `typing.List[int]` | -| `Struct("i" / Byte)` | `Container[typing.Any]` | `Dict[str, typing.Any]` or `None` | +| Construct | parses to (ParsedType) | builds from (BuildTypes) | +| -------------------- | ------------------------------ | ---------------------------------------- | +| `Int16ub` | `int` | `int` | +| `Bytes` | `bytes` | `bytes`, `bytearray` or `memoryview` | +| `Array(5, Int16ub)` | `ListContainer[int]` | `typing.List[int]` | +| `Struct("i" / Byte)` | `Container[typing.Any]` | `typing.Dict[str, typing.Any]` or `None` | The problem is to describe the more complex constructs like: - `Sequence`, `FocusedSeq` which has heterogenous subcons in comparison to an `Array` with only homogenous subcons. @@ -51,45 +51,50 @@ Note: The stubs are based on *construct* in Version 2.10. ### Typed **!!! EXPERIMENTAL VERSION !!!** -To include autocompletion and further enhance the type hints for these complex constructs the **construct_typed** package is used as an extension to the original *construct* package. It is mainly a bunch of Adapters for the original constructs with the focus on type hints. +To include autocompletion and further enhance the type hints for these complex constructs the **construct_typed** package is used as an extension to the original *construct* package. It is mainly a few Adapters with the focus on type hints. It implements the following new constructs: -- `TStruct`, `TBitStruct`: similar to `construct.Struct` but strictly tied to `TContainerMixin` and `@dataclasses.dataclass` +- `DataclassStruct`, `DataclassBitStruct`: similar to `construct.Struct` but strictly tied to `DataclassMixin` and `@dataclasses.dataclass` - `TEnum`: similar to `construct.Enum` but strictly tied to a `TEnumBase` class - `TFlagsEnum`: similar to `construct.FlagsEnum` but strictly tied to a `TFlagsEnumBase` class -These types are strongly typed, which means that there is no difference between the `ParsedType` and the `BuildTypes`. So to build one of the constructs the correct type is enforced. The disadvantage is that the code will be a little bit longer, because you can not for example use a normal `dict` to build an `TStruct`. But the big advantage is, that if you use the correct container type instead of a `dict`, the static code analyses can do its magic and find potential type errors and missing values. +These types are strongly typed, which means that there is no difference between the `ParsedType` and the `BuildTypes`. So to build one of the constructs the correct type is enforced. The disadvantage is that the code will be a little bit longer, because you can not for example use a normal `dict` to build an `DataclassStruct`. But the big advantage is, that if you use the correct container type instead of a `dict`, the static code analyses can do its magic and find potential type errors and missing values without running the code itself. A short example: ```python import dataclasses -import construct as cs -import construct_typed as cst import typing as t +from construct import Array, Byte, Const, Int8ub, this +from construct_typed import DataclassMixin, DataclassStruct, EnumBase, TEnum, csfield -class Orientation(cst.EnumBase): +class Orientation(EnumBase): HORIZONTAL = 0 VERTICAL = 1 @dataclasses.dataclass -class Image(cst.TContainerMixin): - signature: t.Optional[bytes] = cst.sfield(cs.Const(b"BMP")) - orientation: Orientation = cst.sfield(cst.TEnum(cs.Int8ub, Orientation)) - width: int = cst.sfield(cs.Int8ub) - height: int = cst.sfield(cs.Int8ub) - pixels: t.List[int] = cst.sfield(cs.Array(cs.this.width * cs.this.height, cs.Byte)) +class Image(DataclassMixin): + signature: bytes = csfield(Const(b"BMP")) + orientation: Orientation = csfield(TEnum(Int8ub, Orientation)) + width: int = csfield(Int8ub) + height: int = csfield(Int8ub) + pixels: t.List[int] = csfield(Array(this.width * this.height, Byte)) -format = cst.TStruct(Image) -obj = Image(orientation=Orientation.VERTICAL, width=3, height=2, pixels=[7, 8, 9, 11, 12, 13]) +format = DataclassStruct(Image) +obj = Image( + orientation=Orientation.VERTICAL, + width=3, + height=2, + pixels=[7, 8, 9, 11, 12, 13], +) print(format.build(obj)) print(format.parse(b"BMP\x01\x03\x02\x07\x08\t\x0b\x0c\r")) ``` Output: ``` b'BMP\x01\x03\x02\x07\x08\t\x0b\x0c\r' -Container: +Image: signature = b'BMP' (total 3) orientation = Orientation.VERTICAL width = 3 diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index cf82919..00d5093 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -1,3 +1,15 @@ +from .dataclass_struct import ( + DataclassBitStruct, + DataclassMixin, + DataclassStruct, + TBitStruct, + TContainerBase, + TContainerMixin, + TStruct, + TStructField, + csfield, + sfield, +) from .generic_wrapper import ( Adapter, ConstantOrContextLambda, @@ -7,32 +19,26 @@ from .generic_wrapper import ( PathType, ) from .tenum import EnumBase, FlagsEnumBase, TEnum, TFlagsEnum -from .dataclass_struct import DataclassStruct, csfield, TBitStruct, TStruct, sfield, TStructField, TContainerMixin, TContainerBase, DataclassBitStruct, DataclassMixin -from .tunion import TUnion, ufield, TUnionField __all__ = [ - "DataclassStruct", "DataclassBitStruct", - "csfield", "DataclassMixin", - "sfield", - "TStructField", - "TStruct", + "DataclassStruct", "TBitStruct", - "TEnum", - "ufield", - "TUnionField", - "TUnion", - "EnumBase", - "Construct", - "Adapter", - "ListContainer", "TContainerBase", "TContainerMixin", - "Context", - "ConstantOrContextLambda", - "PathType", + "TStruct", + "TStructField", + "csfield", + "sfield", + "EnumBase", + "FlagsEnumBase", + "TEnum", "TFlagsEnum", - "FlagsEnumBase" + "Adapter", + "ConstantOrContextLambda", + "Construct", + "Context", + "ListContainer", + "PathType", ] - diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index a590a1e..444ce21 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -128,7 +128,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): Internally, all fields are converted to a Struct, which does the actual parsing/building. - Parses to a dataclasses.dataclass instance, and builds from such instance (although it also builds from dicts). Size is the sum of all subcon sizes, unless any subcon raises SizeofError. + Parses to a dataclasses.dataclass instance, and builds from such instance. Size is the sum of all subcon sizes, unless any subcon raises SizeofError. :param dc_type: Type of the dataclass, which also inherits from DataclassMixin :param reverse: Flag if the fields of the dataclass should be reversed @@ -213,20 +213,19 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): def _encode( self, obj: DataclassType, context: Context, path: PathType ) -> t.Dict[str, t.Any]: - if isinstance(obj, self.dc_type): - # get all fields from the dataclass - fields = dataclasses.fields(self.dc_type) + if not isinstance(obj, self.dc_type): + raise TypeError(f"'{repr(obj)}' has to be of type {repr(self.dc_type)}") - # extract all fields from the container, that are used for create the dataclass object - ret_dict: t.Dict[str, t.Any] = {} - for field in fields: - value = getattr(obj, field.name) - ret_dict[field.name] = value + # get all fields from the dataclass + fields = dataclasses.fields(self.dc_type) - return ret_dict - raise TypeError( - "'{}' has to be of type {}".format(repr(obj), repr(self.dc_type)) - ) + # extract all fields from the container, that are used for create the dataclass object + ret_dict: t.Dict[str, t.Any] = {} + for field in fields: + value = getattr(obj, field.name) + ret_dict[field.name] = value + + return ret_dict def DataclassBitStruct( diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index d3dab33..aa4af4c 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -1,6 +1,3 @@ -import dataclasses -import enum -import textwrap import typing as t ParsedType = t.TypeVar("ParsedType", covariant=True) diff --git a/construct_typed/tunion.py b/construct_typed/tunion.py deleted file mode 100644 index 5aa8640..0000000 --- a/construct_typed/tunion.py +++ /dev/null @@ -1,48 +0,0 @@ -import dataclasses -import textwrap -import typing as t - -import construct as cs - -from .generic_wrapper import * - -DataclassType = t.TypeVar("DataclassType") - - -def ufield( - subcon: Construct[ParsedType, t.Any], - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, "cs.Context"], None]] = None, -) -> ParsedType: - """ - Create a dataclass field for a "TUnion" from a subcon. - """ - # Rename subcon, if doc or parsed are available - if (doc is not None) or (parsed is not None): - if doc is not None: - doc = textwrap.dedent(doc) - subcon = cs.Renamed(subcon, newdocs=doc, newparsed=parsed) - - if subcon.flagbuildnone is True: - # some subcons have a predefined default value. all other have "None" - default: t.Any = None - if isinstance(subcon, (cs.Const, cs.Default)): - if callable(subcon.value): - raise ValueError("lamda as default is not supported") - default = subcon.value - - # if subcon builds from "None", set default to "None" - field = dataclasses.field( - default=default, - init=False, - metadata={"subcon": cs.Renamed(subcon, newdocs=doc)}, - ) - else: - field = dataclasses.field(metadata={"subcon": subcon}) - - return field # type: ignore - -TUnionField = ufield # also support legacy name - -class TUnion(Adapter[t.Any, t.Any, DataclassType, DataclassType]): - pass # TODO