diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3bdc963..e09f947 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,10 @@ name: CI -on: [push, pull_request] +on: + push: + pull_request: + workflow_dispatch: + workflow_call: jobs: build: @@ -8,7 +12,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'windows-latest'] - python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ] + python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} name: OS ${{ matrix.os }}, Python ${{ matrix.python-version }} @@ -35,6 +39,7 @@ jobs: - name: Install pyright run: | npm install -g pyright + pyright --version # Install this package - name: Install this package @@ -61,3 +66,30 @@ jobs: - name: Run pyright run: | pyright + + create_wheel_and_sdist: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.13' + architecture: x64 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install wheel build + + - name: Build wheel and sdist + run: | + python -m build + + - name: Upload wheel and sdist as artifact + uses: actions/upload-artifact@v4 + with: + name: Package-Distributions-construct-typing + path: dist/ \ No newline at end of file diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 4e1ef42..ea14263 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,6 +1,3 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - name: Upload Python Package on: @@ -8,24 +5,26 @@ on: types: [created] jobs: - deploy: + create_wheel_and_sdist: + name: create_wheel_and_sdist + uses: ./.github/workflows/main.yml + deploy: + needs: [ create_wheel_and_sdist ] runs-on: ubuntu-latest + + environment: pypi + permissions: + id-token: write # IMPORTANT: this permission is mandatory for Trusted Publishing steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + + - name: Download artifacts + uses: actions/download-artifact@v4 with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + name: Package-Distributions-construct-typing + path: ./dist + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index b3d4398..1d9e0fe 100644 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ dmypy.json example_737 example_888 example_ksy.ksy + +# Test stuff +devtest/ \ No newline at end of file diff --git a/README.md b/README.md index b11989c..c4bac18 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,12 @@ +## Modified version of "construct-typing" module used in my projects. +This modification features: +- **[EnhancedDataclassMixin](https://github.com/waszil/construct-typing/commit/479b51344bfd95149596a75ee574ac2e63c032df) with additional features** +- **ConstantOrContextLambda2 type** +- **Typing for Subconstruct** +- **Type hint for Computed** +- **Switch typing fixes** + +The original README.md file was described down below: # construct-typing [![PyPI](https://img.shields.io/pypi/v/construct-typing)](https://pypi.org/project/construct-typing/) ![PyPI - Implementation](https://img.shields.io/pypi/implementation/construct-typing) diff --git a/construct-stubs/core.pyi b/construct-stubs/core.pyi index 4a28191..7ed1af6 100644 --- a/construct-stubs/core.pyi +++ b/construct-stubs/core.pyi @@ -17,7 +17,10 @@ from construct.lib import ( ListType, RebufferedBytesIO, ) -from typing_extensions import Buffer +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.aead import AESCCM, AESGCM, ChaCha20Poly1305 +from cryptography.hazmat.primitives.ciphers.modes import Mode +from typing_extensions import Buffer, TypeAlias # unfortunately, there are a few duplications with "typing", e.g. Union and Optional, which is why the t. prefix must be used everywhere @@ -26,7 +29,7 @@ from typing_extensions import Buffer # - Higher Kinded Types: https://github.com/python/typing/issues/548 # - Higher Kinded Types: https://sobolevn.me/2020/10/higher-kinded-types-in-python -ReadableBuffer: t.TypeAlias = Buffer +ReadableBuffer: TypeAlias = Buffer StreamType = t.IO[bytes] FilenameType = t.Union[str, bytes, os.PathLike[str], os.PathLike[bytes]] PathType = str @@ -67,6 +70,7 @@ class RawCopyError(ConstructError): ... class RotationError(ConstructError): ... class ChecksumError(ConstructError): ... class CancelParsing(ConstructError): ... +class CipherError(ConstructError): ... # =============================================================================== # used internally @@ -86,6 +90,17 @@ def stream_size(stream: StreamType) -> int: ... def stream_iseof(stream: StreamType) -> bool: ... def evaluate(param: ConstantOrContextLambda2[T], context: Context) -> T: ... +class BytesIOWithOffsets(io.BytesIO): + @staticmethod + def from_reading( + stream: StreamType, length: int, path: PathType + ) -> BytesIOWithOffsets: ... + def __init__( + self, contents: bytes, parent_stream: StreamType, offset: int + ) -> None: ... + def tell(self) -> int: ... + def seek(self, offset: int, whence: int = ...) -> int: ... + # =============================================================================== # abstract constructs # =============================================================================== @@ -135,9 +150,20 @@ class Construct(t.Generic[ParsedType, BuildTypes]): ) -> Renamed[ParsedType, BuildTypes]: ... def __add__(self, other: Construct[t.Any, t.Any]) -> Struct: ... def __rshift__(self, other: Construct[t.Any, t.Any]) -> Sequence: ... - def __getitem__( - self, count: t.Union[int, t.Callable[[Context], int]] - ) -> Array[ParsedType, BuildTypes,]: ... + def __getitem__(self, count: t.Union[int, t.Callable[[Context], int]]) -> Array[ + ParsedType, + BuildTypes, + ]: ... + def _parse( + self, stream: StreamType, context: Context, path: PathType + ) -> ParsedType: ... + def _parsereport( + self, stream: StreamType, context: Context, path: PathType + ) -> ParsedType: ... + def _build( + self, obj: BuildTypes, stream: StreamType, context: Context, path: PathType + ) -> int: ... + def _sizeof(self, context: Context, path: PathType) -> int: ... @t.type_check_only class Context(Container[t.Any]): @@ -169,7 +195,7 @@ class Subconstruct( subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... @t.overload - def __init__( + def __init__( # type: ignore self, *args: t.Any, **kwargs: t.Any, @@ -221,24 +247,20 @@ class Compiled(Construct[t.Any, t.Any]): # =============================================================================== # bytes and bits # =============================================================================== -class Bytes(Construct[bytes, t.Union[bytes, int]]): +class Bytes(Construct[bytes, t.Union[bytes, bytearray, int]]): length: ConstantOrContextLambda[int] def __init__( self, length: ConstantOrContextLambda[int], ) -> None: ... -GreedyBytes: Construct[bytes, bytes] +GreedyBytes: Construct[bytes, t.Union[bytes, bytearray]] -def Bitwise( - subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> t.Union[ +def Bitwise(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[ Transformed[SubconParsedType, SubconBuildTypes], Restreamed[SubconParsedType, SubconBuildTypes], ]: ... -def Bytewise( - subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> t.Union[ +def Bytewise(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[ Transformed[SubconParsedType, SubconBuildTypes], Restreamed[SubconParsedType, SubconBuildTypes], ]: ... @@ -600,7 +622,7 @@ class Check(Construct[None, None]): func: ConstantOrContextLambda[bool], ) -> None: ... -Error: Construct[None, None] +Error: Construct[t.NoReturn, t.NoReturn] class FocusedSeq(Construct[t.Any, t.Any]): subcons: t.List[Construct[t.Any, t.Any]] @@ -778,6 +800,10 @@ def If( ) -> IfThenElse[t.Optional[ThenParsedType], t.Optional[ThenBuildTypes]]: ... SwitchType = t.TypeVar("SwitchType") +SwitchParsedType = t.TypeVar("SwitchParsedType") +SwitchBuildTypes = t.TypeVar("SwitchBuildTypes") +SwitchDefaultParsedType = t.TypeVar("SwitchDefaultParsedType") +SwitchDefaultBuildTypes = t.TypeVar("SwitchDefaultBuildTypes") class Switch(Construct[ParsedType, BuildTypes]): keyfunc: ConstantOrContextLambda[t.Any] @@ -785,17 +811,31 @@ class Switch(Construct[ParsedType, BuildTypes]): default: Construct[t.Any, t.Any] @t.overload def __new__( - cls: "type[Switch[int, t.Optional[int]]]", + cls: "type[Switch[SwitchParsedType | None, SwitchBuildTypes | None]]", keyfunc: ConstantOrContextLambda[SwitchType], - cases: t.Dict[SwitchType, Construct[int, int]], - default: t.Optional[Construct[int, int]] = ..., - ) -> Switch[int, t.Optional[int]]: ... + cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], + default: None = ..., + ) -> Switch[SwitchParsedType | None, SwitchBuildTypes | None]: ... + @t.overload + def __new__( + cls: "type[Switch[SwitchParsedType, SwitchBuildTypes]]", + keyfunc: ConstantOrContextLambda[SwitchType], + cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], + default: Construct[t.NoReturn, t.NoReturn], + ) -> Switch[SwitchParsedType, SwitchBuildTypes]: ... + @t.overload + def __new__( + cls: "type[Switch[SwitchParsedType | SwitchDefaultParsedType, SwitchBuildTypes | SwitchDefaultBuildTypes]]", + keyfunc: ConstantOrContextLambda[SwitchType], + cases: dict[t.Any, Construct[SwitchParsedType, SwitchBuildTypes]], + default: Construct[SwitchDefaultParsedType, SwitchDefaultBuildTypes], + ) -> Switch[SwitchParsedType | SwitchDefaultParsedType, SwitchBuildTypes | SwitchDefaultBuildTypes]: ... @t.overload def __new__( cls: "type[Switch[t.Any, t.Any]]", - keyfunc: ConstantOrContextLambda[t.Any], - cases: t.Dict[t.Any, Construct[t.Any, t.Any]], - default: t.Optional[Construct[t.Any, t.Any]] = ..., + keyfunc: ConstantOrContextLambda[SwitchType], + cases: dict[t.Any, Construct[t.Any, t.Any]], + default: Construct[t.Any, t.Any] | None = ..., ) -> Switch[t.Any, t.Any]: ... class StopIf(Construct[None, None]): @@ -876,6 +916,16 @@ class Peek( subcon: Construct[SubconParsedType, SubconBuildTypes], ) -> None: ... +class OffsettedEnd( + Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] +): + endoffset: ConstantOrContextLambda[int] + def __init__( + self, + endoffset: ConstantOrContextLambda[int], + subcon: Construct[SubconParsedType, SubconBuildTypes], + ) -> None: ... + class Seek(Construct[int, None]): at: ConstantOrContextLambda[int] if sys.version_info >= (3, 8): @@ -920,9 +970,7 @@ class RawCopy( def ByteSwapped( subcon: Construct[SubconParsedType, SubconBuildTypes] ) -> Transformed[SubconParsedType, SubconBuildTypes]: ... -def BitsSwapped( - subcon: Construct[SubconParsedType, SubconBuildTypes] -) -> t.Union[ +def BitsSwapped(subcon: Construct[SubconParsedType, SubconBuildTypes]) -> t.Union[ Transformed[SubconParsedType, SubconBuildTypes], Restreamed[SubconParsedType, SubconBuildTypes], ]: ... @@ -942,7 +990,10 @@ class Prefixed( def PrefixedArray( countfield: Construct[int, int], subcon: Construct[SubconParsedType, SubconBuildTypes], -) -> Array[SubconParsedType, SubconBuildTypes,]: ... +) -> Array[ + SubconParsedType, + SubconBuildTypes, +]: ... class FixedSized( Subconstruct[SubconParsedType, SubconBuildTypes, SubconParsedType, SubconBuildTypes] @@ -1091,6 +1142,26 @@ class Rebuffered( tailcutoff: t.Optional[int] = ..., ) -> None: ... +class EncryptedSym(Tunnel[SubconParsedType, SubconBuildTypes]): + cipher: ConstantOrContextLambda2[Cipher[Mode]] + def __init__( + self, + subcon: Construct[SubconParsedType, SubconBuildTypes], + cipher: ConstantOrContextLambda2[Cipher[Mode]], + ) -> None: ... + +class EncryptedSymAead(Tunnel[SubconParsedType, SubconBuildTypes]): + cipher: ConstantOrContextLambda2[t.Union[AESGCM, AESCCM, ChaCha20Poly1305]] + nonce: ConstantOrContextLambda2[bytes] + associated_data: ConstantOrContextLambda2[bytes] + def __init__( + self, + subcon: Construct[SubconParsedType, SubconBuildTypes], + cipher: ConstantOrContextLambda2[t.Union[AESGCM, AESCCM, ChaCha20Poly1305]], + nonce: ConstantOrContextLambda2[bytes], + associated_data: ConstantOrContextLambda2[bytes] = ..., + ) -> None: ... + # =============================================================================== # lazy equivalents # =============================================================================== @@ -1110,9 +1181,9 @@ class Lazy( class LazyContainer(t.Generic[ContainerType], t.Dict[str, ContainerType]): def __getattr__(self, name: str) -> ContainerType: ... def __getitem__(self, index: t.Union[str, int]) -> ContainerType: ... - def keys(self) -> t.Iterator[str]: ... - def values(self) -> t.List[ContainerType]: ... - def items(self) -> t.List[t.Tuple[str, ContainerType]]: ... + def keys(self) -> t.Iterator[str]: ... # type: ignore + def values(self) -> t.List[ContainerType]: ... # type: ignore + def items(self) -> t.List[t.Tuple[str, ContainerType]]: ... # type: ignore class LazyStruct(Construct[LazyContainer[t.Any], t.Optional[t.Dict[str, t.Any]]]): subcons: t.List[Construct[t.Any, t.Any]] diff --git a/construct-stubs/expr.pyi b/construct-stubs/expr.pyi index 3d1b032..a7c1a1a 100644 --- a/construct-stubs/expr.pyi +++ b/construct-stubs/expr.pyi @@ -469,7 +469,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __eq__(self: ExprMixin[float], other: ConstOrCallable[float]) -> BinExpr[bool]: ... @t.overload - def __eq__(self, other: t.Any) -> BinExpr[t.Any]: ... + def __eq__(self, other: ConstOrCallable[t.Any]) -> BinExpr[t.Any]: ... # type: ignore # __ne__ ########################################################################################################### @t.overload @@ -487,7 +487,7 @@ class ExprMixin(t.Generic[ReturnType], object): @t.overload def __ne__(self: ExprMixin[float], other: ConstOrCallable[float]) -> BinExpr[bool]: ... @t.overload - def __ne__(self, other: t.Any) -> BinExpr[t.Any]: ... + def __ne__(self, other: t.Any) -> BinExpr[t.Any]: ... # type: ignore # __neg__ ########################################################################################################## @t.overload diff --git a/construct-stubs/lib/containers.pyi b/construct-stubs/lib/containers.pyi index a50033a..37efc75 100644 --- a/construct-stubs/lib/containers.pyi +++ b/construct-stubs/lib/containers.pyi @@ -19,7 +19,7 @@ def recursion_lock( class Container(t.Generic[ContainerType], t.Dict[str, ContainerType]): def __getattr__(self, name: str) -> ContainerType: ... - def update( + def update( # type: ignore self, seqordict: t.Union[t.Dict[str, ContainerType], t.Tuple[str, ContainerType]], ) -> None: ... diff --git a/construct_typed/__init__.py b/construct_typed/__init__.py index 9ea0ccf..f594f2b 100644 --- a/construct_typed/__init__.py +++ b/construct_typed/__init__.py @@ -9,15 +9,19 @@ from .dataclass_struct import ( TStructField, csfield, sfield, + EnhancedDataclassMixin ) from .generic_wrapper import ( Adapter, ConstantOrContextLambda, + ConstantOrContextLambda2, Construct, Context, ListContainer, PathType, - Array + Array, + Subconstruct, + Computed, ) from .tenum import EnumBase, EnumValue, FlagsEnumBase, TEnum, TFlagsEnum @@ -32,6 +36,7 @@ __all__ = [ "TStructField", "csfield", "sfield", + "EnhancedDataclassMixin", "EnumBase", "EnumValue", "FlagsEnumBase", @@ -39,9 +44,12 @@ __all__ = [ "TFlagsEnum", "Adapter", "ConstantOrContextLambda", + "ConstantOrContextLambda2", "Construct", "Context", "ListContainer", "PathType", - "Array" + "Array", + "Subconstruct", + "Computed" ] diff --git a/construct_typed/dataclass_struct.py b/construct_typed/dataclass_struct.py index 8a4c562..e626085 100644 --- a/construct_typed/dataclass_struct.py +++ b/construct_typed/dataclass_struct.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- # pyright: strict +# pyright: reportIncompatibleVariableOverride=false, reportAny=false import dataclasses import textwrap import typing as t @@ -11,6 +12,7 @@ from construct.lib.containers import ( recursion_lock, ) from construct.lib.py3compat import bytestringtype, reprstring, unicodestringtype +from typing_extensions import override from .generic_wrapper import Adapter, Construct, Context, ParsedType, PathType @@ -27,7 +29,7 @@ class DataclassMixin: methods exists and every name can be used. """ - __dataclass_fields__: "t.ClassVar[t.Dict[str, dataclasses.Field[t.Any]]]" + __dataclass_fields__: "t.ClassVar[dict[str, dataclasses.Field[t.Any]]]" def __getitem__(self, key: str) -> t.Any: return getattr(self, key) @@ -77,8 +79,8 @@ class DataclassMixin: def csfield( subcon: Construct[ParsedType, t.Any], - doc: t.Optional[str] = None, - parsed: t.Optional[t.Callable[[t.Any, Context], None]] = None, + doc: str | None = None, + parsed: t.Callable[[t.Any, Context], None] | None = None, ) -> ParsedType: """ Helper method for "DataclassStruct" and "DataclassBitStruct" to create the dataclass fields. @@ -152,27 +154,14 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): Image(width=1, height=2, pixels=b'12') """ - subcon: "cs.Struct" - if t.TYPE_CHECKING: - - def __new__( - cls, - dc_type: t.Type[DataclassType], - reverse: bool = False, - ) -> "DataclassStruct[DataclassType]": - ... - + subcon: "cs.Struct" # type: ignore def __init__( self, - dc_type: t.Type[DataclassType], + dc_type: type[DataclassType], reverse: bool = False, ) -> None: - if not issubclass(dc_type, DataclassMixin): - raise TypeError(f"'{repr(dc_type)}' has to be a '{repr(DataclassMixin)}'") - if not dataclasses.is_dataclass(dc_type): - raise TypeError(f"'{repr(dc_type)}' has to be a 'dataclasses.dataclass'") - self.dc_type = dc_type - self.reverse = reverse + self.dc_type: type[DataclassType] = dc_type + self.reverse: bool = reverse # get all fields from the dataclass fields = dataclasses.fields(self.dc_type) @@ -180,7 +169,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): fields = tuple(reversed(fields)) # extract the construct formats from the struct_type - subcon_fields = {} + subcon_fields: dict[str, t.Any] = {} for field in fields: subcon_fields[field.name] = field.metadata["subcon"] @@ -190,6 +179,7 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): def __getattr__(self, name: str) -> t.Any: return getattr(self.subcon, name) + @override def _decode( self, obj: "cs.Container[t.Any]", context: Context, path: PathType ) -> DataclassType: @@ -214,9 +204,10 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): return dc # type: ignore + @override def _encode( self, obj: DataclassType, context: Context, path: PathType - ) -> t.Dict[str, t.Any]: + ) -> dict[str, t.Any]: if not isinstance(obj, self.dc_type): raise TypeError(f"'{repr(obj)}' has to be of type {repr(self.dc_type)}") @@ -224,20 +215,16 @@ class DataclassStruct(Adapter[t.Any, t.Any, DataclassType, DataclassType]): fields = dataclasses.fields(self.dc_type) # extract all fields from the container, that are used for create the dataclass object - ret_dict: t.Dict[str, t.Any] = {} + ret_dict: dict[str, t.Any] = {} for field in fields: value = getattr(obj, field.name) ret_dict[field.name] = value return ret_dict - def DataclassBitStruct( - dc_type: t.Type[DataclassType], reverse: bool = False -) -> t.Union[ - "cs.Transformed[DataclassType, DataclassType]", - "cs.Restreamed[DataclassType, DataclassType]", -]: + dc_type: type[DataclassType], reverse: bool = False +) -> "cs.Transformed[DataclassType, DataclassType] | cs.Restreamed[DataclassType, DataclassType]": r""" Makes a DataclassStruct inside a Bitwise. @@ -264,6 +251,29 @@ def DataclassBitStruct( """ return cs.Bitwise(DataclassStruct(dc_type, reverse)) +class EnhancedDataclassMixin(DataclassMixin): + @classmethod + def format(cls): + return DataclassStruct(cls) + + @classmethod + def build(cls, obj: t.Self, **kw: dict[str, t.Any]): + return cls.format().build(obj, **kw) + + @classmethod + def parse(cls, data: bytes | bytearray, **kw: dict[str, t.Any]): + return cls.format().parse(data, **kw) + + @classmethod + def parse_file(cls, file: str, **kw: dict[str, t.Any]): + return cls.format().parse_file(file, **kw) + + @classmethod + def parse_stream(cls, stream: t.IO[bytes], **kw: dict[str, t.Any]): + return cls.format().parse_stream(stream, **kw) + + def build_self(self) -> bytes: + return self.build(self) # support legacy names TStruct = DataclassStruct diff --git a/construct_typed/generic_wrapper.py b/construct_typed/generic_wrapper.py index 742c267..cd4788b 100644 --- a/construct_typed/generic_wrapper.py +++ b/construct_typed/generic_wrapper.py @@ -12,12 +12,14 @@ if t.TYPE_CHECKING: # while type checking, the original classes are already generics, because they are defined like this in the stubs. from construct import Adapter as Adapter from construct import ConstantOrContextLambda as ConstantOrContextLambda + from construct import ConstantOrContextLambda2 as ConstantOrContextLambda2 from construct import Construct as Construct from construct import Context as Context from construct import ListContainer as ListContainer from construct import PathType as PathType from construct import Array as Array - + from construct import Subconstruct as Subconstruct + from construct import Computed as Computed else: import construct as cs @@ -44,5 +46,12 @@ else: ): pass + class Subconstruct(t.Generic[SubconParsedType, SubconBuildTypes, ParsedType, BuildTypes], cs.Subconstruct): + pass + + class Computed(t.Generic[ParsedType], cs.Computed): + pass + ConstantOrContextLambda = t.Union[ValueType, t.Callable[[Context], t.Any]] + ConstantOrContextLambda2 = t.Union[ValueType, t.Callable[[Context], ValueType]] PathType = str diff --git a/construct_typed/tenum.py b/construct_typed/tenum.py index a71fb7f..4417c6b 100644 --- a/construct_typed/tenum.py +++ b/construct_typed/tenum.py @@ -1,7 +1,10 @@ +# pyright: reportAny=false import enum import typing as t -from .generic_wrapper import * +from typing_extensions import Self, override + +from .generic_wrapper import Construct, Adapter, Context, PathType # ## TEnum ############################################################################################################ @@ -10,8 +13,8 @@ class EnumValue: This is a helper class for adding documentation to an enum value. """ - def __init__(self, value: int, doc: t.Optional[str] = None) -> None: - self.value = value + def __init__(self, value: int, doc: str | None = None) -> None: + self.value: int = value self.__doc__ = doc if doc else "" @@ -45,7 +48,7 @@ class EnumBase(enum.IntEnum): 'This is the running state.' """ - def __new__(cls, val: t.Union[EnumValue, int]) -> "EnumBase": + def __new__(cls, val: EnumValue | int) -> "Self": if isinstance(val, EnumValue): obj = int.__new__(cls, val.value) obj._value_ = val.value @@ -60,7 +63,8 @@ class EnumBase(enum.IntEnum): # not found in the enum, a new pseudo member is created. # The idea is taken from: https://stackoverflow.com/a/57179436 @classmethod - def _missing_(cls, value: t.Any) -> t.Optional[enum.Enum]: + @override + def _missing_(cls, value: t.Any) -> enum.Enum | None: if isinstance(value, int): pseudo_member = cls._value2member_map_.get(value, None) if pseudo_member is None: @@ -74,7 +78,8 @@ class EnumBase(enum.IntEnum): return pseudo_member return None # will raise the ValueError in Enum.__new__ - def __reduce_ex__(self, proto: t.Any) -> t.Tuple[t.Any, ...]: + @override + def __reduce_ex__(self, proto: t.Any) -> tuple[t.Any, ...]: """ Pickle enums by value instead of name (restores pre-3.11 behavior). See https://github.com/python/cpython/pull/26658 for why this exists. @@ -89,29 +94,18 @@ class TEnum(Adapter[int, int, EnumType, EnumType]): """ Typed enum. """ - - if t.TYPE_CHECKING: - - def __new__( - cls, subcon: Construct[int, int], enum_type: t.Type[EnumType] - ) -> "TEnum[EnumType]": - ... - - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[EnumType]): - if not issubclass(enum_type, EnumBase): - raise TypeError( - "'{}' has to be a '{}'".format(repr(enum_type), repr(EnumBase)) - ) - + def __init__(self, subcon: Construct[int, int], enum_type: type[EnumType]): # save enum type - self.enum_type = t.cast(t.Type[EnumType], enum_type) # type: ignore + self.enum_type: type[EnumType] = enum_type # init adatper super(TEnum, self).__init__(subcon) # type: ignore + @override def _decode(self, obj: int, context: Context, path: PathType) -> EnumType: return self.enum_type(obj) + @override def _encode( self, obj: EnumType, @@ -158,7 +152,7 @@ class FlagsEnumBase(enum.IntFlag): 'This is option two.' """ - def __new__(cls, val: t.Union[EnumValue, int]) -> "FlagsEnumBase": + def __new__(cls, val: EnumValue | int) -> "Self": if isinstance(val, EnumValue): obj = int.__new__(cls, val.value) obj._value_ = val.value @@ -170,6 +164,7 @@ class FlagsEnumBase(enum.IntFlag): return obj @classmethod + @override def _missing_(cls, value: t.Any) -> t.Any: """ Returns member (possibly creating it) if one can be found for value. @@ -178,7 +173,8 @@ class FlagsEnumBase(enum.IntFlag): new_member.__doc__ = "missing value" return new_member - def __reduce_ex__(self, proto: t.Any) -> t.Tuple[t.Any, ...]: + @override + def __reduce_ex__(self, proto: t.Any) -> tuple[t.Any, ...]: """ Pickle enums by value instead of name (restores pre-3.11 behavior). See https://github.com/python/cpython/pull/26658 for why this exists. @@ -193,29 +189,18 @@ class TFlagsEnum(Adapter[int, int, FlagsEnumType, FlagsEnumType]): """ Typed enum. """ - - if t.TYPE_CHECKING: - - def __new__( - cls, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType] - ) -> "TFlagsEnum[FlagsEnumType]": - ... - - def __init__(self, subcon: Construct[int, int], enum_type: t.Type[FlagsEnumType]): - if not issubclass(enum_type, FlagsEnumBase): - raise TypeError( - "'{}' has to be a '{}'".format(repr(enum_type), repr(FlagsEnumBase)) - ) - + def __init__(self, subcon: Construct[int, int], enum_type: type[FlagsEnumType]): # save enum type - self.enum_type = t.cast(t.Type[FlagsEnumType], enum_type) # type: ignore + self.enum_type: type[FlagsEnumType] = enum_type # init adatper super(TFlagsEnum, self).__init__(subcon) # type: ignore + @override def _decode(self, obj: int, context: Context, path: PathType) -> FlagsEnumType: return self.enum_type(obj) + @override def _encode( self, obj: FlagsEnumType, diff --git a/construct_typed/version.py b/construct_typed/version.py index 721c684..38a2845 100644 --- a/construct_typed/version.py +++ b/construct_typed/version.py @@ -1,2 +1,2 @@ -version = (0, 6, 1) -version_string = "0.6.1" +version = (0, 7, 0) +version_string = "0.7.0+wrapper" diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 3412486..0000000 --- a/mypy.ini +++ /dev/null @@ -1,3 +0,0 @@ -[mypy] -strict = True -warn_unused_ignores = False \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8a2689b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,76 @@ + +[build-system] +requires = ["setuptools >= 75.8.0"] +build-backend = "setuptools.build_meta" + +[project] +name="construct-typing" +dynamic = ["version"] +license = { file = "LICENSE" } +description="Extension for the python package 'construct' that adds typing features" +readme = "README.md" +authors=[{ name = "Tim Riddermann" }] +requires-python = ">=3.9" +dependencies = [ + "construct==2.10.70", + "typing_extensions>=4.6.0" +] +keywords = [ + "construct", + "kaitai", + "declarative", + "data structure", + "struct", + "binary", + "symmetric", + "parser", + "builder", + "parsing", + "building", + "pack", + "unpack", + "packer", + "unpacker", + "bitstring", + "bytestring", + "annotation", + "type hint", + "typing", + "typed", + "bitstruct", + "PEP 561", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Build Tools", + "Topic :: Software Development :: Code Generators", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Typing :: Typed", +] + +[project.urls] +"Homepage" = "https://github.com/timrid/construct-typing" +"Bug Reports" = "https://github.com/timrid/construct-typing/issues" + +[tool.setuptools] +packages=[ + "construct-stubs", + "construct-stubs.lib", + "construct_typed" +] + +[tool.setuptools.dynamic] +version = {attr = "construct_typed.version.version_string"} + +[tool.mypy] +strict = true +warn_unused_ignores = false \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index d1c8be1..2514c06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -construct==2.10.68 +construct==2.10.70 pytest>=6.2.0 numpy arrow @@ -7,4 +7,8 @@ cloudpickle lz4 black isort -mypy \ No newline at end of file +mypy +cryptography +build +setuptools +wheel diff --git a/setup.py b/setup.py deleted file mode 100644 index aa7b612..0000000 --- a/setup.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup - -version_string = "?.?.?" -exec(open("./construct_typed/version.py").read()) - -setup( - name="construct-typing", - version=version_string, - packages=["construct-stubs", "construct_typed"], - package_data={ - "construct-stubs": ["*.pyi", "lib/*.pyi"], - "construct_typed": ["py.typed"], - }, - license="MIT", - license_files=("LICENSE",), - description="Extension for the python package 'construct' that adds typing features", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - platforms=["POSIX", "Windows"], - url="https://github.com/timrid/construct-typing", - author="Tim Riddermann", - python_requires=">=3.7", - install_requires=[ - "construct==2.10.68", - "typing_extensions>=4.6.0" - ], - keywords=[ - "construct", - "kaitai", - "declarative", - "data structure", - "struct", - "binary", - "symmetric", - "parser", - "builder", - "parsing", - "building", - "pack", - "unpack", - "packer", - "unpacker", - "bitstring", - "bytestring", - "annotation", - "type hint", - "typing", - "typed", - "bitstruct", - "PEP 561", - ], - classifiers=[ - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: MIT License", - "Intended Audience :: Developers", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development :: Build Tools", - "Topic :: Software Development :: Code Generators", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: Implementation :: CPython", - "Typing :: Typed", - ], -) diff --git a/tests/test_core.py b/tests/test_core.py index b31cd6b..602a899 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -151,17 +151,29 @@ def test_formatfield_bool_issue_901() -> None: assert d.sizeof() == 1 def test_bytesinteger() -> None: + d = BytesInteger(0) + assert raises(d.parse, b"") == IntegerError + assert raises(d.build, 0) == IntegerError d = BytesInteger(4, signed=True, swapped=False) common(d, b"\x01\x02\x03\x04", 0x01020304, 4) common(d, b"\xff\xff\xff\xff", -1, 4) d = BytesInteger(4, signed=False, swapped=this.swapped) common(d, b"\x01\x02\x03\x04", 0x01020304, 4, swapped=False) common(d, b"\x04\x03\x02\x01", 0x01020304, 4, swapped=True) + assert raises(BytesInteger(-1).parse, b"") == IntegerError + assert raises(BytesInteger(-1).build, 0) == IntegerError + assert raises(BytesInteger(8).build, None) == IntegerError + assert raises(BytesInteger(8, signed=False).build, -1) == IntegerError + assert raises(BytesInteger(8, True).build, -2**64) == IntegerError + assert raises(BytesInteger(8, True).build, 2**64) == IntegerError + assert raises(BytesInteger(8, False).build, -2**64) == IntegerError + assert raises(BytesInteger(8, False).build, 2**64) == IntegerError assert raises(BytesInteger(this.missing).sizeof) == SizeofError - assert raises(BytesInteger(4, signed=False).build, -1) == IntegerError - common(BytesInteger(0), b"", 0, 0) def test_bitsinteger() -> None: + d = BitsInteger(0) + assert raises(d.parse, b"") == IntegerError + assert raises(d.build, 0) == IntegerError d = BitsInteger(8) common(d, b"\x01\x01\x01\x01\x01\x01\x01\x01", 255, 8) d = BitsInteger(8, signed=True) @@ -171,9 +183,17 @@ def test_bitsinteger() -> None: d = BitsInteger(16, swapped=this.swapped) common(d, b"\x01\x01\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00", 0xff00, 16, swapped=False) common(d, b"\x00\x00\x00\x00\x00\x00\x00\x00\x01\x01\x01\x01\x01\x01\x01\x01", 0xff00, 16, swapped=True) - assert raises(BitsInteger(this.missing).sizeof) == SizeofError + assert raises(BitsInteger(-1).parse, b"") == IntegerError + assert raises(BitsInteger(-1).build, 0) == IntegerError + assert raises(BitsInteger(5, swapped=True).parse, bytes(5)) == IntegerError + assert raises(BitsInteger(5, swapped=True).build, 0) == IntegerError + assert raises(BitsInteger(8).build, None) == IntegerError assert raises(BitsInteger(8, signed=False).build, -1) == IntegerError - common(BitsInteger(0), b"", 0, 0) + assert raises(BitsInteger(8, True).build, -2**64) == IntegerError + assert raises(BitsInteger(8, True).build, 2**64) == IntegerError + assert raises(BitsInteger(8, False).build, -2**64) == IntegerError + assert raises(BitsInteger(8, False).build, 2**64) == IntegerError + assert raises(BitsInteger(this.missing).sizeof) == SizeofError def test_varint() -> None: d = VarInt @@ -926,6 +946,17 @@ def test_peek() -> None: assert d4.build(Container(a=0x01, b=0x0102)) == b"" assert d4.sizeof() == 0 +def test_offsettedend() -> None: + d1 = Struct( + "header" / Bytes(2), + "data" / OffsettedEnd(-2, GreedyBytes), + "footer" / Bytes(2), + ) + common(d1, b"\x01\x02\x03\x04\x05\x06\x07", Container(header=b'\x01\x02', data=b'\x03\x04\x05', footer=b'\x06\x07')) + + d2 = OffsettedEnd(0, Byte) + assert raises(d2.sizeof) == SizeofError + def test_seek() -> None: d = Seek(5) assert d.parse(b"") == 5 @@ -1334,6 +1365,105 @@ def test_compressed_prefixed() -> None: assert st.parse(st.build(Container(one=zeros,two=zeros))) == Container(one=zeros,two=zeros) assert raises(d.sizeof) == SizeofError +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsym() -> None: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + iv = b"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + nonce = iv + + # AES 128/256 bit - ECB + d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.ECB())) + common(d, b"\xf4\x0f\x54\xb7\x6a\x7a\xf1\xdb\x92\x73\x14\xde\x2f\xa0\x3e\x2d", b'Secret Message..', key=key128, iv=iv) + common(d, b"\x82\x6b\x01\x82\x90\x02\xa1\x9e\x35\x0a\xe2\xc3\xee\x1a\x42\xf5", b'Secret Message..', key=key256, iv=iv) + + # AES 128/256 bit - CBC + d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.CBC(ctx.iv))) + common(d, b"\xba\x79\xc2\x62\x22\x08\x29\xb9\xfb\xd3\x90\xc4\x04\xb7\x55\x87", b'Secret Message..', key=key128, iv=iv) + common(d, b"\x60\xc2\x45\x0d\x7e\x41\xd4\xf8\x85\xd4\x8a\x64\xd1\x45\x49\xe3", b'Secret Message..', key=key256, iv=iv) + + # AES 128/256 bit - CTR + d = EncryptedSym(GreedyBytes, lambda ctx: Cipher(algorithms.AES(ctx.key), modes.CTR(ctx.nonce))) + common(d, b"\x80\x78\xb6\x0c\x07\xf5\x0c\x90\xce\xa2\xbf\xcb\x5b\x22\xb9\xb5", b'Secret Message..', key=key128, nonce=nonce) + common(d, b"\x6a\xae\x7b\x86\x1a\xa6\xe0\x6a\x49\x02\x02\x1b\xf2\x3c\xd8\x0d", b'Secret Message..', key=key256, nonce=nonce) + + assert raises(EncryptedSym(GreedyBytes, "AES").build, b"") == CipherError # type: ignore + assert raises(EncryptedSym(GreedyBytes, "AES").parse, b"") == CipherError # type: ignore + +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsym_cbc_example() -> None: + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + d = Struct( + "iv" / Default(Bytes(16), os.urandom(16)), + "enc_data" / EncryptedSym( + Aligned(16, + Struct( + "width" / Int16ul, + "height" / Int16ul + ) + ), + lambda ctx: Cipher(algorithms.AES(ctx._.key), modes.CBC(ctx.iv)) + ) + ) + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + byts = d.build({"enc_data": {"width": 5, "height": 4}}, key=key128) + obj = d.parse(byts, key=key128) + assert obj.enc_data == Container(width=5, height=4) + +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsymaead() -> None: + from cryptography.hazmat.primitives.ciphers import aead + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + key256 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + nonce = b"\x20\x21\x22\x23\x24\x25\x26\x27\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" + + # AES 128/256 bit - GCM + d = Struct( + "associated_data" / Bytes(21), + "data" / EncryptedSymAead( + GreedyBytes, + lambda ctx: aead.AESGCM(ctx._.key), + this._.nonce, + this.associated_data + ) + ) + common( + d, + b"This is authenticated\xb6\xd3\x64\x0c\x7a\x31\xaa\x16\xa3\x58\xec\x17\x39\x99\x2e\xf8\x4e\x41\x17\x76\x3f\xd1\x06\x47\x04\x9f\x42\x1c\xf4\xa9\xfd\x99\x9c\xe9", + Container(associated_data=b"This is authenticated", data=b"The secret message"), + key=key128, + nonce=nonce + ) + common( + d, + b"This is authenticated\xde\xb4\x41\x79\xc8\x7f\xea\x8d\x0e\x41\xf6\x44\x2f\x93\x21\xe6\x37\xd1\xd3\x29\xa4\x97\xc3\xb5\xf4\x81\x72\xa1\x7f\x3b\x9b\x53\x24\xe4", + Container(associated_data=b"This is authenticated", data=b"The secret message"), + key=key256, + nonce=nonce + ) + assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).build, b"") == CipherError # type: ignore + assert raises(EncryptedSymAead(GreedyBytes, "AESGCM", bytes(16)).parse, b"") == CipherError # type: ignore + +@pytest.mark.xfail(ONWINDOWS and PYPY, reason="no wheel for 'cryptography' is currently available for pypy on windows") +def test_encryptedsymaead_gcm_example() -> None: + from cryptography.hazmat.primitives.ciphers import aead + d = Struct( + "nonce" / Default(Bytes(16), os.urandom(16)), + "associated_data" / Bytes(21), + "enc_data" / EncryptedSymAead( + GreedyBytes, + lambda ctx: aead.AESGCM(ctx._.key), + this.nonce, + this.associated_data + ) + ) + key128 = b"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + byts = d.build({"associated_data": b"This is authenticated", "enc_data": b"The secret message"}, key=key128) + obj = d.parse(byts, key=key128) + assert obj.enc_data == b"The secret message" + assert obj.associated_data == b"This is authenticated" + def test_rebuffered() -> None: data = b"0" * 1000 assert Rebuffered(Array(1000,Byte)).parse_stream(io.BytesIO(data)) == [48]*1000 diff --git a/tests/test_typed.py b/tests/test_typed.py index b85058a..df8cc84 100644 --- a/tests/test_typed.py +++ b/tests/test_typed.py @@ -2,9 +2,11 @@ # pyright: strict import dataclasses import enum +import textwrap import typing as t import construct as cs + import construct_typed as cst from construct_typed import DataclassBitStruct, DataclassMixin, DataclassStruct, csfield @@ -72,16 +74,20 @@ def test_dataclass_str_repr() -> None: == "Image: \n signature = b'BMP' (total 3)\n width = 3\n height = 2" ) + def test_dataclass_ifthenelse() -> None: @dataclasses.dataclass class IfThenElseTest(DataclassMixin): test_if: t.Optional[int] = csfield(cs.If(False, cs.Int8ub)) - test_ifthenelse: t.Optional[int] = csfield(cs.IfThenElse(True, cs.Int8ub, cs.Pass)) + test_ifthenelse: t.Optional[int] = csfield( + cs.IfThenElse(True, cs.Int8ub, cs.Pass) + ) a = IfThenElseTest(test_if=None, test_ifthenelse=None) assert a.test_if == None assert a.test_ifthenelse == None + def test_dataclass_struct() -> None: @dataclasses.dataclass class Image(DataclassMixin): @@ -395,9 +401,10 @@ def test_tenum_no_enumbase() -> None: def test_tenum_asdict() -> None: # see: https://github.com/timrid/construct-typing/issues/21 - import construct_typed as cst import dataclasses + import construct_typed as cst + class TestEnum(cst.EnumBase): one = 1 two = 2 @@ -436,9 +443,9 @@ def test_tenum_docstring() -> None: Value_NoDoc = cst.EnumValue(2) Value_NoDoc2 = 3 - assert ( - TestEnum.__doc__ - == """ + assert TestEnum.__doc__ is not None + assert textwrap.dedent(TestEnum.__doc__) == textwrap.dedent( + """ This is an test enum. """ ) @@ -508,9 +515,10 @@ def test_tenum_flags() -> None: def test_tenum_flags_asdict() -> None: - import construct_typed as cst import dataclasses + import construct_typed as cst + class TestEnum(cst.FlagsEnumBase): one = 1 two = 2 @@ -549,9 +557,9 @@ def test_tenum_flags_docstring() -> None: Value_NoDoc = cst.EnumValue(2) Value_NoDoc2 = 4 - assert ( - TestEnum.__doc__ - == """ + assert TestEnum.__doc__ is not None + assert textwrap.dedent(TestEnum.__doc__) == textwrap.dedent( + """ This is an test flags enum. """ )