From e5e364b97a0b2d25f6feeebdbe1f4dc7da7a76af Mon Sep 17 00:00:00 2001 From: Gergely Polonkai Date: Wed, 1 Nov 2023 07:22:29 +0100 Subject: [PATCH] ci: Add and configure mypy, and make it happy --- examples/test_client.py | 11 +++-- examples/test_server.py | 5 +- poetry.lock | 48 ++++++++++++++++++- pyproject.toml | 1 + ssb/muxrpc.py | 101 ++++++++++++++++++++++++++++------------ ssb/packet_stream.py | 26 ++++++++--- ssb/util.py | 10 +++- 7 files changed, 155 insertions(+), 47 deletions(-) diff --git a/examples/test_client.py b/examples/test_client.py index 6a93925..da5cad2 100644 --- a/examples/test_client.py +++ b/examples/test_client.py @@ -8,9 +8,10 @@ import struct import time from colorlog import ColoredFormatter +from nacl.signing import SigningKey from secret_handshake.network import SHSClient -from ssb.muxrpc import MuxRPCAPI, MuxRPCAPIException +from ssb.muxrpc import MuxRPCAPI, MuxRPCAPIException, MuxRPCRequest from ssb.packet_stream import PacketStream, PSMessageType from ssb.util import load_ssb_secret @@ -18,7 +19,7 @@ api = MuxRPCAPI() @api.define("createHistoryStream") -def create_history_stream(connection, msg): # pylint: disable=unused-argument +def create_history_stream(connection: PacketStream, msg: MuxRPCRequest) -> None: # pylint: disable=unused-argument """Handle the createHistoryStream RPC call""" print("create_history_stream", msg) @@ -27,13 +28,13 @@ def create_history_stream(connection, msg): # pylint: disable=unused-argument @api.define("blobs.createWants") -def create_wants(connection, msg): # pylint: disable=unused-argument +def create_wants(connection: PacketStream, msg: MuxRPCRequest) -> None: # pylint: disable=unused-argument """Handle the createWants RPC call""" print("create_wants", msg) -async def test_client(): +async def test_client() -> None: """The actual client implementation""" async for msg in api.call( @@ -69,7 +70,7 @@ async def test_client(): f.write(img_data) -async def main(keypair): +async def main(keypair: SigningKey) -> None: """The main function to run""" client = SHSClient("127.0.0.1", 8008, keypair, bytes(keypair.verify_key)) diff --git a/examples/test_server.py b/examples/test_server.py index 4c5b74a..8de8181 100644 --- a/examples/test_server.py +++ b/examples/test_server.py @@ -5,6 +5,7 @@ import logging from colorlog import ColoredFormatter from secret_handshake import SHSServer +from secret_handshake.network import SHSDuplexStream from ssb.muxrpc import MuxRPCAPI from ssb.packet_stream import PacketStream @@ -13,7 +14,7 @@ from ssb.util import load_ssb_secret api = MuxRPCAPI() -async def on_connect(conn): +async def on_connect(conn: SHSDuplexStream) -> None: """Incoming connection handler""" packet_stream = PacketStream(conn) @@ -25,7 +26,7 @@ async def on_connect(conn): print(msg) -async def main(): +async def main() -> None: """The main function to run""" server = SHSServer("127.0.0.1", 8008, load_ssb_secret()["keypair"]) server.on_connect(on_connect) diff --git a/poetry.lock b/poetry.lock index 80fda8f..8f78f5b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -638,6 +638,52 @@ files = [ {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, ] +[[package]] +name = "mypy" +version = "1.6.1" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e5012e5cc2ac628177eaac0e83d622b2dd499e28253d4107a08ecc59ede3fc2c"}, + {file = "mypy-1.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8fbb68711905f8912e5af474ca8b78d077447d8f3918997fecbf26943ff3cbb"}, + {file = "mypy-1.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21a1ad938fee7d2d96ca666c77b7c494c3c5bd88dff792220e1afbebb2925b5e"}, + {file = "mypy-1.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b96ae2c1279d1065413965c607712006205a9ac541895004a1e0d4f281f2ff9f"}, + {file = "mypy-1.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:40b1844d2e8b232ed92e50a4bd11c48d2daa351f9deee6c194b83bf03e418b0c"}, + {file = "mypy-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81af8adaa5e3099469e7623436881eff6b3b06db5ef75e6f5b6d4871263547e5"}, + {file = "mypy-1.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8c223fa57cb154c7eab5156856c231c3f5eace1e0bed9b32a24696b7ba3c3245"}, + {file = "mypy-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8032e00ce71c3ceb93eeba63963b864bf635a18f6c0c12da6c13c450eedb183"}, + {file = "mypy-1.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c46b51de523817a0045b150ed11b56f9fff55f12b9edd0f3ed35b15a2809de0"}, + {file = "mypy-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:19f905bcfd9e167159b3d63ecd8cb5e696151c3e59a1742e79bc3bcb540c42c7"}, + {file = "mypy-1.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:82e469518d3e9a321912955cc702d418773a2fd1e91c651280a1bda10622f02f"}, + {file = "mypy-1.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d4473c22cc296425bbbce7e9429588e76e05bc7342da359d6520b6427bf76660"}, + {file = "mypy-1.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59a0d7d24dfb26729e0a068639a6ce3500e31d6655df8557156c51c1cb874ce7"}, + {file = "mypy-1.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfd13d47b29ed3bbaafaff7d8b21e90d827631afda134836962011acb5904b71"}, + {file = "mypy-1.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:eb4f18589d196a4cbe5290b435d135dee96567e07c2b2d43b5c4621b6501531a"}, + {file = "mypy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:41697773aa0bf53ff917aa077e2cde7aa50254f28750f9b88884acea38a16169"}, + {file = "mypy-1.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7274b0c57737bd3476d2229c6389b2ec9eefeb090bbaf77777e9d6b1b5a9d143"}, + {file = "mypy-1.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbaf4662e498c8c2e352da5f5bca5ab29d378895fa2d980630656178bd607c46"}, + {file = "mypy-1.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bb8ccb4724f7d8601938571bf3f24da0da791fe2db7be3d9e79849cb64e0ae85"}, + {file = "mypy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:68351911e85145f582b5aa6cd9ad666c8958bcae897a1bfda8f4940472463c45"}, + {file = "mypy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:49ae115da099dcc0922a7a895c1eec82c1518109ea5c162ed50e3b3594c71208"}, + {file = "mypy-1.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8b27958f8c76bed8edaa63da0739d76e4e9ad4ed325c814f9b3851425582a3cd"}, + {file = "mypy-1.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:925cd6a3b7b55dfba252b7c4561892311c5358c6b5a601847015a1ad4eb7d332"}, + {file = "mypy-1.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8f57e6b6927a49550da3d122f0cb983d400f843a8a82e65b3b380d3d7259468f"}, + {file = "mypy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a43ef1c8ddfdb9575691720b6352761f3f53d85f1b57d7745701041053deff30"}, + {file = "mypy-1.6.1-py3-none-any.whl", hash = "sha256:4cbe68ef919c28ea561165206a2dcb68591c50f3bcf777932323bc208d949cf1"}, + {file = "mypy-1.6.1.tar.gz", hash = "sha256:4d01c00d09a0be62a4ca3f933e315455bde83f37f892ba4b08ce92f3cf44bcc1"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +reports = ["lxml"] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -1269,4 +1315,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "4c675f7bcb417a7a7d8cf065f804c796ccc476edffc79e99dbed152ff94c6d58" +content-hash = "b503e50c4ab977c6785c68bc1e5bf2efb7ab88a1fd33770e84a7f612a85d2641" diff --git a/pyproject.toml b/pyproject.toml index ab1b21e..ba91826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ pytest-mock = "^3.12.0" commitizen = "^3.12.0" black = "^23.10.1" pylint = "^3.0.2" +mypy = "^1.6.1" [tool.poetry.group.docs.dependencies] Sphinx = "^2.1.1" diff --git a/ssb/muxrpc.py b/ssb/muxrpc.py index b0abf65..1f5f83b 100644 --- a/ssb/muxrpc.py +++ b/ssb/muxrpc.py @@ -1,6 +1,15 @@ """MuxRPC""" -from ssb.packet_stream import PSMessageType +from typing import Any, AsyncIterator, Callable, Dict, Generator, List, Literal, Optional, Union + +from typing_extensions import Self + +from .packet_stream import PacketStream, PSMessage, PSMessageType, PSRequestHandler, PSStreamHandler + +MuxRPCJSON = Dict[str, Any] +MuxRPCCallType = Literal["async", "duplex", "sink", "source", "sync"] +MuxRPCRequestHandlerType = Callable[[PacketStream, "MuxRPCRequest"], None] +MuxRPCRequestParam = Union[bytes, str, MuxRPCJSON] # pylint: disable=invalid-name class MuxRPCAPIException(Exception): @@ -10,7 +19,7 @@ class MuxRPCAPIException(Exception): class MuxRPCHandler: # pylint: disable=too-few-public-methods """Base MuxRPC handler class""" - def check_message(self, msg): + def check_message(self, msg: PSMessage) -> None: """Check message validity""" body = msg.body @@ -18,30 +27,48 @@ class MuxRPCHandler: # pylint: disable=too-few-public-methods if isinstance(body, dict) and "name" in body and body["name"] == "Error": raise MuxRPCAPIException(body["message"]) + def __await__(self): + raise NotImplementedError() -class MuxRPCRequestHandler(MuxRPCHandler): + def __aiter__(self): + raise NotImplementedError() + + async def __anext__(self): + raise NotImplementedError() + + def send(self, msg: Any, msg_type: PSMessageType = PSMessageType.JSON, end: bool = False) -> None: + """Send a message through the stream""" + + raise NotImplementedError() + + +class MuxRPCRequestHandler(MuxRPCHandler): # pylint: disable=abstract-method """Base class for MuxRPC request handlers""" - def __init__(self, ps_handler): + def __init__(self, ps_handler: PSRequestHandler): self.ps_handler = ps_handler def __await__(self): msg = yield from self.ps_handler.__await__() self.check_message(msg) + return msg -class MuxRPCSourceHandler(MuxRPCHandler): +class MuxRPCSourceHandler(MuxRPCHandler): # pylint: disable=abstract-method """MuxRPC handler for sources""" - def __init__(self, ps_handler): + def __init__(self, ps_handler: PSStreamHandler): self.ps_handler = ps_handler - def __aiter__(self): + def __aiter__(self) -> AsyncIterator[Optional[PSMessage]]: return self - async def __anext__(self): + async def __anext__(self) -> Optional[PSMessage]: msg = await self.ps_handler.__anext__() + + assert msg + self.check_message(msg) return msg @@ -50,64 +77,72 @@ class MuxRPCSourceHandler(MuxRPCHandler): class MuxRPCSinkHandlerMixin: # pylint: disable=too-few-public-methods """Mixin for sink-type MuxRPC handlers""" - def send(self, msg, msg_type=PSMessageType.JSON, end=False): + connection: PacketStream + req: int + + def send(self, msg: Any, msg_type: PSMessageType = PSMessageType.JSON, end: bool = False) -> None: """Send a message through the stream""" self.connection.send(msg, stream=True, msg_type=msg_type, req=self.req, end_err=end) -class MuxRPCDuplexHandler(MuxRPCSinkHandlerMixin, MuxRPCSourceHandler): +class MuxRPCDuplexHandler(MuxRPCSinkHandlerMixin, MuxRPCSourceHandler): # pylint: disable=abstract-method """MuxRPC handler for duplex streams""" - def __init__(self, ps_handler, connection, req): + def __init__(self, ps_handler: PSStreamHandler, connection: PacketStream, req: int): super().__init__(ps_handler) self.connection = connection self.req = req -class MuxRPCSinkHandler(MuxRPCHandler, MuxRPCSinkHandlerMixin): +class MuxRPCSinkHandler(MuxRPCHandler, MuxRPCSinkHandlerMixin): # pylint: disable=abstract-method """MuxRPC handler for sinks""" - def __init__(self, connection, req): + def __init__(self, connection: PacketStream, req: int): self.connection = connection self.req = req -def _get_appropriate_api_handler(type_, connection, ps_handler, req): +def _get_appropriate_api_handler( + type_: MuxRPCCallType, connection: PacketStream, ps_handler: Union[PSRequestHandler, PSStreamHandler], req: int +) -> MuxRPCHandler: """Find the appropriate MuxRPC handler""" if type_ in {"sync", "async"}: + assert isinstance(ps_handler, PSRequestHandler) return MuxRPCRequestHandler(ps_handler) if type_ == "source": + assert isinstance(ps_handler, PSStreamHandler) return MuxRPCSourceHandler(ps_handler) if type_ == "sink": return MuxRPCSinkHandler(connection, req) if type_ == "duplex": + assert isinstance(ps_handler, PSStreamHandler) return MuxRPCDuplexHandler(ps_handler, connection, req) - return None + raise TypeError(f"Unknown request type {type_}") class MuxRPCRequest: """MuxRPC request""" @classmethod - def from_message(cls, message): + def from_message(cls, message: PSMessage) -> Self: """Initialise a request from a raw packet stream message""" body = message.body return cls(".".join(body["name"]), body["args"]) - def __init__(self, name, args): + def __init__(self, name: str, args: List[MuxRPCRequestParam]): self.name = name self.args = args - def __repr__(self): + def __repr__(self) -> str: return f"" @@ -115,7 +150,7 @@ class MuxRPCMessage: """MuxRPC message""" @classmethod - def from_message(cls, message): + def from_message(cls, message: PSMessage) -> Self: """Initialise a MuxRPC message from a raw packet stream message""" return cls(message.body) @@ -123,21 +158,23 @@ class MuxRPCMessage: def __init__(self, body): self.body = body - def __repr__(self): + def __repr__(self) -> str: return f"" class MuxRPCAPI: """Generit MuxRPC API""" - def __init__(self): - self.handlers = {} - self.connection = None + def __init__(self) -> None: + self.handlers: Dict[str, MuxRPCRequestHandlerType] = {} + self.connection: Optional[PacketStream] = None - def __aiter__(self): + def __aiter__(self) -> AsyncIterator[None]: return self - async def __anext__(self): + async def __anext__(self) -> None: + assert self.connection + req_message = await self.connection.__anext__() if req_message is None: @@ -148,25 +185,25 @@ class MuxRPCAPI: if isinstance(body, dict) and body.get("name"): self.process(self.connection, MuxRPCRequest.from_message(req_message)) - def __await__(self): + def __await__(self) -> Generator[None, None, None]: yield from self.__anext__().__await__() - def add_connection(self, connection): + def add_connection(self, connection: PacketStream) -> None: """Set the packet stream connection of this RPC API""" self.connection = connection - def define(self, name): + def define(self, name: str) -> Callable[[MuxRPCRequestHandlerType], MuxRPCRequestHandlerType]: """Decorator to define an RPC method handler""" - def _handle(f): + def _handle(f: MuxRPCRequestHandlerType) -> MuxRPCRequestHandlerType: self.handlers[name] = f return f return _handle - def process(self, connection, request): + def process(self, connection: PacketStream, request: MuxRPCRequest) -> None: """Process an incoming request""" handler = self.handlers.get(request.name) @@ -176,9 +213,11 @@ class MuxRPCAPI: handler(connection, request) - def call(self, name, args, type_="sync"): + def call(self, name: str, args: List[MuxRPCRequestParam], type_: MuxRPCCallType = "sync") -> MuxRPCHandler: """Call an RPC method""" + assert self.connection + if not self.connection.is_connected: raise Exception("not connected") # pylint: disable=broad-exception-raised diff --git a/ssb/packet_stream.py b/ssb/packet_stream.py index a78b5ad..b1bcaff 100644 --- a/ssb/packet_stream.py +++ b/ssb/packet_stream.py @@ -6,11 +6,15 @@ import logging from math import ceil import struct from time import time +from typing import Any, AsyncIterator, Dict, Optional, Union +from secret_handshake.network import SHSDuplexStream import simplejson logger = logging.getLogger("packet_stream") +PSMessageData = Union[bytes, bool, Dict[str, Any], str] + class PSMessageType(Enum): """Available message types""" @@ -41,7 +45,7 @@ class PSStreamHandler: def __aiter__(self): return self - async def __anext__(self): + async def __anext__(self) -> Optional["PSMessage"]: elem = await self.queue.get() if not elem: @@ -93,7 +97,7 @@ class PSMessage: return cls(type_, body, bool(flags & 0x08), bool(flags & 0x04), req=req) @property - def data(self): + def data(self) -> bytes: """The raw message data""" if self.type == PSMessageType.TEXT: @@ -127,7 +131,7 @@ class PSMessage: class PacketStream: """SSB Packet stream""" - def __init__(self, connection): + def __init__(self, connection: SHSDuplexStream): self.connection = connection self.req_counter = 1 self._event_map = {} @@ -144,10 +148,10 @@ class PacketStream: return self.connection.is_connected - def __aiter__(self): + def __aiter__(self) -> AsyncIterator[Optional[PSMessage]]: return self - async def __anext__(self): + async def __anext__(self) -> Optional[PSMessage]: msg = await self.read() if not msg: @@ -202,7 +206,7 @@ class PacketStream: logger.info("RESPONSE [%d]: EOS", -msg.req) return msg - def _write(self, msg): + def _write(self, msg: PSMessage) -> None: logger.info("SEND [%d]: %r", msg.req, msg) header = struct.pack( ">BIi", (int(msg.stream) << 3) | (int(msg.end_err) << 2) | msg.type.value, len(msg.data), msg.req @@ -213,11 +217,17 @@ class PacketStream: logger.debug("WRITE DATA: %s", msg.data) def send( # pylint: disable=too-many-arguments - self, data, msg_type=PSMessageType.JSON, stream=False, end_err=False, req=None + self, + data: Any, + msg_type: PSMessageType = PSMessageType.JSON, + stream: bool = False, + end_err: bool = False, + req: Optional[int] = None, ): """Send data through the packet stream""" update_counter = False + if req is None: update_counter = True req = self.req_counter @@ -231,10 +241,12 @@ class PacketStream: handler = PSStreamHandler(self.req_counter) else: handler = PSRequestHandler(self.req_counter) + self.register_handler(handler) if update_counter: self.req_counter += 1 + return handler def disconnect(self): diff --git a/ssb/util.py b/ssb/util.py index 7d969c3..181c3d1 100644 --- a/ssb/util.py +++ b/ssb/util.py @@ -2,11 +2,19 @@ from base64 import b64decode, b64encode import os +from typing import TypedDict from nacl.signing import SigningKey import yaml +class SSBSecret(TypedDict): + """Dictionary to hold an SSB identity""" + + keypair: SigningKey + id: str + + class ConfigException(Exception): """Exception to raise if there is a problem with the configuration data""" @@ -17,7 +25,7 @@ def tag(key): return b"@" + b64encode(bytes(key)) + b".ed25519" -def load_ssb_secret(): +def load_ssb_secret() -> SSBSecret: """Load SSB keys from ~/.ssb""" with open(os.path.expanduser("~/.ssb/secret"), encoding="utf-8") as f: