This commit is contained in:
Gergely Polonkai 2022-04-12 15:25:29 +02:00
parent 61b4270332
commit 03de86597f
No known key found for this signature in database
GPG Key ID: 2D2885533B869ED4
15 changed files with 1164 additions and 501 deletions

View File

@ -16,3 +16,4 @@ __version__ = '0.1.0'
from .identity import Identity from .identity import Identity
from .document import Document from .document import Document
from .replica import Replica

84
earthsnake/compare.py Normal file
View File

@ -0,0 +1,84 @@
"""Comparison functions"""
from enum import Enum
from typing import Any, List, Literal, Optional
class Cmp(Enum):
"""Comparison results"""
LT = -1
EQ = 0
GT = 1
def deep_compare(val_a: Any, val_b: Any) -> Cmp:
"""Compare two dictionaries key by key"""
if isinstance(val_a, dict):
for key, elem_a in val_a.items():
elem_b = val_b[key]
cmp = deep_compare(elem_a, elem_b)
if cmp == Cmp.EQ:
continue
return cmp
return compare_arrays(list(val_a.keys()), list(val_b.keys()))
if val_a > val_b:
return Cmp.GT
if val_a < val_b:
return Cmp.LT
return Cmp.EQ
def compare_basic(val_a: Any, val_b: Any, order: Literal['ASC', 'DESC'] = 'ASC') -> Cmp:
"""Compare two basic values"""
cmp = deep_compare(val_a, val_b)
if cmp == Cmp.EQ:
return Cmp.EQ
if order == 'ASC':
return cmp
return Cmp.LT if cmp == Cmp.GT else Cmp.GT
def compare_arrays(
arr_a: List[Any],
arr_b: List[Any],
sort_orders: Optional[List[Literal['ASC', 'DESC']]] = None,
) -> Cmp:
"""Compare two arrays"""
sort_orders = sort_orders or []
for idx, (elem_a, elem_b) in enumerate(zip(arr_a, arr_b)):
try:
sort_order = sort_orders[idx]
except IndexError:
sort_order = 'ASC'
elem_cmp = compare_basic(elem_a, elem_b, sort_order)
if elem_cmp != Cmp.EQ:
return elem_cmp
if len(arr_a) == len(arr_b):
return Cmp.EQ
idx = min(len(arr_a), len(arr_b))
try:
sort_order = sort_orders[idx]
except IndexError:
sort_order = 'ASC'
return compare_basic(len(arr_a), len(arr_b), sort_order)

View File

@ -1,4 +1,4 @@
"""Format validator for raw (JSON) documents in the es.4 format""" """Document class for the es.4 format"""
from datetime import datetime, timezone from datetime import datetime, timezone
from hashlib import sha256 from hashlib import sha256
@ -29,9 +29,7 @@ class RawDocument(TypedDict, total=False):
class Es4Document(Document): # pylint: disable=too-many-instance-attributes class Es4Document(Document): # pylint: disable=too-many-instance-attributes
"""Validator for the 'es.4' format """An es.4 format document
Checks if documents are spec-compliant before ingesting, and signs them according to spec.
See https://earthstar-project.org/specs/data-spec See https://earthstar-project.org/specs/data-spec
""" """
@ -129,7 +127,7 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
signature: Optional[str] = None, signature: Optional[str] = None,
delete_after: Optional[datetime] = None, delete_after: Optional[datetime] = None,
): ):
self.author: Identity = author self.author = author
self.path = path self.path = path
self.signature = signature self.signature = signature
self._content = content or '' self._content = content or ''
@ -142,6 +140,8 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
def from_json(cls, raw_document: Dict[str, Any]) -> 'Es4Document': def from_json(cls, raw_document: Dict[str, Any]) -> 'Es4Document':
"""Validate raw_document as an es.4 document and create an ``Es4Document`` from it """Validate raw_document as an es.4 document and create an ``Es4Document`` from it
Checks if documents are spec-compliant before ingesting, and signs them according to spec.
:returns: a new ``Es4Document`` :returns: a new ``Es4Document``
:raises ValidationError: if anything is wrong :raises ValidationError: if anything is wrong
""" """
@ -245,14 +245,24 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
hasher = sha256() hasher = sha256()
for key, value in sorted(hash_keys.items(), key=lambda elem: elem[0]): for key, value in sorted(hash_keys.items(), key=lambda elem: elem[0]):
# Skip null fields
if value is None: if value is None:
continue continue
# Otherwise, append the fieldname and value.
# Tab and newline are our field separators.
# Convert integers to strings here.
# (The newline is included on the last field.)
hasher.update(f'{key}\t{value}\n'.encode('utf-8')) hasher.update(f'{key}\t{value}\n'.encode('utf-8'))
# Binary digest, not hex digest string! Then convert bytes to Earthstar b32 format with
# leading 'b'.
return base32_bytes_to_string(hasher.digest()) return base32_bytes_to_string(hasher.digest())
def sign(self, identity: Optional[Identity] = None) -> None: def sign(self, identity: Optional[Identity] = None) -> None:
"""Sign the document and store the signature into the document (mutating it)
"""
if identity and identity != self.author: if identity and identity != self.author:
raise ValidationError( raise ValidationError(
"when signing a document, keypair address must match document author" "when signing a document, keypair address must match document author"
@ -358,3 +368,21 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
if content is not None and calculated_hash != content_hash: if content is not None and calculated_hash != content_hash:
raise ValidationError("content does not match contentHash") raise ValidationError("content does not match contentHash")
@staticmethod
def compare_newest_first(doc_a, doc_b):
"""Compare two documents based on their time stamp"""
if doc_a.timestamp < doc_b.timestamp:
return Cmp.LT
if doc_b.timestamp > doc_b.timestamp:
return Cmp.GT
if doc_a.signature < doc_b.signature:
return Cmp.LT
if doc_a.signature > doc_b.signature:
return Cmp.GT
return Cmp.EQ

6
earthsnake/peer.py Normal file
View File

@ -0,0 +1,6 @@
from .syncer import SyncerBase
class Peer:
def __init__(self, syncer: SyncerBase) -> None:
pass

File diff suppressed because it is too large Load Diff

View File

@ -105,3 +105,67 @@ class Replica(ABC):
:returns: the number of documents changed. :returns: the number of documents changed.
""" """
"""Workspace drivers"""
from abc import ABC, ABCMeta
from typing import List, Optional
from ...document import Document
from .. import Workspace
from ...query import Query
class WorkspaceDriverConfig(ABC):
"""Base class for configurable workspace drivers"""
def get_config(self, key: str) -> Optional[str]:
"""Get a configuration value"""
raise NotImplementedError()
def set_config(self, key: str, value: str) -> None:
"""Set a configuration value"""
raise NotImplementedError()
def list_config_keys(self) -> List[str]:
"""List all configuration keys"""
raise NotImplementedError()
def delete_config(self, key: str) -> bool:
"""Delete a configuration value"""
raise NotImplementedError()
class WorkspaceDriverBase(WorkspaceDriverConfig, metaclass=ABCMeta):
"""Base class for workspace drivers"""
workspace: Workspace
@property
def is_closed(self) -> bool:
"""Tells if a workspace is closed"""
raise NotImplementedError()
def close(self, erase: bool) -> None:
"""Close the workspace"""
raise NotImplementedError()
def get_max_local_index(self) -> int:
"""Get the maximum local index count"""
raise NotImplementedError()
def query_docs(self, query: Query) -> List[Document]:
"""Query a list of documents"""
raise NotImplementedError()
def upsert(self, doc: Document) -> Document:
"""Insert or update a document"""
raise NotImplementedError()

View File

@ -1,16 +1,21 @@
"""A share driver that stores data in memory
"""
from typing import Dict, List, Tuple from typing import Dict, List, Tuple
from ..document import Document from ..document import Document
from ..exc import ReplicaIsClosedError from ..exc import ReplicaIsClosedError
from ..identity import Identity from ..identity import Identity
from ..query import HistoryMode, Query
from ..path import Path from ..path import Path
from ..query import HistoryMode, Query
from ..share import Share from ..share import Share
from . import Replica from . import Replica
class InMemoryReplica(Replica): class InMemoryReplica(Replica):
def __init__(self, share: Share, **driver_kwargs): """In-memory Replica"""
def __init__(self, share: Share, **driver_kwargs) -> None:
self.share = share self.share = share
self._is_closed = False self._is_closed = False
self._max_local_index = -1 self._max_local_index = -1
@ -18,10 +23,13 @@ class InMemoryReplica(Replica):
self._documents: List[Tuple[int, Document]] = {} self._documents: List[Tuple[int, Document]] = {}
@property @property
def is_closed(self): def is_closed(self) -> bool:
return self._is_closed return self._is_closed
def close(self, erase: bool = False): def close(self, erase: bool = False) -> None:
if self._is_closed:
raise ReplicaIsClosedError()
if erase: if erase:
self._local_index = -1 self._local_index = -1
self._documents = [] self._documents = []
@ -30,15 +38,22 @@ class InMemoryReplica(Replica):
@property @property
def max_local_index(self) -> int: def max_local_index(self) -> int:
if self._is_closed:
raise ReplicaIsClosedError()
return self._max_local_index return self._max_local_index
def _get_all_docs(self) -> List[Document]: def _get_all_docs(self) -> List[Document]:
"""Get all documents"""
if self._is_closed: if self._is_closed:
raise ReplicaIsClosedError() raise ReplicaIsClosedError()
return [document for _, document in self._documents] return [document for _, document in self._documents]
def _get_latest_docs(self) -> List[Document]: def _get_latest_docs(self) -> List[Document]:
"""Get the latest version of each document"""
if self._is_closed: if self._is_closed:
raise ReplicaIsClosedError() raise ReplicaIsClosedError()
@ -54,6 +69,8 @@ class InMemoryReplica(Replica):
return list(docs_by_path.values()) return list(docs_by_path.values())
def query_docs(self, query: Query) -> List[Document]: def query_docs(self, query: Query) -> List[Document]:
"""Query a list of documents"""
if self._is_closed: if self._is_closed:
raise ReplicaIsClosedError() raise ReplicaIsClosedError()
@ -72,7 +89,7 @@ class InMemoryReplica(Replica):
if self._is_closed: if self._is_closed:
raise ReplicaIsClosedError() raise ReplicaIsClosedError()
self._local_index += 1 self._max_local_index += 1
self._documents = [ self._documents = [
(local_index, document) (local_index, document)

View File

@ -0,0 +1,104 @@
"""Workspace syncing classes"""
from enum import Enum, auto
from typing import Any, Callable, Dict, Generic, Optional, TypeVar
from ..watchable import Watchable, WatchableSet
FnsBag = Dict[str, Callable[[Any], Any]]
Thunk = Callable[[], None]
T = TypeVar('T')
class TransportStatus(Enum):
OPEN = auto()
CLOSED = auto()
class ConnectionStatus(Enum):
CONNECTING = auto()
CLOSED = auto()
class TransportBase:
"""Base class for workspace syncers"""
status: Watchable[TransportStatus]
is_closed: bool
methods: FnsBag
device_id: str
connections: WatchableSet[Connection]
def __init__(self, device_id: str, methods: FnsBag):
raise NotImplementedError()
def on_close(self, cb: Thunk) -> Thunk:
"""Set a handler for when the connection closes"""
raise NotImplementedError()
def close(self) -> None:
"""Close the syncers connection"""
raise NotImplementedError()
class TransportLocal(TransportBase):
def __init__(self, device_id: str, methods: BagType, description: str) -> None:
self.device_id = device_id
self.methods = methods
self.description = description
@property
def is_closed(self) -> bool:
return self.status == TransportStatus.CLOSED
def on_close(self, func: Thunk) -> Thunk:
return self.status.on_change_to(TransportStatus.CLOSED)(func)
def close() -> None:
if self.is_closed:
return
self.status.set(TransportStatus.CLOSED)
for conn in self.connections:
conn.close()
self.connections.clear()
def add_connection(self, other_trans: TransportLocal[BagType]) -> Tuple[Connection, Connection]:
if self.is_closed:
raise Exception('Cant use a transport after its closed')
this_conn: Connection[BagType]
other_conn: Connection[BagType]
this_conn = Connection(
description=f'conn {self.device_id} to {other_trans.device_id}',
transport=self,
device_id=self.device_id,
methods=self.methods,
send_envelope=lambda conn: ConnectionBase[BagType], env: Envelope[BagType]: other_conn.handle_incoming_envelope(env),
)
other_conn = Connection(
description=f'conn other_trans.device_id to {this.device_id}',
transport: other_trans,
device_id: other_trans.device_id,
methods: other_trans.methods,
send_envelope: lambda conn: ConnectionBase[BagType], env: Envelope[BagType]: this_conn.handle_incoming_envelope(env),
)
@this_conn.on_close
def close_other():
other_conn.close()
self.connections.delete(this_conn)
@other_conn.on_close
def close_this():
this_conn.close()
self.connections.add(this_conn)
other_trans.connections.add(other_conn)
return this_conn, other_conn

View File

@ -0,0 +1,118 @@
from typing import Any, Callable, Dict, Generic, Optional, Set, TypeVar
from .connection import Connection, ConnectionStatus
from .envelope import Envelope, EnvelopeNotify, EnvelopeRequest
from .util import make_id
from .watchable import Watchable
from . import TransportBase
T = TypeVar('T')
Thunk = Callable[[], None]
class Connection(Generic[T]):
def __init__(self, transport: TransportBase, device_id: str, description: str, methods: T, send_envelope: Callable[[Connection[T], Envelope], None]) -> None:
self.status = Watchable(ConnectionStatus.CONNECTING)
self._close_cbs: Set[Thunk] = set()
self.description = description
self._transport = transport
self._device_id = device_id
self._other_device_id: Optional[str] = None
self._methods = methods
self._send_envelope = send_envelope
self._deferred_requests: Dict[str, Deferred[Any]] = {}
self._last_seen = 0
@property
def is_closed(self) -> bool:
return self.status == ConnectionStatus.CLOSED
def on_close(self, func: Thunk) -> Thunk:
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
self._close_cbs.add(func)
def del_cb():
self._close_cbs.remove(func)
return del_cb
def close(self) -> None:
if self.is_closed:
return
self.status = ConnectionStatus.CLOSED
for func in self._close_cbs:
func()
self._close_cbs.clear()
def handle_incoming_envelope(env: Envelope[T]) -> None:
# TODO: maybe this function should be in a lock to ensure it only runs one at a time
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
# TODO: throw error if status is ERROR?
if env.kind == 'NOTIFY':
if not hasattr(self._methods, env.method):
logger.warn(f'> error in NOTIFY handler: no notify method called {env.method}')
else:
try:
self._methods[env.method](*env.args)
except BaseException as exc:
logger.warn(f'> error when running NOTIFY method: {env} {exc}')
elif env.kind == 'REQUEST':
try:
if not hasattr(self._methods, env.method):
raise RpcErrorUnknownMethod(f'unknown method in REQUEST: {env.method}')
data = self._methods[env.method](*env.args)
response_env_data = EnvelopeResponseWithData(kind='RESPONSE', from_device_id=self._device_id, envelope_id=env.envelope_id, data=data)
self._send_envelope(self, response_env_data)
except BaseException as exc:
response_env_error = EnvelopeResponseWithError(kind='RESPONSE', from_device_id=self.device_id, envelope_id=env.envelope_id, error=str(exc))
self._send_envelope(self, response_error)
elif env.kind == 'RESPONSE':
deferred = self._deferred_requests.get(env.envelope_id)
if not deferred:
return
if env.data:
deferred.resolve(env.data)
elif env.error:
deferred.reject(RpcErrorFromMethod(env.error))
else:
logger.warn('> RESPONSE has neither data nor error. This should never happen')
deferred.reject(RpcError('> RESPONSE has neither data nor error??'))
self._deferred_requests.remove(env.envelope_id)
def notify(self, method: MethodKey, *args: Parameters[BagType[MethodKey]]) -> None:
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
env = EnvelopeNotify(
kind='NOTIFY',
from_device_id=self._device_id,
envelope_id=f'env:{make_id()}',
method=method,
args=args
)
self._send_envelope(self, env)
def request(self, method: str, *args: Parameters[BagType[MethodKey]]) -> ReturnType[BagType[MethodKey]]:
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
env = EnvelopeRequest(kind='REQUEST', from_device_id=self._device_id, envelope_id=f'env:{make_id}', method=method, args=args)
deferred = make_deferred()
self._deferred_requests[env.envelope_id] = deferred
self._send_envelope(self, env)
return deferred.promise

View File

@ -0,0 +1,115 @@
"""RPC Envelope handling"""
from enum import Enum, auto
from typing import Any, Callable, Dict, List, Literal, Union
Fn = Callable[..., Any]
FnsBag = Dict[str, Fn]
# pylint: disable=too-few-public-methods
class EnvelopeKind(Enum):
"""Types of envelopes"""
NOTIFY = auto()
REQUEST = auto()
RESPONSE = auto()
class EnvelopeBase:
"""Base envelope type"""
kind: EnvelopeKind
from_device_id: str
envelope_id: str
def as_dict(self) -> Dict[str, Any]:
"""Convert the envelope to a dictionary"""
return {
'kind': self.kind.name,
'fromDeviceId': self.from_device_id,
'envelopeId': self.envelope_id,
}
class EnvelopeNotify(EnvelopeBase):
"""Envelope type for a notification"""
kind: Literal[EnvelopeKind.NOTIFY]
method: str
args: List[str]
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'method': self.method,
'args': self.args,
}
)
return envelope
class EnvelopeRequest(EnvelopeBase):
"""Envelope type for a request"""
kind: Literal[EnvelopeKind.REQUEST]
method: str
args: List[str]
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'method': self.method,
'args': self.args,
}
)
return envelope
class EnvelopeResponseWithData(EnvelopeBase):
"""Envelope type for a data response"""
kind: Literal[EnvelopeKind.RESPONSE]
data: List[str]
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'data': self.data,
}
)
return envelope
class EnvelopeResponseWithError(EnvelopeBase):
"""Envelope type for an error response"""
kind: Literal[EnvelopeKind.RESPONSE]
error: str
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'error': self.error,
}
)
EvelopeResponse = Union[EnvelopeResponseWithData, EnvelopeResponseWithError]
Envelope = Union[
EnvelopeNotify,
EnvelopeRequest,
EnvelopeResponseWithData,
EnvelopeResponseWithError,
]

View File

View File

@ -0,0 +1,2 @@
def make_id() -> str:
return str(randint(0, 999999999999999)).zfill(15)

View File

@ -1,8 +1,15 @@
"""Generic types and definitions """Generic types and definitions
""" """
from typing import Literal
ALPHA_LOWER = 'abcdefghijklmnopqrstuvwxyz' ALPHA_LOWER = 'abcdefghijklmnopqrstuvwxyz'
ALPHA_UPPER = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' ALPHA_UPPER = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
DIGIT = '0123456789' DIGIT = '0123456789'
B32_CHAR = ALPHA_LOWER + '234567' B32_CHAR = ALPHA_LOWER + '234567'
ALPHA_LOWER_OR_DIGIT = ALPHA_LOWER + DIGIT ALPHA_LOWER_OR_DIGIT = ALPHA_LOWER + DIGIT
PRINTABLE_ASCII = bytes(range(32, 127)).decode('utf-8')
ASCII = bytes(range(128))
Cmp = Literal[-1, 0, 1]

11
earthsnake/util.py Normal file
View File

@ -0,0 +1,11 @@
from datetime import datetime
from random import random
def microsecond_now() -> int:
return int(datetime.utcnow().timestamp() * 1000)
def random_id() -> str:
# TODO: better randomness here
return f'{random()}{random()}'

134
earthsnake/watchable.py Normal file
View File

@ -0,0 +1,134 @@
"""Watchable variables"""
from typing import Callable, Dict, Generic, Iterable, Optional, Set, TypeVar
Thunk = Callable[[], None]
T = TypeVar('T')
CbOldNew = Callable[[T, T], None]
CbValue = Callable[[T], None]
class Watchable(Generic[T]):
"""A non-seamless proxy to watch a variables value"""
def __init__(self, value: T):
self._cbs: Set[CbOldNew[T]] = set()
self._cbs_by_target: Dict[T, Set[CbOldNew[T]]] = {}
self.value = value
def get(self) -> T:
"""Get the current value of the variable"""
return self.value
def set(self, new_val: T) -> None:
"""Set the variable to a new value"""
old_val = self.value
self.value = new_val
if new_val != old_val:
for func in self._cbs:
func(old_val, new_val)
for target_func in self._cbs_by_target.get(new_val, []):
target_func(old_val, new_val)
def on_change(self, func: CbOldNew[T]) -> Thunk:
"""Add a callback to be called when the variable changes"""
self._cbs.add(func)
def del_cb() -> None:
self._cbs.remove(func)
return del_cb
def on_change_to(self, target: T) -> Callable[[CbOldNew[T]], Thunk]:
"""Add a callback to be called when the variable is set to a specific value"""
def decorator(func: CbOldNew[T]) -> Thunk:
self._cbs_by_target[target].add(func)
def del_cb() -> None:
self._cbs_by_target[target].remove(func)
return del_cb
return decorator
class WatchableSet(Set[T]):
"""A set that can be watched for changes"""
def __init__(self, iterable: Optional[Iterable[T]] = None) -> None:
if iterable is None:
super().__init__()
else:
super().__init__(iterable)
self._add_cbs: Set[CbValue[T]] = set()
self._remove_cbs: Set[CbValue[T]] = set()
self._change_cbs: Set[Thunk] = set()
def add(self, value: T) -> None:
had = value in self
super().add(value)
if not had:
for func in self._add_cbs:
func(value)
for change_func in self._change_cbs:
change_func()
def remove(self, value: T) -> None:
had = value in self
super().remove(value)
if had:
for func in self._remove_cbs:
func(value)
for change_func in self._change_cbs:
change_func()
def clear(self) -> None:
for value in super().copy():
super().remove(value)
for func in self._remove_cbs:
func(value)
for change_func in self._change_cbs:
change_func()
def on_add(self, func: CbValue[T]) -> Thunk:
"""Add a callback function to be called when an item gets added to the set"""
self._add_cbs.add(func)
def del_cb() -> None:
self._add_cbs.remove(func)
return del_cb
def on_remove(self, func: CbValue[T]) -> Thunk:
"""Add a callback function to be called when an item gets removed from the set"""
self._remove_cbs.add(func)
def del_cb() -> None:
self._remove_cbs.remove(func)
return del_cb
def on_change(self, func: Thunk) -> Thunk:
"""Add a callback function to be called when the set changes"""
self._change_cbs.add(func)
def del_cb() -> None:
self._change_cbs.remove(func)
return del_cb