The SSH host key has changed on 8 April, 2022 to this one: SHA256:573uTBSeh74kvOo0HJXi5ijdzRm8me27suzNEDlGyrQ
Gergely Polonkai 4 months ago
parent 61b4270332
commit 03de86597f
No known key found for this signature in database
GPG Key ID: 2D2885533B869ED4
  1. 1
      earthsnake/__init__.py
  2. 84
      earthsnake/compare.py
  3. 38
      earthsnake/document/es4.py
  4. 6
      earthsnake/peer.py
  5. 866
      earthsnake/replica.py
  6. 64
      earthsnake/replica/__init__.py
  7. 27
      earthsnake/replica/memory.py
  8. 104
      earthsnake/syncer/__init__.py
  9. 118
      earthsnake/syncer/connection.py
  10. 115
      earthsnake/syncer/envelope.py
  11. 0
      earthsnake/syncer/http.py
  12. 2
      earthsnake/syncer/util.py
  13. 7
      earthsnake/types.py
  14. 11
      earthsnake/util.py
  15. 134
      earthsnake/watchable.py

@ -16,3 +16,4 @@ __version__ = '0.1.0'
from .identity import Identity
from .document import Document
from .replica import Replica

@ -0,0 +1,84 @@
"""Comparison functions"""
from enum import Enum
from typing import Any, List, Literal, Optional
class Cmp(Enum):
"""Comparison results"""
LT = -1
EQ = 0
GT = 1
def deep_compare(val_a: Any, val_b: Any) -> Cmp:
"""Compare two dictionaries key by key"""
if isinstance(val_a, dict):
for key, elem_a in val_a.items():
elem_b = val_b[key]
cmp = deep_compare(elem_a, elem_b)
if cmp == Cmp.EQ:
continue
return cmp
return compare_arrays(list(val_a.keys()), list(val_b.keys()))
if val_a > val_b:
return Cmp.GT
if val_a < val_b:
return Cmp.LT
return Cmp.EQ
def compare_basic(val_a: Any, val_b: Any, order: Literal['ASC', 'DESC'] = 'ASC') -> Cmp:
"""Compare two basic values"""
cmp = deep_compare(val_a, val_b)
if cmp == Cmp.EQ:
return Cmp.EQ
if order == 'ASC':
return cmp
return Cmp.LT if cmp == Cmp.GT else Cmp.GT
def compare_arrays(
arr_a: List[Any],
arr_b: List[Any],
sort_orders: Optional[List[Literal['ASC', 'DESC']]] = None,
) -> Cmp:
"""Compare two arrays"""
sort_orders = sort_orders or []
for idx, (elem_a, elem_b) in enumerate(zip(arr_a, arr_b)):
try:
sort_order = sort_orders[idx]
except IndexError:
sort_order = 'ASC'
elem_cmp = compare_basic(elem_a, elem_b, sort_order)
if elem_cmp != Cmp.EQ:
return elem_cmp
if len(arr_a) == len(arr_b):
return Cmp.EQ
idx = min(len(arr_a), len(arr_b))
try:
sort_order = sort_orders[idx]
except IndexError:
sort_order = 'ASC'
return compare_basic(len(arr_a), len(arr_b), sort_order)

@ -1,4 +1,4 @@
"""Format validator for raw (JSON) documents in the es.4 format"""
"""Document class for the es.4 format"""
from datetime import datetime, timezone
from hashlib import sha256
@ -29,9 +29,7 @@ class RawDocument(TypedDict, total=False):
class Es4Document(Document): # pylint: disable=too-many-instance-attributes
"""Validator for the 'es.4' format
Checks if documents are spec-compliant before ingesting, and signs them according to spec.
"""An es.4 format document
See https://earthstar-project.org/specs/data-spec
"""
@ -129,7 +127,7 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
signature: Optional[str] = None,
delete_after: Optional[datetime] = None,
):
self.author: Identity = author
self.author = author
self.path = path
self.signature = signature
self._content = content or ''
@ -142,6 +140,8 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
def from_json(cls, raw_document: Dict[str, Any]) -> 'Es4Document':
"""Validate raw_document as an es.4 document and create an ``Es4Document`` from it
Checks if documents are spec-compliant before ingesting, and signs them according to spec.
:returns: a new ``Es4Document``
:raises ValidationError: if anything is wrong
"""
@ -245,14 +245,24 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
hasher = sha256()
for key, value in sorted(hash_keys.items(), key=lambda elem: elem[0]):
# Skip null fields
if value is None:
continue
# Otherwise, append the fieldname and value.
# Tab and newline are our field separators.
# Convert integers to strings here.
# (The newline is included on the last field.)
hasher.update(f'{key}\t{value}\n'.encode('utf-8'))
# Binary digest, not hex digest string! Then convert bytes to Earthstar b32 format with
# leading 'b'.
return base32_bytes_to_string(hasher.digest())
def sign(self, identity: Optional[Identity] = None) -> None:
"""Sign the document and store the signature into the document (mutating it)
"""
if identity and identity != self.author:
raise ValidationError(
"when signing a document, keypair address must match document author"
@ -358,3 +368,21 @@ class Es4Document(Document): # pylint: disable=too-many-instance-attributes
if content is not None and calculated_hash != content_hash:
raise ValidationError("content does not match contentHash")
@staticmethod
def compare_newest_first(doc_a, doc_b):
"""Compare two documents based on their time stamp"""
if doc_a.timestamp < doc_b.timestamp:
return Cmp.LT
if doc_b.timestamp > doc_b.timestamp:
return Cmp.GT
if doc_a.signature < doc_b.signature:
return Cmp.LT
if doc_a.signature > doc_b.signature:
return Cmp.GT
return Cmp.EQ

@ -0,0 +1,6 @@
from .syncer import SyncerBase
class Peer:
def __init__(self, syncer: SyncerBase) -> None:
pass

File diff suppressed because it is too large Load Diff

@ -105,3 +105,67 @@ class Replica(ABC):
:returns: the number of documents changed.
"""
"""Workspace drivers"""
from abc import ABC, ABCMeta
from typing import List, Optional
from ...document import Document
from .. import Workspace
from ...query import Query
class WorkspaceDriverConfig(ABC):
"""Base class for configurable workspace drivers"""
def get_config(self, key: str) -> Optional[str]:
"""Get a configuration value"""
raise NotImplementedError()
def set_config(self, key: str, value: str) -> None:
"""Set a configuration value"""
raise NotImplementedError()
def list_config_keys(self) -> List[str]:
"""List all configuration keys"""
raise NotImplementedError()
def delete_config(self, key: str) -> bool:
"""Delete a configuration value"""
raise NotImplementedError()
class WorkspaceDriverBase(WorkspaceDriverConfig, metaclass=ABCMeta):
"""Base class for workspace drivers"""
workspace: Workspace
@property
def is_closed(self) -> bool:
"""Tells if a workspace is closed"""
raise NotImplementedError()
def close(self, erase: bool) -> None:
"""Close the workspace"""
raise NotImplementedError()
def get_max_local_index(self) -> int:
"""Get the maximum local index count"""
raise NotImplementedError()
def query_docs(self, query: Query) -> List[Document]:
"""Query a list of documents"""
raise NotImplementedError()
def upsert(self, doc: Document) -> Document:
"""Insert or update a document"""
raise NotImplementedError()

@ -1,16 +1,21 @@
"""A share driver that stores data in memory
"""
from typing import Dict, List, Tuple
from ..document import Document
from ..exc import ReplicaIsClosedError
from ..identity import Identity
from ..query import HistoryMode, Query
from ..path import Path
from ..query import HistoryMode, Query
from ..share import Share
from . import Replica
class InMemoryReplica(Replica):
def __init__(self, share: Share, **driver_kwargs):
"""In-memory Replica"""
def __init__(self, share: Share, **driver_kwargs) -> None:
self.share = share
self._is_closed = False
self._max_local_index = -1
@ -18,10 +23,13 @@ class InMemoryReplica(Replica):
self._documents: List[Tuple[int, Document]] = {}
@property
def is_closed(self):
def is_closed(self) -> bool:
return self._is_closed
def close(self, erase: bool = False):
def close(self, erase: bool = False) -> None:
if self._is_closed:
raise ReplicaIsClosedError()
if erase:
self._local_index = -1
self._documents = []
@ -30,15 +38,22 @@ class InMemoryReplica(Replica):
@property
def max_local_index(self) -> int:
if self._is_closed:
raise ReplicaIsClosedError()
return self._max_local_index
def _get_all_docs(self) -> List[Document]:
"""Get all documents"""
if self._is_closed:
raise ReplicaIsClosedError()
return [document for _, document in self._documents]
def _get_latest_docs(self) -> List[Document]:
"""Get the latest version of each document"""
if self._is_closed:
raise ReplicaIsClosedError()
@ -54,6 +69,8 @@ class InMemoryReplica(Replica):
return list(docs_by_path.values())
def query_docs(self, query: Query) -> List[Document]:
"""Query a list of documents"""
if self._is_closed:
raise ReplicaIsClosedError()
@ -72,7 +89,7 @@ class InMemoryReplica(Replica):
if self._is_closed:
raise ReplicaIsClosedError()
self._local_index += 1
self._max_local_index += 1
self._documents = [
(local_index, document)

@ -0,0 +1,104 @@
"""Workspace syncing classes"""
from enum import Enum, auto
from typing import Any, Callable, Dict, Generic, Optional, TypeVar
from ..watchable import Watchable, WatchableSet
FnsBag = Dict[str, Callable[[Any], Any]]
Thunk = Callable[[], None]
T = TypeVar('T')
class TransportStatus(Enum):
OPEN = auto()
CLOSED = auto()
class ConnectionStatus(Enum):
CONNECTING = auto()
CLOSED = auto()
class TransportBase:
"""Base class for workspace syncers"""
status: Watchable[TransportStatus]
is_closed: bool
methods: FnsBag
device_id: str
connections: WatchableSet[Connection]
def __init__(self, device_id: str, methods: FnsBag):
raise NotImplementedError()
def on_close(self, cb: Thunk) -> Thunk:
"""Set a handler for when the connection closes"""
raise NotImplementedError()
def close(self) -> None:
"""Close the syncer’s connection"""
raise NotImplementedError()
class TransportLocal(TransportBase):
def __init__(self, device_id: str, methods: BagType, description: str) -> None:
self.device_id = device_id
self.methods = methods
self.description = description
@property
def is_closed(self) -> bool:
return self.status == TransportStatus.CLOSED
def on_close(self, func: Thunk) -> Thunk:
return self.status.on_change_to(TransportStatus.CLOSED)(func)
def close() -> None:
if self.is_closed:
return
self.status.set(TransportStatus.CLOSED)
for conn in self.connections:
conn.close()
self.connections.clear()
def add_connection(self, other_trans: TransportLocal[BagType]) -> Tuple[Connection, Connection]:
if self.is_closed:
raise Exception('Can’t use a transport after it’s closed')
this_conn: Connection[BagType]
other_conn: Connection[BagType]
this_conn = Connection(
description=f'conn {self.device_id} to {other_trans.device_id}',
transport=self,
device_id=self.device_id,
methods=self.methods,
send_envelope=lambda conn: ConnectionBase[BagType], env: Envelope[BagType]: other_conn.handle_incoming_envelope(env),
)
other_conn = Connection(
description=f'conn other_trans.device_id to {this.device_id}',
transport: other_trans,
device_id: other_trans.device_id,
methods: other_trans.methods,
send_envelope: lambda conn: ConnectionBase[BagType], env: Envelope[BagType]: this_conn.handle_incoming_envelope(env),
)
@this_conn.on_close
def close_other():
other_conn.close()
self.connections.delete(this_conn)
@other_conn.on_close
def close_this():
this_conn.close()
self.connections.add(this_conn)
other_trans.connections.add(other_conn)
return this_conn, other_conn

@ -0,0 +1,118 @@
from typing import Any, Callable, Dict, Generic, Optional, Set, TypeVar
from .connection import Connection, ConnectionStatus
from .envelope import Envelope, EnvelopeNotify, EnvelopeRequest
from .util import make_id
from .watchable import Watchable
from . import TransportBase
T = TypeVar('T')
Thunk = Callable[[], None]
class Connection(Generic[T]):
def __init__(self, transport: TransportBase, device_id: str, description: str, methods: T, send_envelope: Callable[[Connection[T], Envelope], None]) -> None:
self.status = Watchable(ConnectionStatus.CONNECTING)
self._close_cbs: Set[Thunk] = set()
self.description = description
self._transport = transport
self._device_id = device_id
self._other_device_id: Optional[str] = None
self._methods = methods
self._send_envelope = send_envelope
self._deferred_requests: Dict[str, Deferred[Any]] = {}
self._last_seen = 0
@property
def is_closed(self) -> bool:
return self.status == ConnectionStatus.CLOSED
def on_close(self, func: Thunk) -> Thunk:
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
self._close_cbs.add(func)
def del_cb():
self._close_cbs.remove(func)
return del_cb
def close(self) -> None:
if self.is_closed:
return
self.status = ConnectionStatus.CLOSED
for func in self._close_cbs:
func()
self._close_cbs.clear()
def handle_incoming_envelope(env: Envelope[T]) -> None:
# TODO: maybe this function should be in a lock to ensure it only runs one at a time
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
# TODO: throw error if status is ERROR?
if env.kind == 'NOTIFY':
if not hasattr(self._methods, env.method):
logger.warn(f'> error in NOTIFY handler: no notify method called {env.method}')
else:
try:
self._methods[env.method](*env.args)
except BaseException as exc:
logger.warn(f'> error when running NOTIFY method: {env} {exc}')
elif env.kind == 'REQUEST':
try:
if not hasattr(self._methods, env.method):
raise RpcErrorUnknownMethod(f'unknown method in REQUEST: {env.method}')
data = self._methods[env.method](*env.args)
response_env_data = EnvelopeResponseWithData(kind='RESPONSE', from_device_id=self._device_id, envelope_id=env.envelope_id, data=data)
self._send_envelope(self, response_env_data)
except BaseException as exc:
response_env_error = EnvelopeResponseWithError(kind='RESPONSE', from_device_id=self.device_id, envelope_id=env.envelope_id, error=str(exc))
self._send_envelope(self, response_error)
elif env.kind == 'RESPONSE':
deferred = self._deferred_requests.get(env.envelope_id)
if not deferred:
return
if env.data:
deferred.resolve(env.data)
elif env.error:
deferred.reject(RpcErrorFromMethod(env.error))
else:
logger.warn('> RESPONSE has neither data nor error. This should never happen')
deferred.reject(RpcError('> RESPONSE has neither data nor error??'))
self._deferred_requests.remove(env.envelope_id)
def notify(self, method: MethodKey, *args: Parameters[BagType[MethodKey]]) -> None:
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
env = EnvelopeNotify(
kind='NOTIFY',
from_device_id=self._device_id,
envelope_id=f'env:{make_id()}',
method=method,
args=args
)
self._send_envelope(self, env)
def request(self, method: str, *args: Parameters[BagType[MethodKey]]) -> ReturnType[BagType[MethodKey]]:
if self.is_closed:
raise RpcErrorUseAfterClose('the connection is closed')
env = EnvelopeRequest(kind='REQUEST', from_device_id=self._device_id, envelope_id=f'env:{make_id}', method=method, args=args)
deferred = make_deferred()
self._deferred_requests[env.envelope_id] = deferred
self._send_envelope(self, env)
return deferred.promise

@ -0,0 +1,115 @@
"""RPC Envelope handling"""
from enum import Enum, auto
from typing import Any, Callable, Dict, List, Literal, Union
Fn = Callable[..., Any]
FnsBag = Dict[str, Fn]
# pylint: disable=too-few-public-methods
class EnvelopeKind(Enum):
"""Types of envelopes"""
NOTIFY = auto()
REQUEST = auto()
RESPONSE = auto()
class EnvelopeBase:
"""Base envelope type"""
kind: EnvelopeKind
from_device_id: str
envelope_id: str
def as_dict(self) -> Dict[str, Any]:
"""Convert the envelope to a dictionary"""
return {
'kind': self.kind.name,
'fromDeviceId': self.from_device_id,
'envelopeId': self.envelope_id,
}
class EnvelopeNotify(EnvelopeBase):
"""Envelope type for a notification"""
kind: Literal[EnvelopeKind.NOTIFY]
method: str
args: List[str]
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'method': self.method,
'args': self.args,
}
)
return envelope
class EnvelopeRequest(EnvelopeBase):
"""Envelope type for a request"""
kind: Literal[EnvelopeKind.REQUEST]
method: str
args: List[str]
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'method': self.method,
'args': self.args,
}
)
return envelope
class EnvelopeResponseWithData(EnvelopeBase):
"""Envelope type for a data response"""
kind: Literal[EnvelopeKind.RESPONSE]
data: List[str]
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'data': self.data,
}
)
return envelope
class EnvelopeResponseWithError(EnvelopeBase):
"""Envelope type for an error response"""
kind: Literal[EnvelopeKind.RESPONSE]
error: str
def as_dict(self) -> Dict[str, Any]:
envelope = super().as_dict()
envelope.update(
{
'error': self.error,
}
)
EvelopeResponse = Union[EnvelopeResponseWithData, EnvelopeResponseWithError]
Envelope = Union[
EnvelopeNotify,
EnvelopeRequest,
EnvelopeResponseWithData,
EnvelopeResponseWithError,
]

@ -0,0 +1,2 @@
def make_id() -> str:
return str(randint(0, 999999999999999)).zfill(15)

@ -1,8 +1,15 @@
"""Generic types and definitions
"""
from typing import Literal
ALPHA_LOWER = 'abcdefghijklmnopqrstuvwxyz'
ALPHA_UPPER = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
DIGIT = '0123456789'
B32_CHAR = ALPHA_LOWER + '234567'
ALPHA_LOWER_OR_DIGIT = ALPHA_LOWER + DIGIT
PRINTABLE_ASCII = bytes(range(32, 127)).decode('utf-8')
ASCII = bytes(range(128))
Cmp = Literal[-1, 0, 1]

@ -0,0 +1,11 @@
from datetime import datetime
from random import random
def microsecond_now() -> int:
return int(datetime.utcnow().timestamp() * 1000)
def random_id() -> str:
# TODO: better randomness here
return f'{random()}{random()}'

@ -0,0 +1,134 @@
"""Watchable variables"""
from typing import Callable, Dict, Generic, Iterable, Optional, Set, TypeVar
Thunk = Callable[[], None]
T = TypeVar('T')
CbOldNew = Callable[[T, T], None]
CbValue = Callable[[T], None]
class Watchable(Generic[T]):
"""A non-seamless proxy to watch a variable’s value"""
def __init__(self, value: T):
self._cbs: Set[CbOldNew[T]] = set()
self._cbs_by_target: Dict[T, Set[CbOldNew[T]]] = {}
self.value = value
def get(self) -> T:
"""Get the current value of the variable"""
return self.value
def set(self, new_val: T) -> None:
"""Set the variable to a new value"""
old_val = self.value
self.value = new_val
if new_val != old_val:
for func in self._cbs:
func(old_val, new_val)
for target_func in self._cbs_by_target.get(new_val, []):
target_func(old_val, new_val)
def on_change(self, func: CbOldNew[T]) -> Thunk:
"""Add a callback to be called when the variable changes"""
self._cbs.add(func)
def del_cb() -> None:
self._cbs.remove(func)
return del_cb
def on_change_to(self, target: T) -> Callable[[CbOldNew[T]], Thunk]:
"""Add a callback to be called when the variable is set to a specific value"""
def decorator(func: CbOldNew[T]) -> Thunk:
self._cbs_by_target[target].add(func)
def del_cb() -> None:
self._cbs_by_target[target].remove(func)
return del_cb
return decorator
class WatchableSet(Set[T]):
"""A set that can be watched for changes"""
def __init__(self, iterable: Optional[Iterable[T]] = None) -> None:
if iterable is None:
super().__init__()
else:
super().__init__(iterable)
self._add_cbs: Set[CbValue[T]] = set()
self._remove_cbs: Set[CbValue[T]] = set()
self._change_cbs: Set[Thunk] = set()
def add(self, value: T) -> None:
had = value in self
super().add(value)
if not had:
for func in self._add_cbs:
func(value)
for change_func in self._change_cbs:
change_func()
def remove(self, value: T) -> None:
had = value in self
super().remove(value)
if had:
for func in self._remove_cbs:
func(value)
for change_func in self._change_cbs:
change_func()
def clear(self) -> None:
for value in super().copy():
super().remove(value)
for func in self._remove_cbs:
func(value)
for change_func in self._change_cbs:
change_func()
def on_add(self, func: CbValue[T]) -> Thunk:
"""Add a callback function to be called when an item gets added to the set"""
self._add_cbs.add(func)
def del_cb() -> None:
self._add_cbs.remove(func)
return del_cb
def on_remove(self, func: CbValue[T]) -> Thunk:
"""Add a callback function to be called when an item gets removed from the set"""
self._remove_cbs.add(func)
def del_cb() -> None:
self._remove_cbs.remove(func)
return del_cb
def on_change(self, func: Thunk) -> Thunk:
"""Add a callback function to be called when the set changes"""
self._change_cbs.add(func)
def del_cb() -> None:
self._change_cbs.remove(func)
return del_cb
Loading…
Cancel
Save