final

earthsnake/__init__.py

@@ -16,3 +16,4 @@ __version__ = '0.1.0'
 
 from .identity import Identity
 from .document import Document
+from .replica import Replica

earthsnake/compare.py

@@ -0,0 +1,84 @@
+"""Comparison functions"""
+
+from enum import Enum
+from typing import Any, List, Literal, Optional
+
+
+class Cmp(Enum):
+    """Comparison results"""
+
+    LT = -1
+    EQ = 0
+    GT = 1
+
+
+def deep_compare(val_a: Any, val_b: Any) -> Cmp:
+    """Compare two dictionaries key by key"""
+
+    if isinstance(val_a, dict):
+        for key, elem_a in val_a.items():
+            elem_b = val_b[key]
+
+            cmp = deep_compare(elem_a, elem_b)
+
+            if cmp == Cmp.EQ:
+                continue
+
+            return cmp
+
+        return compare_arrays(list(val_a.keys()), list(val_b.keys()))
+
+    if val_a > val_b:
+        return Cmp.GT
+
+    if val_a < val_b:
+        return Cmp.LT
+
+    return Cmp.EQ
+
+
+def compare_basic(val_a: Any, val_b: Any, order: Literal['ASC', 'DESC'] = 'ASC') -> Cmp:
+    """Compare two basic values"""
+
+    cmp = deep_compare(val_a, val_b)
+
+    if cmp == Cmp.EQ:
+        return Cmp.EQ
+
+    if order == 'ASC':
+        return cmp
+
+    return Cmp.LT if cmp == Cmp.GT else Cmp.GT
+
+
+def compare_arrays(
+    arr_a: List[Any],
+    arr_b: List[Any],
+    sort_orders: Optional[List[Literal['ASC', 'DESC']]] = None,
+) -> Cmp:
+    """Compare two arrays"""
+
+    sort_orders = sort_orders or []
+
+    for idx, (elem_a, elem_b) in enumerate(zip(arr_a, arr_b)):
+        try:
+            sort_order = sort_orders[idx]
+        except IndexError:
+            sort_order = 'ASC'
+
+        elem_cmp = compare_basic(elem_a, elem_b, sort_order)
+
+        if elem_cmp != Cmp.EQ:
+            return elem_cmp
+
+    if len(arr_a) == len(arr_b):
+        return Cmp.EQ
+
+    idx = min(len(arr_a), len(arr_b))
+
+    try:
+        sort_order = sort_orders[idx]
+    except IndexError:
+        sort_order = 'ASC'
+
+    return compare_basic(len(arr_a), len(arr_b), sort_order)

earthsnake/document/es4.py

@@ -1,4 +1,4 @@
-"""Format validator for raw (JSON) documents in the es.4 format"""
+"""Document class for the es.4 format"""
 
 from datetime import datetime, timezone
 from hashlib import sha256
@@ -29,9 +29,7 @@ class RawDocument(TypedDict, total=False):
 
 
 class Es4Document(Document):  # pylint: disable=too-many-instance-attributes
-    """Validator for the 'es.4' format
+    """An es.4 format document
-
-    Checks if documents are spec-compliant before ingesting, and signs them according to spec.
 
     See https://earthstar-project.org/specs/data-spec
     """
@@ -129,7 +127,7 @@ class Es4Document(Document):  # pylint: disable=too-many-instance-attributes
         signature: Optional[str] = None,
         delete_after: Optional[datetime] = None,
     ):
-        self.author: Identity = author
+        self.author = author
         self.path = path
         self.signature = signature
         self._content = content or ''
@@ -142,6 +140,8 @@ class Es4Document(Document):  # pylint: disable=too-many-instance-attributes
     def from_json(cls, raw_document: Dict[str, Any]) -> 'Es4Document':
         """Validate raw_document as an es.4 document and create an ``Es4Document`` from it
 
+        Checks if documents are spec-compliant before ingesting, and signs them according to spec.
+
         :returns: a new ``Es4Document``
         :raises ValidationError: if anything is wrong
         """
@@ -245,14 +245,24 @@ class Es4Document(Document):  # pylint: disable=too-many-instance-attributes
         hasher = sha256()
 
         for key, value in sorted(hash_keys.items(), key=lambda elem: elem[0]):
+            # Skip null fields
             if value is None:
                 continue
 
+            # Otherwise, append the fieldname and value.
+            # Tab and newline are our field separators.
+            # Convert integers to strings here.
+            # (The newline is included on the last field.)
             hasher.update(f'{key}\t{value}\n'.encode('utf-8'))
 
+        # Binary digest, not hex digest string!  Then convert bytes to Earthstar b32 format with
+        # leading 'b'.
         return base32_bytes_to_string(hasher.digest())
 
     def sign(self, identity: Optional[Identity] = None) -> None:
+        """Sign the document and store the signature into the document (mutating it)
+        """
+
         if identity and identity != self.author:
             raise ValidationError(
                 "when signing a document, keypair address must match document author"
@@ -358,3 +368,21 @@ class Es4Document(Document):  # pylint: disable=too-many-instance-attributes
 
         if content is not None and calculated_hash != content_hash:
             raise ValidationError("content does not match contentHash")
+
+    @staticmethod
+    def compare_newest_first(doc_a, doc_b):
+        """Compare two documents based on their time stamp"""
+
+        if doc_a.timestamp < doc_b.timestamp:
+            return Cmp.LT
+
+        if doc_b.timestamp > doc_b.timestamp:
+            return Cmp.GT
+
+        if doc_a.signature < doc_b.signature:
+            return Cmp.LT
+
+        if doc_a.signature > doc_b.signature:
+            return Cmp.GT
+
+        return Cmp.EQ

earthsnake/peer.py

@@ -0,0 +1,6 @@
+from .syncer import SyncerBase
+
+
+class Peer:
+    def __init__(self, syncer: SyncerBase) -> None:
+        pass

earthsnake/replica/__init__.py

@@ -105,3 +105,67 @@ class Replica(ABC):
 
         :returns: the number of documents changed.
         """
+"""Workspace drivers"""
+
+from abc import ABC, ABCMeta
+from typing import List, Optional
+
+from ...document import Document
+from .. import Workspace
+from ...query import Query
+
+
+class WorkspaceDriverConfig(ABC):
+    """Base class for configurable workspace drivers"""
+
+    def get_config(self, key: str) -> Optional[str]:
+        """Get a configuration value"""
+
+        raise NotImplementedError()
+
+    def set_config(self, key: str, value: str) -> None:
+        """Set a configuration value"""
+
+        raise NotImplementedError()
+
+    def list_config_keys(self) -> List[str]:
+        """List all configuration keys"""
+
+        raise NotImplementedError()
+
+    def delete_config(self, key: str) -> bool:
+        """Delete a configuration value"""
+
+        raise NotImplementedError()
+
+
+class WorkspaceDriverBase(WorkspaceDriverConfig, metaclass=ABCMeta):
+    """Base class for workspace drivers"""
+
+    workspace: Workspace
+
+    @property
+    def is_closed(self) -> bool:
+        """Tells if a workspace is closed"""
+
+        raise NotImplementedError()
+
+    def close(self, erase: bool) -> None:
+        """Close the workspace"""
+
+        raise NotImplementedError()
+
+    def get_max_local_index(self) -> int:
+        """Get the maximum local index count"""
+
+        raise NotImplementedError()
+
+    def query_docs(self, query: Query) -> List[Document]:
+        """Query a list of documents"""
+
+        raise NotImplementedError()
+
+    def upsert(self, doc: Document) -> Document:
+        """Insert or update a document"""
+
+        raise NotImplementedError()

earthsnake/replica/memory.py

@@ -1,16 +1,21 @@
+"""A share driver that stores data in memory
+"""
+
 from typing import Dict, List, Tuple
 
 from ..document import Document
 from ..exc import ReplicaIsClosedError
 from ..identity import Identity
-from ..query import HistoryMode, Query
 from ..path import Path
+from ..query import HistoryMode, Query
 from ..share import Share
 from . import Replica
 
 
 class InMemoryReplica(Replica):
-    def __init__(self, share: Share, **driver_kwargs):
+    """In-memory Replica"""
+
+    def __init__(self, share: Share, **driver_kwargs) -> None:
         self.share = share
         self._is_closed = False
         self._max_local_index = -1
@@ -18,10 +23,13 @@ class InMemoryReplica(Replica):
         self._documents: List[Tuple[int, Document]] = {}
 
     @property
-    def is_closed(self):
+    def is_closed(self) -> bool:
         return self._is_closed
 
-    def close(self, erase: bool = False):
+    def close(self, erase: bool = False) -> None:
+        if self._is_closed:
+            raise ReplicaIsClosedError()
+
         if erase:
             self._local_index = -1
             self._documents = []
@@ -30,15 +38,22 @@ class InMemoryReplica(Replica):
 
     @property
     def max_local_index(self) -> int:
+        if self._is_closed:
+            raise ReplicaIsClosedError()
+
         return self._max_local_index
 
     def _get_all_docs(self) -> List[Document]:
+        """Get all documents"""
+
         if self._is_closed:
             raise ReplicaIsClosedError()
 
         return [document for _, document in self._documents]
 
     def _get_latest_docs(self) -> List[Document]:
+        """Get the latest version of each document"""
+
         if self._is_closed:
             raise ReplicaIsClosedError()
 
@@ -54,6 +69,8 @@ class InMemoryReplica(Replica):
         return list(docs_by_path.values())
 
     def query_docs(self, query: Query) -> List[Document]:
+        """Query a list of documents"""
+
         if self._is_closed:
             raise ReplicaIsClosedError()
 
@@ -72,7 +89,7 @@ class InMemoryReplica(Replica):
         if self._is_closed:
             raise ReplicaIsClosedError()
 
-        self._local_index += 1
+        self._max_local_index += 1
 
         self._documents = [
             (local_index, document)

earthsnake/syncer/__init__.py

@@ -0,0 +1,104 @@
+"""Workspace syncing classes"""
+
+from enum import Enum, auto
+from typing import Any, Callable, Dict, Generic, Optional, TypeVar
+
+from ..watchable import Watchable, WatchableSet
+
+FnsBag = Dict[str, Callable[[Any], Any]]
+Thunk = Callable[[], None]
+T = TypeVar('T')
+
+
+class TransportStatus(Enum):
+    OPEN = auto()
+    CLOSED = auto()
+
+
+class ConnectionStatus(Enum):
+    CONNECTING = auto()
+    CLOSED = auto()
+
+
+class TransportBase:
+    """Base class for workspace syncers"""
+
+    status: Watchable[TransportStatus]
+    is_closed: bool
+    methods: FnsBag
+    device_id: str
+    connections: WatchableSet[Connection]
+
+    def __init__(self, device_id: str, methods: FnsBag):
+        raise NotImplementedError()
+
+    def on_close(self, cb: Thunk) -> Thunk:
+        """Set a handler for when the connection closes"""
+
+        raise NotImplementedError()
+
+    def close(self) -> None:
+        """Close the syncer’s connection"""
+
+        raise NotImplementedError()
+
+
+class TransportLocal(TransportBase):
+    def __init__(self, device_id: str, methods: BagType, description: str) -> None:
+        self.device_id = device_id
+        self.methods = methods
+        self.description = description
+
+    @property
+    def is_closed(self) -> bool:
+        return self.status == TransportStatus.CLOSED
+
+    def on_close(self, func: Thunk) -> Thunk:
+        return self.status.on_change_to(TransportStatus.CLOSED)(func)
+
+    def close() -> None:
+        if self.is_closed:
+            return
+
+        self.status.set(TransportStatus.CLOSED)
+
+        for conn in self.connections:
+            conn.close()
+
+        self.connections.clear()
+
+    def add_connection(self, other_trans: TransportLocal[BagType]) -> Tuple[Connection, Connection]:
+        if self.is_closed:
+            raise Exception('Can’t use a transport after it’s closed')
+
+        this_conn: Connection[BagType]
+        other_conn: Connection[BagType]
+
+        this_conn = Connection(
+            description=f'conn {self.device_id} to {other_trans.device_id}',
+            transport=self,
+            device_id=self.device_id,
+            methods=self.methods,
+            send_envelope=lambda conn: ConnectionBase[BagType], env: Envelope[BagType]: other_conn.handle_incoming_envelope(env),
+        )
+        other_conn = Connection(
+            description=f'conn other_trans.device_id to {this.device_id}',
+            transport: other_trans,
+            device_id: other_trans.device_id,
+            methods: other_trans.methods,
+            send_envelope: lambda conn: ConnectionBase[BagType], env: Envelope[BagType]: this_conn.handle_incoming_envelope(env),
+        )
+
+        @this_conn.on_close
+        def close_other():
+            other_conn.close()
+            self.connections.delete(this_conn)
+
+        @other_conn.on_close
+        def close_this():
+            this_conn.close()
+
+        self.connections.add(this_conn)
+        other_trans.connections.add(other_conn)
+
+        return this_conn, other_conn

earthsnake/syncer/connection.py

@@ -0,0 +1,118 @@
+from typing import Any, Callable, Dict, Generic, Optional, Set, TypeVar
+
+from .connection import Connection, ConnectionStatus
+from .envelope import Envelope, EnvelopeNotify, EnvelopeRequest
+from .util import make_id
+from .watchable import Watchable
+from . import TransportBase
+
+T = TypeVar('T')
+Thunk = Callable[[], None]
+
+
+class Connection(Generic[T]):
+    def __init__(self, transport: TransportBase, device_id: str, description: str, methods: T, send_envelope: Callable[[Connection[T], Envelope], None]) -> None:
+        self.status = Watchable(ConnectionStatus.CONNECTING)
+        self._close_cbs: Set[Thunk] = set()
+        self.description = description
+        self._transport = transport
+        self._device_id = device_id
+        self._other_device_id: Optional[str] = None
+        self._methods = methods
+        self._send_envelope = send_envelope
+        self._deferred_requests: Dict[str, Deferred[Any]] = {}
+        self._last_seen = 0
+
+    @property
+    def is_closed(self) -> bool:
+        return self.status == ConnectionStatus.CLOSED
+
+    def on_close(self, func: Thunk) -> Thunk:
+        if self.is_closed:
+            raise RpcErrorUseAfterClose('the connection is closed')
+
+        self._close_cbs.add(func)
+
+        def del_cb():
+            self._close_cbs.remove(func)
+
+        return del_cb
+
+    def close(self) -> None:
+        if self.is_closed:
+            return
+
+        self.status = ConnectionStatus.CLOSED
+
+        for func in self._close_cbs:
+            func()
+
+        self._close_cbs.clear()
+
+    def handle_incoming_envelope(env: Envelope[T]) -> None:
+        # TODO: maybe this function should be in a lock to ensure it only runs one at a time
+        if self.is_closed:
+            raise RpcErrorUseAfterClose('the connection is closed')
+
+        # TODO: throw error if status is ERROR?
+        if env.kind == 'NOTIFY':
+            if not hasattr(self._methods, env.method):
+                logger.warn(f'> error in NOTIFY handler: no notify method called {env.method}')
+            else:
+                try:
+                    self._methods[env.method](*env.args)
+                except BaseException as exc:
+                    logger.warn(f'> error when running NOTIFY method: {env} {exc}')
+
+        elif env.kind == 'REQUEST':
+            try:
+                if not hasattr(self._methods, env.method):
+                    raise RpcErrorUnknownMethod(f'unknown method in REQUEST: {env.method}')
+
+                data = self._methods[env.method](*env.args)
+                response_env_data = EnvelopeResponseWithData(kind='RESPONSE', from_device_id=self._device_id, envelope_id=env.envelope_id, data=data)
+                self._send_envelope(self, response_env_data)
+            except BaseException as exc:
+                response_env_error = EnvelopeResponseWithError(kind='RESPONSE', from_device_id=self.device_id, envelope_id=env.envelope_id, error=str(exc))
+
+                self._send_envelope(self, response_error)
+        elif env.kind == 'RESPONSE':
+            deferred = self._deferred_requests.get(env.envelope_id)
+
+            if not deferred:
+                return
+
+            if env.data:
+                deferred.resolve(env.data)
+            elif env.error:
+                deferred.reject(RpcErrorFromMethod(env.error))
+            else:
+                logger.warn('> RESPONSE has neither data nor error. This should never happen')
+
+                deferred.reject(RpcError('> RESPONSE has neither data nor error??'))
+
+            self._deferred_requests.remove(env.envelope_id)
+
+    def notify(self, method: MethodKey, *args: Parameters[BagType[MethodKey]]) -> None:
+        if self.is_closed:
+            raise RpcErrorUseAfterClose('the connection is closed')
+
+        env = EnvelopeNotify(
+            kind='NOTIFY',
+            from_device_id=self._device_id,
+            envelope_id=f'env:{make_id()}',
+            method=method,
+            args=args
+        )
+        self._send_envelope(self, env)
+
+    def request(self, method: str, *args: Parameters[BagType[MethodKey]]) -> ReturnType[BagType[MethodKey]]:
+        if self.is_closed:
+            raise RpcErrorUseAfterClose('the connection is closed')
+
+        env = EnvelopeRequest(kind='REQUEST', from_device_id=self._device_id, envelope_id=f'env:{make_id}', method=method, args=args)
+        deferred = make_deferred()
+        self._deferred_requests[env.envelope_id] = deferred
+        self._send_envelope(self, env)
+
+        return deferred.promise

earthsnake/syncer/envelope.py

@@ -0,0 +1,115 @@
+"""RPC Envelope handling"""
+
+from enum import Enum, auto
+from typing import Any, Callable, Dict, List, Literal, Union
+
+Fn = Callable[..., Any]
+FnsBag = Dict[str, Fn]
+
+# pylint: disable=too-few-public-methods
+
+
+class EnvelopeKind(Enum):
+    """Types of envelopes"""
+
+    NOTIFY = auto()
+    REQUEST = auto()
+    RESPONSE = auto()
+
+
+class EnvelopeBase:
+    """Base envelope type"""
+
+    kind: EnvelopeKind
+    from_device_id: str
+    envelope_id: str
+
+    def as_dict(self) -> Dict[str, Any]:
+        """Convert the envelope to a dictionary"""
+
+        return {
+            'kind': self.kind.name,
+            'fromDeviceId': self.from_device_id,
+            'envelopeId': self.envelope_id,
+        }
+
+
+class EnvelopeNotify(EnvelopeBase):
+    """Envelope type for a notification"""
+
+    kind: Literal[EnvelopeKind.NOTIFY]
+    method: str
+    args: List[str]
+
+    def as_dict(self) -> Dict[str, Any]:
+        envelope = super().as_dict()
+        envelope.update(
+            {
+                'method': self.method,
+                'args': self.args,
+            }
+        )
+
+        return envelope
+
+
+class EnvelopeRequest(EnvelopeBase):
+    """Envelope type for a request"""
+
+    kind: Literal[EnvelopeKind.REQUEST]
+    method: str
+    args: List[str]
+
+    def as_dict(self) -> Dict[str, Any]:
+        envelope = super().as_dict()
+        envelope.update(
+            {
+                'method': self.method,
+                'args': self.args,
+            }
+        )
+
+        return envelope
+
+
+class EnvelopeResponseWithData(EnvelopeBase):
+    """Envelope type for a data response"""
+
+    kind: Literal[EnvelopeKind.RESPONSE]
+    data: List[str]
+
+    def as_dict(self) -> Dict[str, Any]:
+        envelope = super().as_dict()
+        envelope.update(
+            {
+                'data': self.data,
+            }
+        )
+
+        return envelope
+
+
+class EnvelopeResponseWithError(EnvelopeBase):
+    """Envelope type for an error response"""
+
+    kind: Literal[EnvelopeKind.RESPONSE]
+    error: str
+
+    def as_dict(self) -> Dict[str, Any]:
+        envelope = super().as_dict()
+        envelope.update(
+            {
+                'error': self.error,
+            }
+        )
+
+
+EvelopeResponse = Union[EnvelopeResponseWithData, EnvelopeResponseWithError]
+
+
+Envelope = Union[
+    EnvelopeNotify,
+    EnvelopeRequest,
+    EnvelopeResponseWithData,
+    EnvelopeResponseWithError,
+]

earthsnake/syncer/util.py

@@ -0,0 +1,2 @@
+def make_id() -> str:
+    return str(randint(0, 999999999999999)).zfill(15)

earthsnake/types.py

@@ -1,8 +1,15 @@
 """Generic types and definitions
 """
 
+from typing import Literal
+
 ALPHA_LOWER = 'abcdefghijklmnopqrstuvwxyz'
 ALPHA_UPPER = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
 DIGIT = '0123456789'
 B32_CHAR = ALPHA_LOWER + '234567'
 ALPHA_LOWER_OR_DIGIT = ALPHA_LOWER + DIGIT
+PRINTABLE_ASCII = bytes(range(32, 127)).decode('utf-8')
+
+ASCII = bytes(range(128))
+
+Cmp = Literal[-1, 0, 1]

earthsnake/util.py

@@ -0,0 +1,11 @@
+from datetime import datetime
+from random import random
+
+
+def microsecond_now() -> int:
+    return int(datetime.utcnow().timestamp() * 1000)
+
+
+def random_id() -> str:
+    # TODO: better randomness here
+    return f'{random()}{random()}'

earthsnake/watchable.py

@@ -0,0 +1,134 @@
+"""Watchable variables"""
+
+from typing import Callable, Dict, Generic, Iterable, Optional, Set, TypeVar
+
+
+Thunk = Callable[[], None]
+T = TypeVar('T')
+CbOldNew = Callable[[T, T], None]
+CbValue = Callable[[T], None]
+
+
+class Watchable(Generic[T]):
+    """A non-seamless proxy to watch a variable’s value"""
+
+    def __init__(self, value: T):
+        self._cbs: Set[CbOldNew[T]] = set()
+        self._cbs_by_target: Dict[T, Set[CbOldNew[T]]] = {}
+        self.value = value
+
+    def get(self) -> T:
+        """Get the current value of the variable"""
+
+        return self.value
+
+    def set(self, new_val: T) -> None:
+        """Set the variable to a new value"""
+
+        old_val = self.value
+        self.value = new_val
+
+        if new_val != old_val:
+            for func in self._cbs:
+                func(old_val, new_val)
+
+            for target_func in self._cbs_by_target.get(new_val, []):
+                target_func(old_val, new_val)
+
+    def on_change(self, func: CbOldNew[T]) -> Thunk:
+        """Add a callback to be called when the variable changes"""
+
+        self._cbs.add(func)
+
+        def del_cb() -> None:
+            self._cbs.remove(func)
+
+        return del_cb
+
+    def on_change_to(self, target: T) -> Callable[[CbOldNew[T]], Thunk]:
+        """Add a callback to be called when the variable is set to a specific value"""
+
+        def decorator(func: CbOldNew[T]) -> Thunk:
+            self._cbs_by_target[target].add(func)
+
+            def del_cb() -> None:
+                self._cbs_by_target[target].remove(func)
+
+            return del_cb
+
+        return decorator
+
+
+class WatchableSet(Set[T]):
+    """A set that can be watched for changes"""
+
+    def __init__(self, iterable: Optional[Iterable[T]] = None) -> None:
+        if iterable is None:
+            super().__init__()
+        else:
+            super().__init__(iterable)
+
+        self._add_cbs: Set[CbValue[T]] = set()
+        self._remove_cbs: Set[CbValue[T]] = set()
+        self._change_cbs: Set[Thunk] = set()
+
+    def add(self, value: T) -> None:
+        had = value in self
+        super().add(value)
+
+        if not had:
+            for func in self._add_cbs:
+                func(value)
+
+            for change_func in self._change_cbs:
+                change_func()
+
+    def remove(self, value: T) -> None:
+        had = value in self
+        super().remove(value)
+
+        if had:
+            for func in self._remove_cbs:
+                func(value)
+
+            for change_func in self._change_cbs:
+                change_func()
+
+    def clear(self) -> None:
+        for value in super().copy():
+            super().remove(value)
+            for func in self._remove_cbs:
+                func(value)
+
+        for change_func in self._change_cbs:
+            change_func()
+
+    def on_add(self, func: CbValue[T]) -> Thunk:
+        """Add a callback function to be called when an item gets added to the set"""
+
+        self._add_cbs.add(func)
+
+        def del_cb() -> None:
+            self._add_cbs.remove(func)
+
+        return del_cb
+
+    def on_remove(self, func: CbValue[T]) -> Thunk:
+        """Add a callback function to be called when an item gets removed from the set"""
+
+        self._remove_cbs.add(func)
+
+        def del_cb() -> None:
+            self._remove_cbs.remove(func)
+
+        return del_cb
+
+    def on_change(self, func: Thunk) -> Thunk:
+        """Add a callback function to be called when the set changes"""
+
+        self._change_cbs.add(func)
+
+        def del_cb() -> None:
+            self._change_cbs.remove(func)
+
+        return del_cb