From 053b2aef0f4546a978cc330f626078ab9c8889f1 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 15:42:41 +0300 Subject: [PATCH 01/12] add multiaddr as subpackage --- hivemind/dht/dht.py | 3 +- hivemind/dht/node.py | 2 +- hivemind/p2p/multiaddr/__init__.py | 5 + hivemind/p2p/multiaddr/codecs/__init__.py | 22 ++ hivemind/p2p/multiaddr/codecs/cid.py | 132 ++++++++++ hivemind/p2p/multiaddr/codecs/domain.py | 18 ++ hivemind/p2p/multiaddr/codecs/fspath.py | 15 ++ hivemind/p2p/multiaddr/codecs/ip4.py | 13 + hivemind/p2p/multiaddr/codecs/ip6.py | 13 + hivemind/p2p/multiaddr/codecs/onion.py | 37 +++ hivemind/p2p/multiaddr/codecs/onion3.py | 37 +++ hivemind/p2p/multiaddr/codecs/uint16be.py | 20 ++ hivemind/p2p/multiaddr/codecs/utf8.py | 19 ++ hivemind/p2p/multiaddr/exceptions.py | 94 +++++++ hivemind/p2p/multiaddr/multiaddr.py | 238 +++++++++++++++++ hivemind/p2p/multiaddr/protocols.py | 303 ++++++++++++++++++++++ hivemind/p2p/multiaddr/transforms.py | 96 +++++++ hivemind/p2p/p2p_daemon.py | 2 +- 18 files changed, 1065 insertions(+), 4 deletions(-) create mode 100755 hivemind/p2p/multiaddr/__init__.py create mode 100644 hivemind/p2p/multiaddr/codecs/__init__.py create mode 100644 hivemind/p2p/multiaddr/codecs/cid.py create mode 100644 hivemind/p2p/multiaddr/codecs/domain.py create mode 100644 hivemind/p2p/multiaddr/codecs/fspath.py create mode 100644 hivemind/p2p/multiaddr/codecs/ip4.py create mode 100644 hivemind/p2p/multiaddr/codecs/ip6.py create mode 100644 hivemind/p2p/multiaddr/codecs/onion.py create mode 100644 hivemind/p2p/multiaddr/codecs/onion3.py create mode 100644 hivemind/p2p/multiaddr/codecs/uint16be.py create mode 100644 hivemind/p2p/multiaddr/codecs/utf8.py create mode 100644 hivemind/p2p/multiaddr/exceptions.py create mode 100644 hivemind/p2p/multiaddr/multiaddr.py create mode 100644 hivemind/p2p/multiaddr/protocols.py create mode 100644 hivemind/p2p/multiaddr/transforms.py diff --git a/hivemind/dht/dht.py b/hivemind/dht/dht.py index 85b371d1c..eb3f8aeb0 100644 --- a/hivemind/dht/dht.py +++ b/hivemind/dht/dht.py @@ -7,9 +7,8 @@ from functools import partial from typing import Awaitable, Callable, Iterable, List, Optional, Sequence, TypeVar, Union -from multiaddr import Multiaddr - from hivemind.dht.node import DEFAULT_NUM_WORKERS, DHTNode +from hivemind.p2p.multiaddr import Multiaddr from hivemind.dht.routing import DHTKey, DHTValue, Subkey from hivemind.dht.validation import CompositeValidator, RecordValidatorBase from hivemind.p2p import P2P, PeerID diff --git a/hivemind/dht/node.py b/hivemind/dht/node.py index ee56da11a..0f8ec0cc4 100644 --- a/hivemind/dht/node.py +++ b/hivemind/dht/node.py @@ -23,7 +23,6 @@ Union, ) -from multiaddr import Multiaddr from sortedcontainers import SortedSet from hivemind.dht.crypto import DHTRecord, RecordValidatorBase @@ -32,6 +31,7 @@ from hivemind.dht.storage import DictionaryDHTValue from hivemind.dht.traverse import traverse_dht from hivemind.p2p import P2P, PeerID +from hivemind.p2p.multiaddr import Multiaddr from hivemind.utils import MSGPackSerializer, SerializerBase, get_logger from hivemind.utils.auth import AuthorizerBase from hivemind.utils.timed_storage import DHTExpiration, TimedStorage, ValueWithExpiration diff --git a/hivemind/p2p/multiaddr/__init__.py b/hivemind/p2p/multiaddr/__init__.py new file mode 100755 index 000000000..b9d71ad56 --- /dev/null +++ b/hivemind/p2p/multiaddr/__init__.py @@ -0,0 +1,5 @@ +from .multiaddr import Multiaddr # NOQA + +__author__ = 'Steven Buss' +__email__ = 'steven.buss@gmail.com' +__version__ = '0.0.9' diff --git a/hivemind/p2p/multiaddr/codecs/__init__.py b/hivemind/p2p/multiaddr/codecs/__init__.py new file mode 100644 index 000000000..d30127931 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/__init__.py @@ -0,0 +1,22 @@ +import importlib + + +# These are special sizes +LENGTH_PREFIXED_VAR_SIZE = -1 + + +class NoneCodec: + SIZE = 0 + IS_PATH = False + + +CODEC_CACHE = {} + + +def codec_by_name(name): + if name is None: # Special “do nothing – expect nothing” pseudo-codec + return NoneCodec + codec = CODEC_CACHE.get(name) + if not codec: + codec = CODEC_CACHE[name] = importlib.import_module(".{0}".format(name), __name__) + return codec diff --git a/hivemind/p2p/multiaddr/codecs/cid.py b/hivemind/p2p/multiaddr/codecs/cid.py new file mode 100644 index 000000000..fbef5fe30 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/cid.py @@ -0,0 +1,132 @@ +import base58 +# import cid + +from . import LENGTH_PREFIXED_VAR_SIZE + + +SIZE = LENGTH_PREFIXED_VAR_SIZE +IS_PATH = False + + +# Spec: https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation +CIDv0_PREFIX_TO_LENGTH = { + # base58btc prefixes for valid lengths 1 – 42 with the identity “hash” function + '12': [5, 12, 19, 23, 30, 41, 52, 56], + '13': [9, 16, 34, 45], + '14': [27, 38, 49, 60], + '15': [3, 6, 20], + '16': [3, 6, 13, 20, 31, 42, 53], + '17': [3, 13, 42], + '18': [3], + '19': [3, 24, 57], + '1A': [24, 35, 46], + '1B': [35], + '1D': [17], + '1E': [10, 17], + '1F': [10], + '1G': [10, 28, 50], + '1H': [28, 39], + '1P': [21], + '1Q': [21], + '1R': [21, 54], + '1S': [54], + '1T': [7, 32, 43], + '1U': [7, 32, 43], + '1V': [7], + '1W': [7, 14], + '1X': [7, 14], + '1Y': [7, 14], + '1Z': [7, 14], + '1f': [4], + '1g': [4, 58], + '1h': [4, 25, 58], + '1i': [4, 25], + '1j': [4, 25], + '1k': [4, 25, 47], + '1m': [4, 36, 47], + '1n': [4, 36], + '1o': [4, 36], + '1p': [4], + '1q': [4], + '1r': [4], + '1s': [4], + '1t': [4], + '1u': [4], + '1v': [4], + '1w': [4], + '1x': [4], + '1y': [4], + '1z': [4, 18], + + # base58btc prefix for length 42 with the sha256 hash function + 'Qm': [46], +} + +PROTO_NAME_TO_CIDv1_CODEC = { + # The “p2p” multiaddr protocol requires all keys to use the “libp2p-key” multicodec + "p2p": "libp2p-key", +} + + +def to_bytes(proto, string): + expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name) + + if len(string) in CIDv0_PREFIX_TO_LENGTH.get(string[0:2], ()): # CIDv0 + # Upgrade the wire (binary) representation of any received CIDv0 string + # to CIDv1 if we can determine which multicodec value to use + if expected_codec: + return cid.make_cid(1, expected_codec, base58.b58decode(string)).buffer + + return base58.b58decode(string) + else: # CIDv1+ + parsed = cid.from_string(string) + + # Ensure CID has correct codec for protocol + if expected_codec and parsed.codec != expected_codec: + raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec" + .format(proto.name, expected_codec)) + + return parsed.buffer + + +def _is_binary_cidv0_multihash(buf): + if buf.startswith(b"\x12\x20") and len(buf) == 34: # SHA2-256 + return True + + if (buf[0] == 0x00 and buf[1] in range(43)) and len(buf) == (buf[1] + 2): # Identity hash + return True + + return False + + +def to_string(proto, buf): + expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name) + + if _is_binary_cidv0_multihash(buf): # CIDv0 + if not expected_codec: + # Simply encode as base58btc as there is nothing better to do + return base58.b58encode(buf).decode('ascii') + + # “Implementations SHOULD display peer IDs using the first (raw + # base58btc encoded multihash) format until the second format is + # widely supported.” + # + # In the future the following line should instead convert the multihash + # to CIDv1 and with the `expected_codec` and wrap it in base32: + # return cid.make_cid(1, expected_codec, buf).encode("base32").decode("ascii") + return base58.b58encode(buf).decode("ascii") + else: # CIDv1+ + parsed = cid.from_bytes(buf) + + # Ensure CID has correct codec for protocol + if expected_codec and parsed.codec != expected_codec: + raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec" + .format(proto.name, expected_codec)) + + # “Implementations SHOULD display peer IDs using the first (raw + # base58btc encoded multihash) format until the second format is + # widely supported.” + if expected_codec and _is_binary_cidv0_multihash(parsed.multihash): + return base58.b58encode(parsed.multihash).decode("ascii") + + return parsed.encode("base32").decode("ascii") diff --git a/hivemind/p2p/multiaddr/codecs/domain.py b/hivemind/p2p/multiaddr/codecs/domain.py new file mode 100644 index 000000000..7d616366d --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/domain.py @@ -0,0 +1,18 @@ +import idna + +from . import LENGTH_PREFIXED_VAR_SIZE + + +SIZE = LENGTH_PREFIXED_VAR_SIZE +IS_PATH = False + + +def to_bytes(proto, string): + return idna.uts46_remap(string).encode("utf-8") + + +def to_string(proto, buf): + string = buf.decode("utf-8") + for label in string.split("."): + idna.check_label(label) + return string diff --git a/hivemind/p2p/multiaddr/codecs/fspath.py b/hivemind/p2p/multiaddr/codecs/fspath.py new file mode 100644 index 000000000..36b6859a4 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/fspath.py @@ -0,0 +1,15 @@ +import os + +from . import LENGTH_PREFIXED_VAR_SIZE + + +SIZE = LENGTH_PREFIXED_VAR_SIZE +IS_PATH = True + + +def to_bytes(proto, string): + return os.fsencode(string) + + +def to_string(proto, buf): + return os.fsdecode(buf) diff --git a/hivemind/p2p/multiaddr/codecs/ip4.py b/hivemind/p2p/multiaddr/codecs/ip4.py new file mode 100644 index 000000000..1a7f64556 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/ip4.py @@ -0,0 +1,13 @@ +import netaddr + + +SIZE = 32 +IS_PATH = False + + +def to_bytes(proto, string): + return netaddr.IPAddress(string, version=4).packed + + +def to_string(proto, buf): + return str(netaddr.IPAddress(int.from_bytes(buf, byteorder='big'), version=4)) diff --git a/hivemind/p2p/multiaddr/codecs/ip6.py b/hivemind/p2p/multiaddr/codecs/ip6.py new file mode 100644 index 000000000..7ae71b0d0 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/ip6.py @@ -0,0 +1,13 @@ +import netaddr + + +SIZE = 128 +IS_PATH = False + + +def to_bytes(proto, string): + return netaddr.IPAddress(string, version=6).packed + + +def to_string(proto, buf): + return str(netaddr.IPAddress(int.from_bytes(buf, byteorder='big'), version=6)) diff --git a/hivemind/p2p/multiaddr/codecs/onion.py b/hivemind/p2p/multiaddr/codecs/onion.py new file mode 100644 index 000000000..55c2078d3 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/onion.py @@ -0,0 +1,37 @@ +import base64 +import struct + + +SIZE = 96 +IS_PATH = False + + +def to_bytes(proto, string): + addr = string.split(":") + if len(addr) != 2: + raise ValueError("Does not contain a port number") + + # onion address without the ".onion" substring + if len(addr[0]) != 16: + raise ValueError("Invalid onion host address length (must be 16 characters)") + try: + onion_host_bytes = base64.b32decode(addr[0].upper()) + except Exception as exc: + raise ValueError("Cannot decode {0!r} as base32: {1}".format(addr[0], exc)) from exc + + # onion port number + try: + port = int(addr[1], 10) + except ValueError as exc: + raise ValueError("Port number is not a base 10 integer") from exc + if port not in range(1, 65536): + raise ValueError("Port number is not in range(1, 65536)") + + return b''.join((onion_host_bytes, struct.pack('>H', port))) + + +def to_string(proto, buf): + addr_bytes, port_bytes = (buf[:-2], buf[-2:]) + addr = base64.b32encode(addr_bytes).decode('ascii').lower() + port = str(struct.unpack('>H', port_bytes)[0]) + return ':'.join([addr, port]) diff --git a/hivemind/p2p/multiaddr/codecs/onion3.py b/hivemind/p2p/multiaddr/codecs/onion3.py new file mode 100644 index 000000000..d12bae588 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/onion3.py @@ -0,0 +1,37 @@ +import base64 +import struct + + +SIZE = 296 +IS_PATH = False + + +def to_bytes(proto, string): + addr = string.split(":") + if len(addr) != 2: + raise ValueError("Does not contain a port number") + + # onion3 address without the ".onion" substring + if len(addr[0]) != 56: + raise ValueError("Invalid onion3 host address length (must be 56 characters)") + try: + onion3_host_bytes = base64.b32decode(addr[0].upper()) + except Exception as exc: + raise ValueError("Cannot decode {0!r} as base32: {1}".format(addr[0], exc)) from exc + + # onion3 port number + try: + port = int(addr[1], 10) + except ValueError as exc: + raise ValueError("Port number is not a base 10 integer") from exc + if port not in range(1, 65536): + raise ValueError("Port number is not in range(1, 65536)") + + return b''.join((onion3_host_bytes, struct.pack('>H', port))) + + +def to_string(proto, buf): + addr_bytes, port_bytes = (buf[:-2], buf[-2:]) + addr = base64.b32encode(addr_bytes).decode('ascii').lower() + port = str(struct.unpack('>H', port_bytes)[0]) + return ':'.join([addr, port]) diff --git a/hivemind/p2p/multiaddr/codecs/uint16be.py b/hivemind/p2p/multiaddr/codecs/uint16be.py new file mode 100644 index 000000000..73442e430 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/uint16be.py @@ -0,0 +1,20 @@ +import struct + + +SIZE = 16 +IS_PATH = False + + +def to_bytes(proto, string): + try: + return struct.pack('>H', int(string, 10)) + except ValueError as exc: + raise ValueError("Not a base 10 integer") from exc + except struct.error as exc: + raise ValueError("Integer not in range(65536)") from exc + + +def to_string(proto, buf): + if len(buf) != 2: + raise ValueError("Invalid integer length (must be 2 bytes / 16 bits)") + return str(struct.unpack('>H', buf)[0]) diff --git a/hivemind/p2p/multiaddr/codecs/utf8.py b/hivemind/p2p/multiaddr/codecs/utf8.py new file mode 100644 index 000000000..5a85e2793 --- /dev/null +++ b/hivemind/p2p/multiaddr/codecs/utf8.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +from . import LENGTH_PREFIXED_VAR_SIZE + + +SIZE = LENGTH_PREFIXED_VAR_SIZE +IS_PATH = False + + +def to_bytes(proto, string): + if len(string) == 0: + raise ValueError("{0} value must not be empty".format(proto.name)) + return string.encode('utf-8') + + +def to_string(proto, buf): + if len(buf) == 0: + raise ValueError("invalid length (should be > 0)") + return buf.decode('utf-8') diff --git a/hivemind/p2p/multiaddr/exceptions.py b/hivemind/p2p/multiaddr/exceptions.py new file mode 100644 index 000000000..c92fb8b35 --- /dev/null +++ b/hivemind/p2p/multiaddr/exceptions.py @@ -0,0 +1,94 @@ +class Error(Exception): + pass + + +class LookupError(LookupError, Error): + pass + + +class ProtocolLookupError(LookupError): + """ + MultiAddr did not contain a protocol with the requested code + """ + + def __init__(self, proto, string): + self.proto = proto + self.string = string + + super().__init__( + "MultiAddr {0!r} does not contain protocol {1}".format(string, proto) + ) + + +class ParseError(ValueError, Error): + pass + + +class StringParseError(ParseError): + """ + MultiAddr string representation could not be parsed + """ + + def __init__(self, message, string, protocol=None, original=None): + self.message = message + self.string = string + self.protocol = protocol + self.original = original + + if protocol: + message = "Invalid MultiAddr {0!r} protocol {1}: {2}".format(string, protocol, message) + else: + message = "Invalid MultiAddr {0!r}: {1}".format(string, message) + + super().__init__(message) + + +class BinaryParseError(ParseError): + """ + MultiAddr binary representation could not be parsed + """ + + def __init__(self, message, binary, protocol, original=None): + self.message = message + self.binary = binary + self.protocol = protocol + self.original = original + + message = "Invalid binary MultiAddr protocol {0}: {1}".format(protocol, message) + + super().__init__(message) + + +class ProtocolRegistryError(Error): + pass + + +ProtocolManagerError = ProtocolRegistryError + + +class ProtocolRegistryLocked(Error): + """Protocol registry was locked and doesn't allow any further additions""" + def __init__(self): + super().__init__("Protocol registry is locked and does not accept any new values") + + +class ProtocolExistsError(ProtocolRegistryError): + """Protocol with the given name or code already exists""" + def __init__(self, proto, kind="name"): + self.proto = proto + self.kind = kind + + super().__init__( + "Protocol with {0} {1!r} already exists".format(kind, getattr(proto, kind)) + ) + + +class ProtocolNotFoundError(ProtocolRegistryError): + """No protocol with the given name or code found""" + def __init__(self, value, kind="name"): + self.value = value + self.kind = kind + + super().__init__( + "No protocol with {0} {1!r} found".format(kind, value) + ) diff --git a/hivemind/p2p/multiaddr/multiaddr.py b/hivemind/p2p/multiaddr/multiaddr.py new file mode 100644 index 000000000..52f51a055 --- /dev/null +++ b/hivemind/p2p/multiaddr/multiaddr.py @@ -0,0 +1,238 @@ +import collections.abc + +import varint + +from . import exceptions, protocols + +from .transforms import bytes_iter +from .transforms import string_to_bytes +from .transforms import bytes_to_string + + +__all__ = ("Multiaddr",) + + +class MultiAddrKeys(collections.abc.KeysView, collections.abc.Sequence): + def __contains__(self, proto): + proto = self._mapping.registry.find(proto) + return collections.abc.Sequence.__contains__(self, proto) + + def __getitem__(self, idx): + if idx < 0: + idx = len(self)+idx + for idx2, proto in enumerate(self): + if idx2 == idx: + return proto + raise IndexError("Protocol list index out of range") + + __hash__ = collections.abc.KeysView._hash + + def __iter__(self): + for _, proto, _, _ in bytes_iter(self._mapping.to_bytes()): + yield proto + + +class MultiAddrItems(collections.abc.ItemsView, collections.abc.Sequence): + def __contains__(self, item): + proto, value = item + proto = self._mapping.registry.find(proto) + return collections.abc.Sequence.__contains__(self, (proto, value)) + + def __getitem__(self, idx): + if idx < 0: + idx = len(self)+idx + for idx2, item in enumerate(self): + if idx2 == idx: + return item + raise IndexError("Protocol item list index out of range") + + def __iter__(self): + for _, proto, codec, part in bytes_iter(self._mapping.to_bytes()): + if codec.SIZE != 0: + try: + # If we have an address, return it + yield proto, codec.to_string(proto, part) + except Exception as exc: + raise exceptions.BinaryParseError( + str(exc), + self._mapping.to_bytes(), + proto.name, + exc, + ) from exc + else: + # We were given something like '/utp', which doesn't have + # an address, so return None + yield proto, None + + +class MultiAddrValues(collections.abc.ValuesView, collections.abc.Sequence): + __contains__ = collections.abc.Sequence.__contains__ + + def __getitem__(self, idx): + if idx < 0: + idx = len(self)+idx + for idx2, proto in enumerate(self): + if idx2 == idx: + return proto + raise IndexError("Protocol value list index out of range") + + def __iter__(self): + for _, value in MultiAddrItems(self._mapping): + yield value + + +class Multiaddr(collections.abc.Mapping): + """Multiaddr is a representation of multiple nested internet addresses. + + Multiaddr is a cross-protocol, cross-platform format for representing + internet addresses. It emphasizes explicitness and self-description. + + Learn more here: https://multiformats.io/multiaddr/ + + Multiaddrs have both a binary and string representation. + + >>> from multiaddr import Multiaddr + >>> addr = Multiaddr("/ip4/1.2.3.4/tcp/80") + + Multiaddr objects are immutable, so `encapsulate` and `decapsulate` + return new objects rather than modify internal state. + """ + + __slots__ = ("_bytes", "registry") + + def __init__(self, addr, *, registry=protocols.REGISTRY): + """Instantiate a new Multiaddr. + + Args: + addr : A string-encoded or a byte-encoded Multiaddr + + """ + self.registry = registry + if isinstance(addr, str): + self._bytes = string_to_bytes(addr) + elif isinstance(addr, bytes): + self._bytes = addr + elif isinstance(addr, Multiaddr): + self._bytes = addr.to_bytes() + else: + raise TypeError("MultiAddr must be bytes, str or another MultiAddr instance") + + @classmethod + def join(cls, *addrs): + """Concatenate the values of the given MultiAddr strings or objects, + encapsulating each successive MultiAddr value with the previous ones.""" + return cls(b"".join(map(lambda a: cls(a).to_bytes(), addrs))) + + def __eq__(self, other): + """Checks if two Multiaddr objects are exactly equal.""" + return self._bytes == other._bytes + + def __str__(self): + """Return the string representation of this Multiaddr. + + May raise a :class:`~multiaddr.exceptions.BinaryParseError` if the + stored MultiAddr binary representation is invalid.""" + return bytes_to_string(self._bytes) + + def __contains__(self, proto): + return proto in MultiAddrKeys(self) + + def __iter__(self): + return iter(MultiAddrKeys(self)) + + def __len__(self): + return sum(1 for _ in bytes_iter(self.to_bytes())) + + def __repr__(self): + return "" % str(self) + + def __hash__(self): + return self._bytes.__hash__() + + def to_bytes(self): + """Returns the byte array representation of this Multiaddr.""" + return self._bytes + + __bytes__ = to_bytes + + def protocols(self): + """Returns a list of Protocols this Multiaddr includes.""" + return MultiAddrKeys(self) + + def split(self, maxsplit=-1): + """Returns the list of individual path components this MultiAddr is made + up of.""" + final_split_offset = -1 + results = [] + for idx, (offset, proto, codec, part_value) in enumerate(bytes_iter(self._bytes)): + # Split at most `maxplit` times + if idx == maxsplit: + final_split_offset = offset + break + + # Re-assemble binary MultiAddr representation + part_size = varint.encode(len(part_value)) if codec.SIZE < 0 else b"" + part = b"".join((proto.vcode, part_size, part_value)) + + # Add MultiAddr with the given value + results.append(self.__class__(part)) + # Add final item with remainder of MultiAddr if there is anything left + if final_split_offset >= 0: + results.append(self.__class__(self._bytes[final_split_offset:])) + + return results + + keys = protocols + + def items(self): + return MultiAddrItems(self) + + def values(self): + return MultiAddrValues(self) + + def encapsulate(self, other): + """Wrap this Multiaddr around another. + + For example: + /ip4/1.2.3.4 encapsulate /tcp/80 = /ip4/1.2.3.4/tcp/80 + """ + return self.join(self, other) + + def decapsulate(self, other): + """Remove a Multiaddr wrapping. + + For example: + /ip4/1.2.3.4/tcp/80 decapsulate /ip4/1.2.3.4 = /tcp/80 + """ + s1 = self.to_bytes() + s2 = Multiaddr(other).to_bytes() + try: + idx = s1.rindex(s2) + except ValueError: + # if multiaddr not contained, returns a copy + return Multiaddr(self) + return Multiaddr(s1[:idx]) + + def value_for_protocol(self, proto): + """Return the value (if any) following the specified protocol + + Returns + ------- + Union[object, NoneType] + The parsed protocol value for the given protocol code or ``None`` + if the given protocol does not require any value + + Raises + ------ + ~multiaddr.exceptions.BinaryParseError + The stored MultiAddr binary representation is invalid + ~multiaddr.exceptions.ProtocolLookupError + MultiAddr does not contain any instance of this protocol + """ + proto = self.registry.find(proto) + for proto2, value in self.items(): + if proto2 is proto or proto2 == proto: + return value + raise exceptions.ProtocolLookupError(proto, str(self)) + + __getitem__ = value_for_protocol diff --git a/hivemind/p2p/multiaddr/protocols.py b/hivemind/p2p/multiaddr/protocols.py new file mode 100644 index 000000000..ba385ba69 --- /dev/null +++ b/hivemind/p2p/multiaddr/protocols.py @@ -0,0 +1,303 @@ +import varint + +from . import exceptions +from .codecs import codec_by_name + +__all__ = ("Protocol", "PROTOCOLS", "REGISTRY") + + +# source of protocols https://github.com/multiformats/multicodec/blob/master/table.csv#L382 +# replicating table here to: +# 1. avoid parsing the csv +# 2. ensuring errors in the csv don't screw up code. +# 3. changing a number has to happen in two places. +P_IP4 = 0x04 +P_IP6 = 0x29 +P_IP6ZONE = 0x2A +P_TCP = 0x06 +P_UDP = 0x0111 +P_DCCP = 0x21 +P_SCTP = 0x84 +P_UDT = 0x012D +P_UTP = 0x012E +P_P2P = 0x01A5 +P_HTTP = 0x01E0 +P_HTTPS = 0x01BB +P_TLS = 0x01C0 +P_QUIC = 0x01CC +P_QUIC1 = 0x01CD +P_WS = 0x01DD +P_WSS = 0x01DE +P_ONION = 0x01BC +P_ONION3 = 0x01BD +P_P2P_CIRCUIT = 0x0122 +P_DNS = 0x35 +P_DNS4 = 0x36 +P_DNS6 = 0x37 +P_DNSADDR = 0x38 +P_P2P_WEBSOCKET_STAR = 0x01DF +P_P2P_WEBRTC_STAR = 0x0113 +P_P2P_WEBRTC_DIRECT = 0x0114 +P_UNIX = 0x0190 + + +class Protocol: + __slots__ = [ + "code", # int + "name", # string + "codec", # string + ] + + def __init__(self, code, name, codec): + if not isinstance(code, int): + raise TypeError("code must be an integer") + if not isinstance(name, str): + raise TypeError("name must be a string") + if not isinstance(codec, str) and codec is not None: + raise TypeError("codec must be a string or None") + + self.code = code + self.name = name + self.codec = codec + + @property + def size(self): + return codec_by_name(self.codec).SIZE + + @property + def path(self): + return codec_by_name(self.codec).IS_PATH + + @property + def vcode(self): + return varint.encode(self.code) + + def __eq__(self, other): + if not isinstance(other, Protocol): + return NotImplemented + + return all((self.code == other.code, + self.name == other.name, + self.codec == other.codec, + self.path == other.path)) + + def __hash__(self): + return self.code + + def __repr__(self): + return "Protocol(code={code!r}, name={name!r}, codec={codec!r})".format( + code=self.code, + name=self.name, + codec=self.codec, + ) + + +# List of multiaddr protocols supported by this module by default +PROTOCOLS = [ + Protocol(P_IP4, 'ip4', 'ip4'), + Protocol(P_TCP, 'tcp', 'uint16be'), + Protocol(P_UDP, 'udp', 'uint16be'), + Protocol(P_DCCP, 'dccp', 'uint16be'), + Protocol(P_IP6, 'ip6', 'ip6'), + Protocol(P_IP6ZONE, 'ip6zone', 'utf8'), + Protocol(P_DNS, 'dns', 'domain'), + Protocol(P_DNS4, 'dns4', 'domain'), + Protocol(P_DNS6, 'dns6', 'domain'), + Protocol(P_DNSADDR, 'dnsaddr', 'domain'), + Protocol(P_SCTP, 'sctp', 'uint16be'), + Protocol(P_UDT, 'udt', None), + Protocol(P_UTP, 'utp', None), + Protocol(P_P2P, 'p2p', 'cid'), + Protocol(P_ONION, 'onion', 'onion'), + Protocol(P_ONION3, 'onion3', 'onion3'), + Protocol(P_QUIC, 'quic', None), + Protocol(P_QUIC1, 'quic-v1', None), + Protocol(P_HTTP, 'http', None), + Protocol(P_HTTPS, 'https', None), + Protocol(P_TLS, 'tls', None), + Protocol(P_WS, 'ws', None), + Protocol(P_WSS, 'wss', None), + Protocol(P_P2P_WEBSOCKET_STAR, 'p2p-websocket-star', None), + Protocol(P_P2P_WEBRTC_STAR, 'p2p-webrtc-star', None), + Protocol(P_P2P_WEBRTC_DIRECT, 'p2p-webrtc-direct', None), + Protocol(P_P2P_CIRCUIT, 'p2p-circuit', None), + Protocol(P_UNIX, 'unix', 'fspath'), +] + + +class ProtocolRegistry: + """A collection of individual Multiaddr protocols indexed for fast lookup""" + __slots__ = ("_codes_to_protocols", "_locked", "_names_to_protocols") + + def __init__(self, protocols=()): + self._locked = False + self._codes_to_protocols = {proto.code: proto for proto in protocols} + self._names_to_protocols = {proto.name: proto for proto in protocols} + + def add(self, proto): + """Add the given protocol description to this registry + + Raises + ------ + ~multiaddr.exceptions.ProtocolRegistryLocked + Protocol registry is locked and does not accept any new entries. + + You can use `.copy(unlock=True)` to copy an existing locked registry + and unlock it. + ~multiaddr.exceptions.ProtocolExistsError + A protocol with the given name or code already exists. + """ + if self._locked: + raise exceptions.ProtocolRegistryLocked() + + if proto.name in self._names_to_protocols: + raise exceptions.ProtocolExistsError(proto, "name") + + if proto.code in self._codes_to_protocols: + raise exceptions.ProtocolExistsError(proto, "code") + + self._names_to_protocols[proto.name] = proto + self._codes_to_protocols[proto.code] = proto + return proto + + def add_alias_name(self, proto, alias_name): + """Add an alternate name for an existing protocol description to the registry + + Raises + ------ + ~multiaddr.exceptions.ProtocolRegistryLocked + Protocol registry is locked and does not accept any new entries. + + You can use `.copy(unlock=True)` to copy an existing locked registry + and unlock it. + ~multiaddr.exceptions.ProtocolExistsError + A protocol with the given name already exists. + ~multiaddr.exceptions.ProtocolNotFoundError + No protocol matching *proto* could be found. + """ + if self._locked: + raise exceptions.ProtocolRegistryLocked() + + proto = self.find(proto) + assert self._names_to_protocols.get(proto.name) is proto, \ + "Protocol to alias must have already been added to the registry" + + if alias_name in self._names_to_protocols: + raise exceptions.ProtocolExistsError(self._names_to_protocols[alias_name], "name") + + self._names_to_protocols[alias_name] = proto + + def add_alias_code(self, proto, alias_code): + """Add an alternate code for an existing protocol description to the registry + + Raises + ------ + ~multiaddr.exceptions.ProtocolRegistryLocked + Protocol registry is locked and does not accept any new entries. + + You can use `.copy(unlock=True)` to copy an existing locked registry + and unlock it. + ~multiaddr.exceptions.ProtocolExistsError + A protocol with the given code already exists. + ~multiaddr.exceptions.ProtocolNotFoundError + No protocol matching *proto* could be found. + """ + if self._locked: + raise exceptions.ProtocolRegistryLocked() + + proto = self.find(proto) + assert self._codes_to_protocols.get(proto.code) is proto, \ + "Protocol to alias must have already been added to the registry" + + if alias_code in self._codes_to_protocols: + raise exceptions.ProtocolExistsError(self._codes_to_protocols[alias_code], "name") + + self._codes_to_protocols[alias_code] = proto + + def lock(self): + """Lock this registry instance to deny any further changes""" + self._locked = True + + @property + def locked(self): + return self._locked + + def copy(self, *, unlock=False): + """Create a copy of this protocol registry + + Arguments + --------- + unlock + Create the copied registry unlocked even if the current one is locked? + """ + registry = ProtocolRegistry() + registry._locked = self._locked and not unlock + registry._codes_to_protocols = self._codes_to_protocols.copy() + registry._names_to_protocols = self._names_to_protocols.copy() + return registry + + __copy__ = copy + + def find_by_name(self, name): + """Look up a protocol by its human-readable name + + Raises + ------ + ~multiaddr.exceptions.ProtocolNotFoundError + """ + if name not in self._names_to_protocols: + raise exceptions.ProtocolNotFoundError(name, "name") + return self._names_to_protocols[name] + + def find_by_code(self, code): + """Look up a protocol by its binary representation code + + Raises + ------ + ~multiaddr.exceptions.ProtocolNotFoundError + """ + if code not in self._codes_to_protocols: + raise exceptions.ProtocolNotFoundError(code, "code") + return self._codes_to_protocols[code] + + def find(self, proto): + """Look up a protocol by its name or code, return existing protocol objects unchanged + + Raises + ------ + ~multiaddr.exceptions.ProtocolNotFoundError + """ + if isinstance(proto, Protocol): + return proto + elif isinstance(proto, str): + return self.find_by_name(proto) + elif isinstance(proto, int): + return self.find_by_code(proto) + else: + raise TypeError("Protocol object, name or code expected, got {0!r}".format(proto)) + + +REGISTRY = ProtocolRegistry(PROTOCOLS) +REGISTRY.add_alias_name("p2p", "ipfs") +REGISTRY.lock() + + +def protocol_with_name(name): + return REGISTRY.find_by_name(name) + + +def protocol_with_code(code): + return REGISTRY.find_by_code(code) + + +def protocol_with_any(proto): + return REGISTRY.find(proto) + + +def protocols_with_string(string): + """Return a list of protocols matching given string.""" + ret = [] + for name in string.split("/"): + if len(name) > 0: + ret.append(protocol_with_name(name)) + return ret diff --git a/hivemind/p2p/multiaddr/transforms.py b/hivemind/p2p/multiaddr/transforms.py new file mode 100644 index 000000000..951c824c6 --- /dev/null +++ b/hivemind/p2p/multiaddr/transforms.py @@ -0,0 +1,96 @@ +import io +import varint + +from . import exceptions + +from .codecs import LENGTH_PREFIXED_VAR_SIZE +from .codecs import codec_by_name + +from .protocols import protocol_with_code +from .protocols import protocol_with_name + + +def string_to_bytes(string): + bs = [] + for proto, codec, value in string_iter(string): + bs.append(varint.encode(proto.code)) + if value is not None: + try: + buf = codec.to_bytes(proto, value) + except Exception as exc: + raise exceptions.StringParseError(str(exc), string, proto.name, exc) from exc + if codec.SIZE == LENGTH_PREFIXED_VAR_SIZE: + bs.append(varint.encode(len(buf))) + bs.append(buf) + return b''.join(bs) + + +def bytes_to_string(buf): + st = [''] # start with empty string so we get a leading slash on join() + for _, proto, codec, part in bytes_iter(buf): + st.append(proto.name) + if codec.SIZE != 0: + try: + value = codec.to_string(proto, part) + except Exception as exc: + raise exceptions.BinaryParseError(str(exc), buf, proto.name, exc) from exc + if codec.IS_PATH and value[0] == '/': + st.append(value[1:]) + else: + st.append(value) + return '/'.join(st) + + +def size_for_addr(codec, buf_io): + if codec.SIZE >= 0: + return codec.SIZE // 8 + else: + return varint.decode_stream(buf_io) + + +def string_iter(string): + if not string.startswith('/'): + raise exceptions.StringParseError("Must begin with /", string) + # consume trailing slashes + string = string.rstrip('/') + sp = string.split('/') + + # skip the first element, since it starts with / + sp.pop(0) + while sp: + element = sp.pop(0) + try: + proto = protocol_with_name(element) + codec = codec_by_name(proto.codec) + except (ImportError, exceptions.ProtocolNotFoundError) as exc: + raise exceptions.StringParseError("Unknown Protocol", string, element) from exc + value = None + if codec.SIZE != 0: + if len(sp) < 1: + raise exceptions.StringParseError("Protocol requires address", string, proto.name) + if codec.IS_PATH: + value = "/" + "/".join(sp) + sp.clear() + else: + value = sp.pop(0) + yield proto, codec, value + + +def bytes_iter(buf): + buf_io = io.BytesIO(buf) + while buf_io.tell() < len(buf): + offset = buf_io.tell() + code = varint.decode_stream(buf_io) + proto = None + try: + proto = protocol_with_code(code) + codec = codec_by_name(proto.codec) + except (ImportError, exceptions.ProtocolNotFoundError) as exc: + raise exceptions.BinaryParseError( + "Unknown Protocol", + buf, + proto.name if proto else code, + ) from exc + + size = size_for_addr(codec, buf_io) + yield offset, proto, codec, buf_io.read(size) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 4acea4d72..0db4714aa 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -12,13 +12,13 @@ from typing import Any, AsyncIterator, Awaitable, Callable, List, Optional, Sequence, Tuple, Type, TypeVar, Union from google.protobuf.message import Message -from multiaddr import Multiaddr import hivemind.hivemind_cli as cli import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.control import DEFAULT_MAX_MSG_SIZE, P2PDaemonError, P2PHandlerError from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure +from hivemind.p2p.multiaddr import Multiaddr from hivemind.proto import crypto_pb2 from hivemind.proto.p2pd_pb2 import RPCError from hivemind.utils.asyncio import as_aiter, asingle From 372fbe86ea24bb6983d26c960cb9c3e748ca2ff8 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 16:07:14 +0300 Subject: [PATCH 02/12] fix imports --- hivemind/p2p/p2p_daemon_bindings/control.py | 3 +-- hivemind/p2p/p2p_daemon_bindings/datastructures.py | 1 - hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 2 +- hivemind/utils/networking.py | 3 +-- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index 4f229bbdb..a93b3394a 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -9,9 +9,8 @@ from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Optional, Sequence, Tuple from uuid import UUID, uuid4 -from multiaddr import Multiaddr, protocols - from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo +from hivemind.p2p.multiaddr import Multiaddr from hivemind.p2p.p2p_daemon_bindings.utils import ( DispatchFailure, P2PDaemonError, diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 920aa920b..2998bc4fb 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -10,7 +10,6 @@ import base58 import multihash from cryptography.hazmat.primitives import serialization -from multiaddr import Multiaddr, protocols from hivemind.proto import crypto_pb2, p2pd_pb2 diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index 7fee00477..a78050a2d 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -8,8 +8,8 @@ from contextlib import asynccontextmanager from typing import AsyncIterator, Iterable, Sequence, Tuple -from multiaddr import Multiaddr +from hivemind.p2p.multiaddr import Multiaddr from hivemind.p2p.p2p_daemon_bindings.control import ( DEFAULT_MAX_MSG_SIZE, ControlClient, diff --git a/hivemind/utils/networking.py b/hivemind/utils/networking.py index dc9a60440..228e6c596 100644 --- a/hivemind/utils/networking.py +++ b/hivemind/utils/networking.py @@ -1,9 +1,8 @@ from ipaddress import ip_address from typing import List, Sequence -from multiaddr import Multiaddr - from hivemind.utils.logging import TextStyle, get_logger +from hivemind.p2p.multiaddr import Multiaddr LOCALHOST = "127.0.0.1" From b410f8453f929473d27fb8c9d6f1ff9b4fc9df54 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 16:14:41 +0300 Subject: [PATCH 03/12] fix import --- hivemind/p2p/p2p_daemon_bindings/datastructures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 2998bc4fb..03e10f51d 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -12,6 +12,7 @@ from cryptography.hazmat.primitives import serialization from hivemind.proto import crypto_pb2, p2pd_pb2 +from hivemind.p2p.multiaddr import Multiaddr class PeerID: From 6e8dfb46e98563c28915606999574567048a0109 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 16:19:18 +0300 Subject: [PATCH 04/12] fix yet another import --- hivemind/p2p/p2p_daemon_bindings/control.py | 2 +- hivemind/p2p/p2p_daemon_bindings/datastructures.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index a93b3394a..39fc9692d 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -10,7 +10,7 @@ from uuid import UUID, uuid4 from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo -from hivemind.p2p.multiaddr import Multiaddr +from hivemind.p2p.multiaddr import Multiaddr, protocols from hivemind.p2p.p2p_daemon_bindings.utils import ( DispatchFailure, P2PDaemonError, diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 03e10f51d..6813ce4e3 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -12,7 +12,7 @@ from cryptography.hazmat.primitives import serialization from hivemind.proto import crypto_pb2, p2pd_pb2 -from hivemind.p2p.multiaddr import Multiaddr +from hivemind.p2p.multiaddr import Multiaddr, protocols class PeerID: From 2d6f8836017d20b3fcb7b4d81da5765f06c49bcd Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 16:41:57 +0300 Subject: [PATCH 05/12] fix import inside multiaddr --- hivemind/p2p/multiaddr/codecs/cid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hivemind/p2p/multiaddr/codecs/cid.py b/hivemind/p2p/multiaddr/codecs/cid.py index fbef5fe30..260ebfbdc 100644 --- a/hivemind/p2p/multiaddr/codecs/cid.py +++ b/hivemind/p2p/multiaddr/codecs/cid.py @@ -1,5 +1,5 @@ import base58 -# import cid +import cid from . import LENGTH_PREFIXED_VAR_SIZE From 1b7ffd9282da4a861e5a3557ed0e986b1a455919 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 17:11:02 +0300 Subject: [PATCH 06/12] fix imports in tests --- tests/test_dht.py | 3 ++- tests/test_dht_protocol.py | 2 +- tests/test_p2p_daemon.py | 2 +- tests/test_p2p_daemon_bindings.py | 2 +- tests/test_utils/dht_swarms.py | 2 +- tests/test_utils/p2p_daemon.py | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_dht.py b/tests/test_dht.py index 63dee8983..0b80911de 100644 --- a/tests/test_dht.py +++ b/tests/test_dht.py @@ -4,13 +4,14 @@ import time import pytest -from multiaddr import Multiaddr import hivemind from test_utils.dht_swarms import launch_dht_instances from test_utils.networking import get_free_port +from hivemind.p2p.multiaddr import Multiaddr + @pytest.mark.asyncio async def test_startup_error(): diff --git a/tests/test_dht_protocol.py b/tests/test_dht_protocol.py index 908f8f3e5..d8567013e 100644 --- a/tests/test_dht_protocol.py +++ b/tests/test_dht_protocol.py @@ -5,13 +5,13 @@ from typing import List, Sequence, Tuple import pytest -from multiaddr import Multiaddr import hivemind from hivemind import P2P, PeerID, get_dht_time, get_logger from hivemind.dht import DHTID from hivemind.dht.protocol import DHTProtocol from hivemind.dht.storage import DictionaryDHTValue +from hivemind.p2p.multiaddr import Multiaddr logger = get_logger(__name__) diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 55ff36af5..eaa0b469d 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -9,9 +9,9 @@ import numpy as np import pytest -from multiaddr import Multiaddr from hivemind.p2p import P2P, P2PDaemonError, P2PHandlerError +from hivemind.p2p.multiaddr import Multiaddr from hivemind.proto import dht_pb2, test_pb2 from hivemind.utils.serializer import MSGPackSerializer diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 71658f2ef..006114962 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -4,8 +4,8 @@ import pytest from google.protobuf.message import EncodeError -from multiaddr import Multiaddr, protocols +from hivemind.p2p.multiaddr import Multiaddr, protocols from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, parse_conn_protocol from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo from hivemind.p2p.p2p_daemon_bindings.utils import ( diff --git a/tests/test_utils/dht_swarms.py b/tests/test_utils/dht_swarms.py index 732c5b294..97ca7482c 100644 --- a/tests/test_utils/dht_swarms.py +++ b/tests/test_utils/dht_swarms.py @@ -5,11 +5,11 @@ import threading from typing import Dict, List, Tuple -from multiaddr import Multiaddr from hivemind.dht import DHT from hivemind.dht.node import DHTID, DHTNode from hivemind.p2p import PeerID +from hivemind.p2p.multiaddr import Multiaddr def run_node(initial_peers: List[Multiaddr], info_queue: mp.Queue, **kwargs): diff --git a/tests/test_utils/p2p_daemon.py b/tests/test_utils/p2p_daemon.py index bc889bd30..c929e15df 100644 --- a/tests/test_utils/p2p_daemon.py +++ b/tests/test_utils/p2p_daemon.py @@ -7,10 +7,10 @@ from contextlib import asynccontextmanager, suppress from typing import NamedTuple -from multiaddr import Multiaddr, protocols from pkg_resources import resource_filename from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client +from hivemind.p2p.multiaddr import Multiaddr from test_utils.networking import get_free_port From 7567d3de6cee2e436467e1b94b0d9a8d64760ff3 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 17:18:57 +0300 Subject: [PATCH 07/12] yet another fix --- tests/test_utils/p2p_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/p2p_daemon.py b/tests/test_utils/p2p_daemon.py index c929e15df..d48d1ff40 100644 --- a/tests/test_utils/p2p_daemon.py +++ b/tests/test_utils/p2p_daemon.py @@ -10,7 +10,7 @@ from pkg_resources import resource_filename from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client -from hivemind.p2p.multiaddr import Multiaddr +from hivemind.p2p.multiaddr import Multiaddr, protocols from test_utils.networking import get_free_port From 227d62831ee6e67c8582c53c2bb36c4bbc335da9 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 18:00:15 +0300 Subject: [PATCH 08/12] tests, isort, black --- hivemind/dht/dht.py | 2 +- hivemind/p2p/multiaddr/codecs/__init__.py | 1 - hivemind/p2p/multiaddr/codecs/cid.py | 1 - hivemind/p2p/multiaddr/codecs/domain.py | 1 - hivemind/p2p/multiaddr/codecs/fspath.py | 1 - hivemind/p2p/multiaddr/codecs/ip4.py | 1 - hivemind/p2p/multiaddr/codecs/ip6.py | 1 - hivemind/p2p/multiaddr/codecs/onion.py | 1 - hivemind/p2p/multiaddr/codecs/onion3.py | 1 - hivemind/p2p/multiaddr/codecs/uint16be.py | 1 - hivemind/p2p/multiaddr/codecs/utf8.py | 1 - hivemind/p2p/multiaddr/multiaddr.py | 6 +- hivemind/p2p/multiaddr/transforms.py | 9 +- hivemind/p2p/p2p_daemon.py | 2 +- hivemind/p2p/p2p_daemon_bindings/control.py | 2 +- .../p2p/p2p_daemon_bindings/datastructures.py | 2 +- hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 1 - hivemind/utils/networking.py | 2 +- tests/test_dht.py | 3 +- tests/test_multiaddr.py | 379 ++++++++++++++++++ tests/test_protocols.py | 182 +++++++++ tests/test_transforms.py | 251 ++++++++++++ tests/test_utils/dht_swarms.py | 1 - tests/test_utils/p2p_daemon.py | 2 +- 24 files changed, 823 insertions(+), 31 deletions(-) create mode 100644 tests/test_multiaddr.py create mode 100644 tests/test_protocols.py create mode 100644 tests/test_transforms.py diff --git a/hivemind/dht/dht.py b/hivemind/dht/dht.py index 67aa8f979..24a5d7fa6 100644 --- a/hivemind/dht/dht.py +++ b/hivemind/dht/dht.py @@ -8,10 +8,10 @@ from typing import Awaitable, Callable, Iterable, List, Optional, Sequence, TypeVar, Union from hivemind.dht.node import DEFAULT_NUM_WORKERS, DHTNode -from hivemind.p2p.multiaddr import Multiaddr from hivemind.dht.routing import DHTKey, DHTValue, Subkey from hivemind.dht.validation import CompositeValidator, RecordValidatorBase from hivemind.p2p import P2P, PeerID +from hivemind.p2p.multiaddr import Multiaddr from hivemind.utils import MPFuture, get_logger, switch_to_uvloop from hivemind.utils.timed_storage import DHTExpiration, ValueWithExpiration diff --git a/hivemind/p2p/multiaddr/codecs/__init__.py b/hivemind/p2p/multiaddr/codecs/__init__.py index d30127931..ce76d6ea2 100644 --- a/hivemind/p2p/multiaddr/codecs/__init__.py +++ b/hivemind/p2p/multiaddr/codecs/__init__.py @@ -1,6 +1,5 @@ import importlib - # These are special sizes LENGTH_PREFIXED_VAR_SIZE = -1 diff --git a/hivemind/p2p/multiaddr/codecs/cid.py b/hivemind/p2p/multiaddr/codecs/cid.py index 260ebfbdc..985c25b6f 100644 --- a/hivemind/p2p/multiaddr/codecs/cid.py +++ b/hivemind/p2p/multiaddr/codecs/cid.py @@ -3,7 +3,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/domain.py b/hivemind/p2p/multiaddr/codecs/domain.py index 7d616366d..129ad6888 100644 --- a/hivemind/p2p/multiaddr/codecs/domain.py +++ b/hivemind/p2p/multiaddr/codecs/domain.py @@ -2,7 +2,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/fspath.py b/hivemind/p2p/multiaddr/codecs/fspath.py index 36b6859a4..a93f5f6dc 100644 --- a/hivemind/p2p/multiaddr/codecs/fspath.py +++ b/hivemind/p2p/multiaddr/codecs/fspath.py @@ -2,7 +2,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = True diff --git a/hivemind/p2p/multiaddr/codecs/ip4.py b/hivemind/p2p/multiaddr/codecs/ip4.py index 1a7f64556..4384d17d6 100644 --- a/hivemind/p2p/multiaddr/codecs/ip4.py +++ b/hivemind/p2p/multiaddr/codecs/ip4.py @@ -1,6 +1,5 @@ import netaddr - SIZE = 32 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/ip6.py b/hivemind/p2p/multiaddr/codecs/ip6.py index 7ae71b0d0..6b87e5237 100644 --- a/hivemind/p2p/multiaddr/codecs/ip6.py +++ b/hivemind/p2p/multiaddr/codecs/ip6.py @@ -1,6 +1,5 @@ import netaddr - SIZE = 128 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/onion.py b/hivemind/p2p/multiaddr/codecs/onion.py index 55c2078d3..e864190b8 100644 --- a/hivemind/p2p/multiaddr/codecs/onion.py +++ b/hivemind/p2p/multiaddr/codecs/onion.py @@ -1,7 +1,6 @@ import base64 import struct - SIZE = 96 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/onion3.py b/hivemind/p2p/multiaddr/codecs/onion3.py index d12bae588..827c1a185 100644 --- a/hivemind/p2p/multiaddr/codecs/onion3.py +++ b/hivemind/p2p/multiaddr/codecs/onion3.py @@ -1,7 +1,6 @@ import base64 import struct - SIZE = 296 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/uint16be.py b/hivemind/p2p/multiaddr/codecs/uint16be.py index 73442e430..3a7f05bf7 100644 --- a/hivemind/p2p/multiaddr/codecs/uint16be.py +++ b/hivemind/p2p/multiaddr/codecs/uint16be.py @@ -1,6 +1,5 @@ import struct - SIZE = 16 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/utf8.py b/hivemind/p2p/multiaddr/codecs/utf8.py index 5a85e2793..e4edcf2b0 100644 --- a/hivemind/p2p/multiaddr/codecs/utf8.py +++ b/hivemind/p2p/multiaddr/codecs/utf8.py @@ -2,7 +2,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/multiaddr.py b/hivemind/p2p/multiaddr/multiaddr.py index 52f51a055..8a74ea52f 100644 --- a/hivemind/p2p/multiaddr/multiaddr.py +++ b/hivemind/p2p/multiaddr/multiaddr.py @@ -3,11 +3,7 @@ import varint from . import exceptions, protocols - -from .transforms import bytes_iter -from .transforms import string_to_bytes -from .transforms import bytes_to_string - +from .transforms import bytes_iter, bytes_to_string, string_to_bytes __all__ = ("Multiaddr",) diff --git a/hivemind/p2p/multiaddr/transforms.py b/hivemind/p2p/multiaddr/transforms.py index 951c824c6..d46ab9a16 100644 --- a/hivemind/p2p/multiaddr/transforms.py +++ b/hivemind/p2p/multiaddr/transforms.py @@ -1,13 +1,10 @@ import io + import varint from . import exceptions - -from .codecs import LENGTH_PREFIXED_VAR_SIZE -from .codecs import codec_by_name - -from .protocols import protocol_with_code -from .protocols import protocol_with_name +from .codecs import LENGTH_PREFIXED_VAR_SIZE, codec_by_name +from .protocols import protocol_with_code, protocol_with_name def string_to_bytes(string): diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 0db4714aa..66a87ae09 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -15,10 +15,10 @@ import hivemind.hivemind_cli as cli import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient +from hivemind.p2p.multiaddr import Multiaddr from hivemind.p2p.p2p_daemon_bindings.control import DEFAULT_MAX_MSG_SIZE, P2PDaemonError, P2PHandlerError from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure -from hivemind.p2p.multiaddr import Multiaddr from hivemind.proto import crypto_pb2 from hivemind.proto.p2pd_pb2 import RPCError from hivemind.utils.asyncio import as_aiter, asingle diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index 01ef012a8..fa5239d72 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -9,8 +9,8 @@ from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Optional, Sequence, Tuple from uuid import UUID, uuid4 -from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo from hivemind.p2p.multiaddr import Multiaddr, protocols +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo from hivemind.p2p.p2p_daemon_bindings.utils import ( DispatchFailure, P2PDaemonError, diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 6813ce4e3..6ad7b054d 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -11,8 +11,8 @@ import multihash from cryptography.hazmat.primitives import serialization -from hivemind.proto import crypto_pb2, p2pd_pb2 from hivemind.p2p.multiaddr import Multiaddr, protocols +from hivemind.proto import crypto_pb2, p2pd_pb2 class PeerID: diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index a78050a2d..677101be9 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -8,7 +8,6 @@ from contextlib import asynccontextmanager from typing import AsyncIterator, Iterable, Sequence, Tuple - from hivemind.p2p.multiaddr import Multiaddr from hivemind.p2p.p2p_daemon_bindings.control import ( DEFAULT_MAX_MSG_SIZE, diff --git a/hivemind/utils/networking.py b/hivemind/utils/networking.py index 228e6c596..a348404d9 100644 --- a/hivemind/utils/networking.py +++ b/hivemind/utils/networking.py @@ -1,8 +1,8 @@ from ipaddress import ip_address from typing import List, Sequence -from hivemind.utils.logging import TextStyle, get_logger from hivemind.p2p.multiaddr import Multiaddr +from hivemind.utils.logging import TextStyle, get_logger LOCALHOST = "127.0.0.1" diff --git a/tests/test_dht.py b/tests/test_dht.py index 0b80911de..c980adc53 100644 --- a/tests/test_dht.py +++ b/tests/test_dht.py @@ -6,12 +6,11 @@ import pytest import hivemind +from hivemind.p2p.multiaddr import Multiaddr from test_utils.dht_swarms import launch_dht_instances from test_utils.networking import get_free_port -from hivemind.p2p.multiaddr import Multiaddr - @pytest.mark.asyncio async def test_startup_error(): diff --git a/tests/test_multiaddr.py b/tests/test_multiaddr.py new file mode 100644 index 000000000..63a72d263 --- /dev/null +++ b/tests/test_multiaddr.py @@ -0,0 +1,379 @@ +import pytest + +from hivemind.p2p.multiaddr.exceptions import ( + BinaryParseError, + ProtocolLookupError, + ProtocolNotFoundError, + StringParseError, +) +from hivemind.p2p.multiaddr.multiaddr import Multiaddr +from hivemind.p2p.multiaddr.protocols import ( + P_DNS, + P_IP4, + P_IP6, + P_P2P, + P_TCP, + P_UDP, + P_UNIX, + P_UTP, + protocol_with_name, + protocols_with_string, +) + + +@pytest.mark.parametrize( + "addr_str", + [ + "/ip4", + "/ip4/::1", + "/ip4/fdpsofodsajfdoisa", + "/ip6", + "/ip6zone", + "/ip6zone/", + "/ip6zone//ip6/fe80::1", + "/udp", + "/tcp", + "/sctp", + "/udp/65536", + "/tcp/65536", + "/onion/9imaq4ygg2iegci7:80", + "/onion/aaimaq4ygg2iegci7:80", + "/onion/timaq4ygg2iegci7:0", + "/onion/timaq4ygg2iegci7:-1", + "/onion/timaq4ygg2iegci7", + "/onion/timaq4ygg2iegci@:666", + "/onion3/9ww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd7:80", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:0", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:a", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:-1", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy@:666", + "/udp/1234/sctp", + "/udp/1234/udt/1234", + "/udp/1234/utp/1234", + "/ip4/127.0.0.1/udp/jfodsajfidosajfoidsa", + "/ip4/127.0.0.1/udp", + "/ip4/127.0.0.1/tcp/jfodsajfidosajfoidsa", + "/ip4/127.0.0.1/tcp", + "/ip4/127.0.0.1/p2p", + "/ip4/127.0.0.1/p2p/tcp", + "/unix", + "/ip4/1.2.3.4/tcp/80/unix", + "/ip4/127.0.0.1/tcp/9090/http/p2p-webcrt-direct", + "/dns", + "/dns4", + "/dns6", + "/cancer", + ], +) +def test_invalid(addr_str): + with pytest.raises(StringParseError): + Multiaddr(addr_str) + + +@pytest.mark.parametrize( + "addr_str", + [ + "/ip4/1.2.3.4", + "/ip4/0.0.0.0", + "/ip6/::1", + "/ip6/2601:9:4f81:9700:803e:ca65:66e8:c21", + "/ip6zone/x/ip6/fe80::1", + "/ip6zone/x%y/ip6/fe80::1", + "/ip6zone/x%y/ip6/::", + "/onion/timaq4ygg2iegci7:1234", + "/onion/timaq4ygg2iegci7:80/http", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:1234", + "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80/http", + "/udp/0", + "/tcp/0", + "/sctp/0", + "/udp/1234", + "/tcp/1234", + "/sctp/1234", + "/utp", + "/udp/65535", + "/tcp/65535", + "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC", + "/udp/1234/sctp/1234", + "/udp/1234/udt", + "/udp/1234/utp", + "/tcp/1234/http", + "/tcp/1234/https", + "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", + "/ip4/127.0.0.1/udp/1234", + "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", + "/unix/a/b/c/d/e", + "/unix/Überrschung!/大柱", + "/unix/stdio", + "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f", + "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC" "/tcp/1234/unix/stdio", + "/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct", + "/dns/example.com", + "/dns4/موقع.وزارة-الاتصالات.مصر", + ], +) # nopep8 +def test_valid(addr_str): + ma = Multiaddr(addr_str) + assert str(ma) == addr_str.rstrip("/") + + +def test_eq(): + m1 = Multiaddr("/ip4/127.0.0.1/udp/1234") + m2 = Multiaddr("/ip4/127.0.0.1/tcp/1234") + m3 = Multiaddr("/ip4/127.0.0.1/tcp/1234") + m4 = Multiaddr("/ip4/127.0.0.1/tcp/1234/") + + assert m1 != m2 + assert m2 != m1 + + assert m2 == m3 + assert m3 == m2 + + assert m1 == m1 + + assert m2 == m4 + assert m4 == m2 + assert m3 == m4 + assert m4 == m3 + + +def test_protocols(): + ma = Multiaddr("/ip4/127.0.0.1/udp/1234") + protos = ma.protocols() + assert protos[0].code == protocol_with_name("ip4").code + assert protos[1].code == protocol_with_name("udp").code + + +@pytest.mark.parametrize( + "proto_string,expected", + [ + ("/ip4", [protocol_with_name("ip4")]), + ("/ip4/tcp", [protocol_with_name("ip4"), protocol_with_name("tcp")]), + ( + "ip4/tcp/udp/ip6", + [ + protocol_with_name("ip4"), + protocol_with_name("tcp"), + protocol_with_name("udp"), + protocol_with_name("ip6"), + ], + ), + ("////////ip4/tcp", [protocol_with_name("ip4"), protocol_with_name("tcp")]), + ("ip4/udp/////////", [protocol_with_name("ip4"), protocol_with_name("udp")]), + ("////////ip4/tcp////////", [protocol_with_name("ip4"), protocol_with_name("tcp")]), + ("////////ip4/////////tcp////////", [protocol_with_name("ip4"), protocol_with_name("tcp")]), + ], +) +def test_protocols_with_string(proto_string, expected): + protos = protocols_with_string(proto_string) + assert protos == expected + + +@pytest.mark.parametrize("proto_string", ["dsijafd", "/ip4/tcp/fidosafoidsa", "////////ip4/tcp/21432141/////////"]) +def test_invalid_protocols_with_string(proto_string): + with pytest.raises(ProtocolNotFoundError): + protocols_with_string(proto_string) + + +@pytest.mark.parametrize( + "proto_string,maxsplit,expected", + [ + ("/ip4/1.2.3.4", -1, ("/ip4/1.2.3.4",)), + ("/ip4/0.0.0.0", 0, ("/ip4/0.0.0.0",)), + ("/ip6/::1", 1, ("/ip6/::1",)), + ("/onion/timaq4ygg2iegci7:80/http", 0, ("/onion/timaq4ygg2iegci7:80/http",)), + ( + "/ip4/127.0.0.1/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234", + 1, + ("/ip4/127.0.0.1", "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234"), + ), + ("/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f", -1, ("/ip4/1.2.3.4", "/tcp/80", "/unix/a/b/c/d/e/f")), + ], +) +def test_split(proto_string, maxsplit, expected): + assert tuple(map(str, Multiaddr(proto_string).split(maxsplit))) == expected + + +@pytest.mark.parametrize( + "proto_parts,expected", + [ + (("/ip4/1.2.3.4",), "/ip4/1.2.3.4"), + ((b"\x04\x00\x00\x00\x00",), "/ip4/0.0.0.0"), + (("/ip6/::1",), "/ip6/::1"), + (("/onion/timaq4ygg2iegci7:80/http",), "/onion/timaq4ygg2iegci7:80/http"), + ( + ( + b"\x04\x7F\x00\x00\x01", + "/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234", + ), + "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", + ), + (("/ip4/1.2.3.4", "/tcp/80", "/unix/a/b/c/d/e/f"), "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f"), + ], +) +def test_join(proto_parts, expected): + assert str(Multiaddr.join(*proto_parts)) == expected + + +def test_encapsulate(): + m1 = Multiaddr("/ip4/127.0.0.1/udp/1234") + m2 = Multiaddr("/udp/5678") + + encapsulated = m1.encapsulate(m2) + assert str(encapsulated) == "/ip4/127.0.0.1/udp/1234/udp/5678" + + m3 = Multiaddr("/udp/5678") + decapsulated = encapsulated.decapsulate(m3) + assert str(decapsulated) == "/ip4/127.0.0.1/udp/1234" + + m4 = Multiaddr("/ip4/127.0.0.1") + decapsulated_2 = decapsulated.decapsulate(m4) + assert str(decapsulated_2) == "" + + m5 = Multiaddr("/ip6/::1") + decapsulated_3 = decapsulated.decapsulate(m5) + + assert str(decapsulated_3) == "/ip4/127.0.0.1/udp/1234" + + +def assert_value_for_proto(multi, proto, expected): + assert multi.value_for_protocol(proto) == expected + + +def test_get_value(): + ma = Multiaddr( + "/ip4/127.0.0.1/utp/tcp/5555/udp/1234/utp/" "p2p/bafzbeigalb34xlqdtvyklzqa5ibmn6pssqsdskc4ty2e4jxy2kamquh22y" + ) + + assert_value_for_proto(ma, P_IP4, "127.0.0.1") + assert_value_for_proto(ma, P_UTP, None) + assert_value_for_proto(ma, P_TCP, "5555") + assert_value_for_proto(ma, P_UDP, "1234") + assert_value_for_proto(ma, P_P2P, "QmbHVEEepCi7rn7VL7Exxpd2Ci9NNB6ifvqwhsrbRMgQFP") + assert_value_for_proto(ma, "ip4", "127.0.0.1") + assert_value_for_proto(ma, "utp", None) + assert_value_for_proto(ma, "tcp", "5555") + assert_value_for_proto(ma, "udp", "1234") + assert_value_for_proto(ma, protocol_with_name("ip4"), "127.0.0.1") + assert_value_for_proto(ma, protocol_with_name("utp"), None) + assert_value_for_proto(ma, protocol_with_name("tcp"), "5555") + assert_value_for_proto(ma, protocol_with_name("udp"), "1234") + + with pytest.raises(ProtocolLookupError): + ma.value_for_protocol(P_IP6) + with pytest.raises(ProtocolLookupError): + ma.value_for_protocol("ip6") + with pytest.raises(ProtocolLookupError): + ma.value_for_protocol(protocol_with_name("ip6")) + + a = Multiaddr(b"\x35\x03a:b") # invalid protocol value + with pytest.raises(BinaryParseError): + a.value_for_protocol(P_DNS) + + a = Multiaddr("/ip4/0.0.0.0") # only one addr + assert_value_for_proto(a, P_IP4, "0.0.0.0") + + a = Multiaddr("/ip4/0.0.0.0/ip4/0.0.0.0/ip4/0.0.0.0") # same sub-addr + assert_value_for_proto(a, P_IP4, "0.0.0.0") + + a = Multiaddr("/ip4/0.0.0.0/udp/12345/utp") # ending in a no-value one. + assert_value_for_proto(a, P_IP4, "0.0.0.0") + assert_value_for_proto(a, P_UDP, "12345") + assert_value_for_proto(a, P_UTP, None) + + a = Multiaddr("/ip4/0.0.0.0/unix/a/b/c/d") # ending in a path one. + assert_value_for_proto(a, P_IP4, "0.0.0.0") + assert_value_for_proto(a, P_UNIX, "/a/b/c/d") + + a = Multiaddr("/unix/studio") + assert_value_for_proto(a, P_UNIX, "/studio") # only a path. + + +def test_views(): + ma = Multiaddr( + "/ip4/127.0.0.1/utp/tcp/5555/udp/1234/utp/" "p2p/bafzbeigalb34xlqdtvyklzqa5ibmn6pssqsdskc4ty2e4jxy2kamquh22y" + ) + + for idx, (proto1, proto2, item, value) in enumerate(zip(ma, ma.keys(), ma.items(), ma.values())): # noqa: E501 + assert (proto1, value) == (proto2, value) == item + assert proto1 in ma + assert proto2 in ma.keys() + assert item in ma.items() + assert value in ma.values() + assert ma.keys()[idx] == ma.keys()[idx - len(ma)] == proto1 == proto2 + assert ma.items()[idx] == ma.items()[idx - len(ma)] == item + assert ma.values()[idx] == ma.values()[idx - len(ma)] == ma[proto1] == value + + assert len(ma.keys()) == len(ma.items()) == len(ma.values()) == len(ma) + assert len(list(ma.keys())) == len(ma.keys()) + assert len(list(ma.items())) == len(ma.items()) + assert len(list(ma.values())) == len(ma.values()) + + with pytest.raises(IndexError): + ma.keys()[len(ma)] + with pytest.raises(IndexError): + ma.items()[len(ma)] + with pytest.raises(IndexError): + ma.values()[len(ma)] + + +def test_bad_initialization_no_params(): + with pytest.raises(TypeError): + Multiaddr() + + +def test_bad_initialization_too_many_params(): + with pytest.raises(TypeError): + Multiaddr("/ip4/0.0.0.0", "") + + +def test_bad_initialization_wrong_type(): + with pytest.raises(TypeError): + Multiaddr(42) + + +def test_value_for_protocol_argument_wrong_type(): + a = Multiaddr("/ip4/127.0.0.1/udp/1234") + with pytest.raises(ProtocolNotFoundError): + a.value_for_protocol("str123") + + with pytest.raises(TypeError): + a.value_for_protocol(None) + + +def test_multi_addr_str_corruption(): + a = Multiaddr("/ip4/127.0.0.1/udp/1234") + a._bytes = b"047047047" + + with pytest.raises(BinaryParseError): + str(a) + + +def test_decapsulate(): + a = Multiaddr("/ip4/127.0.0.1/udp/1234") + u = Multiaddr("/udp/1234") + assert a.decapsulate(u) == Multiaddr("/ip4/127.0.0.1") + + +def test__repr(): + a = Multiaddr("/ip4/127.0.0.1/udp/1234") + assert repr(a) == "" % str(a) + + +def test_zone(): + ip6_string = "/ip6zone/eth0/ip6/::1" + ip6_bytes = b"\x2a\x04eth0\x29\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01" + + maddr_from_str = Multiaddr(ip6_string) + assert maddr_from_str.to_bytes() == ip6_bytes + + maddr_from_bytes = Multiaddr(ip6_bytes) + assert str(maddr_from_bytes) == ip6_string + + +def test_hash(): + addr_bytes = Multiaddr("/ip4/127.0.0.1/udp/1234").to_bytes() + + assert hash(Multiaddr(addr_bytes)) == hash(addr_bytes) diff --git a/tests/test_protocols.py b/tests/test_protocols.py new file mode 100644 index 000000000..474194a32 --- /dev/null +++ b/tests/test_protocols.py @@ -0,0 +1,182 @@ +import pytest +import varint + +from hivemind.p2p.multiaddr import exceptions, protocols + + +def test_code_to_varint(): + vi = varint.encode(5) + assert vi == b"\x05" + vi = varint.encode(150) + assert vi == b"\x96\x01" + + +def test_varint_to_code(): + cc = varint.decode_bytes(b"\x05") + assert cc == 5 + cc = varint.decode_bytes(b"\x96\x01") + assert cc == 150 + + +@pytest.fixture +def valid_params(): + return {"code": protocols.P_IP4, "name": "ipb4", "codec": "ipb"} + + +def test_valid(valid_params): + proto = protocols.Protocol(**valid_params) + for key in valid_params: + assert getattr(proto, key) == valid_params[key] + + +@pytest.mark.parametrize("invalid_code", ["abc"]) +def test_invalid_code(valid_params, invalid_code): + valid_params["code"] = invalid_code + with pytest.raises(TypeError): + protocols.Protocol(**valid_params) + + +@pytest.mark.parametrize("invalid_name", [123, 1.0]) +def test_invalid_name(valid_params, invalid_name): + valid_params["name"] = invalid_name + with pytest.raises(TypeError): + protocols.Protocol(**valid_params) + + +@pytest.mark.parametrize("invalid_codec", [b"ip4", 123, 0.123]) +def test_invalid_codec(valid_params, invalid_codec): + valid_params["codec"] = invalid_codec + with pytest.raises(TypeError): + protocols.Protocol(**valid_params) + + +@pytest.mark.parametrize("name", ["foo-str", "foo-u"]) +def test_valid_names(valid_params, name): + valid_params["name"] = name + test_valid(valid_params) + + +@pytest.mark.parametrize("codec", ["ip4", "ip6"]) +def test_valid_codecs(valid_params, codec): + valid_params["codec"] = codec + test_valid(valid_params) + + +def test_protocol_with_name(): + proto = protocols.protocol_with_name("ip4") + assert proto.name == "ip4" + assert proto.code == protocols.P_IP4 + assert proto.size == 32 + assert proto.vcode == varint.encode(protocols.P_IP4) + assert hash(proto) == protocols.P_IP4 + assert protocols.protocol_with_any("ip4") == proto + assert protocols.protocol_with_any(proto) == proto + + with pytest.raises(exceptions.ProtocolNotFoundError): + proto = protocols.protocol_with_name("foo") + + +def test_protocol_with_code(): + proto = protocols.protocol_with_code(protocols.P_IP4) + assert proto.name == "ip4" + assert proto.code == protocols.P_IP4 + assert proto.size == 32 + assert proto.vcode == varint.encode(protocols.P_IP4) + assert hash(proto) == protocols.P_IP4 + assert protocols.protocol_with_any(protocols.P_IP4) == proto + assert protocols.protocol_with_any(proto) == proto + + with pytest.raises(exceptions.ProtocolNotFoundError): + proto = protocols.protocol_with_code(1234) + + +def test_protocol_equality(): + proto1 = protocols.protocol_with_name("ip4") + proto2 = protocols.protocol_with_code(protocols.P_IP4) + proto3 = protocols.protocol_with_name("onion") + proto4 = protocols.protocol_with_name("onion3") + + assert proto1 == proto2 + assert proto1 != proto3 + assert proto3 != proto4 + assert proto1 is not None + assert proto2 != str(proto2) + + +@pytest.mark.parametrize("names", [["ip4"], ["ip4", "tcp"], ["ip4", "tcp", "udp"]]) +def test_protocols_with_string(names): + expected = [protocols.protocol_with_name(name) for name in names] + ins = "/".join(names) + assert protocols.protocols_with_string(ins) == expected + assert protocols.protocols_with_string("/" + ins) == expected + assert protocols.protocols_with_string("/" + ins + "/") == expected + + +@pytest.mark.parametrize("invalid_name", ["", "/", "//"]) +def test_protocols_with_string_invalid(invalid_name): + assert protocols.protocols_with_string(invalid_name) == [] + + +def test_protocols_with_string_mixed(): + names = ["ip4"] + ins = "/".join(names) + test_protocols_with_string(names) + with pytest.raises(exceptions.ProtocolNotFoundError): + names.append("foo") + ins = "/".join(names) + protocols.protocols_with_string(ins) + + +def test_add_protocol(valid_params): + registry = protocols.ProtocolRegistry() + proto = protocols.Protocol(**valid_params) + registry.add(proto) + assert proto.name in registry._names_to_protocols + assert proto.code in registry._codes_to_protocols + assert registry.find(proto.name) is registry.find(proto.code) is proto + + +def test_add_protocol_twice(valid_params): + registry = protocols.ProtocolRegistry() + proto = registry.add(protocols.Protocol(**valid_params)) + + with pytest.raises(exceptions.ProtocolExistsError): + registry.add(proto) + del registry._names_to_protocols[proto.name] + with pytest.raises(exceptions.ProtocolExistsError): + registry.add(proto) + del registry._codes_to_protocols[proto.code] + registry.add(proto) + + +def test_add_protocol_alias(): + registry = protocols.REGISTRY.copy(unlock=True) + registry.add_alias_name("tcp", "abcd") + registry.add_alias_code("tcp", 123456) + + with pytest.raises(exceptions.ProtocolExistsError): + registry.add_alias_name("tcp", "abcd") + with pytest.raises(exceptions.ProtocolExistsError): + registry.add_alias_code("tcp", 123456) + + assert registry.find("tcp") is registry.find("abcd") + assert registry.find("tcp") is registry.find(123456) + + +def test_add_protocol_lock(valid_params): + registry = protocols.REGISTRY.copy(unlock=True) + assert not registry.locked + registry.lock() + assert registry.locked + + with pytest.raises(exceptions.ProtocolRegistryLocked): + registry.add(protocols.Protocol(**valid_params)) + with pytest.raises(exceptions.ProtocolRegistryLocked): + registry.add_alias_name("tcp", "abcdef") + with pytest.raises(exceptions.ProtocolRegistryLocked): + registry.add_alias_code(0x4, 0x123456) + + +def test_protocol_repr(): + proto = protocols.protocol_with_name("ip4") + assert "Protocol(code=4, name='ip4', codec='ip4')" == repr(proto) diff --git a/tests/test_transforms.py b/tests/test_transforms.py new file mode 100644 index 000000000..2fc4aa385 --- /dev/null +++ b/tests/test_transforms.py @@ -0,0 +1,251 @@ +import io + +import pytest + +import hivemind.p2p.multiaddr.protocols +from hivemind.p2p.multiaddr.codecs import codec_by_name +from hivemind.p2p.multiaddr.exceptions import BinaryParseError, StringParseError +from hivemind.p2p.multiaddr.protocols import REGISTRY, Protocol +from hivemind.p2p.multiaddr.transforms import bytes_iter, bytes_to_string, size_for_addr, string_to_bytes + +# These test values were generated by running them through the go implementation +# of multiaddr (https://github.com/multiformats/go-multiaddr) +# +# All values are bijective. +ADDR_BYTES_STR_TEST_DATA = [ + (REGISTRY.find("ip4"), b"\x0a\x0b\x0c\x0d", "10.11.12.13"), + ( + REGISTRY.find("ip6"), + b"\x1a\xa1\x2b\xb2\x3c\xc3\x4d\xd4\x5e\xe5\x6f\xf6\x7a\xb7\x8a\xc8", + "1aa1:2bb2:3cc3:4dd4:5ee5:6ff6:7ab7:8ac8", + ), + (REGISTRY.find("tcp"), b"\xab\xcd", "43981"), + (REGISTRY.find("onion"), b"\x9a\x18\x08\x73\x06\x36\x90\x43\x09\x1f\x04\xd2", "timaq4ygg2iegci7:1234"), + ( + REGISTRY.find("p2p"), + b"\x01\x72\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f" + b"\xf2\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b", + "QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC", + ), + # Additional test data + ( + REGISTRY.find("dns4"), + b"\xd9\x85\xd9\x88\xd9\x82\xd8\xb9.\xd9\x88\xd8\xb2\xd8\xa7\xd8\xb1\xd8\xa9" + b"-\xd8\xa7\xd9\x84\xd8\xa7\xd8\xaa\xd8\xb5\xd8\xa7\xd9\x84\xd8\xa7\xd8\xaa" + b".\xd9\x85\xd8\xb5\xd8\xb1", + # Explicitly mark this as unicode, as the “u” forces the text to be displayed LTR in editors + "موقع.وزارة-الاتصالات.مصر", + ), + ( + REGISTRY.find("dns4"), + b"fu\xc3\x9fball.example", + "fußball.example", + ), # This will fail if IDNA-2003/NamePrep is used +] + +ADDR_BYTES_FROM_STR_TEST_DATA = [ + # New CIDv1 string to new CIDv1 binary format mapping (non-bijective) + ( + REGISTRY.find("p2p"), + b"\x01\x72\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f" + b"\xf2\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b", + "bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm", + ), +] + +ADDR_BYTES_TO_STR_TEST_DATA = [ + # Old CIDv0 binary to old CIDv0 string format mapping (non-bijective) + ( + REGISTRY.find("p2p"), + b"\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f\xf2" + b"\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b", + "QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC", + ), +] + +BYTES_MAP_STR_TEST_DATA = [ + ("/ip4/127.0.0.1/udp/1234", b"\x04\x7f\x00\x00\x01\x91\x02\x04\xd2"), + ("/ip4/127.0.0.1/tcp/4321", b"\x04\x7f\x00\x00\x01\x06\x10\xe1"), + ( + "/ip4/127.0.0.1/udp/1234/ip4/127.0.0.1/tcp/4321", + b"\x04\x7f\x00\x00\x01\x91\x02\x04\xd2\x04\x7f\x00\x00\x01\x06\x10\xe1", + ), +] + + +@pytest.mark.parametrize( + "codec_name, buf, expected", + [ + (None, b"\x01\x02\x03", (0, 0)), + ("ip4", b"\x01\x02\x03", (4, 0)), + ("cid", b"\x40\x50\x60\x51", (64, 1)), + ], +) +def test_size_for_addr(codec_name, buf, expected): + buf_io = io.BytesIO(buf) + assert (size_for_addr(codec_by_name(codec_name), buf_io), buf_io.tell()) == expected + + +@pytest.mark.parametrize( + "buf, expected", + [ + # "/ip4/127.0.0.1/udp/1234/ip4/127.0.0.1/tcp/4321" + ( + b"\x04\x7f\x00\x00\x01\x91\x02\x04\xd2\x04\x7f\x00\x00\x01\x06\x10\xe1", + [ + (REGISTRY.find("ip4"), b"\x7f\x00\x00\x01"), + (REGISTRY.find("udp"), b"\x04\xd2"), + (REGISTRY.find("ip4"), b"\x7f\x00\x00\x01"), + (REGISTRY.find("tcp"), b"\x10\xe1"), + ], + ), + ], +) +def test_bytes_iter(buf, expected): + assert list((proto, val) for _, proto, _, val in bytes_iter(buf)) == expected + + +@pytest.mark.parametrize("proto, buf, expected", ADDR_BYTES_STR_TEST_DATA + ADDR_BYTES_TO_STR_TEST_DATA) +def test_codec_to_string(proto, buf, expected): + assert codec_by_name(proto.codec).to_string(proto, buf) == expected + + +@pytest.mark.parametrize("proto, expected, string", ADDR_BYTES_STR_TEST_DATA + ADDR_BYTES_FROM_STR_TEST_DATA) +def test_codec_to_bytes(proto, string, expected): + assert codec_by_name(proto.codec).to_bytes(proto, string) == expected + + +@pytest.mark.parametrize("string, buf", BYTES_MAP_STR_TEST_DATA) +def test_string_to_bytes(string, buf): + assert string_to_bytes(string) == buf + + +@pytest.mark.parametrize("string, buf", BYTES_MAP_STR_TEST_DATA) +def test_bytes_to_string(string, buf): + assert bytes_to_string(buf) == string + + +class DummyProtocol(Protocol): + def __init__(self, code, name, codec=None): + self.code = code + self.name = name + self.codec = codec + + +class UnparsableProtocol(DummyProtocol): + def __init__(self): + super().__init__(333, "unparsable", "?") + + +@pytest.fixture +def protocol_extension(monkeypatch): + # “Add” additional non-parsable protocol to protocols from code list + registry = multiaddr.protocols.REGISTRY.copy(unlock=True) + registry.add(UnparsableProtocol()) + monkeypatch.setattr(multiaddr.protocols, "REGISTRY", registry) + + +@pytest.mark.parametrize("string", ["test", "/ip4/", "/unparsable/5"]) +def test_string_to_bytes_value_error(protocol_extension, string): + with pytest.raises(StringParseError): + string_to_bytes(string) + + +@pytest.mark.parametrize("bytes", [b"\xcd\x02\x0c\x0d", b"\x35\x03a:b"]) +def test_bytes_to_string_value_error(protocol_extension, bytes): + with pytest.raises(BinaryParseError): + bytes_to_string(bytes) + + +@pytest.mark.parametrize( + "proto, address", + [ + (REGISTRY.find("ip4"), "1124.2.3"), + (REGISTRY.find("ip6"), "123.123.123.123"), + (REGISTRY.find("tcp"), "a"), + (REGISTRY.find("tcp"), "100000"), + (REGISTRY.find("onion"), "100000"), + (REGISTRY.find("onion"), "1234567890123456:0"), + (REGISTRY.find("onion"), "timaq4ygg2iegci7:a"), + (REGISTRY.find("onion"), "timaq4ygg2iegci7:0"), + (REGISTRY.find("onion"), "timaq4ygg2iegci7:71234"), + (REGISTRY.find("p2p"), "15230d52ebb89d85b02a284948203a"), + ( + REGISTRY.find("p2p"), # CID type != "libp2p-key": + "bafyaajaiaejcbrrv5vds2whn3c464rsb5r2vpxeanneinzlijenlac77cju2pptf", + ), + (REGISTRY.find("ip6zone"), ""), + ], +) +def test_codec_to_bytes_value_error(proto, address): + # Codecs themselves may raise any exception type – it will then be converted + # to `StringParseError` by a higher level + with pytest.raises(Exception): + codec_by_name(proto.codec).to_bytes(proto, address) + + +@pytest.mark.parametrize( + "proto, buf", + [ + (REGISTRY.find("tcp"), b"\xff\xff\xff\xff"), + ( + REGISTRY.find("p2p"), # CID type != "libp2p-key": + b"\x01\x70\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" + b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", + ), + (REGISTRY.find("ip6zone"), b""), + ], +) +def test_codec_to_string_value_error(proto, buf): + # Codecs themselves may raise any exception type – it will then be converted + # to `BinaryParseError` by a higher level + with pytest.raises(Exception): + codec_by_name(proto.codec).to_string(proto, buf) + + +@pytest.mark.parametrize( + "proto, string, expected", + [ + ( + REGISTRY.find("p2p"), # This one gets autoconverted to CIDv1 + "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", + b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" + b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", + ), + ( + REGISTRY.find("ip6"), # Others do not + "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", + b"\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41\xec\x75" + b"\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", + ), + ], +) +def test_cid_autoconvert_to_bytes(proto, string, expected): + assert codec_by_name("cid").to_bytes(proto, string) == expected + + +@pytest.mark.parametrize( + "proto, buf, expected", + [ + ( + REGISTRY.find("p2p"), # This one gets autoconverted to CIDv0 + b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" + b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", + "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", + ), + ( + REGISTRY.find("ip6"), # Others do not + b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" + b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", + "bafzaajaiaejcbrrv5vds2whn3c464rsb5r2vpxeanneinzlijenlac77cju2pptf", + ), + ( + REGISTRY.find("ip6"), # (Needed to put identity conversion test somewhere) + b"\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41\xec\x75" + b"\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", + "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", + ), + ], +) +def test_cid_autoconvert_to_string(proto, buf, expected): + assert codec_by_name("cid").to_string(proto, buf) == expected diff --git a/tests/test_utils/dht_swarms.py b/tests/test_utils/dht_swarms.py index 97ca7482c..2538de4d6 100644 --- a/tests/test_utils/dht_swarms.py +++ b/tests/test_utils/dht_swarms.py @@ -5,7 +5,6 @@ import threading from typing import Dict, List, Tuple - from hivemind.dht import DHT from hivemind.dht.node import DHTID, DHTNode from hivemind.p2p import PeerID diff --git a/tests/test_utils/p2p_daemon.py b/tests/test_utils/p2p_daemon.py index d48d1ff40..84d16edb4 100644 --- a/tests/test_utils/p2p_daemon.py +++ b/tests/test_utils/p2p_daemon.py @@ -9,8 +9,8 @@ from pkg_resources import resource_filename -from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client from hivemind.p2p.multiaddr import Multiaddr, protocols +from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client from test_utils.networking import get_free_port From e61e597fd29353c323b9bbb04d8c9cd17ab4cd37 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 4 Nov 2024 19:09:06 +0300 Subject: [PATCH 09/12] change multiaddr version --- hivemind/p2p/multiaddr/codecs/__init__.py | 1 + hivemind/p2p/multiaddr/codecs/cid.py | 1 + hivemind/p2p/multiaddr/codecs/domain.py | 1 + hivemind/p2p/multiaddr/codecs/fspath.py | 1 + hivemind/p2p/multiaddr/codecs/ip4.py | 1 + hivemind/p2p/multiaddr/codecs/ip6.py | 1 + hivemind/p2p/multiaddr/codecs/onion.py | 1 + hivemind/p2p/multiaddr/codecs/onion3.py | 1 + hivemind/p2p/multiaddr/codecs/uint16be.py | 1 + hivemind/p2p/multiaddr/codecs/utf8.py | 1 + hivemind/p2p/multiaddr/multiaddr.py | 6 +- hivemind/p2p/multiaddr/transforms.py | 9 +- tests/test_multiaddr.py | 21 -- tests/test_protocols.py | 182 ---------------- tests/test_transforms.py | 251 ---------------------- 15 files changed, 21 insertions(+), 458 deletions(-) delete mode 100644 tests/test_protocols.py delete mode 100644 tests/test_transforms.py diff --git a/hivemind/p2p/multiaddr/codecs/__init__.py b/hivemind/p2p/multiaddr/codecs/__init__.py index ce76d6ea2..d30127931 100644 --- a/hivemind/p2p/multiaddr/codecs/__init__.py +++ b/hivemind/p2p/multiaddr/codecs/__init__.py @@ -1,5 +1,6 @@ import importlib + # These are special sizes LENGTH_PREFIXED_VAR_SIZE = -1 diff --git a/hivemind/p2p/multiaddr/codecs/cid.py b/hivemind/p2p/multiaddr/codecs/cid.py index 985c25b6f..260ebfbdc 100644 --- a/hivemind/p2p/multiaddr/codecs/cid.py +++ b/hivemind/p2p/multiaddr/codecs/cid.py @@ -3,6 +3,7 @@ from . import LENGTH_PREFIXED_VAR_SIZE + SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/domain.py b/hivemind/p2p/multiaddr/codecs/domain.py index 129ad6888..7d616366d 100644 --- a/hivemind/p2p/multiaddr/codecs/domain.py +++ b/hivemind/p2p/multiaddr/codecs/domain.py @@ -2,6 +2,7 @@ from . import LENGTH_PREFIXED_VAR_SIZE + SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/fspath.py b/hivemind/p2p/multiaddr/codecs/fspath.py index a93f5f6dc..36b6859a4 100644 --- a/hivemind/p2p/multiaddr/codecs/fspath.py +++ b/hivemind/p2p/multiaddr/codecs/fspath.py @@ -2,6 +2,7 @@ from . import LENGTH_PREFIXED_VAR_SIZE + SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = True diff --git a/hivemind/p2p/multiaddr/codecs/ip4.py b/hivemind/p2p/multiaddr/codecs/ip4.py index 4384d17d6..1a7f64556 100644 --- a/hivemind/p2p/multiaddr/codecs/ip4.py +++ b/hivemind/p2p/multiaddr/codecs/ip4.py @@ -1,5 +1,6 @@ import netaddr + SIZE = 32 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/ip6.py b/hivemind/p2p/multiaddr/codecs/ip6.py index 6b87e5237..7ae71b0d0 100644 --- a/hivemind/p2p/multiaddr/codecs/ip6.py +++ b/hivemind/p2p/multiaddr/codecs/ip6.py @@ -1,5 +1,6 @@ import netaddr + SIZE = 128 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/onion.py b/hivemind/p2p/multiaddr/codecs/onion.py index e864190b8..55c2078d3 100644 --- a/hivemind/p2p/multiaddr/codecs/onion.py +++ b/hivemind/p2p/multiaddr/codecs/onion.py @@ -1,6 +1,7 @@ import base64 import struct + SIZE = 96 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/onion3.py b/hivemind/p2p/multiaddr/codecs/onion3.py index 827c1a185..d12bae588 100644 --- a/hivemind/p2p/multiaddr/codecs/onion3.py +++ b/hivemind/p2p/multiaddr/codecs/onion3.py @@ -1,6 +1,7 @@ import base64 import struct + SIZE = 296 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/uint16be.py b/hivemind/p2p/multiaddr/codecs/uint16be.py index 3a7f05bf7..73442e430 100644 --- a/hivemind/p2p/multiaddr/codecs/uint16be.py +++ b/hivemind/p2p/multiaddr/codecs/uint16be.py @@ -1,5 +1,6 @@ import struct + SIZE = 16 IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/utf8.py b/hivemind/p2p/multiaddr/codecs/utf8.py index e4edcf2b0..5a85e2793 100644 --- a/hivemind/p2p/multiaddr/codecs/utf8.py +++ b/hivemind/p2p/multiaddr/codecs/utf8.py @@ -2,6 +2,7 @@ from . import LENGTH_PREFIXED_VAR_SIZE + SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/multiaddr.py b/hivemind/p2p/multiaddr/multiaddr.py index 8a74ea52f..52f51a055 100644 --- a/hivemind/p2p/multiaddr/multiaddr.py +++ b/hivemind/p2p/multiaddr/multiaddr.py @@ -3,7 +3,11 @@ import varint from . import exceptions, protocols -from .transforms import bytes_iter, bytes_to_string, string_to_bytes + +from .transforms import bytes_iter +from .transforms import string_to_bytes +from .transforms import bytes_to_string + __all__ = ("Multiaddr",) diff --git a/hivemind/p2p/multiaddr/transforms.py b/hivemind/p2p/multiaddr/transforms.py index d46ab9a16..951c824c6 100644 --- a/hivemind/p2p/multiaddr/transforms.py +++ b/hivemind/p2p/multiaddr/transforms.py @@ -1,10 +1,13 @@ import io - import varint from . import exceptions -from .codecs import LENGTH_PREFIXED_VAR_SIZE, codec_by_name -from .protocols import protocol_with_code, protocol_with_name + +from .codecs import LENGTH_PREFIXED_VAR_SIZE +from .codecs import codec_by_name + +from .protocols import protocol_with_code +from .protocols import protocol_with_name def string_to_bytes(string): diff --git a/tests/test_multiaddr.py b/tests/test_multiaddr.py index 63a72d263..e7cb2bab3 100644 --- a/tests/test_multiaddr.py +++ b/tests/test_multiaddr.py @@ -36,19 +36,6 @@ "/sctp", "/udp/65536", "/tcp/65536", - "/onion/9imaq4ygg2iegci7:80", - "/onion/aaimaq4ygg2iegci7:80", - "/onion/timaq4ygg2iegci7:0", - "/onion/timaq4ygg2iegci7:-1", - "/onion/timaq4ygg2iegci7", - "/onion/timaq4ygg2iegci@:666", - "/onion3/9ww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd7:80", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:0", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:a", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:-1", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyy@:666", "/udp/1234/sctp", "/udp/1234/udt/1234", "/udp/1234/utp/1234", @@ -82,10 +69,6 @@ def test_invalid(addr_str): "/ip6zone/x/ip6/fe80::1", "/ip6zone/x%y/ip6/fe80::1", "/ip6zone/x%y/ip6/::", - "/onion/timaq4ygg2iegci7:1234", - "/onion/timaq4ygg2iegci7:80/http", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:1234", - "/onion3/vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd:80/http", "/udp/0", "/tcp/0", "/sctp/0", @@ -110,8 +93,6 @@ def test_invalid(addr_str): "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f", "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC" "/tcp/1234/unix/stdio", "/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct", - "/dns/example.com", - "/dns4/موقع.وزارة-الاتصالات.مصر", ], ) # nopep8 def test_valid(addr_str): @@ -183,7 +164,6 @@ def test_invalid_protocols_with_string(proto_string): ("/ip4/1.2.3.4", -1, ("/ip4/1.2.3.4",)), ("/ip4/0.0.0.0", 0, ("/ip4/0.0.0.0",)), ("/ip6/::1", 1, ("/ip6/::1",)), - ("/onion/timaq4ygg2iegci7:80/http", 0, ("/onion/timaq4ygg2iegci7:80/http",)), ( "/ip4/127.0.0.1/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234", 1, @@ -202,7 +182,6 @@ def test_split(proto_string, maxsplit, expected): (("/ip4/1.2.3.4",), "/ip4/1.2.3.4"), ((b"\x04\x00\x00\x00\x00",), "/ip4/0.0.0.0"), (("/ip6/::1",), "/ip6/::1"), - (("/onion/timaq4ygg2iegci7:80/http",), "/onion/timaq4ygg2iegci7:80/http"), ( ( b"\x04\x7F\x00\x00\x01", diff --git a/tests/test_protocols.py b/tests/test_protocols.py deleted file mode 100644 index 474194a32..000000000 --- a/tests/test_protocols.py +++ /dev/null @@ -1,182 +0,0 @@ -import pytest -import varint - -from hivemind.p2p.multiaddr import exceptions, protocols - - -def test_code_to_varint(): - vi = varint.encode(5) - assert vi == b"\x05" - vi = varint.encode(150) - assert vi == b"\x96\x01" - - -def test_varint_to_code(): - cc = varint.decode_bytes(b"\x05") - assert cc == 5 - cc = varint.decode_bytes(b"\x96\x01") - assert cc == 150 - - -@pytest.fixture -def valid_params(): - return {"code": protocols.P_IP4, "name": "ipb4", "codec": "ipb"} - - -def test_valid(valid_params): - proto = protocols.Protocol(**valid_params) - for key in valid_params: - assert getattr(proto, key) == valid_params[key] - - -@pytest.mark.parametrize("invalid_code", ["abc"]) -def test_invalid_code(valid_params, invalid_code): - valid_params["code"] = invalid_code - with pytest.raises(TypeError): - protocols.Protocol(**valid_params) - - -@pytest.mark.parametrize("invalid_name", [123, 1.0]) -def test_invalid_name(valid_params, invalid_name): - valid_params["name"] = invalid_name - with pytest.raises(TypeError): - protocols.Protocol(**valid_params) - - -@pytest.mark.parametrize("invalid_codec", [b"ip4", 123, 0.123]) -def test_invalid_codec(valid_params, invalid_codec): - valid_params["codec"] = invalid_codec - with pytest.raises(TypeError): - protocols.Protocol(**valid_params) - - -@pytest.mark.parametrize("name", ["foo-str", "foo-u"]) -def test_valid_names(valid_params, name): - valid_params["name"] = name - test_valid(valid_params) - - -@pytest.mark.parametrize("codec", ["ip4", "ip6"]) -def test_valid_codecs(valid_params, codec): - valid_params["codec"] = codec - test_valid(valid_params) - - -def test_protocol_with_name(): - proto = protocols.protocol_with_name("ip4") - assert proto.name == "ip4" - assert proto.code == protocols.P_IP4 - assert proto.size == 32 - assert proto.vcode == varint.encode(protocols.P_IP4) - assert hash(proto) == protocols.P_IP4 - assert protocols.protocol_with_any("ip4") == proto - assert protocols.protocol_with_any(proto) == proto - - with pytest.raises(exceptions.ProtocolNotFoundError): - proto = protocols.protocol_with_name("foo") - - -def test_protocol_with_code(): - proto = protocols.protocol_with_code(protocols.P_IP4) - assert proto.name == "ip4" - assert proto.code == protocols.P_IP4 - assert proto.size == 32 - assert proto.vcode == varint.encode(protocols.P_IP4) - assert hash(proto) == protocols.P_IP4 - assert protocols.protocol_with_any(protocols.P_IP4) == proto - assert protocols.protocol_with_any(proto) == proto - - with pytest.raises(exceptions.ProtocolNotFoundError): - proto = protocols.protocol_with_code(1234) - - -def test_protocol_equality(): - proto1 = protocols.protocol_with_name("ip4") - proto2 = protocols.protocol_with_code(protocols.P_IP4) - proto3 = protocols.protocol_with_name("onion") - proto4 = protocols.protocol_with_name("onion3") - - assert proto1 == proto2 - assert proto1 != proto3 - assert proto3 != proto4 - assert proto1 is not None - assert proto2 != str(proto2) - - -@pytest.mark.parametrize("names", [["ip4"], ["ip4", "tcp"], ["ip4", "tcp", "udp"]]) -def test_protocols_with_string(names): - expected = [protocols.protocol_with_name(name) for name in names] - ins = "/".join(names) - assert protocols.protocols_with_string(ins) == expected - assert protocols.protocols_with_string("/" + ins) == expected - assert protocols.protocols_with_string("/" + ins + "/") == expected - - -@pytest.mark.parametrize("invalid_name", ["", "/", "//"]) -def test_protocols_with_string_invalid(invalid_name): - assert protocols.protocols_with_string(invalid_name) == [] - - -def test_protocols_with_string_mixed(): - names = ["ip4"] - ins = "/".join(names) - test_protocols_with_string(names) - with pytest.raises(exceptions.ProtocolNotFoundError): - names.append("foo") - ins = "/".join(names) - protocols.protocols_with_string(ins) - - -def test_add_protocol(valid_params): - registry = protocols.ProtocolRegistry() - proto = protocols.Protocol(**valid_params) - registry.add(proto) - assert proto.name in registry._names_to_protocols - assert proto.code in registry._codes_to_protocols - assert registry.find(proto.name) is registry.find(proto.code) is proto - - -def test_add_protocol_twice(valid_params): - registry = protocols.ProtocolRegistry() - proto = registry.add(protocols.Protocol(**valid_params)) - - with pytest.raises(exceptions.ProtocolExistsError): - registry.add(proto) - del registry._names_to_protocols[proto.name] - with pytest.raises(exceptions.ProtocolExistsError): - registry.add(proto) - del registry._codes_to_protocols[proto.code] - registry.add(proto) - - -def test_add_protocol_alias(): - registry = protocols.REGISTRY.copy(unlock=True) - registry.add_alias_name("tcp", "abcd") - registry.add_alias_code("tcp", 123456) - - with pytest.raises(exceptions.ProtocolExistsError): - registry.add_alias_name("tcp", "abcd") - with pytest.raises(exceptions.ProtocolExistsError): - registry.add_alias_code("tcp", 123456) - - assert registry.find("tcp") is registry.find("abcd") - assert registry.find("tcp") is registry.find(123456) - - -def test_add_protocol_lock(valid_params): - registry = protocols.REGISTRY.copy(unlock=True) - assert not registry.locked - registry.lock() - assert registry.locked - - with pytest.raises(exceptions.ProtocolRegistryLocked): - registry.add(protocols.Protocol(**valid_params)) - with pytest.raises(exceptions.ProtocolRegistryLocked): - registry.add_alias_name("tcp", "abcdef") - with pytest.raises(exceptions.ProtocolRegistryLocked): - registry.add_alias_code(0x4, 0x123456) - - -def test_protocol_repr(): - proto = protocols.protocol_with_name("ip4") - assert "Protocol(code=4, name='ip4', codec='ip4')" == repr(proto) diff --git a/tests/test_transforms.py b/tests/test_transforms.py deleted file mode 100644 index 2fc4aa385..000000000 --- a/tests/test_transforms.py +++ /dev/null @@ -1,251 +0,0 @@ -import io - -import pytest - -import hivemind.p2p.multiaddr.protocols -from hivemind.p2p.multiaddr.codecs import codec_by_name -from hivemind.p2p.multiaddr.exceptions import BinaryParseError, StringParseError -from hivemind.p2p.multiaddr.protocols import REGISTRY, Protocol -from hivemind.p2p.multiaddr.transforms import bytes_iter, bytes_to_string, size_for_addr, string_to_bytes - -# These test values were generated by running them through the go implementation -# of multiaddr (https://github.com/multiformats/go-multiaddr) -# -# All values are bijective. -ADDR_BYTES_STR_TEST_DATA = [ - (REGISTRY.find("ip4"), b"\x0a\x0b\x0c\x0d", "10.11.12.13"), - ( - REGISTRY.find("ip6"), - b"\x1a\xa1\x2b\xb2\x3c\xc3\x4d\xd4\x5e\xe5\x6f\xf6\x7a\xb7\x8a\xc8", - "1aa1:2bb2:3cc3:4dd4:5ee5:6ff6:7ab7:8ac8", - ), - (REGISTRY.find("tcp"), b"\xab\xcd", "43981"), - (REGISTRY.find("onion"), b"\x9a\x18\x08\x73\x06\x36\x90\x43\x09\x1f\x04\xd2", "timaq4ygg2iegci7:1234"), - ( - REGISTRY.find("p2p"), - b"\x01\x72\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f" - b"\xf2\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b", - "QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC", - ), - # Additional test data - ( - REGISTRY.find("dns4"), - b"\xd9\x85\xd9\x88\xd9\x82\xd8\xb9.\xd9\x88\xd8\xb2\xd8\xa7\xd8\xb1\xd8\xa9" - b"-\xd8\xa7\xd9\x84\xd8\xa7\xd8\xaa\xd8\xb5\xd8\xa7\xd9\x84\xd8\xa7\xd8\xaa" - b".\xd9\x85\xd8\xb5\xd8\xb1", - # Explicitly mark this as unicode, as the “u” forces the text to be displayed LTR in editors - "موقع.وزارة-الاتصالات.مصر", - ), - ( - REGISTRY.find("dns4"), - b"fu\xc3\x9fball.example", - "fußball.example", - ), # This will fail if IDNA-2003/NamePrep is used -] - -ADDR_BYTES_FROM_STR_TEST_DATA = [ - # New CIDv1 string to new CIDv1 binary format mapping (non-bijective) - ( - REGISTRY.find("p2p"), - b"\x01\x72\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f" - b"\xf2\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b", - "bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm", - ), -] - -ADDR_BYTES_TO_STR_TEST_DATA = [ - # Old CIDv0 binary to old CIDv0 string format mapping (non-bijective) - ( - REGISTRY.find("p2p"), - b"\x12\x20\xd5\x2e\xbb\x89\xd8\x5b\x02\xa2\x84\x94\x82\x03\xa6\x2f\xf2" - b"\x83\x89\xc5\x7c\x9f\x42\xbe\xec\x4e\xc2\x0d\xb7\x6a\x68\x91\x1c\x0b", - "QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC", - ), -] - -BYTES_MAP_STR_TEST_DATA = [ - ("/ip4/127.0.0.1/udp/1234", b"\x04\x7f\x00\x00\x01\x91\x02\x04\xd2"), - ("/ip4/127.0.0.1/tcp/4321", b"\x04\x7f\x00\x00\x01\x06\x10\xe1"), - ( - "/ip4/127.0.0.1/udp/1234/ip4/127.0.0.1/tcp/4321", - b"\x04\x7f\x00\x00\x01\x91\x02\x04\xd2\x04\x7f\x00\x00\x01\x06\x10\xe1", - ), -] - - -@pytest.mark.parametrize( - "codec_name, buf, expected", - [ - (None, b"\x01\x02\x03", (0, 0)), - ("ip4", b"\x01\x02\x03", (4, 0)), - ("cid", b"\x40\x50\x60\x51", (64, 1)), - ], -) -def test_size_for_addr(codec_name, buf, expected): - buf_io = io.BytesIO(buf) - assert (size_for_addr(codec_by_name(codec_name), buf_io), buf_io.tell()) == expected - - -@pytest.mark.parametrize( - "buf, expected", - [ - # "/ip4/127.0.0.1/udp/1234/ip4/127.0.0.1/tcp/4321" - ( - b"\x04\x7f\x00\x00\x01\x91\x02\x04\xd2\x04\x7f\x00\x00\x01\x06\x10\xe1", - [ - (REGISTRY.find("ip4"), b"\x7f\x00\x00\x01"), - (REGISTRY.find("udp"), b"\x04\xd2"), - (REGISTRY.find("ip4"), b"\x7f\x00\x00\x01"), - (REGISTRY.find("tcp"), b"\x10\xe1"), - ], - ), - ], -) -def test_bytes_iter(buf, expected): - assert list((proto, val) for _, proto, _, val in bytes_iter(buf)) == expected - - -@pytest.mark.parametrize("proto, buf, expected", ADDR_BYTES_STR_TEST_DATA + ADDR_BYTES_TO_STR_TEST_DATA) -def test_codec_to_string(proto, buf, expected): - assert codec_by_name(proto.codec).to_string(proto, buf) == expected - - -@pytest.mark.parametrize("proto, expected, string", ADDR_BYTES_STR_TEST_DATA + ADDR_BYTES_FROM_STR_TEST_DATA) -def test_codec_to_bytes(proto, string, expected): - assert codec_by_name(proto.codec).to_bytes(proto, string) == expected - - -@pytest.mark.parametrize("string, buf", BYTES_MAP_STR_TEST_DATA) -def test_string_to_bytes(string, buf): - assert string_to_bytes(string) == buf - - -@pytest.mark.parametrize("string, buf", BYTES_MAP_STR_TEST_DATA) -def test_bytes_to_string(string, buf): - assert bytes_to_string(buf) == string - - -class DummyProtocol(Protocol): - def __init__(self, code, name, codec=None): - self.code = code - self.name = name - self.codec = codec - - -class UnparsableProtocol(DummyProtocol): - def __init__(self): - super().__init__(333, "unparsable", "?") - - -@pytest.fixture -def protocol_extension(monkeypatch): - # “Add” additional non-parsable protocol to protocols from code list - registry = multiaddr.protocols.REGISTRY.copy(unlock=True) - registry.add(UnparsableProtocol()) - monkeypatch.setattr(multiaddr.protocols, "REGISTRY", registry) - - -@pytest.mark.parametrize("string", ["test", "/ip4/", "/unparsable/5"]) -def test_string_to_bytes_value_error(protocol_extension, string): - with pytest.raises(StringParseError): - string_to_bytes(string) - - -@pytest.mark.parametrize("bytes", [b"\xcd\x02\x0c\x0d", b"\x35\x03a:b"]) -def test_bytes_to_string_value_error(protocol_extension, bytes): - with pytest.raises(BinaryParseError): - bytes_to_string(bytes) - - -@pytest.mark.parametrize( - "proto, address", - [ - (REGISTRY.find("ip4"), "1124.2.3"), - (REGISTRY.find("ip6"), "123.123.123.123"), - (REGISTRY.find("tcp"), "a"), - (REGISTRY.find("tcp"), "100000"), - (REGISTRY.find("onion"), "100000"), - (REGISTRY.find("onion"), "1234567890123456:0"), - (REGISTRY.find("onion"), "timaq4ygg2iegci7:a"), - (REGISTRY.find("onion"), "timaq4ygg2iegci7:0"), - (REGISTRY.find("onion"), "timaq4ygg2iegci7:71234"), - (REGISTRY.find("p2p"), "15230d52ebb89d85b02a284948203a"), - ( - REGISTRY.find("p2p"), # CID type != "libp2p-key": - "bafyaajaiaejcbrrv5vds2whn3c464rsb5r2vpxeanneinzlijenlac77cju2pptf", - ), - (REGISTRY.find("ip6zone"), ""), - ], -) -def test_codec_to_bytes_value_error(proto, address): - # Codecs themselves may raise any exception type – it will then be converted - # to `StringParseError` by a higher level - with pytest.raises(Exception): - codec_by_name(proto.codec).to_bytes(proto, address) - - -@pytest.mark.parametrize( - "proto, buf", - [ - (REGISTRY.find("tcp"), b"\xff\xff\xff\xff"), - ( - REGISTRY.find("p2p"), # CID type != "libp2p-key": - b"\x01\x70\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" - b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", - ), - (REGISTRY.find("ip6zone"), b""), - ], -) -def test_codec_to_string_value_error(proto, buf): - # Codecs themselves may raise any exception type – it will then be converted - # to `BinaryParseError` by a higher level - with pytest.raises(Exception): - codec_by_name(proto.codec).to_string(proto, buf) - - -@pytest.mark.parametrize( - "proto, string, expected", - [ - ( - REGISTRY.find("p2p"), # This one gets autoconverted to CIDv1 - "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", - b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" - b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", - ), - ( - REGISTRY.find("ip6"), # Others do not - "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", - b"\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41\xec\x75" - b"\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", - ), - ], -) -def test_cid_autoconvert_to_bytes(proto, string, expected): - assert codec_by_name("cid").to_bytes(proto, string) == expected - - -@pytest.mark.parametrize( - "proto, buf, expected", - [ - ( - REGISTRY.find("p2p"), # This one gets autoconverted to CIDv0 - b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" - b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", - "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", - ), - ( - REGISTRY.find("ip6"), # Others do not - b"\x01\x72\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41" - b"\xec\x75\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", - "bafzaajaiaejcbrrv5vds2whn3c464rsb5r2vpxeanneinzlijenlac77cju2pptf", - ), - ( - REGISTRY.find("ip6"), # (Needed to put identity conversion test somewhere) - b"\x00\x24\x08\x01\x12\x20\xc6\x35\xed\x47\x2d\x58\xed\xd8\xb9\xee\x46\x41\xec\x75" - b"\x57\xdc\x80\x6b\x48\x86\xe5\x68\x49\x1a\xb0\x0b\xff\x12\x69\xa7\xbe\x65", - "12D3KooWPA6ax6t3jqTyGq73Zm1RmwppYqxaXzrtarfcTWGp5Wzx", - ), - ], -) -def test_cid_autoconvert_to_string(proto, buf, expected): - assert codec_by_name("cid").to_string(proto, buf) == expected From b0030e1d99b247e4dc4ca6566cb7f59de65267d2 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 17 Nov 2024 12:06:37 +0300 Subject: [PATCH 10/12] black and isort --- hivemind/p2p/multiaddr/__init__.py | 6 +- hivemind/p2p/multiaddr/codecs/__init__.py | 1 - hivemind/p2p/multiaddr/codecs/cid.py | 104 +++++++++++----------- hivemind/p2p/multiaddr/codecs/domain.py | 1 - hivemind/p2p/multiaddr/codecs/fspath.py | 1 - hivemind/p2p/multiaddr/codecs/ip4.py | 3 +- hivemind/p2p/multiaddr/codecs/ip6.py | 3 +- hivemind/p2p/multiaddr/codecs/onion.py | 9 +- hivemind/p2p/multiaddr/codecs/onion3.py | 9 +- hivemind/p2p/multiaddr/codecs/uint16be.py | 5 +- hivemind/p2p/multiaddr/codecs/utf8.py | 5 +- hivemind/p2p/multiaddr/exceptions.py | 15 ++-- hivemind/p2p/multiaddr/multiaddr.py | 22 ++--- hivemind/p2p/multiaddr/protocols.py | 78 ++++++++-------- hivemind/p2p/multiaddr/transforms.py | 31 +++---- 15 files changed, 136 insertions(+), 157 deletions(-) diff --git a/hivemind/p2p/multiaddr/__init__.py b/hivemind/p2p/multiaddr/__init__.py index b9d71ad56..d3eafe3bc 100755 --- a/hivemind/p2p/multiaddr/__init__.py +++ b/hivemind/p2p/multiaddr/__init__.py @@ -1,5 +1,5 @@ from .multiaddr import Multiaddr # NOQA -__author__ = 'Steven Buss' -__email__ = 'steven.buss@gmail.com' -__version__ = '0.0.9' +__author__ = "Steven Buss" +__email__ = "steven.buss@gmail.com" +__version__ = "0.0.9" diff --git a/hivemind/p2p/multiaddr/codecs/__init__.py b/hivemind/p2p/multiaddr/codecs/__init__.py index d30127931..ce76d6ea2 100644 --- a/hivemind/p2p/multiaddr/codecs/__init__.py +++ b/hivemind/p2p/multiaddr/codecs/__init__.py @@ -1,6 +1,5 @@ import importlib - # These are special sizes LENGTH_PREFIXED_VAR_SIZE = -1 diff --git a/hivemind/p2p/multiaddr/codecs/cid.py b/hivemind/p2p/multiaddr/codecs/cid.py index 260ebfbdc..c9ad682fa 100644 --- a/hivemind/p2p/multiaddr/codecs/cid.py +++ b/hivemind/p2p/multiaddr/codecs/cid.py @@ -3,7 +3,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False @@ -11,55 +10,54 @@ # Spec: https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation CIDv0_PREFIX_TO_LENGTH = { # base58btc prefixes for valid lengths 1 – 42 with the identity “hash” function - '12': [5, 12, 19, 23, 30, 41, 52, 56], - '13': [9, 16, 34, 45], - '14': [27, 38, 49, 60], - '15': [3, 6, 20], - '16': [3, 6, 13, 20, 31, 42, 53], - '17': [3, 13, 42], - '18': [3], - '19': [3, 24, 57], - '1A': [24, 35, 46], - '1B': [35], - '1D': [17], - '1E': [10, 17], - '1F': [10], - '1G': [10, 28, 50], - '1H': [28, 39], - '1P': [21], - '1Q': [21], - '1R': [21, 54], - '1S': [54], - '1T': [7, 32, 43], - '1U': [7, 32, 43], - '1V': [7], - '1W': [7, 14], - '1X': [7, 14], - '1Y': [7, 14], - '1Z': [7, 14], - '1f': [4], - '1g': [4, 58], - '1h': [4, 25, 58], - '1i': [4, 25], - '1j': [4, 25], - '1k': [4, 25, 47], - '1m': [4, 36, 47], - '1n': [4, 36], - '1o': [4, 36], - '1p': [4], - '1q': [4], - '1r': [4], - '1s': [4], - '1t': [4], - '1u': [4], - '1v': [4], - '1w': [4], - '1x': [4], - '1y': [4], - '1z': [4, 18], - + "12": [5, 12, 19, 23, 30, 41, 52, 56], + "13": [9, 16, 34, 45], + "14": [27, 38, 49, 60], + "15": [3, 6, 20], + "16": [3, 6, 13, 20, 31, 42, 53], + "17": [3, 13, 42], + "18": [3], + "19": [3, 24, 57], + "1A": [24, 35, 46], + "1B": [35], + "1D": [17], + "1E": [10, 17], + "1F": [10], + "1G": [10, 28, 50], + "1H": [28, 39], + "1P": [21], + "1Q": [21], + "1R": [21, 54], + "1S": [54], + "1T": [7, 32, 43], + "1U": [7, 32, 43], + "1V": [7], + "1W": [7, 14], + "1X": [7, 14], + "1Y": [7, 14], + "1Z": [7, 14], + "1f": [4], + "1g": [4, 58], + "1h": [4, 25, 58], + "1i": [4, 25], + "1j": [4, 25], + "1k": [4, 25, 47], + "1m": [4, 36, 47], + "1n": [4, 36], + "1o": [4, 36], + "1p": [4], + "1q": [4], + "1r": [4], + "1s": [4], + "1t": [4], + "1u": [4], + "1v": [4], + "1w": [4], + "1x": [4], + "1y": [4], + "1z": [4, 18], # base58btc prefix for length 42 with the sha256 hash function - 'Qm': [46], + "Qm": [46], } PROTO_NAME_TO_CIDv1_CODEC = { @@ -83,8 +81,7 @@ def to_bytes(proto, string): # Ensure CID has correct codec for protocol if expected_codec and parsed.codec != expected_codec: - raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec" - .format(proto.name, expected_codec)) + raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec".format(proto.name, expected_codec)) return parsed.buffer @@ -105,7 +102,7 @@ def to_string(proto, buf): if _is_binary_cidv0_multihash(buf): # CIDv0 if not expected_codec: # Simply encode as base58btc as there is nothing better to do - return base58.b58encode(buf).decode('ascii') + return base58.b58encode(buf).decode("ascii") # “Implementations SHOULD display peer IDs using the first (raw # base58btc encoded multihash) format until the second format is @@ -120,8 +117,7 @@ def to_string(proto, buf): # Ensure CID has correct codec for protocol if expected_codec and parsed.codec != expected_codec: - raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec" - .format(proto.name, expected_codec)) + raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec".format(proto.name, expected_codec)) # “Implementations SHOULD display peer IDs using the first (raw # base58btc encoded multihash) format until the second format is diff --git a/hivemind/p2p/multiaddr/codecs/domain.py b/hivemind/p2p/multiaddr/codecs/domain.py index 7d616366d..129ad6888 100644 --- a/hivemind/p2p/multiaddr/codecs/domain.py +++ b/hivemind/p2p/multiaddr/codecs/domain.py @@ -2,7 +2,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False diff --git a/hivemind/p2p/multiaddr/codecs/fspath.py b/hivemind/p2p/multiaddr/codecs/fspath.py index 36b6859a4..a93f5f6dc 100644 --- a/hivemind/p2p/multiaddr/codecs/fspath.py +++ b/hivemind/p2p/multiaddr/codecs/fspath.py @@ -2,7 +2,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = True diff --git a/hivemind/p2p/multiaddr/codecs/ip4.py b/hivemind/p2p/multiaddr/codecs/ip4.py index 1a7f64556..bcef47979 100644 --- a/hivemind/p2p/multiaddr/codecs/ip4.py +++ b/hivemind/p2p/multiaddr/codecs/ip4.py @@ -1,6 +1,5 @@ import netaddr - SIZE = 32 IS_PATH = False @@ -10,4 +9,4 @@ def to_bytes(proto, string): def to_string(proto, buf): - return str(netaddr.IPAddress(int.from_bytes(buf, byteorder='big'), version=4)) + return str(netaddr.IPAddress(int.from_bytes(buf, byteorder="big"), version=4)) diff --git a/hivemind/p2p/multiaddr/codecs/ip6.py b/hivemind/p2p/multiaddr/codecs/ip6.py index 7ae71b0d0..5b8edf2c0 100644 --- a/hivemind/p2p/multiaddr/codecs/ip6.py +++ b/hivemind/p2p/multiaddr/codecs/ip6.py @@ -1,6 +1,5 @@ import netaddr - SIZE = 128 IS_PATH = False @@ -10,4 +9,4 @@ def to_bytes(proto, string): def to_string(proto, buf): - return str(netaddr.IPAddress(int.from_bytes(buf, byteorder='big'), version=6)) + return str(netaddr.IPAddress(int.from_bytes(buf, byteorder="big"), version=6)) diff --git a/hivemind/p2p/multiaddr/codecs/onion.py b/hivemind/p2p/multiaddr/codecs/onion.py index 55c2078d3..77df5c049 100644 --- a/hivemind/p2p/multiaddr/codecs/onion.py +++ b/hivemind/p2p/multiaddr/codecs/onion.py @@ -1,7 +1,6 @@ import base64 import struct - SIZE = 96 IS_PATH = False @@ -27,11 +26,11 @@ def to_bytes(proto, string): if port not in range(1, 65536): raise ValueError("Port number is not in range(1, 65536)") - return b''.join((onion_host_bytes, struct.pack('>H', port))) + return b"".join((onion_host_bytes, struct.pack(">H", port))) def to_string(proto, buf): addr_bytes, port_bytes = (buf[:-2], buf[-2:]) - addr = base64.b32encode(addr_bytes).decode('ascii').lower() - port = str(struct.unpack('>H', port_bytes)[0]) - return ':'.join([addr, port]) + addr = base64.b32encode(addr_bytes).decode("ascii").lower() + port = str(struct.unpack(">H", port_bytes)[0]) + return ":".join([addr, port]) diff --git a/hivemind/p2p/multiaddr/codecs/onion3.py b/hivemind/p2p/multiaddr/codecs/onion3.py index d12bae588..c489ead8f 100644 --- a/hivemind/p2p/multiaddr/codecs/onion3.py +++ b/hivemind/p2p/multiaddr/codecs/onion3.py @@ -1,7 +1,6 @@ import base64 import struct - SIZE = 296 IS_PATH = False @@ -27,11 +26,11 @@ def to_bytes(proto, string): if port not in range(1, 65536): raise ValueError("Port number is not in range(1, 65536)") - return b''.join((onion3_host_bytes, struct.pack('>H', port))) + return b"".join((onion3_host_bytes, struct.pack(">H", port))) def to_string(proto, buf): addr_bytes, port_bytes = (buf[:-2], buf[-2:]) - addr = base64.b32encode(addr_bytes).decode('ascii').lower() - port = str(struct.unpack('>H', port_bytes)[0]) - return ':'.join([addr, port]) + addr = base64.b32encode(addr_bytes).decode("ascii").lower() + port = str(struct.unpack(">H", port_bytes)[0]) + return ":".join([addr, port]) diff --git a/hivemind/p2p/multiaddr/codecs/uint16be.py b/hivemind/p2p/multiaddr/codecs/uint16be.py index 73442e430..a4b3edde5 100644 --- a/hivemind/p2p/multiaddr/codecs/uint16be.py +++ b/hivemind/p2p/multiaddr/codecs/uint16be.py @@ -1,13 +1,12 @@ import struct - SIZE = 16 IS_PATH = False def to_bytes(proto, string): try: - return struct.pack('>H', int(string, 10)) + return struct.pack(">H", int(string, 10)) except ValueError as exc: raise ValueError("Not a base 10 integer") from exc except struct.error as exc: @@ -17,4 +16,4 @@ def to_bytes(proto, string): def to_string(proto, buf): if len(buf) != 2: raise ValueError("Invalid integer length (must be 2 bytes / 16 bits)") - return str(struct.unpack('>H', buf)[0]) + return str(struct.unpack(">H", buf)[0]) diff --git a/hivemind/p2p/multiaddr/codecs/utf8.py b/hivemind/p2p/multiaddr/codecs/utf8.py index 5a85e2793..6b84077c3 100644 --- a/hivemind/p2p/multiaddr/codecs/utf8.py +++ b/hivemind/p2p/multiaddr/codecs/utf8.py @@ -2,7 +2,6 @@ from . import LENGTH_PREFIXED_VAR_SIZE - SIZE = LENGTH_PREFIXED_VAR_SIZE IS_PATH = False @@ -10,10 +9,10 @@ def to_bytes(proto, string): if len(string) == 0: raise ValueError("{0} value must not be empty".format(proto.name)) - return string.encode('utf-8') + return string.encode("utf-8") def to_string(proto, buf): if len(buf) == 0: raise ValueError("invalid length (should be > 0)") - return buf.decode('utf-8') + return buf.decode("utf-8") diff --git a/hivemind/p2p/multiaddr/exceptions.py b/hivemind/p2p/multiaddr/exceptions.py index c92fb8b35..d65c3a402 100644 --- a/hivemind/p2p/multiaddr/exceptions.py +++ b/hivemind/p2p/multiaddr/exceptions.py @@ -15,9 +15,7 @@ def __init__(self, proto, string): self.proto = proto self.string = string - super().__init__( - "MultiAddr {0!r} does not contain protocol {1}".format(string, proto) - ) + super().__init__("MultiAddr {0!r} does not contain protocol {1}".format(string, proto)) class ParseError(ValueError, Error): @@ -68,27 +66,26 @@ class ProtocolRegistryError(Error): class ProtocolRegistryLocked(Error): """Protocol registry was locked and doesn't allow any further additions""" + def __init__(self): super().__init__("Protocol registry is locked and does not accept any new values") class ProtocolExistsError(ProtocolRegistryError): """Protocol with the given name or code already exists""" + def __init__(self, proto, kind="name"): self.proto = proto self.kind = kind - super().__init__( - "Protocol with {0} {1!r} already exists".format(kind, getattr(proto, kind)) - ) + super().__init__("Protocol with {0} {1!r} already exists".format(kind, getattr(proto, kind))) class ProtocolNotFoundError(ProtocolRegistryError): """No protocol with the given name or code found""" + def __init__(self, value, kind="name"): self.value = value self.kind = kind - super().__init__( - "No protocol with {0} {1!r} found".format(kind, value) - ) + super().__init__("No protocol with {0} {1!r} found".format(kind, value)) diff --git a/hivemind/p2p/multiaddr/multiaddr.py b/hivemind/p2p/multiaddr/multiaddr.py index 52f51a055..0f9a0f491 100644 --- a/hivemind/p2p/multiaddr/multiaddr.py +++ b/hivemind/p2p/multiaddr/multiaddr.py @@ -3,11 +3,7 @@ import varint from . import exceptions, protocols - -from .transforms import bytes_iter -from .transforms import string_to_bytes -from .transforms import bytes_to_string - +from .transforms import bytes_iter, bytes_to_string, string_to_bytes __all__ = ("Multiaddr",) @@ -19,7 +15,7 @@ def __contains__(self, proto): def __getitem__(self, idx): if idx < 0: - idx = len(self)+idx + idx = len(self) + idx for idx2, proto in enumerate(self): if idx2 == idx: return proto @@ -40,7 +36,7 @@ def __contains__(self, item): def __getitem__(self, idx): if idx < 0: - idx = len(self)+idx + idx = len(self) + idx for idx2, item in enumerate(self): if idx2 == idx: return item @@ -54,11 +50,11 @@ def __iter__(self): yield proto, codec.to_string(proto, part) except Exception as exc: raise exceptions.BinaryParseError( - str(exc), - self._mapping.to_bytes(), - proto.name, - exc, - ) from exc + str(exc), + self._mapping.to_bytes(), + proto.name, + exc, + ) from exc else: # We were given something like '/utp', which doesn't have # an address, so return None @@ -70,7 +66,7 @@ class MultiAddrValues(collections.abc.ValuesView, collections.abc.Sequence): def __getitem__(self, idx): if idx < 0: - idx = len(self)+idx + idx = len(self) + idx for idx2, proto in enumerate(self): if idx2 == idx: return proto diff --git a/hivemind/p2p/multiaddr/protocols.py b/hivemind/p2p/multiaddr/protocols.py index ba385ba69..57321860d 100644 --- a/hivemind/p2p/multiaddr/protocols.py +++ b/hivemind/p2p/multiaddr/protocols.py @@ -43,8 +43,8 @@ class Protocol: __slots__ = [ - "code", # int - "name", # string + "code", # int + "name", # string "codec", # string ] @@ -76,10 +76,9 @@ def __eq__(self, other): if not isinstance(other, Protocol): return NotImplemented - return all((self.code == other.code, - self.name == other.name, - self.codec == other.codec, - self.path == other.path)) + return all( + (self.code == other.code, self.name == other.name, self.codec == other.codec, self.path == other.path) + ) def __hash__(self): return self.code @@ -94,39 +93,40 @@ def __repr__(self): # List of multiaddr protocols supported by this module by default PROTOCOLS = [ - Protocol(P_IP4, 'ip4', 'ip4'), - Protocol(P_TCP, 'tcp', 'uint16be'), - Protocol(P_UDP, 'udp', 'uint16be'), - Protocol(P_DCCP, 'dccp', 'uint16be'), - Protocol(P_IP6, 'ip6', 'ip6'), - Protocol(P_IP6ZONE, 'ip6zone', 'utf8'), - Protocol(P_DNS, 'dns', 'domain'), - Protocol(P_DNS4, 'dns4', 'domain'), - Protocol(P_DNS6, 'dns6', 'domain'), - Protocol(P_DNSADDR, 'dnsaddr', 'domain'), - Protocol(P_SCTP, 'sctp', 'uint16be'), - Protocol(P_UDT, 'udt', None), - Protocol(P_UTP, 'utp', None), - Protocol(P_P2P, 'p2p', 'cid'), - Protocol(P_ONION, 'onion', 'onion'), - Protocol(P_ONION3, 'onion3', 'onion3'), - Protocol(P_QUIC, 'quic', None), - Protocol(P_QUIC1, 'quic-v1', None), - Protocol(P_HTTP, 'http', None), - Protocol(P_HTTPS, 'https', None), - Protocol(P_TLS, 'tls', None), - Protocol(P_WS, 'ws', None), - Protocol(P_WSS, 'wss', None), - Protocol(P_P2P_WEBSOCKET_STAR, 'p2p-websocket-star', None), - Protocol(P_P2P_WEBRTC_STAR, 'p2p-webrtc-star', None), - Protocol(P_P2P_WEBRTC_DIRECT, 'p2p-webrtc-direct', None), - Protocol(P_P2P_CIRCUIT, 'p2p-circuit', None), - Protocol(P_UNIX, 'unix', 'fspath'), + Protocol(P_IP4, "ip4", "ip4"), + Protocol(P_TCP, "tcp", "uint16be"), + Protocol(P_UDP, "udp", "uint16be"), + Protocol(P_DCCP, "dccp", "uint16be"), + Protocol(P_IP6, "ip6", "ip6"), + Protocol(P_IP6ZONE, "ip6zone", "utf8"), + Protocol(P_DNS, "dns", "domain"), + Protocol(P_DNS4, "dns4", "domain"), + Protocol(P_DNS6, "dns6", "domain"), + Protocol(P_DNSADDR, "dnsaddr", "domain"), + Protocol(P_SCTP, "sctp", "uint16be"), + Protocol(P_UDT, "udt", None), + Protocol(P_UTP, "utp", None), + Protocol(P_P2P, "p2p", "cid"), + Protocol(P_ONION, "onion", "onion"), + Protocol(P_ONION3, "onion3", "onion3"), + Protocol(P_QUIC, "quic", None), + Protocol(P_QUIC1, "quic-v1", None), + Protocol(P_HTTP, "http", None), + Protocol(P_HTTPS, "https", None), + Protocol(P_TLS, "tls", None), + Protocol(P_WS, "ws", None), + Protocol(P_WSS, "wss", None), + Protocol(P_P2P_WEBSOCKET_STAR, "p2p-websocket-star", None), + Protocol(P_P2P_WEBRTC_STAR, "p2p-webrtc-star", None), + Protocol(P_P2P_WEBRTC_DIRECT, "p2p-webrtc-direct", None), + Protocol(P_P2P_CIRCUIT, "p2p-circuit", None), + Protocol(P_UNIX, "unix", "fspath"), ] class ProtocolRegistry: """A collection of individual Multiaddr protocols indexed for fast lookup""" + __slots__ = ("_codes_to_protocols", "_locked", "_names_to_protocols") def __init__(self, protocols=()): @@ -179,8 +179,9 @@ def add_alias_name(self, proto, alias_name): raise exceptions.ProtocolRegistryLocked() proto = self.find(proto) - assert self._names_to_protocols.get(proto.name) is proto, \ - "Protocol to alias must have already been added to the registry" + assert ( + self._names_to_protocols.get(proto.name) is proto + ), "Protocol to alias must have already been added to the registry" if alias_name in self._names_to_protocols: raise exceptions.ProtocolExistsError(self._names_to_protocols[alias_name], "name") @@ -206,8 +207,9 @@ def add_alias_code(self, proto, alias_code): raise exceptions.ProtocolRegistryLocked() proto = self.find(proto) - assert self._codes_to_protocols.get(proto.code) is proto, \ - "Protocol to alias must have already been added to the registry" + assert ( + self._codes_to_protocols.get(proto.code) is proto + ), "Protocol to alias must have already been added to the registry" if alias_code in self._codes_to_protocols: raise exceptions.ProtocolExistsError(self._codes_to_protocols[alias_code], "name") diff --git a/hivemind/p2p/multiaddr/transforms.py b/hivemind/p2p/multiaddr/transforms.py index 951c824c6..dd95bdf6a 100644 --- a/hivemind/p2p/multiaddr/transforms.py +++ b/hivemind/p2p/multiaddr/transforms.py @@ -1,13 +1,10 @@ import io + import varint from . import exceptions - -from .codecs import LENGTH_PREFIXED_VAR_SIZE -from .codecs import codec_by_name - -from .protocols import protocol_with_code -from .protocols import protocol_with_name +from .codecs import LENGTH_PREFIXED_VAR_SIZE, codec_by_name +from .protocols import protocol_with_code, protocol_with_name def string_to_bytes(string): @@ -22,11 +19,11 @@ def string_to_bytes(string): if codec.SIZE == LENGTH_PREFIXED_VAR_SIZE: bs.append(varint.encode(len(buf))) bs.append(buf) - return b''.join(bs) + return b"".join(bs) def bytes_to_string(buf): - st = [''] # start with empty string so we get a leading slash on join() + st = [""] # start with empty string so we get a leading slash on join() for _, proto, codec, part in bytes_iter(buf): st.append(proto.name) if codec.SIZE != 0: @@ -34,11 +31,11 @@ def bytes_to_string(buf): value = codec.to_string(proto, part) except Exception as exc: raise exceptions.BinaryParseError(str(exc), buf, proto.name, exc) from exc - if codec.IS_PATH and value[0] == '/': + if codec.IS_PATH and value[0] == "/": st.append(value[1:]) else: st.append(value) - return '/'.join(st) + return "/".join(st) def size_for_addr(codec, buf_io): @@ -49,11 +46,11 @@ def size_for_addr(codec, buf_io): def string_iter(string): - if not string.startswith('/'): + if not string.startswith("/"): raise exceptions.StringParseError("Must begin with /", string) # consume trailing slashes - string = string.rstrip('/') - sp = string.split('/') + string = string.rstrip("/") + sp = string.split("/") # skip the first element, since it starts with / sp.pop(0) @@ -87,10 +84,10 @@ def bytes_iter(buf): codec = codec_by_name(proto.codec) except (ImportError, exceptions.ProtocolNotFoundError) as exc: raise exceptions.BinaryParseError( - "Unknown Protocol", - buf, - proto.name if proto else code, - ) from exc + "Unknown Protocol", + buf, + proto.name if proto else code, + ) from exc size = size_for_addr(codec, buf_io) yield offset, proto, codec, buf_io.read(size) From 8df748fc4ac4093c36e0a1490039e2430fd2feb1 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Tue, 3 Dec 2024 21:28:31 +0300 Subject: [PATCH 11/12] update protobuf requiresments --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f32fc94c8..951eb31da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ msgpack>=0.5.6 sortedcontainers uvloop>=0.14.0 grpcio-tools>=1.33.2 -protobuf>=3.12.2,<5.28.0 +protobuf>=3.12.2,<5.29.0 configargparse>=1.2.3 py-multihash>=0.2.3 multiaddr @ git+https://github.com/multiformats/py-multiaddr.git@e01dbd38f2c0464c0f78b556691d655265018cce From 5b29f81bc6834871f8b80dfc1a40b5997d3a8ffb Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Tue, 3 Dec 2024 21:39:29 +0300 Subject: [PATCH 12/12] style --- .../p2p/p2p_daemon_bindings/datastructures.py | 1 - tests/test_multiaddr.py | 358 ------------------ 2 files changed, 359 deletions(-) delete mode 100644 tests/test_multiaddr.py diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index b52714ee5..b0be1bb38 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -10,7 +10,6 @@ import base58 import multihash from cryptography.hazmat.primitives import serialization - from multiaddr import Multiaddr from hivemind.p2p.multiaddr import Multiaddr, protocols diff --git a/tests/test_multiaddr.py b/tests/test_multiaddr.py deleted file mode 100644 index e7cb2bab3..000000000 --- a/tests/test_multiaddr.py +++ /dev/null @@ -1,358 +0,0 @@ -import pytest - -from hivemind.p2p.multiaddr.exceptions import ( - BinaryParseError, - ProtocolLookupError, - ProtocolNotFoundError, - StringParseError, -) -from hivemind.p2p.multiaddr.multiaddr import Multiaddr -from hivemind.p2p.multiaddr.protocols import ( - P_DNS, - P_IP4, - P_IP6, - P_P2P, - P_TCP, - P_UDP, - P_UNIX, - P_UTP, - protocol_with_name, - protocols_with_string, -) - - -@pytest.mark.parametrize( - "addr_str", - [ - "/ip4", - "/ip4/::1", - "/ip4/fdpsofodsajfdoisa", - "/ip6", - "/ip6zone", - "/ip6zone/", - "/ip6zone//ip6/fe80::1", - "/udp", - "/tcp", - "/sctp", - "/udp/65536", - "/tcp/65536", - "/udp/1234/sctp", - "/udp/1234/udt/1234", - "/udp/1234/utp/1234", - "/ip4/127.0.0.1/udp/jfodsajfidosajfoidsa", - "/ip4/127.0.0.1/udp", - "/ip4/127.0.0.1/tcp/jfodsajfidosajfoidsa", - "/ip4/127.0.0.1/tcp", - "/ip4/127.0.0.1/p2p", - "/ip4/127.0.0.1/p2p/tcp", - "/unix", - "/ip4/1.2.3.4/tcp/80/unix", - "/ip4/127.0.0.1/tcp/9090/http/p2p-webcrt-direct", - "/dns", - "/dns4", - "/dns6", - "/cancer", - ], -) -def test_invalid(addr_str): - with pytest.raises(StringParseError): - Multiaddr(addr_str) - - -@pytest.mark.parametrize( - "addr_str", - [ - "/ip4/1.2.3.4", - "/ip4/0.0.0.0", - "/ip6/::1", - "/ip6/2601:9:4f81:9700:803e:ca65:66e8:c21", - "/ip6zone/x/ip6/fe80::1", - "/ip6zone/x%y/ip6/fe80::1", - "/ip6zone/x%y/ip6/::", - "/udp/0", - "/tcp/0", - "/sctp/0", - "/udp/1234", - "/tcp/1234", - "/sctp/1234", - "/utp", - "/udp/65535", - "/tcp/65535", - "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC", - "/udp/1234/sctp/1234", - "/udp/1234/udt", - "/udp/1234/utp", - "/tcp/1234/http", - "/tcp/1234/https", - "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", - "/ip4/127.0.0.1/udp/1234", - "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", - "/unix/a/b/c/d/e", - "/unix/Überrschung!/大柱", - "/unix/stdio", - "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f", - "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC" "/tcp/1234/unix/stdio", - "/ip4/127.0.0.1/tcp/9090/http/p2p-webrtc-direct", - ], -) # nopep8 -def test_valid(addr_str): - ma = Multiaddr(addr_str) - assert str(ma) == addr_str.rstrip("/") - - -def test_eq(): - m1 = Multiaddr("/ip4/127.0.0.1/udp/1234") - m2 = Multiaddr("/ip4/127.0.0.1/tcp/1234") - m3 = Multiaddr("/ip4/127.0.0.1/tcp/1234") - m4 = Multiaddr("/ip4/127.0.0.1/tcp/1234/") - - assert m1 != m2 - assert m2 != m1 - - assert m2 == m3 - assert m3 == m2 - - assert m1 == m1 - - assert m2 == m4 - assert m4 == m2 - assert m3 == m4 - assert m4 == m3 - - -def test_protocols(): - ma = Multiaddr("/ip4/127.0.0.1/udp/1234") - protos = ma.protocols() - assert protos[0].code == protocol_with_name("ip4").code - assert protos[1].code == protocol_with_name("udp").code - - -@pytest.mark.parametrize( - "proto_string,expected", - [ - ("/ip4", [protocol_with_name("ip4")]), - ("/ip4/tcp", [protocol_with_name("ip4"), protocol_with_name("tcp")]), - ( - "ip4/tcp/udp/ip6", - [ - protocol_with_name("ip4"), - protocol_with_name("tcp"), - protocol_with_name("udp"), - protocol_with_name("ip6"), - ], - ), - ("////////ip4/tcp", [protocol_with_name("ip4"), protocol_with_name("tcp")]), - ("ip4/udp/////////", [protocol_with_name("ip4"), protocol_with_name("udp")]), - ("////////ip4/tcp////////", [protocol_with_name("ip4"), protocol_with_name("tcp")]), - ("////////ip4/////////tcp////////", [protocol_with_name("ip4"), protocol_with_name("tcp")]), - ], -) -def test_protocols_with_string(proto_string, expected): - protos = protocols_with_string(proto_string) - assert protos == expected - - -@pytest.mark.parametrize("proto_string", ["dsijafd", "/ip4/tcp/fidosafoidsa", "////////ip4/tcp/21432141/////////"]) -def test_invalid_protocols_with_string(proto_string): - with pytest.raises(ProtocolNotFoundError): - protocols_with_string(proto_string) - - -@pytest.mark.parametrize( - "proto_string,maxsplit,expected", - [ - ("/ip4/1.2.3.4", -1, ("/ip4/1.2.3.4",)), - ("/ip4/0.0.0.0", 0, ("/ip4/0.0.0.0",)), - ("/ip6/::1", 1, ("/ip6/::1",)), - ( - "/ip4/127.0.0.1/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234", - 1, - ("/ip4/127.0.0.1", "/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234"), - ), - ("/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f", -1, ("/ip4/1.2.3.4", "/tcp/80", "/unix/a/b/c/d/e/f")), - ], -) -def test_split(proto_string, maxsplit, expected): - assert tuple(map(str, Multiaddr(proto_string).split(maxsplit))) == expected - - -@pytest.mark.parametrize( - "proto_parts,expected", - [ - (("/ip4/1.2.3.4",), "/ip4/1.2.3.4"), - ((b"\x04\x00\x00\x00\x00",), "/ip4/0.0.0.0"), - (("/ip6/::1",), "/ip6/::1"), - ( - ( - b"\x04\x7F\x00\x00\x01", - "/p2p/bafzbeigvf25ytwc3akrijfecaotc74udrhcxzh2cx3we5qqnw5vgrei4bm/tcp/1234", - ), - "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234", - ), - (("/ip4/1.2.3.4", "/tcp/80", "/unix/a/b/c/d/e/f"), "/ip4/1.2.3.4/tcp/80/unix/a/b/c/d/e/f"), - ], -) -def test_join(proto_parts, expected): - assert str(Multiaddr.join(*proto_parts)) == expected - - -def test_encapsulate(): - m1 = Multiaddr("/ip4/127.0.0.1/udp/1234") - m2 = Multiaddr("/udp/5678") - - encapsulated = m1.encapsulate(m2) - assert str(encapsulated) == "/ip4/127.0.0.1/udp/1234/udp/5678" - - m3 = Multiaddr("/udp/5678") - decapsulated = encapsulated.decapsulate(m3) - assert str(decapsulated) == "/ip4/127.0.0.1/udp/1234" - - m4 = Multiaddr("/ip4/127.0.0.1") - decapsulated_2 = decapsulated.decapsulate(m4) - assert str(decapsulated_2) == "" - - m5 = Multiaddr("/ip6/::1") - decapsulated_3 = decapsulated.decapsulate(m5) - - assert str(decapsulated_3) == "/ip4/127.0.0.1/udp/1234" - - -def assert_value_for_proto(multi, proto, expected): - assert multi.value_for_protocol(proto) == expected - - -def test_get_value(): - ma = Multiaddr( - "/ip4/127.0.0.1/utp/tcp/5555/udp/1234/utp/" "p2p/bafzbeigalb34xlqdtvyklzqa5ibmn6pssqsdskc4ty2e4jxy2kamquh22y" - ) - - assert_value_for_proto(ma, P_IP4, "127.0.0.1") - assert_value_for_proto(ma, P_UTP, None) - assert_value_for_proto(ma, P_TCP, "5555") - assert_value_for_proto(ma, P_UDP, "1234") - assert_value_for_proto(ma, P_P2P, "QmbHVEEepCi7rn7VL7Exxpd2Ci9NNB6ifvqwhsrbRMgQFP") - assert_value_for_proto(ma, "ip4", "127.0.0.1") - assert_value_for_proto(ma, "utp", None) - assert_value_for_proto(ma, "tcp", "5555") - assert_value_for_proto(ma, "udp", "1234") - assert_value_for_proto(ma, protocol_with_name("ip4"), "127.0.0.1") - assert_value_for_proto(ma, protocol_with_name("utp"), None) - assert_value_for_proto(ma, protocol_with_name("tcp"), "5555") - assert_value_for_proto(ma, protocol_with_name("udp"), "1234") - - with pytest.raises(ProtocolLookupError): - ma.value_for_protocol(P_IP6) - with pytest.raises(ProtocolLookupError): - ma.value_for_protocol("ip6") - with pytest.raises(ProtocolLookupError): - ma.value_for_protocol(protocol_with_name("ip6")) - - a = Multiaddr(b"\x35\x03a:b") # invalid protocol value - with pytest.raises(BinaryParseError): - a.value_for_protocol(P_DNS) - - a = Multiaddr("/ip4/0.0.0.0") # only one addr - assert_value_for_proto(a, P_IP4, "0.0.0.0") - - a = Multiaddr("/ip4/0.0.0.0/ip4/0.0.0.0/ip4/0.0.0.0") # same sub-addr - assert_value_for_proto(a, P_IP4, "0.0.0.0") - - a = Multiaddr("/ip4/0.0.0.0/udp/12345/utp") # ending in a no-value one. - assert_value_for_proto(a, P_IP4, "0.0.0.0") - assert_value_for_proto(a, P_UDP, "12345") - assert_value_for_proto(a, P_UTP, None) - - a = Multiaddr("/ip4/0.0.0.0/unix/a/b/c/d") # ending in a path one. - assert_value_for_proto(a, P_IP4, "0.0.0.0") - assert_value_for_proto(a, P_UNIX, "/a/b/c/d") - - a = Multiaddr("/unix/studio") - assert_value_for_proto(a, P_UNIX, "/studio") # only a path. - - -def test_views(): - ma = Multiaddr( - "/ip4/127.0.0.1/utp/tcp/5555/udp/1234/utp/" "p2p/bafzbeigalb34xlqdtvyklzqa5ibmn6pssqsdskc4ty2e4jxy2kamquh22y" - ) - - for idx, (proto1, proto2, item, value) in enumerate(zip(ma, ma.keys(), ma.items(), ma.values())): # noqa: E501 - assert (proto1, value) == (proto2, value) == item - assert proto1 in ma - assert proto2 in ma.keys() - assert item in ma.items() - assert value in ma.values() - assert ma.keys()[idx] == ma.keys()[idx - len(ma)] == proto1 == proto2 - assert ma.items()[idx] == ma.items()[idx - len(ma)] == item - assert ma.values()[idx] == ma.values()[idx - len(ma)] == ma[proto1] == value - - assert len(ma.keys()) == len(ma.items()) == len(ma.values()) == len(ma) - assert len(list(ma.keys())) == len(ma.keys()) - assert len(list(ma.items())) == len(ma.items()) - assert len(list(ma.values())) == len(ma.values()) - - with pytest.raises(IndexError): - ma.keys()[len(ma)] - with pytest.raises(IndexError): - ma.items()[len(ma)] - with pytest.raises(IndexError): - ma.values()[len(ma)] - - -def test_bad_initialization_no_params(): - with pytest.raises(TypeError): - Multiaddr() - - -def test_bad_initialization_too_many_params(): - with pytest.raises(TypeError): - Multiaddr("/ip4/0.0.0.0", "") - - -def test_bad_initialization_wrong_type(): - with pytest.raises(TypeError): - Multiaddr(42) - - -def test_value_for_protocol_argument_wrong_type(): - a = Multiaddr("/ip4/127.0.0.1/udp/1234") - with pytest.raises(ProtocolNotFoundError): - a.value_for_protocol("str123") - - with pytest.raises(TypeError): - a.value_for_protocol(None) - - -def test_multi_addr_str_corruption(): - a = Multiaddr("/ip4/127.0.0.1/udp/1234") - a._bytes = b"047047047" - - with pytest.raises(BinaryParseError): - str(a) - - -def test_decapsulate(): - a = Multiaddr("/ip4/127.0.0.1/udp/1234") - u = Multiaddr("/udp/1234") - assert a.decapsulate(u) == Multiaddr("/ip4/127.0.0.1") - - -def test__repr(): - a = Multiaddr("/ip4/127.0.0.1/udp/1234") - assert repr(a) == "" % str(a) - - -def test_zone(): - ip6_string = "/ip6zone/eth0/ip6/::1" - ip6_bytes = b"\x2a\x04eth0\x29\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01" - - maddr_from_str = Multiaddr(ip6_string) - assert maddr_from_str.to_bytes() == ip6_bytes - - maddr_from_bytes = Multiaddr(ip6_bytes) - assert str(maddr_from_bytes) == ip6_string - - -def test_hash(): - addr_bytes = Multiaddr("/ip4/127.0.0.1/udp/1234").to_bytes() - - assert hash(Multiaddr(addr_bytes)) == hash(addr_bytes)