Add multiaddr as subpackage

hivemind/dht/dht.py

@@ -7,12 +7,11 @@
 from functools import partial
 from typing import Awaitable, Callable, Iterable, List, Optional, Sequence, TypeVar, Union
 
-from multiaddr import Multiaddr
-
 from hivemind.dht.node import DEFAULT_NUM_WORKERS, DHTNode
 from hivemind.dht.routing import DHTKey, DHTValue, Subkey
 from hivemind.dht.validation import CompositeValidator, RecordValidatorBase
 from hivemind.p2p import P2P, PeerID
+from hivemind.p2p.multiaddr import Multiaddr
 from hivemind.utils import MPFuture, get_logger, switch_to_uvloop
 from hivemind.utils.timed_storage import DHTExpiration, ValueWithExpiration
 

hivemind/dht/node.py

@@ -23,7 +23,6 @@
     Union,
 )
 
-from multiaddr import Multiaddr
 from sortedcontainers import SortedSet
 
 from hivemind.dht.crypto import DHTRecord, RecordValidatorBase
@@ -32,6 +31,7 @@
 from hivemind.dht.storage import DictionaryDHTValue
 from hivemind.dht.traverse import traverse_dht
 from hivemind.p2p import P2P, PeerID
+from hivemind.p2p.multiaddr import Multiaddr
 from hivemind.utils import MSGPackSerializer, SerializerBase, get_logger
 from hivemind.utils.auth import AuthorizerBase
 from hivemind.utils.timed_storage import DHTExpiration, TimedStorage, ValueWithExpiration

hivemind/p2p/multiaddr/__init__.py

@@ -0,0 +1,5 @@
+from .multiaddr import Multiaddr  # NOQA
+
+__author__ = "Steven Buss"
+__email__ = "[email protected]"
+__version__ = "0.0.9"

hivemind/p2p/multiaddr/codecs/__init__.py

@@ -0,0 +1,21 @@
+import importlib
+
+# These are special sizes
+LENGTH_PREFIXED_VAR_SIZE = -1
+
+
+class NoneCodec:
+    SIZE = 0
+    IS_PATH = False
+
+
+CODEC_CACHE = {}
+
+
+def codec_by_name(name):
+    if name is None:  # Special “do nothing – expect nothing” pseudo-codec
+        return NoneCodec
+    codec = CODEC_CACHE.get(name)
+    if not codec:
+        codec = CODEC_CACHE[name] = importlib.import_module(".{0}".format(name), __name__)
+    return codec

hivemind/p2p/multiaddr/codecs/cid.py

@@ -0,0 +1,128 @@
+import base58
+import cid
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = False
+
+
+# Spec: https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation
+CIDv0_PREFIX_TO_LENGTH = {
+    # base58btc prefixes for valid lengths 1 – 42 with the identity “hash” function
+    "12": [5, 12, 19, 23, 30, 41, 52, 56],
+    "13": [9, 16, 34, 45],
+    "14": [27, 38, 49, 60],
+    "15": [3, 6, 20],
+    "16": [3, 6, 13, 20, 31, 42, 53],
+    "17": [3, 13, 42],
+    "18": [3],
+    "19": [3, 24, 57],
+    "1A": [24, 35, 46],
+    "1B": [35],
+    "1D": [17],
+    "1E": [10, 17],
+    "1F": [10],
+    "1G": [10, 28, 50],
+    "1H": [28, 39],
+    "1P": [21],
+    "1Q": [21],
+    "1R": [21, 54],
+    "1S": [54],
+    "1T": [7, 32, 43],
+    "1U": [7, 32, 43],
+    "1V": [7],
+    "1W": [7, 14],
+    "1X": [7, 14],
+    "1Y": [7, 14],
+    "1Z": [7, 14],
+    "1f": [4],
+    "1g": [4, 58],
+    "1h": [4, 25, 58],
+    "1i": [4, 25],
+    "1j": [4, 25],
+    "1k": [4, 25, 47],
+    "1m": [4, 36, 47],
+    "1n": [4, 36],
+    "1o": [4, 36],
+    "1p": [4],
+    "1q": [4],
+    "1r": [4],
+    "1s": [4],
+    "1t": [4],
+    "1u": [4],
+    "1v": [4],
+    "1w": [4],
+    "1x": [4],
+    "1y": [4],
+    "1z": [4, 18],
+    # base58btc prefix for length 42 with the sha256 hash function
+    "Qm": [46],
+}
+
+PROTO_NAME_TO_CIDv1_CODEC = {
+    # The “p2p” multiaddr protocol requires all keys to use the “libp2p-key” multicodec
+    "p2p": "libp2p-key",
+}
+
+
+def to_bytes(proto, string):
+    expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name)
+
+    if len(string) in CIDv0_PREFIX_TO_LENGTH.get(string[0:2], ()):  # CIDv0
+        # Upgrade the wire (binary) representation of any received CIDv0 string
+        # to CIDv1 if we can determine which multicodec value to use
+        if expected_codec:
+            return cid.make_cid(1, expected_codec, base58.b58decode(string)).buffer
+
+        return base58.b58decode(string)
+    else:  # CIDv1+
+        parsed = cid.from_string(string)
+
+        # Ensure CID has correct codec for protocol
+        if expected_codec and parsed.codec != expected_codec:
+            raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec".format(proto.name, expected_codec))
+
+        return parsed.buffer
+
+
+def _is_binary_cidv0_multihash(buf):
+    if buf.startswith(b"\x12\x20") and len(buf) == 34:  # SHA2-256
+        return True
+
+    if (buf[0] == 0x00 and buf[1] in range(43)) and len(buf) == (buf[1] + 2):  # Identity hash
+        return True
+
+    return False
+
+
+def to_string(proto, buf):
+    expected_codec = PROTO_NAME_TO_CIDv1_CODEC.get(proto.name)
+
+    if _is_binary_cidv0_multihash(buf):  # CIDv0
+        if not expected_codec:
+            # Simply encode as base58btc as there is nothing better to do
+            return base58.b58encode(buf).decode("ascii")
+
+        # “Implementations SHOULD display peer IDs using the first (raw
+        #  base58btc encoded multihash) format until the second format is
+        #  widely supported.”
+        #
+        # In the future the following line should instead convert the multihash
+        # to CIDv1 and with the `expected_codec` and wrap it in base32:
+        #   return cid.make_cid(1, expected_codec, buf).encode("base32").decode("ascii")
+        return base58.b58encode(buf).decode("ascii")
+    else:  # CIDv1+
+        parsed = cid.from_bytes(buf)
+
+        # Ensure CID has correct codec for protocol
+        if expected_codec and parsed.codec != expected_codec:
+            raise ValueError("“{0}” multiaddr CIDs must use the “{1}” multicodec".format(proto.name, expected_codec))
+
+        # “Implementations SHOULD display peer IDs using the first (raw
+        #  base58btc encoded multihash) format until the second format is
+        #  widely supported.”
+        if expected_codec and _is_binary_cidv0_multihash(parsed.multihash):
+            return base58.b58encode(parsed.multihash).decode("ascii")
+
+        return parsed.encode("base32").decode("ascii")

hivemind/p2p/multiaddr/codecs/domain.py

@@ -0,0 +1,17 @@
+import idna
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    return idna.uts46_remap(string).encode("utf-8")
+
+
+def to_string(proto, buf):
+    string = buf.decode("utf-8")
+    for label in string.split("."):
+        idna.check_label(label)
+    return string

hivemind/p2p/multiaddr/codecs/fspath.py

@@ -0,0 +1,14 @@
+import os
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = True
+
+
+def to_bytes(proto, string):
+    return os.fsencode(string)
+
+
+def to_string(proto, buf):
+    return os.fsdecode(buf)

hivemind/p2p/multiaddr/codecs/ip4.py

@@ -0,0 +1,12 @@
+import netaddr
+
+SIZE = 32
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    return netaddr.IPAddress(string, version=4).packed
+
+
+def to_string(proto, buf):
+    return str(netaddr.IPAddress(int.from_bytes(buf, byteorder="big"), version=4))

hivemind/p2p/multiaddr/codecs/ip6.py

@@ -0,0 +1,12 @@
+import netaddr
+
+SIZE = 128
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    return netaddr.IPAddress(string, version=6).packed
+
+
+def to_string(proto, buf):
+    return str(netaddr.IPAddress(int.from_bytes(buf, byteorder="big"), version=6))

hivemind/p2p/multiaddr/codecs/onion.py

@@ -0,0 +1,36 @@
+import base64
+import struct
+
+SIZE = 96
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    addr = string.split(":")
+    if len(addr) != 2:
+        raise ValueError("Does not contain a port number")
+
+    # onion address without the ".onion" substring
+    if len(addr[0]) != 16:
+        raise ValueError("Invalid onion host address length (must be 16 characters)")
+    try:
+        onion_host_bytes = base64.b32decode(addr[0].upper())
+    except Exception as exc:
+        raise ValueError("Cannot decode {0!r} as base32: {1}".format(addr[0], exc)) from exc
+
+    # onion port number
+    try:
+        port = int(addr[1], 10)
+    except ValueError as exc:
+        raise ValueError("Port number is not a base 10 integer") from exc
+    if port not in range(1, 65536):
+        raise ValueError("Port number is not in range(1, 65536)")
+
+    return b"".join((onion_host_bytes, struct.pack(">H", port)))
+
+
+def to_string(proto, buf):
+    addr_bytes, port_bytes = (buf[:-2], buf[-2:])
+    addr = base64.b32encode(addr_bytes).decode("ascii").lower()
+    port = str(struct.unpack(">H", port_bytes)[0])
+    return ":".join([addr, port])

hivemind/p2p/multiaddr/codecs/onion3.py

@@ -0,0 +1,36 @@
+import base64
+import struct
+
+SIZE = 296
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    addr = string.split(":")
+    if len(addr) != 2:
+        raise ValueError("Does not contain a port number")
+
+    # onion3 address without the ".onion" substring
+    if len(addr[0]) != 56:
+        raise ValueError("Invalid onion3 host address length (must be 56 characters)")
+    try:
+        onion3_host_bytes = base64.b32decode(addr[0].upper())
+    except Exception as exc:
+        raise ValueError("Cannot decode {0!r} as base32: {1}".format(addr[0], exc)) from exc
+
+    # onion3 port number
+    try:
+        port = int(addr[1], 10)
+    except ValueError as exc:
+        raise ValueError("Port number is not a base 10 integer") from exc
+    if port not in range(1, 65536):
+        raise ValueError("Port number is not in range(1, 65536)")
+
+    return b"".join((onion3_host_bytes, struct.pack(">H", port)))
+
+
+def to_string(proto, buf):
+    addr_bytes, port_bytes = (buf[:-2], buf[-2:])
+    addr = base64.b32encode(addr_bytes).decode("ascii").lower()
+    port = str(struct.unpack(">H", port_bytes)[0])
+    return ":".join([addr, port])

hivemind/p2p/multiaddr/codecs/uint16be.py

@@ -0,0 +1,19 @@
+import struct
+
+SIZE = 16
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    try:
+        return struct.pack(">H", int(string, 10))
+    except ValueError as exc:
+        raise ValueError("Not a base 10 integer") from exc
+    except struct.error as exc:
+        raise ValueError("Integer not in range(65536)") from exc
+
+
+def to_string(proto, buf):
+    if len(buf) != 2:
+        raise ValueError("Invalid integer length (must be 2 bytes / 16 bits)")
+    return str(struct.unpack(">H", buf)[0])

hivemind/p2p/multiaddr/codecs/utf8.py

@@ -0,0 +1,18 @@
+from __future__ import absolute_import
+
+from . import LENGTH_PREFIXED_VAR_SIZE
+
+SIZE = LENGTH_PREFIXED_VAR_SIZE
+IS_PATH = False
+
+
+def to_bytes(proto, string):
+    if len(string) == 0:
+        raise ValueError("{0} value must not be empty".format(proto.name))
+    return string.encode("utf-8")
+
+
+def to_string(proto, buf):
+    if len(buf) == 0:
+        raise ValueError("invalid length (should be > 0)")
+    return buf.decode("utf-8")