From 2652a920ee3e155da022d41c6a082e2c65e4e60d Mon Sep 17 00:00:00 2001 From: Arondondon Date: Tue, 1 Oct 2024 20:13:17 +0300 Subject: [PATCH 1/4] Implemented receiving metadata and proto files from the lighthouse. --- requirements.txt | 3 +- snet/sdk/__init__.py | 37 +++------ snet/sdk/client_lib_generator.py | 32 ++------ snet/sdk/config.py | 4 +- .../ipfs_metadata_provider.py | 54 ------------- .../metadata_provider/metadata_provider.py | 12 --- .../__init__.py | 0 .../service_metadata.py | 3 + snet/sdk/storage_provider/storage_provider.py | 77 +++++++++++++++++++ snet/sdk/utils/ipfs_utils.py | 36 --------- snet/sdk/utils/utils.py | 36 ++++++++- 11 files changed, 136 insertions(+), 158 deletions(-) delete mode 100644 snet/sdk/metadata_provider/ipfs_metadata_provider.py delete mode 100644 snet/sdk/metadata_provider/metadata_provider.py rename snet/sdk/{metadata_provider => storage_provider}/__init__.py (100%) rename snet/sdk/{metadata_provider => storage_provider}/service_metadata.py (99%) create mode 100644 snet/sdk/storage_provider/storage_provider.py diff --git a/requirements.txt b/requirements.txt index 218b126..028ddbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,5 @@ argcomplete==3.1.2 grpcio-health-checking==1.59.0 jsonschema==4.0.0 eth-account==0.9.0 -snet.contracts==0.1.1 \ No newline at end of file +snet.contracts==0.1.1 +lighthouseweb3>=0.1.4 \ No newline at end of file diff --git a/snet/sdk/__init__.py b/snet/sdk/__init__.py index e07bcc1..8ba490b 100644 --- a/snet/sdk/__init__.py +++ b/snet/sdk/__init__.py @@ -1,5 +1,6 @@ import importlib import os +from logging import DEBUG from pathlib import Path import sys from typing import Any, NewType @@ -13,7 +14,7 @@ # Suppress the eth-typing package`s warnings related to some new networks warnings.filterwarnings("ignore", "Network .* does not have a valid ChainId. eth-typing should be " "updated with the latest networks.", UserWarning) - from snet.sdk.metadata_provider.ipfs_metadata_provider import IPFSMetadataProvider + from snet.sdk.storage_provider.storage_provider import StorageProvider from snet.sdk.payment_strategies.default_payment_strategy import DefaultPaymentStrategy from snet.sdk.client_lib_generator import ClientLibGenerator @@ -28,7 +29,6 @@ _sym_db.RegisterMessage = lambda x: None import web3 -import ipfshttpclient from snet.sdk.service_client import ServiceClient from snet.sdk.account import Account @@ -36,8 +36,8 @@ from snet.contracts import get_contract_object -from snet.sdk.metadata_provider.service_metadata import mpe_service_metadata_from_json -from snet.sdk.utils.ipfs_utils import bytesuri_to_hash, get_from_ipfs_and_checkhash +from snet.sdk.storage_provider.service_metadata import mpe_service_metadata_from_json +from snet.sdk.utils.ipfs_utils import get_from_ipfs_and_checkhash from snet.sdk.utils.utils import find_file_by_keyword ModuleName = NewType('ModuleName', str) @@ -66,10 +66,6 @@ def __init__(self, sdk_config: Config, metadata_provider=None): else: self.mpe_contract = MPEContract(self.web3, _mpe_contract_address) - # Instantiate IPFS client - ipfs_endpoint = self._sdk_config["ipfs_endpoint"] - self.ipfs_client = ipfshttpclient.connect(ipfs_endpoint) - # Get Registry contract address from config if specified; mostly for local testing _registry_contract_address = self._sdk_config.get("registry_contract_address", None) if _registry_contract_address is None: @@ -77,6 +73,9 @@ def __init__(self, sdk_config: Config, metadata_provider=None): else: self.registry_contract = get_contract_object(self.web3, "Registry", _registry_contract_address) + if self._metadata_provider is None: + self._metadata_provider = StorageProvider(self._sdk_config, self.registry_contract) + self.account = Account(self.web3, sdk_config, self.mpe_contract) self.payment_channel_provider = PaymentChannelProvider(self.web3, self.mpe_contract) @@ -88,7 +87,7 @@ def create_service_client(self, org_id: str, service_id: str, group_name=None, concurrent_calls=1): # Create and instance of the Config object, so we can create an instance of ClientLibGenerator - lib_generator = ClientLibGenerator(self._sdk_config, self.registry_contract, org_id, service_id) + lib_generator = ClientLibGenerator(self._metadata_provider, org_id, service_id) # Download the proto file and generate stubs if needed force_update = self._sdk_config.get('force_update', False) @@ -113,9 +112,6 @@ def create_service_client(self, org_id: str, service_id: str, group_name=None, options['email'] = self._sdk_config.get("email", "") options['concurrency'] = self._sdk_config.get("concurrency", True) - if self._metadata_provider is None: - self._metadata_provider = IPFSMetadataProvider(self.ipfs_client, self.registry_contract) - service_metadata = self._metadata_provider.enhance_service_metadata(org_id, service_id) group = self._get_service_group_details(service_metadata, group_name) strategy = payment_channel_management_strategy @@ -124,9 +120,9 @@ def create_service_client(self, org_id: str, service_id: str, group_name=None, pb2_module = self.get_module_by_keyword(org_id, service_id, keyword="pb2.py") - service_client = ServiceClient(org_id, service_id, service_metadata, group, service_stub, strategy, + _service_client = ServiceClient(org_id, service_id, service_metadata, group, service_stub, strategy, options, self.mpe_contract, self.account, self.web3, pb2_module, self.payment_channel_provider) - return service_client + return _service_client def get_service_stub(self, org_id: str, service_id: str) -> ServiceStub: path_to_pb_files = self.get_path_to_pb_files(org_id, service_id) @@ -154,18 +150,7 @@ def get_module_by_keyword(self, org_id: str, service_id: str, keyword: str) -> M return ModuleName(module_name) def get_service_metadata(self, org_id, service_id): - (found, registration_id, metadata_uri) = self.registry_contract.functions.getServiceRegistrationById( - bytes(org_id, "utf-8"), - bytes(service_id, "utf-8") - ).call() - - if found is not True: - raise Exception('No service "{}" found in organization "{}"'.format(service_id, org_id)) - - metadata_hash = bytesuri_to_hash(metadata_uri) - metadata_json = get_from_ipfs_and_checkhash(self.ipfs_client, metadata_hash) - metadata = mpe_service_metadata_from_json(metadata_json) - return metadata + return self._metadata_provider.get_service_metadata(org_id, service_id) def _get_first_group(self, service_metadata): return service_metadata['groups'][0] diff --git a/snet/sdk/client_lib_generator.py b/snet/sdk/client_lib_generator.py index 8623162..3933ba1 100644 --- a/snet/sdk/client_lib_generator.py +++ b/snet/sdk/client_lib_generator.py @@ -1,16 +1,15 @@ import os from pathlib import Path, PurePath +from snet.sdk import StorageProvider from snet.sdk.utils import ipfs_utils from snet.sdk.utils.utils import compile_proto, type_converter -from snet.sdk.metadata_provider.service_metadata import mpe_service_metadata_from_json class ClientLibGenerator: - def __init__(self, sdk_config, registry_contract, org_id, service_id): - self.sdk_config = sdk_config - self.registry_contract = registry_contract + def __init__(self, metadata_provider, org_id, service_id): + self._metadata_provider = metadata_provider self.org_id = org_id self.service_id = service_id self.language = "python" @@ -36,12 +35,11 @@ def generate_client_library(self): library_language) try: - metadata = self._get_service_metadata_from_registry() - model_ipfs_hash = metadata["model_ipfs_hash"] + metadata = self._metadata_provider.fetch_service_metadata(self.org_id, self.service_id) + service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash") # Receive proto files - ipfs_utils.safe_extract_proto_from_ipfs(ipfs_utils.get_ipfs_client(self.sdk_config), - model_ipfs_hash, library_dir_path) + self._metadata_provider.fetch_and_extract_proto(service_api_source, library_dir_path) # Compile proto files compile_proto(Path(library_dir_path), library_dir_path, target_language=self.language) @@ -50,21 +48,3 @@ def generate_client_library(self): f'generated at {library_dir_path}') except Exception as e: print(e) - - def _get_service_metadata_from_registry(self): - rez = self._get_service_registration() - metadata_hash = ipfs_utils.bytesuri_to_hash(rez["metadataURI"]) - metadata = ipfs_utils.get_from_ipfs_and_checkhash(ipfs_utils.get_ipfs_client(self.sdk_config), metadata_hash) - metadata = metadata.decode("utf-8") - metadata = mpe_service_metadata_from_json(metadata) - return metadata - - def _get_service_registration(self): - params = [type_converter("bytes32")(self.org_id), type_converter("bytes32")(self.service_id)] - rez = self.registry_contract.functions.getServiceRegistrationById(*params).call() - if not rez[0]: - raise Exception("Cannot find Service with id=%s in Organization with id=%s" % ( - self.service_id, self.org_id)) - return {"metadataURI": rez[2]} - - diff --git a/snet/sdk/config.py b/snet/sdk/config.py index eff168f..e8f8b04 100644 --- a/snet/sdk/config.py +++ b/snet/sdk/config.py @@ -22,7 +22,8 @@ def __init__(self, "mpe_contract_address": mpe_contract_address, "token_contract_address": token_contract_address, "registry_contract_address": registry_contract_address, - "signer_private_key": signer_private_key + "signer_private_key": signer_private_key, + "lighthouse_token": " " } def __getitem__(self, key): @@ -34,4 +35,3 @@ def get(self, key, default=None): def get_ipfs_endpoint(self): return self["ipfs_endpoint"] - diff --git a/snet/sdk/metadata_provider/ipfs_metadata_provider.py b/snet/sdk/metadata_provider/ipfs_metadata_provider.py deleted file mode 100644 index a1c7067..0000000 --- a/snet/sdk/metadata_provider/ipfs_metadata_provider.py +++ /dev/null @@ -1,54 +0,0 @@ -import json -import web3 - -from snet.sdk.metadata_provider.service_metadata import mpe_service_metadata_from_json -from snet.sdk.utils.ipfs_utils import bytesuri_to_hash, get_from_ipfs_and_checkhash - - -class IPFSMetadataProvider(object): - - def __init__(self, ipfs_client, registry_contract): - self.registry_contract = registry_contract - self._ipfs_client = ipfs_client - - def fetch_org_metadata(self, org_id): - org = web3.Web3.to_bytes(text=org_id).ljust(32, b"\0") - found, id, metadata_uri, owner, members, service_ids = self.registry_contract.functions.getOrganizationById( - org - ).call() - if found is not True: - raise Exception('Organization with org ID "{}" not found '.format(org_id)) - - metadata_hash = bytesuri_to_hash(metadata_uri) - metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, metadata_hash) - org_metadata = json.loads(metadata_json) - return org_metadata - - def fetch_service_metadata(self, org_id, service_id): - org = web3.Web3.to_bytes(text=org_id).ljust(32, b"\0") - service = web3.Web3.to_bytes(text=service_id).ljust(32, b"\0") - - found, registration_id, metadata_uri = self.registry_contract.functions.getServiceRegistrationById( - org, service).call() - - if found is not True: - raise Exception('No service "{}" found in organization "{}"'.format(service_id, org_id)) - - metadata_hash = bytesuri_to_hash(metadata_uri) - metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, metadata_hash) - metadata = mpe_service_metadata_from_json(metadata_json) - return metadata - - def enhance_service_metadata(self, org_id, service_id): - service_metadata = self.fetch_service_metadata(org_id, service_id) - org_metadata = self.fetch_org_metadata(org_id) - - org_group_map = {} - for group in org_metadata['groups']: - org_group_map[group['group_name']] = group - - for group in service_metadata.m['groups']: - # merge service group with org_group - group['payment'] = org_group_map[group['group_name']]['payment'] - - return service_metadata diff --git a/snet/sdk/metadata_provider/metadata_provider.py b/snet/sdk/metadata_provider/metadata_provider.py deleted file mode 100644 index e0c7f6a..0000000 --- a/snet/sdk/metadata_provider/metadata_provider.py +++ /dev/null @@ -1,12 +0,0 @@ -class MetadataProvider(object): - - def fetch_org_metadata(self,org_id): - pass - - - def fetch_service_metadata(self,org_id,service_id): - pass - - def enhance_service_group_details(self, service_metadata, org_metadata): - pass - diff --git a/snet/sdk/metadata_provider/__init__.py b/snet/sdk/storage_provider/__init__.py similarity index 100% rename from snet/sdk/metadata_provider/__init__.py rename to snet/sdk/storage_provider/__init__.py diff --git a/snet/sdk/metadata_provider/service_metadata.py b/snet/sdk/storage_provider/service_metadata.py similarity index 99% rename from snet/sdk/metadata_provider/service_metadata.py rename to snet/sdk/storage_provider/service_metadata.py index 1772e6e..e39ca46 100644 --- a/snet/sdk/metadata_provider/service_metadata.py +++ b/snet/sdk/storage_provider/service_metadata.py @@ -310,6 +310,9 @@ def __getitem__(self, key): def __contains__(self, key): return key in self.m + def get(self, key, default=None): + return self.m.get(key, default) + def get_group_name_nonetrick(self, group_name=None): """ In all getter function in case of single payment group, group_name can be None """ groups = self.m["groups"] diff --git a/snet/sdk/storage_provider/storage_provider.py b/snet/sdk/storage_provider/storage_provider.py new file mode 100644 index 0000000..4dc03a1 --- /dev/null +++ b/snet/sdk/storage_provider/storage_provider.py @@ -0,0 +1,77 @@ +import web3 +from lighthouseweb3 import Lighthouse +import json + +from snet.sdk.utils.ipfs_utils import get_ipfs_client, get_from_ipfs_and_checkhash +from snet.sdk.utils.utils import bytesuri_to_hash, safe_extract_proto +from snet.sdk.storage_provider.service_metadata import mpe_service_metadata_from_json + +class StorageProvider(object): + def __init__(self, config, registry_contract): + self._registry_contract = registry_contract + self._ipfs_client = get_ipfs_client(config) + self.lighthouse_client = Lighthouse(config["lighthouse_token"]) + + def fetch_org_metadata(self,org_id): + org = web3.Web3.to_bytes(text=org_id).ljust(32, b"\0") + + found, _, org_metadata_uri, _, _, _ = self._registry_contract.functions.getOrganizationById(org).call() + if found is not True: + raise Exception('Organization with org ID "{}" not found '.format(org_id)) + + org_provider_type, org_metadata_hash = bytesuri_to_hash(org_metadata_uri) + + if org_provider_type == "ipfs": + org_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, org_metadata_hash) + else: + _, org_metadata_json = self.lighthouse_client.download(org_metadata_uri) + org_metadata = json.loads(org_metadata_json) + + return org_metadata + + + def fetch_service_metadata(self,org_id,service_id): + org = web3.Web3.to_bytes(text=org_id).ljust(32, b"\0") + service = web3.Web3.to_bytes(text=service_id).ljust(32, b"\0") + + found, _, service_metadata_uri = self._registry_contract.functions.getServiceRegistrationById(org, service).call() + if found is not True: + raise Exception('No service "{}" found in organization "{}"'.format(service_id, org_id)) + + service_provider_type, service_metadata_hash = bytesuri_to_hash(service_metadata_uri) + + if service_provider_type == "ipfs": + service_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, service_metadata_hash) + else: + _, service_metadata_json = self.lighthouse_client.download(service_metadata_uri) + service_metadata = mpe_service_metadata_from_json(service_metadata_json) + + return service_metadata + + def enhance_service_metadata(self,org_id,service_id): + service_metadata = self.fetch_service_metadata(org_id, service_id) + org_metadata = self.fetch_org_metadata(org_id) + + org_group_map = {} + for group in org_metadata['groups']: + org_group_map[group['group_name']] = group + + for group in service_metadata.m['groups']: + # merge service group with org_group + group['payment'] = org_group_map[group['group_name']]['payment'] + + return service_metadata + + def fetch_and_extract_proto(self, service_api_source, protodir): + try: + proto_provider_type, service_api_source = bytesuri_to_hash(service_api_source) + except Exception: + proto_provider_type = "ipfs" + + if proto_provider_type == "ipfs": + spec_tar = get_from_ipfs_and_checkhash(self._ipfs_client, service_api_source) + else: + _, spec_tar = self.lighthouse_client.download(service_api_source) + + safe_extract_proto(spec_tar, protodir) + diff --git a/snet/sdk/utils/ipfs_utils.py b/snet/sdk/utils/ipfs_utils.py index 4e18c95..280bfae 100644 --- a/snet/sdk/utils/ipfs_utils.py +++ b/snet/sdk/utils/ipfs_utils.py @@ -1,8 +1,4 @@ """ Utilities related to ipfs """ -import tarfile -import io -import os - import base58 import ipfshttpclient import multihash @@ -40,39 +36,7 @@ def get_from_ipfs_and_checkhash(ipfs_client, ipfs_hash_base58, validate=True): data = ipfs_client.cat(ipfs_hash_base58) return data - -def bytesuri_to_hash(s): - s = s.rstrip(b"\0").decode('ascii') - if not s.startswith("ipfs://"): - raise Exception("We support only ipfs uri in Registry") - return s[7:] - - -def safe_extract_proto_from_ipfs(ipfs_client, ipfs_hash, protodir): - """ - Tar files might be dangerous (see https://bugs.python.org/issue21109, - and https://docs.python.org/3/library/tarfile.html, TarFile.extractall warning) - we extract only simple files - """ - spec_tar = get_from_ipfs_and_checkhash(ipfs_client, ipfs_hash) - with tarfile.open(fileobj=io.BytesIO(spec_tar)) as f: - for m in f.getmembers(): - if os.path.dirname(m.name) != "": - raise Exception( - "tarball has directories. We do not support it.") - if not m.isfile(): - raise Exception( - "tarball contains %s which is not a file" % m.name) - fullname = os.path.join(protodir, m.name) - if os.path.exists(fullname): - os.remove(fullname) - print("%s removed." % fullname) - # now it is safe to call extractall - f.extractall(protodir) - - def get_ipfs_client(config): ipfs_endpoint = config.get_ipfs_endpoint() return ipfshttpclient.connect(ipfs_endpoint) - diff --git a/snet/sdk/utils/utils.py b/snet/sdk/utils/utils.py index ebb7cc1..d6fc8f6 100644 --- a/snet/sdk/utils/utils.py +++ b/snet/sdk/utils/utils.py @@ -1,10 +1,12 @@ import json -import os import subprocess import sys import importlib.resources from urllib.parse import urlparse from pathlib import Path, PurePath +import os +import tarfile +import io import web3 from grpc_tools.protoc import main as protoc @@ -145,3 +147,35 @@ def find_file_by_keyword(directory, keyword): for file in files: if keyword in file: return file + + +def bytesuri_to_hash(s): + s = s.rstrip(b"\0").decode('ascii') + if s.startswith("ipfs://"): + return "ipfs", s[7:] + elif s.startswith("lighthouse://"): + return "lighthouse", s[13:] + else: + raise Exception("We support only ipfs and lighthouse uri in Registry") + + +def safe_extract_proto(spec_tar, protodir): + """ + Tar files might be dangerous (see https://bugs.python.org/issue21109, + and https://docs.python.org/3/library/tarfile.html, TarFile.extractall warning) + we extract only simple files + """ + with tarfile.open(fileobj=io.BytesIO(spec_tar)) as f: + for m in f.getmembers(): + if os.path.dirname(m.name) != "": + raise Exception( + "tarball has directories. We do not support it.") + if not m.isfile(): + raise Exception( + "tarball contains %s which is not a file" % m.name) + fullname = os.path.join(protodir, m.name) + if os.path.exists(fullname): + os.remove(fullname) + print("%s removed." % fullname) + # now it is safe to call extractall + f.extractall(protodir) \ No newline at end of file From e816af23514865633acf59feba38a951a5c7b96d Mon Sep 17 00:00:00 2001 From: Arondondon Date: Thu, 3 Oct 2024 11:01:36 +0300 Subject: [PATCH 2/4] Changed uri/api_source prefix from "lighthouse" to "filecoin". Several minor fixes. --- snet/sdk/__init__.py | 2 +- snet/sdk/storage_provider/storage_provider.py | 4 ++-- snet/sdk/utils/utils.py | 13 +++++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/snet/sdk/__init__.py b/snet/sdk/__init__.py index 8ba490b..395a8bc 100644 --- a/snet/sdk/__init__.py +++ b/snet/sdk/__init__.py @@ -150,7 +150,7 @@ def get_module_by_keyword(self, org_id: str, service_id: str, keyword: str) -> M return ModuleName(module_name) def get_service_metadata(self, org_id, service_id): - return self._metadata_provider.get_service_metadata(org_id, service_id) + return self._metadata_provider.fetch_service_metadata(org_id, service_id) def _get_first_group(self, service_metadata): return service_metadata['groups'][0] diff --git a/snet/sdk/storage_provider/storage_provider.py b/snet/sdk/storage_provider/storage_provider.py index 4dc03a1..f2c6066 100644 --- a/snet/sdk/storage_provider/storage_provider.py +++ b/snet/sdk/storage_provider/storage_provider.py @@ -64,14 +64,14 @@ def enhance_service_metadata(self,org_id,service_id): def fetch_and_extract_proto(self, service_api_source, protodir): try: - proto_provider_type, service_api_source = bytesuri_to_hash(service_api_source) + proto_provider_type, service_api_source = bytesuri_to_hash(service_api_source, to_decode=False) except Exception: proto_provider_type = "ipfs" if proto_provider_type == "ipfs": spec_tar = get_from_ipfs_and_checkhash(self._ipfs_client, service_api_source) else: - _, spec_tar = self.lighthouse_client.download(service_api_source) + spec_tar, _ = self.lighthouse_client.download(service_api_source) safe_extract_proto(spec_tar, protodir) diff --git a/snet/sdk/utils/utils.py b/snet/sdk/utils/utils.py index d6fc8f6..21c4369 100644 --- a/snet/sdk/utils/utils.py +++ b/snet/sdk/utils/utils.py @@ -149,14 +149,15 @@ def find_file_by_keyword(directory, keyword): return file -def bytesuri_to_hash(s): - s = s.rstrip(b"\0").decode('ascii') +def bytesuri_to_hash(s, to_decode=True): + if to_decode: + s = s.rstrip(b"\0").decode('ascii') if s.startswith("ipfs://"): return "ipfs", s[7:] - elif s.startswith("lighthouse://"): - return "lighthouse", s[13:] + elif s.startswith("filecoin://"): + return "filecoin", s[11:] else: - raise Exception("We support only ipfs and lighthouse uri in Registry") + raise Exception("We support only ipfs and filecoin uri in Registry") def safe_extract_proto(spec_tar, protodir): @@ -178,4 +179,4 @@ def safe_extract_proto(spec_tar, protodir): os.remove(fullname) print("%s removed." % fullname) # now it is safe to call extractall - f.extractall(protodir) \ No newline at end of file + f.extractall(protodir) From 47957f23fc67fe81f1cbb5046445a03b74d67a83 Mon Sep 17 00:00:00 2001 From: Arondondon Date: Tue, 8 Oct 2024 18:15:34 +0300 Subject: [PATCH 3/4] Fixed StorageProvider class and changed get_from_ipfs_and_checkhash() function. --- requirements.txt | 2 +- snet/sdk/storage_provider/storage_provider.py | 4 +- snet/sdk/utils/ipfs_utils.py | 40 ++++++++----------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index 028ddbb..3423c8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,4 @@ grpcio-health-checking==1.59.0 jsonschema==4.0.0 eth-account==0.9.0 snet.contracts==0.1.1 -lighthouseweb3>=0.1.4 \ No newline at end of file +lighthouseweb3==0.1.4 \ No newline at end of file diff --git a/snet/sdk/storage_provider/storage_provider.py b/snet/sdk/storage_provider/storage_provider.py index f2c6066..032221d 100644 --- a/snet/sdk/storage_provider/storage_provider.py +++ b/snet/sdk/storage_provider/storage_provider.py @@ -24,7 +24,7 @@ def fetch_org_metadata(self,org_id): if org_provider_type == "ipfs": org_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, org_metadata_hash) else: - _, org_metadata_json = self.lighthouse_client.download(org_metadata_uri) + org_metadata_json, _ = self.lighthouse_client.download(org_metadata_uri) org_metadata = json.loads(org_metadata_json) return org_metadata @@ -43,7 +43,7 @@ def fetch_service_metadata(self,org_id,service_id): if service_provider_type == "ipfs": service_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, service_metadata_hash) else: - _, service_metadata_json = self.lighthouse_client.download(service_metadata_uri) + service_metadata_json, _ = self.lighthouse_client.download(service_metadata_uri) service_metadata = mpe_service_metadata_from_json(service_metadata_json) return service_metadata diff --git a/snet/sdk/utils/ipfs_utils.py b/snet/sdk/utils/ipfs_utils.py index 280bfae..64cdf9a 100644 --- a/snet/sdk/utils/ipfs_utils.py +++ b/snet/sdk/utils/ipfs_utils.py @@ -6,34 +6,26 @@ def get_from_ipfs_and_checkhash(ipfs_client, ipfs_hash_base58, validate=True): """ - Get file from ipfs - We must check the hash becasue we cannot believe that ipfs_client wasn't been compromise + Get file from IPFS. If validate is True, verify the integrity of the file using its hash. """ - if validate: - from snet.sdk.resources.proto.unixfs_pb2 import Data - from snet.sdk.resources.proto.merckledag_pb2 import MerkleNode - # No nice Python library to parse ipfs blocks, so do it ourselves. + data = ipfs_client.cat(ipfs_hash_base58) + + if validate: block_data = ipfs_client.block.get(ipfs_hash_base58) - mn = MerkleNode() - mn.ParseFromString(block_data) - unixfs_data = Data() - unixfs_data.ParseFromString(mn.Data) - assert unixfs_data.Type == unixfs_data.DataType.Value( - 'File'), "IPFS hash must be a file" - data = unixfs_data.Data - - # multihash has a badly registered base58 codec, overwrite it... - multihash.CodecReg.register( - 'base58', base58.b58encode, base58.b58decode) - # create a multihash object from our ipfs hash - mh = multihash.decode(ipfs_hash_base58.encode('ascii'), 'base58') - - # Convenience method lets us directly use a multihash to verify data - if not mh.verify(block_data): + + # print(f"IPFS hash (Base58): {ipfs_hash_base58}") + # print(f"Block data length: {len(block_data)}") + + # Decode Base58 bash to multihash + try: + mh = multihash.decode(ipfs_hash_base58.encode('ascii'), "base58") + except Exception as e: + raise ValueError(f"Invalid multihash for IPFS hash: {ipfs_hash_base58}. Error: {str(e)}") from e + + if not mh.verify(block_data): # Correctly using mh instance for verification raise Exception("IPFS hash mismatch with data") - else: - data = ipfs_client.cat(ipfs_hash_base58) + return data def get_ipfs_client(config): From 59e81fc16eb715e4849187507bad4a70dbd411c8 Mon Sep 17 00:00:00 2001 From: Arondondon Date: Wed, 9 Oct 2024 13:35:41 +0300 Subject: [PATCH 4/4] Fixed bug with fetching service metadata from filecoin. --- snet/sdk/storage_provider/storage_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snet/sdk/storage_provider/storage_provider.py b/snet/sdk/storage_provider/storage_provider.py index 032221d..29f03f1 100644 --- a/snet/sdk/storage_provider/storage_provider.py +++ b/snet/sdk/storage_provider/storage_provider.py @@ -43,7 +43,7 @@ def fetch_service_metadata(self,org_id,service_id): if service_provider_type == "ipfs": service_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, service_metadata_hash) else: - service_metadata_json, _ = self.lighthouse_client.download(service_metadata_uri) + service_metadata_json, _ = self.lighthouse_client.download(service_metadata_hash) service_metadata = mpe_service_metadata_from_json(service_metadata_json) return service_metadata