Merge pull request #72 from singnet/filecoin-storage

Add filecoin storage

requirements.txt

@@ -17,4 +17,5 @@ argcomplete==3.1.2
 grpcio-health-checking==1.59.0
 jsonschema==4.0.0
 eth-account==0.9.0
-snet.contracts==0.1.1
+snet.contracts==0.1.1
+lighthouseweb3==0.1.4

snet/sdk/__init__.py

@@ -1,5 +1,6 @@
 import importlib
 import os
+from logging import DEBUG
 from pathlib import Path
 import sys
 from typing import Any, NewType
@@ -13,7 +14,7 @@
     # Suppress the eth-typing package`s warnings related to some new networks
     warnings.filterwarnings("ignore", "Network .* does not have a valid ChainId. eth-typing should be "
                                       "updated with the latest networks.", UserWarning)
-    from snet.sdk.metadata_provider.ipfs_metadata_provider import IPFSMetadataProvider
+    from snet.sdk.storage_provider.storage_provider import StorageProvider
 
 from snet.sdk.payment_strategies.default_payment_strategy import DefaultPaymentStrategy
 from snet.sdk.client_lib_generator import ClientLibGenerator
@@ -28,16 +29,15 @@
 _sym_db.RegisterMessage = lambda x: None
 
 import web3
-import ipfshttpclient
 
 from snet.sdk.service_client import ServiceClient
 from snet.sdk.account import Account
 from snet.sdk.mpe.mpe_contract import MPEContract
 
 from snet.contracts import get_contract_object
 
-from snet.sdk.metadata_provider.service_metadata import mpe_service_metadata_from_json
-from snet.sdk.utils.ipfs_utils import bytesuri_to_hash, get_from_ipfs_and_checkhash
+from snet.sdk.storage_provider.service_metadata import mpe_service_metadata_from_json
+from snet.sdk.utils.ipfs_utils import get_from_ipfs_and_checkhash
 from snet.sdk.utils.utils import find_file_by_keyword
 
 ModuleName = NewType('ModuleName', str)
@@ -66,17 +66,16 @@ def __init__(self, sdk_config: Config, metadata_provider=None):
         else:
             self.mpe_contract = MPEContract(self.web3, _mpe_contract_address)
 
-        # Instantiate IPFS client
-        ipfs_endpoint = self._sdk_config["ipfs_endpoint"]
-        self.ipfs_client = ipfshttpclient.connect(ipfs_endpoint)
-
         # Get Registry contract address from config if specified; mostly for local testing
         _registry_contract_address = self._sdk_config.get("registry_contract_address", None)
         if _registry_contract_address is None:
             self.registry_contract = get_contract_object(self.web3, "Registry")
         else:
             self.registry_contract = get_contract_object(self.web3, "Registry", _registry_contract_address)
 
+        if self._metadata_provider is None:
+            self._metadata_provider = StorageProvider(self._sdk_config, self.registry_contract)
+
         self.account = Account(self.web3, sdk_config, self.mpe_contract)
         self.payment_channel_provider = PaymentChannelProvider(self.web3, self.mpe_contract)
 
@@ -88,7 +87,7 @@ def create_service_client(self, org_id: str, service_id: str, group_name=None,
                               concurrent_calls=1):
 
         # Create and instance of the Config object, so we can create an instance of ClientLibGenerator
-        lib_generator = ClientLibGenerator(self._sdk_config, self.registry_contract, org_id, service_id)
+        lib_generator = ClientLibGenerator(self._metadata_provider, org_id, service_id)
 
         # Download the proto file and generate stubs if needed
         force_update = self._sdk_config.get('force_update', False)
@@ -113,9 +112,6 @@ def create_service_client(self, org_id: str, service_id: str, group_name=None,
         options['email'] = self._sdk_config.get("email", "")
         options['concurrency'] = self._sdk_config.get("concurrency", True)
 
-        if self._metadata_provider is None:
-            self._metadata_provider = IPFSMetadataProvider(self.ipfs_client, self.registry_contract)
-
         service_metadata = self._metadata_provider.enhance_service_metadata(org_id, service_id)
         group = self._get_service_group_details(service_metadata, group_name)
         strategy = payment_channel_management_strategy
@@ -124,9 +120,9 @@ def create_service_client(self, org_id: str, service_id: str, group_name=None,
 
         pb2_module = self.get_module_by_keyword(org_id, service_id, keyword="pb2.py")
 
-        service_client = ServiceClient(org_id, service_id, service_metadata, group, service_stub, strategy,
+        _service_client = ServiceClient(org_id, service_id, service_metadata, group, service_stub, strategy,
                                        options, self.mpe_contract, self.account, self.web3, pb2_module, self.payment_channel_provider)
-        return service_client
+        return _service_client
 
     def get_service_stub(self, org_id: str, service_id: str) -> ServiceStub:
         path_to_pb_files = self.get_path_to_pb_files(org_id, service_id)
@@ -154,18 +150,7 @@ def get_module_by_keyword(self, org_id: str, service_id: str, keyword: str) -> M
         return ModuleName(module_name)
 
     def get_service_metadata(self, org_id, service_id):
-        (found, registration_id, metadata_uri) = self.registry_contract.functions.getServiceRegistrationById(
-            bytes(org_id, "utf-8"),
-            bytes(service_id, "utf-8")
-        ).call()
-
-        if found is not True:
-            raise Exception('No service "{}" found in organization "{}"'.format(service_id, org_id))
-
-        metadata_hash = bytesuri_to_hash(metadata_uri)
-        metadata_json = get_from_ipfs_and_checkhash(self.ipfs_client, metadata_hash)
-        metadata = mpe_service_metadata_from_json(metadata_json)
-        return metadata
+        return self._metadata_provider.fetch_service_metadata(org_id, service_id)
 
     def _get_first_group(self, service_metadata):
         return service_metadata['groups'][0]

snet/sdk/client_lib_generator.py

@@ -1,16 +1,15 @@
 import os
 from pathlib import Path, PurePath
 
+from snet.sdk import StorageProvider
 from snet.sdk.utils import ipfs_utils
 from snet.sdk.utils.utils import compile_proto, type_converter
-from snet.sdk.metadata_provider.service_metadata import mpe_service_metadata_from_json
 
 
 class ClientLibGenerator:
 
-    def __init__(self, sdk_config, registry_contract, org_id, service_id):
-        self.sdk_config = sdk_config
-        self.registry_contract = registry_contract
+    def __init__(self, metadata_provider, org_id, service_id):
+        self._metadata_provider = metadata_provider
         self.org_id = org_id
         self.service_id = service_id
         self.language = "python"
@@ -36,12 +35,11 @@ def generate_client_library(self):
                                                                    library_language)
 
         try:
-            metadata = self._get_service_metadata_from_registry()
-            model_ipfs_hash = metadata["model_ipfs_hash"]
+            metadata = self._metadata_provider.fetch_service_metadata(self.org_id, self.service_id)
+            service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash")
 
             # Receive proto files
-            ipfs_utils.safe_extract_proto_from_ipfs(ipfs_utils.get_ipfs_client(self.sdk_config),
-                                                    model_ipfs_hash, library_dir_path)
+            self._metadata_provider.fetch_and_extract_proto(service_api_source, library_dir_path)
 
             # Compile proto files
             compile_proto(Path(library_dir_path), library_dir_path, target_language=self.language)
@@ -50,21 +48,3 @@ def generate_client_library(self):
                   f'generated at {library_dir_path}')
         except Exception as e:
             print(e)
-
-    def _get_service_metadata_from_registry(self):
-        rez = self._get_service_registration()
-        metadata_hash = ipfs_utils.bytesuri_to_hash(rez["metadataURI"])
-        metadata = ipfs_utils.get_from_ipfs_and_checkhash(ipfs_utils.get_ipfs_client(self.sdk_config), metadata_hash)
-        metadata = metadata.decode("utf-8")
-        metadata = mpe_service_metadata_from_json(metadata)
-        return metadata
-
-    def _get_service_registration(self):
-        params = [type_converter("bytes32")(self.org_id), type_converter("bytes32")(self.service_id)]
-        rez = self.registry_contract.functions.getServiceRegistrationById(*params).call()
-        if not rez[0]:
-            raise Exception("Cannot find Service with id=%s in Organization with id=%s" % (
-                self.service_id, self.org_id))
-        return {"metadataURI": rez[2]}
-
-

snet/sdk/config.py

@@ -22,7 +22,8 @@ def __init__(self,
             "mpe_contract_address": mpe_contract_address,
             "token_contract_address": token_contract_address,
             "registry_contract_address": registry_contract_address,
-            "signer_private_key": signer_private_key
+            "signer_private_key": signer_private_key,
+            "lighthouse_token": " "
         }
 
     def __getitem__(self, key):
@@ -34,4 +35,3 @@ def get(self, key, default=None):
     def get_ipfs_endpoint(self):
         return self["ipfs_endpoint"]
 
-

snet/sdk/metadata_provider/service_metadata.py → snet/sdk/storage_provider/service_metadata.py

@@ -310,6 +310,9 @@ def __getitem__(self, key):
     def __contains__(self, key):
         return key in self.m
 
+    def get(self, key, default=None):
+        return self.m.get(key, default)
+
     def get_group_name_nonetrick(self, group_name=None):
         """ In all getter function in case of single payment group, group_name can be None """
         groups = self.m["groups"]

snet/sdk/storage_provider/storage_provider.py

@@ -0,0 +1,77 @@
+import web3
+from lighthouseweb3 import Lighthouse
+import json
+
+from snet.sdk.utils.ipfs_utils import get_ipfs_client, get_from_ipfs_and_checkhash
+from snet.sdk.utils.utils import bytesuri_to_hash, safe_extract_proto
+from snet.sdk.storage_provider.service_metadata import mpe_service_metadata_from_json
+
+class StorageProvider(object):
+    def __init__(self, config, registry_contract):
+        self._registry_contract = registry_contract
+        self._ipfs_client = get_ipfs_client(config)
+        self.lighthouse_client = Lighthouse(config["lighthouse_token"])
+
+    def fetch_org_metadata(self,org_id):
+        org = web3.Web3.to_bytes(text=org_id).ljust(32, b"\0")
+
+        found, _, org_metadata_uri, _, _, _ = self._registry_contract.functions.getOrganizationById(org).call()
+        if found is not True:
+            raise Exception('Organization with org ID "{}" not found '.format(org_id))
+
+        org_provider_type, org_metadata_hash = bytesuri_to_hash(org_metadata_uri)
+
+        if org_provider_type == "ipfs":
+            org_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, org_metadata_hash)
+        else:
+            org_metadata_json, _ = self.lighthouse_client.download(org_metadata_uri)
+        org_metadata = json.loads(org_metadata_json)
+
+        return org_metadata
+
+
+    def fetch_service_metadata(self,org_id,service_id):
+        org = web3.Web3.to_bytes(text=org_id).ljust(32, b"\0")
+        service = web3.Web3.to_bytes(text=service_id).ljust(32, b"\0")
+
+        found, _, service_metadata_uri = self._registry_contract.functions.getServiceRegistrationById(org, service).call()
+        if found is not True:
+            raise Exception('No service "{}" found in organization "{}"'.format(service_id, org_id))
+
+        service_provider_type, service_metadata_hash = bytesuri_to_hash(service_metadata_uri)
+
+        if service_provider_type == "ipfs":
+            service_metadata_json = get_from_ipfs_and_checkhash(self._ipfs_client, service_metadata_hash)
+        else:
+            service_metadata_json, _ = self.lighthouse_client.download(service_metadata_hash)
+        service_metadata = mpe_service_metadata_from_json(service_metadata_json)
+
+        return service_metadata
+
+    def enhance_service_metadata(self,org_id,service_id):
+        service_metadata = self.fetch_service_metadata(org_id, service_id)
+        org_metadata = self.fetch_org_metadata(org_id)
+
+        org_group_map = {}
+        for group in org_metadata['groups']:
+            org_group_map[group['group_name']] = group
+
+        for group in service_metadata.m['groups']:
+            # merge service group with org_group
+            group['payment'] = org_group_map[group['group_name']]['payment']
+
+        return service_metadata
+
+    def fetch_and_extract_proto(self, service_api_source, protodir):
+        try:
+            proto_provider_type, service_api_source = bytesuri_to_hash(service_api_source, to_decode=False)
+        except Exception:
+            proto_provider_type = "ipfs"
+
+        if proto_provider_type == "ipfs":
+            spec_tar = get_from_ipfs_and_checkhash(self._ipfs_client, service_api_source)
+        else:
+            spec_tar, _ = self.lighthouse_client.download(service_api_source)
+
+        safe_extract_proto(spec_tar, protodir)
+