Skip to content

Commit

Permalink
[FSTORE-1478] Deduplicate project (logicalclocks#285)
Browse files Browse the repository at this point in the history 
* Move project-related files to hopsworks-common

* Make aliases and adapt the moved code
  • Loading branch information
@aversey
aversey authored Aug 19, 2024
1 parent 7906541 commit 9413e6e
Show file tree
Hide file tree
Showing 9 changed files with 696 additions and 707 deletions.
90 changes: 6 additions & 84 deletions python/hopsworks/core/opensearch_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,89 +14,11 @@
# limitations under the License.
#

from furl import furl
from hopsworks import client, constants
from hopsworks.client.exceptions import OpenSearchException
from hopsworks.core import variable_api
from hopsworks_common.core.opensearch_api import (
OpenSearchApi,
)


class OpenSearchApi:
def __init__(self):
self._variable_api = variable_api.VariableApi()

def _get_opensearch_url(self):
if client._is_external():
external_domain = self._variable_api.get_loadbalancer_external_domain(
"opensearch"
)
return f"https://{external_domain}:9200"
else:
service_discovery_domain = self._variable_api.get_variable(
"service_discovery_domain"
)
if service_discovery_domain == "":
raise OpenSearchException(
"Client could not locate service_discovery_domain "
"in cluster configuration or variable is empty."
)
return f"https://rest.elastic.service.{service_discovery_domain}:9200"

def get_project_index(self, index):
"""
This helper method prefixes the supplied index name with the project name to avoid index name clashes.
Args:
:index: the opensearch index to interact with.
Returns:
A valid opensearch index name.
"""
_client = client.get_instance()
return (_client._project_name + "_" + index).lower()

def get_default_py_config(self):
"""
Get the required opensearch configuration to setup a connection using the *opensearch-py* library.
```python
import hopsworks
from opensearchpy import OpenSearch
project = hopsworks.login()
opensearch_api = project.get_opensearch_api()
client = OpenSearch(**opensearch_api.get_default_py_config())
```
Returns:
A dictionary with required configuration.
"""
url = furl(self._get_opensearch_url())
return {
constants.OPENSEARCH_CONFIG.HOSTS: [{"host": url.host, "port": url.port}],
constants.OPENSEARCH_CONFIG.HTTP_COMPRESS: False,
constants.OPENSEARCH_CONFIG.HEADERS: {
"Authorization": self._get_authorization_token()
},
constants.OPENSEARCH_CONFIG.USE_SSL: True,
constants.OPENSEARCH_CONFIG.VERIFY_CERTS: True,
constants.OPENSEARCH_CONFIG.SSL_ASSERT_HOSTNAME: False,
constants.OPENSEARCH_CONFIG.CA_CERTS: client.get_instance()._get_ca_chain_path(),
}

def _get_authorization_token(self):
"""Get opensearch jwt token.
# Returns
`str`: OpenSearch jwt token
# Raises
`RestAPIError`: If unable to get the token
"""

_client = client.get_instance()
path_params = ["elastic", "jwt", _client._project_id]

headers = {"content-type": "application/json"}
return _client._send_request("GET", path_params, headers=headers)["token"]
__all__ = [
"OpenSearchApi",
]
109 changes: 6 additions & 103 deletions python/hopsworks/core/project_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,108 +14,11 @@
# limitations under the License.
#

import json
from hopsworks_common.core.project_api import (
ProjectApi,
)

from hopsworks import client, constants, project
from hopsworks.client.exceptions import RestAPIError


class ProjectApi:
def _exists(self, name: str):
"""Check if a project exists.
# Arguments
name: Name of the project.
# Returns
`bool`: True if project exists, otherwise False
"""
try:
self._get_project(name)
return True
except RestAPIError:
return False

def _get_projects(self):
"""Get all projects accessible by the user.
# Returns
`List[Project]`: List of Project objects
# Raises
`RestAPIError`: If unable to get the projects
"""
_client = client.get_instance()
path_params = [
"project",
]
project_team_json = _client._send_request("GET", path_params)
projects = []
for project_team in project_team_json:
projects.append(self._get_project(project_team["project"]["name"]))
return projects

def _get_project(self, name: str):
"""Get a project.
# Arguments
name: Name of the project.
# Returns
`Project`: The Project object
# Raises
`RestAPIError`: If unable to get the project
"""
_client = client.get_instance()
path_params = [
"project",
"getProjectInfo",
name,
]
project_json = _client._send_request("GET", path_params)
return project.Project.from_response_json(project_json)

def _create_project(
self, name: str, description: str = None, feature_store_topic: str = None
):
"""Create a new project.
# Arguments
name: Name of the project.
description: Description of the project.
feature_store_topic: Feature store topic name.
# Returns
`Project`: The Project object
# Raises
`RestAPIError`: If unable to create the project
"""
_client = client.get_instance()

path_params = ["project"]
query_params = {"projectName": name}
headers = {"content-type": "application/json"}

data = {
"projectName": name,
"services": constants.SERVICES.LIST,
"description": description,
"featureStoreTopic": feature_store_topic,
}
_client._send_request(
"POST",
path_params,
headers=headers,
query_params=query_params,
data=json.dumps(data),
)

# The return of the project creation is not a ProjectDTO, so get the correct object after creation
project = self._get_project(name)
print("Project created successfully, explore it at " + project.get_url())
return project

def get_client(self):
_client = client.get_instance()
path_params = [
"project",
_client._project_id,
"client",
]
return _client._send_request("GET", path_params, stream=True)
__all__ = [
"ProjectApi",
]
158 changes: 7 additions & 151 deletions python/hopsworks/core/secret_api.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#
# Copyright 2022 Logical Clocks AB
# Copyright 2024 Hopsworks AB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -14,155 +14,11 @@
# limitations under the License.
#

import getpass
import json
from hopsworks_common.core.secret_api import (
SecretsApi,
)

from hopsworks import client, secret, util
from hopsworks.client.exceptions import RestAPIError
from hopsworks.core import project_api


class SecretsApi:
def __init__(
self,
):
self._project_api = project_api.ProjectApi()

def get_secrets(self):
"""Get all secrets
# Returns
`List[Secret]`: List of all accessible secrets
# Raises
`RestAPIError`: If unable to get the secrets
"""
_client = client.get_instance()
path_params = [
"users",
"secrets",
]
return secret.Secret.from_response_json(
_client._send_request("GET", path_params)
)

def get_secret(self, name: str, owner: str = None) -> secret.Secret:
"""Get a secret.
# Arguments
name: Name of the secret.
owner: username of the owner for a secret shared with the current project. Users can find their username in the Account Settings > Profile section.
# Returns
`Secret`: The Secret object
# Raises
`RestAPIError`: If unable to get the secret
"""
_client = client.get_instance()
query_params = None
if owner is None:
path_params = [
"users",
"secrets",
name,
]
else:
query_params = {"name": name, "owner": owner}
path_params = [
"users",
"secrets",
"shared",
]

return secret.Secret.from_response_json(
_client._send_request("GET", path_params, query_params=query_params)
)[0]

def get(self, name: str, owner: str = None) -> str:
"""Get the secret's value.
If the secret does not exist, it prompts the user to create the secret if the application is running interactively
# Arguments
name: Name of the secret.
owner: email of the owner for a secret shared with the current project.
# Returns
`str`: The secret value
# Raises
`RestAPIError`: If unable to get the secret
"""
try:
return self.get_secret(name=name, owner=owner).value
except RestAPIError as e:
if (
e.response.json().get("errorCode", "") == 160048
and e.response.status_code == 404
and util.is_interactive()
):
secret_input = getpass.getpass(
prompt="\nCould not find secret, enter value here to create it: "
)
return self.create_secret(name, secret_input).value
else:
raise e

def create_secret(
self, name: str, value: str, project: str = None
) -> secret.Secret:
"""Create a new secret.
```python
import hopsworks
connection = hopsworks.connection()
secrets_api = connection.get_secrets_api()
secret = secrets_api.create_secret("my_secret", "Fk3MoPlQXCQvPo")
```
# Arguments
name: Name of the secret.
value: The secret value.
project: Name of the project to share the secret with.
# Returns
`Secret`: The Secret object
# Raises
`RestAPIError`: If unable to create the secret
"""
_client = client.get_instance()

secret_config = {"name": name, "secret": value}

if project is None:
secret_config["visibility"] = "PRIVATE"
else:
scope_project = self._project_api._get_project(project)
secret_config["scope"] = scope_project.id
secret_config["visibility"] = "PROJECT"

path_params = [
"users",
"secrets",
]

headers = {"content-type": "application/json"}

_client._send_request(
"POST", path_params, headers=headers, data=json.dumps(secret_config)
)

created_secret = self.get_secret(name)
print(f"Secret created successfully, explore it at {created_secret.get_url()}")
return created_secret

def _delete(self, name: str):
"""Delete the secret.
:param name: name of the secret
:type name: Secret
"""
_client = client.get_instance()
path_params = [
"users",
"secrets",
name,
]
_client._send_request("DELETE", path_params)
__all__ = [
"SecretsApi",
]
Loading

0 comments on commit 9413e6e

Please sign in to comment.