From 0786f2aa679c399d6dd96b38fbe76cf488c0218a Mon Sep 17 00:00:00 2001 From: Lazlo Westerhof Date: Tue, 17 Dec 2024 10:07:32 +0100 Subject: [PATCH] YDA-6062: fix name clash and integration tests --- __init__.py | 3 +- arb.py | 81 ++++++++++++++++++++++++++++++ integration_tests.py | 46 +++++++++-------- setup.cfg | 2 +- resources.py => stats.py | 74 --------------------------- util/__init__.py | 2 +- util/{resource.py => resources.py} | 0 util/yoda_names.py | 3 +- 8 files changed, 111 insertions(+), 100 deletions(-) create mode 100644 arb.py rename resources.py => stats.py (89%) rename util/{resource.py => resources.py} (100%) diff --git a/__init__.py b/__init__.py index fc16f3409..9d79d903d 100644 --- a/__init__.py +++ b/__init__.py @@ -29,6 +29,7 @@ # so that they become visible to iRODS. from admin import * +from arb import * from browse import * from folder import * from groups import * @@ -39,10 +40,10 @@ from meta_form import * from provenance import * from research import * -from resources import * from schema import * from schema_transformation import * from schema_transformations import * +from stats import * from publication_troubleshoot import * from vault import * from datacite import * diff --git a/arb.py b/arb.py new file mode 100644 index 000000000..a77949a7a --- /dev/null +++ b/arb.py @@ -0,0 +1,81 @@ +"""Functions for automatic resource balancing module.""" + +__copyright__ = 'Copyright (c) 2019-2024, Utrecht University' +__license__ = 'GPLv3, see LICENSE' + +from util import * + +__all__ = ['rule_resource_update_resc_arb_data', + 'rule_resource_update_misc_arb_data'] + + +@rule.make(inputs=[0, 1, 2], outputs=[]) +def rule_resource_update_resc_arb_data(ctx: rule.Context, resc_name: str, bytes_free: int, bytes_total: int) -> None: + """ + Update ARB data for a specific resource + + :param ctx: Combined type of a callback and rei struct + :param resc_name: Name of a particular unixfilesystem resource + :param bytes_free: Free size on this resource, in bytes + :param bytes_total: Total size of this resource, in bytes + """ + if user.user_type(ctx) != 'rodsadmin': + log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") + return + + if not resources.exists(ctx, resc_name): + log.write(ctx, "Error: could not find resource named '{}' for ARB update.".format(resc_name)) + return + + bytes_free_gb = int(bytes_free) / 2 ** 30 + bytes_free_percent = 100 * (float(bytes_free) / float(bytes_total)) + + if resc_name in config.arb_exempt_resources: + arb_status = constants.arb_status.EXEMPT + elif bytes_free_gb >= config.arb_min_gb_free and bytes_free_percent > config.arb_min_percent_free: + arb_status = constants.arb_status.AVAILABLE + else: + arb_status = constants.arb_status.FULL + + parent_resc_name = resources.get_parent_by_name(ctx, resc_name) + + manager = arb_data_manager.ARBDataManager() + manager.put(ctx, resc_name, constants.arb_status.IGNORE) + + if parent_resc_name is not None and resources.get_type_by_name(ctx, parent_resc_name) == "passthru": + manager.put(ctx, parent_resc_name, arb_status) + + +@rule.make() +def rule_resource_update_misc_arb_data(ctx: rule.Context) -> None: + """Update ARB data for resources that are not covered by the regular process. That is, + all resources that are neither unixfilesystem nor passthrough resources, as well as + passthrough resources that do not have a unixfilesystem child resource. + + :param ctx: Combined type of a callback and rei struct + """ + if user.user_type(ctx) != 'rodsadmin': + log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") + return + + manager = arb_data_manager.ARBDataManager() + + all_resources = resources.get_all_resource_names(ctx) + ufs_resources = set(resources.get_resource_names_by_type(ctx, "unixfilesystem") + + resources.get_resource_names_by_type(ctx, "unix file system")) + pt_resources = set(resources.get_resource_names_by_type(ctx, "passthru")) + + for resc in all_resources: + if resc in ufs_resources: + pass + elif resc not in pt_resources: + manager.put(ctx, resc, constants.arb_status.IGNORE) + else: + child_resources = resources.get_children_by_name(ctx, resc) + child_found = False + for child_resource in child_resources: + if child_resource in ufs_resources: + child_found = True + # Ignore the passthrough resource if it does not have a UFS child resource + if not child_found: + manager.put(ctx, resc, constants.arb_status.IGNORE) diff --git a/integration_tests.py b/integration_tests.py index c65cea82d..d38612b34 100644 --- a/integration_tests.py +++ b/integration_tests.py @@ -17,7 +17,7 @@ import groups import meta import schema -from util import avu, collection, config, constants, data_object, group, jsonutil, log, msi, resource, rule, user +from util import avu, collection, config, constants, data_object, group, jsonutil, log, msi, resources, rule, user def _call_msvc_stat_vault(ctx, resc_name, data_path): @@ -41,7 +41,11 @@ def _call_msvc_json_arrayops(ctx, jsonstr, val, ops, index, argument_index): def _call_msvc_json_objops(ctx, jsonstr, val, ops, argument_index): """Returns an output argument from the json_objops microservice""" - return ctx.msi_json_objops(jsonstr, val, ops)["arguments"][argument_index] + result = ctx.msi_json_objops(jsonstr, val, ops)["arguments"] + if ops == "get": + return list(result[argument_index].key), list(result[argument_index].value) + else: + return result[argument_index] def _create_tmp_object(ctx): @@ -666,7 +670,7 @@ def _test_folder_secure_func(ctx, func): "check": lambda x: x["DATA_SIZE"].isdigit()}, # Using the resource_id as data_id to ensure no existing data object uses this occupied identifier {"name": "util.data_object.get_properties.no_data_object", - "test": lambda ctx: data_object.get_properties(ctx, resource.id_from_name(ctx, "irodsResc"), "irodsResc"), + "test": lambda ctx: data_object.get_properties(ctx, resources.id_from_name(ctx, "irodsResc"), "irodsResc"), "check": lambda x: x is None}, {"name": "util.data_object.owner", "test": lambda ctx: data_object.owner(ctx, "/tempZone/home/research-initial/testdata/lorem.txt"), @@ -701,29 +705,29 @@ def _test_folder_secure_func(ctx, func): {"name": "util.group.members.doesnotexist", "test": lambda ctx: user.exists(ctx, "research-doesnotexist"), "check": lambda x: x is False}, - {"name": "util.resource.exists.yes", - "test": lambda ctx: resource.exists(ctx, "irodsResc"), + {"name": "util.resources.exists.yes", + "test": lambda ctx: resources.exists(ctx, "irodsResc"), "check": lambda x: x}, - {"name": "util.resource.exists.no", - "test": lambda ctx: resource.exists(ctx, "bananaResc"), + {"name": "util.resources.exists.no", + "test": lambda ctx: resources.exists(ctx, "bananaResc"), "check": lambda x: not x}, - {"name": "util.resource.get_all_resource_names", - "test": lambda ctx: resource.get_all_resource_names(ctx), - "check": lambda x: len(x) == 16}, - {"name": "util.resource.get_children_by_name", - "test": lambda ctx: resource.get_children_by_name(ctx, "dev001_p1"), + {"name": "util.resources.get_all_resource_names", + "test": lambda ctx: resources.get_all_resource_names(ctx), + "check": lambda x: len(x) == 15}, + {"name": "util.resources.get_children_by_name", + "test": lambda ctx: resources.get_children_by_name(ctx, "dev001_p1"), "check": lambda x: x == ["dev001_1"]}, - {"name": "util.resource.get_parent_by_name", - "test": lambda ctx: resource.get_parent_by_name(ctx, "dev001_1"), + {"name": "util.resources.get_parent_by_name", + "test": lambda ctx: resources.get_parent_by_name(ctx, "dev001_1"), "check": lambda x: x == "dev001_p1"}, - {"name": "util.resource.get_resource_names_by_type", - "test": lambda ctx: resource.get_resource_names_by_type(ctx, "unixfilesystem"), - "check": lambda x: sorted(x) == sorted(['bundleResc', 'demoResc', 'dev001_1', 'dev001_2', 'dev002_1'])}, - {"name": "util.resource.get_type_by_name", - "test": lambda ctx: resource.get_type_by_name(ctx, "dev001_1"), + {"name": "util.resources.get_resource_names_by_type", + "test": lambda ctx: resources.get_resource_names_by_type(ctx, "unixfilesystem"), + "check": lambda x: sorted(x) == sorted(['bundleResc', 'dev001_1', 'dev001_2', 'dev002_1'])}, + {"name": "util.resources.get_type_by_name", + "test": lambda ctx: resources.get_type_by_name(ctx, "dev001_1"), "check": lambda x: x == "unixfilesystem"}, - {"name": "util.resource.to_from_id", - "test": lambda ctx: resource.name_from_id(ctx, resource.id_from_name(ctx, "irodsResc")), + {"name": "util.resources.to_from_id", + "test": lambda ctx: resources.name_from_id(ctx, resources.id_from_name(ctx, "irodsResc")), "check": lambda x: x == "irodsResc"}, {"name": "util.user.exists.yes", "test": lambda ctx: user.exists(ctx, "rods"), diff --git a/setup.cfg b/setup.cfg index 880b648c2..bf7b852e9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ strictness=short docstring_style=sphinx max-line-length=127 exclude=__init__.py,tools,tests/env/ -application-import-names=avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils +application-import-names=arb,avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,stats,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resources,yoda_names,policies_utils [mypy] exclude = tools|unit-tests diff --git a/resources.py b/stats.py similarity index 89% rename from resources.py rename to stats.py index 830ef953e..dd37f3163 100644 --- a/resources.py +++ b/stats.py @@ -17,8 +17,6 @@ 'api_resource_full_year_differentiated_group_storage', 'rule_resource_store_storage_statistics', 'rule_resource_research', - 'rule_resource_update_resc_arb_data', - 'rule_resource_update_misc_arb_data', 'rule_resource_vault'] @@ -556,78 +554,6 @@ def rule_resource_store_storage_statistics(ctx: rule.Context) -> str: return 'ok' -@rule.make(inputs=[0, 1, 2], outputs=[]) -def rule_resource_update_resc_arb_data(ctx: rule.Context, resc_name: str, bytes_free: int, bytes_total: int) -> None: - """ - Update ARB data for a specific resource - - :param ctx: Combined type of a callback and rei struct - :param resc_name: Name of a particular unixfilesystem resource - :param bytes_free: Free size on this resource, in bytes - :param bytes_total: Total size of this resource, in bytes - """ - if user.user_type(ctx) != 'rodsadmin': - log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") - return - - if not resource.exists(ctx, resc_name): - log.write(ctx, "Error: could not find resource named '{}' for ARB update.".format(resc_name)) - return - - bytes_free_gb = int(bytes_free) / 2 ** 30 - bytes_free_percent = 100 * (float(bytes_free) / float(bytes_total)) - - if resc_name in config.arb_exempt_resources: - arb_status = constants.arb_status.EXEMPT - elif bytes_free_gb >= config.arb_min_gb_free and bytes_free_percent > config.arb_min_percent_free: - arb_status = constants.arb_status.AVAILABLE - else: - arb_status = constants.arb_status.FULL - - parent_resc_name = resource.get_parent_by_name(ctx, resc_name) - - manager = arb_data_manager.ARBDataManager() - manager.put(ctx, resc_name, constants.arb_status.IGNORE) - - if parent_resc_name is not None and resource.get_type_by_name(ctx, parent_resc_name) == "passthru": - manager.put(ctx, parent_resc_name, arb_status) - - -@rule.make() -def rule_resource_update_misc_arb_data(ctx: rule.Context) -> None: - """Update ARB data for resources that are not covered by the regular process. That is, - all resources that are neither unixfilesystem nor passthrough resources, as well as - passthrough resources that do not have a unixfilesystem child resource. - - :param ctx: Combined type of a callback and rei struct - """ - if user.user_type(ctx) != 'rodsadmin': - log.write(ctx, "Error: insufficient permissions to run ARB data update rule.") - return - - manager = arb_data_manager.ARBDataManager() - - all_resources = resource.get_all_resource_names(ctx) - ufs_resources = set(resource.get_resource_names_by_type(ctx, "unixfilesystem") - + resource.get_resource_names_by_type(ctx, "unix file system")) - pt_resources = set(resource.get_resource_names_by_type(ctx, "passthru")) - - for resc in all_resources: - if resc in ufs_resources: - pass - elif resc not in pt_resources: - manager.put(ctx, resc, constants.arb_status.IGNORE) - else: - child_resources = resource.get_children_by_name(ctx, resc) - child_found = False - for child_resource in child_resources: - if child_resource in ufs_resources: - child_found = True - # Ignore the passthrough resource if it does not have a UFS child resource - if not child_found: - manager.put(ctx, resc, constants.arb_status.IGNORE) - - def get_categories(ctx: rule.Context) -> List: """Get all categories for current user. diff --git a/util/__init__.py b/util/__init__.py index 2353fecbb..d41b93d44 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -34,7 +34,7 @@ import group import avu import misc - import resource + import resources import arb_data_manager import cached_data_manager import irods_type_info diff --git a/util/resource.py b/util/resources.py similarity index 100% rename from util/resource.py rename to util/resources.py diff --git a/util/yoda_names.py b/util/yoda_names.py index ebc1e6d6e..8b2306710 100644 --- a/util/yoda_names.py +++ b/util/yoda_names.py @@ -1,5 +1,4 @@ -"""This class contains utility functions that process names of Yoda entities (e.g. category names, user names, etc.) -""" +"""This class contains utility functions that process names of Yoda entities (e.g. category names, user names, etc.)""" __copyright__ = 'Copyright (c) 2019-2024, Utrecht University' __license__ = 'GPLv3, see LICENSE'