YDA-6062: fix name clash and integration tests

__init__.py

@@ -29,6 +29,7 @@
 # so that they become visible to iRODS.
 
 from admin                    import *
+from arb                      import *
 from browse                   import *
 from folder                   import *
 from groups                   import *
@@ -39,10 +40,10 @@
 from meta_form                import *
 from provenance               import *
 from research                 import *
-from resources                import *
 from schema                   import *
 from schema_transformation    import *
 from schema_transformations   import *
+from stats                    import *
 from publication_troubleshoot import *
 from vault                    import *
 from datacite                 import *

arb.py

@@ -0,0 +1,81 @@
+"""Functions for automatic resource balancing module."""
+
+__copyright__ = 'Copyright (c) 2019-2024, Utrecht University'
+__license__   = 'GPLv3, see LICENSE'
+
+from util import *
+
+__all__ = ['rule_resource_update_resc_arb_data',
+           'rule_resource_update_misc_arb_data']
+
+
+@rule.make(inputs=[0, 1, 2], outputs=[])
+def rule_resource_update_resc_arb_data(ctx: rule.Context, resc_name: str, bytes_free: int, bytes_total: int) -> None:
+    """
+    Update ARB data for a specific resource
+
+    :param ctx:         Combined type of a callback and rei struct
+    :param resc_name:   Name of a particular unixfilesystem resource
+    :param bytes_free:  Free size on this resource, in bytes
+    :param bytes_total: Total size of this resource, in bytes
+    """
+    if user.user_type(ctx) != 'rodsadmin':
+        log.write(ctx, "Error: insufficient permissions to run ARB data update rule.")
+        return
+
+    if not resources.exists(ctx, resc_name):
+        log.write(ctx, "Error: could not find resource named '{}' for ARB update.".format(resc_name))
+        return
+
+    bytes_free_gb = int(bytes_free) / 2 ** 30
+    bytes_free_percent = 100 * (float(bytes_free) / float(bytes_total))
+
+    if resc_name in config.arb_exempt_resources:
+        arb_status = constants.arb_status.EXEMPT
+    elif bytes_free_gb >= config.arb_min_gb_free and bytes_free_percent > config.arb_min_percent_free:
+        arb_status = constants.arb_status.AVAILABLE
+    else:
+        arb_status = constants.arb_status.FULL
+
+    parent_resc_name = resources.get_parent_by_name(ctx, resc_name)
+
+    manager = arb_data_manager.ARBDataManager()
+    manager.put(ctx, resc_name, constants.arb_status.IGNORE)
+
+    if parent_resc_name is not None and resources.get_type_by_name(ctx, parent_resc_name) == "passthru":
+        manager.put(ctx, parent_resc_name, arb_status)
+
+
+@rule.make()
+def rule_resource_update_misc_arb_data(ctx: rule.Context) -> None:
+    """Update ARB data for resources that are not covered by the regular process. That is,
+       all resources that are neither unixfilesystem nor passthrough resources, as well as
+       passthrough resources that do not have a unixfilesystem child resource.
+
+       :param ctx: Combined type of a callback and rei struct
+    """
+    if user.user_type(ctx) != 'rodsadmin':
+        log.write(ctx, "Error: insufficient permissions to run ARB data update rule.")
+        return
+
+    manager = arb_data_manager.ARBDataManager()
+
+    all_resources = resources.get_all_resource_names(ctx)
+    ufs_resources = set(resources.get_resource_names_by_type(ctx, "unixfilesystem")
+                        + resources.get_resource_names_by_type(ctx, "unix file system"))
+    pt_resources  = set(resources.get_resource_names_by_type(ctx, "passthru"))
+
+    for resc in all_resources:
+        if resc in ufs_resources:
+            pass
+        elif resc not in pt_resources:
+            manager.put(ctx, resc, constants.arb_status.IGNORE)
+        else:
+            child_resources = resources.get_children_by_name(ctx, resc)
+            child_found = False
+            for child_resource in child_resources:
+                if child_resource in ufs_resources:
+                    child_found = True
+            # Ignore the passthrough resource if it does not have a UFS child resource
+            if not child_found:
+                manager.put(ctx, resc, constants.arb_status.IGNORE)

integration_tests.py

@@ -17,7 +17,7 @@
 import groups
 import meta
 import schema
-from util import avu, collection, config, constants, data_object, group, jsonutil, log, msi, resource, rule, user
+from util import avu, collection, config, constants, data_object, group, jsonutil, log, msi, resources, rule, user
 
 
 def _call_msvc_stat_vault(ctx, resc_name, data_path):
@@ -41,7 +41,11 @@ def _call_msvc_json_arrayops(ctx, jsonstr, val, ops, index, argument_index):
 
 def _call_msvc_json_objops(ctx, jsonstr, val, ops, argument_index):
     """Returns an output argument from the json_objops microservice"""
-    return ctx.msi_json_objops(jsonstr, val, ops)["arguments"][argument_index]
+    result = ctx.msi_json_objops(jsonstr, val, ops)["arguments"]
+    if ops == "get":
+        return list(result[argument_index].key), list(result[argument_index].value)
+    else:
+        return result[argument_index]
 
 
 def _create_tmp_object(ctx):
@@ -666,7 +670,7 @@ def _test_folder_secure_func(ctx, func):
      "check": lambda x: x["DATA_SIZE"].isdigit()},
     # Using the resource_id as data_id to ensure no existing data object uses this occupied identifier
     {"name":   "util.data_object.get_properties.no_data_object",
-     "test": lambda ctx: data_object.get_properties(ctx, resource.id_from_name(ctx, "irodsResc"), "irodsResc"),
+     "test": lambda ctx: data_object.get_properties(ctx, resources.id_from_name(ctx, "irodsResc"), "irodsResc"),
      "check": lambda x: x is None},
     {"name":   "util.data_object.owner",
      "test": lambda ctx: data_object.owner(ctx, "/tempZone/home/research-initial/testdata/lorem.txt"),
@@ -701,29 +705,29 @@ def _test_folder_secure_func(ctx, func):
     {"name":   "util.group.members.doesnotexist",
      "test": lambda ctx: user.exists(ctx, "research-doesnotexist"),
      "check": lambda x: x is False},
-    {"name":   "util.resource.exists.yes",
-     "test": lambda ctx: resource.exists(ctx, "irodsResc"),
+    {"name":   "util.resources.exists.yes",
+     "test": lambda ctx: resources.exists(ctx, "irodsResc"),
      "check": lambda x: x},
-    {"name":   "util.resource.exists.no",
-     "test": lambda ctx: resource.exists(ctx, "bananaResc"),
+    {"name":   "util.resources.exists.no",
+     "test": lambda ctx: resources.exists(ctx, "bananaResc"),
      "check": lambda x: not x},
-    {"name":   "util.resource.get_all_resource_names",
-     "test": lambda ctx: resource.get_all_resource_names(ctx),
-     "check": lambda x: len(x) == 16},
-    {"name":   "util.resource.get_children_by_name",
-     "test": lambda ctx: resource.get_children_by_name(ctx, "dev001_p1"),
+    {"name":   "util.resources.get_all_resource_names",
+     "test": lambda ctx: resources.get_all_resource_names(ctx),
+     "check": lambda x: len(x) == 15},
+    {"name":   "util.resources.get_children_by_name",
+     "test": lambda ctx: resources.get_children_by_name(ctx, "dev001_p1"),
      "check": lambda x: x == ["dev001_1"]},
-    {"name":   "util.resource.get_parent_by_name",
-     "test": lambda ctx: resource.get_parent_by_name(ctx, "dev001_1"),
+    {"name":   "util.resources.get_parent_by_name",
+     "test": lambda ctx: resources.get_parent_by_name(ctx, "dev001_1"),
      "check": lambda x: x == "dev001_p1"},
-    {"name":   "util.resource.get_resource_names_by_type",
-     "test": lambda ctx: resource.get_resource_names_by_type(ctx, "unixfilesystem"),
-     "check": lambda x: sorted(x) == sorted(['bundleResc', 'demoResc', 'dev001_1', 'dev001_2', 'dev002_1'])},
-    {"name":   "util.resource.get_type_by_name",
-     "test": lambda ctx: resource.get_type_by_name(ctx, "dev001_1"),
+    {"name":   "util.resources.get_resource_names_by_type",
+     "test": lambda ctx: resources.get_resource_names_by_type(ctx, "unixfilesystem"),
+     "check": lambda x: sorted(x) == sorted(['bundleResc', 'dev001_1', 'dev001_2', 'dev002_1'])},
+    {"name":   "util.resources.get_type_by_name",
+     "test": lambda ctx: resources.get_type_by_name(ctx, "dev001_1"),
      "check": lambda x: x == "unixfilesystem"},
-    {"name":   "util.resource.to_from_id",
-     "test": lambda ctx: resource.name_from_id(ctx, resource.id_from_name(ctx, "irodsResc")),
+    {"name":   "util.resources.to_from_id",
+     "test": lambda ctx: resources.name_from_id(ctx, resources.id_from_name(ctx, "irodsResc")),
      "check": lambda x: x == "irodsResc"},
     {"name":   "util.user.exists.yes",
      "test": lambda ctx: user.exists(ctx, "rods"),

setup.cfg

@@ -5,7 +5,7 @@ strictness=short
 docstring_style=sphinx
 max-line-length=127
 exclude=__init__.py,tools,tests/env/
-application-import-names=avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resource,yoda_names,policies_utils
+application-import-names=arb,avu,conftest,util,api,config,constants,data_access_token,datacite,datarequest,data_object,epic,error,folder,groups,groups_import,json_datacite,json_landing_page,jsonutil,log,mail,meta,meta_form,msi,notifications,schema,schema_transformation,schema_transformations,settings,stats,pathutil,provenance,policies_intake,policies_datamanager,policies_datapackage_status,policies_folder_status,policies_datarequest_status,publication,query,replication,revisions,revision_strategies,revision_utils,rule,user,vault,sram,arb_data_manager,cached_data_manager,resources,yoda_names,policies_utils
 
 [mypy]
 exclude = tools|unit-tests

resources.py → stats.py

@@ -17,8 +17,6 @@
            'api_resource_full_year_differentiated_group_storage',
            'rule_resource_store_storage_statistics',
            'rule_resource_research',
-           'rule_resource_update_resc_arb_data',
-           'rule_resource_update_misc_arb_data',
            'rule_resource_vault']
 
 
@@ -556,78 +554,6 @@ def rule_resource_store_storage_statistics(ctx: rule.Context) -> str:
     return 'ok'
 
 
-@rule.make(inputs=[0, 1, 2], outputs=[])
-def rule_resource_update_resc_arb_data(ctx: rule.Context, resc_name: str, bytes_free: int, bytes_total: int) -> None:
-    """
-    Update ARB data for a specific resource
-
-    :param ctx:         Combined type of a callback and rei struct
-    :param resc_name:   Name of a particular unixfilesystem resource
-    :param bytes_free:  Free size on this resource, in bytes
-    :param bytes_total: Total size of this resource, in bytes
-    """
-    if user.user_type(ctx) != 'rodsadmin':
-        log.write(ctx, "Error: insufficient permissions to run ARB data update rule.")
-        return
-
-    if not resource.exists(ctx, resc_name):
-        log.write(ctx, "Error: could not find resource named '{}' for ARB update.".format(resc_name))
-        return
-
-    bytes_free_gb = int(bytes_free) / 2 ** 30
-    bytes_free_percent = 100 * (float(bytes_free) / float(bytes_total))
-
-    if resc_name in config.arb_exempt_resources:
-        arb_status = constants.arb_status.EXEMPT
-    elif bytes_free_gb >= config.arb_min_gb_free and bytes_free_percent > config.arb_min_percent_free:
-        arb_status = constants.arb_status.AVAILABLE
-    else:
-        arb_status = constants.arb_status.FULL
-
-    parent_resc_name = resource.get_parent_by_name(ctx, resc_name)
-
-    manager = arb_data_manager.ARBDataManager()
-    manager.put(ctx, resc_name, constants.arb_status.IGNORE)
-
-    if parent_resc_name is not None and resource.get_type_by_name(ctx, parent_resc_name) == "passthru":
-        manager.put(ctx, parent_resc_name, arb_status)
-
-
-@rule.make()
-def rule_resource_update_misc_arb_data(ctx: rule.Context) -> None:
-    """Update ARB data for resources that are not covered by the regular process. That is,
-       all resources that are neither unixfilesystem nor passthrough resources, as well as
-       passthrough resources that do not have a unixfilesystem child resource.
-
-       :param ctx: Combined type of a callback and rei struct
-    """
-    if user.user_type(ctx) != 'rodsadmin':
-        log.write(ctx, "Error: insufficient permissions to run ARB data update rule.")
-        return
-
-    manager = arb_data_manager.ARBDataManager()
-
-    all_resources = resource.get_all_resource_names(ctx)
-    ufs_resources = set(resource.get_resource_names_by_type(ctx, "unixfilesystem")
-                        + resource.get_resource_names_by_type(ctx, "unix file system"))
-    pt_resources  = set(resource.get_resource_names_by_type(ctx, "passthru"))
-
-    for resc in all_resources:
-        if resc in ufs_resources:
-            pass
-        elif resc not in pt_resources:
-            manager.put(ctx, resc, constants.arb_status.IGNORE)
-        else:
-            child_resources = resource.get_children_by_name(ctx, resc)
-            child_found = False
-            for child_resource in child_resources:
-                if child_resource in ufs_resources:
-                    child_found = True
-            # Ignore the passthrough resource if it does not have a UFS child resource
-            if not child_found:
-                manager.put(ctx, resc, constants.arb_status.IGNORE)
-
-
 def get_categories(ctx: rule.Context) -> List:
     """Get all categories for current user.
 

util/__init__.py

@@ -34,7 +34,7 @@
     import group
     import avu
     import misc
-    import resource
+    import resources
     import arb_data_manager
     import cached_data_manager
     import irods_type_info

util/yoda_names.py

@@ -1,5 +1,4 @@
-"""This class contains utility functions that process names of Yoda entities (e.g. category names, user names, etc.)
-"""
+"""This class contains utility functions that process names of Yoda entities (e.g. category names, user names, etc.)"""
 
 __copyright__ = 'Copyright (c) 2019-2024, Utrecht University'
 __license__   = 'GPLv3, see LICENSE'