Add automatic conversion of native specifications from Torque to Slurm

bidscoin/bcoin.py

@@ -51,6 +51,29 @@ def update_to(self, b=1, bsize=1, tsize=None):
         self.update(b * bsize - self.n)  # will also set self.n = b * bsize
 
 
+def drmaa_nativespec(specs: str) -> str:
+    """
+    Converts native Torque walltime and memory specifications to the DRMAA implementation (currently only Slurm is supported)
+
+    :param specs: Native Torque walltime and memory specifications, e.g. '-l walltime=00:10:00,mem=2gb'
+    :return:      The converted native specifications
+    """
+
+    import drmaa    # Lazy import to avoid import error on non-HPC systems
+
+    with drmaa.Session() as session:
+        implementation = session.drmaaImplementation
+
+    if 'Slurm' in implementation and '-l ' in specs:
+        specs = (specs.replace('-l ', '')
+                      .replace(',', ' ')
+                      .replace('walltime', '--time')
+                      .replace('mem', '--mem')
+                      .replace('gb','000'))
+
+    return specs.strip()
+
+
 def synchronize(pbatch, jobids: list, wait: int=15):
     """
     Shows tqdm progress bars for queued and running DRMAA jobs. Waits until all jobs have finished +

bidscoin/bidsapps/deface.py

@@ -61,6 +61,7 @@ def deface(bidsfolder: str, pattern: str, participant: list, force: bool, output
     # Prepare the HPC job submission
     if cluster:
         from drmaa import Session as drmaasession
+        from bidscoin.bcoin import drmaa_nativespec
     else:
         from contextlib import nullcontext as drmaasession                                      # Use a dummy context manager
     with drmaasession() as pbatch:
@@ -69,7 +70,7 @@ def deface(bidsfolder: str, pattern: str, participant: list, force: bool, output
             jt                     = pbatch.createJobTemplate()
             jt.jobEnvironment      = os.environ
             jt.remoteCommand       = shutil.which('pydeface')
-            jt.nativeSpecification = cluster
+            jt.nativeSpecification = drmaa_nativespec(cluster)
             jt.joinFiles           = True
 
         # Loop over bids subject/session-directories

bidscoin/bidsapps/medeface.py

@@ -67,6 +67,7 @@ def medeface(bidsfolder: str, pattern: str, maskpattern: str, participant: list,
     tmp_combined = f"{next(tempfile._get_candidate_names())}_echocombined_deface.nii"
     if cluster:
         from drmaa import Session as drmaasession
+        from bidscoin.bcoin import drmaa_nativespec
     else:
         from contextlib import nullcontext as drmaasession                                      # Use a dummy context manager
     with drmaasession() as pbatch:
@@ -75,7 +76,7 @@ def medeface(bidsfolder: str, pattern: str, maskpattern: str, participant: list,
             jt                     = pbatch.createJobTemplate()
             jt.jobEnvironment      = os.environ
             jt.remoteCommand       = shutil.which('pydeface')
-            jt.nativeSpecification = cluster
+            jt.nativeSpecification = drmaa_nativespec(cluster)
             jt.joinFiles           = True
 
         # Loop over bids subject/session-directories to first get all the echo-combined deface masks

bidscoin/bidsapps/skullstrip.py

@@ -75,6 +75,7 @@ def skullstrip(bidsfolder: str, pattern: str, participant: list, masked: str, ou
     # Prepare the HPC job submission
     if cluster:
         from drmaa import Session as drmaasession
+        from bidscoin.bcoin import drmaa_nativespec
     else:
         from contextlib import nullcontext as drmaasession                                      # Use a dummy context manager
     with drmaasession() as pbatch:
@@ -83,7 +84,7 @@ def skullstrip(bidsfolder: str, pattern: str, participant: list, masked: str, ou
             jt                     = pbatch.createJobTemplate()
             jt.jobEnvironment      = os.environ
             jt.remoteCommand       = shutil.which('mri_synthstrip')
-            jt.nativeSpecification = cluster
+            jt.nativeSpecification = drmaa_nativespec(cluster)
             jt.joinFiles           = True
 
         # Loop over bids subject/session-directories

bidscoin/bidsapps/slicereport.py

@@ -114,6 +114,7 @@ def slicer_append(inputimage: Path, operations: str, outlineimage: Path, mainopt
     # Run the command on the HPC cluster or directly in the shell
     if cluster:
         from drmaa import Session as drmaasession   # Lazy import to avoid import error on non-HPC systems
+        from bidscoin.bcoin import drmaa_nativespec
 
         script = workdir/'slicereport.sh'
         script.write_text('#!/bin/bash\n' + command)
@@ -122,7 +123,7 @@ def slicer_append(inputimage: Path, operations: str, outlineimage: Path, mainopt
             jt                     = pbatch.createJobTemplate()
             jt.jobEnvironment      = os.environ
             jt.remoteCommand       = str(script)
-            jt.nativeSpecification = cluster
+            jt.nativeSpecification = drmaa_nativespec(cluster)
             jt.joinFiles           = True
             jt.jobName             = 'slicereport'
             jt.outputPath          = f"{os.getenv('HOSTNAME')}:{workdir}/{jt.jobName}.out"

bidscoin/bidscoiner.py

@@ -134,6 +134,7 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force:
     if cluster:
 
         from drmaa import Session as drmaasession           # NB: Importing drmaa for non-HPC users may cause import errors
+        from bidscoin.bcoin import drmaa_nativespec
 
         LOGGER.info('')
         LOGGER.info('============== HPC START ==============')
@@ -142,7 +143,7 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force:
             jt                     = pbatch.createJobTemplate()
             jt.jobEnvironment      = os.environ
             jt.remoteCommand       = shutil.which('bidscoiner') or __file__
-            jt.nativeSpecification = cluster
+            jt.nativeSpecification = drmaa_nativespec(cluster)
             jt.joinFiles           = True
             jobids                 = []