diff --git a/em_workflows/dm_conversion/flow.py b/em_workflows/dm_conversion/flow.py index 486e9760..d4aef73c 100644 --- a/em_workflows/dm_conversion/flow.py +++ b/em_workflows/dm_conversion/flow.py @@ -36,44 +36,9 @@ def _calculate_shrink_factor(filepath: Path, enforce_2d=True) -> float: return min_xy / LARGE_DIM -@task( - name="Convert DM to MRC", - on_failure=[utils.collect_exception_task_hook], -) -def convert_dms_to_mrc(file_path: FilePath) -> FilePath: - if file_path.fp_in.suffix.strip(".").lower() not in DMS_EXT: - return file_path - dm_as_mrc = file_path.gen_output_fp(out_fname="dm_as_mrc.mrc") - msg = f"Using dir: {file_path.fp_in}, : creating dm_as_mrc {dm_as_mrc}" - utils.log(msg=msg) - log_file = f"{dm_as_mrc.parent}/dm2mrc.log" - cmd = [DMConfig.dm2mrc_loc, file_path.fp_in.as_posix(), dm_as_mrc.as_posix()] - # utils.log(f"Generated cmd {cmd}") - FilePath.run(cmd=cmd, log_file=log_file) - return file_path +def _newstack_mrc_to_tiff(input_fn: Path, output_fn: Path, log_fn: Path, use_float=True) -> None: - -@task( - name="Convert 16bit to 8bit tiffs", - on_failure=[utils.collect_exception_task_hook], -) -def convert_if_int16_tiff(file_path: FilePath) -> None: - """ - accepts a tiff Path obj - tests if 16 bit - if 16 bit convert (write to assets_dir) & return Path - else return orig Path - """ - if not ( - file_path.fp_in.suffix.strip(".").lower() in TIFS_EXT - and is_16bit(file_path.fp_in) - ): - return - tif_8_bit = file_path.gen_output_fp(out_fname="as_8_bit.tif") - log_fp = f"{file_path.working_dir}/{file_path.base}_as_8_bit.log" - utils.log(f"{file_path.fp_in} is a 16 bit tiff, converting to {tif_8_bit}") - - shrink_factor = _calculate_shrink_factor(file_path.fp_in) + shrink_factor = _calculate_shrink_factor(input_fn) cmd = [ "env", @@ -85,83 +50,60 @@ def convert_if_int16_tiff(file_path: FilePath) -> None: "6", "-mode", "0", - # From the newstack manual for "-float": Enter 1 for each section to fill the data range. - "-float", - "1", - file_path.fp_in.as_posix(), - str(tif_8_bit), - ] - utils.log(f"Generated cmd {cmd}") - FilePath.run(cmd, log_fp) + ] + + if use_float: + cmd.extend(["-float", "1"]) + else: + cmd.extend(["-meansd", "140,50"]) + cmd.extend([ + input_fn.as_posix(), + output_fn.as_posix(), + ]) + FilePath.run(cmd, str(log_fn)) @task( - name="Convert 2D MRC to tiff", + name="Convert EM images to tiff", on_failure=[utils.collect_exception_task_hook], ) -def convert_2d_mrc_to_tiff(file_path: FilePath) -> None: - """ - Checks Projects dir for mrc inputs. We assume anything in Projects will be 2D. - Converts to tiff file, 1024 in size (using constant LARGE_DIM) +def convert_em_to_tiff(file_path: FilePath) -> FilePath: - env IMOD_OUTPUT_FORMAT=TIF newstack \ - --shrink $shrink_factor$ -antialias 6 -mode 0 -meansd 140,50 f_in.mrc f_out.tif - where shrink_factor is input_size/1024 - used the smaller of x and y (4092/1024) - """ - # nifti_fp = neuroglancer.gen_niftis.__wrapped__(file_path) - # utils.log(f"+++++++++++++++++++++++++++++++++++++++++++++") - # min_max_histo = neuroglancer.gen_min_max_histo(file_path) - # utils.log(min_max_histo) - # utils.log(f"+++++++++++++++++++++++++++++++++++++++++++++") - - # work out shrink_factor - if file_path.fp_in.suffix.strip(".").lower() not in MRCS_EXT: - return - shrink_factor = _calculate_shrink_factor( file_path.fp_in) - # round to 3 decimal places - shrink_factor_3 = f"{shrink_factor:.3f}" - out_fp = f"{file_path.working_dir}/{file_path.base}_mrc_as_tiff.tiff" - log_fp = f"{file_path.working_dir}/{file_path.base}_mrc_as_tiff.log" - utils.log(f"{file_path.fp_in.as_posix()} is a mrc file, will convert to {out_fp}.") - # work out meansd - cmd = [ - "env", - "IMOD_OUTPUT_FORMAT=TIF", - DMConfig.newstack_loc, - "-shrink", - shrink_factor_3, - "-antialias", - "6", - "-mode", - "0", - "-meansd", - "140,50", - file_path.fp_in.as_posix(), - out_fp, - ] - utils.log(f"Generated cmd {cmd}") - FilePath.run(cmd, log_fp) + if file_path.fp_in.suffix.strip(".").lower() in MRCS_EXT: + out_fp = file_path.gen_output_fp(out_fname="_as_tiff.tiff") + log_fp = file_path.working_dir/f"{file_path.base}_as_tiff.log" + utils.log(f"{file_path.fp_in.as_posix()} is a mrc file, will convert to {out_fp}.") -@task( - name="Convert DM MRC to jpeg", - on_failure=[utils.collect_exception_task_hook], -) -def convert_dm_mrc_to_jpeg(file_path: FilePath) -> None: - """ - converts previously generated mrc file, (derived from a dm file) to jpeg. - note, ignores everything that's NOT dm_as_mrc.mrc - that is, convert_dm_mrc_to_jpeg does not process ALL mrc files. - """ - dm_as_mrc = file_path.gen_output_fp(out_fname="dm_as_mrc.mrc") - if dm_as_mrc.exists(): - mrc_as_jpg = file_path.gen_output_fp(out_fname="mrc_as_jpg.jpeg") - log_fp = f"{mrc_as_jpg.parent}/mrc2tif.log" - cmd = [DMConfig.mrc2tif_loc, "-j", dm_as_mrc.as_posix(), mrc_as_jpg.as_posix()] - utils.log(f"Generated cmd {cmd}") - FilePath.run(cmd, log_fp) + _newstack_mrc_to_tiff(file_path.fp_in, out_fp, log_fp, use_float=False) + + elif ( + file_path.fp_in.suffix.strip(".").lower() in TIFS_EXT + and is_16bit(file_path.fp_in) + ): + + tif_8_bit = file_path.gen_output_fp(out_fname="_as_tiff.tif") + log_fp = file_path.working_dir/f"{file_path.base}_as_tiff.log" + utils.log(f"{file_path.fp_in} is a 16 bit tiff, converting to {tif_8_bit}") + + _newstack_mrc_to_tiff(file_path.fp_in, tif_8_bit, log_fp, use_float=True) + + elif file_path.fp_in.suffix.strip(".").lower() in DMS_EXT: + dm_as_mrc = file_path.gen_output_fp(out_fname="dm_as_mrc.mrc") + utils.log(msg=f"{file_path.fp_in} is a dm file, will convert to {dm_as_mrc}.") + log_file = file_path.working_dir/"dm2mrc.log" + + cmd = [DMConfig.dm2mrc_loc, file_path.fp_in.as_posix(), dm_as_mrc.as_posix()] + FilePath.run(cmd=cmd, log_file=str(log_file)) + + out_fp = file_path.working_dir/f"{file_path.base}_as_tiff.tiff" + log_fp = file_path.working_dir/f"{file_path.base}_as_tiff.log" + utils.log(f"{dm_as_mrc} convert to {out_fp}.") + + _newstack_mrc_to_tiff(dm_as_mrc, out_fp, log_fp, use_float=True) + + return file_path @task( @@ -173,15 +115,10 @@ def scale_jpegs(file_path: FilePath, size: str) -> Optional[dict]: generates keyThumbnail and keyImage looks for file names _mrc_as_jpg.jpeg """ - mrc_as_jpg = file_path.gen_output_fp(out_fname="mrc_as_jpg.jpeg") - tif_8_bit = file_path.gen_output_fp(out_fname="as_8_bit.tif") - mrc_as_tiff = Path(f"{file_path.working_dir}/{file_path.base}_mrc_as_tiff.tiff") - if mrc_as_jpg.exists(): - cur = mrc_as_jpg - elif tif_8_bit.exists(): - cur = tif_8_bit - elif mrc_as_tiff.exists(): - cur = mrc_as_tiff # can ignore here if scaled + + as_tiff = Path(f"{file_path.working_dir}/{file_path.base}_as_tiff.tiff") + if as_tiff.exists(): + cur = as_tiff elif file_path.fp_in.suffix.strip(".").lower() in KEYIMG_EXTS: cur = file_path.fp_in else: @@ -287,27 +224,18 @@ def dm_flow( share_name=file_share, input_dir=input_dir_fp, fps_in=input_fps ) - # TODO move the three conversion block into if-else based on filetypes - tiff_results = convert_if_int16_tiff.map(file_path=fps) - - # mrc is intermed format, to jpeg conversion - dm_to_mrc = convert_dms_to_mrc.map(file_path=fps) - mrc_results = convert_dm_mrc_to_jpeg.map(dm_to_mrc) - - mrc_tiff_results = convert_2d_mrc_to_tiff.map( - file_path=fps, wait_for=[allow_failure(mrc_results)] - ) + tiff_results = convert_em_to_tiff.map(file_path=fps) # Finally generate all valid suffixed results keyimg_assets = scale_jpegs.map( fps, size=unmapped("l"), - wait_for=[allow_failure(tiff_results), allow_failure(mrc_tiff_results)], + wait_for=[allow_failure(tiff_results), allow_failure(tiff_results)], ) thumb_assets = scale_jpegs.map( fps, size=unmapped("s"), - wait_for=[allow_failure(tiff_results), allow_failure(mrc_tiff_results)], + wait_for=[allow_failure(tiff_results), allow_failure(tiff_results)], ) prim_fps = utils.gen_prim_fps.map(fp_in=fps)