Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP - DM convert refactor #486

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

WIP - DM convert refactor #486

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 54 additions & 126 deletions em_workflows/dm_conversion/flow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,44 +36,9 @@ def _calculate_shrink_factor(filepath: Path, enforce_2d=True) -> float:
return min_xy / LARGE_DIM


@task(
name="Convert DM to MRC",
on_failure=[utils.collect_exception_task_hook],
)
def convert_dms_to_mrc(file_path: FilePath) -> FilePath:
if file_path.fp_in.suffix.strip(".").lower() not in DMS_EXT:
return file_path
dm_as_mrc = file_path.gen_output_fp(out_fname="dm_as_mrc.mrc")
msg = f"Using dir: {file_path.fp_in}, : creating dm_as_mrc {dm_as_mrc}"
utils.log(msg=msg)
log_file = f"{dm_as_mrc.parent}/dm2mrc.log"
cmd = [DMConfig.dm2mrc_loc, file_path.fp_in.as_posix(), dm_as_mrc.as_posix()]
# utils.log(f"Generated cmd {cmd}")
FilePath.run(cmd=cmd, log_file=log_file)
return file_path
def _newstack_mrc_to_tiff(input_fn: Path, output_fn: Path, log_fn: Path, use_float=True) -> None:


@task(
name="Convert 16bit to 8bit tiffs",
on_failure=[utils.collect_exception_task_hook],
)
def convert_if_int16_tiff(file_path: FilePath) -> None:
"""
accepts a tiff Path obj
tests if 16 bit
if 16 bit convert (write to assets_dir) & return Path
else return orig Path
"""
if not (
file_path.fp_in.suffix.strip(".").lower() in TIFS_EXT
and is_16bit(file_path.fp_in)
):
return
tif_8_bit = file_path.gen_output_fp(out_fname="as_8_bit.tif")
log_fp = f"{file_path.working_dir}/{file_path.base}_as_8_bit.log"
utils.log(f"{file_path.fp_in} is a 16 bit tiff, converting to {tif_8_bit}")

shrink_factor = _calculate_shrink_factor(file_path.fp_in)
shrink_factor = _calculate_shrink_factor(input_fn)

cmd = [
"env",
Expand All @@ -85,83 +50,60 @@ def convert_if_int16_tiff(file_path: FilePath) -> None:
"6",
"-mode",
"0",
# From the newstack manual for "-float": Enter 1 for each section to fill the data range.
"-float",
"1",
file_path.fp_in.as_posix(),
str(tif_8_bit),
]
utils.log(f"Generated cmd {cmd}")
FilePath.run(cmd, log_fp)
]

if use_float:
cmd.extend(["-float", "1"])
else:
cmd.extend(["-meansd", "140,50"])

cmd.extend([
input_fn.as_posix(),
output_fn.as_posix(),
])

FilePath.run(cmd, str(log_fn))

@task(
name="Convert 2D MRC to tiff",
name="Convert EM images to tiff",
on_failure=[utils.collect_exception_task_hook],
)
def convert_2d_mrc_to_tiff(file_path: FilePath) -> None:
"""
Checks Projects dir for mrc inputs. We assume anything in Projects will be 2D.
Converts to tiff file, 1024 in size (using constant LARGE_DIM)
def convert_em_to_tiff(file_path: FilePath) -> FilePath:

env IMOD_OUTPUT_FORMAT=TIF newstack \
--shrink $shrink_factor$ -antialias 6 -mode 0 -meansd 140,50 f_in.mrc f_out.tif
where shrink_factor is input_size/1024
used the smaller of x and y (4092/1024)
"""
# nifti_fp = neuroglancer.gen_niftis.__wrapped__(file_path)
# utils.log(f"+++++++++++++++++++++++++++++++++++++++++++++")
# min_max_histo = neuroglancer.gen_min_max_histo(file_path)
# utils.log(min_max_histo)
# utils.log(f"+++++++++++++++++++++++++++++++++++++++++++++")

# work out shrink_factor
if file_path.fp_in.suffix.strip(".").lower() not in MRCS_EXT:
return
shrink_factor = _calculate_shrink_factor( file_path.fp_in)
# round to 3 decimal places
shrink_factor_3 = f"{shrink_factor:.3f}"
out_fp = f"{file_path.working_dir}/{file_path.base}_mrc_as_tiff.tiff"
log_fp = f"{file_path.working_dir}/{file_path.base}_mrc_as_tiff.log"
utils.log(f"{file_path.fp_in.as_posix()} is a mrc file, will convert to {out_fp}.")
# work out meansd
cmd = [
"env",
"IMOD_OUTPUT_FORMAT=TIF",
DMConfig.newstack_loc,
"-shrink",
shrink_factor_3,
"-antialias",
"6",
"-mode",
"0",
"-meansd",
"140,50",
file_path.fp_in.as_posix(),
out_fp,
]
utils.log(f"Generated cmd {cmd}")
FilePath.run(cmd, log_fp)
if file_path.fp_in.suffix.strip(".").lower() in MRCS_EXT:

out_fp = file_path.gen_output_fp(out_fname="_as_tiff.tiff")
log_fp = file_path.working_dir/f"{file_path.base}_as_tiff.log"
utils.log(f"{file_path.fp_in.as_posix()} is a mrc file, will convert to {out_fp}.")

@task(
name="Convert DM MRC to jpeg",
on_failure=[utils.collect_exception_task_hook],
)
def convert_dm_mrc_to_jpeg(file_path: FilePath) -> None:
"""
converts previously generated mrc file, (derived from a dm file) to jpeg.
note, ignores everything that's NOT <name>dm_as_mrc.mrc
that is, convert_dm_mrc_to_jpeg does not process ALL mrc files.
"""
dm_as_mrc = file_path.gen_output_fp(out_fname="dm_as_mrc.mrc")
if dm_as_mrc.exists():
mrc_as_jpg = file_path.gen_output_fp(out_fname="mrc_as_jpg.jpeg")
log_fp = f"{mrc_as_jpg.parent}/mrc2tif.log"
cmd = [DMConfig.mrc2tif_loc, "-j", dm_as_mrc.as_posix(), mrc_as_jpg.as_posix()]
utils.log(f"Generated cmd {cmd}")
FilePath.run(cmd, log_fp)
_newstack_mrc_to_tiff(file_path.fp_in, out_fp, log_fp, use_float=False)

elif (
file_path.fp_in.suffix.strip(".").lower() in TIFS_EXT
and is_16bit(file_path.fp_in)
):

tif_8_bit = file_path.gen_output_fp(out_fname="_as_tiff.tif")
log_fp = file_path.working_dir/f"{file_path.base}_as_tiff.log"
utils.log(f"{file_path.fp_in} is a 16 bit tiff, converting to {tif_8_bit}")

_newstack_mrc_to_tiff(file_path.fp_in, tif_8_bit, log_fp, use_float=True)

elif file_path.fp_in.suffix.strip(".").lower() in DMS_EXT:
dm_as_mrc = file_path.gen_output_fp(out_fname="dm_as_mrc.mrc")
utils.log(msg=f"{file_path.fp_in} is a dm file, will convert to {dm_as_mrc}.")
log_file = file_path.working_dir/"dm2mrc.log"

cmd = [DMConfig.dm2mrc_loc, file_path.fp_in.as_posix(), dm_as_mrc.as_posix()]
FilePath.run(cmd=cmd, log_file=str(log_file))

out_fp = file_path.working_dir/f"{file_path.base}_as_tiff.tiff"
log_fp = file_path.working_dir/f"{file_path.base}_as_tiff.log"
utils.log(f"{dm_as_mrc} convert to {out_fp}.")

_newstack_mrc_to_tiff(dm_as_mrc, out_fp, log_fp, use_float=True)

return file_path


@task(
Expand All @@ -173,15 +115,10 @@ def scale_jpegs(file_path: FilePath, size: str) -> Optional[dict]:
generates keyThumbnail and keyImage
looks for file names <something>_mrc_as_jpg.jpeg
"""
mrc_as_jpg = file_path.gen_output_fp(out_fname="mrc_as_jpg.jpeg")
tif_8_bit = file_path.gen_output_fp(out_fname="as_8_bit.tif")
mrc_as_tiff = Path(f"{file_path.working_dir}/{file_path.base}_mrc_as_tiff.tiff")
if mrc_as_jpg.exists():
cur = mrc_as_jpg
elif tif_8_bit.exists():
cur = tif_8_bit
elif mrc_as_tiff.exists():
cur = mrc_as_tiff # can ignore here if scaled

as_tiff = Path(f"{file_path.working_dir}/{file_path.base}_as_tiff.tiff")
if as_tiff.exists():
cur = as_tiff
elif file_path.fp_in.suffix.strip(".").lower() in KEYIMG_EXTS:
cur = file_path.fp_in
else:
Expand Down Expand Up @@ -287,27 +224,18 @@ def dm_flow(
share_name=file_share, input_dir=input_dir_fp, fps_in=input_fps
)

# TODO move the three conversion block into if-else based on filetypes
tiff_results = convert_if_int16_tiff.map(file_path=fps)

# mrc is intermed format, to jpeg conversion
dm_to_mrc = convert_dms_to_mrc.map(file_path=fps)
mrc_results = convert_dm_mrc_to_jpeg.map(dm_to_mrc)

mrc_tiff_results = convert_2d_mrc_to_tiff.map(
file_path=fps, wait_for=[allow_failure(mrc_results)]
)
tiff_results = convert_em_to_tiff.map(file_path=fps)

# Finally generate all valid suffixed results
keyimg_assets = scale_jpegs.map(
fps,
size=unmapped("l"),
wait_for=[allow_failure(tiff_results), allow_failure(mrc_tiff_results)],
wait_for=[allow_failure(tiff_results), allow_failure(tiff_results)],
)
thumb_assets = scale_jpegs.map(
fps,
size=unmapped("s"),
wait_for=[allow_failure(tiff_results), allow_failure(mrc_tiff_results)],
wait_for=[allow_failure(tiff_results), allow_failure(tiff_results)],
)

prim_fps = utils.gen_prim_fps.map(fp_in=fps)
Expand Down
Loading