From daf5d46490b4196d7e263994f735f046ca08676a Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 26 Apr 2024 06:07:13 -0700 Subject: [PATCH 1/3] ruff fixes --- fireworks/core/firework.py | 30 +++++++++---------- fireworks/core/fworker.py | 4 +-- fireworks/core/launchpad.py | 12 ++++---- fireworks/core/rocket.py | 4 +-- fireworks/core/rocket_launcher.py | 4 +-- fireworks/features/fw_report.py | 2 +- fireworks/features/multi_launcher.py | 2 +- fireworks/flask_site/app.py | 4 +-- fireworks/queue/queue_launcher.py | 6 ++-- fireworks/scripts/lpad_run.py | 16 +++++----- .../user_objects/firetasks/script_task.py | 2 +- fireworks/utilities/filepad.py | 4 +-- fireworks/utilities/fw_utilities.py | 2 +- 13 files changed, 46 insertions(+), 46 deletions(-) diff --git a/fireworks/core/firework.py b/fireworks/core/firework.py index b043d0102..69405d084 100644 --- a/fireworks/core/firework.py +++ b/fireworks/core/firework.py @@ -153,13 +153,13 @@ def __init__( not only to direct children, but to all dependent FireWorks down to the Workflow's leaves. """ - mod_spec = mod_spec if mod_spec is not None else [] - additions = additions if additions is not None else [] - detours = detours if detours is not None else [] + mod_spec = mod_spec or [] + additions = additions or [] + detours = detours or [] - self.stored_data = stored_data if stored_data else {} + self.stored_data = stored_data or {} self.exit = exit - self.update_spec = update_spec if update_spec else {} + self.update_spec = update_spec or {} self.mod_spec = mod_spec if isinstance(mod_spec, (list, tuple)) else [mod_spec] self.additions = additions if isinstance(additions, (list, tuple)) else [additions] self.detours = detours if isinstance(detours, (list, tuple)) else [detours] @@ -267,13 +267,13 @@ def __init__( NEGATIVE_FWID_CTR -= 1 self.fw_id = NEGATIVE_FWID_CTR - self.launches = launches if launches else [] - self.archived_launches = archived_launches if archived_launches else [] + self.launches = launches or [] + self.archived_launches = archived_launches or [] self.created_on = created_on or datetime.utcnow() self.updated_on = updated_on or datetime.utcnow() parents = [parents] if isinstance(parents, Firework) else parents - self.parents = parents if parents else [] + self.parents = parents or [] self._state = state @@ -476,9 +476,9 @@ def __init__( self.fworker = fworker or FWorker() self.host = host or get_my_host() self.ip = ip or get_my_ip() - self.trackers = trackers if trackers else [] - self.action = action if action else None - self.state_history = state_history if state_history else [] + self.trackers = trackers or [] + self.action = action or None + self.state_history = state_history or [] self.state = state self.launch_id = launch_id self.fw_id = fw_id @@ -579,7 +579,7 @@ def reservedtime_secs(self): """ start = self.time_reserved if start: - end = self.time_start if self.time_start else datetime.utcnow() + end = self.time_start or datetime.utcnow() return (end - start).total_seconds() return None @@ -986,7 +986,7 @@ def rerun_fw(self, fw_id, updated_ids=None): Returns: list[int]: list of Firework ids that were updated. """ - updated_ids = updated_ids if updated_ids else set() + updated_ids = updated_ids or set() m_fw = self.id_fw[fw_id] m_fw._rerun() updated_ids.add(fw_id) @@ -1089,7 +1089,7 @@ def refresh(self, fw_id, updated_ids=None): set(int): list of Firework ids that were updated """ # these are the fw_ids to re-enter into the database - updated_ids = updated_ids if updated_ids else set() + updated_ids = updated_ids or set() fw = self.id_fw[fw_id] prev_state = fw.state @@ -1346,7 +1346,7 @@ def from_firework(cls, fw: Firework, name: str | None = None, metadata=None) -> Returns: Workflow """ - name = name if name else fw.name + name = name or fw.name return Workflow([fw], None, name=name, metadata=metadata, created_on=fw.created_on, updated_on=fw.updated_on) def __str__(self) -> str: diff --git a/fireworks/core/fworker.py b/fireworks/core/fworker.py index 08c0395ac..ce9ec69ac 100644 --- a/fireworks/core/fworker.py +++ b/fireworks/core/fworker.py @@ -35,8 +35,8 @@ def __init__(self, name="Automatically generated Worker", category="", query=Non """ self.name = name self.category = category - self._query = query if query else {} - self.env = env if env else {} + self._query = query or {} + self.env = env or {} @recursive_serialize def to_dict(self): diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py index c43a343dd..5350de9e3 100644 --- a/fireworks/core/launchpad.py +++ b/fireworks/core/launchpad.py @@ -189,11 +189,11 @@ def __init__( # set up logger self.logdir = logdir - self.strm_lvl = strm_lvl if strm_lvl else "INFO" + self.strm_lvl = strm_lvl or "INFO" self.m_logger = get_fw_logger("launchpad", l_dir=self.logdir, stream_level=self.strm_lvl) - self.user_indices = user_indices if user_indices else [] - self.wf_user_indices = wf_user_indices if wf_user_indices else [] + self.user_indices = user_indices or [] + self.wf_user_indices = wf_user_indices or [] # get connection if uri_mode: @@ -344,7 +344,7 @@ def maintain(self, infinite=True, maintain_interval=None) -> None: infinite (bool) maintain_interval (seconds): sleep time """ - maintain_interval = maintain_interval if maintain_interval else MAINTAIN_INTERVAL + maintain_interval = maintain_interval or MAINTAIN_INTERVAL while True: self.m_logger.info("Performing maintenance on Launchpad...") @@ -729,7 +729,7 @@ def get_fw_ids(self, query=None, sort=None, limit=0, count_only=False, launches_ list: list of firework ids matching the query """ coll = "launches" if launches_mode else "fireworks" - criteria = query if query else {} + criteria = query or {} if launches_mode: lids = self._get_active_launch_ids() criteria["launch_id"] = {"$in": lids} @@ -775,7 +775,7 @@ def get_wf_ids(self, query=None, sort=None, limit=0, count_only=False): Returns: list: list of firework ids """ - criteria = query if query else {} + criteria = query or {} aggregation = [] if criteria is not None: diff --git a/fireworks/core/rocket.py b/fireworks/core/rocket.py index a6edd4fcd..a98695be3 100644 --- a/fireworks/core/rocket.py +++ b/fireworks/core/rocket.py @@ -51,7 +51,7 @@ def do_ping(launchpad: LaunchPad, launch_id: int) -> None: launchpad.ping_launch(launch_id) else: with open("FW_ping.json", "w") as f: - f.write('{"ping_time": "%s"}' % datetime.utcnow().isoformat()) + f.write(f'{{"ping_time": "{datetime.utcnow().isoformat()}"}}') def ping_launch(launchpad: LaunchPad, launch_id: int, stop_event: Event, master_thread: Thread) -> None: @@ -468,7 +468,7 @@ def decorate_fwaction( for k, v in my_spec.get("_files_out").items(): files = glob.glob(os.path.join(launch_dir, v)) if files: - filepath = sorted(files)[-1] + filepath = max(files) fwaction.mod_spec.append({"_set": {f"_files_prev->{k:s}": filepath}}) return fwaction diff --git a/fireworks/core/rocket_launcher.py b/fireworks/core/rocket_launcher.py index 15ac7ebb1..9d9553b21 100644 --- a/fireworks/core/rocket_launcher.py +++ b/fireworks/core/rocket_launcher.py @@ -85,8 +85,8 @@ def rapidfire( local_redirect (bool): redirect standard input and output to local file pdb_on_exception (bool): if True, python will start the debugger on a firework exception """ - sleep_time = sleep_time if sleep_time else RAPIDFIRE_SLEEP_SECS - curdir = m_dir if m_dir else os.getcwd() + sleep_time = sleep_time or RAPIDFIRE_SLEEP_SECS + curdir = m_dir or os.getcwd() l_logger = get_fw_logger("rocket.launcher", l_dir=launchpad.get_logdir(), stream_level=strm_lvl) nlaunches = -1 if nlaunches == "infinite" else int(nlaunches) fworker = get_fworker(fworker) diff --git a/fireworks/features/fw_report.py b/fireworks/features/fw_report.py index d40ea1dde..1b82ee50d 100644 --- a/fireworks/features/fw_report.py +++ b/fireworks/features/fw_report.py @@ -69,7 +69,7 @@ def get_stats(self, coll="fireworks", interval="days", num_intervals=5, addition coll = self.db[coll] pipeline = [] - match_q = additional_query if additional_query else {} + match_q = additional_query or {} if num_intervals: now_time = datetime.utcnow() start_time = now_time - relativedelta(**{interval: num_intervals}) diff --git a/fireworks/features/multi_launcher.py b/fireworks/features/multi_launcher.py index 546b846a4..28e0d7e3c 100644 --- a/fireworks/features/multi_launcher.py +++ b/fireworks/features/multi_launcher.py @@ -67,7 +67,7 @@ def rapidfire_process( FWData().NODE_LIST = node_list FWData().SUB_NPROCS = sub_nproc FWData().Running_IDs = running_ids_dict - sleep_time = sleep if sleep else RAPIDFIRE_SLEEP_SECS + sleep_time = sleep or RAPIDFIRE_SLEEP_SECS l_dir = launchpad.get_logdir() if launchpad else None l_logger = get_fw_logger("rocket.launcher", l_dir=l_dir, stream_level=loglvl) # Record the start time for timeout update diff --git a/fireworks/flask_site/app.py b/fireworks/flask_site/app.py index baf8c9335..347e5b400 100644 --- a/fireworks/flask_site/app.py +++ b/fireworks/flask_site/app.py @@ -96,8 +96,8 @@ def pluralize(number, singular="", plural="s"): def home(): fw_querystr = request.args.get("fw_query") wf_querystr = request.args.get("wf_query") - fw_querystr = fw_querystr if fw_querystr else "" - wf_querystr = wf_querystr if wf_querystr else "" + fw_querystr = fw_querystr or "" + wf_querystr = wf_querystr or "" session["fw_filt"] = parse_querystr(fw_querystr, app.lp.fireworks) if fw_querystr else {} session["wf_filt"] = parse_querystr(wf_querystr, app.lp.workflows) if wf_querystr else {} diff --git a/fireworks/queue/queue_launcher.py b/fireworks/queue/queue_launcher.py index b99827c93..3a271a9b2 100644 --- a/fireworks/queue/queue_launcher.py +++ b/fireworks/queue/queue_launcher.py @@ -58,7 +58,7 @@ def launch_rocket_to_queue( (only in non-reservation mode) fw_id (int): specific fw_id to reserve (reservation mode only) """ - fworker = fworker if fworker else FWorker() + fworker = fworker or FWorker() launcher_dir = os.path.abspath(launcher_dir) l_logger = get_fw_logger("queue.launcher", l_dir=launchpad.logdir, stream_level=strm_lvl) @@ -201,7 +201,7 @@ def rapidfire( fill_mode (bool): whether to submit jobs even when there is nothing to run (only in non-reservation mode) """ - sleep_time = sleep_time if sleep_time else RAPIDFIRE_SLEEP_SECS + sleep_time = sleep_time or RAPIDFIRE_SLEEP_SECS launch_dir = os.path.abspath(launch_dir) nlaunches = -1 if nlaunches == "infinite" else int(nlaunches) l_logger = get_fw_logger("queue.launcher", l_dir=launchpad.logdir, stream_level=strm_lvl) @@ -334,5 +334,5 @@ def setup_offline_job(launchpad, fw, launch_id) -> None: # separate this function out for reuse in unit testing fw.to_file("FW.json") with open("FW_offline.json", "w") as f: - f.write('{"launch_id":%s}' % launch_id) + f.write(f'{{"launch_id":{launch_id}}}') launchpad.add_offline_run(launch_id, fw.fw_id, fw.name) diff --git a/fireworks/scripts/lpad_run.py b/fireworks/scripts/lpad_run.py index 5e21cc452..d4badd6f2 100644 --- a/fireworks/scripts/lpad_run.py +++ b/fireworks/scripts/lpad_run.py @@ -281,10 +281,10 @@ def get_fw_ids_helper(lp: LaunchPad, args: Namespace, count_only: bool | None = raise ValueError("Please specify exactly one of (fw_id, name, state, query)") if sum(bool(x) for x in [args.fw_id, args.name, args.state, args.query]) == 0: args.query = "{}" - args.display_format = args.display_format if args.display_format else "ids" + args.display_format = args.display_format or "ids" if sum(bool(x) for x in [args.fw_id, args.name, args.qid]) > 1: raise ValueError("Please specify exactly one of (fw_id, name, qid)") - args.display_format = args.display_format if args.display_format else "more" + args.display_format = args.display_format or "more" if args.fw_id: query = {"fw_id": {"$in": args.fw_id}} @@ -370,10 +370,10 @@ def get_fws_in_wfs(args: Namespace) -> None: raise ValueError("Please specify exactly one of (fw_id, name, state, query)") if sum(bool(x) for x in [args.fw_fw_id, args.fw_name, args.fw_state, args.fw_query]) == 0: args.fw_query = "{}" - args.display_format = args.display_format if args.display_format else "ids" + args.display_format = args.display_format or "ids" if sum(bool(x) for x in [args.fw_fw_id, args.fw_name, args.qid]) > 1: raise ValueError("Please specify exactly one of (fw_id, name, qid)") - args.display_format = args.display_format if args.display_format else "more" + args.display_format = args.display_format or "more" if args.fw_fw_id: fw_query = {"fw_id": {"$in": args.fw_fw_id}} @@ -425,9 +425,9 @@ def get_wfs(args: Namespace) -> None: raise ValueError("Please specify exactly one of (fw_id, name, state, query)") if sum(bool(x) for x in [args.fw_id, args.name, args.state, args.query]) == 0: args.query = "{}" - args.display_format = args.display_format if args.display_format else "ids" + args.display_format = args.display_format or "ids" else: - args.display_format = args.display_format if args.display_format else "more" + args.display_format = args.display_format or "more" if args.fw_id: query = {"nodes": {"$in": args.fw_id}} @@ -733,8 +733,8 @@ def webgui(args: Namespace) -> None: def add_scripts(args: Namespace) -> None: lp = get_lp(args) - args.names = args.names if args.names else [None] * len(args.scripts) - args.wf_name = args.wf_name if args.wf_name else args.names[0] + args.names = args.names or [None] * len(args.scripts) + args.wf_name = args.wf_name or args.names[0] fws = [] links = {} for idx, s in enumerate(args.scripts): diff --git a/fireworks/user_objects/firetasks/script_task.py b/fireworks/user_objects/firetasks/script_task.py index b86dc3294..e2ec4b9a1 100644 --- a/fireworks/user_objects/firetasks/script_task.py +++ b/fireworks/user_objects/firetasks/script_task.py @@ -122,7 +122,7 @@ def _load_params(self, d) -> None: @classmethod def from_str(cls, shell_cmd, parameters=None): - parameters = parameters if parameters else {} + parameters = parameters or {} parameters["script"] = [shell_cmd] parameters["use_shell"] = True return cls(parameters) diff --git a/fireworks/utilities/filepad.py b/fireworks/utilities/filepad.py index e69f765fb..734a829e1 100644 --- a/fireworks/utilities/filepad.py +++ b/fireworks/utilities/filepad.py @@ -99,7 +99,7 @@ def __init__( # logging self.logdir = logdir - self.strm_lvl = strm_lvl if strm_lvl else "INFO" + self.strm_lvl = strm_lvl or "INFO" self.logger = get_fw_logger("filepad", l_dir=self.logdir, stream_level=self.strm_lvl) # build indexes @@ -113,7 +113,7 @@ def build_indexes(self, indexes=None, background=True) -> None: indexes (list): list of single field indexes to be built. background (bool): Run in the background or not. """ - indexes = indexes if indexes else ["identifier", "gfs_id"] + indexes = indexes or ["identifier", "gfs_id"] for i in indexes: self.filepad.create_index(i, unique=True, background=background) diff --git a/fireworks/utilities/fw_utilities.py b/fireworks/utilities/fw_utilities.py index 3b63ce2ae..03b79aeaa 100644 --- a/fireworks/utilities/fw_utilities.py +++ b/fireworks/utilities/fw_utilities.py @@ -48,7 +48,7 @@ def get_fw_logger( logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) # anything debug and above passes through to the handler level - stream_level = stream_level if stream_level else "CRITICAL" + stream_level = stream_level or "CRITICAL" # add handlers for the file_levels if l_dir: for lvl in file_levels: From 2b357d531e519f87193fe00f26b7c64ee13f615a Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 26 Apr 2024 06:37:33 -0700 Subject: [PATCH 2/3] refactor dict.update() --- fireworks/core/firework.py | 2 +- fireworks/core/launchpad.py | 6 +++--- fireworks/features/stats.py | 10 +++++----- fireworks/queue/queue_launcher.py | 2 +- .../user_objects/queue_adapters/common_adapter.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/fireworks/core/firework.py b/fireworks/core/firework.py index 69405d084..7368fa39c 100644 --- a/fireworks/core/firework.py +++ b/fireworks/core/firework.py @@ -643,7 +643,7 @@ def _update_state_history(self, state) -> None: now_time = datetime.utcnow() new_history_entry = {"state": state, "created_on": now_time} if state != "COMPLETED" and last_checkpoint: - new_history_entry.update({"checkpoint": last_checkpoint}) + new_history_entry.update(checkpoint=last_checkpoint) self.state_history.append(new_history_entry) if state in ["RUNNING", "RESERVED"]: self.touch_history() # add updated_on key diff --git a/fireworks/core/launchpad.py b/fireworks/core/launchpad.py index 5350de9e3..02a9fb43b 100644 --- a/fireworks/core/launchpad.py +++ b/fireworks/core/launchpad.py @@ -898,7 +898,7 @@ def future_run_exists(self, fworker=None) -> bool: return True # retrieve all [RUNNING/RESERVED] fireworks q = fworker.query if fworker else {} - q.update({"state": {"$in": ["RUNNING", "RESERVED"]}}) + q.update(state={"$in": ["RUNNING", "RESERVED"]}) active = self.get_fw_ids(q) # then check if they have WAITING children for fw_id in active: @@ -1670,7 +1670,7 @@ def rerun_fw(self, fw_id, rerun_duplicates=True, recover_launch=None, recover_mo # Launch recovery if recover_launch is not None: recovery = self.get_recovery(fw_id, recover_launch) - recovery.update({"_mode": recover_mode}) + recovery.update(_mode=recover_mode) set_spec = recursive_dict({"$set": {"spec._recovery": recovery}}) if recover_mode == "prev_dir": prev_dir = self.get_launch_by_id(recovery.get("_launch_id")).launch_dir @@ -1714,7 +1714,7 @@ def get_recovery(self, fw_id, launch_id="last"): m_fw = self.get_fw_by_id(fw_id) launch = m_fw.launches[-1] if launch_id == "last" else self.get_launch_by_id(launch_id) recovery = launch.state_history[-1].get("checkpoint") - recovery.update({"_prev_dir": launch.launch_dir, "_launch_id": launch.launch_id}) + recovery.update(_prev_dir=launch.launch_dir, _launch_id=launch.launch_id) return recovery def _refresh_wf(self, fw_id) -> None: diff --git a/fireworks/features/stats.py b/fireworks/features/stats.py index c381e64d6..402e5ef6b 100644 --- a/fireworks/features/stats.py +++ b/fireworks/features/stats.py @@ -197,8 +197,8 @@ def group_fizzled_fireworks( "created_on": self._query_datetime_range(start_time=query_start, end_time=query_end, **args), } if include_ids: - project_query.update({"fw_id": 1}) - group_query.update({"fw_id": {"$push": "$fw_id"}}) + project_query.update(fw_id=1) + group_query.update(fw_id={"$push": "$fw_id"}) if query: match_query.update(query) return self._aggregate( @@ -306,11 +306,11 @@ def _get_summary( } match_query.update(query) if runtime_stats: - project_query.update({"runtime_secs": 1}) + project_query.update(runtime_secs=1) group_query.update(RUNTIME_STATS) if include_ids: project_query.update({id_field: 1}) - group_query.update({"ids": {"$push": "$" + id_field}}) + group_query.update(ids={"$push": "$" + id_field}) return self._aggregate( coll=coll, match=match_query, @@ -357,7 +357,7 @@ def _aggregate( for arg in [match, project, unwind, group_op]: if arg is None: arg = {} - group_op.update({"_id": "$" + group_by}) + group_op.update(_id=f"${group_by}") if sort is None: sort_query = ("_id", 1) query = [{"$match": match}, {"$project": project}, {"$group": group_op}, {"$sort": SON([sort_query])}] diff --git a/fireworks/queue/queue_launcher.py b/fireworks/queue/queue_launcher.py index 3a271a9b2..e7d461abc 100644 --- a/fireworks/queue/queue_launcher.py +++ b/fireworks/queue/queue_launcher.py @@ -96,7 +96,7 @@ def launch_rocket_to_queue( # update qadapter job_name based on FW name job_name = get_slug(fw.name)[0:QUEUE_JOBNAME_MAXLEN] - qadapter.update({"job_name": job_name}) + qadapter.update(job_name=job_name) if "_queueadapter" in fw.spec: l_logger.debug("updating queue params using Firework spec..") diff --git a/fireworks/user_objects/queue_adapters/common_adapter.py b/fireworks/user_objects/queue_adapters/common_adapter.py index 6e1380b25..259d6ada7 100644 --- a/fireworks/user_objects/queue_adapters/common_adapter.py +++ b/fireworks/user_objects/queue_adapters/common_adapter.py @@ -66,7 +66,7 @@ def __init__(self, q_type, q_name=None, template_file=None, timeout=None, **kwar ) self.q_name = q_name or q_type self.timeout = timeout or 5 - self.update(dict(kwargs)) + self.update(kwargs) self.q_commands = copy.deepcopy(CommonAdapter.default_q_commands) if "_q_commands_override" in self: From 35a47ca940dda860267f9229e9a8c2a5e3a491b2 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 26 Apr 2024 07:05:54 -0700 Subject: [PATCH 3/3] fix FWAction init --- fireworks/core/firework.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fireworks/core/firework.py b/fireworks/core/firework.py index 7368fa39c..842a97b16 100644 --- a/fireworks/core/firework.py +++ b/fireworks/core/firework.py @@ -153,9 +153,9 @@ def __init__( not only to direct children, but to all dependent FireWorks down to the Workflow's leaves. """ - mod_spec = mod_spec or [] - additions = additions or [] - detours = detours or [] + mod_spec = mod_spec if mod_spec is not None else [] + additions = additions if additions is not None else [] + detours = detours if detours is not None else [] self.stored_data = stored_data or {} self.exit = exit