From d73c1da9a180ac21682ab9891ac6bf0b36ca0439 Mon Sep 17 00:00:00 2001 From: Whisperity Date: Tue, 20 Aug 2024 15:03:52 +0200 Subject: [PATCH] feat(cmd): Implemented a CLI for task management This patch extends `CodeChecker cmd` with a new sub-command, `serverside-tasks`, which lets users and administrators deal with querying the status of running server-side tasks. By default, the CLI queries the information of the task(s) specified by their token(s) in the `--token` argument from the server using `getTaskInfo(token)`, and shows this information in either verbose "plain text" (available if precisely **one** task was specified), "table" or JSON formats. In addition to `--token`, it also supports 19 more parameters, each of which correspond to a filter option in the `TaskFilter` API type. If any filters in addition to `--token` is specified, it will exercise `getTasks(filter)` instead. This mode is only available to administrators. The resulting, more detailed information structs are printed in "table" or JSON formats. Apart from querying the current status, two additional flags are available, irrespective of which query method is used to obtain a list of "matching tasks": * `--kill` will call `cancelTask(token)` for each task. * `--await` will block execution until the specified task(s) terminate (in one way or another). `--await` is implemented by calling the new **`await_task_termination`** library function, which is implemented with the goal of being reusable by other clients later. --- codechecker_common/typehints.py | 35 + codechecker_common/util.py | 4 +- docs/web/user_guide.md | 290 +++++++++ web/client/codechecker_client/client.py | 36 +- web/client/codechecker_client/cmd/cmd.py | 271 +++++++- .../codechecker_client/helpers/tasks.py | 49 ++ web/client/codechecker_client/task_client.py | 596 ++++++++++++++++++ .../task_executors/abstract_task.py | 7 + .../codechecker_server/task_executors/main.py | 8 +- 9 files changed, 1282 insertions(+), 14 deletions(-) create mode 100644 codechecker_common/typehints.py create mode 100644 web/client/codechecker_client/helpers/tasks.py create mode 100644 web/client/codechecker_client/task_client.py diff --git a/codechecker_common/typehints.py b/codechecker_common/typehints.py new file mode 100644 index 0000000000..642d5ce0ec --- /dev/null +++ b/codechecker_common/typehints.py @@ -0,0 +1,35 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- +""" +Type hint (`typing`) extensions. +""" +from typing import Any, Protocol, TypeVar + + +_T_contra = TypeVar("_T_contra", contravariant=True) + + +class LTComparable(Protocol[_T_contra]): + def __lt__(self, other: _T_contra, /) -> bool: ... + + +class LEComparable(Protocol[_T_contra]): + def __le__(self, other: _T_contra, /) -> bool: ... + + +class GTComparable(Protocol[_T_contra]): + def __gt__(self, other: _T_contra, /) -> bool: ... + + +class GEComparable(Protocol[_T_contra]): + def __ge__(self, other: _T_contra, /) -> bool: ... + + +class Orderable(LTComparable[Any], LEComparable[Any], + GTComparable[Any], GEComparable[Any], Protocol): + """Type hint for something that supports rich comparison operators.""" diff --git a/codechecker_common/util.py b/codechecker_common/util.py index c9075d5599..a9e966393d 100644 --- a/codechecker_common/util.py +++ b/codechecker_common/util.py @@ -20,6 +20,8 @@ from codechecker_common.logger import get_logger +from .typehints import Orderable + LOG = get_logger('system') @@ -35,7 +37,7 @@ def arg_match(options, args): return matched_args -def clamp(min_: int, value: int, max_: int) -> int: +def clamp(min_: Orderable, value: Orderable, max_: Orderable) -> Orderable: """Clamps ``value`` such that ``min_ <= value <= max_``.""" if min_ > max_: raise ValueError("min <= max required") diff --git a/docs/web/user_guide.md b/docs/web/user_guide.md index bca75ee459..bda19706a2 100644 --- a/docs/web/user_guide.md +++ b/docs/web/user_guide.md @@ -39,6 +39,7 @@ - [Manage product configuration of a server (`products`)](#manage-product-configuration-of-a-server-products) - [Query authorization settings (`permissions`)](#query-authorization-settings-permissions) - [Authenticate to the server (`login`)](#authenticate-to-the-server-login) + - [Server-side task management (`serverside-tasks`)](#server-side-task-management-serverside-tasks) - [Exporting source code suppression to suppress file](#exporting-source-code-suppression-to-suppress-file) - [Export comments and review statuses (`export`)](#export-comments-and-review-statuses-export) - [Import comments and review statuses into Codechecker (`import`)](#import-comments-and-review-statuses-into-codechecker-import) @@ -1394,6 +1395,295 @@ can be used normally. The password can be saved on the disk. If such "preconfigured" password is not found, the user will be asked, in the command-line, to provide credentials. +#### Server-side task management (`serverside-tasks`) +
+ + $ CodeChecker cmd serverside-tasks --help (click to expand) + + +``` +usage: CodeChecker cmd serverside-tasks [-h] [-t [TOKEN [TOKEN ...]]] + [--await] [--kill] + [--output {plaintext,table,json}] + [--machine-id [MACHINE_ID [MACHINE_ID ...]]] + [--type [TYPE [TYPE ...]]] + [--status [{allocated,enqueued,running,completed,failed,cancelled,dropped} [{allocated,enqueued,running,completed,failed,cancelled,dropped} ...]]] + [--username [USERNAME [USERNAME ...]] + | --no-username] + [--product [PRODUCT [PRODUCT ...]] | + --no-product] + [--enqueued-before TIMESTAMP] + [--enqueued-after TIMESTAMP] + [--started-before TIMESTAMP] + [--started-after TIMESTAMP] + [--finished-before TIMESTAMP] + [--finished-after TIMESTAMP] + [--last-seen-before TIMESTAMP] + [--last-seen-after TIMESTAMP] + [--only-cancelled | --no-cancelled] + [--only-consumed | --no-consumed] + [--url SERVER_URL] + [--verbose {info,debug_analyzer,debug}] + +Query the status of and otherwise filter information for server-side +background tasks executing on a CodeChecker server. In addition, for server +administartors, allows requesting tasks to cancel execution. + +Normally, the querying of a task's status is available only to the following +users: + - The user who caused the creation of the task. + - For tasks that are associated with a specific product, the PRODUCT_ADMIN + users of that product. + - Accounts with SUPERUSER rights (server administrators). + +optional arguments: + -h, --help show this help message and exit + -t [TOKEN [TOKEN ...]], --token [TOKEN [TOKEN ...]] + The identifying token(s) of the task(s) to query. Each + task is associated with a unique token. (default: + None) + --await Instead of querying the status and reporting that, + followed by an exit, block execution of the + 'CodeChecker cmd serverside-tasks' program until the + queried task(s) terminate(s). Makes the CLI's return + code '0' if the task(s) completed successfully, and + non-zero otherwise. If '--kill' is also specified, the + CLI will await the shutdown of the task(s), but will + return '0' if the task(s) were successfully killed as + well. (default: False) + --kill Request the co-operative and graceful termination of + the tasks matching the filter(s) specified. '--kill' + is only available to SUPERUSERs! Note, that this + action only submits a *REQUEST* of termination to the + server, and tasks are free to not support in-progress + kills. Even for tasks that support getting killed, due + to its graceful nature, it might take a considerable + time for the killing to conclude. Killing a task that + has not started RUNNING yet results in it + automatically terminating before it would start. + (default: False) + +output arguments: + --output {plaintext,table,json} + The format of the output to use when showing the + result of the request. (default: plaintext) + +task list filter arguments: + These options can be used to obtain and filter the list of tasks + associated with the 'CodeChecker server' specified by '--url', based on the + various information columns stored for tasks. + + '--token' is usable with the following filters as well. + + Filters with a variable number of options (e.g., '--machine-id A B') will be + in a Boolean OR relation with each other (meaning: machine ID is either "A" + or "B"). + Specifying multiple filters (e.g., '--machine-id A B --username John') will + be considered in a Boolean AND relation (meaning: [machine ID is either "A" or + "B"] and [the task was created by "John"]). + + Listing is only available for the following, privileged users: + - For tasks that are associated with a specific product, the PRODUCT_ADMINs + of that product. + - Server administrators (SUPERUSERs). + + Unprivileged users MUST use only the task's token to query information about + the task. + + + --machine-id [MACHINE_ID [MACHINE_ID ...]] + The IDs of the server instance executing the tasks. + This is an internal identifier set by server + administrators via the 'CodeChecker server' command. + (default: None) + --type [TYPE [TYPE ...]] + The descriptive, but still machine-readable "type" of + the tasks to filter for. (default: None) + --status [{allocated,enqueued,running,completed,failed,cancelled,dropped} [{allocated,enqueued,running,completed,failed,cancelled,dropped} ...]] + The task's execution status(es) in the pipeline. + (default: None) + --username [USERNAME [USERNAME ...]] + The user(s) who executed the action that caused the + tasks' creation. (default: None) + --no-username Filter for tasks without a responsible user that + created them. (default: False) + --product [PRODUCT [PRODUCT ...]] + Filter for tasks that execute in the context of + products specified by the given ENDPOINTs. This query + is only available if you are a PRODUCT_ADMIN of the + specified product(s). (default: None) + --no-product Filter for server-wide tasks (not associated with any + products). This query is only available to SUPERUSERs. + (default: False) + --enqueued-before TIMESTAMP + Filter for tasks that were created BEFORE (or on) the + specified TIMESTAMP, which is given in the format of + 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --enqueued-after TIMESTAMP + Filter for tasks that were created AFTER (or on) the + specified TIMESTAMP, which is given in the format of + 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --started-before TIMESTAMP + Filter for tasks that were started execution BEFORE + (or on) the specified TIMESTAMP, which is given in the + format of 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --started-after TIMESTAMP + Filter for tasks that were started execution AFTER (or + on) the specified TIMESTAMP, which is given in the + format of 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --finished-before TIMESTAMP + Filter for tasks that concluded execution BEFORE (or + on) the specified TIMESTAMP, which is given in the + format of 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --finished-after TIMESTAMP + Filter for tasks that concluded execution execution + AFTER (or on) the specified TIMESTAMP, which is given + in the format of 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --last-seen-before TIMESTAMP + Filter for tasks that reported actual forward progress + in its execution ("heartbeat") BEFORE (or on) the + specified TIMESTAMP, which is given in the format of + 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --last-seen-after TIMESTAMP + Filter for tasks that reported actual forward progress + in its execution ("heartbeat") AFTER (or on) the + specified TIMESTAMP, which is given in the format of + 'year:month:day' or + 'year:month:day:hour:minute:second'. If the "time" + part (':hour:minute:second') is not given, 00:00:00 + (midnight) is assumed instead. Timestamps for tasks + are always understood as Coordinated Universal Time + (UTC). (default: None) + --only-cancelled Show only tasks that received a cancel request from a + SUPERUSER (see '--kill'). (default: False) + --no-cancelled Show only tasks that had not received a cancel request + from a SUPERUSER (see '--kill'). (default: False) + --only-consumed Show only tasks that concluded their execution and the + responsible user (see '--username') "downloaded" this + fact. (default: False) + --no-consumed Show only tasks that concluded their execution but the + responsible user (see '--username') did not "check" on + the task. (default: False) + +common arguments: + --url SERVER_URL The URL of the server to access, in the format of + '[http[s]://]host:port'. (default: localhost:8001) + --verbose {info,debug_analyzer,debug} + Set verbosity level. + +The return code of 'CodeChecker cmd serverside-tasks' is almost always '0', +unless there is an error. +If **EXACTLY** one '--token' is specified in the arguments without the use of +'--await' or '--kill', the return code is based on the current status of the +task, as identified by the token: + - 0: The task completed successfully. + - 1: (Reserved for operational errors.) + - 2: (Reserved for command-line errors.) + - 4: The task failed to complete due to an error during execution. + - 8: The task is still running... + - 16: The task was cancelled by the administrators, or the server was shut + down. +``` +
+ +The `serverside-tasks` subcommand allows users and administrators to query the status of (and for administrators, request the cancellation) of **server-side background tasks**. +These background tasks are created by a limited set of user actions, where the user's client not waiting for the completion of the task can be beneficial. +A task is always identified by its **token**, which is a random generated value. +This token is presented to the user when appropriate. + +##### Querying the status of a single job + +The primary purpose of `CodeChecker cmd serverside-tasks` is to query the status of a running task, with the `--token TOKEN` flag, e.g., `CodeChecker cmd serverside-tasks --token ABCDEF`. +This will return the task's details: + +``` +Task 'ABCDEF': + - Type: TaskService::DummyTask + - Summary: Dummy task for testing purposes + - Status: CANCELLED + - Enqueued at: 2024-08-19 15:55:34 + - Started at: 2024-08-19 15:55:34 + - Last seen: 2024-08-19 15:55:35 + - Completed at: 2024-08-19 15:55:35 + +Comments on task '8b62497c7d1b7e3945445f5b9c3951d97ae07e58f97cad60a0187221e7d1e2ba': +... +``` + +If `--await` is also specified, the execution of `CodeChecker cmd serverside-task` blocks the caller prompt or script until the task terminates on the server. +This is useful in situations where the side effect of a task is needed to be ready before the script may process further instructions. + +A task can have the following statuses: + + * **Allocated**: The task's token was minted, but the complete input to the task has not yet fully processed. + * **Enqueued**: The task is ready for execution, and the system is waiting for free resources to begin running the implementation. + * **Running**: The task is actively executing. + * **Completed**: The task successfully finished executing. (The side effects of the operations are available at this point.) + * **Failed**: The task's execution was started, but failed for some reason. This could be an error detected in the input, a database issue, or any other _Exception_. The "Comments" field of the task, when queried, will likely contain the details of the error. + * **Cancelled**: The task was cancelled by an administrator ([see later](#requesting-the-termination-of-a-task-only-for-SUPERUSERs)) and the task shut down to this request. + * **Dropped**: The task's execution was interrupted due to an external reason (system crash, service shutdown). + +##### Querying multiple tasks via filters + +For product and server administrators (`PRODUCT_ADMIN` and `SUPERUSER` rights), the `serverside-tasks` subcommand exposes various filter options, which can be used to create even a combination of criteria tasks must match to be returned. +Please refer to the `--help` of the subcommand for the exact list of filters available. +In this mode, the statuses of the tasks are printed in a concise table. + +```sh +$ CodeChecker cmd serverside-tasks --enqueued-after 2024:08:19 --status cancelled + +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +Token | Machine | Type | Summary | Status | Product | User | Enqueued | Started | Last seen | Completed | Cancelled? +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +8b62497c7d1b7e3945445f5b9c3951d97ae07e58f97cad60a0187221e7d1e2ba | xxxxxxxxxxxxx:8001 | TaskService::DummyTask | Dummy task for testing purposes | CANCELLED | | | 2024-08-19 15:55:34 | 2024-08-19 15:55:34 | 2024-08-19 15:55:35 | 2024-08-19 15:55:35 | Yes +6fa0097a9bd1799572c7ccd2afc0272684ed036c11145da7eaf40cc8a07c7241 | xxxxxxxxxxxxx:8001 | TaskService::DummyTask | Dummy task for testing purposes | CANCELLED | | | 2024-08-19 15:55:53 | 2024-08-19 15:55:53 | 2024-08-19 15:55:53 | 2024-08-19 15:55:53 | Yes +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +``` + +##### Requesting the termination of a task (only for `SUPERUSER`s) + +Tasks matching the query filters can be requested for termination ("killed") by specifying `--kill` in addition to the filters. +This will send a request to the server to shut the tasks down. + +**Note**, that this shutdown is not deterministic and is not immediate. +Due to technical reasons, it is up for the task's implementation to find the appropriate position to honour the shutdown request. +Depending on the task's semantics, the input, or simply circumstance, a task may completely ignore the shutdown request and decide to nevertheless complete. + ### Exporting source code suppression to suppress file diff --git a/web/client/codechecker_client/client.py b/web/client/codechecker_client/client.py index 0bbe2f69fe..570d7e28dc 100644 --- a/web/client/codechecker_client/client.py +++ b/web/client/codechecker_client/client.py @@ -23,10 +23,11 @@ from codechecker_web.shared import env from codechecker_web.shared.version import CLIENT_API -from codechecker_client.helpers.authentication import ThriftAuthHelper -from codechecker_client.helpers.product import ThriftProductHelper -from codechecker_client.helpers.results import ThriftResultsHelper from .credential_manager import UserCredentials +from .helpers.authentication import ThriftAuthHelper +from .helpers.product import ThriftProductHelper +from .helpers.results import ThriftResultsHelper +from .helpers.tasks import ThriftServersideTaskHelper from .product import split_product_url LOG = get_logger('system') @@ -65,7 +66,7 @@ def setup_auth_client(protocol, host, port, session_token=None): session token for the session. """ client = ThriftAuthHelper(protocol, host, port, - '/v' + CLIENT_API + '/Authentication', + f"/v{CLIENT_API}/Authentication", session_token) return client @@ -78,7 +79,7 @@ def login_user(protocol, host, port, username, login=False): """ session = UserCredentials() auth_client = ThriftAuthHelper(protocol, host, port, - '/v' + CLIENT_API + '/Authentication') + f"/v{CLIENT_API}/Authentication") if not login: logout_done = auth_client.destroySession() @@ -213,7 +214,7 @@ def setup_product_client(protocol, host, port, auth_client=None, # as "viewpoint" from which the product service is called. product_client = ThriftProductHelper( protocol, host, port, - '/' + product_name + '/v' + CLIENT_API + '/Products', + f"/{product_name}/v{CLIENT_API}/Products", session_token, lambda: get_new_token(protocol, host, port, cred_manager)) @@ -263,3 +264,26 @@ def setup_client(product_url) -> ThriftResultsHelper: f"/{product_name}/v{CLIENT_API}/CodeCheckerService", session_token, lambda: get_new_token(protocol, host, port, cred_manager)) + + +def setup_task_client(protocol, host, port, auth_client=None, + session_token=None): + """ + Setup the Thrift client for the server-side task management endpoint. + """ + cred_manager = UserCredentials() + session_token = cred_manager.get_token(host, port) + + if not session_token: + auth_client = setup_auth_client(protocol, host, port) + session_token = perform_auth_for_handler(auth_client, host, port, + cred_manager) + + # Attach to the server-wide task management service. + task_client = ThriftServersideTaskHelper( + protocol, host, port, + f"/v{CLIENT_API}/Tasks", + session_token, + lambda: get_new_token(protocol, host, port, cred_manager)) + + return task_client diff --git a/web/client/codechecker_client/cmd/cmd.py b/web/client/codechecker_client/cmd/cmd.py index 6be5de36b6..f68b8a148e 100644 --- a/web/client/codechecker_client/cmd/cmd.py +++ b/web/client/codechecker_client/cmd/cmd.py @@ -14,13 +14,17 @@ import argparse import getpass import datetime +import os import sys from codechecker_api.codeCheckerDBAccess_v6 import ttypes -from codechecker_client import cmd_line_client -from codechecker_client import product_client -from codechecker_client import permission_client, source_component_client, \ +from codechecker_client import \ + cmd_line_client, \ + permission_client, \ + product_client, \ + source_component_client, \ + task_client, \ token_client from codechecker_common import arg, logger, util @@ -1227,6 +1231,231 @@ def __register_permissions(parser): help="The output format to use in showing the data.") +def __register_tasks(parser): + """ + Add `argparse` subcommand `parser` options for the "handle server-side + tasks" action. + """ + if "TEST_WORKSPACE" in os.environ: + testing_args = parser.add_argument_group("testing arguments") + testing_args.add_argument("--create-dummy-task", + dest="dummy_task_args", + metavar="ARG", + default=argparse.SUPPRESS, + type=str, + nargs=2, + help=""" +Exercises the 'createDummyTask(int timeout, bool shouldFail)' API endpoint. +Used for testing purposes. +Note, that the server **MUST** be started in a testing environment as well, +otherwise, the request will be rejected by the server! +""") + + parser.add_argument("-t", "--token", + dest="token", + metavar="TOKEN", + type=str, + nargs='*', + help="The identifying token(s) of the task(s) to " + "query. Each task is associated with a unique " + "token.") + + parser.add_argument("--await", + dest="wait_and_block", + action="store_true", + help=""" +Instead of querying the status and reporting that, followed by an exit, block +execution of the 'CodeChecker cmd serverside-tasks' program until the queried +task(s) terminate(s). +Makes the CLI's return code '0' if the task(s) completed successfully, and +non-zero otherwise. +If '--kill' is also specified, the CLI will await the shutdown of the task(s), +but will return '0' if the task(s) were successfully killed as well. +""") + + parser.add_argument("--kill", + dest="cancel_task", + action="store_true", + help=""" +Request the co-operative and graceful termination of the tasks matching the +filter(s) specified. +'--kill' is only available to SUPERUSERs! +Note, that this action only submits a *REQUEST* of termination to the server, +and tasks are free to not support in-progress kills. +Even for tasks that support getting killed, due to its graceful nature, it +might take a considerable time for the killing to conclude. +Killing a task that has not started RUNNING yet results in it automatically +terminating before it would start. +""") + + output = parser.add_argument_group("output arguments") + output.add_argument("--output", + dest="output_format", + required=False, + default="plaintext", + choices=["plaintext", "table", "json"], + help="The format of the output to use when showing " + "the result of the request.") + + task_list = parser.add_argument_group( + "task list filter arguments", + """These options can be used to obtain and filter the list of tasks +associated with the 'CodeChecker server' specified by '--url', based on the +various information columns stored for tasks. + +'--token' is usable with the following filters as well. + +Filters with a variable number of options (e.g., '--machine-id A B') will be +in a Boolean OR relation with each other (meaning: machine ID is either "A" +or "B"). +Specifying multiple filters (e.g., '--machine-id A B --username John') will +be considered in a Boolean AND relation (meaning: [machine ID is either "A" or +"B"] and [the task was created by "John"]). + +Listing is only available for the following, privileged users: + - For tasks that are associated with a specific product, the PRODUCT_ADMINs + of that product. + - Server administrators (SUPERUSERs). + +Unprivileged users MUST use only the task's token to query information about +the task. + """) + + task_list.add_argument("--machine-id", + type=str, + nargs='*', + help="The IDs of the server instance executing " + "the tasks. This is an internal identifier " + "set by server administrators via the " + "'CodeChecker server' command.") + + task_list.add_argument("--type", + type=str, + nargs='*', + help="The descriptive, but still " + "machine-readable \"type\" of the tasks to " + "filter for.") + + task_list.add_argument("--status", + type=str, + nargs='*', + choices=["allocated", "enqueued", "running", + "completed", "failed", "cancelled", + "dropped"], + help="The task's execution status(es) in the " + "pipeline.") + + username = task_list.add_mutually_exclusive_group(required=False) + username.add_argument("--username", + type=str, + nargs='*', + help="The user(s) who executed the action that " + "caused the tasks' creation.") + username.add_argument("--no-username", + action="store_true", + help="Filter for tasks without a responsible user " + "that created them.") + + product = task_list.add_mutually_exclusive_group(required=False) + product.add_argument("--product", + type=str, + nargs='*', + help="Filter for tasks that execute in the context " + "of products specified by the given ENDPOINTs. " + "This query is only available if you are a " + "PRODUCT_ADMIN of the specified product(s).") + product.add_argument("--no-product", + action="store_true", + help="Filter for server-wide tasks (not associated " + "with any products). This query is only " + "available to SUPERUSERs.") + + timestamp_documentation: str = """ +TIMESTAMP, which is given in the format of 'year:month:day' or +'year:month:day:hour:minute:second'. +If the "time" part (':hour:minute:second') is not given, 00:00:00 (midnight) +is assumed instead. +Timestamps for tasks are always understood as Coordinated Universal Time (UTC). +""" + + task_list.add_argument("--enqueued-before", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that were created BEFORE " + "(or on) the specified " + + timestamp_documentation) + task_list.add_argument("--enqueued-after", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that were created AFTER " + "(or on) the specified " + + timestamp_documentation) + + task_list.add_argument("--started-before", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that were started " + "execution BEFORE (or on) the specified " + + timestamp_documentation) + task_list.add_argument("--started-after", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that were started " + "execution AFTER (or on) the specified " + + timestamp_documentation) + + task_list.add_argument("--finished-before", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that concluded execution " + "BEFORE (or on) the specified " + + timestamp_documentation) + task_list.add_argument("--finished-after", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that concluded execution " + "execution AFTER (or on) the specified " + + timestamp_documentation) + + task_list.add_argument("--last-seen-before", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that reported actual " + "forward progress in its execution " + "(\"heartbeat\") BEFORE (or on) the " + "specified " + timestamp_documentation) + task_list.add_argument("--last-seen-after", + type=valid_time, + metavar="TIMESTAMP", + help="Filter for tasks that reported actual " + "forward progress in its execution " + "(\"heartbeat\") AFTER (or on) the " + "specified " + timestamp_documentation) + + cancel = task_list.add_mutually_exclusive_group(required=False) + cancel.add_argument("--only-cancelled", + action="store_true", + help="Show only tasks that received a cancel request " + "from a SUPERUSER (see '--kill').") + cancel.add_argument("--no-cancelled", + action="store_true", + help="Show only tasks that had not received a " + "cancel request from a SUPERUSER " + "(see '--kill').") + + consumed = task_list.add_mutually_exclusive_group(required=False) + consumed.add_argument("--only-consumed", + action="store_true", + help="Show only tasks that concluded their " + "execution and the responsible user (see " + "'--username') \"downloaded\" this fact.") + consumed.add_argument("--no-consumed", + action="store_true", + help="Show only tasks that concluded their " + "execution but the responsible user (see " + "'--username') did not \"check\" on the task.") + + def __register_token(parser): """ Add argparse subcommand parser for the "handle token" action. @@ -1538,5 +1767,41 @@ def add_arguments_to_parser(parser): permissions.set_defaults(func=permission_client.handle_permissions) __add_common_arguments(permissions, needs_product_url=False) + tasks = subcommands.add_parser( + "serverside-tasks", + formatter_class=arg.RawDescriptionDefaultHelpFormatter, + description=""" +Query the status of and otherwise filter information for server-side +background tasks executing on a CodeChecker server. In addition, for server +administartors, allows requesting tasks to cancel execution. + +Normally, the querying of a task's status is available only to the following +users: + - The user who caused the creation of the task. + - For tasks that are associated with a specific product, the PRODUCT_ADMIN + users of that product. + - Accounts with SUPERUSER rights (server administrators). +""", + help="Await, query, and cancel background tasks executing on the " + "server.", + epilog=""" +The return code of 'CodeChecker cmd serverside-tasks' is almost always '0', +unless there is an error. +If **EXACTLY** one '--token' is specified in the arguments without the use of +'--await' or '--kill', the return code is based on the current status of the +task, as identified by the token: + - 0: The task completed successfully. + - 1: (Reserved for operational errors.) + - 2: (Reserved for command-line errors.) + - 4: The task failed to complete due to an error during execution. + - 8: The task is still running... + - 16: The task was cancelled by the administrators, or the server was shut + down. +""" + ) + __register_tasks(tasks) + tasks.set_defaults(func=task_client.handle_tasks) + __add_common_arguments(tasks, needs_product_url=False) + # 'cmd' does not have a main() method in itself, as individual subcommands are # handled later on separately. diff --git a/web/client/codechecker_client/helpers/tasks.py b/web/client/codechecker_client/helpers/tasks.py new file mode 100644 index 0000000000..026b2665fa --- /dev/null +++ b/web/client/codechecker_client/helpers/tasks.py @@ -0,0 +1,49 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- +""" +Helper for the "serverside tasks" Thrift API. +""" +from typing import Callable, List, Optional + +from codechecker_api.codeCheckerServersideTasks_v6 import \ + codeCheckerServersideTaskService +from codechecker_api.codeCheckerServersideTasks_v6.ttypes import \ + AdministratorTaskInfo, TaskFilter, TaskInfo + +from ..thrift_call import thrift_client_call +from .base import BaseClientHelper + + +# These names are inherited from Thrift stubs. +# pylint: disable=invalid-name +class ThriftServersideTaskHelper(BaseClientHelper): + """Clientside Thrift stub for the `codeCheckerServersideTaskService`.""" + + def __init__(self, protocol: str, host: str, port: int, uri: str, + session_token: Optional[str] = None, + get_new_token: Optional[Callable] = None): + super().__init__(protocol, host, port, uri, + session_token, get_new_token) + + self.client = codeCheckerServersideTaskService.Client(self.protocol) + + @thrift_client_call + def getTaskInfo(self, _token: str) -> TaskInfo: + raise NotImplementedError("Should have called Thrift code!") + + @thrift_client_call + def getTasks(self, _filters: TaskFilter) -> List[AdministratorTaskInfo]: + raise NotImplementedError("Should have called Thrift code!") + + @thrift_client_call + def cancelTask(self, _token: str) -> bool: + raise NotImplementedError("Should have called Thrift code!") + + @thrift_client_call + def createDummyTask(self, _timeout: int, _should_fail: bool) -> str: + raise NotImplementedError("Should have called Thrift code!") diff --git a/web/client/codechecker_client/task_client.py b/web/client/codechecker_client/task_client.py new file mode 100644 index 0000000000..f6666923ef --- /dev/null +++ b/web/client/codechecker_client/task_client.py @@ -0,0 +1,596 @@ +# ------------------------------------------------------------------------- +# +# Part of the CodeChecker project, under the Apache License v2.0 with +# LLVM Exceptions. See LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ------------------------------------------------------------------------- +""" +Implementation for the ``CodeChecker cmd serverside-tasks`` subcommand. +""" +from argparse import Namespace +from copy import deepcopy +from datetime import datetime, timedelta, timezone +import json +import os +import sys +import time +from typing import Callable, Dict, List, Optional, Tuple, cast + +from codechecker_api_shared.ttypes import Ternary +from codechecker_api.ProductManagement_v6.ttypes import Product +from codechecker_api.codeCheckerServersideTasks_v6.ttypes import \ + AdministratorTaskInfo, TaskFilter, TaskInfo, TaskStatus + +from codechecker_common import logger +from codechecker_common.util import clamp +from codechecker_report_converter import twodim + +from .client import setup_product_client, setup_task_client +from .helpers.product import ThriftProductHelper +from .helpers.tasks import ThriftServersideTaskHelper +from .product import split_server_url + + +# Needs to be set in the handler functions. +LOG: Optional[logger.logging.Logger] = None + + +def init_logger(level, stream=None, logger_name="system"): + logger.setup_logger(level, stream) + + global LOG + LOG = logger.get_logger(logger_name) + + +class TaskTimeoutError(Exception): + """Indicates that `await_task_termination` timed out.""" + + def __init__(self, token: str, task_status: int, delta: timedelta): + super().__init__(f"Task '{token}' is still " + f"'{TaskStatus._VALUES_TO_NAMES[task_status]}', " + f"but did not have any progress for '{delta}' " + f"({delta.total_seconds()} seconds)!") + + +def await_task_termination( + log: logger.logging.Logger, + token: str, + probe_delta_min: timedelta = timedelta(seconds=5), + probe_delta_max: timedelta = timedelta(minutes=2), + timeout_from_last_task_progress: Optional[timedelta] = timedelta(hours=1), + max_consecutive_request_failures: Optional[int] = 10, + task_api_client: Optional[ThriftServersideTaskHelper] = None, + server_address: Optional[Tuple[str, str, str]] = None, +) -> str: + """ + Blocks the execution of the current process until the task specified by + `token` terminates. + When terminated, returns the task's `TaskStatus` as a string. + + `await_task_termination` sleeps the current process (as with + `time.sleep`), and periodically wakes up, with a distance of wake-ups + calculated between `probe_delta_min` and `probe_delta_max`, to check the + status of the task by downloading ``getTaskInfo()`` result from the + server. + + The server to use is specified by either providing a valid + `task_api_client`, at which point the connection of the existing client + will be reused; or by providing a + ``(protocol: str, host: str, port: int)`` tuple under `server_address`, + which will cause `await_task_termination` to set the Task API client up + internally. + + This call blocks the caller stack frame indefinitely, unless + `timeout_from_last_task_progress` is specified. + If so, the function will unblock by raising `TaskTimeoutError` if the + specified time has elapsed since the queried task last exhibited forward + progress. + Forward progress is calculated from the task's ``startedAt`` timestamp, + the ``completedAt`` timestamp, or the ``lastHeartbeat``, whichever is + later in time. + For tasks that had not started executing yet (their ``startedAt`` is + `None`), this timeout does not apply. + + This function is resillient against network problems and request failures + through the connection to the server, if + `max_consecutive_request_failures` is specified. + If so, it will wait the given number of Thrift client failures before + giving up. + """ + if not task_api_client and not server_address: + raise ValueError("Specify 'task_api_client' or 'server_address' " + "to point the function at a server to probe!") + if not task_api_client and server_address: + protocol, host, port = server_address + task_api_client = setup_task_client(protocol, host, port) + if not task_api_client: + raise ConnectionError("Failed to set up Task API client!") + + probe_distance: timedelta = deepcopy(probe_delta_min) + request_failures: int = 0 + last_forward_progress_by_task: Optional[datetime] = None + task_status: int = TaskStatus.ALLOCATED + + def _query_task_status(): + while True: + nonlocal request_failures + try: + ti = task_api_client.getTaskInfo(token) + request_failures = 0 + break + except SystemExit: + # getTaskInfo() is decorated by @thrift_client_call, which + # raises SystemExit by calling sys.exit() internally, if + # something fails. + request_failures += 1 + if max_consecutive_request_failures and request_failures > \ + max_consecutive_request_failures: + raise + log.info("Retrying task status query [%d / %d retries] ...", + request_failures, max_consecutive_request_failures) + + last_forward_progress_by_task: Optional[datetime] = None + epoch_to_consider: int = 0 + if ti.completedAtEpoch: + epoch_to_consider = ti.completedAtEpoch + elif ti.lastHeartbeatEpoch: + epoch_to_consider = ti.lastHeartbeatEpoch + elif ti.startedAtEpoch: + epoch_to_consider = ti.startedAtEpoch + if epoch_to_consider: + last_forward_progress_by_task = cast( + datetime, _utc_epoch_to_datetime(epoch_to_consider)) + + task_status = cast(int, ti.status) + + return last_forward_progress_by_task, task_status + + while True: + task_forward_progressed_at, task_status = _query_task_status() + if task_status in [TaskStatus.COMPLETED, TaskStatus.FAILED, + TaskStatus.CANCELLED, TaskStatus.DROPPED]: + break + + if task_forward_progressed_at: + time_since_last_progress = datetime.now(timezone.utc) \ + - task_forward_progressed_at + if timeout_from_last_task_progress and \ + time_since_last_progress >= \ + timeout_from_last_task_progress: + log.error("'%s' timeout elapsed since task last progressed " + "at '%s', considering " + "hung/locked out/lost/failed...", + timeout_from_last_task_progress, + task_forward_progressed_at) + raise TaskTimeoutError(token, task_status, + time_since_last_progress) + + if last_forward_progress_by_task: + # Tune the next probe's wait period in a fashion similar to + # TCP's low-level AIMD (addition increment, + # multiplicative decrement) algorithm. + time_between_last_two_progresses = \ + last_forward_progress_by_task - task_forward_progressed_at + if not time_between_last_two_progresses: + # No progress since the last probe, increase the timeout + # until the next probe, and hope that some progress will + # have been made by that time. + probe_distance += timedelta(seconds=1) + elif time_between_last_two_progresses <= 2 * probe_distance: + # time_between_last_two_progresses is always at least + # probe_distance, because it is the distance between two + # queried and observed forward progress measurements. + # However, if they are "close enough" to each other, it + # means that the server is progressing well with the task + # and it is likely that the task might be finished "soon". + # + # In this case, it is beneficial to INCREASE the probing + # frequency, in order not to make the user wait "too much" + # before observing a "likely" soon available success. + probe_distance /= 2 + else: + # If the progresses detected from the server are + # "far apart", it can indicate that the server is busy + # with processing the task. + # + # In this case, DECREASING the frequency if beneficial, + # because it is "likely" that a final result will not + # arrive soon, and keeping the current frequency would + # just keep "flooding" the server with queries that do + # not return a meaningfully different result. + probe_distance += timedelta(seconds=1) + else: + # If the forward progress has not been observed yet at all, + # increase the timeout until the next probe, and hope that + # some progress will have been made by that time. + probe_distance += timedelta(seconds=1) + + # At any rate, always keep the probe_distance between the + # requested limits. + probe_distance = \ + clamp(probe_delta_min, probe_distance, probe_delta_max) + + last_forward_progress_by_task = task_forward_progressed_at + + log.debug("Waiting %f seconds (%s) before querying the server...", + probe_distance.total_seconds(), probe_distance) + time.sleep(probe_distance.total_seconds()) + + return TaskStatus._VALUES_TO_NAMES[task_status] + + +def _datetime_to_utc_epoch(d: Optional[datetime]) -> Optional[int]: + return int(d.replace(tzinfo=timezone.utc).timestamp()) if d else None + + +def _utc_epoch_to_datetime(s: Optional[int]) -> Optional[datetime]: + return datetime.fromtimestamp(s, timezone.utc) if s else None + + +def _datetime_to_str(d: Optional[datetime]) -> Optional[str]: + return d.strftime("%Y-%m-%d %H:%M:%S") if d else None + + +def _build_filter(args: Namespace, + product_id_to_endpoint: Dict[int, str], + get_product_api: Callable[[], ThriftProductHelper]) \ + -> Optional[TaskFilter]: + """Build a `TaskFilter` from the command-line `args`.""" + filter_: Optional[TaskFilter] = None + + def get_filter() -> TaskFilter: + nonlocal filter_ + if not filter_: + filter_ = TaskFilter() + return filter_ + + if args.machine_id: + get_filter().machineIDs = args.machine_id + if args.type: + get_filter().kinds = args.type + if args.status: + get_filter().statuses = [TaskStatus._NAMES_TO_VALUES[s.upper()] + for s in args.status] + if args.username: + get_filter().usernames = args.username + elif args.no_username: + get_filter().filterForNoUsername = True + if args.product: + # Users specify products via ENDPOINTs for U.X. friendliness, but the + # API works with product IDs. + def _get_product_id_or_log(endpoint: str) -> Optional[int]: + try: + products: List[Product] = cast( + List[Product], + get_product_api().getProducts(endpoint, None)) + # Endpoints substring-match. + product = next(p for p in products if p.endpoint == endpoint) + p_id = cast(int, product.id) + product_id_to_endpoint[p_id] = endpoint + return p_id + except StopIteration: + LOG.warning("No product with endpoint '%s', omitting it from " + "the query.", + endpoint) + return None + + get_filter().productIDs = list(filter(lambda i: i is not None, + map(_get_product_id_or_log, + args.product))) + elif args.no_product: + get_filter().filterForNoProductID = True + if args.enqueued_before: + get_filter().enqueuedBeforeEpoch = _datetime_to_utc_epoch( + args.enqueued_before) + if args.enqueued_after: + get_filter().enqueuedAfterEpoch = _datetime_to_utc_epoch( + args.enqueued_after) + if args.started_before: + get_filter().startedBeforeEpoch = _datetime_to_utc_epoch( + args.started_before) + if args.started_after: + get_filter().startedAfterEpoch = _datetime_to_utc_epoch( + args.started_after) + if args.finished_before: + get_filter().completedBeforeEpoch = _datetime_to_utc_epoch( + args.finished_before) + if args.finished_after: + get_filter().completedAfterEpoch = _datetime_to_utc_epoch( + args.finished_after) + if args.last_seen_before: + get_filter().heartbeatBeforeEpoch = _datetime_to_utc_epoch( + args.started_before) + if args.last_seen_after: + get_filter().heartbeatAfterEpoch = _datetime_to_utc_epoch( + args.started_after) + if args.only_cancelled: + get_filter().cancelFlag = Ternary._NAMES_TO_VALUES["ON"] + elif args.no_cancelled: + get_filter().cancelFlag = Ternary._NAMES_TO_VALUES["OFF"] + if args.only_consumed: + get_filter().consumedFlag = Ternary._NAMES_TO_VALUES["ON"] + elif args.no_consumed: + get_filter().consumedFlag = Ternary._NAMES_TO_VALUES["OFF"] + + return filter_ + + +def _unapi_info(ti: TaskInfo) -> dict: + """ + Converts a `TaskInfo` API structure into a flat Pythonic `dict` of + non-API types. + """ + return {**{k: v + for k, v in ti.__dict__.items() + if k != "status" and not k.endswith("Epoch")}, + **{k.replace("Epoch", "", 1): + _datetime_to_str(_utc_epoch_to_datetime(v)) + for k, v in ti.__dict__.items() + if k.endswith("Epoch")}, + **{"status": TaskStatus._VALUES_TO_NAMES[cast(int, ti.status)]}, + } + + +def _unapi_admin_info(ati: AdministratorTaskInfo) -> dict: + """ + Converts a `AdministratorTaskInfo` API structure into a flat Pythonic + `dict` of non-API types. + """ + return {**{k: v + for k, v in ati.__dict__.items() + if k != "normalInfo"}, + **_unapi_info(cast(TaskInfo, ati.normalInfo)), + } + + +def _transform_product_ids_to_endpoints( + task_infos: List[dict], + product_id_to_endpoint: Dict[int, str], + get_product_api: Callable[[], ThriftProductHelper] +): + """Replace ``task_infos[N]["productId"]`` with + ``task_infos[N]["productEndpoint"]`` for all elements. + """ + for ti in task_infos: + try: + ti["productEndpoint"] = \ + product_id_to_endpoint[ti["productId"]] \ + if ti["productId"] != 0 else None + except KeyError: + # Take the slow path, and get the ID->Endpoint map from the server. + product_id_to_endpoint = { + product.id: product.endpoint + for product + in get_product_api().getProducts(None, None)} + del ti["productId"] + + +def handle_tasks(args: Namespace) -> int: + """Main method for the ``CodeChecker cmd serverside-tasks`` subcommand.""" + # If the given output format is not `table`, redirect the logger's output + # to standard error. + init_logger(args.verbose if "verbose" in args else None, + "stderr" if "output_format" in args + and args.output_format != "table" + else None) + + rc: int = 0 + protocol, host, port = split_server_url(args.server_url) + api = setup_task_client(protocol, host, port) + + if "TEST_WORKSPACE" in os.environ and "dummy_task_args" in args: + timeout, should_fail = \ + int(args.dummy_task_args[0]), \ + args.dummy_task_args[1].lower() in ["y", "yes", "true", "1", "on"] + + dummy_task_token = api.createDummyTask(timeout, should_fail) + LOG.info("Dummy task created with token '%s'.", dummy_task_token) + if not args.token: + args.token = [dummy_task_token] + else: + args.token.append(dummy_task_token) + + # Lazily initialise a Product manager API client as well, it can be needed + # if products are being put into a request filter, or product-specific + # tasks appear on the output. + product_api: Optional[ThriftProductHelper] = None + product_id_to_endpoint: Dict[int, str] = {} + + def get_product_api() -> ThriftProductHelper: + nonlocal product_api + if not product_api: + product_api = setup_product_client(protocol, host, port) + return product_api + + tokens_of_tasks: List[str] = [] + task_filter = _build_filter(args, + product_id_to_endpoint, + get_product_api) + if task_filter: + # If the "filtering" API must be used, the args.token list should also + # be part of the filter. + task_filter.tokens = args.token + + admin_task_infos: List[AdministratorTaskInfo] = \ + api.getTasks(task_filter) + + # Save the tokens of matched tasks for later, in case we have to do + # some further processing. + if args.cancel_task or args.wait_and_block: + tokens_of_tasks = [cast(str, ti.normalInfo.token) + for ti in admin_task_infos] + + task_info_for_print = list(map(_unapi_admin_info, + admin_task_infos)) + _transform_product_ids_to_endpoints(task_info_for_print, + product_id_to_endpoint, + get_product_api) + + if args.output_format == "json": + print(json.dumps(task_info_for_print)) + else: + if args.output_format == "plaintext": + # For the listing of the tasks, the "table" format is more + # appropriate, so we intelligently switch over to that. + args.output_format = "table" + + headers = ["Token", "Machine", "Type", "Summary", "Status", + "Product", "User", "Enqueued", "Started", "Last seen", + "Completed", "Cancelled?"] + rows = [] + for ti in task_info_for_print: + rows.append((ti["token"], + ti["machineId"], + ti["taskKind"], + ti["summary"], + ti["status"], + ti["productEndpoint"] or "", + ti["actorUsername"] or "", + ti["enqueuedAt"] or "", + ti["startedAt"] or "", + ti["lastHeartbeat"] or "", + ti["completedAt"] or "", + "Yes" if ti["cancelFlagSet"] else "", + )) + print(twodim.to_str(args.output_format, headers, rows)) + else: + # If the filtering API was not used, we need to query the tasks + # directly, based on their token. + if not args.token: + LOG.error("ERROR! To use 'CodeChecker cmd serverside-tasks', " + "a '--token' list or some other filter criteria " + "**MUST** be specified!") + sys.exit(2) # Simulate argparse error code. + + # Otherwise, query the tasks, and print their info. + task_infos: List[TaskInfo] = [api.getTaskInfo(token) + for token in args.token] + if not task_infos: + LOG.error("No tasks retrieved for the specified tokens!") + return 1 + + if args.wait_and_block or args.cancel_task: + # If we need to do something with the tasks later, save the tokens. + tokens_of_tasks = args.token + + task_info_for_print = list(map(_unapi_info, task_infos)) + _transform_product_ids_to_endpoints(task_info_for_print, + product_id_to_endpoint, + get_product_api) + + if len(task_infos) == 1: + # If there was exactly one task in the query, the return code + # of the program should be based on the status of the task. + ti = task_info_for_print[0] + if ti["status"] == "COMPLETED": + rc = 0 + elif ti["status"] == "FAILED": + rc = 4 + elif ti["status"] in ["ALLOCATED", "ENQUEUED", "RUNNING"]: + rc = 8 + elif ti["status"] in ["CANCELLED", "DROPPED"]: + rc = 16 + else: + raise ValueError(f"Unknown task status '{ti['status']}'!") + + if args.output_format == "json": + print(json.dumps(task_info_for_print)) + else: + if len(task_infos) > 1 or args.output_format != "plaintext": + if args.output_format == "plaintext": + # For the listing of the tasks, if there are multiple, the + # "table" format is more appropriate, so we intelligently + # switch over to that. + args.output_format = "table" + + headers = ["Token", "Type", "Summary", "Status", "Product", + "User", "Enqueued", "Started", "Last seen", + "Completed", "Cancelled by administrators?"] + rows = [] + for ti in task_info_for_print: + rows.append((ti["token"], + ti["taskKind"], + ti["summary"], + ti["status"], + ti["productEndpoint"] or "", + ti["actorUsername"] or "", + ti["enqueuedAt"] or "", + ti["startedAt"] or "", + ti["lastHeartbeat"] or "", + ti["completedAt"] or "", + "Yes" if ti["cancelFlagSet"] else "", + )) + + print(twodim.to_str(args.output_format, headers, rows)) + else: + # Otherwise, for exactly ONE task, in "plaintext" mode, print + # the details for humans to read. + ti = task_info_for_print[0] + product_line = \ + f" - Product: {ti['productEndpoint']}\n" \ + if ti["productEndpoint"] else "" + user_line = f" - User: {ti['actorUsername']}\n" \ + if ti["actorUsername"] else "" + cancel_line = " - Cancelled by administrators!\n" \ + if ti["cancelFlagSet"] else "" + print(f"Task '{ti['token']}':\n" + f" - Type: {ti['taskKind']}\n" + f" - Summary: {ti['summary']}\n" + f" - Status: {ti['status']}\n" + f"{product_line}" + f"{user_line}" + f" - Enqueued at: {ti['enqueuedAt'] or ''}\n" + f" - Started at: {ti['startedAt'] or ''}\n" + f" - Last seen: {ti['lastHeartbeat'] or ''}\n" + f" - Completed at: {ti['completedAt'] or ''}\n" + f"{cancel_line}" + ) + if ti["comments"]: + print(f"Comments on task '{ti['token']}':\n") + for line in ti["comments"].split("\n"): + if not line or line == "----------": + # Empty or separator lines. + print(line) + elif " at " in line and line.endswith(":"): + # Lines with the "header" for who made the comment + # and when. + print(line) + else: + print(f"> {line}") + + if args.cancel_task: + for token in tokens_of_tasks: + this_call_cancelled = api.cancelTask(token) + if this_call_cancelled: + LOG.info("Submitted cancellation request for task '%s'.", + token) + else: + LOG.debug("Task '%s' had already been cancelled.", token) + + if args.wait_and_block: + rc = 0 + for token in tokens_of_tasks: + LOG.info("Awaiting the completion of task '%s' ...", token) + status: str = await_task_termination( + cast(logger.logging.Logger, LOG), + token, + task_api_client=api) + if status != "COMPLETED": + if args.cancel_task: + # If '--kill' was specified, keep the return code 0 + # if the task was successfully cancelled as well. + if status != "CANCELLED": + LOG.error("Task '%s' error status: %s!", + token, status) + rc = 1 + else: + LOG.info("Task '%s' terminated in status: %s.", + token, status) + else: + LOG.error("Task '%s' error status: %s!", token, status) + rc = 1 + else: + LOG.info("Task '%s' terminated in status: %s.", token, status) + + return rc diff --git a/web/server/codechecker_server/task_executors/abstract_task.py b/web/server/codechecker_server/task_executors/abstract_task.py index 34e1cd4d7a..f38830ad33 100644 --- a/web/server/codechecker_server/task_executors/abstract_task.py +++ b/web/server/codechecker_server/task_executors/abstract_task.py @@ -106,6 +106,13 @@ def execute(self, task_manager: "TaskManager") -> None: injected `task_manager`) and logging failures accordingly. """ if task_manager.should_cancel(self): + def _log_cancel_and_abandon(db_task: DBTask): + db_task.add_comment("CANCEL!\nTask cancelled before " + "execution began!", + "SYSTEM[AbstractTask::execute()]") + db_task.set_abandoned(force_dropped_status=False) + + task_manager._mutate_task_record(self, _log_cancel_and_abandon) return try: diff --git a/web/server/codechecker_server/task_executors/main.py b/web/server/codechecker_server/task_executors/main.py index dc9dd4e543..580a27c4b9 100644 --- a/web/server/codechecker_server/task_executors/main.py +++ b/web/server/codechecker_server/task_executors/main.py @@ -77,9 +77,8 @@ def executor_hangup_handler(signum: int, _frame): except Empty: continue - import pprint - LOG.info("Executor #%d received task object:\n\n%s:\n%s\n\n", - os.getpid(), t, pprint.pformat(t.__dict__)) + LOG.debug("Executor PID %d popped task '%s' (%s) ...", + os.getpid(), t.token, str(t)) t.execute(tm) @@ -135,7 +134,8 @@ def _drop_task_at_shutdown(t: AbstractTask): try: config_db_engine.dispose() except Exception as ex: - LOG.error("Failed to shut down task executor!\n%s", str(ex)) + LOG.error("Failed to shut down task executor %d!\n%s", + os.getpid(), str(ex)) return LOG.debug("Task executor subprocess PID %d exited main loop.",