diff --git a/d2go/checkpoint/fsdp_checkpoint.py b/d2go/checkpoint/fsdp_checkpoint.py index 5c1f1f2b..2c33c888 100644 --- a/d2go/checkpoint/fsdp_checkpoint.py +++ b/d2go/checkpoint/fsdp_checkpoint.py @@ -222,9 +222,13 @@ def _save_file(self, data, filename): def _load_file(self, f: str): if isinstance(self.model, FSDPWrapper): with ( - interleave_by_rank(concurrency_limit=self._concurrency_limit_fetcher()) - if self.model.state_dict_type != StateDictType.FULL_STATE_DICT - else nullcontext() # FULL_STATE_DICT doesn't need interleaving + ( + interleave_by_rank( + concurrency_limit=self._concurrency_limit_fetcher() + ) + if self.model.state_dict_type != StateDictType.FULL_STATE_DICT + else nullcontext() + ) # FULL_STATE_DICT doesn't need interleaving ): # use mmap for FSDP checkpoints return torch.load(f, map_location=torch.device("cpu"), mmap=True) diff --git a/d2go/config/utils.py b/d2go/config/utils.py index 406cf6ae..610d1d21 100644 --- a/d2go/config/utils.py +++ b/d2go/config/utils.py @@ -209,7 +209,6 @@ def get_diff_cfg_rec(old_cfg, new_cfg, out): out[key] = out.__class__() out[key] = get_diff_cfg_rec(old_cfg[key], new_cfg[key], out[key]) else: - out[key] = new_cfg[key] return out diff --git a/d2go/data/disk_cache.py b/d2go/data/disk_cache.py index 9ff02e4d..1a96bed3 100644 --- a/d2go/data/disk_cache.py +++ b/d2go/data/disk_cache.py @@ -80,7 +80,6 @@ def _initialize_diskcache(self): # self._cache.cache.clear(retry=True) # seems faster if index exists if comm.get_local_rank() == 0: - if self._diskcache_strategy == "naive": for i, item in enumerate(self._lst): ret = self._write_to_local_db((i, item)) diff --git a/d2go/data/extended_coco.py b/d2go/data/extended_coco.py index d017e59d..ba722e95 100644 --- a/d2go/data/extended_coco.py +++ b/d2go/data/extended_coco.py @@ -69,7 +69,6 @@ def extract_archive_file(archive_fn: str, im_dir: str): def save_converted_json(target_json, convert_coco_dict): - if target_json in COCOTEXT_DATASET_CONVERSION_STATUS: return @@ -339,7 +338,6 @@ def convert_to_dict_list( filter_empty_annotations: Optional[bool] = True, _post_process_: Optional[Callable] = None, ) -> List[Dict]: - ann_error_report = { name: ErrorEntry(name, msg, 0) for name, msg in [ diff --git a/d2go/modeling/backbone/fbnet_v2.py b/d2go/modeling/backbone/fbnet_v2.py index fcfb50b9..4060e5e1 100644 --- a/d2go/modeling/backbone/fbnet_v2.py +++ b/d2go/modeling/backbone/fbnet_v2.py @@ -59,8 +59,8 @@ def _merge_fbnetv2_arch_def(cfg): def _parse_arch_def(cfg): arch = cfg.MODEL.FBNET_V2.ARCH arch_def = cfg.MODEL.FBNET_V2.ARCH_DEF - assert (arch != "" and not arch_def) ^ ( - not arch and arch_def != [] + assert ( + (arch != "" and not arch_def) ^ (not arch and arch_def != []) ), "Only allow one unset node between MODEL.FBNET_V2.ARCH ({}) and MODEL.FBNET_V2.ARCH_DEF ({})".format( arch, arch_def ) diff --git a/d2go/modeling/meta_arch/rcnn.py b/d2go/modeling/meta_arch/rcnn.py index a50ad144..a4939fb6 100644 --- a/d2go/modeling/meta_arch/rcnn.py +++ b/d2go/modeling/meta_arch/rcnn.py @@ -132,7 +132,6 @@ def default_rcnn_prepare_for_export(self, cfg, inputs, predictor_type): def _apply_eager_mode_quant(cfg, model): - if isinstance(model, GeneralizedRCNN): """Wrap each quantized part of the model to insert Quant and DeQuant in-place""" diff --git a/d2go/modeling/subclass.py b/d2go/modeling/subclass.py index 2fcae567..bb863a99 100644 --- a/d2go/modeling/subclass.py +++ b/d2go/modeling/subclass.py @@ -30,9 +30,7 @@ def add_subclass_configs(cfg): _C.MODEL.SUBCLASS.NUM_SUBCLASSES = 0 # must be set _C.MODEL.SUBCLASS.NUM_LAYERS = 1 _C.MODEL.SUBCLASS.SUBCLASS_ID_FETCHER = "SubclassFetcher" # ABC, must be set - _C.MODEL.SUBCLASS.SUBCLASS_MAPPING = ( - [] - ) # subclass mapping from model output to annotation + _C.MODEL.SUBCLASS.SUBCLASS_MAPPING = [] # subclass mapping from model output to annotation class SubclassFetcher(ABC): diff --git a/d2go/registry/bootstrap.py b/d2go/registry/bootstrap.py index 6c5cb666..e0c3905c 100644 --- a/d2go/registry/bootstrap.py +++ b/d2go/registry/bootstrap.py @@ -87,7 +87,6 @@ def _match_any(name, module_full_names, match_submodule=False): def _import_mock(name, globals=None, locals=None, fromlist=(), level=0): - use_orig_import = False # enable some first-party packages diff --git a/d2go/trainer/fsdp.py b/d2go/trainer/fsdp.py index d3a46e8e..035100fd 100644 --- a/d2go/trainer/fsdp.py +++ b/d2go/trainer/fsdp.py @@ -130,7 +130,6 @@ def postorder_fsdp_wrap( parent_module: Optional[nn.Module], ignore_branch: bool, ): - rank = dist.get_rank() # don't traverse branches of specified ignored modules if module in modules_not_to_fsdp: diff --git a/d2go/utils/logging.py b/d2go/utils/logging.py index c446da4d..ca4e9bc5 100644 --- a/d2go/utils/logging.py +++ b/d2go/utils/logging.py @@ -69,13 +69,11 @@ def _log_exit(category: str, name: str, unique_id: str, duration: float) -> None def log_interval( category: Optional[str] = None, name: Optional[str] = None ) -> Callable[[Callable[..., _T]], Callable[..., _T]]: - _unique_id = uuid.uuid1().int >> 97 _overwrite_category = category _overwrite_name = name def log_interval_deco(func: Callable[..., _T]) -> Callable[..., _T]: - _category = _overwrite_category or func.__qualname__.split(".")[0] _name = _overwrite_name or func.__name__ diff --git a/projects_oss/detr/detr/backbone/pit.py b/projects_oss/detr/detr/backbone/pit.py index dd4081c6..d7cce5cc 100644 --- a/projects_oss/detr/detr/backbone/pit.py +++ b/projects_oss/detr/detr/backbone/pit.py @@ -97,7 +97,6 @@ def __init__( self.fc = nn.Linear(in_feature, out_feature) def forward(self, x, cls_token): - x = self.conv(x) cls_token = self.fc(cls_token) diff --git a/projects_oss/detr/detr/datasets/coco.py b/projects_oss/detr/detr/datasets/coco.py index 43622437..7bfc4ec6 100644 --- a/projects_oss/detr/detr/datasets/coco.py +++ b/projects_oss/detr/detr/datasets/coco.py @@ -6,6 +6,7 @@ Mostly copy-paste from https://github.com/pytorch/vision/blob/13b35ff/references/detection/coco_utils.py """ + import os from pathlib import Path @@ -126,7 +127,6 @@ def __call__(self, image, target): def make_coco_transforms(image_set): - normalize = T.Compose( [T.ToTensor(), T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])] ) diff --git a/projects_oss/detr/detr/datasets/coco_eval.py b/projects_oss/detr/detr/datasets/coco_eval.py index adfe11b2..c8c5e91a 100644 --- a/projects_oss/detr/detr/datasets/coco_eval.py +++ b/projects_oss/detr/detr/datasets/coco_eval.py @@ -8,6 +8,7 @@ The difference is that there is less copy-pasting from pycocotools in the end of the file, as python3 can suppress prints with contextlib """ + import contextlib import copy import os diff --git a/projects_oss/detr/detr/datasets/transforms.py b/projects_oss/detr/detr/datasets/transforms.py index c3dc44dc..a87d5366 100644 --- a/projects_oss/detr/detr/datasets/transforms.py +++ b/projects_oss/detr/detr/datasets/transforms.py @@ -4,6 +4,7 @@ """ Transforms and data augmentation for both image + bbox. """ + import random import PIL diff --git a/projects_oss/detr/detr/engine.py b/projects_oss/detr/detr/engine.py index ca771f4c..180257db 100644 --- a/projects_oss/detr/detr/engine.py +++ b/projects_oss/detr/detr/engine.py @@ -2,6 +2,7 @@ """ Train and eval functions used in main.py """ + import math import os import sys diff --git a/projects_oss/detr/detr/functions/ms_deform_attn_func.py b/projects_oss/detr/detr/functions/ms_deform_attn_func.py index 40862a5e..73c27509 100644 --- a/projects_oss/detr/detr/functions/ms_deform_attn_func.py +++ b/projects_oss/detr/detr/functions/ms_deform_attn_func.py @@ -18,7 +18,6 @@ class MSDeformAttnFunction(Function): - # The @custom_fwd and @custom_bwd decorators are used in this case to allow enabling of # Automatic Mixed Precision when we do not have implementations of custom CUDA kernels for # all the precision types. diff --git a/projects_oss/detr/detr/models/backbone.py b/projects_oss/detr/detr/models/backbone.py index f6bebac4..6a99e0a4 100644 --- a/projects_oss/detr/detr/models/backbone.py +++ b/projects_oss/detr/detr/models/backbone.py @@ -12,6 +12,7 @@ """ Backbone modules. """ + from collections import OrderedDict from typing import Dict, List diff --git a/projects_oss/detr/detr/models/deformable_detr.py b/projects_oss/detr/detr/models/deformable_detr.py index e8d312f6..d345fae4 100644 --- a/projects_oss/detr/detr/models/deformable_detr.py +++ b/projects_oss/detr/detr/models/deformable_detr.py @@ -10,6 +10,7 @@ """ Deformable DETR model and criterion classes. """ + import copy import math diff --git a/projects_oss/detr/detr/models/detr.py b/projects_oss/detr/detr/models/detr.py index 91103df3..9df0e4f7 100644 --- a/projects_oss/detr/detr/models/detr.py +++ b/projects_oss/detr/detr/models/detr.py @@ -4,6 +4,7 @@ """ DETR model and criterion classes. """ + import torch import torch.nn.functional as F from detectron2.config import configurable diff --git a/projects_oss/detr/detr/models/matcher.py b/projects_oss/detr/detr/models/matcher.py index ff2040b8..15d2a160 100644 --- a/projects_oss/detr/detr/models/matcher.py +++ b/projects_oss/detr/detr/models/matcher.py @@ -4,6 +4,7 @@ """ Modules to compute the matching cost and solve the corresponding LSAP. """ + import torch from detr.util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou from scipy.optimize import linear_sum_assignment diff --git a/projects_oss/detr/detr/models/position_encoding.py b/projects_oss/detr/detr/models/position_encoding.py index e122c855..10bae9c3 100644 --- a/projects_oss/detr/detr/models/position_encoding.py +++ b/projects_oss/detr/detr/models/position_encoding.py @@ -4,6 +4,7 @@ """ Various positional encodings for the transformer. """ + import math import torch @@ -63,14 +64,10 @@ def forward(self, tensor_list: NestedTensor): pos_y = y_embed[:, :, :, None] / dim_t pos_x = torch.stack( (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 - ).flatten( - 3 - ) # shape (B, H, W, N) + ).flatten(3) # shape (B, H, W, N) pos_y = torch.stack( (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 - ).flatten( - 3 - ) # shape (B, H, W, N) + ).flatten(3) # shape (B, H, W, N) pos = torch.cat((pos_y, pos_x), dim=3).permute( 0, 3, 1, 2 ) # shape (B, 2*N, H, W) diff --git a/projects_oss/detr/detr/models/segmentation.py b/projects_oss/detr/detr/models/segmentation.py index a8f48fad..9ffedd80 100644 --- a/projects_oss/detr/detr/models/segmentation.py +++ b/projects_oss/detr/detr/models/segmentation.py @@ -4,6 +4,7 @@ """ This file provides the definition of the convolutional heads used to predict masks, as well as the losses """ + import io from collections import defaultdict from typing import List, Optional diff --git a/projects_oss/detr/detr/models/setcriterion.py b/projects_oss/detr/detr/models/setcriterion.py index 221dfffe..386c2c8f 100644 --- a/projects_oss/detr/detr/models/setcriterion.py +++ b/projects_oss/detr/detr/models/setcriterion.py @@ -247,7 +247,6 @@ def get_foreground_background_loss( return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) def _forward(self, outputs, outputs_without_aux, targets): - # Retrieve the matching between the outputs of the last layer and the targets # A list where each item is [row_indices, col_indices] indices = self.matcher(outputs_without_aux, targets) diff --git a/projects_oss/detr/detr/models/transformer.py b/projects_oss/detr/detr/models/transformer.py index 7c21ffeb..3c6831f4 100644 --- a/projects_oss/detr/detr/models/transformer.py +++ b/projects_oss/detr/detr/models/transformer.py @@ -9,6 +9,7 @@ * extra LN at the end of encoder is removed * decoder returns a stack of activations from all decoding layers """ + import copy from typing import Optional diff --git a/projects_oss/detr/detr/util/box_ops.py b/projects_oss/detr/detr/util/box_ops.py index e84e06e9..f9e22421 100644 --- a/projects_oss/detr/detr/util/box_ops.py +++ b/projects_oss/detr/detr/util/box_ops.py @@ -4,6 +4,7 @@ """ Utilities for bounding box manipulation and GIoU. """ + import torch from torchvision.ops.boxes import box_area diff --git a/projects_oss/detr/detr/util/misc.py b/projects_oss/detr/detr/util/misc.py index 0a1983ef..1f8c3af8 100644 --- a/projects_oss/detr/detr/util/misc.py +++ b/projects_oss/detr/detr/util/misc.py @@ -6,6 +6,7 @@ Mostly copy-paste from torchvision references. """ + import datetime import os import pickle diff --git a/projects_oss/detr/detr/util/plot_utils.py b/projects_oss/detr/detr/util/plot_utils.py index 5bb06c69..4477d4df 100644 --- a/projects_oss/detr/detr/util/plot_utils.py +++ b/projects_oss/detr/detr/util/plot_utils.py @@ -3,6 +3,7 @@ """ Plotting utilities to visualize training logs. """ + from pathlib import Path, PurePath import matplotlib.pyplot as plt diff --git a/projects_oss/detr/test_all.py b/projects_oss/detr/test_all.py index b214d19a..08e90acd 100644 --- a/projects_oss/detr/test_all.py +++ b/projects_oss/detr/test_all.py @@ -195,7 +195,6 @@ def run_model( ) def ort_validate(self, onnx_io, inputs, outputs, tolerate_small_mismatch=False): - inputs, _ = torch.jit._flatten(inputs) outputs, _ = torch.jit._flatten(outputs) diff --git a/projects_oss/detr/test_op.py b/projects_oss/detr/test_op.py index 03695796..ca4345d8 100644 --- a/projects_oss/detr/test_op.py +++ b/projects_oss/detr/test_op.py @@ -120,7 +120,6 @@ def test_forward_equal_with_pytorch_float(self): def test_gradient_numerical( self, channels=4, grad_value=True, grad_sampling_loc=True, grad_attn_weight=True ): - value = torch.rand(N, S, M, channels).cuda() * 0.01 sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 diff --git a/tests/data/test_d2go_datasets.py b/tests/data/test_d2go_datasets.py index a28caa13..4717d70d 100644 --- a/tests/data/test_d2go_datasets.py +++ b/tests/data/test_d2go_datasets.py @@ -303,7 +303,6 @@ def LineMetadata(): @tempdir def test_coco_create_adhoc_class_to_use_dataset(self, tmp_dir): - image_dir, json_file = create_test_images_and_dataset_json( tmp_dir, num_classes=2 ) @@ -339,7 +338,6 @@ def test_coco_create_adhoc_class_to_use_dataset(self, tmp_dir): @tempdir def test_register_coco_dataset_registry(self, tmp_dir): - dummy_buffer = [] @COCO_REGISTER_FUNCTION_REGISTRY.register() @@ -370,7 +368,6 @@ def _register_dummy_function_coco(dataset_name, split_dict): @tempdir def test_adhoc_register_coco_dataset_registry(self, tmp_dir): - dummy_buffer = [] def _dummy_load_func(): @@ -378,7 +375,6 @@ def _dummy_load_func(): @COCO_REGISTER_FUNCTION_REGISTRY.register() def _register_dummy_function_coco_adhoc(dataset_name, split_dict): - json_file = split_dict[ANN_FN] image_root = split_dict[IM_DIR] diff --git a/tests/export/test_torchscript.py b/tests/export/test_torchscript.py index ec44d27e..e9a5b0bb 100644 --- a/tests/export/test_torchscript.py +++ b/tests/export/test_torchscript.py @@ -14,7 +14,6 @@ def mock_export(cls, model, input_args, save_path, export_method, **export_kwarg class TestTorchscriptExportMethods(unittest.TestCase): def test_update_export_kwargs_from_export_method(self): - _empty_export_kwargs = {} def try_mock_export(export_method: str, export_kwargs=_empty_export_kwargs): diff --git a/tests/misc/test_config.py b/tests/misc/test_config.py index a495956e..1e8eafac 100755 --- a/tests/misc/test_config.py +++ b/tests/misc/test_config.py @@ -258,7 +258,6 @@ def test_not_scale_for_zero_world_size(self): class TestConfigDefaultsGen(unittest.TestCase): def test_case1(self): - # register in local scope @CONFIG_UPDATER_REGISTRY.register() def _test1(cfg):