diff --git a/.pylintrc b/.pylintrc index ea607054..5f640ccc 100644 --- a/.pylintrc +++ b/.pylintrc @@ -104,12 +104,14 @@ disable=abstract-method, no-self-use, no-value-for-parameter, # gin causes false positives nonzero-method, + not-callable, # lots of false positives oct-method, old-division, old-ne-operator, old-octal-literal, old-raise-syntax, parameter-unpacking, + possibly-used-before-assignment, # false positives with control flow print-statement, raising-string, range-builtin-not-iterating, @@ -132,6 +134,7 @@ disable=abstract-method, too-many-instance-attributes, too-many-locals, too-many-nested-blocks, + too-many-positional-arguments, too-many-public-methods, too-many-return-statements, too-many-statements, @@ -140,6 +143,7 @@ disable=abstract-method, unicode-builtin, unnecessary-pass, unpacking-in-except, + used-before-assignment, useless-else-on-loop, useless-object-inheritance, useless-suppression, @@ -425,7 +429,7 @@ valid-metaclass-classmethod-first-arg=mcs # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=StandardError, - Exception, - BaseException +overgeneral-exceptions=builtins.StandardError, + builtins.Exception, + builtins.BaseException diff --git a/Pipfile b/Pipfile index 02205064..0501b65e 100644 --- a/Pipfile +++ b/Pipfile @@ -12,7 +12,7 @@ tensorflow = "==2.12.0" dm-reverb = "==0.11.0" [dev-packages] -pylint = "==2.14.1" +pylint = "==3.3.2" pytest = "==7.1.2" pytype = "==2022.06.06" yapf = "==0.43.0" diff --git a/Pipfile.lock b/Pipfile.lock index 335daff8..8fb03b24 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "d6c198a2acfcd4a25fb0d0335fa6478f50e94a25602b50a5b839f40d46301173" + "sha256": "e6605491ff6cd7776fbfd5252bec0b0e98c62e8417cc1be4cd20a0bc842e4e12" }, "pipfile-spec": 6, "requires": {}, @@ -1354,11 +1354,11 @@ "develop": { "astroid": { "hashes": [ - "sha256:86b0a340a512c65abf4368b80252754cda17c02cdbbd3f587dddf98112233e7b", - "sha256:bb24615c77f4837c707669d16907331374ae8a964650a66999da3f5ca68dc946" + "sha256:6aaea045f938c735ead292204afdb977a36e989522b7833ef6fea94de743f442", + "sha256:db676dc4f3ae6bfe31cda227dc60e03438378d7a896aec57422c95634e8d722f" ], - "markers": "python_full_version >= '3.6.2'", - "version": "==2.11.7" + "markers": "python_full_version >= '3.9.0'", + "version": "==3.3.6" }, "attrs": { "hashes": [ @@ -1370,11 +1370,11 @@ }, "dill": { "hashes": [ - "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0", - "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373" + "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a", + "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c" ], - "markers": "python_version >= '3.7'", - "version": "==0.3.6" + "markers": "python_version >= '3.8'", + "version": "==0.3.9" }, "importlab": { "hashes": [ @@ -1394,11 +1394,11 @@ }, "isort": { "hashes": [ - "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504", - "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6" + "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", + "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6" ], "markers": "python_full_version >= '3.8.0'", - "version": "==5.12.0" + "version": "==5.13.2" }, "lazy-object-proxy": { "hashes": [ @@ -1566,11 +1566,12 @@ }, "pylint": { "hashes": [ - "sha256:549261e0762c3466cc001024c4419c08252cb8c8d40f5c2c6966fea690e7fe2a", - "sha256:bb71e6d169506de585edea997e48d9ff20c0dc0e2fbc1d166bad6b640120326b" + "sha256:77f068c287d49b8683cd7c6e624243c74f92890f767f106ffa1ddf3c0a54cb7a", + "sha256:9ec054ec992cd05ad30a6df1676229739a73f8feeabf3912c995d17601052b01" ], "index": "pypi", - "version": "==2.14.1" + "markers": "python_full_version >= '3.9.0'", + "version": "==3.3.2" }, "pyparsing": { "hashes": [ @@ -1717,19 +1718,19 @@ }, "tomlkit": { "hashes": [ - "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171", - "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3" + "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", + "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79" ], - "markers": "python_version >= '3.7'", - "version": "==0.11.8" + "markers": "python_version >= '3.8'", + "version": "==0.13.2" }, "typing-extensions": { "hashes": [ - "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", - "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" + "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", + "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8" ], - "markers": "python_version >= '3.7'", - "version": "==4.5.0" + "markers": "python_version >= '3.8'", + "version": "==4.12.2" }, "typing-inspect": { "hashes": [ diff --git a/compiler_opt/distributed/buffered_scheduler_test.py b/compiler_opt/distributed/buffered_scheduler_test.py index aca46647..c0e3052c 100644 --- a/compiler_opt/distributed/buffered_scheduler_test.py +++ b/compiler_opt/distributed/buffered_scheduler_test.py @@ -63,16 +63,18 @@ def square(self, the_value, extra_factor=1): [x * x for x in range(10)]) _, futures = buffered_scheduler.schedule_on_worker_pool( - lambda w, v: w.square(**v), [dict(the_value=v) for v in range(10)], - pool) + lambda w, v: w.square(**v), [{ + 'the_value': v + } for v in range(10)], pool) worker.wait_for(futures) self.assertListEqual([f.result() for f in futures], [x * x for x in range(10)]) # same idea, but mix some kwargs _, futures = buffered_scheduler.schedule_on_worker_pool( - lambda w, v: w.square(v[0], **v[1]), - [(v, dict(extra_factor=10)) for v in range(10)], pool) + lambda w, v: w.square(v[0], **v[1]), [(v, { + 'extra_factor': 10 + }) for v in range(10)], pool) worker.wait_for(futures) self.assertListEqual([f.result() for f in futures], [x * x * 10 for x in range(10)]) diff --git a/compiler_opt/distributed/worker_test.py b/compiler_opt/distributed/worker_test.py index df4b5cd5..68e3f8fc 100644 --- a/compiler_opt/distributed/worker_test.py +++ b/compiler_opt/distributed/worker_test.py @@ -35,8 +35,11 @@ def test_gin_args(self): gin.bind_parameter('_test.SomeType.argument', 42) real_args = worker.get_full_worker_args( SomeType, more_args=2, even_more_args='hi') - self.assertDictEqual(real_args, - dict(argument=42, more_args=2, even_more_args='hi')) + self.assertDictEqual(real_args, { + 'argument': 42, + 'more_args': 2, + 'even_more_args': 'hi' + }) if __name__ == '__main__': diff --git a/compiler_opt/es/blackbox_learner.py b/compiler_opt/es/blackbox_learner.py index ab67ae4e..81f61f19 100644 --- a/compiler_opt/es/blackbox_learner.py +++ b/compiler_opt/es/blackbox_learner.py @@ -168,7 +168,7 @@ def _get_perturbations(self) -> List[npt.NDArray[np.float32]]: rng = np.random.default_rng(seed=self._seed) for _ in range(self._config.total_num_perturbations): perturbations.append( - rng.normal(size=(len(self._model_weights))) * + rng.normal(size=len(self._model_weights)) * self._config.precision_parameter) return perturbations diff --git a/compiler_opt/es/blackbox_optimizers.py b/compiler_opt/es/blackbox_optimizers.py index bac0e67c..b0f1e32a 100644 --- a/compiler_opt/es/blackbox_optimizers.py +++ b/compiler_opt/es/blackbox_optimizers.py @@ -983,16 +983,14 @@ def trust_region_test(self, current_input: FloatArray, str(tr_imp_ratio)) if should_reject: self.radius *= self.params['reject_factor'] - if self.radius < self.params['minimum_radius']: - self.radius = self.params['minimum_radius'] + self.radius = max(self.radius, self.params['minimum_radius']) self.is_returned_step = True print('Step rejected. Shrink: ' + str(self.radius) + log_message) return False else: # accept step if should_shrink: self.radius *= self.params['shrink_factor'] - if self.radius < self.params['minimum_radius']: - self.radius = self.params['minimum_radius'] + self.radius = max(self.radius, self.params['minimum_radius']) print('Shrink: ' + str(self.radius) + log_message) elif should_grow: self.radius *= self.params['grow_factor'] diff --git a/compiler_opt/rl/compilation_runner.py b/compiler_opt/rl/compilation_runner.py index 26231244..2381115a 100644 --- a/compiler_opt/rl/compilation_runner.py +++ b/compiler_opt/rl/compilation_runner.py @@ -122,7 +122,7 @@ def __init__(self): Exception.__init__(self) -def kill_process_ignore_exceptions(p: 'subprocess.Popen[bytes]'): +def kill_process_ignore_exceptions(p: 'subprocess.Popen[bytes]'): # pylint: disable=useless-return # kill the process and ignore exceptions. Exceptions would be thrown if the # process has already been killed/finished (which is inherently in a race # condition with us killing it) @@ -130,7 +130,7 @@ def kill_process_ignore_exceptions(p: 'subprocess.Popen[bytes]'): p.kill() p.wait() finally: - return # pylint: disable=lost-exception + return # pylint: disable=lost-exception,return-in-finally class WorkerCancellationManager: diff --git a/compiler_opt/rl/local_data_collector.py b/compiler_opt/rl/local_data_collector.py index 7e9a4365..4a69397b 100644 --- a/compiler_opt/rl/local_data_collector.py +++ b/compiler_opt/rl/local_data_collector.py @@ -105,13 +105,12 @@ def _schedule_jobs(self, policy: policy_saver.Policy, model_id: int, # by now, all the pending work, which was signaled to cancel, must've # finished self._join_pending_jobs() - jobs = [ - dict( - loaded_module_spec=loaded_module_spec, - policy=policy, - reward_stat=self._reward_stat_map[loaded_module_spec.name], - model_id=model_id) for loaded_module_spec in sampled_modules - ] + jobs = [{ + 'loaded_module_spec': loaded_module_spec, + 'policy': policy, + 'reward_stat': self._reward_stat_map[loaded_module_spec.name], + 'model_id': model_id + } for loaded_module_spec in sampled_modules] (self._workers, self._current_futures) = buffered_scheduler.schedule_on_worker_pool( diff --git a/compiler_opt/type_map.py b/compiler_opt/type_map.py index c5def40e..0194ee79 100644 --- a/compiler_opt/type_map.py +++ b/compiler_opt/type_map.py @@ -17,11 +17,10 @@ from typing import List, Tuple, Union import tensorflow as tf -ScalarCType = Union['type[ctypes.c_float]', 'type[ctypes.c_double]', - 'type[ctypes.c_int8]', 'type[ctypes.c_int16]', - 'type[ctypes.c_uint16]', 'type[ctypes.c_int32]', - 'type[ctypes.c_uint32]', 'type[ctypes.c_int64]', - 'type[ctypes.c_uint64]'] +ScalarCType = Union[ # pylint: disable=invalid-name + 'type[ctypes.c_float]', 'type[ctypes.c_double]', 'type[ctypes.c_int8]', + 'type[ctypes.c_int16]', 'type[ctypes.c_uint16]', 'type[ctypes.c_int32]', + 'type[ctypes.c_uint32]', 'type[ctypes.c_int64]', 'type[ctypes.c_uint64]'] TYPE_ASSOCIATIONS: List[Tuple[str, ScalarCType, tf.DType]] = [