From 48ad8e01915f27ec9a232f8058aa1c1f238a4762 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 17 Jun 2022 12:32:22 -0500 Subject: [PATCH 01/18] add api-spec generation/validation to pants --- pants-plugins/api_spec/BUILD | 1 + pants-plugins/api_spec/__init__.py | 0 pants-plugins/api_spec/register.py | 24 +++ pants-plugins/api_spec/rules.py | 265 +++++++++++++++++++++++++ pants-plugins/api_spec/target_types.py | 29 +++ pants.toml | 1 + st2common/st2common/BUILD | 14 +- st2common/st2common/openapi.yaml | 2 +- 8 files changed, 333 insertions(+), 3 deletions(-) create mode 100644 pants-plugins/api_spec/BUILD create mode 100644 pants-plugins/api_spec/__init__.py create mode 100644 pants-plugins/api_spec/register.py create mode 100644 pants-plugins/api_spec/rules.py create mode 100644 pants-plugins/api_spec/target_types.py diff --git a/pants-plugins/api_spec/BUILD b/pants-plugins/api_spec/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/api_spec/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/api_spec/__init__.py b/pants-plugins/api_spec/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/api_spec/register.py b/pants-plugins/api_spec/register.py new file mode 100644 index 0000000000..5982e32e28 --- /dev/null +++ b/pants-plugins/api_spec/register.py @@ -0,0 +1,24 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from api_spec.rules import rules as api_spec_rules +from api_spec.target_types import APISpec + + +def rules(): + return [*api_spec_rules()] + + +def target_types(): + return [APISpec] diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py new file mode 100644 index 0000000000..5161973db9 --- /dev/null +++ b/pants-plugins/api_spec/rules.py @@ -0,0 +1,265 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from textwrap import dedent + +from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules.pex import ( + PexRequest, + VenvPex, + VenvPexProcess, +) +from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest +from pants.backend.python.util_rules.python_sources import ( + PythonSourceFiles, + PythonSourceFilesRequest, +) +from pants.core.goals.fmt import FmtResult, FmtRequest +from pants.core.goals.lint import LintResult, LintResults, LintTargetsRequest +from pants.core.target_types import FileSourceField, ResourceSourceField +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.engine.addresses import Address +from pants.engine.fs import ( + CreateDigest, + Digest, + FileContent, + MergeDigests, + Snapshot, +) +from pants.engine.process import FallibleProcessResult, ProcessResult +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import ( + FieldSet, + SourcesField, + TransitiveTargets, + TransitiveTargetsRequest, +) +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from api_spec.target_types import ( + APISpecSourceField, + APISpec, +) + + +GENERATE_SCRIPT = "generate_api_spec" +VALIDATE_SCRIPT = "validate_api_spec" + +SPEC_HEADER = b"""\ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY +# Edit st2common/st2common/openapi.yaml.j2 and then run +# ./pants fmt st2common/st2common/openapi.yaml +# to generate the final spec file +""" + + +@dataclass(frozen=True) +class APISpecFieldSet(FieldSet): + required_fields = (APISpecSourceField,) + + source: APISpecSourceField + + +class GenerateAPISpecViaFmtRequest(FmtRequest): + field_set_type = APISpecFieldSet + name = GENERATE_SCRIPT + + +class ValidateAPISpecRequest(LintTargetsRequest): + field_set_type = APISpecFieldSet + name = VALIDATE_SCRIPT + + +@rule( + desc="Update openapi.yaml with st2-generate-api-spec", + level=LogLevel.DEBUG, +) +async def generate_api_spec_via_fmt( + request: GenerateAPISpecViaFmtRequest, +) -> FmtResult: + # There will only be one target+field_set, but we iterate + # to satisfy how fmt expects that there could be more than one. + # If there is more than one, they will all get the same contents. + + # Find all the dependencies of our target + transitive_targets = await Get( + TransitiveTargets, + TransitiveTargetsRequest( + [field_set.address for field_set in request.field_sets] + ), + ) + + dependency_files_get = Get( + SourceFiles, + SourceFilesRequest( + sources_fields=[ + tgt.get(SourcesField) for tgt in transitive_targets.dependencies + ], + for_sources_types=(FileSourceField, ResourceSourceField), + # enable_codegen=True, + ), + ) + + source_files_get = Get( + SourceFiles, + SourceFilesRequest(field_set.source for field_set in request.field_sets), + ) + + # actually generate it with an external script. + # Generation cannot be inlined here because it needs to import the st2 code. + pex_get = Get( + VenvPex, + PexFromTargetsRequest( + [ + Address( + "st2common/st2common/cmd", + target_name="cmd", + relative_file_path=f"{GENERATE_SCRIPT}.py", + ), + ], + output_filename=f"{GENERATE_SCRIPT}.pex", + internal_only=True, + main=EntryPoint.parse(f"st2common.cmd.{GENERATE_SCRIPT}:main"), + ), + ) + + pex, dependency_files, source_files = await MultiGet( + pex_get, dependency_files_get, source_files_get + ) + + # If we were given an input digest from a previous formatter for the source files, then we + # should use that input digest instead of the one we read from the filesystem. + source_files_snapshot = ( + source_files.snapshot if request.snapshot is None else request.snapshot + ) + + input_digest = await Get( + Digest, + MergeDigests((dependency_files.snapshot.digest, source_files_snapshot.digest)), + ) + + result = await Get( + ProcessResult, + VenvPexProcess( + pex, + argv=( + "--config-file", + "conf/st2.dev.conf", + ), + input_digest=input_digest, + description=f"Regenerating openapi.yaml api spec", + level=LogLevel.DEBUG, + ), + ) + + contents = [ + FileContent( + f"{field_set.address.spec_path}/{field_set.source.value}", + SPEC_HEADER + result.stdout, + ) + for field_set in request.field_sets + ] + + output_digest = await Get(Digest, CreateDigest(contents)) + output_snapshot = await Get(Snapshot, Digest, output_digest) + return FmtResult.create(request, result, output_snapshot, strip_chroot_path=True) + + +@rule( + desc="Validate openapi.yaml with st2-validate-api-spec", + level=LogLevel.DEBUG, +) +async def validate_api_spec( + request: ValidateAPISpecRequest, +) -> LintResults: + # There will only be one target+field_set, but we iterate + # to satisfy how lint expects that there could be more than one. + # If there is more than one, they will all get the same contents. + + # Find all the dependencies of our target + transitive_targets = await Get( + TransitiveTargets, + TransitiveTargetsRequest( + [field_set.address for field_set in request.field_sets] + ), + ) + + dependency_files_get = Get( + SourceFiles, + SourceFilesRequest( + sources_fields=[ + tgt.get(SourcesField) for tgt in transitive_targets.dependencies + ], + for_sources_types=(FileSourceField, ResourceSourceField), + # enable_codegen=True, + ), + ) + + source_files_get = Get( + SourceFiles, + SourceFilesRequest(field_set.source for field_set in request.field_sets), + ) + + # actually validate it with an external script. + # Validation cannot be inlined here because it needs to import the st2 code. + pex_get = Get( + VenvPex, + PexFromTargetsRequest( + [ + Address( + "st2common/st2common/cmd", + target_name="cmd", + relative_file_path=f"{VALIDATE_SCRIPT}.py", + ), + ], + output_filename=f"{VALIDATE_SCRIPT}.pex", + internal_only=True, + main=EntryPoint.parse(f"st2common.cmd.{VALIDATE_SCRIPT}:main"), + ), + ) + + pex, dependency_files, source_files = await MultiGet( + pex_get, dependency_files_get, source_files_get + ) + + input_digest = await Get( + Digest, + MergeDigests((dependency_files.snapshot.digest, source_files.snapshot.digest)), + ) + + process_result = await Get( + FallibleProcessResult, + VenvPexProcess( + pex, + argv=( + "--config-file", + "conf/st2.dev.conf", + ), + input_digest=input_digest, + description=f"Validating openapi.yaml api spec", + level=LogLevel.DEBUG, + ), + ) + + result = LintResult.from_fallible_process_result(process_result) + return LintResults([result], linter_name=request.name) + + +def rules(): + return [ + *collect_rules(), + UnionRule(FmtRequest, GenerateAPISpecViaFmtRequest), + UnionRule(LintTargetsRequest, ValidateAPISpecRequest), + ] diff --git a/pants-plugins/api_spec/target_types.py b/pants-plugins/api_spec/target_types.py new file mode 100644 index 0000000000..bc1c3e893f --- /dev/null +++ b/pants-plugins/api_spec/target_types.py @@ -0,0 +1,29 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + SingleSourceField, + Target, +) + + +class APISpecSourceField(SingleSourceField): + default = "openapi.yaml" + + +class APISpec(Target): + alias = "api_spec" + core_fields = (*COMMON_TARGET_FIELDS, Dependencies, APISpecSourceField) + help = "Generate openapi.yaml file from Jinja2 template and python sources." diff --git a/pants.toml b/pants.toml index 3de5e4904f..922e7984d9 100644 --- a/pants.toml +++ b/pants.toml @@ -24,6 +24,7 @@ backend_packages = [ # internal plugins in pants-plugins/ "pants.backend.plugin_development", + "api_spec", "schemas", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. diff --git a/st2common/st2common/BUILD b/st2common/st2common/BUILD index 38259c4c32..c40efc6ef3 100644 --- a/st2common/st2common/BUILD +++ b/st2common/st2common/BUILD @@ -5,7 +5,17 @@ python_sources( ) # These may be loaded with st2common.util.spec_loader -resources( +resource( + name="openapi_spec_template", + source="openapi.yaml.j2", +) +api_spec( name="openapi_spec", - sources=["openapi.yaml", "openapi.yaml.j2"], + source="openapi.yaml", + dependencies=[ + ":openapi_spec_template", + "st2common/st2common/cmd/generate_api_spec.py", # st2-generate-api-spec + "st2common/st2common/cmd/validate_api_spec.py", # st2-validate-api-spec + "//conf:st2_dev_conf", # used for both generate and validate + ], ) diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index e86e42727d..c3a135db8a 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -1,6 +1,6 @@ # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # Edit st2common/st2common/openapi.yaml.j2 and then run -# make .generate-api-spec +# ./pants fmt st2common/st2common/openapi.yaml # to generate the final spec file swagger: '2.0' From 38c455553cf4a40803a0856f03d7eaa107e05740 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 17 Jun 2022 17:32:13 -0500 Subject: [PATCH 02/18] add note about missing x-api-model validation of openapi spec There are a lot of issues, so we're only partially validating the spec now. We still validate with prance, but skip checking x-api-model because there are so many legacy issues. --- pants-plugins/api_spec/rules.py | 2 ++ st2common/st2common/openapi.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index 5161973db9..a452d189fa 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -246,6 +246,8 @@ async def validate_api_spec( argv=( "--config-file", "conf/st2.dev.conf", + # "--validate-defs", # check for x-api-model in definitions + # "--verbose", # show definitions on failure ), input_digest=input_digest, description=f"Validating openapi.yaml api spec", diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index c3a135db8a..e86e42727d 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -1,6 +1,6 @@ # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # Edit st2common/st2common/openapi.yaml.j2 and then run -# ./pants fmt st2common/st2common/openapi.yaml +# make .generate-api-spec # to generate the final spec file swagger: '2.0' From cb42c19af3bee86097273e3daa7a6329b49348d7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Jun 2022 14:10:56 -0500 Subject: [PATCH 03/18] resolve lint issues in pants-plugins --- pants-plugins/api_spec/rules.py | 15 +++------------ pants-plugins/api_spec/target_types.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index a452d189fa..a37b18b1a0 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -12,19 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. from dataclasses import dataclass -from textwrap import dedent from pants.backend.python.target_types import EntryPoint from pants.backend.python.util_rules.pex import ( - PexRequest, VenvPex, VenvPexProcess, ) from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest -from pants.backend.python.util_rules.python_sources import ( - PythonSourceFiles, - PythonSourceFilesRequest, -) from pants.core.goals.fmt import FmtResult, FmtRequest from pants.core.goals.lint import LintResult, LintResults, LintTargetsRequest from pants.core.target_types import FileSourceField, ResourceSourceField @@ -48,10 +42,7 @@ from pants.engine.unions import UnionRule from pants.util.logging import LogLevel -from api_spec.target_types import ( - APISpecSourceField, - APISpec, -) +from api_spec.target_types import APISpecSourceField GENERATE_SCRIPT = "generate_api_spec" @@ -159,7 +150,7 @@ async def generate_api_spec_via_fmt( "conf/st2.dev.conf", ), input_digest=input_digest, - description=f"Regenerating openapi.yaml api spec", + description="Regenerating openapi.yaml api spec", level=LogLevel.DEBUG, ), ) @@ -250,7 +241,7 @@ async def validate_api_spec( # "--verbose", # show definitions on failure ), input_digest=input_digest, - description=f"Validating openapi.yaml api spec", + description="Validating openapi.yaml api spec", level=LogLevel.DEBUG, ), ) diff --git a/pants-plugins/api_spec/target_types.py b/pants-plugins/api_spec/target_types.py index bc1c3e893f..2b5101565b 100644 --- a/pants-plugins/api_spec/target_types.py +++ b/pants-plugins/api_spec/target_types.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from pants.backend.python.target_types import PythonResolveField from pants.engine.target import ( COMMON_TARGET_FIELDS, Dependencies, @@ -25,5 +26,13 @@ class APISpecSourceField(SingleSourceField): class APISpec(Target): alias = "api_spec" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, APISpecSourceField) + core_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + APISpecSourceField, + # hack: work around an issue in the pylint backend that tries to + # use this field on the api_spec target, possibly because + # it depends on python files. + PythonResolveField, + ) help = "Generate openapi.yaml file from Jinja2 template and python sources." From 6bcbb5d380ac1819c1788df79f99a5c472c33a5e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Jun 2022 20:06:06 -0500 Subject: [PATCH 04/18] make api_spec generate Resource targets --- pants-plugins/api_spec/target_types.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pants-plugins/api_spec/target_types.py b/pants-plugins/api_spec/target_types.py index 2b5101565b..841b90b556 100644 --- a/pants-plugins/api_spec/target_types.py +++ b/pants-plugins/api_spec/target_types.py @@ -15,12 +15,14 @@ from pants.engine.target import ( COMMON_TARGET_FIELDS, Dependencies, - SingleSourceField, Target, ) +from pants.core.target_types import ( + ResourceSourceField, +) -class APISpecSourceField(SingleSourceField): +class APISpecSourceField(ResourceSourceField): default = "openapi.yaml" From 19faf3b5c5afd947f5228aa02b7d0aa1bf0be9a4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 11 Oct 2022 20:04:08 -0500 Subject: [PATCH 05/18] pants plugins updates --- pants-plugins/api_spec/rules.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index a37b18b1a0..adf0737db8 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -19,7 +19,7 @@ VenvPexProcess, ) from pants.backend.python.util_rules.pex_from_targets import PexFromTargetsRequest -from pants.core.goals.fmt import FmtResult, FmtRequest +from pants.core.goals.fmt import FmtResult, FmtTargetsRequest from pants.core.goals.lint import LintResult, LintResults, LintTargetsRequest from pants.core.target_types import FileSourceField, ResourceSourceField from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest @@ -63,7 +63,7 @@ class APISpecFieldSet(FieldSet): source: APISpecSourceField -class GenerateAPISpecViaFmtRequest(FmtRequest): +class GenerateAPISpecViaFmtTargetsRequest(FmtTargetsRequest): field_set_type = APISpecFieldSet name = GENERATE_SCRIPT @@ -78,7 +78,7 @@ class ValidateAPISpecRequest(LintTargetsRequest): level=LogLevel.DEBUG, ) async def generate_api_spec_via_fmt( - request: GenerateAPISpecViaFmtRequest, + request: GenerateAPISpecViaFmtTargetsRequest, ) -> FmtResult: # There will only be one target+field_set, but we iterate # to satisfy how fmt expects that there could be more than one. @@ -253,6 +253,6 @@ async def validate_api_spec( def rules(): return [ *collect_rules(), - UnionRule(FmtRequest, GenerateAPISpecViaFmtRequest), + UnionRule(FmtTargetsRequest, GenerateAPISpecViaFmtTargetsRequest), UnionRule(LintTargetsRequest, ValidateAPISpecRequest), ] From 21d522ce43dc5ffd59849dad4183b8581e6582fe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 16:21:27 -0600 Subject: [PATCH 06/18] add tests for pants-plugins/api_spec generate --- pants-plugins/api_spec/rules.py | 29 +++-- pants-plugins/api_spec/rules_test.py | 162 +++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 pants-plugins/api_spec/rules_test.py diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index adf0737db8..29104abbe5 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -45,8 +45,12 @@ from api_spec.target_types import APISpecSourceField -GENERATE_SCRIPT = "generate_api_spec" -VALIDATE_SCRIPT = "validate_api_spec" +# these constants are also used in the tests +CMD_SOURCE_ROOT = "st2common" +CMD_DIR = "st2common/st2common/cmd" +CMD_MODULE = "st2common.cmd" +GENERATE_CMD = "generate_api_spec" +VALIDATE_CMD = "validate_api_spec" SPEC_HEADER = b"""\ # NOTE: This file is auto-generated - DO NOT EDIT MANUALLY @@ -65,12 +69,12 @@ class APISpecFieldSet(FieldSet): class GenerateAPISpecViaFmtTargetsRequest(FmtTargetsRequest): field_set_type = APISpecFieldSet - name = GENERATE_SCRIPT + name = GENERATE_CMD class ValidateAPISpecRequest(LintTargetsRequest): field_set_type = APISpecFieldSet - name = VALIDATE_SCRIPT + name = VALIDATE_CMD @rule( @@ -115,14 +119,14 @@ async def generate_api_spec_via_fmt( PexFromTargetsRequest( [ Address( - "st2common/st2common/cmd", + CMD_DIR, target_name="cmd", - relative_file_path=f"{GENERATE_SCRIPT}.py", + relative_file_path=f"{GENERATE_CMD}.py", ), ], - output_filename=f"{GENERATE_SCRIPT}.pex", + output_filename=f"{GENERATE_CMD}.pex", internal_only=True, - main=EntryPoint.parse(f"st2common.cmd.{GENERATE_SCRIPT}:main"), + main=EntryPoint.parse(f"{CMD_MODULE}.{GENERATE_CMD}:main"), ), ) @@ -165,6 +169,7 @@ async def generate_api_spec_via_fmt( output_digest = await Get(Digest, CreateDigest(contents)) output_snapshot = await Get(Snapshot, Digest, output_digest) + # TODO: Drop result.stdout since we already wrote it to a file? return FmtResult.create(request, result, output_snapshot, strip_chroot_path=True) @@ -210,14 +215,14 @@ async def validate_api_spec( PexFromTargetsRequest( [ Address( - "st2common/st2common/cmd", + CMD_DIR, target_name="cmd", - relative_file_path=f"{VALIDATE_SCRIPT}.py", + relative_file_path=f"{VALIDATE_CMD}.py", ), ], - output_filename=f"{VALIDATE_SCRIPT}.pex", + output_filename=f"{VALIDATE_CMD}.pex", internal_only=True, - main=EntryPoint.parse(f"st2common.cmd.{VALIDATE_SCRIPT}:main"), + main=EntryPoint.parse(f"{CMD_MODULE}.{VALIDATE_CMD}:main"), ), ) diff --git a/pants-plugins/api_spec/rules_test.py b/pants-plugins/api_spec/rules_test.py new file mode 100644 index 0000000000..944b5ed0a1 --- /dev/null +++ b/pants-plugins/api_spec/rules_test.py @@ -0,0 +1,162 @@ +# Copyright 2022 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os + +import pytest + +from pants.backend.python import target_types_rules +from pants.backend.python.target_types import PythonSourcesGeneratorTarget + +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.engine.addresses import Address +from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot +from pants.engine.target import Target +from pants.core.goals.fmt import FmtResult +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .rules import ( + CMD_DIR, + CMD_SOURCE_ROOT, + GENERATE_CMD, + APISpecFieldSet, + GenerateAPISpecViaFmtTargetsRequest, + rules as api_spec_rules, +) +from .target_types import APISpec + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *api_spec_rules(), + *target_types_rules.rules(), + QueryRule(FmtResult, (GenerateAPISpecViaFmtTargetsRequest,)), + QueryRule(SourceFiles, (SourceFilesRequest,)), + ], + target_types=[APISpec, PythonSourcesGeneratorTarget], + ) + + +def run_st2_generate_api_spec( + rule_runner: RuleRunner, + targets: list[Target], + *, + extra_args: list[str] | None = None, +) -> FmtResult: + rule_runner.set_options( + [ + "--backend-packages=api_spec", + f"--source-root-patterns=/{CMD_SOURCE_ROOT}", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + field_sets = [APISpecFieldSet.create(tgt) for tgt in targets] + input_sources = rule_runner.request( + SourceFiles, + [ + SourceFilesRequest(field_set.sources for field_set in field_sets), + ], + ) + fmt_result = rule_runner.request( + FmtResult, + [ + GenerateAPISpecViaFmtTargetsRequest( + field_sets, snapshot=input_sources.snapshot + ), + ], + ) + return fmt_result + + +# copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py +def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: + files = [ + FileContent(path, content.encode()) for path, content in source_files.items() + ] + digest = rule_runner.request(Digest, [CreateDigest(files)]) + return rule_runner.request(Snapshot, [digest]) + + +# add dummy script at st2common/st2common/cmd/generate_api_spec.py that the test can load. +GENERATE_API_SPEC_PY = """ +import os + + +def main(): + api_spec_text = "{api_spec_text}" + print(api_spec_text) +""" + + +def write_generate_files( + api_spec_dir: str, api_spec_file: str, before: str, after: str, rule_runner: RuleRunner +) -> None: + files = { + f"{api_spec_dir}/{api_spec_file}": before, + f"{api_spec_dir}/BUILD": "api_spec(name='t')", + # add in the target that's hard-coded in the generate_api_spec_via_fmt rue + f"{CMD_DIR}/{GENERATE_CMD}.py": GENERATE_API_SPEC_PY.format( + api_spec_dir=api_spec_dir, api_spec_text=after + ), + f"{CMD_DIR}/BUILD": "python_sources()", + } + + module = CMD_DIR + while module != CMD_SOURCE_ROOT: + files[f"{module}/__init__.py"] = "" + module = os.path.dirname(module) + + rule_runner.write_files(files) + + +def test_generate_changed(rule_runner: RuleRunner) -> None: + write_generate_files( + api_spec_dir="my_dir", + api_spec_file="dummy.yaml", + before="BEFORE", + after="AFTER", + rule_runner=rule_runner, + ) + + tgt = rule_runner.get_target( + Address("my_dir", target_name="t", relative_file_path="dummy.yaml") + ) + fmt_result = run_st2_generate_api_spec(rule_runner, [tgt]) + assert fmt_result.output == get_snapshot( + rule_runner, {"my_dir/dummy.yaml": "AFTER"} + ) + assert fmt_result.did_change is True + + +def test_generate_unchanged(rule_runner: RuleRunner) -> None: + write_generate_files( + api_spec_dir="my_dir", + api_spec_file="dummy.yaml", + before="AFTER", + after="AFTER", + rule_runner=rule_runner, + ) + + tgt = rule_runner.get_target( + Address("my_dir", target_name="t", relative_file_path="dummy.yaml") + ) + fmt_result = run_st2_generate_api_spec(rule_runner, [tgt]) + assert fmt_result.output == get_snapshot( + rule_runner, {"my_dir/dummy.yaml": "AFTER"} + ) + assert fmt_result.did_change is False From eefc40ac58c930ecf87e34413c52cecf9c3189a4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 20:58:25 -0600 Subject: [PATCH 07/18] add tests for pants-plugins/api_spec validate --- pants-plugins/api_spec/rules_test.py | 101 +++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/pants-plugins/api_spec/rules_test.py b/pants-plugins/api_spec/rules_test.py index 944b5ed0a1..03015f340f 100644 --- a/pants-plugins/api_spec/rules_test.py +++ b/pants-plugins/api_spec/rules_test.py @@ -31,6 +31,7 @@ CMD_DIR, CMD_SOURCE_ROOT, GENERATE_CMD, + VALIDATE_CMD, APISpecFieldSet, GenerateAPISpecViaFmtTargetsRequest, rules as api_spec_rules, @@ -83,6 +84,38 @@ def run_st2_generate_api_spec( return fmt_result +def run_st2_validate_api_spec( + rule_runner: RuleRunner, + targets: list[Target], + *, + extra_args: list[str] | None = None, +) -> LintResults: + rule_runner.set_options( + [ + "--backend-packages=api_spec", + f"--source-root-patterns=/{CMD_SOURCE_ROOT}", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + field_sets = [APISpecFieldSet.create(tgt) for tgt in targets] + input_sources = rule_runner.request( + SourceFiles, + [ + SourceFilesRequest(field_set.sources for field_set in field_sets), + ], + ) + lint_results = rule_runner.request( + LintResults, + [ + ValidateAPISpecRequest( + field_sets, snapshot=input_sources.snapshot + ), + ], + ) + return lint_results + + # copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py def get_snapshot(rule_runner: RuleRunner, source_files: dict[str, str]) -> Snapshot: files = [ @@ -160,3 +193,71 @@ def test_generate_unchanged(rule_runner: RuleRunner) -> None: rule_runner, {"my_dir/dummy.yaml": "AFTER"} ) assert fmt_result.did_change is False + + +# add dummy script at st2common/st2common/cmd/validate_api_spec.py that the test can load. +VALIDATE_API_SPEC_PY = """ +import sys + + +def main(): + sys.exit({rc}) +""" + + +def write_validate_files( + api_spec_dir: str, api_spec_file: str, contents: str, rc: int, rule_runner: RuleRunner +) -> None: + files = { + f"{api_spec_dir}/{api_spec_file}": contents, + f"{api_spec_dir}/BUILD": "api_spec(name='t')", + # add in the target that's hard-coded in the generate_api_spec_via_fmt rue + f"{CMD_DIR}/{VALIDATE_CMD}.py": VALIDATE_API_SPEC_PY.format( + api_spec_dir=api_spec_dir, rc=rc + ), + f"{CMD_DIR}/BUILD": "python_sources()", + } + + module = CMD_DIR + while module != CMD_SOURCE_ROOT: + files[f"{module}/__init__.py"] = "" + module = os.path.dirname(module) + + rule_runner.write_files(files) + + +def test_validate_passed(rule_runner: RuleRunner) -> None: + write_validate_files( + api_spec_dir="my_dir", + api_spec_file="dummy.yaml", + contents="PASS", + rc=0, + rule_runner=rule_runner, + ) + + tgt = rule_runner.get_target( + Address("my_dir", target_name="t", relative_file_path="dummy.yaml") + ) + lint_result = run_st2_validate_api_spec(rule_runner, [tgt]) + assert len(lint_result) == 1 + assert result[0].exit_code == 0 + assert result[0].report == EMPTY_DIGEST + + + +def test_validate_failed(rule_runner: RuleRunner) -> None: + write_validate_files( + api_spec_dir="my_dir", + api_spec_file="dummy.yaml", + contents="FAIL", + rc=1, + rule_runner=rule_runner, + ) + + tgt = rule_runner.get_target( + Address("my_dir", target_name="t", relative_file_path="dummy.yaml") + ) + lint_result = run_st2_validate_api_spec(rule_runner, [tgt]) + assert len(lint_result) == 1 + assert result[0].exit_code == 1 + assert result[0].report == EMPTY_DIGEST From 7786bcb2ecca05a97014defcc242e414a0312c4e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 22:51:14 -0600 Subject: [PATCH 08/18] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f1897ca438..b5771cf182 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 + #5846 #5853 #5848 #5847 #5858 #5857 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 From ec2d43de772432c1ec09229fd0ea2b6192af886c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 23:05:27 -0600 Subject: [PATCH 09/18] add description of api_spec plugin to pants-plugins/README.md --- pants-plugins/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index a20d0b0fa4..29289c32ae 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -9,8 +9,22 @@ The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". These StackStorm-specific plugins are probably only useful for the st2 repo. +- `api_spec` - `schemas` +### `api_spec` plugin + +This plugin wires up pants to make sure `st2common/st2common/openapi.yaml` +gets regenerated if needed. Now, whenever someone runs the `fmt` goal +(eg `./pants fmt st2common/st2common/openapi.yaml`), the api spec will +be regenerated if any of the files used to generate it has changed. +Also, running the `lint` goal will fail if the schemas need to be +regenerated. + +This plugin also wires up pants so that the `lint` goal runs additional +api spec validation on `st2common/st2common/openapi.yaml` with something +like `./pants lint st2common/st2common/openapi.yaml`. + ### `schemas` plugin This plugin wires up pants to make sure `contrib/schemas/*.json` gets From cee6263b61c4125e82e760e96cf9e2e427386786 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 23:17:56 -0600 Subject: [PATCH 10/18] move openapi.yaml header into st2-generate-api-schema --- Makefile | 4 ---- pants-plugins/api_spec/rules.py | 9 +-------- st2common/st2common/cmd/generate_api_spec.py | 11 +++++++++++ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 6120a6f992..6018ee76f2 100644 --- a/Makefile +++ b/Makefile @@ -460,10 +460,6 @@ generate-api-spec: requirements .generate-api-spec @echo @echo "================== Generate openapi.yaml file ====================" @echo - echo "# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY" > st2common/st2common/openapi.yaml - echo "# Edit st2common/st2common/openapi.yaml.j2 and then run" >> st2common/st2common/openapi.yaml - echo "# make .generate-api-spec" >> st2common/st2common/openapi.yaml - echo "# to generate the final spec file" >> st2common/st2common/openapi.yaml . $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml .PHONY: circle-lint-api-spec diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index 29104abbe5..4c1404da26 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -52,13 +52,6 @@ GENERATE_CMD = "generate_api_spec" VALIDATE_CMD = "validate_api_spec" -SPEC_HEADER = b"""\ -# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY -# Edit st2common/st2common/openapi.yaml.j2 and then run -# ./pants fmt st2common/st2common/openapi.yaml -# to generate the final spec file -""" - @dataclass(frozen=True) class APISpecFieldSet(FieldSet): @@ -162,7 +155,7 @@ async def generate_api_spec_via_fmt( contents = [ FileContent( f"{field_set.address.spec_path}/{field_set.source.value}", - SPEC_HEADER + result.stdout, + result.stdout, ) for field_set in request.field_sets ] diff --git a/st2common/st2common/cmd/generate_api_spec.py b/st2common/st2common/cmd/generate_api_spec.py index 7ff7757b71..527fef0f52 100644 --- a/st2common/st2common/cmd/generate_api_spec.py +++ b/st2common/st2common/cmd/generate_api_spec.py @@ -30,12 +30,23 @@ LOG = logging.getLogger(__name__) +# TODO: replace makefile reference with pants equivalent +# ./pants fmt st2common/st2common/openapi.yaml +SPEC_HEADER = """\ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY +# Edit st2common/st2common/openapi.yaml.j2 and then run +# make .generate-api-spec +# to generate the final spec file +""" + + def setup(): common_setup(config=config, setup_db=False, register_mq_exchanges=False) def generate_spec(): spec_string = spec_loader.generate_spec("st2common", "openapi.yaml.j2") + print(SPEC_HEADER) print(spec_string) From 25916462476aefa8bf2ec1f6f5b3a2c92223273c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 23:22:30 -0600 Subject: [PATCH 11/18] ./pants tailor :: --- pants-plugins/api_spec/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pants-plugins/api_spec/BUILD b/pants-plugins/api_spec/BUILD index db46e8d6c9..0eea8b1cf1 100644 --- a/pants-plugins/api_spec/BUILD +++ b/pants-plugins/api_spec/BUILD @@ -1 +1,5 @@ python_sources() + +python_tests( + name="tests", +) From 385c8b4a09f59fe2597f6e128329054e711984b7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 23:27:15 -0600 Subject: [PATCH 12/18] add dependent rules to api_spec plugin --- pants-plugins/api_spec/rules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index 4c1404da26..c0cf95d56f 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from pants.backend.python.target_types import EntryPoint +from pants.backend.python.util_rules import pex, pex_from_targets from pants.backend.python.util_rules.pex import ( VenvPex, VenvPexProcess, @@ -253,4 +254,6 @@ def rules(): *collect_rules(), UnionRule(FmtTargetsRequest, GenerateAPISpecViaFmtTargetsRequest), UnionRule(LintTargetsRequest, ValidateAPISpecRequest), + *pex.rules(), + *pex_from_targets.rules(), ] From ead223ba83ff9666424a96fb91e87989ccc58fb3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 23:55:59 -0600 Subject: [PATCH 13/18] fix tests for pants-plugins/api_spec --- pants-plugins/api_spec/rules_test.py | 41 ++++++++++++---------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/pants-plugins/api_spec/rules_test.py b/pants-plugins/api_spec/rules_test.py index 03015f340f..f48b50dd93 100644 --- a/pants-plugins/api_spec/rules_test.py +++ b/pants-plugins/api_spec/rules_test.py @@ -22,9 +22,10 @@ from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest from pants.engine.addresses import Address -from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot +from pants.engine.fs import CreateDigest, Digest, EMPTY_DIGEST, FileContent, Snapshot from pants.engine.target import Target from pants.core.goals.fmt import FmtResult +from pants.core.goals.lint import LintResult, LintResults from pants.testutil.rule_runner import QueryRule, RuleRunner from .rules import ( @@ -34,6 +35,7 @@ VALIDATE_CMD, APISpecFieldSet, GenerateAPISpecViaFmtTargetsRequest, + ValidateAPISpecRequest, rules as api_spec_rules, ) from .target_types import APISpec @@ -46,6 +48,7 @@ def rule_runner() -> RuleRunner: *api_spec_rules(), *target_types_rules.rules(), QueryRule(FmtResult, (GenerateAPISpecViaFmtTargetsRequest,)), + QueryRule(LintResults, (ValidateAPISpecRequest,)), QueryRule(SourceFiles, (SourceFilesRequest,)), ], target_types=[APISpec, PythonSourcesGeneratorTarget], @@ -70,7 +73,7 @@ def run_st2_generate_api_spec( input_sources = rule_runner.request( SourceFiles, [ - SourceFilesRequest(field_set.sources for field_set in field_sets), + SourceFilesRequest(field_set.source for field_set in field_sets), ], ) fmt_result = rule_runner.request( @@ -89,7 +92,7 @@ def run_st2_validate_api_spec( targets: list[Target], *, extra_args: list[str] | None = None, -) -> LintResults: +) -> Sequence[LintResult]: rule_runner.set_options( [ "--backend-packages=api_spec", @@ -99,21 +102,13 @@ def run_st2_validate_api_spec( env_inherit={"PATH", "PYENV_ROOT", "HOME"}, ) field_sets = [APISpecFieldSet.create(tgt) for tgt in targets] - input_sources = rule_runner.request( - SourceFiles, - [ - SourceFilesRequest(field_set.sources for field_set in field_sets), - ], - ) lint_results = rule_runner.request( LintResults, [ - ValidateAPISpecRequest( - field_sets, snapshot=input_sources.snapshot - ), + ValidateAPISpecRequest(field_sets), ], ) - return lint_results + return lint_results.results # copied from pantsbuild/pants.git/src/python/pants/backend/python/lint/black/rules_integration_test.py @@ -141,7 +136,7 @@ def write_generate_files( ) -> None: files = { f"{api_spec_dir}/{api_spec_file}": before, - f"{api_spec_dir}/BUILD": "api_spec(name='t')", + f"{api_spec_dir}/BUILD": f"api_spec(name='t', source='{api_spec_file}')", # add in the target that's hard-coded in the generate_api_spec_via_fmt rue f"{CMD_DIR}/{GENERATE_CMD}.py": GENERATE_API_SPEC_PY.format( api_spec_dir=api_spec_dir, api_spec_text=after @@ -171,7 +166,7 @@ def test_generate_changed(rule_runner: RuleRunner) -> None: ) fmt_result = run_st2_generate_api_spec(rule_runner, [tgt]) assert fmt_result.output == get_snapshot( - rule_runner, {"my_dir/dummy.yaml": "AFTER"} + rule_runner, {"my_dir/dummy.yaml": "AFTER\n"} ) assert fmt_result.did_change is True @@ -180,8 +175,8 @@ def test_generate_unchanged(rule_runner: RuleRunner) -> None: write_generate_files( api_spec_dir="my_dir", api_spec_file="dummy.yaml", - before="AFTER", - after="AFTER", + before="AFTER\n", + after="AFTER", # print() adds a newline rule_runner=rule_runner, ) @@ -190,7 +185,7 @@ def test_generate_unchanged(rule_runner: RuleRunner) -> None: ) fmt_result = run_st2_generate_api_spec(rule_runner, [tgt]) assert fmt_result.output == get_snapshot( - rule_runner, {"my_dir/dummy.yaml": "AFTER"} + rule_runner, {"my_dir/dummy.yaml": "AFTER\n"} ) assert fmt_result.did_change is False @@ -210,7 +205,7 @@ def write_validate_files( ) -> None: files = { f"{api_spec_dir}/{api_spec_file}": contents, - f"{api_spec_dir}/BUILD": "api_spec(name='t')", + f"{api_spec_dir}/BUILD": f"api_spec(name='t', source='{api_spec_file}')", # add in the target that's hard-coded in the generate_api_spec_via_fmt rue f"{CMD_DIR}/{VALIDATE_CMD}.py": VALIDATE_API_SPEC_PY.format( api_spec_dir=api_spec_dir, rc=rc @@ -240,8 +235,8 @@ def test_validate_passed(rule_runner: RuleRunner) -> None: ) lint_result = run_st2_validate_api_spec(rule_runner, [tgt]) assert len(lint_result) == 1 - assert result[0].exit_code == 0 - assert result[0].report == EMPTY_DIGEST + assert lint_result[0].exit_code == 0 + assert lint_result[0].report == EMPTY_DIGEST @@ -259,5 +254,5 @@ def test_validate_failed(rule_runner: RuleRunner) -> None: ) lint_result = run_st2_validate_api_spec(rule_runner, [tgt]) assert len(lint_result) == 1 - assert result[0].exit_code == 1 - assert result[0].report == EMPTY_DIGEST + assert lint_result[0].exit_code == 1 + assert lint_result[0].report == EMPTY_DIGEST From 2ea3acdd20c96ff18c8e7639211dc25a4472fcc0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 4 Jan 2023 23:59:03 -0600 Subject: [PATCH 14/18] ./pants fmt lint pants-plugins/:: --- pants-plugins/api_spec/rules_test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pants-plugins/api_spec/rules_test.py b/pants-plugins/api_spec/rules_test.py index f48b50dd93..16696ac196 100644 --- a/pants-plugins/api_spec/rules_test.py +++ b/pants-plugins/api_spec/rules_test.py @@ -15,6 +15,8 @@ import os +from typing import Sequence + import pytest from pants.backend.python import target_types_rules @@ -132,7 +134,11 @@ def main(): def write_generate_files( - api_spec_dir: str, api_spec_file: str, before: str, after: str, rule_runner: RuleRunner + api_spec_dir: str, + api_spec_file: str, + before: str, + after: str, + rule_runner: RuleRunner, ) -> None: files = { f"{api_spec_dir}/{api_spec_file}": before, @@ -201,7 +207,11 @@ def main(): def write_validate_files( - api_spec_dir: str, api_spec_file: str, contents: str, rc: int, rule_runner: RuleRunner + api_spec_dir: str, + api_spec_file: str, + contents: str, + rc: int, + rule_runner: RuleRunner, ) -> None: files = { f"{api_spec_dir}/{api_spec_file}": contents, @@ -239,7 +249,6 @@ def test_validate_passed(rule_runner: RuleRunner) -> None: assert lint_result[0].report == EMPTY_DIGEST - def test_validate_failed(rule_runner: RuleRunner) -> None: write_validate_files( api_spec_dir="my_dir", From 43e265b9b458091ba3372f819d59a0e34c193f69 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 5 Jan 2023 00:07:01 -0600 Subject: [PATCH 15/18] fix SPEC_HEADER --- st2common/st2common/cmd/generate_api_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2common/st2common/cmd/generate_api_spec.py b/st2common/st2common/cmd/generate_api_spec.py index 527fef0f52..c1a0c6ccc2 100644 --- a/st2common/st2common/cmd/generate_api_spec.py +++ b/st2common/st2common/cmd/generate_api_spec.py @@ -46,7 +46,7 @@ def setup(): def generate_spec(): spec_string = spec_loader.generate_spec("st2common", "openapi.yaml.j2") - print(SPEC_HEADER) + print(SPEC_HEADER.rstrip()) print(spec_string) From 5ff65acc6e0d67052a614afd88a0b2a372767ce9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 5 Jan 2023 00:24:49 -0600 Subject: [PATCH 16/18] correct Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6018ee76f2..7cdf9c60fc 100644 --- a/Makefile +++ b/Makefile @@ -460,7 +460,7 @@ generate-api-spec: requirements .generate-api-spec @echo @echo "================== Generate openapi.yaml file ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml + . $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf > st2common/st2common/openapi.yaml .PHONY: circle-lint-api-spec circle-lint-api-spec: From 41d04dd7233af5daac46b492c035cb8146126910 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 5 Jan 2023 10:49:45 -0600 Subject: [PATCH 17/18] update pants-plugins/api_spec copyright to 2023 --- pants-plugins/api_spec/register.py | 2 +- pants-plugins/api_spec/rules.py | 2 +- pants-plugins/api_spec/rules_test.py | 2 +- pants-plugins/api_spec/target_types.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pants-plugins/api_spec/register.py b/pants-plugins/api_spec/register.py index 5982e32e28..bf2a9ad1e5 100644 --- a/pants-plugins/api_spec/register.py +++ b/pants-plugins/api_spec/register.py @@ -1,4 +1,4 @@ -# Copyright 2022 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index c0cf95d56f..9653d7e63a 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/api_spec/rules_test.py b/pants-plugins/api_spec/rules_test.py index 16696ac196..0dd6a67486 100644 --- a/pants-plugins/api_spec/rules_test.py +++ b/pants-plugins/api_spec/rules_test.py @@ -1,4 +1,4 @@ -# Copyright 2022 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/api_spec/target_types.py b/pants-plugins/api_spec/target_types.py index 841b90b556..c87bcc6e41 100644 --- a/pants-plugins/api_spec/target_types.py +++ b/pants-plugins/api_spec/target_types.py @@ -1,4 +1,4 @@ -# Copyright 2022 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From dade25a02d8310ecde5149eb8352e8f7eb35967e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 5 Jan 2023 11:02:23 -0600 Subject: [PATCH 18/18] address PR feedback --- pants-plugins/api_spec/rules.py | 6 +++--- pants-plugins/api_spec/rules_test.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pants-plugins/api_spec/rules.py b/pants-plugins/api_spec/rules.py index 9653d7e63a..7d941475df 100644 --- a/pants-plugins/api_spec/rules.py +++ b/pants-plugins/api_spec/rules.py @@ -97,7 +97,6 @@ async def generate_api_spec_via_fmt( tgt.get(SourcesField) for tgt in transitive_targets.dependencies ], for_sources_types=(FileSourceField, ResourceSourceField), - # enable_codegen=True, ), ) @@ -193,7 +192,6 @@ async def validate_api_spec( tgt.get(SourcesField) for tgt in transitive_targets.dependencies ], for_sources_types=(FileSourceField, ResourceSourceField), - # enable_codegen=True, ), ) @@ -236,8 +234,10 @@ async def validate_api_spec( argv=( "--config-file", "conf/st2.dev.conf", + # TODO: Uncomment these as part of a project to fix the (many) issues it identifies. + # We can uncomment --validate-defs (and possibly --verbose) once the spec defs are valid. # "--validate-defs", # check for x-api-model in definitions - # "--verbose", # show definitions on failure + # "--verbose", # show model definitions on failure (only applies to --validate-defs) ), input_digest=input_digest, description="Validating openapi.yaml api spec", diff --git a/pants-plugins/api_spec/rules_test.py b/pants-plugins/api_spec/rules_test.py index 0dd6a67486..0492a5d5dc 100644 --- a/pants-plugins/api_spec/rules_test.py +++ b/pants-plugins/api_spec/rules_test.py @@ -143,7 +143,7 @@ def write_generate_files( files = { f"{api_spec_dir}/{api_spec_file}": before, f"{api_spec_dir}/BUILD": f"api_spec(name='t', source='{api_spec_file}')", - # add in the target that's hard-coded in the generate_api_spec_via_fmt rue + # add in the target that's hard-coded in the generate_api_spec_via_fmt rule f"{CMD_DIR}/{GENERATE_CMD}.py": GENERATE_API_SPEC_PY.format( api_spec_dir=api_spec_dir, api_spec_text=after ), @@ -216,7 +216,7 @@ def write_validate_files( files = { f"{api_spec_dir}/{api_spec_file}": contents, f"{api_spec_dir}/BUILD": f"api_spec(name='t', source='{api_spec_file}')", - # add in the target that's hard-coded in the generate_api_spec_via_fmt rue + # add in the target that's hard-coded in the generate_api_spec_via_fmt rule f"{CMD_DIR}/{VALIDATE_CMD}.py": VALIDATE_API_SPEC_PY.format( api_spec_dir=api_spec_dir, rc=rc ),