diff --git a/assets/src/generated/graphql.ts b/assets/src/generated/graphql.ts index 9cdd75f316..74d987ea38 100644 --- a/assets/src/generated/graphql.ts +++ b/assets/src/generated/graphql.ts @@ -1434,6 +1434,14 @@ export type CrossVersionResourceTarget = { name?: Maybe; }; +export type CustomRunStep = { + __typename?: 'CustomRunStep'; + args?: Maybe>>; + cmd: Scalars['String']['output']; + requireApproval?: Maybe; + stage: StepStage; +}; + export type CustomStackRun = { __typename?: 'CustomStackRun'; /** the list of commands that will be executed */ @@ -1475,6 +1483,13 @@ export type CustomStackRunEdge = { node?: Maybe; }; +export type CustomStepAttributes = { + args?: InputMaybe>>; + cmd: Scalars['String']['input']; + requireApproval?: InputMaybe; + stage?: InputMaybe; +}; + export type DaemonSet = { __typename?: 'DaemonSet'; events?: Maybe>>; @@ -2123,6 +2138,8 @@ export type InfrastructureStack = { /** version/image config for the tool you're using */ configuration: StackConfiguration; customStackRuns?: Maybe; + /** the stack definition in-use by this stack */ + definition?: Maybe; /** the run that physically destroys the stack */ deleteRun?: Maybe; /** whether this stack was previously deleted and is pending cleanup */ @@ -4207,6 +4224,7 @@ export type RootMutationType = { createServiceAccountToken?: Maybe; createServiceDeployment?: Maybe; createStack?: Maybe; + createStackDefinition?: Maybe; createUpgradePolicy?: Maybe; createWebhook?: Maybe; deleteAccessToken?: Maybe; @@ -4238,6 +4256,7 @@ export type RootMutationType = { deleteServiceContext?: Maybe; deleteServiceDeployment?: Maybe; deleteStack?: Maybe; + deleteStackDefinition?: Maybe; deleteUpgradePolicy?: Maybe; deleteUser?: Maybe; deleteWebhook?: Maybe; @@ -4290,6 +4309,8 @@ export type RootMutationType = { signIn?: Maybe; signup?: Maybe; syncGlobalService?: Maybe; + /** start a new run from the newest sha in the stack's run history */ + triggerRun?: Maybe; updateCluster?: Maybe; updateClusterProvider?: Maybe; updateClusterRestore?: Maybe; @@ -4317,6 +4338,7 @@ export type RootMutationType = { updateServiceDeployment?: Maybe; updateSmtp?: Maybe; updateStack?: Maybe; + updateStackDefinition?: Maybe; updateStackRun?: Maybe; updateUser?: Maybe; upsertNotificationRouter?: Maybe; @@ -4548,6 +4570,11 @@ export type RootMutationTypeCreateStackArgs = { }; +export type RootMutationTypeCreateStackDefinitionArgs = { + attributes: StackDefinitionAttributes; +}; + + export type RootMutationTypeCreateUpgradePolicyArgs = { attributes: UpgradePolicyAttributes; }; @@ -4709,6 +4736,11 @@ export type RootMutationTypeDeleteStackArgs = { }; +export type RootMutationTypeDeleteStackDefinitionArgs = { + id: Scalars['ID']['input']; +}; + + export type RootMutationTypeDeleteUpgradePolicyArgs = { id: Scalars['ID']['input']; }; @@ -4925,6 +4957,11 @@ export type RootMutationTypeSyncGlobalServiceArgs = { }; +export type RootMutationTypeTriggerRunArgs = { + id: Scalars['ID']['input']; +}; + + export type RootMutationTypeUpdateClusterArgs = { attributes: ClusterUpdateAttributes; id: Scalars['ID']['input']; @@ -5082,6 +5119,12 @@ export type RootMutationTypeUpdateStackArgs = { }; +export type RootMutationTypeUpdateStackDefinitionArgs = { + attributes: StackDefinitionAttributes; + id: Scalars['ID']['input']; +}; + + export type RootMutationTypeUpdateStackRunArgs = { attributes: StackRunAttributes; id: Scalars['ID']['input']; @@ -5262,6 +5305,7 @@ export type RootQueryType = { serviceStatuses?: Maybe>>; smtp?: Maybe; stack?: Maybe; + stackDefinition?: Maybe; stackRun?: Maybe; statefulSet?: Maybe; /** adds the ability to search/filter through all tag name/value pairs */ @@ -6086,6 +6130,11 @@ export type RootQueryTypeStackArgs = { }; +export type RootQueryTypeStackDefinitionArgs = { + id: Scalars['ID']['input']; +}; + + export type RootQueryTypeStackRunArgs = { id: Scalars['ID']['input']; }; @@ -6240,6 +6289,7 @@ export type RunStep = { insertedAt?: Maybe; logs?: Maybe>>; name: Scalars['String']['output']; + requireApproval?: Maybe; stage: StepStage; status: StepStatus; updatedAt?: Maybe; @@ -6946,6 +6996,8 @@ export type StackAttributes = { configuration: StackConfigurationAttributes; /** id of an scm connection to use for pr callbacks */ connectionId?: InputMaybe; + /** the id of a stack definition to use */ + definitionId?: InputMaybe; environment?: InputMaybe>>; files?: InputMaybe>>; /** reference w/in the repository where the IaC lives */ @@ -7003,6 +7055,24 @@ export type StackConfigurationAttributes = { version?: InputMaybe; }; +export type StackDefinition = { + __typename?: 'StackDefinition'; + configuration: StackConfiguration; + description?: Maybe; + id: Scalars['ID']['output']; + insertedAt?: Maybe; + name: Scalars['String']['output']; + steps?: Maybe>>; + updatedAt?: Maybe; +}; + +export type StackDefinitionAttributes = { + configuration?: InputMaybe; + description?: InputMaybe; + name: Scalars['String']['input']; + steps?: InputMaybe>>; +}; + export type StackEnvironment = { __typename?: 'StackEnvironment'; name: Scalars['String']['output']; @@ -7206,6 +7276,7 @@ export enum StackStatus { export enum StackType { Ansible = 'ANSIBLE', + Custom = 'CUSTOM', Terraform = 'TERRAFORM' } @@ -7307,11 +7378,14 @@ export type SyncConfig = { __typename?: 'SyncConfig'; /** whether the agent should auto-create the namespace for this service */ createNamespace?: Maybe; + /** Whether to require all resources are placed in the same namespace */ + enforceNamespace?: Maybe; namespaceMetadata?: Maybe; }; export type SyncConfigAttributes = { createNamespace?: InputMaybe; + enforceNamespace?: InputMaybe; namespaceMetadata?: InputMaybe; }; diff --git a/lib/console/deployments/pipelines/stage_worker.ex b/lib/console/deployments/pipelines/stage_worker.ex index 7e1871d73b..c180039acc 100644 --- a/lib/console/deployments/pipelines/stage_worker.ex +++ b/lib/console/deployments/pipelines/stage_worker.ex @@ -29,11 +29,14 @@ defmodule Console.Deployments.Pipelines.StageWorker do {:ok, _} -> Logger.info "stage #{stage.id} context applied successfully" {:error, err} -> Logger.info "failed to apply stage context #{stage.id} reason: #{inspect(err)}" - Pipelines.add_stage_error(stage, "context", "Failed to apply stage context with error: #{inspect(err)}") + Pipelines.add_stage_error(stage, "context", "Failed to apply stage context with error: #{format_error(err)}") end {:noreply, state} end + defp format_error(err) when is_binary(err), do: "\n#{err}" + defp format_error(err), do: inspect(err) + def handle_cast(%PipelineStage{} = stage, state) do case Pipelines.build_promotion(stage) do {:ok, _} -> Logger.info "stage #{stage.id} applied successfully" diff --git a/lib/console/deployments/policies/policies.ex b/lib/console/deployments/policies/policies.ex index 5832b7084c..e4547c68b8 100644 --- a/lib/console/deployments/policies/policies.ex +++ b/lib/console/deployments/policies/policies.ex @@ -125,6 +125,18 @@ defmodule Console.Deployments.Policies do def can?(%User{roles: %{admin: true}}, _, _), do: :pass + def can?(%User{id: id} = user, %Stack{actor_id: id, actor_changed: true} = stack, action), + do: can?(user, %{stack | actor_changed: false}, action) + + def can?(_, %Stack{actor_changed: true}, :write), + do: {:error, "you can only set yourself as actor unless you're an admin"} + + def can?(%User{} = user, %Stack{} = stack, :create) do + %{cluster: %Cluster{} = cluster} = Repo.preload(stack, [:cluster]) + + rbac(user, stack, :write) && can?(user, cluster, :write) + end + def can?(%User{} = user, resource, action), do: rbac(resource, user, action) diff --git a/lib/console/deployments/stacks.ex b/lib/console/deployments/stacks.ex index e8ff087fa1..26002f5cf4 100644 --- a/lib/console/deployments/stacks.ex +++ b/lib/console/deployments/stacks.ex @@ -19,7 +19,8 @@ defmodule Console.Deployments.Stacks do GitRepository, PullRequest, ScmConnection, - CustomStackRun + CustomStackRun, + StackDefinition } @preloads [:environment, :files, :observable_metrics] @@ -30,6 +31,7 @@ defmodule Console.Deployments.Stacks do @type step_resp :: {:ok, RunStep.t} | error @type log_resp :: {:ok, RunLog.t} | error @type custom_resp :: {:ok, CustomStackRun.t} | error + @type def_resp :: {:ok, StackDefinition.t} | error @spec get_stack!(binary) :: Stack.t def get_stack!(id), do: Repo.get!(Stack, id) @@ -46,6 +48,17 @@ defmodule Console.Deployments.Stacks do @spec get_custom_run!(binary) :: CustomStackRun.t | nil def get_custom_run!(id), do: Repo.get!(CustomStackRun, id) + @spec get_definition!(binary) :: StackDefinition.t + def get_definition!(id), do: Repo.get!(StackDefinition, id) + + @spec latest_run(binary) :: StackRun.t | nil + def latest_run(stack_id) do + StackRun.for_stack(stack_id) + |> StackRun.ordered(desc: :id) + |> StackRun.limit(1) + |> Repo.one() + end + def preloaded(%Stack{} = stack), do: Repo.preload(stack, @preloads) @spec authorized(binary, Cluster.t) :: run_resp @@ -95,7 +108,7 @@ defmodule Console.Deployments.Stacks do def create_stack(attrs, %User{} = user) do %Stack{status: :queued} |> Stack.changeset(Settings.add_project_id(attrs)) - |> allow(user, :write) + |> allow(user, :create) |> when_ok(:insert) |> notify(:create, user) end @@ -137,6 +150,39 @@ defmodule Console.Deployments.Stacks do |> notify(:delete, user) end + + @doc """ + creates a new stack definition + """ + @spec create_stack_definition(map, User.t) :: def_resp + def create_stack_definition(attrs, %User{} = user) do + %StackDefinition{} + |> StackDefinition.changeset(attrs) + |> allow(user, :write) + |> when_ok(:insert) + end + + @doc """ + updates a stack definition + """ + @spec update_stack_definition(map, binary, User.t) :: def_resp + def update_stack_definition(attrs, id, %User{} = user) do + get_definition!(id) + |> StackDefinition.changeset(attrs) + |> allow(user, :write) + |> when_ok(:update) + end + + @doc """ + deletes a stack definition + """ + @spec delete_stack_definition(binary, User.t) :: def_resp + def delete_stack_definition(id, %User{} = user) do + get_definition!(id) + |> allow(user, :write) + |> when_ok(:delete) + end + @doc """ modifies rbac settings for this stack """ @@ -355,26 +401,27 @@ defmodule Console.Deployments.Stacks do def poll(%Stack{delete_run_id: id}) when is_binary(id), do: {:error, "stack is deleting"} - def poll(%Stack{} = stack) do + def poll(%Stack{polled_sha: ps} = stack) do with {:ok, %Stack{sha: sha, git: git} = stack} <- lock(stack) do %{repository: repo} = stack = Repo.preload(stack, @poll_preloads) - case Discovery.sha(repo, git.ref) do - {:ok, ^sha} -> {:error, "no new commit in repo"} - {:ok, new_sha} -> - with {:ok, new_sha, msg} <- new_changes(repo, git, sha, new_sha), - do: create_run(stack, new_sha, %{message: msg}) - err -> err - end + on_new_sha(repo, git.ref, sha, ps, fn new_sha -> + case new_changes(repo, git, sha, new_sha) do + {:ok, new_sha, msg} -> + create_run(stack, new_sha, %{message: msg}) + err -> + add_polled_sha(stack, new_sha) + err + end + end) |> unlock(stack) end end - def poll(%PullRequest{sha: sha, ref: ref, stack_id: id} = pr) when is_binary(id) do + def poll(%PullRequest{sha: sha, polled_sha: ps, ref: ref, stack_id: id} = pr) when is_binary(id) do %{stack: %{repository: repo} = stack} = pr = Repo.preload(pr, [stack: @poll_preloads]) - case Discovery.sha(repo, ref) do - {:ok, ^sha} -> {:error, "no new commit in repo for branch #{ref}"} - {:ok, new_sha} -> - with {:ok, new_sha, msg} <- new_changes(repo, stack.git, sha, new_sha) do + on_new_sha(repo, ref, sha, ps, fn new_sha -> + case new_changes(repo, stack.git, sha, new_sha) do + {:ok, new_sha, msg} -> start_transaction() |> add_operation(:run, fn _ -> create_run(stack, new_sha, %{pull_request_id: pr.id, message: msg, dry_run: true}) @@ -384,12 +431,28 @@ defmodule Console.Deployments.Stacks do |> Repo.update() end) |> execute(extract: :run) - end + err -> + add_polled_sha(pr, new_sha) + err + end + end) + end + + def poll(_), do: {:error, "invalid parent"} + + defp on_new_sha(repo, ref, sha, ps, fun) do + case Discovery.sha(repo, ref) do + {:ok, ^ps} -> {:error, "this sha has already been polled"} + {:ok, ^sha} -> {:error, "no new commit in repo for branch #{ref}"} + {:ok, found} -> fun.(found) err -> err end end - def poll(_), do: {:error, "invalid parent"} + defp add_polled_sha(record, sha) do + Ecto.Changeset.change(record, %{polled_sha: sha}) + |> Repo.update() + end defp new_changes(repo, %{folder: folder}, sha1, sha2) do case Discovery.changes(repo, sha1, sha2, folder) do @@ -436,11 +499,31 @@ defmodule Console.Deployments.Stacks do end defp template_cmds(commands, _), do: commands + @doc """ + Creates a fresh run from the last sha of the stack + """ + @spec trigger_run(binary, User.t) :: run_resp + def trigger_run(stack_id, %User{} = user) do + start_transaction() + |> add_operation(:stack, fn _ -> + get_stack!(stack_id) + |> allow(user, :write) + end) + |> add_operation(:run, fn %{stack: stack} -> + case latest_run(stack.id) do + %StackRun{git: %{ref: sha}} -> create_run(stack, sha) + _ -> poll(stack) + end + end) + |> execute(extract: :run) + end + @doc """ Creates a new run for the stack with the given sha and optional additional attrs """ @spec create_run(Stack.t, binary) :: run_resp def create_run(%Stack{} = stack, sha, attrs \\ %{}) do + stack = Repo.preload(stack, [:definition]) start_transaction() |> add_operation(:run, fn _ -> %StackRun{stack_id: stack.id, status: :queued} @@ -463,12 +546,25 @@ defmodule Console.Deployments.Stacks do defp stack_attrs(%Stack{} = stack, sha) do Repo.preload(stack, [:environment, :files]) |> Map.take(~w(approval actor_id workdir manage_state dry_run configuration type environment files job_spec repository_id cluster_id)a) + |> Map.put(:configuration, ensure_configuration(stack)) |> Console.clean() |> Map.update(:environment, [], fn env -> Enum.map(env, &Map.delete(&1, :stack_id)) end) |> Map.update(:files, [], fn files -> Enum.map(files, &Map.delete(&1, :stack_id)) end) |> Map.put(:git, %{ref: sha, folder: stack.git.folder}) end + defp ensure_configuration( + %Stack{ + definition: %StackDefinition{configuration: %Stack.Configuration{} = def_conf}, + configuration: conf + } + ) do + (conf || %{}) + |> Map.put_new(:image, def_conf.image) + |> Map.put_new(:tag, def_conf.tag) + end + defp ensure_configuration(%Stack{configuration: configuration}), do: configuration + defp sha_attrs(%StackRun{dry_run: true}, _sha), do: %{} defp sha_attrs(%StackRun{}, sha), do: %{sha: sha} diff --git a/lib/console/deployments/stacks/commands.ex b/lib/console/deployments/stacks/commands.ex index 78c3046fa0..7f677692c0 100644 --- a/lib/console/deployments/stacks/commands.ex +++ b/lib/console/deployments/stacks/commands.ex @@ -1,5 +1,5 @@ defmodule Console.Deployments.Stacks.Commands do - alias Console.Schema.{Stack} + alias Console.Schema.{Stack, StackDefinition} def commands(stack, dry \\ false) @@ -13,6 +13,13 @@ defmodule Console.Deployments.Stacks.Commands do |> stitch_hooks(stack, dry) end + def commands(%Stack{type: :custom}, true), do: [] + def commands(%Stack{type: :custom, definition: %StackDefinition{name: n, steps: steps}} = stack, _) do + Enum.map(steps, &Map.take(&1, ~w(cmd args stage require_approval)a)) + |> Enum.with_index(&Map.merge(&1, %{index: &2, name: "#{n}-#{&2}", status: :pending})) + |> stitch_hooks(stack, false) + end + defp stitch_hooks(commands, %Stack{configuration: %{hooks: [_ | _] = hooks}}, dry) do cmds = group_by_stage(commands) hooks = group_by_stage(hooks) diff --git a/lib/console/graphql/deployments/service.ex b/lib/console/graphql/deployments/service.ex index f951f3c8c2..b2b178ddd9 100644 --- a/lib/console/graphql/deployments/service.ex +++ b/lib/console/graphql/deployments/service.ex @@ -35,6 +35,7 @@ defmodule Console.GraphQl.Deployments.Service do input_object :sync_config_attributes do field :create_namespace, :boolean + field :enforce_namespace, :boolean field :namespace_metadata, :metadata_attributes end @@ -320,6 +321,7 @@ defmodule Console.GraphQl.Deployments.Service do @desc "Advanced configuration of how to sync resources" object :sync_config do field :create_namespace, :boolean, description: "whether the agent should auto-create the namespace for this service" + field :enforce_namespace, :boolean, description: "Whether to require all resources are placed in the same namespace" field :namespace_metadata, :namespace_metadata end diff --git a/lib/console/graphql/deployments/stack.ex b/lib/console/graphql/deployments/stack.ex index f41b709534..692259f628 100644 --- a/lib/console/graphql/deployments/stack.ex +++ b/lib/console/graphql/deployments/stack.ex @@ -23,6 +23,7 @@ defmodule Console.GraphQl.Deployments.Stack do field :actor_id, :id, description: "user id to use for default Plural authentication in this stack" field :project_id, :id, description: "the project id this stack will belong to" field :connection_id, :id, description: "id of an scm connection to use for pr callbacks" + field :definition_id, :id, description: "the id of a stack definition to use" field :read_bindings, list_of(:policy_binding_attributes) field :write_bindings, list_of(:policy_binding_attributes) @@ -107,6 +108,20 @@ defmodule Console.GraphQl.Deployments.Stack do field :dir, :string end + input_object :stack_definition_attributes do + field :name, non_null(:string) + field :description, :string + field :steps, list_of(:custom_step_attributes) + field :configuration, :stack_configuration_attributes + end + + input_object :custom_step_attributes do + field :stage, :step_stage + field :cmd, non_null(:string) + field :args, list_of(:string) + field :require_approval, :boolean + end + object :infrastructure_stack do field :id, :id field :name, non_null(:string), description: "the name of the stack" @@ -143,6 +158,7 @@ defmodule Console.GraphQl.Deployments.Stack do field :project, :project, resolve: dataloader(Deployments), description: "The project this stack belongs to" field :cluster, :cluster, resolve: dataloader(Deployments), description: "the cluster this stack runs on" field :repository, :git_repository, resolve: dataloader(Deployments), description: "the git repository you're sourcing IaC from" + field :definition, :stack_definition, resolve: dataloader(Deployments), description: "the stack definition in-use by this stack" field :actor, :user, resolve: dataloader(User), description: "the actor of this stack (defaults to root console user)" @@ -242,13 +258,14 @@ defmodule Console.GraphQl.Deployments.Stack do end object :run_step do - field :id, non_null(:id) - field :status, non_null(:step_status) - field :stage, non_null(:step_stage) - field :name, non_null(:string) - field :cmd, non_null(:string) - field :args, list_of(non_null(:string)) - field :index, non_null(:integer) + field :id, non_null(:id) + field :status, non_null(:step_status) + field :stage, non_null(:step_stage) + field :name, non_null(:string) + field :cmd, non_null(:string) + field :args, list_of(non_null(:string)) + field :require_approval, :boolean + field :index, non_null(:integer) field :logs, list_of(:run_logs), resolve: dataloader(Deployments) @@ -305,6 +322,25 @@ defmodule Console.GraphQl.Deployments.Stack do timestamps() end + object :stack_definition do + field :id, non_null(:id) + field :name, non_null(:string) + field :description, :string + + field :configuration, non_null(:stack_configuration) + + field :steps, list_of(:custom_run_step) + + timestamps() + end + + object :custom_run_step do + field :cmd, non_null(:string) + field :args, list_of(:string) + field :stage, non_null(:step_stage) + field :require_approval, :boolean + end + object :stack_command do field :cmd, non_null(:string), description: "the executable to call" field :args, list_of(:string), description: "cli args to pass" @@ -381,6 +417,13 @@ defmodule Console.GraphQl.Deployments.Stack do resolve &Deployments.resolve_stack/2 end + field :stack_definition, :stack_definition do + middleware Authenticated + arg :id, non_null(:id) + + resolve &Deployments.resolve_stack_definition/2 + end + connection field :infrastructure_stacks, node_type: :infrastructure_stack do middleware Authenticated arg :q, :string @@ -464,6 +507,28 @@ defmodule Console.GraphQl.Deployments.Stack do resolve &Deployments.delete_custom_stack_run/2 end + field :create_stack_definition, :stack_definition do + middleware Authenticated + arg :attributes, non_null(:stack_definition_attributes) + + resolve &Deployments.create_stack_definition/2 + end + + field :update_stack_definition, :stack_definition do + middleware Authenticated + arg :id, non_null(:id) + arg :attributes, non_null(:stack_definition_attributes) + + resolve &Deployments.update_stack_definition/2 + end + + field :delete_stack_definition, :stack_definition do + middleware Authenticated + arg :id, non_null(:id) + + resolve &Deployments.delete_stack_definition/2 + end + @desc "Creates a custom run, with the given command list, to execute w/in the stack's environment" field :on_demand_run, :stack_run do middleware Authenticated @@ -473,6 +538,14 @@ defmodule Console.GraphQl.Deployments.Stack do resolve &Deployments.create_stack_run/2 end + + @desc "start a new run from the newest sha in the stack's run history" + field :trigger_run, :stack_run do + middleware Authenticated + arg :id, non_null(:id) + + resolve &Deployments.trigger_run/2 + end end object :stack_subscriptions do diff --git a/lib/console/graphql/resolvers/deployments.ex b/lib/console/graphql/resolvers/deployments.ex index d3d5dad72f..9a76ea5b74 100644 --- a/lib/console/graphql/resolvers/deployments.ex +++ b/lib/console/graphql/resolvers/deployments.ex @@ -55,7 +55,8 @@ defmodule Console.GraphQl.Resolvers.Deployments do StackState, ServiceDependency, Project, - ServiceImport + ServiceImport, + StackDefinition } def query(Project, _), do: Project @@ -109,6 +110,7 @@ defmodule Console.GraphQl.Resolvers.Deployments do def query(StackState, _), do: StackState def query(ServiceDependency, _), do: ServiceDependency def query(ServiceImport, _), do: ServiceImport + def query(StackDefinition, _), do: StackDefinition def query(_, _), do: Cluster delegates Console.GraphQl.Resolvers.Deployments.Git diff --git a/lib/console/graphql/resolvers/deployments/stack.ex b/lib/console/graphql/resolvers/deployments/stack.ex index fbfa67e2e6..8b8ef045c2 100644 --- a/lib/console/graphql/resolvers/deployments/stack.ex +++ b/lib/console/graphql/resolvers/deployments/stack.ex @@ -52,6 +52,8 @@ defmodule Console.GraphQl.Resolvers.Deployments.Stack do |> paginate(args) end + def resolve_stack_definition(%{id: id}, _), do: {:ok, Stacks.get_definition!(id)} + def resolve_stack(%{id: id}, ctx) do Stacks.get_stack!(id) |> allow(actor(ctx), :read) @@ -114,9 +116,21 @@ defmodule Console.GraphQl.Resolvers.Deployments.Stack do def delete_custom_stack_run(%{id: id}, %{context: %{current_user: user}}), do: Stacks.delete_custom_stack_run(id, user) + def create_stack_definition(%{attributes: attrs}, %{context: %{current_user: user}}), + do: Stacks.create_stack_definition(attrs, user) + + def update_stack_definition(%{id: id, attributes: attrs}, %{context: %{current_user: user}}), + do: Stacks.update_stack_definition(attrs, id, user) + + def delete_stack_definition(%{id: id}, %{context: %{current_user: user}}), + do: Stacks.delete_stack_definition(id, user) + def create_stack_run(%{stack_id: id, commands: commands} = args, %{context: %{current_user: user}}), do: Stacks.create_custom_run(id, commands, args[:context], user) + def trigger_run(%{id: id}, %{context: %{current_user: user}}), + do: Stacks.trigger_run(id, user) + def job_spec(%StackRun{job_spec: %{} = spec}, _, _), do: {:ok, spec} def job_spec(_, _, _) do case Settings.cached() do diff --git a/lib/console/schema/pull_request.ex b/lib/console/schema/pull_request.ex index d185fee6d8..68bbe48e04 100644 --- a/lib/console/schema/pull_request.ex +++ b/lib/console/schema/pull_request.ex @@ -5,13 +5,14 @@ defmodule Console.Schema.PullRequest do defenum Status, open: 0, merged: 1, closed: 2 schema "pull_requests" do - field :url, :string - field :status, Status, default: :open - field :title, :string - field :creator, :string - field :labels, {:array, :string} - field :ref, :string - field :sha, :string + field :url, :string + field :status, Status, default: :open + field :title, :string + field :creator, :string + field :labels, {:array, :string} + field :ref, :string + field :sha, :string + field :polled_sha, :string field :review_id, :string diff --git a/lib/console/schema/run_step.ex b/lib/console/schema/run_step.ex index db273ae197..428a653133 100644 --- a/lib/console/schema/run_step.ex +++ b/lib/console/schema/run_step.ex @@ -6,12 +6,13 @@ defmodule Console.Schema.RunStep do defenum Stage, plan: 0, verify: 1, apply: 2, init: 3, destroy: 4 schema "run_steps" do - field :name, :string - field :status, Status - field :stage, Stage - field :cmd, :string - field :args, {:array, :string} - field :index, :integer + field :name, :string + field :status, Status + field :stage, Stage + field :cmd, :string + field :args, {:array, :string} + field :index, :integer + field :require_approval, :boolean has_many :logs, RunLog, foreign_key: :step_id @@ -20,7 +21,7 @@ defmodule Console.Schema.RunStep do timestamps() end - @valid ~w(name status stage cmd args index run_id)a + @valid ~w(name status stage cmd args require_approval index run_id)a def changeset(model, attrs \\ %{}) do model diff --git a/lib/console/schema/service.ex b/lib/console/schema/service.ex index df277b0189..6b4f474edd 100644 --- a/lib/console/schema/service.ex +++ b/lib/console/schema/service.ex @@ -119,7 +119,8 @@ defmodule Console.Schema.Service do embeds_one :sync_config, SyncConfig, on_replace: :update do embeds_many :diff_normalizers, DiffNormalizer embeds_one :namespace_metadata, Metadata - field :create_namespace, :boolean, default: true + field :enforce_namespace, :boolean, default: false + field :create_namespace, :boolean, default: true end embeds_one :kustomize, Kustomize, on_replace: :update do @@ -325,7 +326,7 @@ defmodule Console.Schema.Service do def sync_config_changeset(model, attrs \\ %{}) do model - |> cast(attrs, ~w(create_namespace)a) + |> cast(attrs, ~w(create_namespace enforce_namespace)a) |> cast_embed(:namespace_metadata) |> cast_embed(:diff_normalizers) end diff --git a/lib/console/schema/stack.ex b/lib/console/schema/stack.ex index 82dc210e45..f9ff750c30 100644 --- a/lib/console/schema/stack.ex +++ b/lib/console/schema/stack.ex @@ -16,10 +16,11 @@ defmodule Console.Schema.Stack do ObservableMetric, ScmConnection, Tag, - Project + Project, + StackDefinition } - defenum Type, terraform: 0, ansible: 1 + defenum Type, terraform: 0, ansible: 1, custom: 2 defenum Status, queued: 0, pending: 1, @@ -40,15 +41,15 @@ defmodule Console.Schema.Stack do field :tag, :string embeds_many :hooks, Hook, on_replace: :delete do - field :cmd, :string - field :args, {:array, :string} + field :cmd, :string + field :args, {:array, :string} field :after_stage, Console.Schema.RunStep.Stage end end def changeset(model, attrs \\ %{}) do model - |> cast(attrs, ~w(image version)a) + |> cast(attrs, ~w(image version tag)a) |> cast_embed(:hooks, with: &hook_changeset/2) end @@ -71,6 +72,9 @@ defmodule Console.Schema.Stack do field :manage_state, :boolean, default: false field :workdir, :string field :locked_at, :utc_datetime_usec + field :polled_sha, :string + + field :actor_changed, :boolean, virtual: true field :write_policy_id, :binary_id field :read_policy_id, :binary_id @@ -85,6 +89,7 @@ defmodule Console.Schema.Stack do belongs_to :connection, ScmConnection belongs_to :actor, User belongs_to :project, Project + belongs_to :definition, StackDefinition has_one :state, StackState, on_replace: :update, @@ -141,7 +146,7 @@ defmodule Console.Schema.Stack do def stream(query \\ __MODULE__), do: ordered(query, asc: :id) - @valid ~w(name type paused actor_id workdir manage_state status approval project_id connection_id repository_id cluster_id)a + @valid ~w(name type paused actor_id definition_id workdir manage_state status approval project_id connection_id repository_id cluster_id)a @immutable ~w(project_id)a @@ -163,6 +168,7 @@ defmodule Console.Schema.Stack do |> unique_constraint(:name) |> put_new_change(:write_policy_id, &Ecto.UUID.generate/0) |> put_new_change(:read_policy_id, &Ecto.UUID.generate/0) + |> change_markers(actor_id: :actor_changed) |> validate_required(~w(name type status project_id)a) end diff --git a/lib/console/schema/stack_definition.ex b/lib/console/schema/stack_definition.ex new file mode 100644 index 0000000000..fc146fdcc6 --- /dev/null +++ b/lib/console/schema/stack_definition.ex @@ -0,0 +1,35 @@ +defmodule Console.Schema.StackDefinition do + use Piazza.Ecto.Schema + alias Console.Schema.{Stack, RunStep} + + schema "stack_definitions" do + field :name, :string + field :description, :string + + embeds_one :configuration, Stack.Configuration, on_replace: :update + + embeds_many :steps, Step, on_replace: :delete do + field :stage, RunStep.Stage + field :cmd, :string + field :args, {:array, :string} + field :require_approval, :boolean + end + + timestamps() + end + + @valid ~w(name description)a + + def changeset(model, attrs \\ %{}) do + model + |> cast(attrs, @valid) + |> cast_embed(:configuration) + |> cast_embed(:steps, with: &step_changeset/2) + end + + def step_changeset(model, attrs) do + model + |> cast(attrs, ~w(stage cmd args require_approval)a) + |> validate_required(~w(stage cmd args)a) + end +end diff --git a/priv/repo/migrations/20240628202415_add_stack_definitions.exs b/priv/repo/migrations/20240628202415_add_stack_definitions.exs new file mode 100644 index 0000000000..1526f3d071 --- /dev/null +++ b/priv/repo/migrations/20240628202415_add_stack_definitions.exs @@ -0,0 +1,26 @@ +defmodule Console.Repo.Migrations.AddStackDefinitions do + use Ecto.Migration + + def change do + create table(:stack_definitions, primary_key: false) do + add :id, :uuid, primary_key: true + add :name, :string, nil: false + add :description, :string + + add :configuration, :map + add :steps, :map + + timestamps() + end + + create unique_index(:stack_definitions, [:name]) + + alter table(:stacks) do + add :definition_id, references(:stack_definitions, type: :uuid) + end + + alter table(:run_steps) do + add :require_approval, :boolean + end + end +end diff --git a/priv/repo/migrations/20240630124210_add_polled_sha.exs b/priv/repo/migrations/20240630124210_add_polled_sha.exs new file mode 100644 index 0000000000..8fa57df728 --- /dev/null +++ b/priv/repo/migrations/20240630124210_add_polled_sha.exs @@ -0,0 +1,13 @@ +defmodule Console.Repo.Migrations.AddPolledSha do + use Ecto.Migration + + def change do + alter table(:stacks) do + add :polled_sha, :string + end + + alter table(:pull_requests) do + add :polled_sha, :string + end + end +end diff --git a/schema/schema.graphql b/schema/schema.graphql index dae32d1ac5..06fafcccb6 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -364,6 +364,8 @@ type RootQueryType { infrastructureStack(id: ID!): InfrastructureStack + stackDefinition(id: ID!): StackDefinition + infrastructureStacks(after: String, first: Int, before: String, last: Int, q: String, projectId: ID): InfrastructureStackConnection observabilityProvider(id: ID!): ObservabilityProvider @@ -743,9 +745,18 @@ type RootMutationType { deleteCustomStackRun(id: ID!): CustomStackRun + createStackDefinition(attributes: StackDefinitionAttributes!): StackDefinition + + updateStackDefinition(id: ID!, attributes: StackDefinitionAttributes!): StackDefinition + + deleteStackDefinition(id: ID!): StackDefinition + "Creates a custom run, with the given command list, to execute w\/in the stack's environment" onDemandRun(stackId: ID!, commands: [CommandAttributes], context: Json): StackRun + "start a new run from the newest sha in the stack's run history" + triggerRun(id: ID!): StackRun + upsertObservabilityProvider(attributes: ObservabilityProviderAttributes!): ObservabilityProvider deleteObservabilityProvider(id: ID!): ObservabilityProvider @@ -1020,6 +1031,7 @@ enum StackStatus { enum StackType { TERRAFORM ANSIBLE + CUSTOM } enum StepStatus { @@ -1077,6 +1089,9 @@ input StackAttributes { "id of an scm connection to use for pr callbacks" connectionId: ID + "the id of a stack definition to use" + definitionId: ID + readBindings: [PolicyBindingAttributes] writeBindings: [PolicyBindingAttributes] @@ -1205,6 +1220,20 @@ input CommandAttributes { dir: String } +input StackDefinitionAttributes { + name: String! + description: String + steps: [CustomStepAttributes] + configuration: StackConfigurationAttributes +} + +input CustomStepAttributes { + stage: StepStage + cmd: String! + args: [String] + requireApproval: Boolean +} + type InfrastructureStack { id: ID @@ -1275,6 +1304,9 @@ type InfrastructureStack { "the git repository you're sourcing IaC from" repository: GitRepository + "the stack definition in-use by this stack" + definition: StackDefinition + "the actor of this stack (defaults to root console user)" actor: User @@ -1435,6 +1467,7 @@ type RunStep { name: String! cmd: String! args: [String!] + requireApproval: Boolean index: Int! logs: [RunLogs] insertedAt: DateTime @@ -1510,6 +1543,23 @@ type CustomStackRun { updatedAt: DateTime } +type StackDefinition { + id: ID! + name: String! + description: String + configuration: StackConfiguration! + steps: [CustomRunStep] + insertedAt: DateTime + updatedAt: DateTime +} + +type CustomRunStep { + cmd: String! + args: [String] + stage: StepStage! + requireApproval: Boolean +} + type StackCommand { "the executable to call" cmd: String! @@ -2672,6 +2722,7 @@ input ServiceImportAttributes { input SyncConfigAttributes { createNamespace: Boolean + enforceNamespace: Boolean namespaceMetadata: MetadataAttributes } @@ -3105,6 +3156,9 @@ type SyncConfig { "whether the agent should auto-create the namespace for this service" createNamespace: Boolean + "Whether to require all resources are placed in the same namespace" + enforceNamespace: Boolean + namespaceMetadata: NamespaceMetadata } diff --git a/test/console/deployments/stacks_test.exs b/test/console/deployments/stacks_test.exs index b5f5bb0b8b..6c45d2325a 100644 --- a/test/console/deployments/stacks_test.exs +++ b/test/console/deployments/stacks_test.exs @@ -35,8 +35,8 @@ defmodule Console.Deployments.StacksTest do test "project writers can create a stack" do user = insert(:user) - cluster = insert(:cluster) project = insert(:project, write_bindings: [%{user_id: user.id}]) + cluster = insert(:cluster, project: project) repo = insert(:git_repository) {:ok, stack} = Stacks.create_stack(%{ @@ -58,6 +58,23 @@ defmodule Console.Deployments.StacksTest do assert stack.git.folder == "terraform" end + test "non-cluster writers cannot create a stack" do + user = insert(:user) + project = insert(:project, write_bindings: [%{user_id: user.id}]) + cluster = insert(:cluster) + repo = insert(:git_repository) + + {:error, _} = Stacks.create_stack(%{ + name: "my-stack", + type: :terraform, + approval: true, + repository_id: repo.id, + project_id: project.id, + cluster_id: cluster.id, + git: %{ref: "main", folder: "terraform"}, + }, user) + end + test "random users cannot create" do cluster = insert(:cluster) repo = insert(:git_repository) @@ -187,6 +204,20 @@ defmodule Console.Deployments.StacksTest do assert_receive {:event, %PubSub.StackRunCreated{item: ^run}} end + test "it will not create a new run if there are no file changes" do + stack = insert(:stack, + environment: [%{name: "ENV", value: "1"}], + files: [%{path: "test.txt", content: "test"}], + git: %{ref: "main", folder: "terraform"} + ) + expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end) + expect(Discovery, :changes, fn _, _, _, _ -> {:error, "no changes"} end) + + {:error, _} = Stacks.poll(stack) + + assert refetch(stack).polled_sha == "new-sha" + end + test "it can create a new run with hooks interleaved" do stack = insert(:stack, environment: [%{name: "ENV", value: "1"}], @@ -239,6 +270,50 @@ defmodule Console.Deployments.StacksTest do assert_receive {:event, %PubSub.StackRunCreated{item: ^run}} end + test "it can create a new run with a stack definition" do + definition = insert(:stack_definition, steps: [ + %{cmd: "echo", args: ["hello world"], stage: :plan}, + %{cmd: "sleep", args: ["100"], stage: :apply} + ]) + stack = insert(:stack, + definition: definition, + type: :custom, + environment: [%{name: "ENV", value: "1"}], + files: [%{path: "test.txt", content: "test"}], + git: %{ref: "main", folder: "terraform"} + ) + expect(Discovery, :sha, fn _, _ -> {:ok, "new-sha"} end) + expect(Discovery, :changes, fn _, _, _, _ -> {:ok, ["terraform/main.tf"], "a commit message"} end) + + {:ok, run} = Stacks.poll(stack) + + assert run.stack_id == stack.id + assert run.status == :queued + assert run.message == "a commit message" + assert run.cluster_id == stack.cluster_id + assert run.repository_id == stack.repository_id + assert run.git.ref == "new-sha" + assert run.git.folder == stack.git.folder + [first, second] = run.steps + + assert first.cmd == "echo" + assert first.args == ["hello world"] + assert first.index == 0 + assert first.stage == :plan + + assert second.cmd == "sleep" + assert second.args == ["100"] + assert second.index == 1 + assert second.stage == :apply + + stack = refetch(stack) + assert stack.sha == "new-sha" + %{environment: [_], files: [_]} = Console.Repo.preload(stack, [:environment, :files]) + + [_] = StackRun.for_stack(stack.id) |> Console.Repo.all() + assert_receive {:event, %PubSub.StackRunCreated{item: ^run}} + end + test "it can create a new run from a pr if the sha changes" do stack = insert(:stack, environment: [%{name: "ENV", value: "1"}], @@ -383,6 +458,24 @@ defmodule Console.Deployments.StacksTest do end end + describe "#trigger_run/2" do + test "admins can trigger a new run for a stack" do + stack = insert(:stack, git: %{ref: "main", folder: "terraform"}) + run = insert(:stack_run, stack: stack, git: %{ref: "some-sha"}) + + {:ok, new_run} = Stacks.trigger_run(stack.id, admin_user()) + + assert new_run.git.ref == run.git.ref + end + + test "non-admins cannot trigger" do + stack = insert(:stack) + insert(:stack_run, stack: stack, git: %{ref: "some-sha"}) + + {:error, _} = Stacks.trigger_run(stack.id, insert(:user)) + end + end + describe "#dequeue/1" do test "tries to dequeue the next wet run of the stack" do stack = insert(:stack) @@ -699,6 +792,70 @@ defmodule Console.Deployments.StacksTest do {:error, _} = Stacks.delete_custom_stack_run(csr.id, insert(:user)) end end + + describe "#create_stack_definition/2" do + test "admins can create a stack definition" do + {:ok, def} = Stacks.create_stack_definition(%{ + name: "custom", + steps: [ + %{cmd: "some", args: ["arg"], stage: :apply} + ], + configuration: %{image: "some/image", tag: "0.1.0"} + }, admin_user()) + + assert def.name == "custom" + assert hd(def.steps).cmd == "some" + assert hd(def.steps).args == ["arg"] + assert hd(def.steps).stage == :apply + + assert def.configuration.image == "some/image" + assert def.configuration.tag == "0.1.0" + end + + test "non-admins cannot create" do + {:error, _} = Stacks.create_stack_definition(%{ + name: "custom", + steps: [ + %{cmd: "some", args: ["arg"], stage: :apply} + ], + configuration: %{image: "some/image", tag: "0.1.0"} + }, insert(:user)) + end + end + + describe "#update_stack_definition/3" do + test "admins can update a stack definition" do + def = insert(:stack_definition) + + {:ok, updated} = Stacks.update_stack_definition(%{description: "something"}, def.id, admin_user()) + + assert updated.id == def.id + assert updated.description == "something" + end + + test "non-admins cannot update a stack definition" do + def = insert(:stack_definition) + + {:error, _} = Stacks.update_stack_definition(%{description: "something"}, def.id, insert(:user)) + end + end + + describe "#delete_stack_definition/2" do + test "admins can delete a stack definition" do + def = insert(:stack_definition) + + {:ok, deleted} = Stacks.delete_stack_definition(def.id, admin_user()) + + assert deleted.id == def.id + refute refetch(deleted) + end + + test "non-admins cannot delete a stack definition" do + def = insert(:stack_definition) + + {:error, _} = Stacks.delete_stack_definition(def.id, insert(:user)) + end + end end defmodule Console.Deployments.StacksSyncTest do diff --git a/test/console/graphql/mutations/deployments/stack_mutations_test.exs b/test/console/graphql/mutations/deployments/stack_mutations_test.exs index 8797b40f12..f539621595 100644 --- a/test/console/graphql/mutations/deployments/stack_mutations_test.exs +++ b/test/console/graphql/mutations/deployments/stack_mutations_test.exs @@ -352,6 +352,91 @@ defmodule Console.GraphQl.Deployments.StackMutationsTest do end end + describe "createStackDefinition" do + test "admins can create a custom stack run" do + {:ok, %{data: %{"createStackDefinition" => def}}} = run_query(""" + mutation create($attrs: StackDefinitionAttributes!) { + createStackDefinition(attributes: $attrs) { + id + name + configuration { image tag } + steps { cmd args stage } + } + } + """, %{"attrs" => %{ + "name" => "test", + "configuration" => %{"image" => "some/image", "tag" => "0.1.0"}, + "steps" => [%{"cmd" => "echo", "args" => ["Hello World!"], "stage" => "APPLY"}] + }}, %{current_user: admin_user()}) + + assert def["name"] == "test" + assert def["configuration"]["image"] == "some/image" + assert def["configuration"]["tag"] == "0.1.0" + [cmd] = def["steps"] + assert cmd["cmd"] == "echo" + assert cmd["args"] == ["Hello World!"] + assert cmd["stage"] == "APPLY" + end + end + + describe "updateStackDefinition" do + test "admins can update a custom stack run" do + def = insert(:stack_definition) + {:ok, %{data: %{"updateStackDefinition" => def}}} = run_query(""" + mutation update($id: ID!, $attrs: StackDefinitionAttributes!) { + updateStackDefinition(id: $id, attributes: $attrs) { + id + name + steps { cmd args stage } + } + } + """, %{"id" => def.id, "attrs" => %{ + "name" => "test", + "steps" => [%{"cmd" => "echo", "args" => ["Hello World!"], "stage" => "APPLY"}] + }}, %{current_user: admin_user()}) + + assert def["name"] == "test" + [cmd] = def["steps"] + assert cmd["cmd"] == "echo" + assert cmd["args"] == ["Hello World!"] + assert cmd["stage"] == "APPLY" + end + end + + describe "deleteStackDefinition" do + test "it can delete a csr" do + def = insert(:stack_definition) + {:ok, %{data: %{"deleteStackDefinition" => found}}} = run_query(""" + mutation Delete($id: ID!) { + deleteStackDefinition(id: $id) { id } + } + """, %{"id" => def.id}, %{current_user: admin_user()}) + + assert found["id"] == def.id + + refute refetch(def) + end + end + + describe "triggerRun" do + test "admins can trigger a new run for a stack" do + stack = insert(:stack, git: %{ref: "main", folder: "terraform"}) + run = insert(:stack_run, stack: stack, git: %{ref: "some-sha"}) + + {:ok, %{data: %{"triggerRun" => triggered}}} = run_query(""" + mutation Trigger($id: ID!) { + triggerRun(id: $id) { + id + git { ref } + } + } + """, %{"id" => stack.id}, %{current_user: admin_user()}) + + assert triggered["id"] + assert triggered["git"]["ref"] == run.git.ref + end + end + describe "onDemandRun" do test "it can create a custom run" do stack = insert(:stack, sha: "test-sha") diff --git a/test/support/factory.ex b/test/support/factory.ex index 2f622bfafa..5ce9e8597f 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -616,6 +616,14 @@ defmodule Console.Factory do } end + def stack_definition_factory do + %Schema.StackDefinition{ + name: sequence(:def, & "stack-def-#{&1}"), + configuration: %{image: "stack/harness", tag: "0.1.0"}, + steps: [%{cmd: "cmd", args: ["arg"], stage: :apply}] + } + end + def setup_rbac(user, repos \\ ["*"], perms) do role = insert(:role, repositories: repos, permissions: Map.new(perms)) insert(:role_binding, role: role, user: user)