Merge pull request #205 from dbt-labs/job-add-draft-pr-update-docs

Job add draft pr update docs
dbt-labs · Oct 11, 2023 · be39ad4 · be39ad4
2 parents 8c686c1 + 75931cb
commit be39ad4
Show file tree

Hide file tree

Showing 20 changed files with 251 additions and 170 deletions.
diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
@@ -37,4 +37,3 @@ jobs:
           DBT_CLOUD_ACCOUNT_ID: ${{ secrets.TEST_DBT_CLOUD_ACCOUNT_ID }}
           DBT_CLOUD_TOKEN: ${{ secrets.TEST_DBT_CLOUD_TOKEN }}
           DBT_CLOUD_HOST_URL: ${{ secrets.TEST_DBT_CLOUD_HOST_URL }}
-          DBT_LEGACY_JOB_DEFERRAL: 1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,8 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.10...HEAD)
+## [Unreleased](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.11...HEAD)
 
+## [0.2.11](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.10...v0.2.11)
+
+## Changes
+
+- Update docs and examples for jobs and add the ability to set/unset running CI jobs on Draft PRs
 
 ## [0.2.10](https://github.com/dbt-labs/terraform-provider-dbtcloud/compare/v0.2.9...v0.2.10)
 

diff --git a/Makefile b/Makefile
@@ -21,7 +21,7 @@ test: deps
 	go test -mod=readonly -count=1 ./...
 
 test-acceptance: deps
-	TF_ACC=1 TEST_DATABRICKS=false go test -v -mod=readonly -count=1 ./...
+	TF_ACC=1 go test -v -mod=readonly -count=1 ./...
 
 check-docs: docs
 	git diff --exit-code -- docs

diff --git a/docs/data-sources/job.md b/docs/data-sources/job.md
@@ -31,5 +31,6 @@ description: |-
 - `self_deferring` (Boolean) Whether this job defers on a previous run of itself (overrides value in deferring_job_id)
 - `timeout_seconds` (Number) Number of seconds before the job times out
 - `triggers` (Map of Boolean) Flags for which types of triggers to use, keys of github_webhook, git_provider_webhook, schedule, custom_branch_only
+- `triggers_on_draft_pr` (Boolean) Whether the CI job should be automatically triggered on draft PRs
 
 
diff --git a/docs/data-sources/project.md b/docs/data-sources/project.md
@@ -16,16 +16,24 @@ description: |-
 // use dbt_cloud_project instead of dbtcloud_project for the legacy resource names
 // legacy names will be removed from 0.3 onwards
 
+// projects data sources can use the project_id parameter (preferred uniqueness is ensured)
 data "dbtcloud_project" "test_project" {
   project_id = var.dbt_cloud_project_id
 }
+
+// or they can use project names
+// the provider will raise an error if more than one project is found with the same name
+data "dbtcloud_project" "test_project" {
+  name = "My project name"
+}
 ```
 
 <!-- schema generated by tfplugindocs -->
 ## Schema
 
-### Required
+### Optional
 
+- `name` (String) Given name for project
 - `project_id` (Number) ID of the project to represent
 
 ### Read-Only
@@ -34,7 +42,6 @@ data "dbtcloud_project" "test_project" {
 - `docs_job_id` (Number) ID of Job for the documentation
 - `freshness_job_id` (Number) ID of Job for source freshness
 - `id` (String) The ID of this resource.
-- `name` (String) Given name for project
 - `repository_id` (Number) ID of the repository associated with the project
 - `state` (Number) Project state should be 1 = active, as 2 = deleted
 

diff --git a/docs/resources/job.md b/docs/resources/job.md
@@ -7,7 +7,7 @@ description: |-
 
 # dbtcloud_job (Resource)
 
-~> As of September 2023, some CI improvements are being rolled out to dbt Cloud with minor impacts to some jobs:  [more info](https://docs.getdbt.com/docs/dbt-versions/release-notes/june-2023/ci-updates-phase1-rn). 
+~> As of October 2023, CI improvements have been rolled out to dbt Cloud with minor impacts to some jobs:  [more info](https://docs.getdbt.com/docs/dbt-versions/release-notes/june-2023/ci-updates-phase1-rn). 
 <br/>
 <br/>
 Those improvements include modifications to deferral which was historically set at the job level and will now be set at the environment level. 
@@ -19,25 +19,54 @@ Deferral can still be set to "self" by setting `self_deferring` to `true` but wi
 // use dbt_cloud_job instead of dbtcloud_job for the legacy resource names
 // legacy names will be removed from 0.3 onwards
 
+# a job that has github_webhook and git_provider_webhook 
+# set to false will be categorized as a "Deploy Job"
 resource "dbtcloud_job" "test" {
   environment_id = var.dbt_cloud_environment_id
   execute_steps = [
-    "dbt test"
+    "dbt build"
   ]
-  generate_docs        = false
+  generate_docs        = true
   is_active            = true
-  name                 = "Test"
+  name                 = "Daily job"
   num_threads          = 64
   project_id           = data.dbtcloud_project.test_project.id
-  run_generate_sources = false
+  run_generate_sources = true
   target_name          = "default"
   triggers = {
-    "custom_branch_only" : true,
+    "custom_branch_only" : false,
     "github_webhook" : false,
     "git_provider_webhook" : false,
+    "schedule" : true
+  }
+  # this is the default that gets set up when modifying jobs in the UI
+  schedule_days = [0, 1, 2, 3, 4, 5, 6]
+  schedule_type = "days_of_week"
+  schedule_hours = [0]
+}
+
+
+# a job that has github_webhook and git_provider_webhook set 
+# to true will be categorized as a "Continuous Integration Job"
+resource "dbtcloud_job" "ci_job" {
+  environment_id = var.my_ci_environment_id
+  execute_steps = [
+    "dbt build -s state:modified+ --fail-fast"
+  ]
+  generate_docs            = false
+  deferring_environment_id = dbtcloud_environment.my_prod_env.environment_id
+  name                     = "CI Job"
+  num_threads              = 32
+  project_id               = data.dbtcloud_project.test_project.id
+  run_generate_sources     = false
+  triggers = {
+    "custom_branch_only" : true,
+    "github_webhook" : true,
+    "git_provider_webhook" : true,
     "schedule" : false
   }
   # this is the default that gets set up when modifying jobs in the UI
+  # this is not going to be used when schedule is set to false
   schedule_days = [0, 1, 2, 3, 4, 5, 6]
   schedule_type = "days_of_week"
 }
@@ -52,7 +81,7 @@ resource "dbtcloud_job" "test" {
 - `execute_steps` (List of String) List of commands to execute for the job
 - `name` (String) Job name
 - `project_id` (Number) Project ID to create the job in
-- `triggers` (Map of Boolean) Flags for which types of triggers to use, keys of github_webhook, git_provider_webhook, schedule, custom_branch_only
+- `triggers` (Map of Boolean) Flags for which types of triggers to use, possible values are `github_webhook`, `git_provider_webhook`, `schedule` and `custom_branch_only`. <br>`custom_branch_only` is only relevant for CI jobs triggered automatically on PR creation to only trigger a job on a PR to the custom branch of the environment.
 
 ### Optional
 
@@ -72,6 +101,7 @@ resource "dbtcloud_job" "test" {
 - `self_deferring` (Boolean) Whether this job defers on a previous run of itself
 - `target_name` (String) Target name for the dbt profile
 - `timeout_seconds` (Number) Number of seconds to allow the job to run before timing out
+- `triggers_on_draft_pr` (Boolean) Whether the CI job should be automatically triggered on draft PRs
 
 ### Read-Only
 

diff --git a/examples/resources/dbtcloud_job/resource.tf b/examples/resources/dbtcloud_job/resource.tf
@@ -1,25 +1,54 @@
 // use dbt_cloud_job instead of dbtcloud_job for the legacy resource names
 // legacy names will be removed from 0.3 onwards
 
+# a job that has github_webhook and git_provider_webhook 
+# set to false will be categorized as a "Deploy Job"
 resource "dbtcloud_job" "test" {
   environment_id = var.dbt_cloud_environment_id
   execute_steps = [
-    "dbt test"
+    "dbt build"
   ]
-  generate_docs        = false
+  generate_docs        = true
   is_active            = true
-  name                 = "Test"
+  name                 = "Daily job"
   num_threads          = 64
   project_id           = data.dbtcloud_project.test_project.id
-  run_generate_sources = false
+  run_generate_sources = true
   target_name          = "default"
   triggers = {
-    "custom_branch_only" : true,
+    "custom_branch_only" : false,
     "github_webhook" : false,
     "git_provider_webhook" : false,
-    "schedule" : false
+    "schedule" : true
   }
   # this is the default that gets set up when modifying jobs in the UI
   schedule_days = [0, 1, 2, 3, 4, 5, 6]
   schedule_type = "days_of_week"
+  schedule_hours = [0]
 }
+
+
+# a job that has github_webhook and git_provider_webhook set 
+# to true will be categorized as a "Continuous Integration Job"
+resource "dbtcloud_job" "ci_job" {
+  environment_id = var.my_ci_environment_id
+  execute_steps = [
+    "dbt build -s state:modified+ --fail-fast"
+  ]
+  generate_docs            = false
+  deferring_environment_id = dbtcloud_environment.my_prod_env.environment_id
+  name                     = "CI Job"
+  num_threads              = 32
+  project_id               = data.dbtcloud_project.test_project.id
+  run_generate_sources     = false
+  triggers = {
+    "custom_branch_only" : true,
+    "github_webhook" : true,
+    "git_provider_webhook" : true,
+    "schedule" : false
+  }
+  # this is the default that gets set up when modifying jobs in the UI
+  # this is not going to be used when schedule is set to false
+  schedule_days = [0, 1, 2, 3, 4, 5, 6]
+  schedule_type = "days_of_week"
+}
diff --git a/pkg/data_sources/databricks_credential_acceptance_test.go b/pkg/data_sources/databricks_credential_acceptance_test.go
@@ -2,7 +2,6 @@ package data_sources_test
 
 import (
 	"fmt"
-	"os"
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
@@ -11,42 +10,20 @@ import (
 
 func TestAccDbtCloudDatabricksCredentialDataSource(t *testing.T) {
 
-	testDatabricks := os.Getenv("TEST_DATABRICKS")
-
-	var adapterType string
-	if testDatabricks == "true" {
-		adapterType = "databricks"
-	} else {
-		adapterType = "spark"
-	}
 	randomProjectName := acctest.RandStringFromCharSet(5, acctest.CharSetAlphaNum)
-	config := databricks_credential(randomProjectName, "moo", "baa", "maa", 64, adapterType)
+	config := databricks_credential(randomProjectName, "moo", "baa", "maa", 64)
 
-	// TODO: revisit when adapters can be created with a service token
-	// as of now, CI is using a spark adapter and doesn't have a catalog
-	// TEST_DATABRICKS is not set in CI
 	var check resource.TestCheckFunc
 
-	if testDatabricks == "true" {
-		check = resource.ComposeAggregateTestCheckFunc(
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "credential_id"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "project_id"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "adapter_id"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "target_name"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "schema"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "num_threads"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "catalog"),
-		)
-	} else {
-		check = resource.ComposeAggregateTestCheckFunc(
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "credential_id"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "project_id"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "adapter_id"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "target_name"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "schema"),
-			resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "num_threads"),
-		)
-	}
+	check = resource.ComposeAggregateTestCheckFunc(
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "credential_id"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "project_id"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "adapter_id"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "target_name"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "schema"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "num_threads"),
+		resource.TestCheckResourceAttrSet("data.dbtcloud_databricks_credential.test", "catalog"),
+	)
 
 	resource.ParallelTest(t, resource.TestCase{
 		Providers: providers(),
@@ -59,41 +36,36 @@ func TestAccDbtCloudDatabricksCredentialDataSource(t *testing.T) {
 	})
 }
 
-// TODO: revisit when adapters can be created with a service token
-// In CI, the Adapter 123 is of type "spark", but locally, for me it is databricks
-// We can't create adapters right now with service tokens but should revisit when this is updated
-
-func databricks_credential(projectName string, defaultSchema string, username string, password string, numThreads int, adapterType string) string {
-	commonConfig := fmt.Sprintf(`
+func databricks_credential(projectName string, defaultSchema string, username string, password string, numThreads int) string {
+	return fmt.Sprintf(`
     resource "dbtcloud_project" "test_credential_project" {
         name = "%s"
     }
 
+	resource "dbtcloud_connection" "databricks" {
+		project_id = dbtcloud_project.test_credential_project.id
+		type       = "adapter"
+		name       = "Databricks"
+		database   = ""
+		host_name  = "databricks.com"
+		http_path  = "/my/path"
+		catalog    = "moo"
+	  }
+
     data "dbtcloud_databricks_credential" "test" {
         project_id = dbtcloud_project.test_credential_project.id
         credential_id = dbtcloud_databricks_credential.test_cred.credential_id
     }
-    `, projectName)
-
-	if adapterType == "databricks" {
-		credential := `resource "dbtcloud_databricks_credential" "test_cred" {
-			project_id = dbtcloud_project.test_credential_project.id
-			adapter_id = 123
-			token = "abcdefg"
-			schema = "my_schema"
-			adapter_type = "databricks"
-			catalog = "my_catalog"
-		}`
+    
 
-		return fmt.Sprintln(commonConfig, credential)
-	} else {
-		credential := `resource "dbtcloud_databricks_credential" "test_cred" {
-			project_id = dbtcloud_project.test_credential_project.id
-			adapter_id = 123
-			token = "abcdefg"
-			schema = "my_schema"
-			adapter_type = "spark"
-		}`
-		return fmt.Sprintln(commonConfig, credential)
+	resource "dbtcloud_databricks_credential" "test_cred" {
+		project_id = dbtcloud_project.test_credential_project.id
+		adapter_id = dbtcloud_connection.databricks.adapter_id
+		token = "abcdefg"
+		schema = "my_schema"
+		adapter_type = "databricks"
+		catalog = "my_catalog"
 	}
+	`, projectName)
+
 }
diff --git a/pkg/data_sources/job.go b/pkg/data_sources/job.go
@@ -66,6 +66,11 @@ var jobSchema = map[string]*schema.Schema{
 		Computed:    true,
 		Description: "Number of seconds before the job times out",
 	},
+	"triggers_on_draft_pr": &schema.Schema{
+		Type:        schema.TypeBool,
+		Computed:    true,
+		Description: "Whether the CI job should be automatically triggered on draft PRs",
+	},
 }
 
 func DatasourceJob() *schema.Resource {
@@ -123,6 +128,9 @@ func datasourceJobRead(ctx context.Context, d *schema.ResourceData, m interface{
 	if err := d.Set("triggers", triggers); err != nil {
 		return diag.FromErr(err)
 	}
+	if err := d.Set("triggers_on_draft_pr", job.TriggersOnDraftPR); err != nil {
+		return diag.FromErr(err)
+	}
 
 	d.SetId(jobId)
 

diff --git a/pkg/data_sources/job_acceptance_test.go b/pkg/data_sources/job_acceptance_test.go
@@ -20,6 +20,7 @@ func TestDbtCloudJobDataSource(t *testing.T) {
 		resource.TestCheckResourceAttrSet("data.dbtcloud_job.test", "environment_id"),
 		resource.TestCheckResourceAttr("data.dbtcloud_job.test", "name", randomJobName),
 		resource.TestCheckResourceAttr("data.dbtcloud_job.test", "timeout_seconds", "180"),
+		resource.TestCheckResourceAttr("data.dbtcloud_job.test", "triggers_on_draft_pr", "false"),
 	)
 
 	resource.ParallelTest(t, resource.TestCase{