From b9d9530d2b9685e52150ca1e2e944c920957fa49 Mon Sep 17 00:00:00 2001 From: motatoes Date: Mon, 30 Sep 2024 17:59:57 +0100 Subject: [PATCH] support handling job status update --- ee/drift/controllers/ci_jobs.go | 138 ++++++++++++++++++ ee/drift/controllers/drift.go | 12 +- ee/drift/dbmodels/ci_jobs.go | 136 +++++++++++++++++ ee/drift/dbmodels/projects.go | 18 +++ ee/drift/dbmodels/tokens.go | 15 ++ ee/drift/main.go | 2 + ee/drift/middleware/job_token.go | 70 +++++++++ ee/drift/middleware/middleware.go | 1 + ee/drift/model/digger_ci_jobs.gen.go | 2 + ee/drift/model/projects.gen.go | 2 +- .../models_generated/digger_ci_jobs.gen.go | 10 +- ee/drift/models_generated/projects.gen.go | 8 +- 12 files changed, 406 insertions(+), 8 deletions(-) create mode 100644 ee/drift/controllers/ci_jobs.go create mode 100644 ee/drift/dbmodels/ci_jobs.go create mode 100644 ee/drift/middleware/job_token.go diff --git a/ee/drift/controllers/ci_jobs.go b/ee/drift/controllers/ci_jobs.go new file mode 100644 index 000000000..41d9d0a94 --- /dev/null +++ b/ee/drift/controllers/ci_jobs.go @@ -0,0 +1,138 @@ +package controllers + +import ( + "github.com/diggerhq/digger/ee/drift/dbmodels" + "github.com/diggerhq/digger/ee/drift/model" + "github.com/diggerhq/digger/libs/terraform_utils" + "github.com/gin-gonic/gin" + "log" + "net/http" + "time" +) + +type SetJobStatusRequest struct { + Status string `json:"status"` + Timestamp time.Time `json:"timestamp"` + JobSummary *terraform_utils.TerraformSummary `json:"job_summary"` + Footprint *terraform_utils.TerraformPlanFootprint `json:"job_plan_footprint"` + PrCommentUrl string `json:"pr_comment_url"` + TerraformOutput string `json:"terraform_output""` +} + +func (mc MainController) SetJobStatusForProject(c *gin.Context) { + jobId := c.Param("jobId") + //orgId, exists := c.Get(middleware.ORGANISATION_ID_KEY) + + //if !exists { + // c.String(http.StatusForbidden, "Not allowed to access this resource") + // return + //} + + var request SetJobStatusRequest + + err := c.BindJSON(&request) + + if err != nil { + log.Printf("Error binding JSON: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error binding JSON"}) + return + } + + log.Printf("settings tatus for job: %v, new status: %v, tfout: %v, job summary: %v", jobId, request.Status, request.TerraformOutput, request.JobSummary) + + job, err := dbmodels.DB.GetDiggerCiJob(jobId) + if err != nil { + log.Printf("could not get digger ci job, err: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting digger job"}) + return + } + + switch request.Status { + case string(dbmodels.DiggerJobStarted): + job.Status = string(dbmodels.DiggerJobStarted) + err := dbmodels.DB.UpdateDiggerJob(job) + if err != nil { + log.Printf("Error updating job status: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error updating job status"}) + return + } + case string(dbmodels.DiggerJobSucceeded): + job.Status = string(dbmodels.DiggerJobSucceeded) + job.TerraformOutput = request.TerraformOutput + job.ResourcesCreated = int32(request.JobSummary.ResourcesCreated) + job.ResourcesUpdated = int32(request.JobSummary.ResourcesUpdated) + job.ResourcesDeleted = int32(request.JobSummary.ResourcesDeleted) + err := dbmodels.DB.UpdateDiggerJob(job) + if err != nil { + log.Printf("Error updating job status: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error updating job status"}) + return + } + + project, err := dbmodels.DB.GetProjectById(job.ProjectID) + if err != nil { + log.Printf("Error retriving project: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error retrieving project"}) + return + + } + err = ProjectDriftStateMachineApply(*project, job.TerraformOutput, job.ResourcesCreated, job.ResourcesUpdated, job.ResourcesDeleted) + if err != nil { + log.Printf("error while checking drifted project") + } + + case string(dbmodels.DiggerJobFailed): + job.Status = string(dbmodels.DiggerJobFailed) + job.TerraformOutput = request.TerraformOutput + err := dbmodels.DB.UpdateDiggerJob(job) + if err != nil { + log.Printf("Error updating job status: %v", request.Status) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error saving job"}) + return + } + + default: + log.Printf("Unexpected status %v", request.Status) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error saving job"}) + return + } + + job.UpdatedAt = request.Timestamp + err = dbmodels.DB.GormDB.Save(job).Error + if err != nil { + log.Printf("Error saving update job: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error saving job"}) + return + } + + c.JSON(http.StatusOK, gin.H{}) +} + +func ProjectDriftStateMachineApply(project model.Project, tfplan string, resourcesCreated int32, resourcesUpdated int32, resourcesDeleted int32) error { + isEmptyPlan := resourcesCreated == 0 && resourcesUpdated == 0 && resourcesDeleted == 0 + wasEmptyPlan := project.ToCreate == 0 && project.ToUpdate == 0 && project.ToDelete == 0 + if isEmptyPlan { + project.DriftStatus = dbmodels.DriftStatusNoDrift + } + if !isEmptyPlan && wasEmptyPlan { + project.DriftStatus = dbmodels.DriftStatusNewDrift + } + if !isEmptyPlan && !wasEmptyPlan { + if project.DriftTerraformPlan != tfplan { + if project.IsAcknowledged { + project.DriftStatus = dbmodels.DriftStatusNewDrift + } + } + } + + project.DriftTerraformPlan = tfplan + project.ToCreate = resourcesCreated + project.ToUpdate = resourcesUpdated + project.ToDelete = resourcesDeleted + result := dbmodels.DB.GormDB.Save(&project) + if result.Error != nil { + return result.Error + } + log.Printf("project %v, (name: %v) has been updated successfully\n", project.ID, project.Name) + return nil +} diff --git a/ee/drift/controllers/drift.go b/ee/drift/controllers/drift.go index ab5b5f9d2..60ab7b048 100644 --- a/ee/drift/controllers/drift.go +++ b/ee/drift/controllers/drift.go @@ -104,7 +104,8 @@ func (mc MainController) TriggerDriftRunForProject(c *gin.Context) { CommentId: "", Job: jobSpec, Reporter: spec.ReporterSpec{ - ReportingStrategy: "noop", + ReportingStrategy: "noop", + ReportTerraformOutput: true, }, Lock: spec.LockSpec{ LockType: "noop", @@ -128,7 +129,7 @@ func (mc MainController) TriggerDriftRunForProject(c *gin.Context) { PolicyType: "http", }, CommentUpdater: spec.CommentUpdaterSpec{ - CommentUpdaterType: dg_configuration.CommentRenderModeBasic, + CommentUpdaterType: "noop", }, } @@ -163,6 +164,13 @@ func (mc MainController) TriggerDriftRunForProject(c *gin.Context) { } + _, err = dbmodels.DB.CreateCiJobFromSpec(spec, *runName, project.ID) + if err != nil { + log.Printf("error creating the job: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error creating job entry")}) + return + } + err = ciBackend.TriggerWorkflow(spec, *runName, *vcsToken) if err != nil { log.Printf("TriggerWorkflow err: %v\n", err) diff --git a/ee/drift/dbmodels/ci_jobs.go b/ee/drift/dbmodels/ci_jobs.go new file mode 100644 index 000000000..0790558a2 --- /dev/null +++ b/ee/drift/dbmodels/ci_jobs.go @@ -0,0 +1,136 @@ +package dbmodels + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/diggerhq/digger/ee/drift/model" + "github.com/diggerhq/digger/libs/spec" + "github.com/google/uuid" + "gorm.io/gorm" + "log" + "time" +) + +type DiggerJobStatus string + +const ( + DiggerJobCreated DiggerJobStatus = "created" + DiggerJobTriggered DiggerJobStatus = "triggered" + DiggerJobFailed DiggerJobStatus = "failed" + DiggerJobStarted DiggerJobStatus = "started" + DiggerJobSucceeded DiggerJobStatus = "succeeded" + DiggerJobQueuedForRun DiggerJobStatus = "queued" +) + +func (db *Database) GetDiggerCiJob(diggerJobId string) (*model.DiggerCiJob, error) { + log.Printf("GetDiggerCiJob, diggerJobId: %v", diggerJobId) + var ciJob model.DiggerCiJob + + err := db.GormDB.Where("digger_job_id = ?", diggerJobId).First(&ciJob).Error + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("ci job not found") + } + log.Printf("Unknown error occurred while fetching database, %v\n", err) + return nil, err + } + + return &ciJob, nil +} + +func (db *Database) UpdateDiggerJob(job *model.DiggerCiJob) error { + result := db.GormDB.Save(job) + if result.Error != nil { + return result.Error + } + log.Printf("DiggerJob %v, (id: %v) has been updated successfully\n", job.DiggerJobID, job.ID) + return nil +} + +func (db *Database) CreateCiJobFromSpec(spec spec.Spec, runName string, projectId string) (*model.DiggerCiJob, error) { + + marshalledJobSpec, err := json.Marshal(spec.Job) + if err != nil { + log.Printf("failed to marshal job: %v", err) + return nil, err + } + + marshalledReporterSpec, err := json.Marshal(spec.Reporter) + if err != nil { + log.Printf("failed to marshal reporter: %v", err) + return nil, err + } + + marshalledCommentUpdaterSpec, err := json.Marshal(spec.CommentUpdater) + if err != nil { + log.Printf("failed to marshal comment updater: %v", err) + return nil, err + } + + marshalledLockSpec, err := json.Marshal(spec.Lock) + if err != nil { + log.Printf("failed to marshal lockspec: %v", err) + return nil, err + } + + marshalledBackendSpec, err := json.Marshal(spec.Backend) + if err != nil { + log.Printf("failed to marshal backend spec: %v", err) + return nil, err + + } + + marshalledVcsSpec, err := json.Marshal(spec.VCS) + if err != nil { + log.Printf("failed to marshal vcs spec: %v", err) + return nil, err + } + + marshalledPolicySpec, err := json.Marshal(spec.Policy) + if err != nil { + log.Printf("failed to marshal policy spec: %v", err) + return nil, err + } + + marshalledVariablesSpec, err := json.Marshal(spec.Variables) + if err != nil { + log.Printf("failed to marshal variables spec: %v", err) + return nil, err + } + + job := &model.DiggerCiJob{ + ID: uuid.NewString(), + DiggerJobID: spec.JobId, + Spectype: string(spec.SpecType), + Commentid: spec.CommentId, + Runname: runName, + Jobspec: marshalledJobSpec, + Reporterspec: marshalledReporterSpec, + Commentupdaterspec: marshalledCommentUpdaterSpec, + Lockspec: marshalledLockSpec, + Backendspec: marshalledBackendSpec, + Vcsspec: marshalledVcsSpec, + Policyspec: marshalledPolicySpec, + Variablesspec: marshalledVariablesSpec, + CreatedAt: time.Time{}, + UpdatedAt: time.Time{}, + DeletedAt: gorm.DeletedAt{}, + WorkflowFile: spec.VCS.WorkflowFile, + WorkflowURL: "", + Status: string(DiggerJobCreated), + ResourcesCreated: 0, + ResourcesUpdated: 0, + ResourcesDeleted: 0, + ProjectID: projectId, + } + + err = db.GormDB.Create(job).Error + if err != nil { + log.Printf("failed to create job: %v", err) + return nil, err + } + + return job, nil +} diff --git a/ee/drift/dbmodels/projects.go b/ee/drift/dbmodels/projects.go index c7ceaf344..195829e4b 100644 --- a/ee/drift/dbmodels/projects.go +++ b/ee/drift/dbmodels/projects.go @@ -2,6 +2,7 @@ package dbmodels import ( "errors" + "fmt" "github.com/diggerhq/digger/ee/drift/model" "gorm.io/gorm" "log" @@ -13,6 +14,23 @@ var DriftStatusNewDrift = "new drift" var DriftStatusNoDrift = "no drift" var DriftStatusAcknowledgeDrift = "acknowledged drift" +func (db *Database) GetProjectById(projectId string) (*model.Project, error) { + log.Printf("GetProjectById, projectId: %v\n", projectId) + var project model.Project + + err := db.GormDB.Where("id = ?", projectId).First(&project).Error + + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, fmt.Errorf("could not find project") + } + log.Printf("Unknown error occurred while fetching database, %v\n", err) + return nil, err + } + + return &project, nil +} + // GetProjectByName return project for specified org and repo // if record doesn't exist return nil func (db *Database) GetProjectByName(orgId any, repo *model.Repo, name string) (*model.Project, error) { diff --git a/ee/drift/dbmodels/tokens.go b/ee/drift/dbmodels/tokens.go index 15b68b24e..d2e2b7c07 100644 --- a/ee/drift/dbmodels/tokens.go +++ b/ee/drift/dbmodels/tokens.go @@ -1,8 +1,10 @@ package dbmodels import ( + "errors" "github.com/diggerhq/digger/ee/drift/model" "github.com/google/uuid" + "gorm.io/gorm" "log" "time" ) @@ -32,3 +34,16 @@ func (db *Database) CreateDiggerJobToken(organisationId string) (*model.DiggerCi } return jobToken, nil } + +func (db *Database) GetJobToken(tenantId any) (*model.DiggerCiJobToken, error) { + token := &model.DiggerCiJobToken{} + result := db.GormDB.Take(token, "value = ?", tenantId) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, nil + } else { + return nil, result.Error + } + } + return token, nil +} diff --git a/ee/drift/main.go b/ee/drift/main.go index 3bf77949a..065339cec 100644 --- a/ee/drift/main.go +++ b/ee/drift/main.go @@ -74,6 +74,8 @@ func main() { r.POST("github-app-webhook", controller.GithubAppWebHook) r.GET("/github/callback_fe", middleware.WebhookAuth(), controller.GithubAppCallbackPage) + r.POST("/repos/:repo/projects/:projectName/jobs/:jobId/set-status", middleware.JobTokenAuth(), controller.SetJobStatusForProject) + r.POST("/_internal/trigger_drift_for_project", middleware.WebhookAuth(), controller.TriggerDriftRunForProject) port := os.Getenv("DIGGER_PORT") diff --git a/ee/drift/middleware/job_token.go b/ee/drift/middleware/job_token.go new file mode 100644 index 000000000..9f087fd3a --- /dev/null +++ b/ee/drift/middleware/job_token.go @@ -0,0 +1,70 @@ +package middleware + +import ( + "fmt" + "github.com/diggerhq/digger/ee/drift/dbmodels" + model2 "github.com/diggerhq/digger/ee/drift/model" + "github.com/gin-gonic/gin" + "log" + "net/http" + "strings" + "time" +) + +func CheckJobToken(c *gin.Context, token string) (*model2.DiggerCiJobToken, error) { + jobToken, err := dbmodels.DB.GetJobToken(token) + if jobToken == nil { + c.String(http.StatusForbidden, "Invalid bearer token") + c.Abort() + return nil, fmt.Errorf("invalid bearer token") + } + + if time.Now().After(jobToken.Expiry) { + log.Printf("Token has already expired: %v", err) + c.String(http.StatusForbidden, "Token has expired") + c.Abort() + return nil, fmt.Errorf("token has expired") + } + + if err != nil { + log.Printf("Error while fetching token from database: %v", err) + c.String(http.StatusInternalServerError, "Error occurred while fetching database") + c.Abort() + return nil, fmt.Errorf("could not fetch cli token") + } + + return jobToken, nil +} + +func JobTokenAuth() gin.HandlerFunc { + return func(c *gin.Context) { + authHeader := c.Request.Header.Get("Authorization") + if authHeader == "" { + c.String(http.StatusForbidden, "No Authorization header provided") + c.Abort() + return + } + token := strings.TrimPrefix(authHeader, "Bearer ") + if token == authHeader { + c.String(http.StatusForbidden, "Could not find bearer token in Authorization header") + c.Abort() + return + } + + if strings.HasPrefix(token, "cli:") { + if jobToken, err := CheckJobToken(c, token); err != nil { + c.String(http.StatusForbidden, err.Error()) + c.Abort() + return + } else { + c.Set(jobToken.OrganisationID, jobToken.OrganisationID) + c.Set(ACCESS_LEVEL_KEY, jobToken.Type) + } + } else { + c.String(http.StatusForbidden, "Invalid Bearer token") + c.Abort() + return + } + return + } +} diff --git a/ee/drift/middleware/middleware.go b/ee/drift/middleware/middleware.go index 41e34faad..3f3fb079b 100644 --- a/ee/drift/middleware/middleware.go +++ b/ee/drift/middleware/middleware.go @@ -1,3 +1,4 @@ package middleware const ORGANISATION_ID_KEY = "organisation_ID" +const ACCESS_LEVEL_KEY = "access_level" diff --git a/ee/drift/model/digger_ci_jobs.gen.go b/ee/drift/model/digger_ci_jobs.gen.go index 85e7adc1a..f5a266032 100644 --- a/ee/drift/model/digger_ci_jobs.gen.go +++ b/ee/drift/model/digger_ci_jobs.gen.go @@ -36,6 +36,8 @@ type DiggerCiJob struct { ResourcesUpdated int32 `gorm:"column:resources_updated" json:"resources_updated"` ResourcesDeleted int32 `gorm:"column:resources_deleted" json:"resources_deleted"` ProjectID string `gorm:"column:project_id" json:"project_id"` + DiggerJobID string `gorm:"column:digger_job_id" json:"digger_job_id"` + TerraformOutput string `gorm:"column:terraform_output" json:"terraform_output"` } // TableName DiggerCiJob's table name diff --git a/ee/drift/model/projects.gen.go b/ee/drift/model/projects.gen.go index 7f7d8524a..cc7f2d310 100644 --- a/ee/drift/model/projects.gen.go +++ b/ee/drift/model/projects.gen.go @@ -25,7 +25,7 @@ type Project struct { LatestDriftCheck time.Time `gorm:"column:latest_drift_check" json:"latest_drift_check"` DriftTerraformPlan string `gorm:"column:drift_terraform_plan" json:"drift_terraform_plan"` ToUpdate int32 `gorm:"column:to_update" json:"to_update"` - ToChange int32 `gorm:"column:to_change" json:"to_change"` + ToCreate int32 `gorm:"column:to_create" json:"to_create"` ToDelete int32 `gorm:"column:to_delete" json:"to_delete"` IsAcknowledged bool `gorm:"column:is_acknowledged;not null" json:"is_acknowledged"` } diff --git a/ee/drift/models_generated/digger_ci_jobs.gen.go b/ee/drift/models_generated/digger_ci_jobs.gen.go index 5d5a809fc..b90168e18 100644 --- a/ee/drift/models_generated/digger_ci_jobs.gen.go +++ b/ee/drift/models_generated/digger_ci_jobs.gen.go @@ -49,6 +49,8 @@ func newDiggerCiJob(db *gorm.DB, opts ...gen.DOOption) diggerCiJob { _diggerCiJob.ResourcesUpdated = field.NewInt32(tableName, "resources_updated") _diggerCiJob.ResourcesDeleted = field.NewInt32(tableName, "resources_deleted") _diggerCiJob.ProjectID = field.NewString(tableName, "project_id") + _diggerCiJob.DiggerJobID = field.NewString(tableName, "digger_job_id") + _diggerCiJob.TerraformOutput = field.NewString(tableName, "terraform_output") _diggerCiJob.fillFieldMap() @@ -81,6 +83,8 @@ type diggerCiJob struct { ResourcesUpdated field.Int32 ResourcesDeleted field.Int32 ProjectID field.String + DiggerJobID field.String + TerraformOutput field.String fieldMap map[string]field.Expr } @@ -119,6 +123,8 @@ func (d *diggerCiJob) updateTableName(table string) *diggerCiJob { d.ResourcesUpdated = field.NewInt32(table, "resources_updated") d.ResourcesDeleted = field.NewInt32(table, "resources_deleted") d.ProjectID = field.NewString(table, "project_id") + d.DiggerJobID = field.NewString(table, "digger_job_id") + d.TerraformOutput = field.NewString(table, "terraform_output") d.fillFieldMap() @@ -135,7 +141,7 @@ func (d *diggerCiJob) GetFieldByName(fieldName string) (field.OrderExpr, bool) { } func (d *diggerCiJob) fillFieldMap() { - d.fieldMap = make(map[string]field.Expr, 22) + d.fieldMap = make(map[string]field.Expr, 24) d.fieldMap["id"] = d.ID d.fieldMap["spectype"] = d.Spectype d.fieldMap["commentid"] = d.Commentid @@ -158,6 +164,8 @@ func (d *diggerCiJob) fillFieldMap() { d.fieldMap["resources_updated"] = d.ResourcesUpdated d.fieldMap["resources_deleted"] = d.ResourcesDeleted d.fieldMap["project_id"] = d.ProjectID + d.fieldMap["digger_job_id"] = d.DiggerJobID + d.fieldMap["terraform_output"] = d.TerraformOutput } func (d diggerCiJob) clone(db *gorm.DB) diggerCiJob { diff --git a/ee/drift/models_generated/projects.gen.go b/ee/drift/models_generated/projects.gen.go index e786492af..de575c203 100644 --- a/ee/drift/models_generated/projects.gen.go +++ b/ee/drift/models_generated/projects.gen.go @@ -38,7 +38,7 @@ func newProject(db *gorm.DB, opts ...gen.DOOption) project { _project.LatestDriftCheck = field.NewTime(tableName, "latest_drift_check") _project.DriftTerraformPlan = field.NewString(tableName, "drift_terraform_plan") _project.ToUpdate = field.NewInt32(tableName, "to_update") - _project.ToChange = field.NewInt32(tableName, "to_change") + _project.ToCreate = field.NewInt32(tableName, "to_create") _project.ToDelete = field.NewInt32(tableName, "to_delete") _project.IsAcknowledged = field.NewBool(tableName, "is_acknowledged") @@ -62,7 +62,7 @@ type project struct { LatestDriftCheck field.Time DriftTerraformPlan field.String ToUpdate field.Int32 - ToChange field.Int32 + ToCreate field.Int32 ToDelete field.Int32 IsAcknowledged field.Bool @@ -92,7 +92,7 @@ func (p *project) updateTableName(table string) *project { p.LatestDriftCheck = field.NewTime(table, "latest_drift_check") p.DriftTerraformPlan = field.NewString(table, "drift_terraform_plan") p.ToUpdate = field.NewInt32(table, "to_update") - p.ToChange = field.NewInt32(table, "to_change") + p.ToCreate = field.NewInt32(table, "to_create") p.ToDelete = field.NewInt32(table, "to_delete") p.IsAcknowledged = field.NewBool(table, "is_acknowledged") @@ -123,7 +123,7 @@ func (p *project) fillFieldMap() { p.fieldMap["latest_drift_check"] = p.LatestDriftCheck p.fieldMap["drift_terraform_plan"] = p.DriftTerraformPlan p.fieldMap["to_update"] = p.ToUpdate - p.fieldMap["to_change"] = p.ToChange + p.fieldMap["to_create"] = p.ToCreate p.fieldMap["to_delete"] = p.ToDelete p.fieldMap["is_acknowledged"] = p.IsAcknowledged }