feat drift app (#1724)

* modify drift app

.gitignore

@@ -1,5 +1,6 @@
 .idea/
 **/.env
+**/.env*
 .DS_Store
 venv/
 **/__pycache__/

ee/drift/controllers/github.go

@@ -4,20 +4,77 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/diggerhq/digger/backend/utils"
 	"github.com/diggerhq/digger/ee/drift/dbmodels"
 	"github.com/diggerhq/digger/ee/drift/middleware"
 	"github.com/diggerhq/digger/ee/drift/model"
+	"github.com/diggerhq/digger/ee/drift/tasks"
 	next_utils "github.com/diggerhq/digger/next/utils"
 	"github.com/gin-gonic/gin"
 	"github.com/google/go-github/v61/github"
 	"golang.org/x/oauth2"
 	"log"
 	"net/http"
 	"os"
+	"reflect"
 	"strconv"
 	"strings"
 )
 
+func (mc MainController) GithubAppWebHook(c *gin.Context) {
+	c.Header("Content-Type", "application/json")
+	gh := mc.GithubClientProvider
+	log.Printf("GithubAppWebHook")
+
+	payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")))
+	if err != nil {
+		log.Printf("Error validating github app webhook's payload: %v", err)
+		c.String(http.StatusBadRequest, "Error validating github app webhook's payload")
+		return
+	}
+
+	webhookType := github.WebHookType(c.Request)
+	event, err := github.ParseWebHook(webhookType, payload)
+	if err != nil {
+		log.Printf("Failed to parse Github Event. :%v\n", err)
+		c.String(http.StatusInternalServerError, "Failed to parse Github Event")
+		return
+	}
+
+	log.Printf("github event type: %v\n", reflect.TypeOf(event))
+
+	switch event := event.(type) {
+	case *github.PushEvent:
+		log.Printf("Got push event for %d", event.Repo.URL)
+		err := handlePushEvent(gh, event)
+		if err != nil {
+			log.Printf("handlePushEvent error: %v", err)
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
+	default:
+		log.Printf("Unhandled event, event type %v", reflect.TypeOf(event))
+	}
+
+	c.JSON(200, "ok")
+}
+
+func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) error {
+	installationId := *payload.Installation.ID
+	repoName := *payload.Repo.Name
+	repoFullName := *payload.Repo.FullName
+	repoOwner := *payload.Repo.Owner.Login
+	cloneURL := *payload.Repo.CloneURL
+	ref := *payload.Ref
+	defaultBranch := *payload.Repo.DefaultBranch
+
+	if strings.HasSuffix(ref, defaultBranch) {
+		go tasks.LoadProjectsFromGithubRepo(gh, strconv.FormatInt(installationId, 10), repoFullName, repoOwner, repoName, cloneURL, defaultBranch)
+	}
+
+	return nil
+}
+
 func (mc MainController) GithubAppCallbackPage(c *gin.Context) {
 	installationId := c.Request.URL.Query()["installation_id"][0]
 	//setupAction := c.Request.URL.Query()["setup_action"][0]
@@ -80,7 +137,7 @@ func (mc MainController) GithubAppCallbackPage(c *gin.Context) {
 
 	// reset all existing repos (soft delete)
 	var ExistingRepos []model.Repo
-	err = dbmodels.DB.GormDB.Delete(ExistingRepos, "organization_id=?", orgId).Error
+	err = dbmodels.DB.GormDB.Delete(ExistingRepos, "organisation_id=?", orgId).Error
 	if err != nil {
 		log.Printf("could not delete repos: %v", err)
 		c.String(http.StatusInternalServerError, "could not delete repos: %v", err)
@@ -89,20 +146,24 @@ func (mc MainController) GithubAppCallbackPage(c *gin.Context) {
 
 	// here we mark repos that are available one by one
 	for _, repo := range repos {
+		cloneUrl := *repo.CloneURL
+		defaultBranch := *repo.DefaultBranch
 		repoFullName := *repo.FullName
 		repoOwner := strings.Split(*repo.FullName, "/")[0]
 		repoName := *repo.Name
 		repoUrl := fmt.Sprintf("https://github.com/%v", repoFullName)
 
-		_, _, err = dbmodels.CreateOrGetDiggerRepoForGithubRepo(repoFullName, repoOwner, repoName, repoUrl, installationId64, *installation.AppID, *installation.Account.ID, *installation.Account.Login)
+		_, _, err = dbmodels.CreateOrGetDiggerRepoForGithubRepo(repoFullName, repoOwner, repoName, repoUrl, installationId, *installation.AppID, *installation.Account.ID, *installation.Account.Login)
 		if err != nil {
 			log.Printf("createOrGetDiggerRepoForGithubRepo error: %v", err)
 			c.String(http.StatusInternalServerError, "createOrGetDiggerRepoForGithubRepo error: %v", err)
 			return
 		}
+
+		go tasks.LoadProjectsFromGithubRepo(mc.GithubClientProvider, installationId, repoFullName, repoOwner, repoName, cloneUrl, defaultBranch)
 	}
 
-	c.HTML(http.StatusOK, "github_success.tmpl", gin.H{})
+	c.String(http.StatusOK, "success", gin.H{})
 }
 
 // why this validation is needed: https://roadie.io/blog/avoid-leaking-github-org-data/

ee/drift/dbmodels/github.go

@@ -16,7 +16,7 @@ const (
 	GithubAppInstallationLinkInactive GithubAppInstallationLinkStatus = "inactive"
 )
 
-func (db *Database) GetGithubInstallationLinkForInstallationId(installationId int64) (*model.GithubAppInstallationLink, error) {
+func (db *Database) GetGithubInstallationLinkForInstallationId(installationId string) (*model.GithubAppInstallationLink, error) {
 	l := model.GithubAppInstallationLink{}
 	err := db.GormDB.Where("github_installation_id = ? AND status=?", installationId, GithubAppInstallationLinkActive).Find(&l).Error
 	if err != nil {
@@ -28,7 +28,7 @@ func (db *Database) GetGithubInstallationLinkForInstallationId(installationId in
 	return &l, nil
 }
 
-func CreateOrGetDiggerRepoForGithubRepo(ghRepoFullName string, ghRepoOrganisation string, ghRepoName string, ghRepoUrl string, installationId int64, githubAppId int64, accountId int64, login string) (*model.Repo, *model.Organisation, error) {
+func CreateOrGetDiggerRepoForGithubRepo(ghRepoFullName string, ghRepoOrganisation string, ghRepoName string, ghRepoUrl string, installationId string, githubAppId int64, accountId int64, login string) (*model.Repo, *model.Organisation, error) {
 	link, err := DB.GetGithubInstallationLinkForInstallationId(installationId)
 	if err != nil {
 		log.Printf("Error fetching installation link: %v", err)
@@ -45,7 +45,7 @@ func CreateOrGetDiggerRepoForGithubRepo(ghRepoFullName string, ghRepoOrganisatio
 
 	// using Unscoped because we also need to include deleted repos (and undelete them if they exist)
 	var existingRepo model.Repo
-	r := DB.GormDB.Unscoped().Where("organization_id=? AND repos.name=?", orgId, diggerRepoName).Find(&existingRepo)
+	r := DB.GormDB.Unscoped().Where("organisation_id=? AND repos.name=?", orgId, diggerRepoName).Find(&existingRepo)
 
 	if r.Error != nil {
 		if errors.Is(err, gorm.ErrRecordNotFound) {
@@ -71,3 +71,20 @@ func CreateOrGetDiggerRepoForGithubRepo(ghRepoFullName string, ghRepoOrganisatio
 	log.Printf("Created digger repo: %v", repo)
 	return repo, org, nil
 }
+
+// GetGithubAppInstallationLink repoFullName should be in the following format: org/repo_name, for example "diggerhq/github-job-scheduler"
+func (db *Database) GetGithubAppInstallationLink(installationId string) (*model.GithubAppInstallationLink, error) {
+	var link model.GithubAppInstallationLink
+	result := db.GormDB.Where("github_installation_id = ? AND status=?", installationId, GithubAppInstallationLinkActive).Find(&link)
+	if result.Error != nil {
+		if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return nil, result.Error
+		}
+	}
+
+	// If not found, the values will be default values, which means ID will be 0
+	if link.ID == "" {
+		return nil, nil
+	}
+	return &link, nil
+}

ee/drift/dbmodels/projects.go

@@ -0,0 +1,51 @@
+package dbmodels
+
+import (
+	"errors"
+	"github.com/diggerhq/digger/ee/drift/model"
+	"gorm.io/gorm"
+	"log"
+)
+
+type DriftStatus string
+
+var DriftStatusNewDrift = "new drift"
+var DriftStatusNoDrift = "no drift"
+var DriftStatusAcknowledgeDrift = "acknowledged drift"
+
+// GetProjectByName return project for specified org and repo
+// if record doesn't exist return nil
+func (db *Database) GetProjectByName(orgId any, repo *model.Repo, name string) (*model.Project, error) {
+	log.Printf("GetProjectByName, org id: %v, project name: %v\n", orgId, name)
+	var project model.Project
+
+	err := db.GormDB.
+		Joins("INNER JOIN repos ON projects.repo_id = repos.id").
+		Where("repos.id = ?", repo.ID).
+		Where("projects.name = ?", name).First(&project).Error
+
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, nil
+		}
+		log.Printf("Unknown error occurred while fetching database, %v\n", err)
+		return nil, err
+	}
+
+	return &project, nil
+}
+
+func (db *Database) CreateProject(name string, repo *model.Repo) (*model.Project, error) {
+	project := &model.Project{
+		Name:        name,
+		RepoID:      repo.ID,
+		DriftStatus: DriftStatusNewDrift,
+	}
+	result := db.GormDB.Save(project)
+	if result.Error != nil {
+		log.Printf("Failed to create project: %v, error: %v\n", name, result.Error)
+		return nil, result.Error
+	}
+	log.Printf("Project %s, (id: %v) has been created successfully\n", name, project.ID)
+	return project, nil
+}

ee/drift/dbmodels/repos.go

@@ -0,0 +1,53 @@
+package dbmodels
+
+import (
+	"errors"
+	"fmt"
+	"github.com/diggerhq/digger/ee/drift/model"
+	configuration "github.com/diggerhq/digger/libs/digger_config"
+	"gorm.io/gorm"
+	"log"
+)
+
+// GetRepo returns digger repo by organisationId and repo name (diggerhq-digger)
+// it will return an empty object if record doesn't exist in database
+func (db *Database) GetRepo(orgIdKey any, repoName string) (*model.Repo, error) {
+	var repo model.Repo
+
+	err := db.GormDB.Where("organisation_id = ? AND repos.name=?", orgIdKey, repoName).First(&repo).Error
+
+	if err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil, nil
+		}
+		log.Printf("Failed to find digger repo for orgId: %v, and repoName: %v, error: %v\n", orgIdKey, repoName, err)
+		return nil, err
+	}
+	return &repo, nil
+}
+
+func (db *Database) RefreshProjectsFromRepo(orgId string, config configuration.DiggerConfigYaml, repo *model.Repo) error {
+	log.Printf("UpdateRepoDiggerConfig, repo: %v\n", repo)
+
+	err := db.GormDB.Transaction(func(tx *gorm.DB) error {
+		for _, dc := range config.Projects {
+			projectName := dc.Name
+			p, err := db.GetProjectByName(orgId, repo, projectName)
+			if err != nil {
+				return fmt.Errorf("error retriving project by name: %v", err)
+			}
+			if p == nil {
+				_, err := db.CreateProject(projectName, repo)
+				if err != nil {
+					return fmt.Errorf("could not create project: %v", err)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("error while updating projects from config: %v", err)
+	}
+	return nil
+}

ee/drift/dbmodels/storage.go

@@ -21,7 +21,7 @@ func (db *Database) GetOrganisationById(orgId any) (*model.Organisation, error)
 func (db *Database) CreateGithubInstallationLink(orgId string, installationId string) (*model.GithubAppInstallationLink, error) {
 	l := model.GithubAppInstallationLink{}
 	// check if there is already a link to another org, and throw an error in this case
-	result := db.GormDB.Preload("Organisation").Where("github_installation_id = ? AND status=?", installationId, GithubAppInstallationLinkActive).Find(&l)
+	result := db.GormDB.Where("github_installation_id = ? AND status=?", installationId, GithubAppInstallationLinkActive).Find(&l)
 	if result.Error != nil {
 		if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
 			return nil, result.Error
@@ -38,7 +38,7 @@ func (db *Database) CreateGithubInstallationLink(orgId string, installationId st
 
 	var list []model.GithubAppInstallationLink
 	// if there are other installation for this org, we need to make them inactive
-	result = db.GormDB.Preload("Organisation").Where("github_installation_id <> ? AND organisation_id = ? AND status=?", installationId, orgId, GithubAppInstallationLinkActive).Find(&list)
+	result = db.GormDB.Where("github_installation_id <> ? AND organisation_id = ? AND status=?", installationId, orgId, GithubAppInstallationLinkActive).Find(&list)
 	if result.Error != nil {
 		if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
 			return nil, result.Error
@@ -58,10 +58,10 @@ func (db *Database) CreateGithubInstallationLink(orgId string, installationId st
 	return &link, nil
 }
 
-func (db *Database) CreateRepo(name string, repoFullName string, repoOrganisation string, repoName string, repoUrl string, org *model.Organisation, diggerConfig string, githubInstallationId int64, githubAppId int64, accountId int64, login string) (*model.Repo, error) {
+func (db *Database) CreateRepo(name string, repoFullName string, repoOrganisation string, repoName string, repoUrl string, org *model.Organisation, diggerConfig string, githubInstallationId string, githubAppId int64, accountId int64, login string) (*model.Repo, error) {
 	var repo model.Repo
 	// check if repo exist already, do nothing in this case
-	result := db.GormDB.Where("name = ? AND organization_id=?", name, org.ID).Find(&repo)
+	result := db.GormDB.Where("name = ? AND organisation_id=?", name, org.ID).Find(&repo)
 	if result.Error != nil {
 		if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
 			return nil, result.Error
@@ -92,3 +92,20 @@ func (db *Database) CreateRepo(name string, repoFullName string, repoOrganisatio
 	log.Printf("Repo %s, (id: %v) has been created successfully\n", name, repo.ID)
 	return &repo, nil
 }
+
+// GetGithubAppInstallationByIdAndRepo repoFullName should be in the following format: org/repo_name, for example "diggerhq/github-job-scheduler"
+func (db *Database) GetRepoByInstllationIdAndRepoFullName(installationId string, repoFullName string) (*model.Repo, error) {
+	repo := model.Repo{}
+	result := db.GormDB.Where("github_installation_id = ?  AND repo_full_name=?", installationId, repoFullName).Find(&repo)
+	if result.Error != nil {
+		if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
+			return nil, result.Error
+		}
+	}
+
+	// If not found, the values will be default values, which means ID will be 0
+	if repo.ID == "" {
+		return nil, fmt.Errorf("GithubAppInstallation with id=%v doesn't exist", installationId)
+	}
+	return &repo, nil
+}

ee/drift/main.go

@@ -69,6 +69,7 @@ func main() {
 	//authorized := r.Group("/")
 	//authorized.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(dbmodels.CliJobAccessType, dbmodels.AccessPolicyType, models.AdminPolicyType))
 
+	r.POST("github-app-webhook", controller.GithubAppWebHook)
 	r.GET("/github/callback_fe", middleware.WebhookAuth(), controller.GithubAppCallbackPage)
 
 	port := os.Getenv("DIGGER_PORT")