feat: rewrite repos endpoint in go

RHINENG-13556

vmaas/cache.go

@@ -144,3 +144,54 @@ func (c *Cache) packageIDs2Nevras(pkgIDs []int) ([]string, []string) {
 	}
 	return binPackages, sourcePackages
 }
+
+func (c *Cache) buildRepoID2ErratumIDs(modifiedSince *time.Time, showPackages bool) map[RepoID][]ErratumID {
+	if modifiedSince == nil || !showPackages {
+		return nil
+	}
+	repoID2ErrataIDsMap := make(map[RepoID][]ErratumID, len(c.ErratumDetails))
+	for _, erratumDetail := range c.ErratumDetails {
+		if erratumDetail.Updated != nil && !erratumDetail.Updated.After(*modifiedSince) {
+			continue
+		}
+		for repoID := range c.ErratumID2RepoIDs[erratumDetail.ID] {
+			repoID2ErrataIDsMap[repoID] = append(repoID2ErrataIDsMap[repoID], erratumDetail.ID)
+		}
+	}
+	return repoID2ErrataIDsMap
+}
+
+func (c *Cache) cpeIDs2Labels(cpeIDs []CpeID) []string {
+	isDuplicate := make(map[CpeID]bool, len(cpeIDs))
+	cpes := make([]string, 0, len(cpeIDs))
+	for _, cpeID := range cpeIDs {
+		if isDuplicate[cpeID] {
+			continue
+		}
+
+		if cpe, found := c.CpeID2Label[cpeID]; found {
+			cpes = append(cpes, string(cpe))
+			isDuplicate[cpeID] = true
+		}
+	}
+	return cpes
+}
+
+func (c *Cache) erratumIDs2PackageNames(erratumIDs []ErratumID) []string {
+	isDuplicate := make(map[ErratumID]bool, len(erratumIDs))
+	pkgNames := make([]string, 0, len(erratumIDs))
+	for _, erratumID := range erratumIDs {
+		if isDuplicate[erratumID] {
+			continue
+		}
+		isDuplicate[erratumID] = true
+		erratum := c.ErratumID2Name[erratumID]
+		erratumDetail := c.ErratumDetails[erratum]
+		for _, pkgID := range erratumDetail.PkgIDs {
+			pkgDetail := c.PackageDetails[PkgID(pkgID)]
+			pkgName := c.ID2Packagename[pkgDetail.NameID]
+			pkgNames = append(pkgNames, pkgName)
+		}
+	}
+	return pkgNames
+}

vmaas/load.go

@@ -359,10 +359,7 @@ func loadPkgDetails(c *Cache) {
 func loadRepoDetails(c *Cache) { //nolint: funlen
 	defer utils.TimeTrack(time.Now(), "RepoIDs, RepoDetails, RepoLabel2IDs, RepoPath2IDs, ProductID2RepoIDs")
 
-	rows := getAllRows(
-		"repo_detail",
-		"id,label,name,url,COALESCE(basearch,''),COALESCE(releasever,''),product,product_id,revision,last_change,third_party",
-	)
+	rows := getAllRows("repo_detail", "id,label,name,url,COALESCE(basearch,''),COALESCE(releasever,''),product,product_id,COALESCE(revision,''),COALESCE(last_change,''),third_party") //nolint:lll
 	cntRepo := getCount("repo_detail", "*")
 	cntLabel := getCount("repo_detail", "distinct label")
 	cntURL := getCount("repo_detail", "distinct url")
@@ -373,10 +370,12 @@ func loadRepoDetails(c *Cache) { //nolint: funlen
 	prodID2RepoIDs := make(map[int][]RepoID, cntProd)
 	repoIDs := []RepoID{}
 	var repoID RepoID
+	var revision string
+	var lastChange string
 	for rows.Next() {
 		var det RepoDetail
 		err := rows.Scan(&repoID, &det.Label, &det.Name, &det.URL, &det.Basearch, &det.Releasever,
-			&det.Product, &det.ProductID, &det.Revision, &det.LastChange, &det.ThirdParty)
+			&det.Product, &det.ProductID, &revision, &lastChange, &det.ThirdParty)
 		if err != nil {
 			panic(err)
 		}
@@ -408,6 +407,16 @@ func loadRepoDetails(c *Cache) { //nolint: funlen
 			prodID2RepoIDs[det.ProductID] = []RepoID{}
 		}
 		prodID2RepoIDs[det.ProductID] = append(prodID2RepoIDs[det.ProductID], repoID)
+
+		revisionTime, err := time.Parse(time.RFC3339, revision)
+		if err == nil {
+			det.Revision = revisionTime
+		}
+
+		lastChangeTime, err := time.Parse(time.RFC3339, lastChange)
+		if err == nil {
+			det.LastChange = &lastChangeTime
+		}
 	}
 	c.RepoIDs = repoIDs
 	c.RepoDetails = id2repoDetail

vmaas/repos.go

@@ -0,0 +1,123 @@
+package vmaas
+
+import (
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/redhatinsights/vmaas-lib/vmaas/utils"
+)
+
+type RepoDetails map[string][]RepoDetail
+
+type Repos struct {
+	Repos            RepoDetails `json:"repository_list"`
+	LatestRepoChange *time.Time  `json:"latest_repo_change,omitempty"`
+	LastChange       string      `json:"last_change"`
+	utils.PaginationDetails
+}
+
+var RepoPrefixes = strings.Split(os.Getenv("REPO_NAME_PREFIXES"), ",")
+
+func filterInputRepos(c *Cache, repos []string, req *ReposRequest) []string {
+	isDuplicate := make(map[string]bool, len(repos))
+	filteredRepos := make([]string, 0, len(repos))
+	for _, repo := range repos {
+		if repo == "" || isDuplicate[repo] {
+			continue
+		}
+		repoIDs, found := c.RepoLabel2IDs[repo]
+		if !found || len(repoIDs) == 0 {
+			continue
+		}
+		repoDetail, found := c.RepoDetails[repoIDs[len(repoIDs)-1]]
+		if !found {
+			continue
+		}
+
+		if req.ModifiedSince != nil {
+			if req.ModifiedSince.After(time.Now()) {
+				continue
+			}
+			if repoDetail.LastChange != nil && repoDetail.LastChange.Before(*req.ModifiedSince) {
+				continue
+			}
+		}
+
+		if !req.ThirdParty && repoDetail.ThirdParty {
+			continue
+		}
+
+		filteredRepos = append(filteredRepos, repo)
+		isDuplicate[repo] = true
+	}
+	return filteredRepos
+}
+
+func (c *Cache) repoID2CPEs(repoID RepoID, contentSetID ContentSetID) []string {
+	cpeIDs, found := c.RepoID2CpeIDs[repoID]
+	if !found {
+		cpeIDs = c.ContentSetID2CpeIDs[contentSetID]
+	}
+	return c.cpeIDs2Labels(cpeIDs)
+}
+
+func (c *Cache) loadRepoDetailSlice(repo string, repoID2ErratumIDs map[RepoID][]ErratumID) ([]RepoDetail, *time.Time) {
+	repoIDs := c.RepoLabel2IDs[repo]
+	contentSetID := c.Label2ContentSetID[repo]
+	repoDetailSlice := make([]RepoDetail, 0, len(repoIDs))
+	var latestChange *time.Time
+	for _, repoID := range repoIDs {
+		repoDerail := c.RepoDetails[repoID]
+		repoDerail.Label = repo
+		repoDerail.CPEs = c.repoID2CPEs(repoID, contentSetID)
+		erratumIDs := repoID2ErratumIDs[repoID]
+		repoDerail.UpdatedPackageNames = c.erratumIDs2PackageNames(erratumIDs)
+		lastChange := repoDerail.LastChange
+		if latestChange == nil || (lastChange != nil && lastChange.After(*latestChange)) {
+			latestChange = lastChange
+		}
+		repoDetailSlice = append(repoDetailSlice, repoDerail)
+	}
+	return repoDetailSlice, latestChange
+}
+
+func (c *Cache) loadRepoDetails(repos []string, repoID2ErratumIDs map[RepoID][]ErratumID) (RepoDetails, *time.Time, int) {
+	repodDetails := make(RepoDetails, len(repos))
+	var latestRepoChange *time.Time
+	actualPageSize := 0
+	for _, repo := range repos {
+		repoDetailSlice, latestChange := c.loadRepoDetailSlice(repo, repoID2ErratumIDs)
+		if latestRepoChange == nil || (latestChange != nil && latestChange.After(*latestRepoChange)) {
+			latestRepoChange = latestChange
+		}
+		repodDetails[repo] = repoDetailSlice
+		actualPageSize += len(repoDetailSlice)
+	}
+	return repodDetails, latestRepoChange, actualPageSize
+}
+
+func (req *ReposRequest) repos(c *Cache) (*Repos, error) { // TODO: implement opts
+	if len(req.Repos) == 0 {
+		return nil, errors.Wrap(ErrProcessingInput, "'repository_list' is a required property")
+	}
+	repos := utils.StripPrefixes(req.Repos, RepoPrefixes)
+	repos, err := utils.TryExpandRegexPattern(repos, c.RepoLabel2IDs)
+	if err != nil {
+		return nil, errors.Wrap(ErrProcessingInput, "invalid regex pattern")
+	}
+	repos = filterInputRepos(c, repos, req)
+	repos, paginationDetails := utils.Paginate(repos, req.PageNumber, req.PageSize)
+
+	repoID2ErratumIDs := c.buildRepoID2ErratumIDs(req.ModifiedSince, req.ShowPackages)
+	repoDetails, latestRepoChange, actualPageSize := c.loadRepoDetails(repos, repoID2ErratumIDs)
+	paginationDetails.PageSize = actualPageSize
+	res := Repos{
+		Repos:             repoDetails,
+		LatestRepoChange:  latestRepoChange,
+		LastChange:        c.DBChange.LastChange,
+		PaginationDetails: paginationDetails,
+	}
+	return &res, nil
+}

vmaas/types.go

@@ -92,6 +92,15 @@ type ErrataRequest struct {
 	PageSize      int         `json:"page_size"`
 }
 
+type ReposRequest struct {
+	Repos         []string   `json:"repository_list"`
+	ModifiedSince *time.Time `json:"modified_since"`
+	ThirdParty    bool       `json:"third_party"`
+	ShowPackages  bool       `json:"show_packages"`
+	PageNumber    int        `json:"page"`
+	PageSize      int        `json:"page_size"`
+}
+
 type Update struct {
 	Package     string `json:"package"`
 	PackageName string `json:"package_name"`
@@ -202,16 +211,19 @@ type Nevra struct {
 }
 
 type RepoDetail struct {
-	Label      string
-	Name       string
-	URL        string
-	Basearch   string
-	Releasever string
-	Product    string
-	ProductID  int
-	Revision   *string
-	LastChange *string
-	ThirdParty bool
+	Label      string     `json:"label"`
+	Name       string     `json:"name"`
+	URL        string     `json:"url"`
+	Basearch   string     `json:"basearch"`
+	Releasever string     `json:"releasever"`
+	Product    string     `json:"product"`
+	ProductID  int        `json:"-"`
+	Revision   time.Time  `json:"revision"`
+	LastChange *time.Time `json:"-"`
+	ThirdParty bool       `json:"third_party"`
+
+	CPEs                []string `json:"cpes"`
+	UpdatedPackageNames []string `json:"updated_package_names,omitempty"`
 }
 
 type CveDetail struct {

vmaas/utils/repo.go

@@ -0,0 +1,17 @@
+package utils
+
+import (
+	"strings"
+)
+
+func StripPrefixes(repos []string, prefixes []string) []string {
+	processed := make([]string, 0, len(repos))
+	for _, repo := range repos {
+		for _, prefix := range prefixes {
+			if strings.HasPrefix(repo, prefix) {
+				processed = append(processed, repo[len(prefix):])
+			}
+		}
+	}
+	return processed
+}

vmaas/utils/repo_test.go

@@ -0,0 +1,15 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStripPrefixes(t *testing.T) {
+	prefixes := []string{"abc-", "foo-"}
+	repos := []string{"abc-rhel-6-server-rpms", "foo-rhel-6-server-rpms"}
+	repos = StripPrefixes(repos, prefixes)
+	assert.Equal(t, "rhel-6-server-rpms", repos[0])
+	assert.Equal(t, "rhel-6-server-rpms", repos[1])
+}

vmaas/vmaas.go

@@ -62,6 +62,10 @@ func (api *API) Errata(request *ErrataRequest) (*Errata, error) {
 	return request.errata(api.Cache)
 }
 
+func (api *API) Repos(request *ReposRequest) (*Repos, error) {
+	return request.repos(api.Cache)
+}
+
 func (api *API) LoadCacheFromFile(cachePath string) error {
 	var err error
 	api.Cache, err = loadCache(cachePath, api.options)