From a8fc87569435ae9afa4bacd7b5917e9a8cd22168 Mon Sep 17 00:00:00 2001 From: Grzegorz Siewruk Date: Thu, 12 Dec 2024 12:55:37 +0100 Subject: [PATCH] Relase of v1.0.7 --- CHANGELOG.md | 5 ++ .../controller/ThreatIntelController.java | 14 +++ .../service/ThreatIntelService.java | 8 ++ .../mixewayflowapi/db/entity/UserInfo.java | 5 ++ .../mixewayflowapi/db/projection/ItemDTO.java | 19 ++++ .../db/projection/ItemProjection.java | 5 +- .../db/repository/TeamRepository.java | 2 + .../appdatatype/CreateAppDataTypeService.java | 3 + .../coderepo/CreateCodeRepoService.java | 2 +- .../domain/coderepo/FindCodeRepoService.java | 2 + .../domain/finding/FindFindingService.java | 22 ++++- .../domain/team/FindTeamService.java | 4 + .../exceptions/TeamNotFoundException.java | 4 + .../service/ScanManagerService.java | 1 + .../db/changelog/db.changelog-master.sql | 88 ++++++++++++++++++- 15 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemDTO.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 69147fe..4d8cfbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [1.0.7] - 2024-12-12 + +### Introduced +- New API to get Threat Intell for a team + ## [1.0.6] - 2024-11-21 ### Fixed diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/controller/ThreatIntelController.java b/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/controller/ThreatIntelController.java index 1c60e63..d64761a 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/controller/ThreatIntelController.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/controller/ThreatIntelController.java @@ -6,12 +6,15 @@ import io.mixeway.mixewayflowapi.api.threatintel.service.ThreatIntelService; import io.mixeway.mixewayflowapi.db.projection.Item; import io.mixeway.mixewayflowapi.db.projection.ItemProjection; +import io.mixeway.mixewayflowapi.exceptions.TeamNotFoundException; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RestController; import java.security.Principal; @@ -31,6 +34,17 @@ public ResponseEntity getThreats(Principal principal){ return threatIntelService.getThreats(principal); } + @PreAuthorize("hasAuthority('USER')") + @GetMapping(value= "/api/v1/threat-intel/findings/{remoteId}") + public ResponseEntity getThreatsForTeam(Principal principal, @PathVariable("remoteId") String remoteId){ + try { + return threatIntelService.getThreatsForTeam(principal, remoteId); + } catch (TeamNotFoundException e){ + log.warn(e.getLocalizedMessage()); + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + @PreAuthorize("hasAuthority('USER')") @GetMapping(value= "/api/v1/threat-intel/removed") public ResponseEntity> getRemovedThreats(Principal principal){ diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/service/ThreatIntelService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/service/ThreatIntelService.java index d125d5f..2c2ad1f 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/service/ThreatIntelService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/api/threatintel/service/ThreatIntelService.java @@ -35,4 +35,12 @@ public ResponseEntity> getRemovedThreats(Principal public ResponseEntity> getSupressedThreats(Principal principal) { return new ResponseEntity<>(findFindingService.getTopReviewedVulns(principal),HttpStatus.OK); } + + public ResponseEntity getThreatsForTeam(Principal principal, String remoteId) { + ItemListResponse itemListResponse = findFindingService.getThreatIntelFindingsForTeam(principal,remoteId); + itemListResponse.setNumberOfTeams(findTeamService.findAllTeams(principal).size()); + itemListResponse.setNumberOfAllProjects(findCodeRepoService.findCodeRepoForUser(principal).size()); + itemListResponse.setOpenedVulnerabilities(findFindingService.countOpenedVulnerabilities(principal)); + return new ResponseEntity<>(itemListResponse,HttpStatus.OK); + } } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/UserInfo.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/UserInfo.java index 73f016e..d8f747b 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/UserInfo.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/entity/UserInfo.java @@ -35,6 +35,11 @@ public class UserInfo { private final Set roles = new HashSet<>(); @ManyToMany(fetch = FetchType.EAGER) + @JoinTable( + name = "users_teams", + joinColumns = @JoinColumn(name = "user_info_id"), + inverseJoinColumns = @JoinColumn(name = "team_id") + ) private final Set teams = new HashSet<>(); @Column(name = "api_key") diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemDTO.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemDTO.java new file mode 100644 index 0000000..195a1a4 --- /dev/null +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemDTO.java @@ -0,0 +1,19 @@ +package io.mixeway.mixewayflowapi.db.projection; + +import lombok.Data; + +import java.math.BigDecimal; +import java.util.List; + +@Data +public class ItemDTO { + private Long coderepoId; + private String name; + private String urgency; + private int count; + private BigDecimal epss; + private Boolean pii; + private Boolean exploitAvailable; + private List projectNames; + private List projectIds; +} diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemProjection.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemProjection.java index 0f424c4..2262ccc 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemProjection.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/projection/ItemProjection.java @@ -1,6 +1,7 @@ package io.mixeway.mixewayflowapi.db.projection; import java.math.BigDecimal; +import java.util.List; public interface ItemProjection { Long getCoderepoId(); @@ -10,6 +11,6 @@ public interface ItemProjection { BigDecimal getEpss(); boolean isPii(); boolean isExploitAvailable(); - String[] getProjectNames(); - Long[] getProjectIds(); + List getProjectNames(); + List getProjectIds(); } \ No newline at end of file diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/TeamRepository.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/TeamRepository.java index ea0f887..7fbbf80 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/TeamRepository.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/TeamRepository.java @@ -9,5 +9,7 @@ public interface TeamRepository extends CrudRepository { Optional findByName(String name); Optional findById(Long id); + + Optional findByRemoteIdentifier(String remoteIdentifier); List findAll(); } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/appdatatype/CreateAppDataTypeService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/appdatatype/CreateAppDataTypeService.java index 875fe01..93b612f 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/appdatatype/CreateAppDataTypeService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/appdatatype/CreateAppDataTypeService.java @@ -25,6 +25,9 @@ public class CreateAppDataTypeService { @Transactional public void getDataTypesForCodeRepo(CodeRepo codeRepo, BearerScanDataflow bearerScanDataflow){ + codeRepo = codeRepoRepository.findById(codeRepo.getId()) + .orElseThrow(() -> new IllegalArgumentException("CodeRepo not found")); + if (bearerScanDataflow.getDataTypes() != null ){ log.info("[DataFlowAPI] Removing all data entities from {}", codeRepo.getRepourl()); diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/CreateCodeRepoService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/CreateCodeRepoService.java index 115478d..5172663 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/CreateCodeRepoService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/CreateCodeRepoService.java @@ -55,7 +55,7 @@ public void createCodeRepo(CreateCodeRepoRequestDto createCodeRepoRequestDto, Co scanManagerService.scanRepository(finalCodeRepo, finalCodeRepo.getDefaultBranch(), null, null); } else { - throw new TeamNotFoundException(); + throw new TeamNotFoundException("[CreateCodeRepoService] Team " + createCodeRepoRequestDto.getTeam() + " not found"); } log.info("[CodeRepo] Imported Code repo from {} id {} name {}", importCodeRepoResponseDto.getWebUrl(), diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/FindCodeRepoService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/FindCodeRepoService.java index 8b85e7b..cae853f 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/FindCodeRepoService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/FindCodeRepoService.java @@ -75,4 +75,6 @@ public List findByTeam(Team team){ public Optional findAllByUrl(String url) { return codeRepoRepository.findByRepourl(url); } + + } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/finding/FindFindingService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/finding/FindFindingService.java index 5167ea0..60b7a67 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/finding/FindFindingService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/finding/FindFindingService.java @@ -8,7 +8,9 @@ import io.mixeway.mixewayflowapi.db.projection.*; import io.mixeway.mixewayflowapi.db.repository.FindingRepository; import io.mixeway.mixewayflowapi.domain.coderepo.FindCodeRepoService; +import io.mixeway.mixewayflowapi.domain.team.FindTeamService; import io.mixeway.mixewayflowapi.domain.user.FindUserService; +import io.mixeway.mixewayflowapi.exceptions.TeamNotFoundException; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -22,6 +24,7 @@ public class FindFindingService { private final FindingRepository findingRepository; private final FindCodeRepoService findCodeRepoService; private final FindUserService findUserService; + private final FindTeamService findTeamService; public VulnStatsResponseDto countFindingStatsForRepo(CodeRepo codeRepo){ return findingRepository.countFindingsBySource(codeRepo.getId()); @@ -54,6 +57,18 @@ public ItemListResponse getThreatIntelFindings(Principal principal){ }return mapProjectionsToItems(combinedProjections); } + public ItemListResponse getThreatIntelFindingsForTeam(Principal principal, String remoteId){ + Team team = findTeamService.findByRemoteId(remoteId); + if (team == null){ + throw new TeamNotFoundException("[Threat Intell] Trying to find not existing team " + remoteId); + } + List combinedProjections = new ArrayList<>(); + + combinedProjections = findingRepository.findCombinedItems(findCodeRepoService.findByTeam(team).stream().map(CodeRepo::getId).toList()); + + return mapProjectionsToItems(combinedProjections); + } + private ItemListResponse mapProjectionsToItems(List projections) { Map itemMap = new HashMap<>(); List allProjectIds = new ArrayList<>(); @@ -90,9 +105,9 @@ private ItemListResponse mapProjectionsToItems(List projections) } // Map project names and IDs to Project objects - String[] projectNames = projection.getProjectNames(); - Long[] projectIds = projection.getProjectIds(); - allProjectIds.addAll(Arrays.stream(projectIds).toList()); + String[] projectNames = projection.getProjectNames().toArray(String[]::new); + Integer[] projectIds = projection.getProjectIds().toArray(Integer[]::new); + // allProjectIds.addAll(Arrays.stream(projectIds).toList()); if (projectNames != null && projectIds != null) { for (int i = 0; i < projectNames.length; i++) { @@ -119,6 +134,7 @@ private ItemListResponse mapProjectionsToItems(List projections) return response; } + public Long countOpenedVulnerabilities(Principal principal){ return findingRepository.countAllByCodeRepoInAndStatusIn(findCodeRepoService.findCodeRepoForUser(principal), Arrays.asList("NEW", "EXISTING")); } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/team/FindTeamService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/team/FindTeamService.java index b86fa03..60aac3c 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/team/FindTeamService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/team/FindTeamService.java @@ -48,4 +48,8 @@ public Optional findById(Long id) { public List findAll(){ return teamRepository.findAll(); } + + public Team findByRemoteId(String remoteId) { + return teamRepository.findByRemoteIdentifier(remoteId).orElse(null); + } } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/TeamNotFoundException.java b/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/TeamNotFoundException.java index a86b8df..9ee835b 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/TeamNotFoundException.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/TeamNotFoundException.java @@ -3,4 +3,8 @@ public class TeamNotFoundException extends RuntimeException { + public TeamNotFoundException(String errorMessage) { + super(errorMessage); + } + } \ No newline at end of file diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java index b3a9835..a00f417 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java @@ -258,6 +258,7 @@ private Future runSASTScan(String repoDir, CodeRepo codeRepo, CodeRepoBran log.warn("[ScanManagerService] SAST scan interrupted for {}.", codeRepo.getRepourl()); Thread.currentThread().interrupt(); } else { + e.printStackTrace(); log.error("[ScanManagerService] An error occurred during SAST scan for {}.", codeRepo.getRepourl()); } } finally { diff --git a/backend/src/main/resources/db/changelog/db.changelog-master.sql b/backend/src/main/resources/db/changelog/db.changelog-master.sql index e67b19e..4976bca 100644 --- a/backend/src/main/resources/db/changelog/db.changelog-master.sql +++ b/backend/src/main/resources/db/changelog/db.changelog-master.sql @@ -496,4 +496,90 @@ HAVING OR sub_f.has_high = TRUE THEN 'notable' ELSE NULL - END IS NOT NULL; \ No newline at end of file + END IS NOT NULL; + +--changeset siewer:ddl_remove +CREATE TABLE IF NOT EXISTS public.coderepo_languages ( + coderepo_id BIGINT NOT NULL, + percent_of_code INTEGER, + language VARCHAR(255) NOT NULL, + CONSTRAINT coderepo_languages_pkey PRIMARY KEY (coderepo_id, language), + CONSTRAINT fkssqutspukimdtipic5ne8in6q FOREIGN KEY (coderepo_id) + REFERENCES public.coderepo(id) + ); + +--changeset siewer:change-view-redo +DROP VIEW IF EXISTS combined_items_view; +CREATE VIEW combined_items_view AS +SELECT + c.id AS coderepo_id, + v.name AS name, + CASE + WHEN + v.epss > 0.5 + OR (v.epss BETWEEN 0.2 AND 0.5 AND pii_flag = TRUE) + OR (v.epss > 0.1 AND v.exploit_exists = TRUE) + OR has_critical = TRUE + THEN 'urgent' + WHEN + ((v.epss BETWEEN 0.1 AND 0.5) AND pii_flag = FALSE AND v.exploit_exists = FALSE) + OR (v.epss < 0.1 AND v.exploit_exists = TRUE) + OR has_high = TRUE + THEN 'notable' + ELSE NULL + END AS urgency, + COUNT(*) AS count, + v.epss AS epss, + pii_flag AS pii, + v.exploit_exists AS exploitAvailable, + ARRAY_AGG(DISTINCT c.name) AS projectNames, + ARRAY_AGG(DISTINCT c.id) AS projectIds +FROM ( + SELECT + f.coderepo_id, + f.vulnerability_id, + MAX(CASE WHEN f.source IN ('IAC', 'SAST', 'SECRETS') AND f.severity = 'CRITICAL' THEN 1 ELSE 0 END) = 1 AS has_critical, + MAX(CASE WHEN f.source IN ('IAC', 'SAST', 'SECRETS') AND f.severity = 'HIGH' THEN 1 ELSE 0 END) = 1 AS has_high + FROM finding f + WHERE f.status IN ('NEW', 'EXISTING') + GROUP BY f.coderepo_id, f.vulnerability_id + ) AS sub_f +JOIN vulnerability v ON sub_f.vulnerability_id = v.id +JOIN coderepo c ON sub_f.coderepo_id = c.id +LEFT JOIN LATERAL ( + SELECT EXISTS ( + SELECT 1 + FROM app_data_type adt + JOIN app_data_type_category_groups adtcg ON adtcg.app_data_type_id = adt.id + WHERE adt.coderepo_id = c.id + AND adtcg.category_group = 'PII' + ) AS pii_flag +) AS pii_sub ON TRUE +WHERE + v.epss > 0.1 + OR v.exploit_exists = TRUE + OR sub_f.has_critical = TRUE + OR sub_f.has_high = TRUE +GROUP BY + c.id, + v.name, + v.epss, + v.exploit_exists, + pii_sub.pii_flag, + sub_f.has_critical, + sub_f.has_high +HAVING + CASE + WHEN + v.epss > 0.5 + OR (v.epss BETWEEN 0.2 AND 0.5 AND pii_sub.pii_flag = TRUE) + OR (v.epss > 0.1 AND v.exploit_exists = TRUE) + OR sub_f.has_critical = TRUE + THEN 'urgent' + WHEN + ((v.epss BETWEEN 0.1 AND 0.5) AND pii_sub.pii_flag = FALSE AND v.exploit_exists = FALSE) + OR (v.epss < 0.1 AND v.exploit_exists = TRUE) + OR sub_f.has_high = TRUE + THEN 'notable' + ELSE NULL +END IS NOT NULL;