fix: fixes from scs secret scanner (#231)

- chore: move ignore lists into engine config
- Refactor isSecretIgnored function to include allowed values
- plaid-client-id is unstable too #226
- Update stop words in AuthenticatedURL rule

README.md

@@ -170,6 +170,7 @@ Additional Commands:
 Flags:
       --add-special-rule strings      special (non-default) rules to apply.
                                       This list is not affected by the --rule and --ignore-rule flags.
+      --allowed-values strings        allowed secrets values to ignore
       --config string                 config file path
   -h, --help                          help for 2ms
       --ignore-on-exit ignoreOnExit   defines which kind of non-zero exits code should be ignored

cmd/main.go

@@ -28,6 +28,7 @@ const (
 	ruleFlagName               = "rule"
 	ignoreRuleFlagName         = "ignore-rule"
 	ignoreFlagName             = "ignore-result"
+	allowedValuesFlagName      = "allowed-values"
 	specialRulesFlagName       = "add-special-rule"
 	ignoreOnExitFlagName       = "ignore-on-exit"
 	maxTargetMegabytesFlagName = "max-target-megabytes"
@@ -39,7 +40,6 @@ var (
 	reportPathVar      []string
 	stdoutFormatVar    string
 	customRegexRuleVar []string
-	ignoreVar          []string
 	ignoreOnExitVar    = ignoreOnExitNone
 	engineConfigVar    engine.EngineConfig
 	validateVar        bool
@@ -90,7 +90,8 @@ func Execute() (int, error) {
 	rootCmd.PersistentFlags().StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex")
 	rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.SelectedList, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan")
 	rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.IgnoreList, ignoreRuleFlagName, []string{}, "ignore rules by name or tag")
-	rootCmd.PersistentFlags().StringSliceVar(&ignoreVar, ignoreFlagName, []string{}, "ignore specific result by id")
+	rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.IgnoredIds, ignoreFlagName, []string{}, "ignore specific result by id")
+	rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.AllowedValues, allowedValuesFlagName, []string{}, "allowed secrets values to ignore")
 	rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.SpecialList, specialRulesFlagName, []string{}, "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.")
 	rootCmd.PersistentFlags().Var(&ignoreOnExitVar, ignoreOnExitFlagName, "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\nexample: if 'results' is set, only engine errors will make 2ms exit code different from 0")
 	rootCmd.PersistentFlags().IntVar(&engineConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, "files larger than this will be skipped.\nOmit or set to 0 to disable this check.")

cmd/workers.go

@@ -14,7 +14,7 @@ func processItems(engine *engine.Engine) {
 	for item := range channels.Items {
 		report.TotalItemsScanned++
 		wgItems.Add(1)
-		go engine.Detect(item, secretsChan, wgItems, ignoreVar)
+		go engine.Detect(item, secretsChan, wgItems)
 	}
 	wgItems.Wait()
 	close(secretsChan)

docs/list-of-rules.md

@@ -113,8 +113,6 @@ Here is a complete list of all the rules that are currently implemented.
 | okta-access-token | Identified an Okta Access Token, which may compromise identity management services and user authentication data. | access-token |  |
 | openai-api-key | Found an OpenAI API Key, posing a risk of unauthorized access to AI services and data manipulation. | api-key |  |
 | plaid-client-id | Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches. | client-id |  |
-| plaid-secret-key | Detected a Plaid Secret key, risking unauthorized access to financial accounts and sensitive transaction data. | secret-key |  |
-| plaid-api-token | Discovered a Plaid API Token, potentially compromising financial data aggregation and banking services. | api-token |  |
 | planetscale-password | Discovered a PlanetScale password, which could lead to unauthorized database operations and data breaches. | password |  |
 | planetscale-api-token | Identified a PlanetScale API token, potentially compromising database management and operations. | api-token |  |
 | planetscale-oauth-token | Found a PlanetScale OAuth token, posing a risk to database access control and sensitive data integrity. | access-token |  |

engine/engine.go

@@ -24,6 +24,9 @@ type Engine struct {
 	rules     map[string]config.Rule
 	detector  detect.Detector
 	validator validation.Validator
+
+	ignoredIds    []string
+	allowedValues []string
 }
 
 const customRegexRuleIdFormat = "custom-regex-%d"
@@ -34,6 +37,9 @@ type EngineConfig struct {
 	SpecialList  []string
 
 	MaxTargetMegabytes int
+
+	IgnoredIds    []string
+	AllowedValues []string
 }
 
 func Init(engineConfig EngineConfig) (*Engine, error) {
@@ -60,17 +66,20 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 		rules:     rulesToBeApplied,
 		detector:  *detector,
 		validator: *validation.NewValidator(),
+
+		ignoredIds:    engineConfig.IgnoredIds,
+		allowedValues: engineConfig.AllowedValues,
 	}, nil
 }
 
-func (s *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, ignoredIds []string) {
+func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup) {
 	defer wg.Done()
 
 	fragment := detect.Fragment{
 		Raw:      *item.GetContent(),
 		FilePath: item.GetSource(),
 	}
-	for _, value := range s.detector.Detect(fragment) {
+	for _, value := range e.detector.Detect(fragment) {
 		itemId := getFindingId(item, value)
 		secret := &secrets.Secret{
 			ID:          itemId,
@@ -82,15 +91,15 @@ func (s *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
 			EndColumn:   value.EndColumn,
 			Value:       value.Secret,
 		}
-		if !isSecretIgnored(secret, &ignoredIds) {
+		if !isSecretIgnored(secret, &e.ignoredIds, &e.allowedValues) {
 			secretsChannel <- secret
 		} else {
 			log.Debug().Msgf("Secret %s was ignored", secret.ID)
 		}
 	}
 }
 
-func (s *Engine) AddRegexRules(patterns []string) error {
+func (e *Engine) AddRegexRules(patterns []string) error {
 	for idx, pattern := range patterns {
 		regex, err := regexp.Compile(pattern)
 		if err != nil {
@@ -102,7 +111,7 @@ func (s *Engine) AddRegexRules(patterns []string) error {
 			Regex:       regex,
 			Keywords:    []string{},
 		}
-		s.rules[rule.RuleID] = rule
+		e.rules[rule.RuleID] = rule
 	}
 	return nil
 }
@@ -122,7 +131,12 @@ func getFindingId(item plugins.ISourceItem, finding report.Finding) string {
 	return fmt.Sprintf("%x", sha)
 }
 
-func isSecretIgnored(secret *secrets.Secret, ignoredIds *[]string) bool {
+func isSecretIgnored(secret *secrets.Secret, ignoredIds, allowedValues *[]string) bool {
+	for _, allowedValue := range *allowedValues {
+		if secret.Value == allowedValue {
+			return true
+		}
+	}
 	for _, ignoredId := range *ignoredIds {
 		if secret.ID == ignoredId {
 			return true

engine/engine_test.go

@@ -77,7 +77,7 @@ func TestDetector(t *testing.T) {
 		secretsChan := make(chan *secrets.Secret, 1)
 		wg := &sync.WaitGroup{}
 		wg.Add(1)
-		detector.Detect(i, secretsChan, wg, nil)
+		detector.Detect(i, secretsChan, wg)
 		close(secretsChan)
 
 		s := <-secretsChan
@@ -152,7 +152,7 @@ func TestSecrets(t *testing.T) {
 			secretsChan := make(chan *secrets.Secret, 1)
 			wg := &sync.WaitGroup{}
 			wg.Add(1)
-			detector.Detect(item{content: &secret.Content}, secretsChan, wg, nil)
+			detector.Detect(item{content: &secret.Content}, secretsChan, wg)
 			close(secretsChan)
 
 			s := <-secretsChan

engine/rules/authenticated_url.go

@@ -15,7 +15,7 @@ func AuthenticatedURL() *config.Rule {
 		Keywords:    []string{"://"},
 		SecretGroup: 1,
 		Allowlist: config.Allowlist{
-			StopWords: []string{"password"},
+			StopWords: []string{"password", "pass"},
 		},
 	}
 

engine/rules/rules.go

@@ -137,8 +137,8 @@ func getDefaultRules() *[]Rule {
 		{Rule: *rules.OktaAccessToken(), Tags: []string{TagAccessToken}},
 		{Rule: *rules.OpenAI(), Tags: []string{TagApiKey}},
 		{Rule: *rules.PlaidAccessID(), Tags: []string{TagClientId}},
-		{Rule: *rules.PlaidSecretKey(), Tags: []string{TagSecretKey}},
-		{Rule: *rules.PlaidAccessToken(), Tags: []string{TagApiToken}},
+		// {Rule: *rules.PlaidSecretKey(), Tags: []string{TagSecretKey}}, https://github.com/Checkmarx/2ms/issues/226
+		// {Rule: *rules.PlaidAccessToken(), Tags: []string{TagApiToken}}, https://github.com/Checkmarx/2ms/issues/226
 		{Rule: *rules.PlanetScalePassword(), Tags: []string{TagPassword}},
 		{Rule: *rules.PlanetScaleAPIToken(), Tags: []string{TagApiToken}},
 		{Rule: *rules.PlanetScaleOAuthToken(), Tags: []string{TagAccessToken}},