chore(*): -

go.mod

@@ -6,7 +6,7 @@ require (
 	dario.cat/mergo v1.0.1
 	github.com/hueristiq/hq-go-http v0.0.0-20241020113552-532feebd5687
 	github.com/hueristiq/hq-go-limiter v0.0.0-20241020114425-bdc49852dc29
-	github.com/hueristiq/hq-go-url v0.0.0-20241020144539-a9e1f60005ea
+	github.com/hueristiq/hq-go-url v0.0.0-20241021202657-1f82b6fb3ffa
 	github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f
 	github.com/logrusorgru/aurora/v3 v3.0.0
 	github.com/spf13/cast v1.7.0
@@ -20,9 +20,9 @@ require (
 require (
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/hueristiq/hq-go-retrier v0.0.0-20241020110813-ef8a550b01d5 // indirect
+	github.com/hueristiq/hq-go-retrier v0.0.0-20241113111747-73f1e3dac41d // indirect
 	github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -34,10 +34,10 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/term v0.25.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
+	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	golang.org/x/term v0.26.0 // indirect
+	golang.org/x/text v0.20.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )

go.sum

@@ -7,8 +7,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -17,10 +17,10 @@ github.com/hueristiq/hq-go-http v0.0.0-20241020113552-532feebd5687 h1:wbtQCCbsyY
 github.com/hueristiq/hq-go-http v0.0.0-20241020113552-532feebd5687/go.mod h1:4cIeUJTM4gt2NgJ4jOZePenVWaY8337wB1pvsK6sYDs=
 github.com/hueristiq/hq-go-limiter v0.0.0-20241020114425-bdc49852dc29 h1:5MT7d/clp5I73BvazJrF4A9IkcKDCkdhaPyqL7gQrjE=
 github.com/hueristiq/hq-go-limiter v0.0.0-20241020114425-bdc49852dc29/go.mod h1:n1ODyTZYMyUuOIQCqHZgvobG8zWY/gxQaGeMiSnUnZw=
-github.com/hueristiq/hq-go-retrier v0.0.0-20241020110813-ef8a550b01d5 h1:uSIqfeqkXZI/QciepvLVduqbU7Rq+jr+At0ENVjPIN4=
-github.com/hueristiq/hq-go-retrier v0.0.0-20241020110813-ef8a550b01d5/go.mod h1:YkxIHoJHsL0wmzQ3tc0qz4UTr9q9eCicUt5RvMV//xw=
-github.com/hueristiq/hq-go-url v0.0.0-20241020144539-a9e1f60005ea h1:aFUvZ+Bnae4Coo97oThYy6OmuIwEkrQNGesMzAidedc=
-github.com/hueristiq/hq-go-url v0.0.0-20241020144539-a9e1f60005ea/go.mod h1:1q7KVF3MOodsQzUkWwDwqn62L0Yjj8nLDSqZF0oirgQ=
+github.com/hueristiq/hq-go-retrier v0.0.0-20241113111747-73f1e3dac41d h1:ocitia1y84fr6jT77Fcuq04g8bfYwCGkTSDLhcZ29Oo=
+github.com/hueristiq/hq-go-retrier v0.0.0-20241113111747-73f1e3dac41d/go.mod h1:YkxIHoJHsL0wmzQ3tc0qz4UTr9q9eCicUt5RvMV//xw=
+github.com/hueristiq/hq-go-url v0.0.0-20241021202657-1f82b6fb3ffa h1:KL5vQRk2nJxeWi2Xq+dXpzGOtiwEeXW5yjy1qGLriug=
+github.com/hueristiq/hq-go-url v0.0.0-20241021202657-1f82b6fb3ffa/go.mod h1:1q7KVF3MOodsQzUkWwDwqn62L0Yjj8nLDSqZF0oirgQ=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f h1:JAgZOIJ+UbkENpRiOTlfg51CW0UNrUkgwLjUGiH+x9g=
 github.com/hueristiq/hqgolog v0.0.0-20230623113334-a6018965a34f/go.mod h1:S5J3E3Azva5+JKv67uc+Hh3XwLDvkVYDGjEaMTFrIqg=
 github.com/hueristiq/hqgoutils v0.0.0-20231024005153-bd2c47932440 h1:dpHAa9c74HgAXkZ2WPd84q2cCiF76eluuSGRw7bk7To=
@@ -65,16 +65,16 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ
 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
-golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
-golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
+golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
+golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/xurlfind3r/sources/utils_test.go

@@ -7,6 +7,8 @@ import (
 )
 
 func TestFixURL(t *testing.T) {
+	t.Parallel()
+
 	t.Run("Remove Quotes and Spaces", func(t *testing.T) {
 		url := "\"   http://example.com/path   \""
 		expected := "http://example.com/path"
@@ -51,6 +53,8 @@ func TestFindUnbalancedQuote(t *testing.T) {
 }
 
 func TestFindUnbalancedBracket(t *testing.T) {
+	t.Parallel()
+
 	t.Run("Balanced Brackets", func(t *testing.T) {
 		s := "text with [balanced] brackets"
 		assert.Equal(t, -1, findUnbalancedBracket(s, '[', ']'))

pkg/xurlfind3r/xurlfind3r.go

@@ -18,11 +18,7 @@ import (
 // Finder is the main structure that manages the interaction with OSINT sources.
 // It holds the available data sources and the configuration used for searching.
 type Finder struct {
-	// sources is a map of source names to their corresponding implementations.
-	// Each source implements the Source interface, which allows domain searches.
-	sources map[string]sources.Source
-	// configuration contains configuration options such as API keys
-	// and other settings needed by the data sources.
+	sources       map[string]sources.Source
 	configuration *sources.Configuration
 	FilterRegex   *regexp.Regexp
 	MatchRegex    *regexp.Regexp
@@ -38,17 +34,17 @@ func (finder *Finder) IsURLInScope(domain, URL string, subdomainsInScope bool) (
 		return
 	}
 
-	ETLDPlusOne := parsedURL.Domain.Root
+	ETLDPlusOne := parsedURL.Domain.SLD
 
-	if parsedURL.Domain.TopLevel != "" {
-		ETLDPlusOne += "." + parsedURL.Domain.TopLevel
+	if parsedURL.Domain.TLD != "" {
+		ETLDPlusOne += "." + parsedURL.Domain.TLD
 	}
 
 	parsedDomain := dp.Parse(domain)
 
-	expectedETLDPlusOne := parsedDomain.Root
-	if parsedDomain.TopLevel != "" {
-		expectedETLDPlusOne += "." + parsedDomain.TopLevel
+	expectedETLDPlusOne := parsedDomain.SLD
+	if parsedDomain.TLD != "" {
+		expectedETLDPlusOne += "." + parsedDomain.TLD
 	}
 
 	if ETLDPlusOne != expectedETLDPlusOne {
@@ -71,50 +67,35 @@ func (finder *Finder) IsURLInScope(domain, URL string, subdomainsInScope bool) (
 // the sources specified in the configuration. It returns a channel through which
 // the search results (of type Result) are streamed asynchronously.
 func (finder *Finder) Find(domain string) (results chan sources.Result) {
-	// Initialize the results channel where URLs findings are sent.
 	results = make(chan sources.Result)
 
-	// Parse the given domain using a domain parser.
 	parsed := dp.Parse(domain)
 
-	// Rebuild the domain as "root.tld" format.
-	domain = parsed.Root + "." + parsed.TopLevel
+	domain = parsed.SLD + "." + parsed.TLD
 
 	finder.configuration.IsInScope = func(URL string) (isInScope bool) {
 		return finder.IsURLInScope(domain, URL, finder.configuration.IncludeSubdomains)
 	}
 
-	// Launch a goroutine to perform the search concurrently across all sources.
 	go func() {
-		// Ensure the results channel is closed once all search operations complete.
 		defer close(results)
 
-		// A thread-safe map to store already-seen URLs, avoiding duplicates.
 		seenURLs := &sync.Map{}
 
-		// WaitGroup ensures all source goroutines finish before exiting.
 		wg := &sync.WaitGroup{}
 
-		// Iterate over all the sources in the Finder.
 		for name := range finder.sources {
 			wg.Add(1)
 
-			// Start a new goroutine for each source to fetch URLs concurrently.
 			go func(source sources.Source) {
-				// Decrement the WaitGroup counter when this goroutine completes.
 				defer wg.Done()
 
-				// Call the source's Run method to start the subdomain search.
 				sResults := source.Run(finder.configuration, domain)
 
-				// Process each result as it's received from the source.
 				for sResult := range sResults {
-					// If the result is a subdomain, process it.
 					if sResult.Type == sources.ResultURL {
-						// Check if the subdomain has already been seen using sync.Map.
 						_, loaded := seenURLs.LoadOrStore(sResult.Value, struct{}{})
 						if loaded {
-							// If the subdomain is already in the map, skip it.
 							continue
 						}
 
@@ -123,34 +104,26 @@ func (finder *Finder) Find(domain string) (results chan sources.Result) {
 						}
 					}
 
-					// Send the result down the results channel.
 					results <- sResult
 				}
 			}(finder.sources[name])
 		}
 
-		// Wait for all goroutines to finish before exiting.
 		wg.Wait()
 	}()
 
-	// Return the channel that will stream URL results.
 	return
 }
 
 // Configuration holds the configuration for Finder, including
 // the sources to use, sources to exclude, and the necessary API keys.
 type Configuration struct {
 	IncludeSubdomains bool
-
-	// SourcesToUse is a list of source names that should be used for the search.
-	SourcesToUse []string
-	// SourcesToExclude is a list of source names that should be excluded from the search.
-	SourcesToExclude []string
-	// Keys contains the API keys for each data source.
-	Keys sources.Keys
-
-	FilterPattern string
-	MatchPattern  string
+	SourcesToUse      []string
+	SourcesToExclude  []string
+	Keys              sources.Keys
+	FilterPattern     string
+	MatchPattern      string
 }
 
 var (
@@ -162,7 +135,6 @@ var (
 // New creates a new Finder instance based on the provided Configuration.
 // It initializes the Finder with the selected sources and ensures that excluded sources are not used.
 func New(cfg *Configuration) (finder *Finder, err error) {
-	// Initialize a Finder instance with an empty map of sources and the provided configuration.
 	finder = &Finder{
 		sources: map[string]sources.Source{},
 		configuration: &sources.Configuration{
@@ -185,14 +157,11 @@ func New(cfg *Configuration) (finder *Finder, err error) {
 		}
 	}
 
-	// If no specific sources are provided, use the default list of all sources.
 	if len(cfg.SourcesToUse) < 1 {
 		cfg.SourcesToUse = sources.List
 	}
 
-	// Loop through the selected sources and initialize each one
 	for _, source := range cfg.SourcesToUse {
-		// Depending on the source name, initialize the appropriate source and add it to the map.
 		switch source {
 		case sources.BEVIGIL:
 			finder.sources[source] = &bevigil.Source{}
@@ -211,13 +180,11 @@ func New(cfg *Configuration) (finder *Finder, err error) {
 		}
 	}
 
-	// Remove any sources that are specified in the SourcesToExclude list.
 	for index := range cfg.SourcesToExclude {
 		source := cfg.SourcesToExclude[index]
 
 		delete(finder.sources, source)
 	}
 
-	// Return the Finder instance with all the selected sources.
 	return
 }