Bug fixes and optimization

README.md

@@ -7,7 +7,7 @@
 </p>
 
 <p align="center">
-  <a href="https://github.com/snovvcrash/DivideAndScan/blob/main/pyproject.toml#L3"><img src="https://img.shields.io/badge/version-1.0.1-success" alt="version" /></a>
+  <a href="https://github.com/snovvcrash/DivideAndScan/blob/main/pyproject.toml#L3"><img src="https://img.shields.io/badge/version-1.0.2-success" alt="version" /></a>
   <a href="https://github.com/snovvcrash/DivideAndScan/search?l=python"><img src="https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white" alt="python" /></a>
   <a href="https://www.codacy.com/gh/snovvcrash/DivideAndScan/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=snovvcrash/DivideAndScan&amp;utm_campaign=Badge_Grade"><img src="https://app.codacy.com/project/badge/Grade/35f0bdfece9846d7aab3888b01642813" alt="codacy" /></a>
   <a href="https://github.com/snovvcrash/DivideAndScan/actions/workflows/publish-to-pypi.yml"><img src="https://github.com/snovvcrash/DivideAndScan/actions/workflows/publish-to-pypi.yml/badge.svg" alt="pypi" /></a>
@@ -33,7 +33,7 @@ Potential use cases:
 \* Available port scanners:
 
 * [Nmap](https://github.com/nmap/nmap)
-* [Masscan](https://github.com/robertdavidgraham/masscan)
+* [Masscan](https://github.com/ivre/masscan)
 * [RustScan](https://github.com/RustScan/RustScan)
 * [Naabu](https://github.com/projectdiscovery/naabu)
 * [NimScan](https://github.com/elddy/NimScan)
@@ -64,7 +64,7 @@ sudo nmap --script-updatedb
 
 ```bash
 pushd /tmp
-wget https://github.com/robertdavidgraham/masscan/archive/refs/heads/master.zip -O masscan-master.zip
+wget https://github.com/ivre/masscan/archive/refs/heads/master.zip -O masscan-master.zip
 unzip masscan-master.zip
 cd masscan-master
 make
@@ -125,7 +125,7 @@ There's also a [release](https://github.com/snovvcrash/DivideAndScan/releases/la
 
 ```console
 # shiv -e das.divideandscan:main -o das.pyz divideandscan
-~$ wget https://github.com/snovvcrash/DivideAndScan/releases/latest/download/das.pyz
+~$ wget https://github.com/snovvcrash/DivideAndScan/releases/latest/download/das.pyz && chmod +x das.pyz
 ~$ ./das.pyz
 ```
 

das/common.py

@@ -2,7 +2,7 @@
 
 __author__ = '@snovvcrash'
 __site__ = 'https://github.com/snovvcrash/DivideAndScan'
-__version__ = '1.0.1'
+__version__ = '1.0.2'
 
 import time
 import shlex

das/divideandscan.py

@@ -55,7 +55,7 @@ def parse_args():
 
 	scan_epilog = """
 	examples:
-	  das scan -hosts all -show
+	  das scan -hosts {all|new} -show
 	  das scan -ports 22 -show -raw
 	  das scan -hosts 192.168.1.0/24,10.10.13.37 -oA report1 -nmap '-Pn -sVC -O'
 	  das -db testdb scan -ports 22,80,443,445 -oA report2 -parallel
@@ -93,12 +93,13 @@ def parse_args():
 	examples:
 	  das report -hosts all -show
 	  das report -hosts 192.168.1.0/24,10.10.13.37 -oA report1
-	  das report -ports 22,80,443,445 -oA report2
+	  das report -ports 22,80,443,445 -dns -oA report2
 	  das report -ports ports.txt -oA report2
 	""".replace('\t', '')
 	report_parser = subparser.add_parser('report', formatter_class=RawDescriptionHelpFormatter, epilog=report_epilog, help='merge separate Nmap outputs into a single report (https://github.com/CBHue/nMap_Merger)')
 	group_action = report_parser.add_mutually_exclusive_group(required=True)
 	group_action.add_argument('-show', action='store_true', default=False, help='only show Nmap raw reports, do not merge into a file')
+	group_action.add_argument('-dns', action='store_true', default=False, help='if a domain name is present in the DB for a specific host, then add it to the report')
 	group_action.add_argument('-oA', action='store', type=str, default=None, help='final report filename without extension (all formats: HTML, XML, simple text, grepable)')
 	group_action.add_argument('-oX', action='store', type=str, default=None, help='final report filename without extension (XML+HTML formats)')
 	group_action.add_argument('-oN', action='store', type=str, default=None, help='final report filename without extension (simple text format)')
@@ -214,7 +215,7 @@ def main():
 			elif args.ports:
 				sr.nmap_by_ports(args.nmap, parallel)
 
-			nm = NmapMerger(str(P), args.hosts, args.ports, output)
+			nm = NmapMerger(str(P), args.hosts, args.ports, args.dns, output)
 			nm.generate()
 
 	elif args.subparser == 'dns':
@@ -241,7 +242,7 @@ def main():
 			if P.exists():
 				logger.print_info(f'Using DB -> {P.resolve()}')
 
-			nm = NmapMerger(str(P), args.hosts, args.ports, output)
+			nm = NmapMerger(str(P), args.hosts, args.ports, args.dns, output)
 			nm.generate()
 
 	elif args.subparser == 'parse':

das/parsenmap.py

@@ -28,8 +28,8 @@ def __init__(self, db_path, services, dns, raw_output=False):
 		:rtype: das.report.NmapParser
 		"""
 		self.services = services.split(',')
-		self.dns = dns
 
+		self.dns = dns
 		if dns:
 			self.db = TinyDB(db_path)
 			self.ip_domains_dict = {}
@@ -43,7 +43,7 @@ def __init__(self, db_path, services, dns, raw_output=False):
 		P = list(P)
 
 		xml_reports = {x for x in P if not x.stem.startswith('port')}
-		self.xml_reports = [str(r) for r in sorted(xml_reports, key=lambda x: socket.inet_aton(x.stem.replace('-', '.')))]
+		self.xml_reports = [str(r) for r in sorted(xml_reports, key=lambda x: socket.inet_aton(x.stem))]
 
 	def parse(self):
 		"""Print raw Nmap reports in simple text format."""
@@ -58,17 +58,28 @@ def parse(self):
 					for port in nm[ip]['tcp']:
 						if nm[ip]['tcp'][port]['state'] == 'open':
 							service = nm[ip]['tcp'][port]['name']
-							if service in self.services:
+							if service in self.services or service == 'http' and 'https' in self.services:
+								if service == 'http' and 'script' in nm[ip]['tcp'][port] and 'ssl-cert' in nm[ip]['tcp'][port]['script']:
+									if 'https' in self.services:
+										service = 'https'
+									elif 'http' in self.services:
+										continue
+
+								if service == 'http' and port == 80 or service == 'https' and port == 443:
+									port = ''
+								else:
+									port = f':{port}'
+
 								if self.dns:
 									domains = self.ip_domains_dict[ip]
 									if domains:
 										for domain in domains:
 											if not self.raw_output:
-												Logger.print_success(f'IP {ip} -> {service}://{domain}:{port}')
+												Logger.print_success(f'IP {ip} -> {service}://{domain}{port}')
 											else:
-												print(f'{service}://{domain}:{port}')
+												print(f'{service}://{domain}{port}')
 										continue
 								if not self.raw_output:
-									Logger.print_success(f'{service}://{ip}:{port}')
+									Logger.print_success(f'{service}://{ip}{port}')
 								else:
-									print(f'{service}://{ip}:{port}')
+									print(f'{service}://{ip}{port}')

das/parsers/generic.py

@@ -0,0 +1,26 @@
+from das.parsers import IAddPortscanOutput
+
+
+class AddPortscanOutput(IAddPortscanOutput):
+	"""Child class for processing generic scan output."""
+
+	def parse(self):
+		"""
+		Generic scan raw output parser.
+
+		:return: a pair of values (portscan raw output filename, number of hosts added to DB)
+		:rtype: tuple
+		"""
+		items, hosts = [], set()
+		for line in self.portscan_raw:
+			try:
+				ip, port = line.split(':')
+			except Exception:
+				pass
+			else:
+				items.append({'ip': ip, 'port': int(port), 'domains': []})
+				hosts.add(ip)
+
+		self.db.insert_multiple(items)
+
+		return (self.portscan_out, len(hosts))

das/report.py

@@ -19,7 +19,7 @@
 class NmapMerger:
 	"""Class for merging separate Nmap outputs into a single report in different formats."""
 
-	def __init__(self, db_path, hosts, ports, output=None):
+	def __init__(self, db_path, hosts, ports, dns, output=None):
 		"""
 		Constructor.
 
@@ -42,8 +42,12 @@ def __init__(self, db_path, hosts, ports, output=None):
 		else:
 			self.output = None
 
-		self.db = None
-		self.db_path = db_path
+		self.dns = dns
+		if dns:
+			self.db = TinyDB(db_path)
+			self.ip_domains_dict = {}
+			for item in self.db.all():
+				self.ip_domains_dict[item['ip']] = item['domains']
 
 		db_name = Path(db_path).stem
 		P = (Path.home() / '.das' / f'nmap_{db_name}').glob('*.*')
@@ -62,7 +66,7 @@ def __init__(self, db_path, hosts, ports, output=None):
 			else:
 				hosts = hosts.split(',')
 				hosts = [IPNetwork(h) for h in hosts]
-				hosts = [str(ip).replace('.', '-') for ip_obj in hosts for ip in ip_obj]
+				hosts = [str(ip) for ip_obj in hosts for ip in ip_obj]
 				self.nmap_reports = {x for h in hosts for x in P if h == x.stem}
 
 		elif ports:
@@ -104,7 +108,7 @@ def show(self):
 		Logger.print_info(f'Total reports -> {total_reports}')
 
 		try:
-			sorted_reports = [str(r) for r in sorted(text_reports, key=lambda x: socket.inet_aton(x.stem.replace('-', '.')))]
+			sorted_reports = [str(r) for r in sorted(text_reports, key=lambda x: socket.inet_aton(x.stem))]
 		except OSError:
 			sorted_reports = [str(r) for r in sorted(text_reports)]
 
@@ -116,9 +120,6 @@ def show(self):
 
 	def generate(self):
 		"""Perform all the steps needed to generate a single Nmap report."""
-		self.db = TinyDB(self.db_path)
-		self.Host = Query()
-
 		if self.output.format in ('oX', 'oA'):
 			merged_xml = f'{self.output.filename}.xml'
 			for report in self.nmap_reports:
@@ -147,7 +148,7 @@ def generate(self):
 			text_reports = [r for r in self.nmap_reports if r.suffix == '.nmap']
 
 			try:
-				sorted_reports = ' '.join(str(r) for r in sorted(text_reports, key=lambda x: socket.inet_aton(x.stem.replace('-', '.'))))
+				sorted_reports = ' '.join(str(r) for r in sorted(text_reports, key=lambda x: socket.inet_aton(x.stem)))
 			except OSError:
 				sorted_reports = ' '.join(str(r) for r in sorted(text_reports))
 
@@ -164,7 +165,7 @@ def generate(self):
 			grepable_reports = [r for r in self.nmap_reports if r.suffix == '.gnmap']
 
 			try:
-				sorted_reports = ' '.join(str(r) for r in sorted(grepable_reports, key=lambda x: socket.inet_aton(x.stem.replace('-', '.'))))
+				sorted_reports = ' '.join(str(r) for r in sorted(grepable_reports, key=lambda x: socket.inet_aton(x.stem)))
 			except OSError:
 				sorted_reports = ' '.join(str(r) for r in sorted(grepable_reports))
 
@@ -257,11 +258,8 @@ def merge_nmap(self, xml_file, merged_xml):
 					hostnames = host.find('hostnames')
 					if hostnames is not None:
 						ip = host.find('address').attrib['addr']
-						try:
-							domains = self.db.search(self.Host.ip == ip)[0]['domains']
-						except:
-							pass
-						else:
+						if self.dns:
+							domains = self.ip_domains_dict[ip]
 							if domains:
 								for domain in domains:
 									hostname = SubElement(hostnames, 'hostname')

das/scan.py

@@ -35,7 +35,7 @@ def __init__(self, db_path, hosts, ports, limit=None, raw_output=False):
 		"""
 		self.db_name = Path(db_path).stem
 		self.db = TinyDB(db_path)
-		self.Host = Query()
+		#self.Host = Query()
 		self.total_scans = 0
 
 		if hosts:
@@ -48,11 +48,17 @@ def __init__(self, db_path, hosts, ports, limit=None, raw_output=False):
 
 			if hosts == 'all':
 				result = self.db.all()
+			elif hosts == 'new':
+				P = (Path.home() / '.das' / f'nmap_{self.db_name}').glob('*.*')
+				hosts = [ip.stem for ip in P]
+				#result = self.db.search(~(self.Host.ip.one_of(hosts)))
+				result = [item[ip] for item in self.db.all() for ip in hosts if item[ip] not in hosts]
 			else:
 				hosts = hosts.split(',')
 				hosts = [IPNetwork(h) for h in hosts]
 				hosts = [str(ip) for ip_obj in hosts for ip in ip_obj]
-				result = self.db.search(self.Host.ip.one_of(hosts))
+				#result = self.db.search(self.Host.ip.one_of(hosts))
+				result = [item[ip] for item in self.db.all() for ip in hosts]
 
 			self.ip_ports_dict, self.ip_domains_dict = defaultdict(set), {}
 			for item in result:
@@ -73,7 +79,8 @@ def __init__(self, db_path, hosts, ports, limit=None, raw_output=False):
 				result = self.db.all()
 			else:
 				ports = [int(p) for p in ports.split(',')]
-				result = self.db.search(self.Host.port.one_of(ports))
+				#result = self.db.search(self.Host.port.one_of(ports))
+				result = [item[port] for item in self.db.all() for port in ports]
 
 			self.port_ip_dict = defaultdict(set)
 			for item in result:
@@ -115,6 +122,9 @@ def nmap_by_hosts(self, dns):
 			if self.raw_output:
 				for port in sorted_ports:
 					print(f'{ip}:{port}')
+					if dns:
+						for domain in self.ip_domains_dict[ip]:
+							print(f'{domain}:{port}')
 			elif dns:
 				domains = f'[{",".join(self.ip_domains_dict[ip])}]'
 				Logger.print_success(f'IP {ip}, Domains {domains} ({len(ports)}) -> [{",".join([str(p) for p in sorted_ports])}]')
@@ -151,7 +161,7 @@ def nmap_by_hosts(self, nmap_opts, parallel):
 			if not parallel.enabled:
 				Logger.print_separator(f'IP: {ip}', prefix=f'{i}/{self.total_scans}')
 
-			nmap_out = ip.replace('.', '-')
+			nmap_out = ip
 			sorted_ports = ','.join([str(p) for p in sorted(ports)])
 
 			if nmap_opts is None:

docker-install.sh

@@ -8,7 +8,7 @@ nmap --script-updatedb
 # Masscan
 
 pushd /tmp 2>&1 > /dev/null
-wget https://github.com/robertdavidgraham/masscan/archive/refs/heads/master.zip -O masscan-master.zip
+wget https://github.com/ivre/masscan/archive/refs/heads/master.zip -O masscan-master.zip
 unzip -q masscan-master.zip
 cd masscan-master
 make

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "divideandscan"
-version = "1.0.1"
+version = "1.0.2"
 description = "Divide full port scan results and use it for targeted Nmap runs"
 authors = ["Sam Freeside <[email protected]>"]
 license = "BSD-2-Clause"