Merge pull request #90 from OSINT-TECHNOLOGIES/rolling

Stabilized v1.1.2

README.md

@@ -143,8 +143,8 @@ If you have problems with starting installer.sh, you should try to use `dos2unix
 
 # Tasks to complete before new release
 - [x] Rework Google Dorking module in separate mode
-- [ ] Rework Google Dorks list into separate databases with different pre-configured dorks for various purposes
-- [ ] Allow user to create their own dorks DB
+- [x] Rework Google Dorks list into separate databases with different pre-configured dorks for various purposes
+- [x] Allow user to create their own dorks DB
 - [ ] Add separate API search mode with different free APIs
 
 # DPULSE mentions in social medias

datagather_modules/crawl_processor.py

@@ -113,25 +113,27 @@ def sm_gather(url):
                          'VKontakte': [], 'YouTube': [], 'Odnoklassniki': [], 'WeChat': []}
 
     for link in links:
-        if 'facebook.com' in link:
+        parsed_url = urlparse(link)
+        hostname = parsed_url.hostname
+        if hostname and hostname.endswith('facebook.com'):
             categorized_links['Facebook'].append(urllib.parse.unquote(link))
-        elif 'twitter.com' in link:
+        elif hostname and hostname.endswith('twitter.com'):
             categorized_links['Twitter'].append(urllib.parse.unquote(link))
-        elif 'instagram.com' in link:
+        elif hostname and hostname.endswith('instagram.com'):
             categorized_links['Instagram'].append(urllib.parse.unquote(link))
-        elif 't.me' in link:
+        elif hostname and hostname.endswith('t.me'):
             categorized_links['Telegram'].append(urllib.parse.unquote(link))
-        elif 'tiktok.com' in link:
+        elif hostname and hostname.endswith('tiktok.com'):
             categorized_links['TikTok'].append(urllib.parse.unquote(link))
-        elif 'linkedin.com' in link:
+        elif hostname and hostname.endswith('linkedin.com'):
             categorized_links['LinkedIn'].append(urllib.parse.unquote(link))
-        elif 'vk.com' in link:
+        elif hostname and hostname.endswith('vk.com'):
             categorized_links['VKontakte'].append(urllib.parse.unquote(link))
-        elif 'youtube.com' in link:
+        elif hostname and hostname.endswith('youtube.com'):
             categorized_links['YouTube'].append(urllib.parse.unquote(link))
-        elif 'wechat.com' in link:
+        elif hostname and hostname.endswith('wechat.com'):
             categorized_links['WeChat'].append(urllib.parse.unquote(link))
-        elif 'ok.ru' in link:
+        elif hostname and hostname.endswith('ok.ru'):
             categorized_links['Odnoklassniki'].append(urllib.parse.unquote(link))
 
     if not categorized_links['Odnoklassniki']:
@@ -211,25 +213,26 @@ def domains_reverse_research(subdomains, report_file_type):
 
     for inner_list in subdomain_socials_grouped:
         for link in inner_list:
-            if 'facebook.com' in link:
+            hostname = urlparse(link).hostname
+            if hostname and hostname.endswith('facebook.com'):
                 sd_socials['Facebook'].append(urllib.parse.unquote(link))
-            elif 'twitter.com' in link:
+            elif hostname and hostname.endswith('twitter.com'):
                 sd_socials['Twitter'].append(urllib.parse.unquote(link))
-            elif 'instagram.com' in link:
+            elif hostname and hostname.endswith('instagram.com'):
                 sd_socials['Instagram'].append(urllib.parse.unquote(link))
-            elif 't.me' in link:
+            elif hostname and hostname.endswith('t.me'):
                 sd_socials['Telegram'].append(urllib.parse.unquote(link))
-            elif 'tiktok.com' in link:
+            elif hostname and hostname.endswith('tiktok.com'):
                 sd_socials['TikTok'].append(urllib.parse.unquote(link))
-            elif 'linkedin.com' in link:
+            elif hostname and hostname.endswith('linkedin.com'):
                 sd_socials['LinkedIn'].append(urllib.parse.unquote(link))
-            elif 'vk.com' in link:
+            elif hostname and hostname.endswith('vk.com'):
                 sd_socials['VKontakte'].append(urllib.parse.unquote(link))
-            elif 'youtube.com' in link:
+            elif hostname and hostname.endswith('youtube.com'):
                 sd_socials['YouTube'].append(urllib.parse.unquote(link))
-            elif 'wechat.com' in link:
+            elif hostname and hostname.endswith('wechat.com'):
                 sd_socials['WeChat'].append(urllib.parse.unquote(link))
-            elif 'ok.ru' in link:
+            elif hostname and hostname.endswith('ok.ru'):
                 sd_socials['Odnoklassniki'].append(urllib.parse.unquote(link))
 
     sd_socials = {k: list(set(v)) for k, v in sd_socials.items()}

datagather_modules/data_assembler.py

@@ -22,36 +22,46 @@
     sys.exit()
 
 def establishing_dork_db_connection(dorking_flag):
-    if dorking_flag == 'basic':
-        dorking_db_path = 'dorking//basic_dorking.db'
-        table = 'basic_dorks'
-    elif dorking_flag == 'iot':
-        dorking_db_path = 'dorking//iot_dorking.db'
-        table = 'iot_dorks'
-    elif dorking_flag == 'files':
-        dorking_db_path = 'dorking//files_dorking.db'
-        table = 'files_dorks'
+    dorking_db_paths = {
+        'basic': 'dorking//basic_dorking.db',
+        'iot': 'dorking//iot_dorking.db',
+        'files': 'dorking//files_dorking.db',
+        'admins': 'dorking//adminpanels_dorking.db',
+        'web': 'dorking//webstructure_dorking.db',
+    }
+    dorking_tables = {
+        'basic': 'basic_dorks',
+        'iot': 'iot_dorks',
+        'files': 'files_dorks',
+        'admins': 'admins_dorks',
+        'web': 'web_dorks',
+    }
+    if dorking_flag in dorking_db_paths:
+        dorking_db_path = dorking_db_paths[dorking_flag]
+        table = dorking_tables[dorking_flag]
+    elif dorking_flag.startswith('custom'):
+        lst = dorking_flag.split('+')
+        dorking_db_name = lst[1]
+        dorking_db_path = 'dorking//' + dorking_db_name
+        table = 'dorks'
+    else:
+        raise ValueError(f"Invalid dorking flag: {dorking_flag}")
     return dorking_db_path, table
 
 class DataProcessing():
     def report_preprocessing(self, short_domain, report_file_type):
         report_ctime = datetime.now().strftime('%d-%m-%Y, %H:%M:%S')
         files_ctime = datetime.now().strftime('(%d-%m-%Y, %Hh%Mm%Ss)')
         files_body = short_domain.replace(".", "") + '_' + files_ctime
-        if report_file_type == 'pdf':
-            casename = files_body + '.pdf'
-        elif report_file_type == 'xlsx':
-            casename = files_body + '.xlsx'
-        elif report_file_type == 'html':
-            casename = files_body + '.html'
+        casename = f"{files_body}.{report_file_type}"
         foldername = files_body
         db_casename = short_domain.replace(".", "")
         now = datetime.now()
         db_creation_date = str(now.year) + str(now.month) + str(now.day)
-        report_folder = "report_{}".format(foldername)
-        robots_filepath = report_folder + '//01-robots.txt'
-        sitemap_filepath = report_folder + '//02-sitemap.txt'
-        sitemap_links_filepath = report_folder + '//03-sitemap_links.txt'
+        report_folder = f"report_{foldername}"
+        robots_filepath = os.path.join(report_folder, '01-robots.txt')
+        sitemap_filepath = os.path.join(report_folder, '02-sitemap.txt')
+        sitemap_links_filepath = os.path.join(report_folder, '03-sitemap_links.txt')
         os.makedirs(report_folder, exist_ok=True)
         return casename, db_casename, db_creation_date, robots_filepath, sitemap_filepath, sitemap_links_filepath, report_file_type, report_folder, files_ctime, report_ctime
 
@@ -129,20 +139,9 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                 pass
 
             if dorking_flag == 'none':
-                pass
                 dorking_status = 'Google Dorking mode was not selected for this scan'
                 dorking_file_path = 'Google Dorking mode was not selected for this scan'
-            elif dorking_flag == 'basic':
-                dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-            elif dorking_flag == 'iot':
-                dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-            elif dorking_flag == 'files':
+            else:
                 dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
                 dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
@@ -180,23 +179,12 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                 pass
 
             if dorking_flag == 'none':
-                pass
                 dorking_status = 'Google Dorking mode was not selected for this scan'
-                dorking_results = 'Google Dorking mode was not selected for this scan'
-            elif dorking_flag == 'basic':
-                dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_results = dp.transfer_results_to_xlsx(table, dp.get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-            elif dorking_flag == 'iot':
-                dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_results = dp.transfer_results_to_xlsx(table, dp.get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-            elif dorking_flag == 'files':
+                dorking_file_path = 'Google Dorking mode was not selected for this scan'
+            else:
                 dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_results = dp.transfer_results_to_xlsx(table, dp.get_dorking_query(short_domain, dorking_db_path, table))
+                dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
 
             data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
@@ -234,20 +222,9 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                 pass
 
             if dorking_flag == 'none':
-                pass
                 dorking_status = 'Google Dorking mode was not selected for this scan'
                 dorking_file_path = 'Google Dorking mode was not selected for this scan'
-            elif dorking_flag == 'basic':
-                dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-            elif dorking_flag == 'iot':
-                dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-                dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
-            elif dorking_flag == 'files':
+            else:
                 dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
                 dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, dp.get_dorking_query(short_domain, dorking_db_path, table))

datagather_modules/networking_processor.py

@@ -40,6 +40,7 @@ def get_ssl_certificate(short_domain, port=443):
     try:
         logging.info('SSL CERTIFICATE GATHERING: OK')
         context = ssl.create_default_context()
+        context.minimum_version = ssl.TLSVersion.TLSv1_2
         conn = socket.create_connection((short_domain, port))
         sock = context.wrap_socket(conn, server_hostname=short_domain)
         cert = sock.getpeercert()

dorking/db_creator.py

@@ -0,0 +1,36 @@
+import sqlite3
+from colorama import Fore
+import os
+
+def manage_dorks(db_name):
+    db_prep_string = str(db_name) + '.db'
+    if os.path.exists('dorking//' + db_prep_string):
+        print(Fore.RED + f"Sorry, but {db_prep_string} database is already exists. Choose other name for your custom DB")
+        pass
+    else:
+        conn = sqlite3.connect('dorking//' + str(db_prep_string))
+        cursor = conn.cursor()
+
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS dorks (
+            dork_id INTEGER PRIMARY KEY,
+            dork TEXT NOT NULL
+        )
+        ''')
+        conn.commit()
+
+        def add_dork(dork_id, dork):
+            try:
+                cursor.execute('INSERT INTO dorks (dork_id, dork) VALUES (?, ?)', (dork_id, dork))
+                conn.commit()
+                print(Fore.GREEN + "Successfully added new dork")
+            except sqlite3.IntegrityError:
+                print(Fore.RED + "Attention, dork_id variable must be unique")
+
+        while True:
+            dork_id = input(Fore.YELLOW + "Enter dork_id (or 'q' to quit this mode and save changes) >> ")
+            if dork_id.lower() == 'q':
+                break
+            dork = input(Fore.YELLOW + "Enter new dork >> ")
+            add_dork(int(dork_id), dork)
+        conn.close()