wgets.py

#!/usr/bin/env python

from __future__ import print_function

__description__ = "wgets"
__author__ = 'Didier Stevens'
__version__ = '0.0.3'
__date__ = '2023/08/17'

"""

Source code put in the public domain by Didier Stevens, no Copyright
https://DidierStevens.com
Use at your own risk

History:
  2019/04/26: start
  2019/05/21: continue
  2019/06/14: refactor, man page
  2020/03/21: option -H
  2020/10/07: option --firstline
  2021/01/30: fixed -H bug
  2021/01/31: bug fix
  2021/02/01: output file utf8
  2021/02/05: added option --quota
  2021/04/20: continue
  2022/04/07: updated quota logic: status; wait:
  2022/04/19: added FormatTime
  2022/12/10: options.limitrequests
  2023/03/31: options.sleep
  2023/04/01: change GetItem to recursive function to support [] and [integer]
  2023/08/17: 0.0.3 added requery, intermediatesave, includeargs

Todo:
  work out logic to save JSON file at regular intervals
"""

import optparse
import glob
import collections
import time
import sys
import textwrap
import os
import gzip
import re
import fnmatch
from contextlib import contextmanager
import json
import pickle
if sys.version_info[0] >= 3:
    import urllib.request as urllib23
else:
    import urllib2 as urllib23

LIBRARY_EXTENSION = '.library'
HTTP_PROXY = ''
HTTPS_PROXY = ''

def PrintManual():
    manual = '''
Manual:

wgets.py -u "'http://api.ip2proxy.com/?ip=%%s&key=demo&package=PX10' %% line" -D ips.json -Q "action=stop,contains=YOU HAVE EXCEEDED YOUR DEMO QUOTA" ips-2.txt

VT: --quota "action=wait:3600,status=429"

This Python script was developed with Python 2.7 and tested with Python 2.7 and 3.6.

An extra Python script (for example with custom definitions) can be loaded using option -s.

Custom definitions can also be placed in a file named python-per-line.library in the same directory as the program is located, and/or in the current working directory. When present, these 2 files will be loaded upon each execution of the program.

Option -e (execute) is used to execute Python commands before the command is executed. This can, for example, be used to import modules.

Errors occuring when opening a file are reported (and logged if logging is turned on), and the program moves on to the next file.
Errors occuring when reading & processing a file are reported (and logged if logging is turned on), and the program stops unless option ignoreprocessingerrors is used.

The lines are written to standard output, except when option -o is used. When option -o is used, the lines are written to the filename specified by option -o.
Filenames used with option -o starting with # have special meaning.
#c#example.txt will write output both to the console (stdout) and file example.txt.
#g# will write output to a file with a filename generated by the tool like this: toolname-date-time.txt.
#g#KEYWORD will write output to a file with a filename generated by the tool like this: toolname-KEYWORD-date-time.txt.
Use #p#filename to display execution progress.
To process several files while creating seperate output files for each input file, use -o #s#%f%.result *.
This will create output files with the name of the inputfile and extension .result.
There are several variables available when creating separate output files:
 %f% is the full filename (with directory if present)
 %b% is the base name: the filename without directory
 %d% is the directory
 %r% is the root: the filename without extension
 %ru% is the root made unique by appending a counter (if necessary)
 %e% is the extension
Most options can be combined, like #ps# for example.
#l# is used for literal filenames: if the output filename has to start with # (#example.txt for example), use filename #l##example.txt for example.

'''
    for line in manual.split('\n'):
        print(textwrap.fill(line))

DEFAULT_SEPARATOR = ','
QUOTE = '"'

def PrintError(*args, **kwargs):
    print(*args, file=sys.stderr, **kwargs)

#Convert 2 Bytes If Python 3
def C2BIP3(string):
    if sys.version_info[0] > 2:
        return bytes([ord(x) for x in string])
    else:
        return string

#Convert 2 Integer If Python 2
def C2IIP2(data):
    if sys.version_info[0] > 2:
        return data
    else:
        return ord(data)

def File2Strings(filename):
    try:
        f = open(filename, 'r')
    except:
        return None
    try:
        return map(lambda line:line.rstrip('\n'), f.readlines())
    except:
        return None
    finally:
        f.close()

def ProcessAt(argument):
    if argument.startswith('@'):
        strings = File2Strings(argument[1:])
        if strings == None:
            raise Exception('Error reading %s' % argument)
        else:
            return strings
    else:
        return [argument]

# CIC: Call If Callable
def CIC(expression):
    if callable(expression):
        return expression()
    else:
        return expression

# IFF: IF Function
def IFF(expression, valueTrue, valueFalse):
    if expression:
        return CIC(valueTrue)
    else:
        return CIC(valueFalse)

class cVariables():
    def __init__(self, variablesstring='', separator=DEFAULT_SEPARATOR):
        self.dVariables = {}
        if variablesstring == '':
            return
        for variable in variablesstring.split(separator):
            name, value = VariableNameValue(variable)
            self.dVariables[name] = value

    def SetVariable(self, name, value):
        self.dVariables[name] = value

    def Instantiate(self, astring):
        for key, value in self.dVariables.items():
            astring = astring.replace('%' + key + '%', value)
        return astring

class cOutput():
    def __init__(self, filenameOption=None):
        self.starttime = time.time()
        self.filenameOption = filenameOption
        self.separateFiles = False
        self.progress = False
        self.console = False
        self.fOut = None
        self.rootFilenames = {}
        if self.filenameOption:
            if self.ParseHash(self.filenameOption):
                if not self.separateFiles and self.filename != '':
                    self.fOut = open(self.filename, 'w', encoding='utf8')
            elif self.filenameOption != '':
                self.fOut = open(self.filenameOption, 'w', encoding='utf8')

    def ParseHash(self, option):
        if option.startswith('#'):
            position = self.filenameOption.find('#', 1)
            if position > 1:
                switches = self.filenameOption[1:position]
                self.filename = self.filenameOption[position + 1:]
                for switch in switches:
                    if switch == 's':
                        self.separateFiles = True
                    elif switch == 'p':
                        self.progress = True
                    elif switch == 'c':
                        self.console = True
                    elif switch == 'l':
                        pass
                    elif switch == 'g':
                        if self.filename != '':
                            extra = self.filename + '-'
                        else:
                            extra = ''
                        self.filename = '%s-%s%s.txt' % (os.path.splitext(os.path.basename(sys.argv[0]))[0], extra, self.FormatTime())
                    else:
                        return False
                return True
        return False

    @staticmethod
    def FormatTime(epoch=None):
        if epoch == None:
            epoch = time.time()
        return '%04d%02d%02d-%02d%02d%02d' % time.localtime(epoch)[0:6]

    def RootUnique(self, root):
        if not root in self.rootFilenames:
            self.rootFilenames[root] = None
            return root
        iter = 1
        while True:
            newroot = '%s_%04d' % (root, iter)
            if not newroot in self.rootFilenames:
                self.rootFilenames[newroot] = None
                return newroot
            iter += 1

    def Line(self, line, eol='\n'):
        if self.fOut == None or self.console:
            try:
                print(line, end=eol)
            except UnicodeEncodeError:
                encoding = sys.stdout.encoding
                print(line.encode(encoding, errors='backslashreplace').decode(encoding), end=eol)
#            sys.stdout.flush()
        if self.fOut != None:
            self.fOut.write(line + '\n')
            self.fOut.flush()

    def LineTimestamped(self, line):
        self.Line('%s: %s' % (self.FormatTime(), line))

    def Filename(self, filename, index, total):
        self.separateFilename = filename
        if self.progress:
            if index == 0:
                eta = ''
            else:
                seconds = int(float((time.time() - self.starttime) / float(index)) * float(total - index))
                eta = 'estimation %d seconds left, finished %s ' % (seconds, self.FormatTime(time.time() + seconds))
            PrintError('%d/%d %s%s' % (index + 1, total, eta, self.separateFilename))
        if self.separateFiles and self.filename != '':
            oFilenameVariables = cVariables()
            oFilenameVariables.SetVariable('f', self.separateFilename)
            basename = os.path.basename(self.separateFilename)
            oFilenameVariables.SetVariable('b', basename)
            oFilenameVariables.SetVariable('d', os.path.dirname(self.separateFilename))
            root, extension = os.path.splitext(basename)
            oFilenameVariables.SetVariable('r', root)
            oFilenameVariables.SetVariable('ru', self.RootUnique(root))
            oFilenameVariables.SetVariable('e', extension)

            self.Close()
            self.fOut = open(oFilenameVariables.Instantiate(self.filename), 'w')

    def Close(self):
        if self.fOut != None:
            self.fOut.close()
            self.fOut = None

class cExpandFilenameArguments():
    def __init__(self, filenames, literalfilenames=False, recursedir=False, checkfilenames=False, expressionprefix=None):
        self.containsUnixShellStyleWildcards = False
        self.warning = False
        self.message = ''
        self.filenameexpressions = []
        self.expressionprefix = expressionprefix
        self.literalfilenames = literalfilenames

        expression = ''
        if len(filenames) == 0:
            self.filenameexpressions = [['', '']]
        elif literalfilenames:
            self.filenameexpressions = [[filename, ''] for filename in filenames]
        elif recursedir:
            for dirwildcard in filenames:
                if expressionprefix != None and dirwildcard.startswith(expressionprefix):
                    expression = dirwildcard[len(expressionprefix):]
                else:
                    if dirwildcard.startswith('@'):
                        for filename in ProcessAt(dirwildcard):
                            self.filenameexpressions.append([filename, expression])
                    elif os.path.isfile(dirwildcard):
                        self.filenameexpressions.append([dirwildcard, expression])
                    else:
                        if os.path.isdir(dirwildcard):
                            dirname = dirwildcard
                            basename = '*'
                        else:
                            dirname, basename = os.path.split(dirwildcard)
                            if dirname == '':
                                dirname = '.'
                        for path, dirs, files in os.walk(dirname):
                            for filename in fnmatch.filter(files, basename):
                                self.filenameexpressions.append([os.path.join(path, filename), expression])
        else:
            for filename in list(collections.OrderedDict.fromkeys(sum(map(self.Glob, sum(map(ProcessAt, filenames), [])), []))):
                if expressionprefix != None and filename.startswith(expressionprefix):
                    expression = filename[len(expressionprefix):]
                else:
                    self.filenameexpressions.append([filename, expression])
            self.warning = self.containsUnixShellStyleWildcards and len(self.filenameexpressions) == 0
            if self.warning:
                self.message = "Your filename argument(s) contain Unix shell-style wildcards, but no files were matched.\nCheck your wildcard patterns or use option literalfilenames if you don't want wildcard pattern matching."
                return
        if self.filenameexpressions == [] and expression != '':
            self.filenameexpressions = [['', expression]]
        if checkfilenames:
            self.CheckIfFilesAreValid()

    def Glob(self, filename):
        if not ('?' in filename or '*' in filename or ('[' in filename and ']' in filename)):
            return [filename]
        self.containsUnixShellStyleWildcards = True
        return glob.glob(filename)

    def CheckIfFilesAreValid(self):
        valid = []
        doesnotexist = []
        isnotafile = []
        for filename, expression in self.filenameexpressions:
            hashfile = False
            try:
                hashfile = FilenameCheckHash(filename, self.literalfilenames)[0] == FCH_DATA
            except:
                pass
            if filename == '' or hashfile:
                valid.append([filename, expression])
            elif not os.path.exists(filename):
                doesnotexist.append(filename)
            elif not os.path.isfile(filename):
                isnotafile.append(filename)
            else:
                valid.append([filename, expression])
        self.filenameexpressions = valid
        if len(doesnotexist) > 0:
            self.warning = True
            self.message += 'The following files do not exist and will be skipped: ' + ' '.join(doesnotexist) + '\n'
        if len(isnotafile) > 0:
            self.warning = True
            self.message += 'The following files are not regular files and will be skipped: ' + ' '.join(isnotafile) + '\n'

    def Filenames(self):
        if self.expressionprefix == None:
            return [filename for filename, expression in self.filenameexpressions]
        else:
            return self.filenameexpressions

def ToString(value):
    if isinstance(value, str):
        return value
    else:
        return str(value)

def Quote(value, separator, quote):
    value = ToString(value)
    if len(value) > 1 and value[0] == quote and value[-1] == quote:
        return value
    if separator in value or value == '':
        return quote + value + quote
    else:
        return value

def MakeCSVLine(row, separator, quote):
    return separator.join([Quote(value, separator, quote) for value in row])

def Sleep(arg):
# valid inputs: 60, 36s, 5m, 2h, 7d, 01:00:00
    if arg == '':
        return
    elif arg.endswith('s'):
        delay = int(arg[:-1])
    elif arg.endswith('m'):
        delay = int(arg[:-1]) * 60
    elif arg.endswith('h'):
        delay = int(arg[:-1]) * 60 * 60
    elif arg.endswith('d'):
        delay = int(arg[:-1]) * 60 * 60 * 24
    elif ':' in arg:
        hours, minutes, seconds = [int(number) for number in arg.split(':')]
        now = time.localtime()
        delay = time.mktime((now.tm_year, now.tm_mon, now.tm_mday, hours, minutes, seconds, now.tm_wday, now.tm_yday, now.tm_isdst)) - time.time()
        if delay < 0:
            delay += 60 * 60 * 24
    else:
        delay = int(arg)
    print('%s: Sleeping for %d seconds (%s), waking around %s' % (FormatTime(), delay, arg, FormatTime(time.time() + delay)))
    time.sleep(delay)
    print('%s: Sleeping done' % FormatTime())

class cLogfile():
    def __init__(self, keyword, comment):
        self.starttime = time.time()
        self.errors = 0
        if keyword == '':
            self.oOutput = None
        else:
            self.oOutput = cOutput('%s-%s-%s.log' % (os.path.splitext(os.path.basename(sys.argv[0]))[0], keyword, self.FormatTime()))
        self.Line('Start')
        self.Line('UTC', '%04d%02d%02d-%02d%02d%02d' % time.gmtime(time.time())[0:6])
        self.Line('Comment', comment)
        self.Line('Args', repr(sys.argv))
        self.Line('Version', __version__)
        self.Line('Python', repr(sys.version_info))
        self.Line('Platform', sys.platform)
        self.Line('CWD', repr(os.getcwd()))

    @staticmethod
    def FormatTime(epoch=None):
        if epoch == None:
            epoch = time.time()
        return '%04d%02d%02d-%02d%02d%02d' % time.localtime(epoch)[0:6]

    def Line(self, *line):
        if self.oOutput != None:
            self.oOutput.Line(MakeCSVLine((self.FormatTime(), ) + line, DEFAULT_SEPARATOR, QUOTE))

    def LineError(self, *line):
        self.Line('Error', *line)
        self.errors += 1

    def Close(self):
        if self.oOutput != None:
            self.Line('Finish', '%d error(s)' % self.errors, '%d second(s)' % (time.time() - self.starttime))
            self.oOutput.Close()

def ProcessFile(fIn, options, fullread):
    if fIn == None:
        return

    if fullread:
        yield fIn.read()
    else:
        for line in fIn:
            line = line.rstrip('\n\r')
            yield line

def AnalyzeFileError(filename):
    PrintError('Error opening file %s' % filename)
    PrintError(sys.exc_info()[1])
    try:
        if not os.path.exists(filename):
            PrintError('The file does not exist')
        elif os.path.isdir(filename):
            PrintError('The file is a directory')
        elif not os.path.isfile(filename):
            PrintError('The file is not a regular file')
    except:
        pass

@contextmanager
def TextFile(filename, oLogfile):
    if filename == '':
        fIn = sys.stdin
    elif os.path.splitext(filename)[1].lower() == '.gz':
        try:
            fIn = gzip.GzipFile(filename, 'rb')
        except:
            AnalyzeFileError(filename)
            oLogfile.LineError('Opening file %s %s' % (filename, repr(sys.exc_info()[1])))
            fIn = None
    else:
        try:
            fIn = open(filename, 'r')
        except:
            AnalyzeFileError(filename)
            oLogfile.LineError('Opening file %s %s' % (filename, repr(sys.exc_info()[1])))
            fIn = None

    if fIn != None:
        oLogfile.Line('Success', 'Opening file %s' % filename)

    yield fIn

    if fIn != None:
        if sys.exc_info()[1] != None:
            oLogfile.LineError('Reading file %s %s' % (filename, repr(sys.exc_info()[1])))
        if fIn != sys.stdin:
            fIn.close()

def SetProxiesIfNecessary():
    global HTTP_PROXY
    global HTTPS_PROXY

    dProxies = {}
    if HTTP_PROXY != '':
        dProxies['http'] = HTTP_PROXY
    if HTTPS_PROXY != '':
        dProxies['https'] = HTTPS_PROXY
    if os.getenv('http_proxy') != None:
        dProxies['http'] = os.getenv('http_proxy')
    if os.getenv('https_proxy') != None:
        dProxies['https'] = os.getenv('https_proxy')
    if dProxies != {}:
        urllib23.install_opener(urllib23.build_opener(urllib23.ProxyHandler(dProxies)))

def LoadScriptIfExists(filename):
    if os.path.exists(filename):
        exec(open(filename, 'r').read(), globals(), globals())

def HTTPRequest(url, useragent=None, dHeadersArg={}):
    status = None
    dHeaders = {}
    if useragent != None:
        dHeaders['User-Agent'] = useragent
    for key, value in dHeadersArg.items():
        dHeaders[key] = value
    req = urllib23.Request(url=url, headers=dHeaders)
    try:
        if sys.hexversion >= 0x020601F0:
            request = urllib23.urlopen(req, timeout=5)
        else:
            request = urllib23.urlopen(req)
    except urllib23.HTTPError as e:
        try:
            result = e.fp.read()
        except:
            result = None
        return e.code, result
    except:
        return None, None
    try:
        data = request.read()
        status = request.getcode()
    except:
        return request.getcode(), None
    finally:
        request.close()
    return status, data

def DefaultStr(value, default):
    if value == '':
        return default
    else:
        return value

def GetItem(jsondata, path, default=''):
    if path == '':
        return jsondata

    if not '/' in path:
        path += '/'
    index, path = path.split('/', 1)

    if len(index) >= 3 and index[0] == '[' and index[-1] == ']':
        index = int(index[1:-1])
        return GetItem(jsondata[index], path, default)
    elif index == '[]':
        return [GetItem(item, path, default) for item in jsondata]
    elif index in jsondata:
        return GetItem(jsondata[index], path, default)
    else:
        return default

def StartsWithGetRemainder(strIn, strStart):
    if strIn.startswith(strStart):
        return True, strIn[len(strStart):]
    else:
        return False, None

def ParseCommand(command):
    dCommand = {}
    for element in command.split(','):
        name, value = element.split('=', 1)
        name = name.lower().strip()
        dCommand[name] = value
    return dCommand

class cQuota(object):
    def __init__(self, optionvalue):
        self.optionvalue = optionvalue
        if optionvalue == '':
            self.dOptions = {}
            return
        self.dOptions = ParseCommand(optionvalue)
        if not 'status' in self.dOptions and not 'contains' in self.dOptions:
            raise Exception('Missing "status" and "contains": %s' % optionvalue)
        if not 'action' in self.dOptions:
            raise Exception('Missing "action": %s' % optionvalue)
        self.action = self.dOptions['action']

    def CheckOverrun(self, status, result):
        if self.optionvalue == '':
            return False
        if result == None:
            return False
        if 'contains' in self.dOptions and self.dOptions['contains'].encode() in result:
            return True
        if 'status' in self.dOptions and int(self.dOptions['status']) == status:
            return True
        return False

    def Action(self):
        result, remainder = StartsWithGetRemainder(self.dOptions['action'], 'wait:')
        if result:
            return ['wait', int(remainder)]
        else:
            return [self.dOptions['action']]

def ProcessTextFile(filename, oOutput, oLogfile, dDatabase, options):
    rows = []
    requested = False
    requestCounter = 0
    databaseCounter = 0
    oQuota = cQuota(options.quota)
    starttime = time.time()
    with TextFile(filename, oLogfile) as fIn:
        try:
            lines = []
            for line in ProcessFile(fIn, options, False):
                lines.append(line)
            for line in lines:
                # ----- Put your line processing code here -----
                if options.delay != 0 and requested:
                    time.sleep(options.delay)
                if line in dDatabase and not str(dDatabase[line][0]) in options.requery.split(','):
                    status = dDatabase[line][0]
                    resultOriginal = dDatabase[line][1]
                    requested = False
                else:
                    requested = True
                    if options.url == '':
                        url = line
                    else:
                        url = eval(options.url)
                    if options.header != '':
                        key, value = options.header.split(':', 2)
                        dHeaders = {key: value.lstrip()}
                    else:
                        dHeaders = {}
                    status, resultOriginal = HTTPRequest(url, options.useragent, dHeaders)
                    if oQuota.CheckOverrun(status, resultOriginal):
                        if oQuota.Action()[0] == 'stop':
                            print('Quota exceeded, stopping.')
                            break
                        elif oQuota.Action()[0] == 'wait':
                            while True:
                                print('Quota exceeded, waiting %d seconds (time: %s).' % (oQuota.Action()[1], cOutput.FormatTime()))
                                time.sleep(oQuota.Action()[1])
                                status, resultOriginal = HTTPRequest(url, options.useragent, dHeaders)
                                if not oQuota.CheckOverrun(status, resultOriginal):
                                    break
                    if status == 200 or options.alldatabase:
                        if isinstance(resultOriginal, bytes):
                            dDatabase[line] = [status, resultOriginal.decode('latin-1')]
                        else:
                            dDatabase[line] = [status, resultOriginal]
                if resultOriginal == None:
                    results = []
                    if options.pickle == '':
                        oOutput.Line(MakeCSVLine([line, IFF(requested, '1', '0'), status], options.separator, QUOTE))
                elif options.iterate == '':
                    results = [resultOriginal]
                else:
                    results = [json.dumps(item) for item in GetItem(json.loads(resultOriginal), options.iterate)]
                for result in results:
                    jsonresult = {}
                    try:
                        jsonresult = json.loads(result)
                    except:
                        pass
                    row = [line, IFF(requested, '1', '0'), status]
                    if options.result == '':
                        if options.pickle != '':
                            row.append(resultOriginal)
                        else:
                            row.append(repr(resultOriginal))
                    else:
                        if resultOriginal != None:
                            evalresult = eval(options.result)
                        else:
                            evalresult = None
                        if isinstance(evalresult, list):
                            row.extend(evalresult)
                        else:
                            row.append(evalresult)
                    if options.pickle != '':
                        rows.append(row)
                        oOutput.Line('%d: %d %s' % (len(rows), status, line))
                    else:
                        oOutput.Line(MakeCSVLine(row, options.separator, QUOTE))
                if requested:
                    requestCounter += 1
                    if options.database != '' and requestCounter % options.intermediatesave == 0:
                        json.dump(dDatabase, open(options.database, 'w'))
                        elapsed = time.time() - starttime
                        remaining = len(lines) - requestCounter - databaseCounter
                        speed = requestCounter / elapsed
                        remainingSeconds = remaining / speed
                        eta = '%04d/%02d/%02d %02d:%02d:%02d' % time.localtime(time.time() + remainingSeconds)[0:6]
                        print('Intermediate database saved (requestCounter %d elapsed %d requests/second %.02f remaining %d %ds %.02fh %s)' % (requestCounter, elapsed, speed, remaining, remainingSeconds, remainingSeconds / 3600.0, eta))

                    if options.limitrequests != 0 and options.limitrequests <= requestCounter:
                        break
                else:
                    databaseCounter += 1

                # ----------------------------------------------
        except:
            oLogfile.LineError('Processing file %s %s' % (filename, repr(sys.exc_info()[1])))
            if not options.ignoreprocessingerrors:
                raise
            if sys.version_info[0] < 3:
                sys.exc_clear()
    if options.pickle != '':
        pickle.dump(rows, open(options.pickle, 'wb'))

def InstantiateCOutput(options):
    filenameOption = None
    if options.output != '':
        filenameOption = options.output
    return cOutput(filenameOption)

def FormatTime(epoch=None):
    if epoch == '':
        return ''
    if epoch == None:
        epoch = time.time()
    return '%04d-%02d-%02d %02d:%02d:%02d' % time.localtime(epoch)[0:6]

def DictionaryToString(dictionary, separator='~'):
    if not isinstance(dictionary, dict):
        return dictionary
    result = []
    for key, value in dictionary.items():
        result.append('%s=%s' % (key, value))
    return separator.join(result)

def ProcessTextFiles(filenames, oLogfile, options):
    if options.execute != '':
        exec(options.execute, globals())

    if options.script != '':
        LoadScriptIfExists(options.script)
    LoadScriptIfExists(os.path.splitext(sys.argv[0])[0] + LIBRARY_EXTENSION)
    LoadScriptIfExists(os.path.splitext(os.path.basename(sys.argv[0]))[0] + LIBRARY_EXTENSION)

    SetProxiesIfNecessary()

    oOutput = InstantiateCOutput(options)

    if options.database != '':
        try:
            dDatabase = json.load(open(options.database, 'r'))
        except FileNotFoundError:
            dDatabase = {}
    else:
        dDatabase = {}

    if options.firstline != '':
        oOutput.Line(options.firstline)
    if options.includeargs:
        oOutput.Line('#%s' % repr(sys.argv))
        oOutput.Line('#%s' % ' '.join([('"%s"' % arg)  if ' ' in arg else arg for arg in sys.argv]))
    for index, filename in enumerate(filenames):
        oOutput.Filename(filename, index, len(filenames))
        ProcessTextFile(filename, oOutput, oLogfile, dDatabase, options)

    if options.database != '':
        json.dump(dDatabase, open(options.database, 'w'))

    oOutput.Close()

def Main():
    moredesc = '''

Arguments:
@file: process each file listed in the text file specified
wildcards are supported

Source code put in the public domain by Didier Stevens, no Copyright
Use at your own risk
https://DidierStevens.com'''

    oParser = optparse.OptionParser(usage='usage: %prog [options] [[@]file ...]\n' + __description__ + moredesc, version='%prog ' + __version__)
    oParser.add_option('-m', '--man', action='store_true', default=False, help='Print manual')
    oParser.add_option('-o', '--output', type=str, default='', help='Output to file (# supported)')
    oParser.add_option('-u', '--url', type=str, default='', help='Python expression to create URL')
    oParser.add_option('-r', '--result', type=str, default='', help='Python expression to process result')
    oParser.add_option('-i', '--iterate', type=str, default='', help='JSON path to iterate over')
    oParser.add_option('-p', '--pickle', type=str, default='', help='Pickle results to file')
    oParser.add_option('-D', '--database', type=str, default='', help='Database with results')
    oParser.add_option('-A', '--useragent', type=str, default=__description__ + ' ' + __version__, help='User Agent String')
    oParser.add_option('-H', '--header', type=str, default='', help='Header to add')
    oParser.add_option('-d', '--delay', type=float, default=1, help='Delay (in seconds) between queries (use 0 to have no delay)')
    oParser.add_option('-l', '--limitrequests', type=int, default=0, help='Maximum number of requests')
    oParser.add_option('-s', '--script', type=str, default='', help='Script with definitions to include')
    oParser.add_option('-e', '--execute', default='', help='Commands to execute')
    oParser.add_option('-Q', '--quota', default='', help='Quota overrun handling')
    oParser.add_option('-S', '--separator', type=str, default=',', help='CSV separator')
    oParser.add_option('--firstline', default='', help='First line to print')
    oParser.add_option('--alldatabase', action='store_true', default=False, help='Store all results in database, not only 200')
    oParser.add_option('--literalfilenames', action='store_true', default=False, help='Do not interpret filenames')
    oParser.add_option('--recursedir', action='store_true', default=False, help='Recurse directories (wildcards and here files (@...) allowed)')
    oParser.add_option('--checkfilenames', action='store_true', default=False, help='Perform check if files exist prior to file processing')
    oParser.add_option('--logfile', type=str, default='', help='Create logfile with given keyword')
    oParser.add_option('--logcomment', type=str, default='', help='A string with comments to be included in the log file')
    oParser.add_option('--ignoreprocessingerrors', action='store_true', default=False, help='Ignore errors during file processing')
    oParser.add_option('--sleep', default='', help='Sleep before starting queries')
    oParser.add_option('--requery', default='', help='Comma separated list of status codes to requery')
    oParser.add_option('--intermediatesave', type=int, default=100, help='Save database every 100 requests (default)')
    oParser.add_option('--includeargs', action='store_true', default=False, help='Include args')
    (options, args) = oParser.parse_args()

    if options.man:
        oParser.print_help()
        PrintManual()
        return

    Sleep(options.sleep)

    oLogfile = cLogfile(options.logfile, options.logcomment)

    oExpandFilenameArguments = cExpandFilenameArguments(args, options.literalfilenames, options.recursedir, options.checkfilenames)
    oLogfile.Line('FilesCount', str(len(oExpandFilenameArguments.Filenames())))
    oLogfile.Line('Files', repr(oExpandFilenameArguments.Filenames()))
    if oExpandFilenameArguments.warning:
        PrintError('\nWarning:')
        PrintError(oExpandFilenameArguments.message)
        oLogfile.Line('Warning', repr(oExpandFilenameArguments.message))

    if options.separator == '\\t':
        options.separator = '\t'

    ProcessTextFiles(oExpandFilenameArguments.Filenames(), oLogfile, options)

    if oLogfile.errors > 0:
        PrintError('Number of errors: %d' % oLogfile.errors)
    oLogfile.Close()

if __name__ == '__main__':
    Main()