diff --git a/topo/README.md b/topo/README.md index bb41cea92..8e3f17e01 100644 --- a/topo/README.md +++ b/topo/README.md @@ -1,18 +1,28 @@ ## topo Topology-related scripts and utilities; currently mostly ESnet-specific Python code that work with Python 2.x + ### Installation `pip install -r requirements.txt` - Installs the required packages. You will need access to the esnet/neg-tools private repository. -### Running -`esdb_topo.py`: Contacts ESDB and does a good amount of processing. It creates two output files: +### Running + +The `generate_topo.py` script is the main script that calls `esdb_topo.py` and `improve_esdb_topo.py` scripts in sequence ensuring that the output of one is passed as input to the other. + +The `run.sh` shell script in turn passes production-ready parameters to `generate_topo.py` . + +`esdb_topo.py`: Contacts ESDB and does a good amount of processing. It creates three output files: + ``` output/devices.json output/adjacencies.json + output/today.json ``` + These files are meant to be eventually moved to the backend config directory: + ``` .../backend/config/topo/ esnet-devices.json @@ -21,37 +31,30 @@ These files are meant to be eventually moved to the backend config directory: Some SAPs and ports that should be in `devices.json` are missed or misconfigured. -To insert them to our topology correctly we need some more input and further -of processing. Some of this input is gathered from Graphite: +To insert them to our topology correctly we need some more input and further of processing. Some of this input is gathered from Netbeam: ``` -wget -4 https://graphite.es.net/api/west/snmp/?interface_descr= -O input/graphite_ports.json -wget -4 https://graphite.es.net/api/west/sap/ -O input/saps.json +wget -4 https://esnet-netbeam.appspot.com/api/network/esnet/prod/interfaces -O input/netbeam_ports.json +wget -4 https://esnet-netbeam.appspot.com/api/network/esnet/prod/saps -O input/saps.json ``` After this, the files generated will look like this: + ``` input/ lags.json - hand-written config file (*) dual_ports.json - hand-written config file (*) - graphite_ports.json - through wget (*) + netbeam_ports.json - through wget (*) saps.json - through wget (*) - output/ devices.json - generated by esdb_topo.py (*) adjacencies.json - generated by esdb_topo.py - + today.json - generated by esdb_topo.py ``` The last stage of processing is meant to generate an improved `devices.json` file. -`improve_esdb_topo.py` reads the files marked with `(*)` above, then prints a new `devices.json` in STDOUT. - -The `esdb_topo.py` and `improve_esdb_topo.py` scripts have various command-line parameters -to decide where to place their output and where to read their input. - -The `generate_topo.py` script calls both scripts in sequence ensuring the output of one is passed as input -to the other. +`improve_esdb_topo.py`: Reads the files marked with `(*)` above, then prints a new `devices.json` in STDOUT. -The `run.sh` shell script in turn passes production-ready parameters to `generate_topo.py` . \ No newline at end of file +The `esdb_topo.py` and `improve_esdb_topo.py` scripts have various command-line parameters to decide where to place their output and where to read their input. \ No newline at end of file diff --git a/topo/generate_topo.py b/topo/generate_topo.py index c2ce15973..908f8ed05 100755 --- a/topo/generate_topo.py +++ b/topo/generate_topo.py @@ -27,7 +27,7 @@ def main(): parser.add_argument('-f', '--fast', action='count', default=0, help="Fast mode (use saved ports and saps)") parser.add_argument('-s', '--save', action='count', default=0, - help="save ports and saps from graphite for fast run") + help="save ports and saps from netbeam for fast run") parser.add_argument('--tmp-dir', default=defaults['TMP_DIR'], help="Temp dir") @@ -86,27 +86,27 @@ def main(): saps = opts.save_dir + "/" + opts.saps if not opts.fast: - graphite_base_url = "https://graphite.es.net/api/west" - graphite_ifces_url = graphite_base_url + '/snmp/?interface_descr=' - graphite_sap_url = graphite_base_url + '/sap/' + netbeam_base_url = "https://esnet-netbeam.appspot.com/api/network/esnet/prod" + netbeam_ifces_url = netbeam_base_url + '/interfaces' + netbeam_sap_url = netbeam_base_url + '/saps' - graphite_saps = "wget -q -4 " + graphite_sap_url + " -O - | python -m json.tool > " + saps - graphite_ifces = "wget -q -4 " + graphite_ifces_url + " -O - | python -m json.tool > " + ports + netbeam_saps = "wget -q -4 " + netbeam_sap_url + " -O - | python -m json.tool > " + saps + netbeam_ifces = "wget -q -4 " + netbeam_ifces_url + " -O - | python -m json.tool > " + ports if opts.verbose: - print "getting SAPs from graphite" - print graphite_saps + print "getting SAPs from netbeam" + print netbeam_saps - if os.system(graphite_saps) is not 0: - print >> sys.stderr, "error running " + graphite_saps + if os.system(netbeam_saps) is not 0: + print >> sys.stderr, "error running " + netbeam_saps exit(1) if opts.verbose: - print "getting all interfaces from graphite" - print graphite_ifces + print "getting all interfaces from netbeam" + print netbeam_ifces - if os.system(graphite_ifces) is not 0: - print >> sys.stderr, "error running " + graphite_ifces + if os.system(netbeam_ifces) is not 0: + print >> sys.stderr, "error running " + netbeam_ifces exit(1) if opts.save: @@ -127,6 +127,7 @@ def main(): esdb_topo = "./esdb_topo.py " + esdb_verbose_arg + " " + esdb_op_arg + " --output-dir=" + esdb_output_dir \ + " --devices=" + tmp_devs_file + " --adjacencies=" + tmp_adjs_file + if opts.verbose: print "processing today.json" print esdb_topo @@ -140,8 +141,9 @@ def main(): improve_topo = "./improve_esdb_topo.py --lags=" + lags + " --saps=" + saps + " --ports=" + ports \ + " --dual_ports=" + dual_ports + " --input-devices=" + initial_devs \ + " --output-devices=" + output_devs + if opts.verbose: - print "improving today.json with LAGs and dual ports, as well as graphite SAPs and LAGs" + print "improving today.json with LAGs and dual ports, as well as netbeam SAPs and LAGs" print improve_topo if os.system(improve_topo) is not 0: diff --git a/topo/improve_esdb_topo.py b/topo/improve_esdb_topo.py index 73e5256c5..f18b2e924 100755 --- a/topo/improve_esdb_topo.py +++ b/topo/improve_esdb_topo.py @@ -12,7 +12,7 @@ SAPS = 'input/saps.json' LAGS = 'input/lags.json' -PORTS = 'input/graphite_ports.json' +PORTS = 'input/netbeam_ports.json' DUAL_PORTS = 'input/dual_ports.json' OUTPUT_DEVICES = 'output/devices.json' INPUT_DEVICES = 'input/devices.json' @@ -39,13 +39,12 @@ def main(): with open(opts.lags, 'r') as infile: lags = json.load(infile) - # retrieve from https://graphite.es.net/api/west/sap/ + # retrieve from https://esnet-netbeam.appspot.com/api/network/esnet/prod/saps with open(opts.saps, 'r') as infile: saps_in = json.load(infile) - # from - # wget -4 https://graphite.es.net/api/west/snmp/?interface_descr= > input/graphite_ports.json - # note: slow operation + # retrieve from + # wget -4 https://esnet-netbeam.appspot.com/api/network/esnet/prod/interfaces > input/netbeam_ports.json with open(opts.ports, 'r') as infile: ports = json.load(infile) @@ -68,44 +67,42 @@ def main(): 'service': parts[0], 'port': port, 'vlan': parts[2], - 'desc': sap_in['desc'] + 'desc': sap_in['description'] } saps_by_device[device].append(entry) by_device = {} - now = time.time() for port in ports: e = {} - device = port['device_uri'].split('/')[2] - endpoint = port['uri'].split('interface/')[1] + device = port['device'] + endpoint = port['name'] e['device'] = device e['endpoint'] = endpoint - e['ifce'] = port['ifDescr'].strip() - e['description'] = port['ifAlias'].strip() - e['addr'] = port['ipAddr'] - e['mbps'] = port['ifHighSpeed'] - e['end_time'] = port['end_time'] - end_time = port['end_time'] - if now < end_time: - if device not in by_device: - by_device[device] = [] - by_device[device].append(e) - else: - print 'outdated: ' + device + ':' + endpoint + e['ifce'] = port['name'].strip() + e['description'] = port['description'].strip() + # e['addr'] = port['ipAddr'] + e['mbps'] = port['speed'] + if device not in by_device: + by_device[device] = [] + by_device[device].append(e) devices = json.load(open(opts.input_devices, 'r')) untagged = [] for device_entry in devices: device = device_entry['urn'] if device not in by_device: - print "Could not find " + device + " from today.json in graphite\n" \ + print "Could not find " + device + " from today.json in netbeam\n" \ + " - possibly decom'd but " + device + ".rt not cleaned up in PMC.\n" else: - for graphite_port in by_device[device]: - if graphite_port['mbps'] == 0: + for netbeam_port in by_device[device]: + if netbeam_port['mbps'] == 0: + continue + + ifce = str(netbeam_port['ifce']) + description = netbeam_port['description'] + + if "Gig Ethernet" in description: continue - ifce = str(graphite_port['ifce']) - description = graphite_port['description'] relevant = False juniper = False @@ -125,7 +122,6 @@ def main(): if ifce.startswith('et'): relevant = True juniper = True - if ifce.startswith('to_'): relevant = False # until i can get vlan info @@ -137,7 +133,6 @@ def main(): if relevant: used_vlans = [] - if device in saps_by_device: for sap in saps_by_device[device]: if sap['port'] == ifce: @@ -148,7 +143,7 @@ def main(): for u_entry in untagged: if u_entry['device'] == device and u_entry['port'] == ifce: delete = ifce - # print 'found base port for untagged: '+device+':'+ifce + # print 'found base port for untagged: '+ device + ':' + ifce if juniper and '.' in ifce: parts = ifce.split('.') @@ -181,17 +176,17 @@ def main(): found_port_entry['capabilities'].append('ETHERNET') if not found: - description = graphite_port['description'] + description = netbeam_port['description'] if 'oscars-l2circuit' in description: # Hacky hack hack used_vlans = [] reservable_vlans = make_reservable_vlans(used_vlans) port = { - 'reservableIngressBw': graphite_port['mbps'], - 'reservableEgressBw': graphite_port['mbps'], + 'reservableIngressBw': netbeam_port['mbps'], + 'reservableEgressBw': netbeam_port['mbps'], 'tags': [ - graphite_port['description'] + netbeam_port['description'] ], 'urn': ifce_urn, 'capabilities': [ @@ -209,7 +204,7 @@ def main(): delete_entry = port_entry if delete_entry: device_entry['ports'].remove(delete_entry) - # end : for graphite_port in by_device[device]: + # end : for netbeam_port in by_device[device]: for lag in lags: lag_urn = lag['device'] + ':' + lag['name']