From dfa521fd2dc09b683be9de8f29a2bc922e273cff Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 15:48:00 -0400 Subject: [PATCH 01/21] p4 switch --- fimutil/ralph/asset.py | 1 + fimutil/ralph/p4_switch.py | 56 ++++++++++++++++++++++++++++++++++++++ fimutil/ralph/site.py | 24 +++++++++++++++- 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 fimutil/ralph/p4_switch.py diff --git a/fimutil/ralph/asset.py b/fimutil/ralph/asset.py index 93ec863..c72d837 100644 --- a/fimutil/ralph/asset.py +++ b/fimutil/ralph/asset.py @@ -22,6 +22,7 @@ class RalphAssetType(Enum): DPSwitch = auto() Abstract = auto() FPGA = auto() + P4Switch = auto() def __str__(self): return self.name diff --git a/fimutil/ralph/p4_switch.py b/fimutil/ralph/p4_switch.py new file mode 100644 index 0000000..e618c90 --- /dev/null +++ b/fimutil/ralph/p4_switch.py @@ -0,0 +1,56 @@ +import pyjq +import logging +from typing import List + +from fimutil.ralph.ralph_uri import RalphURI +from fimutil.ralph.asset import RalphAsset, RalphAssetType, RalphAssetMimatch +from fimutil.ralph.model import SimpleModel +from fimutil.ralph.ethernetport import EthernetPort + + +class P4Switch(RalphAsset): + """ + Dataplane switch + """ + FIELD_MAP = '{Name: .hostname, SN: .sn, IP: .ipaddresses[0], ' \ + 'AL2S_SWITCH: .custom_fields.al2s_remote_switch_name,' \ + 'AL2S_vlans: .custom_fields.al2s_vlan_ranges, ' \ + 'Local_vlans: .custom_fields.dataplane_vlan_ranges}' + + def __init__(self, *, uri: str, ralph: RalphURI): + super().__init__(uri=uri, ralph=ralph) + self.type = RalphAssetType.P4Switch + self.model = None + self.dp_p4_ports = None + self.switch_ip = None + + def parse(self): + super().parse() + + # find model + model_url = pyjq.one('.model.url', self.raw_json_obj) + self.model = SimpleModel(uri=model_url, ralph=self.ralph) + try: + self.model.parse() + except RalphAssetMimatch: + logging.warning('Unable to parse switch model, continuing') + + try: + port_urls = pyjq.all('.ethernet[].url', self.raw_json_obj) + except ValueError: + logging.warning('Unable to find any ethernet ports in node, continuing') + port_urls = list() + + port_index = 1 + for port in port_urls: + port = EthernetPort(uri=port, ralph=self.ralph) + try: + port.parse() + except RalphAssetMimatch: + continue + self.components['port-' + str(port_index)] = port + port_index += 1 + + def __str__(self): + ret = super().__str__() + return ret + '\n\t' + str(self.model) \ No newline at end of file diff --git a/fimutil/ralph/site.py b/fimutil/ralph/site.py index 2457b35..d4a31eb 100644 --- a/fimutil/ralph/site.py +++ b/fimutil/ralph/site.py @@ -5,6 +5,7 @@ from typing import Dict import json +from fimutil.ralph.p4_switch import P4Switch from fimutil.ralph.ralph_uri import RalphURI from fimutil.ralph.worker_node import WorkerNode from fimutil.ralph.storage import Storage @@ -23,6 +24,7 @@ def __init__(self, *, site_name: str, ralph: RalphURI, config: Dict = None, doma self.workers = list() self.storage = None self.dp_switch = None + self.p4_switch = None self.ptp = False self.name = site_name self.domain = domain @@ -64,6 +66,22 @@ def catalog(self): except ValueError: logging.warning('Unable to find a dataplane switch in site, continuing') + try: + logging.info(f'Searching for P4 switch URL') + query = {'hostname': f'{self.name.lower()}-p4-sw' + self.domain} + results = self.ralph.get_json_object(self.ralph.base_uri + 'data-center-assets/?' + + urlencode(query)) + print(f"KOMAL ---- {results}") + p4_switch_url = pyjq.one('[ .results[0].url ]', results)[0] + self.p4_switch = P4Switch(uri=p4_switch_url, ralph=self.ralph) + self.p4_switch.parse() + print("----------------------------------") + print(self.p4_switch) + print(self.p4_switch.components) + print("----------------------------------") + except ValueError: + logging.warning('Unable to find a p4 switch in site, continuing') + query = {'hostname': f'{self.name.lower()}-time' + self.domain} results = self.ralph.get_json_object(self.ralph.base_uri + 'data-center-assets/?' + urlencode(query)) @@ -134,6 +152,8 @@ def __str__(self): assets.append(str(self.storage)) if self.dp_switch: assets.append(str(self.dp_switch)) + if self.p4_switch: + assets.append(str(self.p4_switch)) for w in self.workers: assets.append(str(w)) return '\n'.join(assets) @@ -144,6 +164,8 @@ def to_json(self): ret["Storage"] = self.storage.fields.copy() if self.dp_switch: ret["DataPlane"] = self.dp_switch.fields.copy() + if self.dp_switch: + ret["P4"] = self.p4_switch.fields.copy() # collect port information from all workers dp_ports = list() for w in self.workers: @@ -159,4 +181,4 @@ def to_json(self): for w in self.workers: n.append(w.to_json()) ret["Nodes"] = n - return ret \ No newline at end of file + return ret From 15ac2dcfd76cb82fadf7e01e3ea924a1dd1bc490 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 15:56:34 -0400 Subject: [PATCH 02/21] remove extra prints --- fimutil/ralph/p4_switch.py | 2 -- fimutil/ralph/site.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/fimutil/ralph/p4_switch.py b/fimutil/ralph/p4_switch.py index e618c90..b939e6d 100644 --- a/fimutil/ralph/p4_switch.py +++ b/fimutil/ralph/p4_switch.py @@ -21,8 +21,6 @@ def __init__(self, *, uri: str, ralph: RalphURI): super().__init__(uri=uri, ralph=ralph) self.type = RalphAssetType.P4Switch self.model = None - self.dp_p4_ports = None - self.switch_ip = None def parse(self): super().parse() diff --git a/fimutil/ralph/site.py b/fimutil/ralph/site.py index d4a31eb..90f5d67 100644 --- a/fimutil/ralph/site.py +++ b/fimutil/ralph/site.py @@ -71,13 +71,11 @@ def catalog(self): query = {'hostname': f'{self.name.lower()}-p4-sw' + self.domain} results = self.ralph.get_json_object(self.ralph.base_uri + 'data-center-assets/?' + urlencode(query)) - print(f"KOMAL ---- {results}") p4_switch_url = pyjq.one('[ .results[0].url ]', results)[0] self.p4_switch = P4Switch(uri=p4_switch_url, ralph=self.ralph) self.p4_switch.parse() print("----------------------------------") print(self.p4_switch) - print(self.p4_switch.components) print("----------------------------------") except ValueError: logging.warning('Unable to find a p4 switch in site, continuing') From 157ca3692c692e96eb9a893aeffa570edf9bf005 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 16:51:46 -0400 Subject: [PATCH 03/21] add p4 switch parsing --- fimutil/ralph/fim_helper.py | 37 +++++++++++++++++++++++++++++++++++++ fimutil/ralph/p4_switch.py | 14 +++++++++++++- fimutil/ralph/site.py | 11 +++++++---- pyproject.toml | 2 +- 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index cc3d977..4654d8b 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -497,6 +497,43 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol interfaces=[sp, v]) link_idx += 1 + # create p4 switch with interfaces and links back to dataplane switch ports + logging.debug('Adding p4 switch') + if site.p4_switch is None: + logging.info(f'P4 Switch was not detected/catalogued') + return topo + + # check if we are using someone else's DP switch + real_switch_site = site.name + + # this prefers an IP address, but uses S/N if IP is None (like in GENI racks) + p4_name = p4_switch_name_id(real_switch_site.lower(), + site.p4_switch.fields['IP'] if site.p4_switch.fields['IP'] else site.p4_switch.fields['SN']) + logging.info(f'Adding P4 switch {dp_name}') + p4 = topo.add_node(name=p4_name[0], + node_id=p4_name[1], + site=site.name, ntype=NodeType.Switch, stitch_node=True) + + p4_service_type = ServiceType.MPLS + p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', + nstype=p4_service_type, stitch_node=True) + + # add switch ports (they are stitch nodes, so just need to get their ids right) + ''' + TODO KOMAL + link_idx = 1 + for k, v in port_map.items(): + sp = p4_ns.add_interface(name=k, + node_id=dp_port_id(dp.name, k), + itype=InterfaceType.TrunkPort, + stitch_node=True) + topo.add_link(name='l' + str(link_idx), + node_id=sp.node_id + '-DAC', + ltype=LinkType.Patch, + interfaces=[sp, v]) + link_idx += 1 + ''' + return topo diff --git a/fimutil/ralph/p4_switch.py b/fimutil/ralph/p4_switch.py index b939e6d..5305ced 100644 --- a/fimutil/ralph/p4_switch.py +++ b/fimutil/ralph/p4_switch.py @@ -51,4 +51,16 @@ def parse(self): def __str__(self): ret = super().__str__() - return ret + '\n\t' + str(self.model) \ No newline at end of file + return ret + '\n\t' + str(self.model) + + def get_dp_ports(self): + """ + Return a list of names of DP switch ports this node is connected to + """ + dp_ports = list() + for n, comp in self.components.items(): + if comp.__dict__.get('type') and comp.type == RalphAssetType.EthernetCardPF: + if comp.fields.get('Peer_port'): + dp_ports.append(comp.fields.get('Peer_port')) + + return dp_ports diff --git a/fimutil/ralph/site.py b/fimutil/ralph/site.py index 90f5d67..2cc8958 100644 --- a/fimutil/ralph/site.py +++ b/fimutil/ralph/site.py @@ -72,11 +72,9 @@ def catalog(self): results = self.ralph.get_json_object(self.ralph.base_uri + 'data-center-assets/?' + urlencode(query)) p4_switch_url = pyjq.one('[ .results[0].url ]', results)[0] + logging.info(f'Identified P4 switch {p4_switch_url=}') self.p4_switch = P4Switch(uri=p4_switch_url, ralph=self.ralph) self.p4_switch.parse() - print("----------------------------------") - print(self.p4_switch) - print("----------------------------------") except ValueError: logging.warning('Unable to find a p4 switch in site, continuing') @@ -162,8 +160,13 @@ def to_json(self): ret["Storage"] = self.storage.fields.copy() if self.dp_switch: ret["DataPlane"] = self.dp_switch.fields.copy() - if self.dp_switch: + if self.p4_switch: ret["P4"] = self.p4_switch.fields.copy() + p4_dp_ports = self.p4_switch.get_dp_ports() + p4_dp_ports = list(set(p4_dp_ports)) + p4_dp_ports.sort() + ret["P4"]["Connected_ports"] = p4_dp_ports + # collect port information from all workers dp_ports = list() for w in self.workers: diff --git a/pyproject.toml b/pyproject.toml index 3be42a7..e9da94c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = ["License :: OSI Approved :: MIT License", dynamic = ["version", "description"] requires-python = '>=3.9' dependencies = [ - "fabric_fim >= 1.6.2", + "fabric_fim >= 1.7.0b1", "pyjq == 2.6.0", "jsonpath_ng == 1.5.3", ] From 54d481831d991865fffcb78652f135c54e0d062b Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 17:25:36 -0400 Subject: [PATCH 04/21] debugging for interfaces --- fimutil/ralph/fim_helper.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index 4654d8b..8ff2fc6 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -509,15 +509,36 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol # this prefers an IP address, but uses S/N if IP is None (like in GENI racks) p4_name = p4_switch_name_id(real_switch_site.lower(), site.p4_switch.fields['IP'] if site.p4_switch.fields['IP'] else site.p4_switch.fields['SN']) - logging.info(f'Adding P4 switch {dp_name}') + logging.info(f'Adding P4 switch {p4_name}') + p4 = topo.add_switch(name=p4_name[0], site=site.name, node_id=p4_name[1]) + print("--------------------------------") + print(f"KOMAL -p4-- {p4}") + print(f"KOMAL -interfaces-- {p4.interfaces}") + print("--------------------------------") + print(site.p4_switch.components) + print("--------------------------------") + ''' p4 = topo.add_node(name=p4_name[0], node_id=p4_name[1], site=site.name, ntype=NodeType.Switch, stitch_node=True) + ''' + # list of DP switch ports connected to the P4 switch in the order of port IDs 0-8 -> p1-p8 + # they would typically be 400G->100G breakouts + dp_to_p4_ports = ['HundredGigeE0/0/0/0/26.1', 'HundredGigeE0/0/0/0/26.2', 'HundredGigeE0/0/0/0/26.3', + 'HundredGigeE0/0/0/0/26.4', + 'HundredGigeE0/0/0/0/27.1', 'HundredGigeE0/0/0/0/27.2', 'HundredGigeE0/0/0/0/27.3', + 'HundredGigeE0/0/0/0/27.4'] - p4_service_type = ServiceType.MPLS - p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', - nstype=p4_service_type, stitch_node=True) - + ''' + # add dp switch ports that link to P4 switch ports (note they are not stitch nodes!!) + for d, p4idx in zip(dp_to_p4_ports, range(1, 8 + 1)): + sp = dp_ns.add_interface(name=d, itype=InterfaceType.TrunkPort, + node_id=dp_port_id(dp.name, d), stitch_node=False) + topo.add_link(name='l' + str(link_idx), ltype=LinkType.Patch, + interfaces=[p4.interfaces[f'p{p4idx}'], sp], + node_id=sp.node_id + '-DAC') + link_idx += 1 + ''' # add switch ports (they are stitch nodes, so just need to get their ids right) ''' TODO KOMAL From 918adba0825c720048df86cb5f3ddb660b3adfda Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 19:00:36 -0400 Subject: [PATCH 05/21] handle port mappings --- fimutil/ralph/fim_helper.py | 59 ++++++++++++++----------------------- fimutil/ralph/p4_switch.py | 5 +--- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index 8ff2fc6..f9f2dde 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -503,33 +503,34 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol logging.info(f'P4 Switch was not detected/catalogued') return topo - # check if we are using someone else's DP switch - real_switch_site = site.name - # this prefers an IP address, but uses S/N if IP is None (like in GENI racks) + logging.info(f'Adding P4 switch {site.name}') + p4_name = p4_switch_name_id(real_switch_site.lower(), site.p4_switch.fields['IP'] if site.p4_switch.fields['IP'] else site.p4_switch.fields['SN']) - logging.info(f'Adding P4 switch {p4_name}') - p4 = topo.add_switch(name=p4_name[0], site=site.name, node_id=p4_name[1]) - print("--------------------------------") - print(f"KOMAL -p4-- {p4}") - print(f"KOMAL -interfaces-- {p4.interfaces}") - print("--------------------------------") - print(site.p4_switch.components) - print("--------------------------------") - ''' + p4 = topo.add_node(name=p4_name[0], node_id=p4_name[1], site=site.name, ntype=NodeType.Switch, stitch_node=True) - ''' - # list of DP switch ports connected to the P4 switch in the order of port IDs 0-8 -> p1-p8 - # they would typically be 400G->100G breakouts - dp_to_p4_ports = ['HundredGigeE0/0/0/0/26.1', 'HundredGigeE0/0/0/0/26.2', 'HundredGigeE0/0/0/0/26.3', - 'HundredGigeE0/0/0/0/26.4', - 'HundredGigeE0/0/0/0/27.1', 'HundredGigeE0/0/0/0/27.2', 'HundredGigeE0/0/0/0/27.3', - 'HundredGigeE0/0/0/0/27.4'] - - ''' + + p4_service_type = ServiceType.MPLS + p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', + nstype=p4_service_type, stitch_node=False) + + index = 1 + dp_to_p4_ports = [] + for c in site.p4_switch.components.values(): + if "Management" in c: + print(f"KOMAL -- skipping component Management interface: {c}") + continue + print(f"KOMAL -- processing component: {c}") + labels = Labels(local_name=f'p{index}') + capacities = Capacities(bw=100) + p4_ns.add_interface(name=f'p{index}', node_id=p4_name[1] + f'-int{index}' if p4_name[1] else None, + itype=InterfaceType.DedicatedPort, + labels=labels, capacities=capacities) + # Build dp_to_p4_ports here + # add dp switch ports that link to P4 switch ports (note they are not stitch nodes!!) for d, p4idx in zip(dp_to_p4_ports, range(1, 8 + 1)): sp = dp_ns.add_interface(name=d, itype=InterfaceType.TrunkPort, @@ -538,22 +539,6 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol interfaces=[p4.interfaces[f'p{p4idx}'], sp], node_id=sp.node_id + '-DAC') link_idx += 1 - ''' - # add switch ports (they are stitch nodes, so just need to get their ids right) - ''' - TODO KOMAL - link_idx = 1 - for k, v in port_map.items(): - sp = p4_ns.add_interface(name=k, - node_id=dp_port_id(dp.name, k), - itype=InterfaceType.TrunkPort, - stitch_node=True) - topo.add_link(name='l' + str(link_idx), - node_id=sp.node_id + '-DAC', - ltype=LinkType.Patch, - interfaces=[sp, v]) - link_idx += 1 - ''' return topo diff --git a/fimutil/ralph/p4_switch.py b/fimutil/ralph/p4_switch.py index 5305ced..0b76907 100644 --- a/fimutil/ralph/p4_switch.py +++ b/fimutil/ralph/p4_switch.py @@ -12,10 +12,7 @@ class P4Switch(RalphAsset): """ Dataplane switch """ - FIELD_MAP = '{Name: .hostname, SN: .sn, IP: .ipaddresses[0], ' \ - 'AL2S_SWITCH: .custom_fields.al2s_remote_switch_name,' \ - 'AL2S_vlans: .custom_fields.al2s_vlan_ranges, ' \ - 'Local_vlans: .custom_fields.dataplane_vlan_ranges}' + FIELD_MAP = '{Name: .hostname, SN: .sn, IP: .ipaddresses[0]}' def __init__(self, *, uri: str, ralph: RalphURI): super().__init__(uri=uri, ralph=ralph) From be69e224ff83ceba15c976d086e376f58ffc5fca Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 21:55:07 -0400 Subject: [PATCH 06/21] p4 to dp port mappings --- fimutil/ralph/fim_helper.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index f9f2dde..a8d807d 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -517,16 +517,39 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', nstype=p4_service_type, stitch_node=False) - index = 1 dp_to_p4_ports = [] + for c in site.p4_switch.components.values(): - if "Management" in c: - print(f"KOMAL -- skipping component Management interface: {c}") - continue print(f"KOMAL -- processing component: {c}") + speed, unit = __parse_speed_spec(c.fields['Speed']) + speed = __normalize_units(speed, unit, 'G') + speed_int = int(speed) + capacities = Capacities(bw=speed_int) + + description = c.fields['Description'] + + # Use regular expression to find the value after "Port" + match = re.search(r'Port (\d+)', description) + if not match: + logging.warning(f"Port could not be determined from Description for component: {c}") + continue + interface_suffix = match.group(1) + index = interface_suffix labels = Labels(local_name=f'p{index}') - capacities = Capacities(bw=100) - p4_ns.add_interface(name=f'p{index}', node_id=p4_name[1] + f'-int{index}' if p4_name[1] else None, + + if "management" in description: + interface_suffix += "mgmt" + + connection = c.fields['Connection'] + + match2 = re.search(r'port\s+(\S+)', connection, re.IGNORECASE) + if not match2: + logging.warning(f"Data Plane port could not be determined from Connection for component: {c}") + continue + + dp_to_p4_ports.append(match2.group(1)) + + p4_ns.add_interface(name=f'p{index}', node_id=p4_name[1] + f'-int{interface_suffix}' if p4_name[1] else None, itype=InterfaceType.DedicatedPort, labels=labels, capacities=capacities) # Build dp_to_p4_ports here From a10545c0f6e2a89a90c79c65cac97d1f60edf632 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:00:02 -0400 Subject: [PATCH 07/21] handle mgmt port --- fimutil/ralph/fim_helper.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index a8d807d..adc2ede 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -520,7 +520,6 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol dp_to_p4_ports = [] for c in site.p4_switch.components.values(): - print(f"KOMAL -- processing component: {c}") speed, unit = __parse_speed_spec(c.fields['Speed']) speed = __normalize_units(speed, unit, 'G') speed_int = int(speed) @@ -528,17 +527,17 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol description = c.fields['Description'] - # Use regular expression to find the value after "Port" - match = re.search(r'Port (\d+)', description) - if not match: - logging.warning(f"Port could not be determined from Description for component: {c}") - continue - interface_suffix = match.group(1) - index = interface_suffix - labels = Labels(local_name=f'p{index}') - if "management" in description: - interface_suffix += "mgmt" + port_name = "mgmt" + else: + # Use regular expression to find the value after "Port" + match = re.search(r'Port (\d+)', description) + if not match: + logging.warning(f"Port could not be determined from Description for component: {c}") + continue + port_name = match.group(1) + + labels = Labels(local_name=f'p{port_name}') connection = c.fields['Connection'] @@ -547,12 +546,12 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol logging.warning(f"Data Plane port could not be determined from Connection for component: {c}") continue + # Build dp_to_p4_ports here dp_to_p4_ports.append(match2.group(1)) - p4_ns.add_interface(name=f'p{index}', node_id=p4_name[1] + f'-int{interface_suffix}' if p4_name[1] else None, + p4_ns.add_interface(name=f'p{port_name}', node_id=p4_name[1] + f'-int{port_name}' if p4_name[1] else None, itype=InterfaceType.DedicatedPort, labels=labels, capacities=capacities) - # Build dp_to_p4_ports here # add dp switch ports that link to P4 switch ports (note they are not stitch nodes!!) for d, p4idx in zip(dp_to_p4_ports, range(1, 8 + 1)): From 0f779c4ef518d7577e74b80fad623d88cfe46d79 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:01:17 -0400 Subject: [PATCH 08/21] pass mac --- fimutil/ralph/fim_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index adc2ede..cade504 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -526,7 +526,6 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol capacities = Capacities(bw=speed_int) description = c.fields['Description'] - if "management" in description: port_name = "mgmt" else: @@ -537,7 +536,7 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol continue port_name = match.group(1) - labels = Labels(local_name=f'p{port_name}') + labels = Labels(local_name=f'p{port_name}', mac=c.fields['MAC']) connection = c.fields['Connection'] From fc48e1fc857290c11b5f3be8f2a27e16ee594e37 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:03:24 -0400 Subject: [PATCH 09/21] only pass mac if not none --- fimutil/ralph/fim_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index cade504..98e7dd3 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -536,7 +536,9 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol continue port_name = match.group(1) - labels = Labels(local_name=f'p{port_name}', mac=c.fields['MAC']) + labels = Labels(local_name=f'p{port_name}') + if c.fields['MAC']: + labels.mac = c.fields['MAC'] connection = c.fields['Connection'] From b8ec6828aeade2111b648714fcaeef1b6ab94f32 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:17:23 -0400 Subject: [PATCH 10/21] ram offset --- README.md | 2 ++ fimutil/ralph/worker_node.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 396db88..6f2106e 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ The config file (by default `.scan-config.json` allows to statically override ce The general format example of the file is as follows (SITE1, SITE2 are all-caps site names): ``` { + "ram_offset": 24 "SITE1": { "dpswitch": { "URL": , @@ -117,6 +118,7 @@ The general format example of the file is as follows (SITE1, SITE2 are all-caps } } ``` +`ram_offset` specifies an offset to subtract from the actual RAM value. This is adjustment needed to for RAM allocated to NOVA on the workers. `mac_offset` intended to be used with OpenStack sites to aid unique MAC generation for vNICs. Note that the first octet of mac_offset must be [even](https://github.com/openstack/neutron-lib/blob/cf494c8be10b36daf238fa12cf7c615656e6640d/neutron_lib/api/validators/__init__.py#L40). diff --git a/fimutil/ralph/worker_node.py b/fimutil/ralph/worker_node.py index ce8741c..eb97682 100644 --- a/fimutil/ralph/worker_node.py +++ b/fimutil/ralph/worker_node.py @@ -98,6 +98,12 @@ def parse(self): if disk: self.model.fields['Disk'] = f'{disk}G' + if self.config and self.config.get('ram_offset'): + ram_offset = self.config.get('ram_offset') + ram -= ram_offset + if ram > 0: + self.model.fields['RAM'] = f'{ram}G' + # override from config if present if self.config and self.config.get(self.site) and self.config.get(self.site).get('workers') and \ self.config.get(self.site).get('workers').get(self.fields['Name']): From 9e43c594082da54e66a58d578ea334580efc2a0b Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:20:21 -0400 Subject: [PATCH 11/21] remove ram offset --- README.md | 2 -- fimutil/ralph/worker_node.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/README.md b/README.md index 6f2106e..396db88 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ The config file (by default `.scan-config.json` allows to statically override ce The general format example of the file is as follows (SITE1, SITE2 are all-caps site names): ``` { - "ram_offset": 24 "SITE1": { "dpswitch": { "URL": , @@ -118,7 +117,6 @@ The general format example of the file is as follows (SITE1, SITE2 are all-caps } } ``` -`ram_offset` specifies an offset to subtract from the actual RAM value. This is adjustment needed to for RAM allocated to NOVA on the workers. `mac_offset` intended to be used with OpenStack sites to aid unique MAC generation for vNICs. Note that the first octet of mac_offset must be [even](https://github.com/openstack/neutron-lib/blob/cf494c8be10b36daf238fa12cf7c615656e6640d/neutron_lib/api/validators/__init__.py#L40). diff --git a/fimutil/ralph/worker_node.py b/fimutil/ralph/worker_node.py index eb97682..ce8741c 100644 --- a/fimutil/ralph/worker_node.py +++ b/fimutil/ralph/worker_node.py @@ -98,12 +98,6 @@ def parse(self): if disk: self.model.fields['Disk'] = f'{disk}G' - if self.config and self.config.get('ram_offset'): - ram_offset = self.config.get('ram_offset') - ram -= ram_offset - if ram > 0: - self.model.fields['RAM'] = f'{ram}G' - # override from config if present if self.config and self.config.get(self.site) and self.config.get(self.site).get('workers') and \ self.config.get(self.site).get('workers').get(self.fields['Name']): From 4f97ded71b4700a6a6154486d30843cfe9d1d4bb Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:27:17 -0400 Subject: [PATCH 12/21] ram offset --- README.md | 2 ++ fimutil/ralph/worker_node.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 396db88..6f2106e 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ The config file (by default `.scan-config.json` allows to statically override ce The general format example of the file is as follows (SITE1, SITE2 are all-caps site names): ``` { + "ram_offset": 24 "SITE1": { "dpswitch": { "URL": , @@ -117,6 +118,7 @@ The general format example of the file is as follows (SITE1, SITE2 are all-caps } } ``` +`ram_offset` specifies an offset to subtract from the actual RAM value. This is adjustment needed to for RAM allocated to NOVA on the workers. `mac_offset` intended to be used with OpenStack sites to aid unique MAC generation for vNICs. Note that the first octet of mac_offset must be [even](https://github.com/openstack/neutron-lib/blob/cf494c8be10b36daf238fa12cf7c615656e6640d/neutron_lib/api/validators/__init__.py#L40). diff --git a/fimutil/ralph/worker_node.py b/fimutil/ralph/worker_node.py index ce8741c..eb97682 100644 --- a/fimutil/ralph/worker_node.py +++ b/fimutil/ralph/worker_node.py @@ -98,6 +98,12 @@ def parse(self): if disk: self.model.fields['Disk'] = f'{disk}G' + if self.config and self.config.get('ram_offset'): + ram_offset = self.config.get('ram_offset') + ram -= ram_offset + if ram > 0: + self.model.fields['RAM'] = f'{ram}G' + # override from config if present if self.config and self.config.get(self.site) and self.config.get(self.site).get('workers') and \ self.config.get(self.site).get('workers').get(self.fields['Name']): From a1ea1009177cbf930339d3399053496ec06e9ad2 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Fri, 10 May 2024 22:47:27 -0400 Subject: [PATCH 13/21] up the version number --- fimutil/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fimutil/__init__.py b/fimutil/__init__.py index b0fe67c..eb85ca4 100644 --- a/fimutil/__init__.py +++ b/fimutil/__init__.py @@ -3,5 +3,5 @@ This is a package of Information Model utilitied for FABRIC for scanning different types of sites """ -__VERSION__ = "1.6.4" +__VERSION__ = "1.7.0b1" __version__ = __VERSION__ From bf50d609df45494c625fc869dd2951019bdedced Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Sun, 12 May 2024 15:20:32 -0400 Subject: [PATCH 14/21] add delegation to P4 switch --- fimutil/__init__.py | 2 +- fimutil/ralph/fim_helper.py | 7 ++++--- fimutil/ralph/p4_switch.py | 1 - pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fimutil/__init__.py b/fimutil/__init__.py index eb85ca4..d09901e 100644 --- a/fimutil/__init__.py +++ b/fimutil/__init__.py @@ -3,5 +3,5 @@ This is a package of Information Model utilitied for FABRIC for scanning different types of sites """ -__VERSION__ = "1.7.0b1" +__VERSION__ = "1.7.0b2" __version__ = __VERSION__ diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index 98e7dd3..46a51f0 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -504,14 +504,15 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol return topo # this prefers an IP address, but uses S/N if IP is None (like in GENI racks) - logging.info(f'Adding P4 switch {site.name}') + logging.debug(f'Adding P4 switch {site.name}') p4_name = p4_switch_name_id(real_switch_site.lower(), site.p4_switch.fields['IP'] if site.p4_switch.fields['IP'] else site.p4_switch.fields['SN']) - + logging.info(f'Adding P4 switch {p4_name}') p4 = topo.add_node(name=p4_name[0], node_id=p4_name[1], - site=site.name, ntype=NodeType.Switch, stitch_node=True) + site=site.name, ntype=NodeType.Switch, stitch_node=False, + capacities=Capacities(units=1)) p4_service_type = ServiceType.MPLS p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', diff --git a/fimutil/ralph/p4_switch.py b/fimutil/ralph/p4_switch.py index 0b76907..1027564 100644 --- a/fimutil/ralph/p4_switch.py +++ b/fimutil/ralph/p4_switch.py @@ -1,6 +1,5 @@ import pyjq import logging -from typing import List from fimutil.ralph.ralph_uri import RalphURI from fimutil.ralph.asset import RalphAsset, RalphAssetType, RalphAssetMimatch diff --git a/pyproject.toml b/pyproject.toml index e9da94c..71e726c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = ["License :: OSI Approved :: MIT License", dynamic = ["version", "description"] requires-python = '>=3.9' dependencies = [ - "fabric_fim >= 1.7.0b1", + "fabric_fim >= 1.7.0b2", "pyjq == 2.6.0", "jsonpath_ng == 1.5.3", ] From 50c41ddb6b0d970728f718139c11ea34ee984a30 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Sun, 12 May 2024 15:25:17 -0400 Subject: [PATCH 15/21] fix units to unit --- fimutil/ralph/fim_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index 46a51f0..f83ccf2 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -512,7 +512,7 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol p4 = topo.add_node(name=p4_name[0], node_id=p4_name[1], site=site.name, ntype=NodeType.Switch, stitch_node=False, - capacities=Capacities(units=1)) + capacities=Capacities(unit=1)) p4_service_type = ServiceType.MPLS p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', From a6f4673ff10d40dc75f2396a84eeb8af53e51d25 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Sun, 12 May 2024 16:40:32 -0400 Subject: [PATCH 16/21] set management ip --- fimutil/ralph/fim_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index f83ccf2..69c3926 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -512,7 +512,7 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol p4 = topo.add_node(name=p4_name[0], node_id=p4_name[1], site=site.name, ntype=NodeType.Switch, stitch_node=False, - capacities=Capacities(unit=1)) + capacities=Capacities(unit=1), management_ip=site.p4_switch.fields['IP']) p4_service_type = ServiceType.MPLS p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', From 153edea98ebcd87f64bff5dfa9833d2d841542ed Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Mon, 20 May 2024 17:01:51 -0400 Subject: [PATCH 17/21] pick location from config --- fimutil/ralph/fim_helper.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index 69c3926..09b72b9 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -1,3 +1,4 @@ +import json from typing import Tuple, List, Dict, Any, Set import logging import re @@ -309,6 +310,10 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol loc = None if address is not None: loc = Location(postal=address) + loc.to_latlon() + + if config and config.get(site.name) and config.get(site.name).get('location'): + loc = Location.from_json(json_string=json.dumps(config.get(site.name).get('location'))) topo = SubstrateTopology() From a89587e25d0651932fbb3f61f2cda4fc84aa0b17 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Mon, 20 May 2024 17:17:58 -0400 Subject: [PATCH 18/21] pick offset per site --- fimutil/ralph/worker_node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fimutil/ralph/worker_node.py b/fimutil/ralph/worker_node.py index eb97682..b9b9f0e 100644 --- a/fimutil/ralph/worker_node.py +++ b/fimutil/ralph/worker_node.py @@ -98,8 +98,8 @@ def parse(self): if disk: self.model.fields['Disk'] = f'{disk}G' - if self.config and self.config.get('ram_offset'): - ram_offset = self.config.get('ram_offset') + if self.config and self.config.get(self.site) and self.config.get(self.site).get('ram_offset'): + ram_offset = self.config.get(self.site).get('ram_offset') ram -= ram_offset if ram > 0: self.model.fields['RAM'] = f'{ram}G' From 0117f69b90d926e48baba94b3703080046b2c203 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Mon, 20 May 2024 17:34:51 -0400 Subject: [PATCH 19/21] up the versions --- fimutil/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fimutil/__init__.py b/fimutil/__init__.py index d09901e..85aab8b 100644 --- a/fimutil/__init__.py +++ b/fimutil/__init__.py @@ -3,5 +3,5 @@ This is a package of Information Model utilitied for FABRIC for scanning different types of sites """ -__VERSION__ = "1.7.0b2" +__VERSION__ = "1.7.0b3" __version__ = __VERSION__ diff --git a/pyproject.toml b/pyproject.toml index 71e726c..7c13ecd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = ["License :: OSI Approved :: MIT License", dynamic = ["version", "description"] requires-python = '>=3.9' dependencies = [ - "fabric_fim >= 1.7.0b2", + "fabric_fim >= 1.7.0b3", "pyjq == 2.6.0", "jsonpath_ng == 1.5.3", ] From 5bded1254325a2663ea87a0674ad664ec1a5feee Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Thu, 30 May 2024 11:35:36 -0400 Subject: [PATCH 20/21] fix the service type for P4 service --- fimutil/__init__.py | 2 +- fimutil/ralph/fim_helper.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fimutil/__init__.py b/fimutil/__init__.py index 85aab8b..c4a0bd6 100644 --- a/fimutil/__init__.py +++ b/fimutil/__init__.py @@ -3,5 +3,5 @@ This is a package of Information Model utilitied for FABRIC for scanning different types of sites """ -__VERSION__ = "1.7.0b3" +__VERSION__ = "1.7.0b4" __version__ = __VERSION__ diff --git a/fimutil/ralph/fim_helper.py b/fimutil/ralph/fim_helper.py index 09b72b9..76e4ca9 100644 --- a/fimutil/ralph/fim_helper.py +++ b/fimutil/ralph/fim_helper.py @@ -519,7 +519,7 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol site=site.name, ntype=NodeType.Switch, stitch_node=False, capacities=Capacities(unit=1), management_ip=site.p4_switch.fields['IP']) - p4_service_type = ServiceType.MPLS + p4_service_type = ServiceType.P4 p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns', nstype=p4_service_type, stitch_node=False) From e447ec8895197be43333f9cb454992691c831d23 Mon Sep 17 00:00:00 2001 From: Komal Thareja Date: Mon, 15 Jul 2024 11:05:31 -0400 Subject: [PATCH 21/21] up dependencies and push to pypi --- fimutil/__init__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fimutil/__init__.py b/fimutil/__init__.py index c4a0bd6..fda12f0 100644 --- a/fimutil/__init__.py +++ b/fimutil/__init__.py @@ -3,5 +3,5 @@ This is a package of Information Model utilitied for FABRIC for scanning different types of sites """ -__VERSION__ = "1.7.0b4" +__VERSION__ = "1.7.0" __version__ = __VERSION__ diff --git a/pyproject.toml b/pyproject.toml index 7c13ecd..6e73d8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = ["License :: OSI Approved :: MIT License", dynamic = ["version", "description"] requires-python = '>=3.9' dependencies = [ - "fabric_fim >= 1.7.0b3", + "fabric_fim >= 1.7.0", "pyjq == 2.6.0", "jsonpath_ng == 1.5.3", ]