Merge pull request #93 from fabric-testbed/rel1.7

Rel1.7 - changes

README.md

@@ -94,6 +94,7 @@ The config file (by default `.scan-config.json` allows to statically override ce
 The general format example of the file is as follows (SITE1, SITE2 are all-caps site names):
 ```
 {
+  "ram_offset": 24
   "SITE1": {
     "dpswitch": {
       "URL": <URL of SITE2's dp switch in Ralph>,
@@ -117,6 +118,7 @@ The general format example of the file is as follows (SITE1, SITE2 are all-caps
   }
 }
 ```
+`ram_offset` specifies an offset to subtract from the actual RAM value. This is adjustment needed to for RAM allocated to NOVA on the workers.
 `mac_offset` intended to be used with OpenStack sites to aid unique MAC generation for vNICs. Note
 that the first octet of mac_offset must be [even](https://github.com/openstack/neutron-lib/blob/cf494c8be10b36daf238fa12cf7c615656e6640d/neutron_lib/api/validators/__init__.py#L40).
 

fimutil/__init__.py

@@ -3,5 +3,5 @@
 This is a package of Information Model utilitied for FABRIC
 for scanning different types of sites
 """
-__VERSION__ = "1.6.4"
+__VERSION__ = "1.7.0"
 __version__ = __VERSION__

fimutil/ralph/asset.py

@@ -22,6 +22,7 @@ class RalphAssetType(Enum):
     DPSwitch = auto()
     Abstract = auto()
     FPGA = auto()
+    P4Switch = auto()
 
     def __str__(self):
         return self.name

fimutil/ralph/fim_helper.py

@@ -1,3 +1,4 @@
+import json
 from typing import Tuple, List, Dict, Any, Set
 import logging
 import re
@@ -309,6 +310,10 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol
     loc = None
     if address is not None:
         loc = Location(postal=address)
+        loc.to_latlon()
+
+    if config and config.get(site.name) and config.get(site.name).get('location'):
+        loc = Location.from_json(json_string=json.dumps(config.get(site.name).get('location')))
 
     topo = SubstrateTopology()
 
@@ -497,6 +502,73 @@ def site_to_fim(site: Site, address: str, config: Dict = None) -> SubstrateTopol
                       interfaces=[sp, v])
         link_idx += 1
 
+    # create p4 switch with interfaces and links back to dataplane switch ports
+    logging.debug('Adding p4 switch')
+    if site.p4_switch is None:
+        logging.info(f'P4 Switch was not detected/catalogued')
+        return topo
+
+    # this prefers an IP address, but uses S/N if IP is None (like in GENI racks)
+    logging.debug(f'Adding P4 switch {site.name}')
+
+    p4_name = p4_switch_name_id(real_switch_site.lower(),
+                                site.p4_switch.fields['IP'] if site.p4_switch.fields['IP'] else site.p4_switch.fields['SN'])
+    logging.info(f'Adding P4 switch {p4_name}')
+    p4 = topo.add_node(name=p4_name[0],
+                       node_id=p4_name[1],
+                       site=site.name, ntype=NodeType.Switch, stitch_node=False,
+                       capacities=Capacities(unit=1), management_ip=site.p4_switch.fields['IP'])
+
+    p4_service_type = ServiceType.P4
+    p4_ns = p4.add_network_service(name=p4.name + '-ns', node_id=p4.node_id + '-ns',
+                                   nstype=p4_service_type, stitch_node=False)
+
+    dp_to_p4_ports = []
+
+    for c in site.p4_switch.components.values():
+        speed, unit = __parse_speed_spec(c.fields['Speed'])
+        speed = __normalize_units(speed, unit, 'G')
+        speed_int = int(speed)
+        capacities = Capacities(bw=speed_int)
+
+        description = c.fields['Description']
+        if "management" in description:
+            port_name = "mgmt"
+        else:
+            # Use regular expression to find the value after "Port"
+            match = re.search(r'Port (\d+)', description)
+            if not match:
+                logging.warning(f"Port could not be determined from Description for component: {c}")
+                continue
+            port_name = match.group(1)
+
+        labels = Labels(local_name=f'p{port_name}')
+        if c.fields['MAC']:
+            labels.mac = c.fields['MAC']
+
+        connection = c.fields['Connection']
+
+        match2 = re.search(r'port\s+(\S+)', connection, re.IGNORECASE)
+        if not match2:
+            logging.warning(f"Data Plane port could not be determined from Connection for component: {c}")
+            continue
+
+        # Build dp_to_p4_ports here
+        dp_to_p4_ports.append(match2.group(1))
+
+        p4_ns.add_interface(name=f'p{port_name}', node_id=p4_name[1] + f'-int{port_name}' if p4_name[1] else None,
+                            itype=InterfaceType.DedicatedPort,
+                            labels=labels, capacities=capacities)
+
+    # add dp switch ports that link to P4 switch ports (note they are not stitch nodes!!)
+    for d, p4idx in zip(dp_to_p4_ports, range(1, 8 + 1)):
+        sp = dp_ns.add_interface(name=d, itype=InterfaceType.TrunkPort,
+                                 node_id=dp_port_id(dp.name, d), stitch_node=False)
+        topo.add_link(name='l' + str(link_idx), ltype=LinkType.Patch,
+                           interfaces=[p4.interfaces[f'p{p4idx}'], sp],
+                           node_id=sp.node_id + '-DAC')
+        link_idx += 1
+
     return topo
 
 

fimutil/ralph/p4_switch.py

@@ -0,0 +1,62 @@
+import pyjq
+import logging
+
+from fimutil.ralph.ralph_uri import RalphURI
+from fimutil.ralph.asset import RalphAsset, RalphAssetType, RalphAssetMimatch
+from fimutil.ralph.model import SimpleModel
+from fimutil.ralph.ethernetport import EthernetPort
+
+
+class P4Switch(RalphAsset):
+    """
+    Dataplane switch
+    """
+    FIELD_MAP = '{Name: .hostname, SN: .sn, IP: .ipaddresses[0]}'
+
+    def __init__(self, *, uri: str, ralph: RalphURI):
+        super().__init__(uri=uri, ralph=ralph)
+        self.type = RalphAssetType.P4Switch
+        self.model = None
+
+    def parse(self):
+        super().parse()
+
+        # find model
+        model_url = pyjq.one('.model.url', self.raw_json_obj)
+        self.model = SimpleModel(uri=model_url, ralph=self.ralph)
+        try:
+            self.model.parse()
+        except RalphAssetMimatch:
+            logging.warning('Unable to parse switch model, continuing')
+
+        try:
+            port_urls = pyjq.all('.ethernet[].url', self.raw_json_obj)
+        except ValueError:
+            logging.warning('Unable to find any ethernet ports in node, continuing')
+            port_urls = list()
+
+        port_index = 1
+        for port in port_urls:
+            port = EthernetPort(uri=port, ralph=self.ralph)
+            try:
+                port.parse()
+            except RalphAssetMimatch:
+                continue
+            self.components['port-' + str(port_index)] = port
+            port_index += 1
+
+    def __str__(self):
+        ret = super().__str__()
+        return ret + '\n\t' + str(self.model)
+
+    def get_dp_ports(self):
+        """
+        Return a list of names of DP switch ports this node is connected to
+        """
+        dp_ports = list()
+        for n, comp in self.components.items():
+            if comp.__dict__.get('type') and comp.type == RalphAssetType.EthernetCardPF:
+                if comp.fields.get('Peer_port'):
+                    dp_ports.append(comp.fields.get('Peer_port'))
+
+        return dp_ports

fimutil/ralph/site.py

@@ -5,6 +5,7 @@
 from typing import Dict
 import json
 
+from fimutil.ralph.p4_switch import P4Switch
 from fimutil.ralph.ralph_uri import RalphURI
 from fimutil.ralph.worker_node import WorkerNode
 from fimutil.ralph.storage import Storage
@@ -23,6 +24,7 @@ def __init__(self, *, site_name: str, ralph: RalphURI, config: Dict = None, doma
         self.workers = list()
         self.storage = None
         self.dp_switch = None
+        self.p4_switch = None
         self.ptp = False
         self.name = site_name
         self.domain = domain
@@ -64,6 +66,18 @@ def catalog(self):
         except ValueError:
             logging.warning('Unable to find a dataplane switch in site, continuing')
 
+        try:
+            logging.info(f'Searching for P4 switch URL')
+            query = {'hostname': f'{self.name.lower()}-p4-sw' + self.domain}
+            results = self.ralph.get_json_object(self.ralph.base_uri + 'data-center-assets/?' +
+                                                 urlencode(query))
+            p4_switch_url = pyjq.one('[ .results[0].url ]', results)[0]
+            logging.info(f'Identified P4 switch {p4_switch_url=}')
+            self.p4_switch = P4Switch(uri=p4_switch_url, ralph=self.ralph)
+            self.p4_switch.parse()
+        except ValueError:
+            logging.warning('Unable to find a p4 switch in site, continuing')
+
         query = {'hostname': f'{self.name.lower()}-time' + self.domain}
         results = self.ralph.get_json_object(self.ralph.base_uri + 'data-center-assets/?' +
                                              urlencode(query))
@@ -134,6 +148,8 @@ def __str__(self):
             assets.append(str(self.storage))
         if self.dp_switch:
             assets.append(str(self.dp_switch))
+        if self.p4_switch:
+            assets.append(str(self.p4_switch))
         for w in self.workers:
             assets.append(str(w))
         return '\n'.join(assets)
@@ -144,6 +160,13 @@ def to_json(self):
             ret["Storage"] = self.storage.fields.copy()
         if self.dp_switch:
             ret["DataPlane"] = self.dp_switch.fields.copy()
+        if self.p4_switch:
+            ret["P4"] = self.p4_switch.fields.copy()
+            p4_dp_ports = self.p4_switch.get_dp_ports()
+            p4_dp_ports = list(set(p4_dp_ports))
+            p4_dp_ports.sort()
+            ret["P4"]["Connected_ports"] = p4_dp_ports
+
         # collect port information from all workers
         dp_ports = list()
         for w in self.workers:
@@ -159,4 +182,4 @@ def to_json(self):
         for w in self.workers:
             n.append(w.to_json())
         ret["Nodes"] = n
-        return ret
+        return ret

fimutil/ralph/worker_node.py

@@ -98,6 +98,12 @@ def parse(self):
         if disk:
             self.model.fields['Disk'] = f'{disk}G'
 
+        if self.config and self.config.get(self.site) and self.config.get(self.site).get('ram_offset'):
+            ram_offset = self.config.get(self.site).get('ram_offset')
+            ram -= ram_offset
+            if ram > 0:
+                self.model.fields['RAM'] = f'{ram}G'
+
         # override from config if present
         if self.config and self.config.get(self.site) and self.config.get(self.site).get('workers') and \
             self.config.get(self.site).get('workers').get(self.fields['Name']):

pyproject.toml

@@ -15,7 +15,7 @@ classifiers = ["License :: OSI Approved :: MIT License",
 dynamic = ["version", "description"]
 requires-python = '>=3.9'
 dependencies = [
-	"fabric_fim >= 1.6.2",
+	"fabric_fim >= 1.7.0",
     "pyjq == 2.6.0",
 	"jsonpath_ng == 1.5.3",
 	]