Diagram class related to #127

autogole-api/packaging/Dockerfile

@@ -1,11 +1,12 @@
 FROM opensciencegrid/software-base:23-al8-release
 
 RUN yum -y install wget epel-release && \
-    yum -y install git python3 python3-pyyaml python3-devel python3-pip gcc openssl-devel cronie python3-pyOpenSSL fetch-crl && \
+    yum -y install git python3 python3-pyyaml python3-devel python3-pip gcc openssl-devel cronie python3-pyOpenSSL fetch-crl graphviz && \
     yum clean all
 
 RUN mkdir -p /opt/ && \
     mkdir -p /srv/ && \
+    mkdir -p /srv/icons/ && \
     mkdir -p /etc/rtmon/templates/ && \
     mkdir -p /var/log/rtmon/ && \
     mkdir -p /etc/grid-security/certificates/
@@ -25,7 +26,9 @@ RUN git clone https://github.com/esnet/sense-rtmon.git /opt/sense-rtmon && \
 
 RUN wget https://raw.githubusercontent.com/sdn-sense/rm-configs/master/CAs/SiteRM.pem -O /etc/grid-security/certificates/e52ac827.0
 
-ADD files/etc/supervisord.d/10-server.conf /etc/supervisord.d/10-server.conf
-
+COPY files/etc/supervisord.d/10-server.conf /etc/supervisord.d/10-server.conf
+COPY icons/host.png /srv/icons/host.png
+COPY icons/switch.png /srv/icons/switch.png
+COPY imageserver.py /srv/imageserver.py
 # Get latest CA's
 RUN fetch-crl || echo "Supress warnings."

autogole-api/packaging/files/etc/rtmon.yaml

@@ -2,11 +2,13 @@
 sleep_timer: 30
 # Work dir for temp files (api directory). Default /srv
 workdir: '/srv/'
-
+image_dir: '/srv/images'
 # Grafana settings (for the API)
 grafana_host: 'https://autogole-grafana.nrp-nautilus.io'
-grafana_api_key: 'REPLACE_ME'
+grafana_api_key: ''
+baseImageURL: 'http://localhost:8080/images'
 grafana_folder: 'Real Time Mon'
+grafana_dev: 'Sunami-Image-Server'
 # FOR DEVELOPMEENT ONLY
 # Enable grafana_dev parameter (can be any string, and will be used as name to create directory inside Grafana)
 # Additionally - all dashboards will have this added to name.
@@ -25,7 +27,7 @@ data_sources:
 
 # Sense endpoints and their auth files.
 sense_endpoints:
-  "sense-o.es.net": "/etc/sense-o-auth-prod.yaml"
+  #"sense-o.es.net": "/etc/sense-o-auth-prod.yaml"
   "sense-o-dev.es.net": "/etc/sense-o-auth.yaml"
 
 # Additional links for the templates

autogole-api/packaging/files/etc/supervisord.d/10-server.conf

@@ -4,3 +4,15 @@ priority=-1
 stdout_logfile=/var/log/rtmon/daemon.log
 stdout_logfile_maxbytes=0
 redirect_stderr=true
+
+[program:image_server]
+command=python3 /srv/imageserver.py
+directory=/opt/devrtmon
+autostart=true
+autorestart=true
+stderr_logfile=/var/log/rtmon/image_server_error.log
+stderr_logfile_maxbytes=0
+stdout_logfile=/var/log/rtmon/image_server.log
+stdout_logfile_maxbytes=0
+priority=1
+redirect_stderr=true

autogole-api/packaging/imageserver.py

@@ -0,0 +1,36 @@
+from flask import Flask, send_from_directory, render_template_string, abort
+import os
+
+app = Flask(__name__)
+
+IMAGE_DIRECTORY = '/srv/images'
+@app.route('/')
+def home():
+    return "<h1>Hello, this is the root page!</h1>"
+
+@app.route('/images')
+def list_images():
+    if not os.path.exists(IMAGE_DIRECTORY):
+        return "<h1>Image directory not found.</h1>", 404
+
+    images = [f for f in os.listdir(IMAGE_DIRECTORY) if os.path.isfile(os.path.join(IMAGE_DIRECTORY, f))]
+
+    html = """
+    <h1>Network Topology</h1>
+    <ul>
+        {% for image in images %}
+        <li><a href="/images/{{ image }}">{{ image }}</a></li>
+        {% endfor %}
+    </ul>
+    """
+    return render_template_string(html, images=images)
+
+@app.route('/images/<filename>')
+def serve_image(filename):
+    if not os.path.isfile(os.path.join(IMAGE_DIRECTORY, filename)):
+        abort(404)
+    return send_from_directory(IMAGE_DIRECTORY, filename)
+
+if __name__ == "__main__":
+    port = 8000
+    app.run(host='0.0.0.0', port=port, debug=True)

autogole-api/packaging/start-dev.sh

@@ -14,5 +14,5 @@ docker run \
   -v $(pwd)/files/etc/grid-security/hostcert.pem:/etc/grid-security/hostcert.pem:ro \
   -v $(pwd)/files/etc/grid-security/hostcert.pem:/etc/grid-security/hostkey.pem:ro \
   --restart always \
-  --net=host \
+  -p 8000:8000 \
   rtmon

autogole-api/requirements.txt

@@ -2,3 +2,5 @@ psutil
 requests
 pyyaml
 grafana-client
+diagrams
+flask

autogole-api/src/python/RTMon/worker.py

@@ -84,8 +84,14 @@ def submit_exe(self, filename, fout):
         self._updateState(filename, fout)
 
     def delete_exe(self, filename, fout):
+
         """Delete Action Execution"""
         self.logger.info('Delete Execution: %s, %s', filename, fout)
+        #Deleting the diagram image
+        diagram_filename = f"{self.config.get('image_dir', '/srv/images')}/diagram_{fout['referenceUUID']}.png"
+        if os.path.exists(diagram_filename):
+            os.remove(diagram_filename)
+            self.logger.info(f"Removed diagram image {diagram_filename}")
         # Delete the dashboard and template from Grafana
         for grafDir, dirVals in self.dashboards.items():
             for dashbName, dashbVals in dirVals.items():

autogole-api/src/python/RTMonLibs/DiagramWorker.py

@@ -0,0 +1,256 @@
+"""
+DiagramWorker Class for Network Topology Visualization
+
+This module contains the DiagramWorker class, which generates network topology diagrams 
+by processing input data that includes hosts and switches. It uses the 'diagrams' library 
+to visualize network components and their interconnections.
+"""
+import os
+from diagrams import Diagram, Cluster, Edge
+from diagrams.custom import Custom
+from RTMonLibs.GeneralLibs import _processName
+# # change later:
+# def _processName(name):
+#     """Process Name for Mermaid and replace all special chars with _"""
+#     for repl in [[" ", "_"], [":", "_"], ["/", "_"], ["-", "_"], [".", "_"], ["?", "_"]]:
+#         name = name.replace(repl[0], repl[1])
+#     return name
+# #########
+
+class DiagramWorker:
+    """
+    DiagramWorker class is responsible for generating network topology diagrams
+    using the input data that contains host and switch information. The class
+    identifies and visualizes links between network components.
+    """
+    HOST_ICON_PATH = '/srv/icons/host.png'
+    SWITCH_ICON_PATH = '/srv/icons/switch.png'
+    # # TEmp
+    # HOST_ICON_PATH = '/Users/sunami/Desktop/publish/sense-rtmon/autogole-api/packaging/icons/host.png'
+    # SWITCH_ICON_PATH = '/Users/sunami/Desktop/publish/sense-rtmon/autogole-api/packaging/icons/switch.png'
+
+    def __init__(self, **kwargs):
+        """
+        Initialize the DiagramWorker with input data.
+
+        :param indata: List of dictionaries containing host and switch details.
+        """
+        self.objects = {}
+        self.added = {}
+        # self.siteoveride = SiteOverride(**kwargs)
+        self.linksadded = set()
+        self.popreverse = None
+        self.instance = kwargs.get("instance")
+
+
+    def d_find_item(self, fval, fkey):
+        """Find Item where fkey == fval"""
+        for key, vals in self.objects.items():
+            if vals.get('data', {}).get(fkey, '') == fval:
+                return key, vals
+        return None, None
+
+    @staticmethod
+    def d_LinkLabel(vals1, vals2):
+        """Get Link Label"""
+        label = ""
+        if vals1.get('data', {}).get('Type', '') == "Host":
+            label = f"Port1: {vals1['data']['Interface']}"
+        elif vals1.get('data', {}).get('Type', '') == "Switch":
+            label = f"Port1: {vals1['data']['Name']}"
+        # Get second side info:
+        if vals2.get('data', {}).get('Type', '') == "Host":
+            label += f"\nPort2: {vals2['data']['Interface']}"
+        elif vals2.get('data', {}).get('Type', '') == "Switch":
+            label += f"\nPort2: {vals2['data']['Name']}"
+        if vals1.get('data', {}).get('Vlan', None):
+            label += f"\nVlan: {vals1['data']['Vlan']}"
+        elif vals2.get('data', {}).get('Vlan', None):
+            label += f"\nVlan: {vals2['data']['Vlan']}"
+        return label
+
+    def d_addLink(self, val1, val2, key, fkey):
+        """Add Link between 2 objects"""
+        if val1 and val2 and key and fkey:
+            if key == fkey:
+                return
+
+            link_keys = tuple(sorted([key, fkey]))
+            if link_keys in self.linksadded:
+                return
+            self.linksadded.add(link_keys)
+
+            val1["obj"] >> Edge(label=self.d_LinkLabel(val1, val2)) << val2["obj"]
+
+    def d_addLinks(self):
+        """Identify Links between items"""
+        for key, vals in self.objects.items():
+            data_type = vals.get('data', {}).get('Type', '')
+            if data_type == "Host":
+                fKey, fItem = self.d_find_item(key, 'PeerHost')
+                if fKey and fItem:
+                    self.d_addLink(vals, fItem, key, fKey)
+            elif data_type == "Switch":
+                if 'Peer' in vals.get('data', {}) and vals['data']['Peer'] != "?peer?":
+                    fKey, fItem = self.d_find_item(vals['data']['Peer'], "Port")
+                    if fKey and fItem:
+                        self.d_addLink(vals, fItem, key, fKey)
+                    ## if Peer to host pair not found for overide
+                    else:
+                        try:
+                            siteName = vals['data']['Peer'].split("::")[0]
+                            switchName = vals['data']['Peer'].split("::")[1].split(":")[0]
+                            portname = vals['data']['Peer'].split("::")[1].split(":")[1]
+                        except:
+                            parts = vals['data']['Peer'].split(":")
+                            for i, part in enumerate(parts):
+                                if part.isdigit() and len(part) == 4:  
+                                    siteName = ":".join(parts[:i+1])  
+                                    remaining = ":".join(parts[i+1:])
+                                    break
+                            if "::" in remaining:
+                                subparts = remaining.split("::")
+                                switchName = subparts[0]
+                                portname = subparts[1].split(":")[0] if len(subparts) > 1 else None
+                            else:
+                                # If no '::', fallback to splitting by ':'
+                                subparts = remaining.split(":")
+                                switchName = subparts[0]
+                                portname = subparts[1] if len(subparts) > 1 else None
+                        with Cluster(siteName):
+                            newSwitch = Custom(switchName, self.SWITCH_ICON_PATH)
+                        newSwitch >> Edge(label="Port 1: " + portname + '\n' + "Port 2: "+ vals['data']["Name"] + '\n' + "Vlan: " +  vals['data']["Vlan"]) << vals["obj"]
+
+
+                elif 'PeerHost' in vals.get('data', {}):
+                    fKey = vals['data']['PeerHost']
+                    fItem = self.objects.get(fKey)
+                    if fItem:
+                        self.d_addLink(vals, fItem, key, fKey)
+
+    def d_addHost(self, item):
+        """
+        Add a host to the network diagram.
+
+        :param item: Dictionary containing host details.
+        :return: Diagram object representing the host.
+        """
+        name = f"Host: {item['Name'].split(':')[1]}"
+        name += f"\nInterface: {item['Interface']}"
+        name += f"\nVlan: {item['Vlan']}"
+        if 'IPv4' in item and item['IPv4'] != "?ipv4?":
+            name += f"\nIPv4: {item['IPv4']}"
+        if 'IPv6' in item and item['IPv6'] != "?ipv6?":
+            name += f"\nIPv6: {item['IPv6']}"
+
+        worker = Custom(name, self.HOST_ICON_PATH)
+        self.objects[item['Name']] = {"obj": worker, "data": item}
+        return worker
+
+    def d_addSwitch(self, item):
+        """
+        Add a switch to the network diagram.
+
+        :param item: Dictionary containing switch details.
+        :return: Diagram object representing the switch.
+        """
+        if item['Node'] in self.added:
+            self.objects[item['Port']] = {
+                                            "obj": self.objects[self.added[item['Node']]]["obj"],
+                                            "data": item
+                                         }
+            return
+        switch1 = Custom(item['Node'].split(":")[1], self.SWITCH_ICON_PATH)
+        if 'Peer' in item and item['Peer'] != "?peer?":
+            self.added[item['Node']] = item['Port']
+            self.objects[item['Port']] = {"obj": switch1, "data": item}
+        elif 'PeerHost' in item:
+            uniqname = _processName(f'{item["Node"]}_{item["Name"]}')
+            self.added[item['Node']] = uniqname
+            self.objects[uniqname] = {"obj": switch1, "data": item}
+         # Add IPv4/IPv6 on the switch
+        ## This is for BGP
+        for ipkey, ipdef in {'IPv4': '?port_ipv4?', 'IPv6': '?port_ipv6?'}.items():
+            if ipkey in item and item[ipkey] != ipdef:
+                ip_node_name = f"{item['Node']}_{ipkey}"
+                ip_label = item[ipkey]
+                ip_label2 = ""
+                sitename = item["Site"]
+
+                if self.instance != None:
+                    tempData = self.instance["intents"]
+                    for flow in tempData:
+                        terminals = flow["json"]["data"]["connections"][0]["terminals"]
+                        for connection in terminals:
+                            if connection["uri"] == sitename:
+                                ip_label2 = connection["ipv6_prefix_list"]
+                                break
+                ip_node = Custom(ip_label + "\n" + ip_label2, self.HOST_ICON_PATH)
+                switch1 >> Edge() << ip_node
+
+        return switch1
+
+
+
+
+    def addItem(self, item):
+        """
+        Add an item (host or switch) to the diagram by identifying its type and location (cluster).
+
+        :param item: Dictionary containing item details.
+        :return: Diagram object representing the item.
+        """
+        site = self.identifySite(item)
+        if item['Type'] == 'Host':
+            with Cluster(site):
+                return self.d_addHost(item)
+        elif item['Type'] == 'Switch':
+            with Cluster(site):
+                return self.d_addSwitch(item)
+
+    def identifySite(self, item):
+        """
+        Identify the site or cluster to which the item (host or switch) belongs.
+
+        :param item: Dictionary containing item details.
+        :return: The name of the site or cluster.
+        """
+        site = None
+        if item['Type'] == 'Host':
+            site = item['Name'].split(':')[0]
+        elif item['Type'] == 'Switch':
+            site = item['Node'].split(':')[0]
+        return site
+
+    def setreverse(self, item):
+        """
+        Set the reverse flag for alternating between the first and last items in the input list.
+
+        :param item: Dictionary containing item details.
+        """
+        if item['Type'] == 'Host' and self.popreverse is None:
+            self.popreverse = False
+        elif item['Type'] == 'Host' and self.popreverse is False:
+            self.popreverse = True
+        elif item['Type'] == 'Host' and self.popreverse is True:
+            self.popreverse = False
+
+    def createGraph(self, output_filename, indata):
+        """
+        Create the network topology diagram and save it to a file.
+
+        :param output_filename: Path where the output diagram will be saved.
+        """
+        output_dir = os.path.dirname(output_filename)
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        with Diagram("Network Topology", show=False, filename=output_filename):
+            while len(indata) > 0:
+                if self.popreverse in (None, False):
+                    item =indata.pop(0)
+                elif self.popreverse == True:
+                    item = indata.pop()
+                self.addItem(item)
+                self.setreverse(item)
+            self.d_addLinks()