inventory: Implement aggregation of multiple variable values

changelogs/fragments/inventory_aggregation.yml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+  - inventory - allow inventory to aggregate multiple hostvars for the same host. (https://github.com/ansible-collections/servicenow.itsm/pull/408)

plugins/inventory/now.py

@@ -120,6 +120,12 @@
     type: bool
     default: false
     version_added: 1.3.0
+  aggregation:
+    description:
+      - Enable multiple variable values aggregations.
+    type: bool
+    default: false
+    version_added: 2.7.0
   inventory_hostname_source:
     type: str
     description:
@@ -329,6 +335,7 @@
 
 
 import os
+import hashlib
 
 from ansible.errors import AnsibleParserError
 from ansible.inventory.group import to_safe_group_name as orig_safe
@@ -352,6 +359,76 @@
 from ..module_utils.table import TableClient
 
 
+class Aggregator:
+    def __init__(self, columns):
+        self.data = dict()
+        self.tmp = None
+
+    def add(self, host, key, value):
+        parent, child = self._split(key)
+        if not self.tmp:
+            self.tmp = dict()
+
+        if not child:
+            self.tmp[parent] = value
+        else:
+            tmp_parent_data = self.tmp.get(parent)
+            if not tmp_parent_data:
+                tmp_parent_data = dict()
+
+            parent_data = dict()
+            parent_data[child] = value
+            if isinstance(tmp_parent_data, str):
+                parent_data[parent] = tmp_parent_data
+            else:
+                for k, v in tmp_parent_data.items():
+                    parent_data[k] = v
+
+            self.tmp[parent] = parent_data
+
+    def commit(self, host):
+        if not self.tmp:
+            return
+        host_data = self.data.get(host, dict())
+        for k, v in self.tmp.items():
+            if k in host_data:
+                vv = host_data.get(k)
+                if isinstance(vv, list):
+                    if not self._is_exists(vv, v):
+                        vv.append(v)
+                continue
+            if isinstance(v, dict):
+                host_data[k] = [v]
+                continue
+            host_data[k] = v
+        self.tmp = None
+        self.data[host] = host_data
+
+    def aggregate(self, inventory):
+        for host, data in self.data.items():
+            for k, v in data.items():
+                inventory.set_variable(host, k, v)
+
+    def _split(self, column):
+        if "." not in column:
+            return column, ""
+        parts = column.split(".")
+        return parts[0], parts[1]
+
+    def _is_exists(self, items, item):
+        hash_v = self._hash_dict(item)
+        for i in items:
+            if self._hash_dict(i) == hash_v:
+                return True
+        return False
+
+    def _hash_dict(self, d):
+        h = hashlib.sha256()
+        d_sorted = str(dict(sorted(d.items())))
+        h.update(d_sorted.encode())
+        return h.hexdigest()
+
+
 def construct_sysparm_query(query, is_encoded_query):
     if is_encoded_query:
         return query
@@ -445,6 +522,11 @@ def set_hostvars(self, host, record, columns):
         for k in columns:
             self.inventory.set_variable(host, k.replace(".", "_"), record[k])
 
+    def set_host_vars_aggregated(self, host, record, columns, aggregator):
+        for k in columns:
+            aggregator.add(host, k, record[k])
+        aggregator.commit(host)
+
     def fill_constructed(
         self,
         records,
@@ -455,16 +537,26 @@ def fill_constructed(
         keyed_groups,
         strict,
         enhanced,
+        aggregation,
     ):
+        if aggregation:
+            aggregator = Aggregator(columns)
+
         for record in records:
             host = self.add_host(record, name_source)
             if host:
-                self.set_hostvars(host, record, columns)
+                if aggregation:
+                    self.set_host_vars_aggregated(host, record, columns, aggregator)
+                else:
+                    self.set_hostvars(host, record, columns)
+
                 self._set_composite_vars(compose, record, host, strict)
                 self._add_host_to_composed_groups(groups, record, host, strict)
                 self._add_host_to_keyed_groups(keyed_groups, record, host, strict)
                 if enhanced:
                     self.fill_enhanced_auto_groups(record, host)
+        if aggregation:
+            aggregator.aggregate(self.inventory)
 
     def fill_enhanced_auto_groups(self, record, host):
         for rel_group in record["relationship_groups"]:
@@ -569,6 +661,7 @@ def parse(self, inventory, loader, path, cache=True):
             raise AnsibleParserError(e)
 
         enhanced = self.get_option("enhanced")
+        aggregation = self.get_option("aggregation")
 
         sysparm_limit = self.get_option("sysparm_limit")
         if sysparm_limit:
@@ -647,4 +740,5 @@ def parse(self, inventory, loader, path, cache=True):
             self.get_option("keyed_groups"),
             self.get_option("strict"),
             enhanced,
+            aggregation,
         )

tests/unit/plugins/inventory/test_now.py

@@ -351,6 +351,7 @@ def test_construction_empty(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -361,6 +362,7 @@ def test_construction_empty(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped"))
@@ -385,6 +387,7 @@ def test_construction_host(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -395,6 +398,7 @@ def test_construction_host(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped"))
@@ -425,6 +429,7 @@ def test_construction_hostvars(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -435,6 +440,7 @@ def test_construction_hostvars(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped"))
@@ -493,6 +499,7 @@ def test_construction_composite_vars(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -503,6 +510,7 @@ def test_construction_composite_vars(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped"))
@@ -547,6 +555,7 @@ def test_construction_composite_vars_strict(self, inventory_plugin):
         keyed_groups = []
         strict = True
         enhanced = False
+        aggregation = False
 
         with pytest.raises(AnsibleError, match="non_existing"):
             inventory_plugin.fill_constructed(
@@ -558,6 +567,7 @@ def test_construction_composite_vars_strict(self, inventory_plugin):
                 keyed_groups,
                 strict,
                 enhanced,
+                aggregation,
             )
 
     def test_construction_composite_vars_ansible_host(self, inventory_plugin):
@@ -579,6 +589,7 @@ def test_construction_composite_vars_ansible_host(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -589,6 +600,7 @@ def test_construction_composite_vars_ansible_host(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped"))
@@ -631,6 +643,7 @@ def test_construction_composed_groups(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -641,6 +654,7 @@ def test_construction_composed_groups(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(
@@ -678,6 +692,7 @@ def test_construction_composed_groups_strict(self, inventory_plugin):
         keyed_groups = []
         strict = True
         enhanced = False
+        aggregation = False
 
         with pytest.raises(AnsibleError, match="cost_usd"):
             inventory_plugin.fill_constructed(
@@ -689,6 +704,7 @@ def test_construction_composed_groups_strict(self, inventory_plugin):
                 keyed_groups,
                 strict,
                 enhanced,
+                aggregation,
             )
 
     def test_construction_keyed_groups(self, inventory_plugin):
@@ -710,6 +726,7 @@ def test_construction_keyed_groups(self, inventory_plugin):
         ]
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -720,6 +737,7 @@ def test_construction_keyed_groups(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(
@@ -760,6 +778,7 @@ def test_construction_keyed_groups_with_parent(self, inventory_plugin):
         ]
         strict = False
         enhanced = False
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -770,6 +789,7 @@ def test_construction_keyed_groups_with_parent(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(
@@ -815,6 +835,7 @@ def test_construction_enhanced(self, inventory_plugin):
         keyed_groups = []
         strict = False
         enhanced = True
+        aggregation = False
 
         inventory_plugin.fill_constructed(
             records,
@@ -825,6 +846,7 @@ def test_construction_enhanced(self, inventory_plugin):
             keyed_groups,
             strict,
             enhanced,
+            aggregation,
         )
 
         assert set(inventory_plugin.inventory.groups) == set(
@@ -853,6 +875,47 @@ def test_construction_enhanced(self, inventory_plugin):
 
         assert a2.vars == dict(inventory_file=None, inventory_dir=None)
 
+    def test_aggragation(self, inventory_plugin):
+        records = [
+            {"sys_id": "1", "app": "tomcat1", "app.env": "dev", "fqdn": "a1"},
+            {"sys_id": "2", "app": "tomcat2", "app.env": "prod", "fqdn": "a1"},
+            {"sys_id": "3", "app": "tomcat3", "app.env": "staging", "fqdn": "a1"},
+            {"sys_i": "4", "app": "tomcat4", "app.env": "dev", "fqdn": "a2"},
+        ]
+
+        columns = [
+            "app",
+            "app.env",
+            "fqdn",
+        ]
+        name_source = "fqdn"
+        compose = {}
+        groups = {}
+        keyed_groups = []
+        strict = False
+        enhanced = False
+        aggregation = True
+
+        inventory_plugin.fill_constructed(
+            records,
+            columns,
+            name_source,
+            compose,
+            groups,
+            keyed_groups,
+            strict,
+            enhanced,
+            aggregation,
+        )
+
+        a1 = inventory_plugin.inventory.get_host("a1")
+        assert isinstance(a1.vars["app"], list)
+        for val in a1.vars["app"]:
+            assert "env" in val.keys()
+            assert "app" in val.keys()
+            assert val["env"] in ["prod", "dev", "staging"]
+            assert val["app"] in ["tomcat1", "tomcat2", "tomcat3"]
+
 
 class TestConstructCacheSuffix:
     def test_from_query(self, inventory_plugin, mocker):