diff --git a/plugins/inventory/now.py b/plugins/inventory/now.py index 5fe5a9ae..d6ec4355 100644 --- a/plugins/inventory/now.py +++ b/plugins/inventory/now.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf8- -*- # Copyright: (c) 2021, XLAB Steampunk # # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) @@ -120,6 +120,12 @@ type: bool default: false version_added: 1.3.0 + experimental_aggregation: + description: + - Enable multiple variable values aggregations. + type: bool + default: false + version_added: 2.7.0 inventory_hostname_source: type: str description: @@ -329,6 +335,7 @@ import os +import hashlib from ansible.errors import AnsibleParserError from ansible.inventory.group import to_safe_group_name as orig_safe @@ -352,6 +359,76 @@ from ..module_utils.table import TableClient +class Aggregator: + def __init__(self, columns): + self.data = dict() + self.tmp = None + + def add(self, host, key, value): + parent, child = self._split(key) + if not self.tmp: + self.tmp = dict() + + if not child: + self.tmp[parent] = value + else: + tmp_parent_data = self.tmp.get(parent) + if not tmp_parent_data: + tmp_parent_data = dict() + + parent_data = dict() + parent_data[child] = value + if isinstance(tmp_parent_data, str): + parent_data[parent] = tmp_parent_data + else: + for k, v in tmp_parent_data.items(): + parent_data[k] = v + + self.tmp[parent] = parent_data + + def commit(self, host): + if not self.tmp: + return + host_data = self.data.get(host, dict()) + for k, v in self.tmp.items(): + if k in host_data: + vv = host_data.get(k) + if isinstance(vv, list): + if not self._is_exists(vv, v): + vv.append(v) + continue + if isinstance(v, dict): + host_data[k] = [v] + continue + host_data[k] = v + self.tmp = None + self.data[host] = host_data + + def aggregate(self, inventory): + for host, data in self.data.items(): + for k, v in data.items(): + inventory.set_variable(host, k, v) + + def _split(self, column): + if "." not in column: + return column, "" + parts = column.split(".") + return parts[0], parts[1] + + def _is_exists(self, items, item): + hash_v = self._hash_dict(item) + for i in items: + if self._hash_dict(i) == hash_v: + return True + return False + + def _hash_dict(self, d): + h = hashlib.sha256() + d_sorted = str(dict(sorted(d.items()))) + h.update(d_sorted.encode()) + return h.hexdigest() + + def construct_sysparm_query(query, is_encoded_query): if is_encoded_query: return query @@ -445,6 +522,11 @@ def set_hostvars(self, host, record, columns): for k in columns: self.inventory.set_variable(host, k.replace(".", "_"), record[k]) + def set_host_vars_aggregated(self, host, record, columns, aggregator): + for k in columns: + aggregator.add(host, k, record[k]) + aggregator.commit(host) + def fill_constructed( self, records, @@ -455,16 +537,26 @@ def fill_constructed( keyed_groups, strict, enhanced, + exp_aggregation, ): + if exp_aggregation: + aggregator = Aggregator(columns) + for record in records: host = self.add_host(record, name_source) if host: - self.set_hostvars(host, record, columns) + if exp_aggregation: + self.set_host_vars_aggregated(host, record, columns, aggregator) + else: + self.set_hostvars(host, record, columns) + self._set_composite_vars(compose, record, host, strict) self._add_host_to_composed_groups(groups, record, host, strict) self._add_host_to_keyed_groups(keyed_groups, record, host, strict) if enhanced: self.fill_enhanced_auto_groups(record, host) + if exp_aggregation: + aggregator.aggregate(self.inventory) def fill_enhanced_auto_groups(self, record, host): for rel_group in record["relationship_groups"]: @@ -569,6 +661,7 @@ def parse(self, inventory, loader, path, cache=True): raise AnsibleParserError(e) enhanced = self.get_option("enhanced") + exp_aggregation = self.get_option("experimental_aggregation") sysparm_limit = self.get_option("sysparm_limit") if sysparm_limit: @@ -636,7 +729,7 @@ def parse(self, inventory, loader, path, cache=True): ) enhance_records_with_rel_groups(records, rel_records) - self._cache[self.cache_key] = {cache_sub_key: records} + self._cache[self.cache_key] = {cache_sub_key: records} self.fill_constructed( records, @@ -647,4 +740,5 @@ def parse(self, inventory, loader, path, cache=True): self.get_option("keyed_groups"), self.get_option("strict"), enhanced, + exp_aggregation, ) diff --git a/tests/unit/plugins/inventory/test_now.py b/tests/unit/plugins/inventory/test_now.py index 38e99a4f..1d437e34 100644 --- a/tests/unit/plugins/inventory/test_now.py +++ b/tests/unit/plugins/inventory/test_now.py @@ -351,6 +351,7 @@ def test_construction_empty(self, inventory_plugin): keyed_groups = [] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -361,6 +362,7 @@ def test_construction_empty(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped")) @@ -385,6 +387,7 @@ def test_construction_host(self, inventory_plugin): keyed_groups = [] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -395,6 +398,7 @@ def test_construction_host(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped")) @@ -425,6 +429,7 @@ def test_construction_hostvars(self, inventory_plugin): keyed_groups = [] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -435,6 +440,7 @@ def test_construction_hostvars(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped")) @@ -493,6 +499,7 @@ def test_construction_composite_vars(self, inventory_plugin): keyed_groups = [] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -503,6 +510,7 @@ def test_construction_composite_vars(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped")) @@ -547,6 +555,7 @@ def test_construction_composite_vars_strict(self, inventory_plugin): keyed_groups = [] strict = True enhanced = False + exp_aggregation = False with pytest.raises(AnsibleError, match="non_existing"): inventory_plugin.fill_constructed( @@ -558,6 +567,7 @@ def test_construction_composite_vars_strict(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) def test_construction_composite_vars_ansible_host(self, inventory_plugin): @@ -579,6 +589,7 @@ def test_construction_composite_vars_ansible_host(self, inventory_plugin): keyed_groups = [] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -589,6 +600,7 @@ def test_construction_composite_vars_ansible_host(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set(("all", "ungrouped")) @@ -631,6 +643,7 @@ def test_construction_composed_groups(self, inventory_plugin): keyed_groups = [] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -641,6 +654,7 @@ def test_construction_composed_groups(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set( @@ -678,6 +692,7 @@ def test_construction_composed_groups_strict(self, inventory_plugin): keyed_groups = [] strict = True enhanced = False + exp_aggregation = False with pytest.raises(AnsibleError, match="cost_usd"): inventory_plugin.fill_constructed( @@ -689,6 +704,7 @@ def test_construction_composed_groups_strict(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) def test_construction_keyed_groups(self, inventory_plugin): @@ -710,6 +726,7 @@ def test_construction_keyed_groups(self, inventory_plugin): ] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -720,6 +737,7 @@ def test_construction_keyed_groups(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set( @@ -760,6 +778,7 @@ def test_construction_keyed_groups_with_parent(self, inventory_plugin): ] strict = False enhanced = False + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -770,6 +789,7 @@ def test_construction_keyed_groups_with_parent(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set( @@ -815,6 +835,7 @@ def test_construction_enhanced(self, inventory_plugin): keyed_groups = [] strict = False enhanced = True + exp_aggregation = False inventory_plugin.fill_constructed( records, @@ -825,6 +846,7 @@ def test_construction_enhanced(self, inventory_plugin): keyed_groups, strict, enhanced, + exp_aggregation, ) assert set(inventory_plugin.inventory.groups) == set( @@ -853,6 +875,47 @@ def test_construction_enhanced(self, inventory_plugin): assert a2.vars == dict(inventory_file=None, inventory_dir=None) + def test_exp_aggragation(self, inventory_plugin): + records = [ + {"sys_id": "1", "app": "tomcat1", "app.env": "dev", "fqdn": "a1"}, + {"sys_id": "2", "app": "tomcat2", "app.env": "prod", "fqdn": "a1"}, + {"sys_id": "3", "app": "tomcat3", "app.env": "staging", "fqdn": "a1"}, + {"sys_i": "4", "app": "tomcat4", "app.env": "dev", "fqdn": "a2"}, + ] + + columns = [ + "app", + "app.env", + "fqdn", + ] + name_source = "fqdn" + compose = {} + groups = {} + keyed_groups = [] + strict = False + enhanced = False + exp_aggregation = True + + inventory_plugin.fill_constructed( + records, + columns, + name_source, + compose, + groups, + keyed_groups, + strict, + enhanced, + exp_aggregation, + ) + + a1 = inventory_plugin.inventory.get_host("a1") + assert isinstance(a1.vars["app"], list) + for val in a1.vars["app"]: + assert "env" in val.keys() + assert "app" in val.keys() + assert val["env"] in ["prod", "dev", "staging"] + assert val["app"] in ["tomcat1", "tomcat2", "tomcat3"] + class TestConstructCacheSuffix: def test_from_query(self, inventory_plugin, mocker):