chore: Upgrades ES to 7.13.3

databuilder/databuilder/publisher/elasticsearch_publisher.py

@@ -90,7 +90,8 @@ def publish_impl(self) -> None:
         cnt = 0
 
         # create new index with mapping
-        self.elasticsearch_client.indices.create(index=self.elasticsearch_new_index, body=self.elasticsearch_mapping)
+        self.elasticsearch_client.indices.create(index=self.elasticsearch_new_index, body=self.elasticsearch_mapping,
+                                                 params={'include_type_name': 'true'})
         for action in actions:
             index_row = dict(index=dict(_index=self.elasticsearch_new_index,
                                         _type=self.elasticsearch_type))

databuilder/requirements.txt

@@ -1,7 +1,7 @@
 # Copyright Contributors to the Amundsen project.
 # SPDX-License-Identifier: Apache-2.0
 
-elasticsearch>=6.2.0,<7.0
+elasticsearch>=6.2.0,<7.13.4
 neo4j-driver>=1.7.2,<2.0
 requests>=2.25.0,<3.0
 

docker-amundsen-local.yml

@@ -18,7 +18,7 @@ services:
       networks:
         - amundsennet
   elasticsearch:
-      image: elasticsearch:6.7.0
+      image: elasticsearch:7.13.3
       container_name: es_amundsen
       ports:
           - 9200:9200
@@ -28,6 +28,8 @@ services:
         nofile:
           soft: 65536
           hard: 65536
+      environment:
+        - discovery.type=single-node
   amundsensearch:
       build:
         context: .

docker-amundsen.yml

@@ -20,7 +20,7 @@ services:
       networks:
         - amundsennet
   elasticsearch:
-      image: elasticsearch:6.7.0
+      image: elasticsearch:7.13.3
       container_name: es_amundsen
       ports:
           - 9200:9200
@@ -32,6 +32,8 @@ services:
         nofile:
           soft: 65536
           hard: 65536
+      environment:
+        - discovery.type=single-node
   amundsensearch:
       image: amundsendev/amundsen-search:2.4.1
       container_name: amundsensearch

search/requirements.txt

@@ -2,5 +2,5 @@
 # SPDX-License-Identifier: Apache-2.0
 
 pyatlasclient==1.0.3
-elasticsearch==6.8.2
-elasticsearch-dsl==6.4.0
+elasticsearch==7.13.3
+elasticsearch-dsl==7.4.0

search/search_service/proxy/elasticsearch.py

@@ -13,6 +13,7 @@
 from elasticsearch import Elasticsearch
 from elasticsearch.exceptions import NotFoundError
 from elasticsearch_dsl import Search, query
+from elasticsearch_dsl.utils import AttrDict
 from flask import current_app
 
 from search_service import config
@@ -284,7 +285,13 @@ def _get_search_result(self, page_index: int,
             except Exception:
                 LOGGING.exception('The record doesnt contain specified field.')
 
-        return search_result_model(total_results=response.hits.total,
+        # This is to support ESv7.x, and newer version of elasticsearch_dsl
+        if isinstance(response.hits.total, AttrDict):
+            _total = response.hits.total.value
+        else:
+            _total = response.hits.total
+
+        return search_result_model(total_results=_total,
                                    results=results)
 
     def _get_instance(self, attr: str, val: Any) -> Any:
@@ -311,6 +318,9 @@ def _search_helper(self, page_index: int,
         :param query_name: name of query to query the ES
         :return:
         """
+        # This is to support ESv7.x
+        # ref: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#track-total-hits-10000-default # noqa: E501
+        client = client.extra(track_total_hits=True)
 
         if query_name:
             q = query.Q(query_name)