From f23d5a46f8e8a5444f952a71f9918d47e8cc0c4f Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Tue, 13 Feb 2024 15:04:14 -0500 Subject: [PATCH 1/8] move job to bootstrap, switch to redis --- bootstrap-app/project.clj | 1 + .../src/cmr/bootstrap/api/routes.clj | 4 +++ bootstrap-app/src/cmr/bootstrap/system.clj | 4 ++- .../services/search/query_execution.clj | 5 ++- .../cmr/search/services/query_execution.clj | 1 + .../granule_counts_results_feature.clj | 1 + .../has_granules_results_feature.clj | 33 ++++++++++++------- search-app/src/cmr/search/system.clj | 3 -- 8 files changed, 36 insertions(+), 16 deletions(-) diff --git a/bootstrap-app/project.clj b/bootstrap-app/project.clj index 1453f6ea14..b4a53ffd78 100644 --- a/bootstrap-app/project.clj +++ b/bootstrap-app/project.clj @@ -25,6 +25,7 @@ [nasa-cmr/cmr-indexer-app "0.1.0-SNAPSHOT"] [nasa-cmr/cmr-metadata-db-app "0.1.0-SNAPSHOT"] [nasa-cmr/cmr-oracle-lib "0.1.0-SNAPSHOT"] + [nasa-cmr/cmr-search-app "0.1.0-SNAPSHOT"] [nasa-cmr/cmr-transmit-lib "0.1.0-SNAPSHOT"] [nasa-cmr/cmr-virtual-product-app "0.1.0-SNAPSHOT"] [org.apache.httpcomponents/httpclient "4.5.13"] diff --git a/bootstrap-app/src/cmr/bootstrap/api/routes.clj b/bootstrap-app/src/cmr/bootstrap/api/routes.clj index fb8ccaf9a4..aab57a2432 100644 --- a/bootstrap-app/src/cmr/bootstrap/api/routes.clj +++ b/bootstrap-app/src/cmr/bootstrap/api/routes.clj @@ -21,6 +21,7 @@ [cmr.common-app.services.search.elastic-search-index-names-cache :as elastic-search-index-names-cache] [cmr.common.log :refer [info]] [cmr.common.generics :as common-generic] + [cmr.search.services.query-execution.has-granules-results-feature :as has-granules-results-feature] [compojure.core :refer :all] [compojure.route :as route] [drift.execute :as drift] @@ -143,6 +144,9 @@ (= keyword-cache-name humanizer-alias-cache/humanizer-alias-cache-key) (humanizer-alias-cache/refresh-entire-cache request-context) + (= keyword-cache-name has-granules-results-feature/has-granule-cache-key) + (has-granules-results-feature/refresh-has-granules-map request-context) + :else (route/not-found "Not Found"))) {:status 200})) diff --git a/bootstrap-app/src/cmr/bootstrap/system.clj b/bootstrap-app/src/cmr/bootstrap/system.clj index 2827ae2823..35c8bfea71 100644 --- a/bootstrap-app/src/cmr/bootstrap/system.clj +++ b/bootstrap-app/src/cmr/bootstrap/system.clj @@ -34,6 +34,7 @@ [cmr.metadata-db.config :as mdb-config] [cmr.metadata-db.system :as mdb-system] [cmr.metadata-db.services.util :as mdb-util] + [cmr.search.services.query-execution.has-granules-results-feature :as has-granules-results-feature] [cmr.transmit.config :as transmit-config])) (defconfig db-batch-size @@ -100,7 +101,8 @@ (b-coll-metadata-cache/update-collections-metadata-cache-job "bootstrap-collections-metadata-cache-update") (coll-gran-acls-caches/refresh-collections-cache-for-granule-acls-job "bootstrap-collections-for-gran-acls-cache-refresh") (elastic-search-index-names-cache/refresh-index-names-cache-job "bootstrap-elastic-search-index-names-cache") - (humanizer-alias-cache/refresh-humanizer-alias-cache-job "boostrap-humanizer-alias-cache-refresh")]) + (humanizer-alias-cache/refresh-humanizer-alias-cache-job "boostrap-humanizer-alias-cache-refresh") + (has-granules-results-feature/refresh-has-granules-map-job "bootstrap-has-granules-map-refresh")]) :queue-broker queue-broker}] (transmit-config/system-with-connections sys [:metadata-db :echo-rest :kms :indexer :access-control :search]))) diff --git a/common-app-lib/src/cmr/common_app/services/search/query_execution.clj b/common-app-lib/src/cmr/common_app/services/search/query_execution.clj index 8c88a69df0..b1218fa0ff 100644 --- a/common-app-lib/src/cmr/common_app/services/search/query_execution.clj +++ b/common-app-lib/src/cmr/common_app/services/search/query_execution.clj @@ -46,6 +46,7 @@ (defmethod post-process-query-result-feature :default [context query elastic-results query-results feature] + (println "inside post-process-query-result-feature :default") ; Does nothing by default query-results) @@ -59,6 +60,7 @@ (defn post-process-query-result-features "Processes result features that execute after a query results have been found." [context query elastic-results query-results] + (println "inside post-process-query-result-features") (reduce (partial post-process-query-result-feature context query elastic-results) query-results (:result-features query))) @@ -79,7 +81,8 @@ (defmethod execute-query :elasticsearch [context query] - (let [[context processed-query] (concept-type-specific-query-processing + (let [_ (println "inside execute-query :elasticsearch") + [context processed-query] (concept-type-specific-query-processing context query) processed-query (pre-process-query-result-features context processed-query) elastic-results (->> processed-query diff --git a/search-app/src/cmr/search/services/query_execution.clj b/search-app/src/cmr/search/services/query_execution.clj index 3956f4da0d..a6c25f73b9 100644 --- a/search-app/src/cmr/search/services/query_execution.clj +++ b/search-app/src/cmr/search/services/query_execution.clj @@ -152,6 +152,7 @@ (defmethod common-qe/concept-type-specific-query-processing :granule [context query] + (println "inside common-qe/concept-type-specific-query-processing :granule") (let [processed-query (r/resolve-collection-queries context query) collection-ids (ce/extract-collection-concept-ids processed-query)] [(assoc context diff --git a/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj b/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj index 131e124c9a..417d2132ec 100644 --- a/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj @@ -145,6 +145,7 @@ ;; This find granule counts per collection. (defmethod query-execution/post-process-query-result-feature :granule-counts [context query elastic-results query-results feature] + (println "inside query-execution/post-process-query-result-feature :granule-counts") (if (zero? (count (query-results->concept-ids query-results))) query-results (->> query-results diff --git a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj index bd8dacfea5..8a0211c676 100644 --- a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj @@ -5,8 +5,8 @@ (:require [cmr.common-app.services.search.query-execution :as query-execution] [cmr.common.jobs :refer [defjob]] [cmr.common.cache :as cache] - [cmr.search.data.elastic-search-index :as idx] - [cmr.common.cache.in-memory-cache :as mem-cache])) + [cmr.redis-utils.redis-cache :as redis-cache] + [cmr.search.data.elastic-search-index :as idx])) (def REFRESH_HAS_GRANULES_MAP_JOB_INTERVAL "The frequency in seconds of the refresh-has-granules-map-job" @@ -19,7 +19,7 @@ (defn create-has-granules-map-cache "Returns a 'cache' which will contain the cached has granules map." [] - (mem-cache/create-in-memory-cache)) + (redis-cache/create-redis-cache)) (defn- collection-granule-counts->has-granules-map "Converts a map of collection ids to granule counts to a map of collection ids to true or false @@ -31,32 +31,43 @@ (defn refresh-has-granules-map "Gets the latest provider holdings and updates the has-granules-map stored in the cache." [context] + (println "inside refresh-has-granules-map") (let [has-granules-map (collection-granule-counts->has-granules-map (idx/get-collection-granule-counts context nil))] + (println "has-granules-map found in elastic = " (pr-str has-granules-map)) (cache/set-value (cache/context->cache context has-granule-cache-key) - :has-granules has-granules-map))) + :has-granules + has-granules-map))) (defn get-has-granules-map "Gets the cached has granules map from the context which contains collection ids to true or false of whether the collections have granules or not. If the has-granules-map has not yet been cached it will retrieve it and cache it." [context] - (let [has-granules-map-cache (cache/context->cache context has-granule-cache-key)] - (cache/get-value has-granules-map-cache - :has-granules - (fn [] - (collection-granule-counts->has-granules-map - (idx/get-collection-granule-counts context nil)))))) + (println "get-has-granules-map func") + (refresh-has-granules-map context) ;; temp + (let [has-granules-map-cache (cache/context->cache context has-granule-cache-key) + _ (println "has-granules-map-cache = " (pr-str has-granules-map-cache)) + has-granules-map (cache/get-value has-granules-map-cache + :has-granules + (fn [] + (collection-granule-counts->has-granules-map + (idx/get-collection-granule-counts context nil)))) + _ (println "has-granules-map found = " (pr-str has-granules-map))] + has-granules-map)) ;; This returns a boolean flag with collection results if a collection has any granules in provider holdings (defmethod query-execution/post-process-query-result-feature :has-granules [context query elastic-results query-results feature] + (println "inside query-execution/post-process-query-result-feature :has-granules") (assoc query-results :has-granules-map (get-has-granules-map context))) (defjob RefreshHasGranulesMapJob [ctx system] (refresh-has-granules-map {:system system})) -(def refresh-has-granules-map-job +(defn refresh-has-granules-map-job + [job-key] {:job-type RefreshHasGranulesMapJob + :job-key job-key :interval REFRESH_HAS_GRANULES_MAP_JOB_INTERVAL}) diff --git a/search-app/src/cmr/search/system.clj b/search-app/src/cmr/search/system.clj index 1f6ff279e6..34f0fb4656 100644 --- a/search-app/src/cmr/search/system.clj +++ b/search-app/src/cmr/search/system.clj @@ -28,7 +28,6 @@ [cmr.search.routes :as routes] [cmr.search.services.humanizers.humanizer-report-service :as hrs] [cmr.search.services.humanizers.humanizer-range-facet-service :as hrfs] - [cmr.search.services.query-execution.has-granules-results-feature :as hgrf] [cmr.search.services.query-execution.has-granules-or-cwic-results-feature :as hgocrf] [cmr.transmit.config :as transmit-config] [cmr.transmit.launchpad-user-cache :as launchpad-user-cache] @@ -117,7 +116,6 @@ ;; Caches a map of tokens to the security identifiers context-augmenter/token-sid-cache-name (context-augmenter/create-token-sid-cache) context-augmenter/token-user-id-cache-name (context-augmenter/create-token-user-id-cache) - :has-granules-map (hgrf/create-has-granules-map-cache) :has-granules-or-cwic-map (hgocrf/create-has-granules-or-cwic-map-cache) :has-granules-or-opensearch-map (hgocrf/create-has-granules-or-opensearch-map-cache) metadata-transformer/xsl-transformer-cache-name (mem-cache/create-in-memory-cache) @@ -145,7 +143,6 @@ :scheduler (jobs/create-scheduler `system-holder [(af/refresh-acl-cache-job "search-acl-cache-refresh") - hgrf/refresh-has-granules-map-job hgocrf/refresh-has-granules-or-cwic-map-job hgocrf/refresh-has-granules-or-opensearch-map-job (metadata-cache/refresh-collections-metadata-cache-job) From 837cd9ab303df39b212457bc7d63d78e2da6344c Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Wed, 14 Feb 2024 11:00:20 -0500 Subject: [PATCH 2/8] move has-granules-map scheduled job to bootstrap --- bootstrap-app/src/cmr/bootstrap/system.clj | 3 ++- .../services/search/query_execution.clj | 5 +--- .../cmr/search/services/query_execution.clj | 1 - .../granule_counts_results_feature.clj | 1 - .../has_granules_results_feature.clj | 26 +++++++------------ 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/bootstrap-app/src/cmr/bootstrap/system.clj b/bootstrap-app/src/cmr/bootstrap/system.clj index 35c8bfea71..4a988e15b2 100644 --- a/bootstrap-app/src/cmr/bootstrap/system.clj +++ b/bootstrap-app/src/cmr/bootstrap/system.clj @@ -93,7 +93,8 @@ cmn-coll-metadata-cache/cache-key (cmn-coll-metadata-cache/create-cache) coll-gran-acls-caches/coll-by-concept-id-cache-key (coll-gran-acls-caches/create-coll-by-concept-id-cache-client) elastic-search-index-names-cache/index-names-cache-key (elastic-search-index-names-cache/create-index-cache) - humanizer-alias-cache/humanizer-alias-cache-key (humanizer-alias-cache/create-cache-client)} + humanizer-alias-cache/humanizer-alias-cache-key (humanizer-alias-cache/create-cache-client) + has-granules-results-feature/has-granule-cache-key (has-granules-results-feature/create-has-granules-map-cache)} :scheduler (jobs/create-scheduler `system-holder [jvm-info/log-jvm-statistics-job (kf/refresh-kms-cache-job "bootstrap-kms-cache-refresh") (provider-cache/refresh-provider-cache-job "bootstrap-provider-cache-refresh") diff --git a/common-app-lib/src/cmr/common_app/services/search/query_execution.clj b/common-app-lib/src/cmr/common_app/services/search/query_execution.clj index b1218fa0ff..8c88a69df0 100644 --- a/common-app-lib/src/cmr/common_app/services/search/query_execution.clj +++ b/common-app-lib/src/cmr/common_app/services/search/query_execution.clj @@ -46,7 +46,6 @@ (defmethod post-process-query-result-feature :default [context query elastic-results query-results feature] - (println "inside post-process-query-result-feature :default") ; Does nothing by default query-results) @@ -60,7 +59,6 @@ (defn post-process-query-result-features "Processes result features that execute after a query results have been found." [context query elastic-results query-results] - (println "inside post-process-query-result-features") (reduce (partial post-process-query-result-feature context query elastic-results) query-results (:result-features query))) @@ -81,8 +79,7 @@ (defmethod execute-query :elasticsearch [context query] - (let [_ (println "inside execute-query :elasticsearch") - [context processed-query] (concept-type-specific-query-processing + (let [[context processed-query] (concept-type-specific-query-processing context query) processed-query (pre-process-query-result-features context processed-query) elastic-results (->> processed-query diff --git a/search-app/src/cmr/search/services/query_execution.clj b/search-app/src/cmr/search/services/query_execution.clj index a6c25f73b9..3956f4da0d 100644 --- a/search-app/src/cmr/search/services/query_execution.clj +++ b/search-app/src/cmr/search/services/query_execution.clj @@ -152,7 +152,6 @@ (defmethod common-qe/concept-type-specific-query-processing :granule [context query] - (println "inside common-qe/concept-type-specific-query-processing :granule") (let [processed-query (r/resolve-collection-queries context query) collection-ids (ce/extract-collection-concept-ids processed-query)] [(assoc context diff --git a/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj b/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj index 417d2132ec..131e124c9a 100644 --- a/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/granule_counts_results_feature.clj @@ -145,7 +145,6 @@ ;; This find granule counts per collection. (defmethod query-execution/post-process-query-result-feature :granule-counts [context query elastic-results query-results feature] - (println "inside query-execution/post-process-query-result-feature :granule-counts") (if (zero? (count (query-results->concept-ids query-results))) query-results (->> query-results diff --git a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj index 8a0211c676..1fc3b06f82 100644 --- a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj @@ -31,35 +31,29 @@ (defn refresh-has-granules-map "Gets the latest provider holdings and updates the has-granules-map stored in the cache." [context] - (println "inside refresh-has-granules-map") (let [has-granules-map (collection-granule-counts->has-granules-map (idx/get-collection-granule-counts context nil))] - (println "has-granules-map found in elastic = " (pr-str has-granules-map)) (cache/set-value (cache/context->cache context has-granule-cache-key) - :has-granules + has-granule-cache-key has-granules-map))) +;; TODO need to continue to trace back to see if a full map is necessary. What is it really being used for anyway? Something related to ES queries. (defn get-has-granules-map "Gets the cached has granules map from the context which contains collection ids to true or false of whether the collections have granules or not. If the has-granules-map has not yet been cached - it will retrieve it and cache it." + it will retrieve it and cache it. + Example) {\"C0000000002-PROV1\" true, \"C0000000003-PROV1\" false}" [context] - (println "get-has-granules-map func") - (refresh-has-granules-map context) ;; temp - (let [has-granules-map-cache (cache/context->cache context has-granule-cache-key) - _ (println "has-granules-map-cache = " (pr-str has-granules-map-cache)) - has-granules-map (cache/get-value has-granules-map-cache - :has-granules - (fn [] - (collection-granule-counts->has-granules-map - (idx/get-collection-granule-counts context nil)))) - _ (println "has-granules-map found = " (pr-str has-granules-map))] - has-granules-map)) + (let [has-granules-map-cache (cache/context->cache context has-granule-cache-key)] + (cache/get-value has-granules-map-cache + has-granule-cache-key + (fn [] + (collection-granule-counts->has-granules-map + (idx/get-collection-granule-counts context nil)))))) ;; This returns a boolean flag with collection results if a collection has any granules in provider holdings (defmethod query-execution/post-process-query-result-feature :has-granules [context query elastic-results query-results feature] - (println "inside query-execution/post-process-query-result-feature :has-granules") (assoc query-results :has-granules-map (get-has-granules-map context))) (defjob RefreshHasGranulesMapJob From 496845ca8187c498777cdecd41515f4e2c56405f Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:47:52 -0500 Subject: [PATCH 3/8] add cache back into search system --- .../services/query_execution/has_granules_results_feature.clj | 1 - search-app/src/cmr/search/system.clj | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj index 1fc3b06f82..393a6ae008 100644 --- a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj @@ -37,7 +37,6 @@ has-granule-cache-key has-granules-map))) -;; TODO need to continue to trace back to see if a full map is necessary. What is it really being used for anyway? Something related to ES queries. (defn get-has-granules-map "Gets the cached has granules map from the context which contains collection ids to true or false of whether the collections have granules or not. If the has-granules-map has not yet been cached diff --git a/search-app/src/cmr/search/system.clj b/search-app/src/cmr/search/system.clj index 34f0fb4656..b3919f384d 100644 --- a/search-app/src/cmr/search/system.clj +++ b/search-app/src/cmr/search/system.clj @@ -28,6 +28,7 @@ [cmr.search.routes :as routes] [cmr.search.services.humanizers.humanizer-report-service :as hrs] [cmr.search.services.humanizers.humanizer-range-facet-service :as hrfs] + [cmr.search.services.query-execution.has-granules-results-feature :as hgrf] [cmr.search.services.query-execution.has-granules-or-cwic-results-feature :as hgocrf] [cmr.transmit.config :as transmit-config] [cmr.transmit.launchpad-user-cache :as launchpad-user-cache] @@ -116,6 +117,7 @@ ;; Caches a map of tokens to the security identifiers context-augmenter/token-sid-cache-name (context-augmenter/create-token-sid-cache) context-augmenter/token-user-id-cache-name (context-augmenter/create-token-user-id-cache) + hgrf/has-granule-cache-key (hgrf/create-has-granules-map-cache) :has-granules-or-cwic-map (hgocrf/create-has-granules-or-cwic-map-cache) :has-granules-or-opensearch-map (hgocrf/create-has-granules-or-opensearch-map-cache) metadata-transformer/xsl-transformer-cache-name (mem-cache/create-in-memory-cache) From 4b3ec0456dd2fed93c76a16abf26502dab70baf7 Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:02:15 -0500 Subject: [PATCH 4/8] update int test --- .../test/cmr/system_int_test/admin/cache_api_test.clj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj b/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj index 7c4ec56df3..766ac0dd5a 100644 --- a/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj +++ b/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj @@ -180,9 +180,8 @@ ["ABC-1" "read"] ["ABC-2" "read"]] "search 1" (url/search-read-caches-url) "acls" ["acls"] - ;; This is now a shared redis cache server, so many keys exist - ;;"search 2" (url/search-read-caches-url) "collections-for-gran-acls" [] - "search 3" (url/search-read-caches-url) "has-granules-map" [] + ;; collections-for-gran-acls-by-concept-id and has-granules-map are purposefully left out because they are + ;; in a shared redis cache server and will have multiple keys dependent on the data now "search 5" (url/search-read-caches-url) "token-imp" [["ABC-1" "read"] ["ABC-2" "read"]] "search 6" (url/search-read-caches-url) "token-sid" ["ABC-2" "ABC-1"] "search 7" (url/search-read-caches-url) "xsl-transformer-templates" [] From 1dec454787c1d76c21edc28ae1a08ad191c40454 Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:41:47 -0500 Subject: [PATCH 5/8] fix compile error --- bootstrap-app/src/cmr/bootstrap/system.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-app/src/cmr/bootstrap/system.clj b/bootstrap-app/src/cmr/bootstrap/system.clj index 689900b605..09cb00ef80 100644 --- a/bootstrap-app/src/cmr/bootstrap/system.clj +++ b/bootstrap-app/src/cmr/bootstrap/system.clj @@ -97,7 +97,7 @@ coll-gran-acls-caches/coll-by-concept-id-cache-key (coll-gran-acls-caches/create-coll-by-concept-id-cache-client) elastic-search-index-names-cache/index-names-cache-key (elastic-search-index-names-cache/create-index-cache) humanizer-alias-cache/humanizer-alias-cache-key (humanizer-alias-cache/create-cache-client) - has-granules-results-feature/has-granule-cache-key (has-granules-results-feature/create-has-granules-map-cache)} + has-granules-results-feature/has-granule-cache-key (has-granules-results-feature/create-has-granules-map-cache) humanizer-report-service/humanizer-report-cache-key (humanizer-report-service/create-humanizer-report-cache-client)} :scheduler (jobs/create-scheduler `system-holder [jvm-info/log-jvm-statistics-job (kf/refresh-kms-cache-job "bootstrap-kms-cache-refresh") @@ -107,7 +107,7 @@ (coll-gran-acls-caches/refresh-collections-cache-for-granule-acls-job "bootstrap-collections-for-gran-acls-cache-refresh") (elastic-search-index-names-cache/refresh-index-names-cache-job "bootstrap-elastic-search-index-names-cache") (humanizer-alias-cache/refresh-humanizer-alias-cache-job "boostrap-humanizer-alias-cache-refresh") - (has-granules-results-feature/refresh-has-granules-map-job "bootstrap-has-granules-map-refresh")]) + (has-granules-results-feature/refresh-has-granules-map-job "bootstrap-has-granules-map-refresh") (humanizer-report-service/refresh-humanizer-report-cache-job "bootstrap-humanizer-report-cache-refresh")]) :queue-broker queue-broker}] (transmit-config/system-with-connections sys [:metadata-db :echo-rest :kms :indexer :access-control :search]))) From 5c96683610d75b39162fa77bb8e6ce03ed2b2261 Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Fri, 16 Feb 2024 10:42:09 -0500 Subject: [PATCH 6/8] fix No method in multimethod 'concept-type+result-format->fields' error --- .../cmr/common_app/services/search/elastic_search_index.clj | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common-app-lib/src/cmr/common_app/services/search/elastic_search_index.clj b/common-app-lib/src/cmr/common_app/services/search/elastic_search_index.clj index f96df7a48a..3ea5b2775e 100644 --- a/common-app-lib/src/cmr/common_app/services/search/elastic_search_index.clj +++ b/common-app-lib/src/cmr/common_app/services/search/elastic_search_index.clj @@ -46,6 +46,12 @@ (fn [concept-type query] [concept-type (qm/base-result-format query)])) +(defmethod concept-type+result-format->fields [:granule :xml] + [concept-type query] + ["granule-ur" + "provider-id" + "concept-id"]) + (defn context->search-index "Returns the search index given a context. This assumes that the search index is always located in a system using the :search-index key." From b89b104c7d7cb2151466b9b8ab464fa29ed5ba37 Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:27:41 -0500 Subject: [PATCH 7/8] add logs --- .../query_execution/has_granules_results_feature.clj | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj index 393a6ae008..77d8b5f9d8 100644 --- a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj @@ -5,6 +5,7 @@ (:require [cmr.common-app.services.search.query-execution :as query-execution] [cmr.common.jobs :refer [defjob]] [cmr.common.cache :as cache] + [cmr.common.log :as log :refer [info]] [cmr.redis-utils.redis-cache :as redis-cache] [cmr.search.data.elastic-search-index :as idx])) @@ -31,11 +32,13 @@ (defn refresh-has-granules-map "Gets the latest provider holdings and updates the has-granules-map stored in the cache." [context] + (info "Refreshing has-granules-map cache.") (let [has-granules-map (collection-granule-counts->has-granules-map (idx/get-collection-granule-counts context nil))] (cache/set-value (cache/context->cache context has-granule-cache-key) has-granule-cache-key - has-granules-map))) + has-granules-map) + (info "Finished has-granules-map cache."))) (defn get-has-granules-map "Gets the cached has granules map from the context which contains collection ids to true or false From a3c1a4625b0ce6d1ed918cb8f7d5c30d51d89994 Mon Sep 17 00:00:00 2001 From: jmaeng72 <44305062+jmaeng72@users.noreply.github.com> Date: Fri, 16 Feb 2024 17:31:50 -0500 Subject: [PATCH 8/8] remove log --- .../query_execution/has_granules_results_feature.clj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj index 77d8b5f9d8..393a6ae008 100644 --- a/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj +++ b/search-app/src/cmr/search/services/query_execution/has_granules_results_feature.clj @@ -5,7 +5,6 @@ (:require [cmr.common-app.services.search.query-execution :as query-execution] [cmr.common.jobs :refer [defjob]] [cmr.common.cache :as cache] - [cmr.common.log :as log :refer [info]] [cmr.redis-utils.redis-cache :as redis-cache] [cmr.search.data.elastic-search-index :as idx])) @@ -32,13 +31,11 @@ (defn refresh-has-granules-map "Gets the latest provider holdings and updates the has-granules-map stored in the cache." [context] - (info "Refreshing has-granules-map cache.") (let [has-granules-map (collection-granule-counts->has-granules-map (idx/get-collection-granule-counts context nil))] (cache/set-value (cache/context->cache context has-granule-cache-key) has-granule-cache-key - has-granules-map) - (info "Finished has-granules-map cache."))) + has-granules-map))) (defn get-has-granules-map "Gets the cached has granules map from the context which contains collection ids to true or false