From ffe42df4828cd962cfb2e3edc581e7b425cb268d Mon Sep 17 00:00:00 2001 From: Chris Durbin Date: Wed, 7 Mar 2018 15:05:49 -0500 Subject: [PATCH] CMR-4781: Temporarily block a very specific query (#471) * CMR-4781: Temporarily block a very specific query * CMR-4781: Control whether we block queries via a configuration parameter. Defaults to true --- common-lib/src/cmr/common/api/errors.clj | 1 + .../src/cmr/search/api/concepts_search.clj | 26 ++++++++++++++++++- .../search/granule_search_test.clj | 22 ++++++++++------ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/common-lib/src/cmr/common/api/errors.clj b/common-lib/src/cmr/common/api/errors.clj index 0a63334e76..114d06eafc 100644 --- a/common-lib/src/cmr/common/api/errors.clj +++ b/common-lib/src/cmr/common/api/errors.clj @@ -15,6 +15,7 @@ :conflict 409 :invalid-content-type 415 :invalid-data 422 + :too-many-requests 429 :service-unavailable 503}) (def CONTENT_TYPE_HEADER "Content-Type") diff --git a/search-app/src/cmr/search/api/concepts_search.clj b/search-app/src/cmr/search/api/concepts_search.clj index 926f27a79d..6ef79312fa 100644 --- a/search-app/src/cmr/search/api/concepts_search.clj +++ b/search-app/src/cmr/search/api/concepts_search.clj @@ -5,6 +5,7 @@ [cmr.common-app.api.routes :as common-routes] [cmr.common-app.services.search :as search] [cmr.common.cache :as cache] + [cmr.common.config :refer [defconfig]] [cmr.common.log :refer (debug info warn error)] [cmr.common.mime-types :as mt] [cmr.common.services.errors :as svc-errors] @@ -47,6 +48,28 @@ results (query-svc/find-concepts-by-json-query ctx concept-type params json-query)] (core-api/search-response ctx results))) +(defconfig block-queries + "Indicates whether we are going to block a specific excessive query." + {:type Boolean + :default true}) + +(defn- block-excessive-queries + "Temporary solution to prevent a specific query from overloading the CMR search resources." + [ctx concept-type result-format params] + (when (and (block-queries) + (= concept-type :granule) + (= :json result-format) + (= "MCD43A4" (:short_name params)) + (contains? params "")) + (warn (format "Blocking %s query from client %s in format %s with params %s." + (name concept-type) + (:client-id ctx) + (rfh/printable-result-format result-format) + (pr-str params))) + (svc-errors/throw-service-error + :too-many-requests + "Excessive query rate. Please contact support@earthdata.nasa.gov."))) + (defn- find-concepts-by-parameters "Invokes query service to parse the parameters query, find results, and return the response" @@ -59,6 +82,7 @@ ctx (assoc ctx :query-string body :scroll-id scroll-id) params (core-api/process-params concept-type params path-w-extension headers mt/xml) result-format (:result-format params) + _ (block-excessive-queries ctx concept-type result-format params) _ (info (format "Searching for %ss from client %s in format %s with params %s." (name concept-type) (:client-id ctx) (rfh/printable-result-format result-format) (pr-str params))) @@ -66,7 +90,7 @@ cached-search-params (lp/process-legacy-psa params)) results (query-svc/find-concepts-by-parameters ctx concept-type search-params)] - (if (:scroll-id results) + (if (:scroll-id results) (core-api/search-response ctx results search-params) (core-api/search-response ctx results)))) diff --git a/system-int-test/test/cmr/system_int_test/search/granule_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule_search_test.clj index debdc0e3a5..34c052634d 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule_search_test.clj @@ -369,11 +369,11 @@ ;; exclude granules by echo_granule_id or concept_id (including parent concept_id) params (deftest exclude-granules-by-echo-granule-n-concept-ids (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E1" - :ShortName "S1" - :Version "V1"})) + :ShortName "S1" + :Version "V1"})) coll2 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E2" - :ShortName "S2" - :Version "V2"})) + :ShortName "S2" + :Version "V2"})) coll1-cid (get-in coll1 [:concept-id]) coll2-cid (get-in coll2 [:concept-id]) gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 coll1-cid {:cloud-cover 0.8})) @@ -414,11 +414,11 @@ ;; Find granules by echo_granule_id, echo_collection_id and concept_id params (deftest search-by-concept-id (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E1" - :ShortName "S1" - :Version "V1"})) + :ShortName "S1" + :Version "V1"})) coll2 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E2" - :ShortName "S2" - :Version "V2"})) + :ShortName "S2" + :Version "V2"})) coll1-cid (get-in coll1 [:concept-id]) coll2-cid (get-in coll2 [:concept-id]) gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 coll1-cid)) @@ -534,3 +534,9 @@ (is (= {:status 400 :errors [(smsg/mixed-arity-parameter-msg :concept-id)]} (search/make-raw-search-query :granule ".json?concept_id=G&concept_id[pattern]=true")))))) + +(deftest block-excessive-queries-test + (testing "Blocking those MCD43A4 queries" + (is (= {:status 429 + :errors ["Excessive query rate. Please contact support@earthdata.nasa.gov."]} + (search/make-raw-search-query :granule ".json?short_name=MCD43A4&&page_size=5")))))