From f189d55ded0c235f19fbfc3597e772e22e4498ad Mon Sep 17 00:00:00 2001 From: sxu123 Date: Thu, 11 Aug 2022 14:25:33 -0400 Subject: [PATCH] Cmr 8473, 8476 migrate service and tool associations (#1639) * CMR-8473-CMR-8476: Migrated service and tool associations to CMR_ASSOCIATIONS table. * CMR-8473-CMR-8476: Fixed some tests. Co-authored-by: siwei xu --- .../data/oracle/concept_tables.clj | 4 +- .../oracle/concepts/service_association.clj | 6 +- .../data/oracle/concepts/tool_association.clj | 6 +- .../cmr/metadata_db/data/oracle/providers.clj | 8 +- .../cmr/metadata_db/data/oracle/search.clj | 8 +- .../083_migrate_service_associations.clj | 93 +++++++++++++++++++ .../084_migrate_tool_associations.clj | 92 ++++++++++++++++++ 7 files changed, 202 insertions(+), 15 deletions(-) create mode 100644 metadata-db-app/src/cmr/metadata_db/migrations/083_migrate_service_associations.clj create mode 100644 metadata-db-app/src/cmr/metadata_db/migrations/084_migrate_tool_associations.clj diff --git a/metadata-db-app/src/cmr/metadata_db/data/oracle/concept_tables.clj b/metadata-db-app/src/cmr/metadata_db/data/oracle/concept_tables.clj index 68f3be6bd7..e9098ecb07 100644 --- a/metadata-db-app/src/cmr/metadata_db/data/oracle/concept_tables.clj +++ b/metadata-db-app/src/cmr/metadata_db/data/oracle/concept_tables.clj @@ -62,11 +62,11 @@ (defmethod get-table-name :service-association [_ _] - "cmr_service_associations") + "cmr_associations") (defmethod get-table-name :tool-association [_ _] - "cmr_tool_associations") + "cmr_associations") (defmethod get-table-name :default [provider concept-type] diff --git a/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/service_association.clj b/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/service_association.clj index d99f397f51..ffdfeb7d93 100644 --- a/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/service_association.clj +++ b/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/service_association.clj @@ -11,7 +11,7 @@ (assoc-in [:extra-fields :associated-revision-id] (when-let [ari (:associated_revision_id result)] (long ari))) - (assoc-in [:extra-fields :service-concept-id] (:service_concept_id result)) + (assoc-in [:extra-fields :service-concept-id] (:source_concept_identifier result)) (assoc :user-id (:user_id result)))) (defn- var-assoc-concept->insert-args @@ -19,8 +19,8 @@ (let [{{:keys [associated-concept-id associated-revision-id service-concept-id]} :extra-fields :keys [user-id]} concept [cols values] (c/concept->common-insert-args concept)] - [(concat cols ["associated_concept_id" "associated_revision_id" "service_concept_id" "user_id"]) - (concat values [associated-concept-id associated-revision-id service-concept-id user-id])])) + [(concat cols ["associated_concept_id" "associated_revision_id" "source_concept_identifier" "user_id" "association_type"]) + (concat values [associated-concept-id associated-revision-id service-concept-id user-id "SERVICE-COLLECTION"])])) (defmethod c/concept->insert-args [:service-association false] [concept _] diff --git a/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/tool_association.clj b/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/tool_association.clj index 931c0c860b..f75804c3ba 100644 --- a/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/tool_association.clj +++ b/metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/tool_association.clj @@ -11,7 +11,7 @@ (assoc-in [:extra-fields :associated-revision-id] (when-let [ari (:associated_revision_id result)] (long ari))) - (assoc-in [:extra-fields :tool-concept-id] (:tool_concept_id result)) + (assoc-in [:extra-fields :tool-concept-id] (:source_concept_identifier result)) (assoc :user-id (:user_id result)))) (defn- var-assoc-concept->insert-args @@ -19,8 +19,8 @@ (let [{{:keys [associated-concept-id associated-revision-id tool-concept-id]} :extra-fields :keys [user-id]} concept [cols values] (c/concept->common-insert-args concept)] - [(concat cols ["associated_concept_id" "associated_revision_id" "tool_concept_id" "user_id"]) - (concat values [associated-concept-id associated-revision-id tool-concept-id user-id])])) + [(concat cols ["associated_concept_id" "associated_revision_id" "source_concept_identifier" "user_id" "association_type"]) + (concat values [associated-concept-id associated-revision-id tool-concept-id user-id "TOOL-COLLECTION"])])) (defmethod c/concept->insert-args [:tool-association false] [concept _] diff --git a/metadata-db-app/src/cmr/metadata_db/data/oracle/providers.clj b/metadata-db-app/src/cmr/metadata_db/data/oracle/providers.clj index 8237507db4..862e419af2 100644 --- a/metadata-db-app/src/cmr/metadata_db/data/oracle/providers.clj +++ b/metadata-db-app/src/cmr/metadata_db/data/oracle/providers.clj @@ -58,16 +58,16 @@ (j/delete! db (ct/get-table-name provider :variable) ["provider_id = ?" provider-id]) ;; Delete the service associations related to the provider via service - (j/db-do-commands db (str "DELETE FROM cmr_service_associations where service_concept_id like 'S%-" provider-id "'")) + (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'SERVICE-COLLECTION' and source_concept_identifier like 'S%-" provider-id "'")) ;; Delete the service associations related to the provider via collection - (j/db-do-commands db (str "DELETE FROM cmr_service_associations where associated_concept_id like 'C%-" provider-id "'")) + (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'SERVICE-COLLECTION' and associated_concept_id like 'C%-" provider-id "'")) ;; Delete services of the provider (j/delete! db (ct/get-table-name provider :service) ["provider_id = ?" provider-id]) ;; Delete the tool associations related to the provider via tool - (j/db-do-commands db (str "DELETE FROM cmr_tool_associations where tool_concept_id like 'TL%-" provider-id "'")) + (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'TOOL-COLLECTION' and source_concept_identifier like 'TL%-" provider-id "'")) ;; Delete the tool associations related to the provider via collection - (j/db-do-commands db (str "DELETE FROM cmr_tool_associations where associated_concept_id like 'C%-" provider-id "'")) + (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'TOOL-COLLECTION' and associated_concept_id like 'C%-" provider-id "'")) ;; Delete tools of the provider (j/delete! db (ct/get-table-name provider :tool) ["provider_id = ?" provider-id]) diff --git a/metadata-db-app/src/cmr/metadata_db/data/oracle/search.clj b/metadata-db-app/src/cmr/metadata_db/data/oracle/search.clj index bb780caf7b..1ff85a2045 100644 --- a/metadata-db-app/src/cmr/metadata_db/data/oracle/search.clj +++ b/metadata-db-app/src/cmr/metadata_db/data/oracle/search.clj @@ -52,10 +52,10 @@ :source_concept_identifier :user_id]) :service-association (into common-columns [:associated_concept_id :associated_revision_id - :service_concept_id :user_id]) + :source_concept_identifier :user_id]) :tool-association (into common-columns [:associated_concept_id :associated_revision_id - :tool_concept_id :user_id])}) + :source_concept_identifier :user_id])}) (def single-table-with-providers-concept-type? "The set of concept types that are stored in a single table with a provider column. These concept @@ -194,7 +194,9 @@ fields (disj (columns-for-find-concept concept-type params) :provider_id) params (params->sql-params concept-type providers (assoc params :provider-id provider-ids)) params (if (or (= :variable-association concept-type) - (= :tag-association concept-type)) + (= :tag-association concept-type) + (= :service-association concept-type) + (= :tool-association concept-type)) (-> params (set/rename-keys (get-in association-concept-type->generic-association [concept-type :kebab-key-mapping])) diff --git a/metadata-db-app/src/cmr/metadata_db/migrations/083_migrate_service_associations.clj b/metadata-db-app/src/cmr/metadata_db/migrations/083_migrate_service_associations.clj new file mode 100644 index 0000000000..ac592af57c --- /dev/null +++ b/metadata-db-app/src/cmr/metadata_db/migrations/083_migrate_service_associations.clj @@ -0,0 +1,93 @@ +(ns cmr.metadata-db.migrations.083-migrate-service-associations + (:require + [config.mdb-migrate-helper :as h])) + +(def ^:private service-assocs-column-sql + "id NUMBER, + concept_id VARCHAR(255) NOT NULL, + native_id VARCHAR(1500) NOT NULL, + service_concept_id VARCHAR(255) NOT NULL, + metadata BLOB NOT NULL, + format VARCHAR(255) NOT NULL, + revision_id INTEGER DEFAULT 1 NOT NULL, + associated_concept_id VARCHAR(255) NOT NULL, + associated_revision_id INTEGER NULL, + revision_date TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL, + deleted INTEGER DEFAULT 0 NOT NULL, + user_id VARCHAR(30), + transaction_id NUMBER DEFAULT 0 NOT NULL") + +(def ^:private service-assocs-constraint-sql + (str "CONSTRAINT s_assoc_pk PRIMARY KEY (id), " + ;; Unique constraint on native id and revision id + "CONSTRAINT s_assoc_con_rev UNIQUE (native_id, revision_id) + USING INDEX (create unique index s_assoc_ucr_i ON CMR_SERVICE_ASSOCIATIONS (native_id, revision_id)), " + + ;; Unique constraint on concept id and revision id + "CONSTRAINT s_assoc_cid_rev UNIQUE (concept_id, revision_id) + USING INDEX (create unique index s_assoc_cri ON CMR_SERVICE_ASSOCIATIONS (concept_id, revision_id))")) + +(defn- create-service-associations-table + [] + (h/sql (format "CREATE TABLE METADATA_DB.CMR_SERVICE_ASSOCIATIONS (%s, %s)" + service-assocs-column-sql service-assocs-constraint-sql))) + +(defn- create-service-associations-indices + [] + (h/sql "CREATE INDEX s_assoc_scid ON cmr_service_associations (service_concept_id, revision_id)") + (h/sql "CREATE INDEX s_assoc_scarid ON cmr_service_associations (service_concept_id, associated_concept_id, associated_revision_id)") + (h/sql "CREATE INDEX s_assoc_crdi ON CMR_SERVICE_ASSOCIATIONS (concept_id, revision_id, deleted)") + (h/sql "CREATE INDEX s_assoc_acari ON CMR_SERVICE_ASSOCIATIONS (associated_concept_id, associated_revision_id)")) + +(defn- create-service-associations-sequence + [] + (h/sql "CREATE SEQUENCE CMR_SERVICE_ASSOCIATIONS_SEQ")) + + +(defn up + "Migrates the database up to version 83." + [] + (println "cmr.metadata-db.migrations.083-migrate-service-associations up...") + + ;; rename CMR_SERVICE_ASSOCIATIONS table to prevent new entries from being inserted. + (h/sql "ALTER TABLE CMR_SERVICE_ASSOCIATIONS RENAME TO CMR_SERVICE_ASSOCIATIONS_OLD") + + ;; Migrate service association data to CMR_ASSOCIATIONS table. + ;; Note: We will need to use the CMR_ASSOCIATIONS_SEQ.nextval for the ID column to make it unique in the new table. + (h/sql "INSERT INTO CMR_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER,ASSOCIATION_TYPE) + SELECT CMR_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SERVICE_CONCEPT_ID,'SERVICE-COLLECTION' + FROM CMR_SERVICE_ASSOCIATIONS_OLD") + + ;; Drop the old table. + (h/sql "DROP TABLE CMR_SERVICE_ASSOCIATIONS_OLD") + + ;; Drop the sequence + (h/sql "DROP SEQUENCE METADATA_DB.cmr_service_associations_seq")) + +(defn down + "Migrates the database down from version 83." + [] + (println "cmr.metadata-db.migrations.083-migrate-service-associations down...") + + ;; rename CMR_ASSOCIATIONS table to prevent new entries from being inserted. + (h/sql "ALTER TABLE CMR_ASSOCIATIONS RENAME TO CMR_ASSOCIATIONS_OLD") + + + ;; Create CMR_SERVICE_ASSOCIATIONS table. + (create-service-associations-table) + (create-service-associations-indices) + (create-service-associations-sequence) + + ;; Migrate service associations back to CMR_SERVICE_ASSOCIATIONS table. + ;; Note: We will now need to use the CMR_SERVICE_ASSOCIATIONS_SEQ.nextval for the ID column so that + ;; when new entries are inserted, all the ids will be unique. + (h/sql "INSERT INTO CMR_SERVICE_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SERVICE_CONCEPT_ID) + SELECT CMR_SERVICE_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER + FROM CMR_ASSOCIATIONS_OLD + WHERE ASSOCIATION_TYPE='SERVICE-COLLECTION'") + + ;; Remove all the service associations migrated. + (h/sql "DELETE CMR_ASSOCIATIONS_OLD WHERE ASSOCIATION_TYPE='SERVICE-COLLECTION'") + + ;; rename CMR_ASSOCIATIONS_OLD table back to CMR_ASSOCIATIONS table.. + (h/sql "ALTER TABLE CMR_ASSOCIATIONS_OLD RENAME TO CMR_ASSOCIATIONS")) diff --git a/metadata-db-app/src/cmr/metadata_db/migrations/084_migrate_tool_associations.clj b/metadata-db-app/src/cmr/metadata_db/migrations/084_migrate_tool_associations.clj new file mode 100644 index 0000000000..f79a6f40b5 --- /dev/null +++ b/metadata-db-app/src/cmr/metadata_db/migrations/084_migrate_tool_associations.clj @@ -0,0 +1,92 @@ +(ns cmr.metadata-db.migrations.084-migrate-tool-associations + (:require + [config.mdb-migrate-helper :as h])) + +(def ^:private tool-assocs-column-sql + "id NUMBER, + concept_id VARCHAR(255) NOT NULL, + native_id VARCHAR(1500) NOT NULL, + tool_concept_id VARCHAR(255) NOT NULL, + metadata BLOB NOT NULL, + format VARCHAR(255) NOT NULL, + revision_id INTEGER DEFAULT 1 NOT NULL, + associated_concept_id VARCHAR(255) NOT NULL, + associated_revision_id INTEGER NULL, + revision_date TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL, + deleted INTEGER DEFAULT 0 NOT NULL, + user_id VARCHAR(30), + transaction_id NUMBER DEFAULT 0 NOT NULL") + +(def ^:private tool-assocs-constraint-sql + (str "CONSTRAINT t_assoc_pk PRIMARY KEY (id), " + ;; Unique constraint on native id and revision id + "CONSTRAINT t_assoc_con_rev UNIQUE (native_id, revision_id) + USING INDEX (create unique index t_assoc_ucr_i ON CMR_TOOL_ASSOCIATIONS (native_id, revision_id)), " + + ;; Unique constraint on concept id and revision id + "CONSTRAINT t_assoc_cid_rev UNIQUE (concept_id, revision_id) + USING INDEX (create unique index t_assoc_cri ON CMR_TOOL_ASSOCIATIONS (concept_id, revision_id))")) + +(defn- create-tool-associations-table + [] + (h/sql (format "CREATE TABLE METADATA_DB.CMR_TOOL_ASSOCIATIONS (%s, %s)" + tool-assocs-column-sql tool-assocs-constraint-sql))) + +(defn- create-tool-associations-indices + [] + (h/sql "CREATE INDEX t_assoc_crdi ON CMR_TOOL_ASSOCIATIONS (concept_id, revision_id, deleted)") + (h/sql "CREATE INDEX t_assoc_acari ON CMR_TOOL_ASSOCIATIONS (associated_concept_id, associated_revision_id)") + (h/sql "CREATE INDEX t_assoc_tcid ON CMR_TOOL_ASSOCIATIONS (tool_concept_id, revision_id)") + (h/sql "CREATE INDEX t_assoc_tcarid ON CMR_TOOL_ASSOCIATIONS (tool_concept_id, associated_concept_id, associated_revision_id)")) + +(defn- create-tool-associations-sequence + [] + (h/sql "CREATE SEQUENCE CMR_TOOL_ASSOCIATIONS_SEQ")) + +(defn up + "Migrates the database up to version 84." + [] + (println "cmr.metadata-db.migrations.084-migrate-tool-associations up...") + + ;; rename CMR_TOOL_ASSOCIATIONS table to prevent new entries from being inserted. + (h/sql "ALTER TABLE CMR_TOOL_ASSOCIATIONS RENAME TO CMR_TOOL_ASSOCIATIONS_OLD") + + ;; Migrate tool association data to CMR_ASSOCIATIONS table. + ;; Note: We will need to use the CMR_ASSOCIATIONS_SEQ.nextval for the ID column to make it unique in the new table. + (h/sql "INSERT INTO CMR_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER,ASSOCIATION_TYPE) + SELECT CMR_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,TOOL_CONCEPT_ID,'TOOL-COLLECTION' + FROM CMR_TOOL_ASSOCIATIONS_OLD") + + ;; Drop the old table. + (h/sql "DROP TABLE CMR_TOOL_ASSOCIATIONS_OLD") + + ;; Drop the sequence + (h/sql "DROP SEQUENCE METADATA_DB.cmr_tool_associations_seq")) + +(defn down + "Migrates the database down from version 84." + [] + (println "cmr.metadata-db.migrations.084-migrate-tool-associations down...") + + ;; rename CMR_ASSOCIATIONS table to prevent new entries from being inserted. + (h/sql "ALTER TABLE CMR_ASSOCIATIONS RENAME TO CMR_ASSOCIATIONS_OLD") + + + ;; Create CMR_TOOL_ASSOCIATIONS table. + (create-tool-associations-table) + (create-tool-associations-indices) + (create-tool-associations-sequence) + + ;; Migrate tool associations back to CMR_TOOL_ASSOCIATIONS table. + ;; Note: We will now need to use the CMR_TOOL_ASSOCIATIONS_SEQ.nextval for the ID column so that + ;; when new entries are inserted, all the ids will be unique. + (h/sql "INSERT INTO CMR_TOOL_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,TOOL_CONCEPT_ID) + SELECT CMR_TOOL_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER + FROM CMR_ASSOCIATIONS_OLD + WHERE ASSOCIATION_TYPE='TOOL-COLLECTION'") + + ;; Remove all the tool associations migrated. + (h/sql "DELETE CMR_ASSOCIATIONS_OLD WHERE ASSOCIATION_TYPE='TOOL-COLLECTION'") + + ;; rename CMR_ASSOCIATIONS_OLD table back to CMR_ASSOCIATIONS table.. + (h/sql "ALTER TABLE CMR_ASSOCIATIONS_OLD RENAME TO CMR_ASSOCIATIONS"))