Cmr 8473, 8476 migrate service and tool associations (#1639)

* CMR-8473-CMR-8476: Migrated service and tool associations to CMR_ASSOCIATIONS table.

* CMR-8473-CMR-8476: Fixed some tests.

Co-authored-by: siwei xu <[email protected]>

metadata-db-app/src/cmr/metadata_db/data/oracle/concept_tables.clj

@@ -62,11 +62,11 @@
 
 (defmethod get-table-name :service-association
   [_ _]
-  "cmr_service_associations")
+  "cmr_associations")
 
 (defmethod get-table-name :tool-association
   [_ _]
-  "cmr_tool_associations")
+  "cmr_associations")
 
 (defmethod get-table-name :default
   [provider concept-type]

metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/service_association.clj

@@ -11,16 +11,16 @@
           (assoc-in [:extra-fields :associated-revision-id]
                     (when-let [ari (:associated_revision_id result)]
                       (long ari)))
-          (assoc-in [:extra-fields :service-concept-id] (:service_concept_id result))
+          (assoc-in [:extra-fields :service-concept-id] (:source_concept_identifier result))
           (assoc :user-id (:user_id result))))
 
 (defn- var-assoc-concept->insert-args
   [concept]
   (let [{{:keys [associated-concept-id associated-revision-id service-concept-id]} :extra-fields
          :keys [user-id]} concept
         [cols values] (c/concept->common-insert-args concept)]
-    [(concat cols ["associated_concept_id" "associated_revision_id" "service_concept_id" "user_id"])
-     (concat values [associated-concept-id associated-revision-id service-concept-id user-id])]))
+    [(concat cols ["associated_concept_id" "associated_revision_id" "source_concept_identifier" "user_id" "association_type"])
+     (concat values [associated-concept-id associated-revision-id service-concept-id user-id "SERVICE-COLLECTION"])]))
 
 (defmethod c/concept->insert-args [:service-association false]
   [concept _]

metadata-db-app/src/cmr/metadata_db/data/oracle/concepts/tool_association.clj

@@ -11,16 +11,16 @@
           (assoc-in [:extra-fields :associated-revision-id]
                     (when-let [ari (:associated_revision_id result)]
                       (long ari)))
-          (assoc-in [:extra-fields :tool-concept-id] (:tool_concept_id result))
+          (assoc-in [:extra-fields :tool-concept-id] (:source_concept_identifier result))
           (assoc :user-id (:user_id result))))
 
 (defn- var-assoc-concept->insert-args
   [concept]
   (let [{{:keys [associated-concept-id associated-revision-id tool-concept-id]} :extra-fields
          :keys [user-id]} concept
         [cols values] (c/concept->common-insert-args concept)]
-    [(concat cols ["associated_concept_id" "associated_revision_id" "tool_concept_id" "user_id"])
-     (concat values [associated-concept-id associated-revision-id tool-concept-id user-id])]))
+    [(concat cols ["associated_concept_id" "associated_revision_id" "source_concept_identifier" "user_id" "association_type"])
+     (concat values [associated-concept-id associated-revision-id tool-concept-id user-id "TOOL-COLLECTION"])]))
 
 (defmethod c/concept->insert-args [:tool-association false]
   [concept _]

metadata-db-app/src/cmr/metadata_db/data/oracle/providers.clj

@@ -58,16 +58,16 @@
     (j/delete! db (ct/get-table-name provider :variable) ["provider_id = ?" provider-id])
 
     ;; Delete the service associations related to the provider via service
-    (j/db-do-commands db (str "DELETE FROM cmr_service_associations where service_concept_id like 'S%-" provider-id "'"))
+    (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'SERVICE-COLLECTION' and source_concept_identifier like 'S%-" provider-id "'"))
     ;; Delete the service associations related to the provider via collection
-    (j/db-do-commands db (str "DELETE FROM cmr_service_associations where associated_concept_id like 'C%-" provider-id "'"))
+    (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'SERVICE-COLLECTION' and associated_concept_id like 'C%-" provider-id "'"))
     ;; Delete services of the provider
     (j/delete! db (ct/get-table-name provider :service) ["provider_id = ?" provider-id])
 
     ;; Delete the tool associations related to the provider via tool
-    (j/db-do-commands db (str "DELETE FROM cmr_tool_associations where tool_concept_id like 'TL%-" provider-id "'"))
+    (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'TOOL-COLLECTION' and source_concept_identifier like 'TL%-" provider-id "'"))
     ;; Delete the tool associations related to the provider via collection
-    (j/db-do-commands db (str "DELETE FROM cmr_tool_associations where associated_concept_id like 'C%-" provider-id "'"))
+    (j/db-do-commands db (str "DELETE FROM CMR_ASSOCIATIONS where association_type = 'TOOL-COLLECTION' and associated_concept_id like 'C%-" provider-id "'"))
     ;; Delete tools of the provider
     (j/delete! db (ct/get-table-name provider :tool) ["provider_id = ?" provider-id])
 

metadata-db-app/src/cmr/metadata_db/data/oracle/search.clj

@@ -52,10 +52,10 @@
                                 :source_concept_identifier :user_id])
    :service-association (into common-columns
                               [:associated_concept_id :associated_revision_id
-                               :service_concept_id :user_id])
+                               :source_concept_identifier :user_id])
    :tool-association (into common-columns
                            [:associated_concept_id :associated_revision_id
-                            :tool_concept_id :user_id])})
+                            :source_concept_identifier :user_id])})
 
 (def single-table-with-providers-concept-type?
   "The set of concept types that are stored in a single table with a provider column. These concept
@@ -194,7 +194,9 @@
         fields (disj (columns-for-find-concept concept-type params) :provider_id)
         params (params->sql-params concept-type providers (assoc params :provider-id provider-ids))
         params (if (or (= :variable-association concept-type)
-                       (= :tag-association concept-type))
+                       (= :tag-association concept-type)
+                       (= :service-association concept-type)
+                       (= :tool-association concept-type))
                  (-> params
                      (set/rename-keys
                       (get-in association-concept-type->generic-association [concept-type :kebab-key-mapping]))

metadata-db-app/src/cmr/metadata_db/migrations/083_migrate_service_associations.clj

@@ -0,0 +1,93 @@
+(ns cmr.metadata-db.migrations.083-migrate-service-associations
+  (:require
+   [config.mdb-migrate-helper :as h]))
+
+(def ^:private service-assocs-column-sql
+  "id NUMBER,
+  concept_id VARCHAR(255) NOT NULL,
+  native_id VARCHAR(1500) NOT NULL,
+  service_concept_id VARCHAR(255) NOT NULL,
+  metadata BLOB NOT NULL,
+  format VARCHAR(255) NOT NULL,
+  revision_id INTEGER DEFAULT 1 NOT NULL,
+  associated_concept_id VARCHAR(255) NOT NULL,
+  associated_revision_id INTEGER NULL,
+  revision_date TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
+  deleted INTEGER DEFAULT 0 NOT NULL,
+  user_id VARCHAR(30),
+  transaction_id NUMBER DEFAULT 0 NOT NULL")
+
+(def ^:private service-assocs-constraint-sql
+  (str "CONSTRAINT s_assoc_pk PRIMARY KEY (id), "
+       ;; Unique constraint on native id and revision id
+       "CONSTRAINT s_assoc_con_rev UNIQUE (native_id, revision_id)
+        USING INDEX (create unique index s_assoc_ucr_i ON CMR_SERVICE_ASSOCIATIONS (native_id, revision_id)), "
+
+       ;; Unique constraint on concept id and revision id
+       "CONSTRAINT s_assoc_cid_rev UNIQUE (concept_id, revision_id)
+        USING INDEX (create unique index s_assoc_cri ON CMR_SERVICE_ASSOCIATIONS (concept_id, revision_id))"))
+
+(defn- create-service-associations-table
+  []
+  (h/sql (format "CREATE TABLE METADATA_DB.CMR_SERVICE_ASSOCIATIONS (%s, %s)"
+                 service-assocs-column-sql service-assocs-constraint-sql)))
+
+(defn- create-service-associations-indices
+  []
+  (h/sql "CREATE INDEX s_assoc_scid ON cmr_service_associations (service_concept_id, revision_id)")
+  (h/sql "CREATE INDEX s_assoc_scarid ON cmr_service_associations (service_concept_id, associated_concept_id, associated_revision_id)")
+  (h/sql "CREATE INDEX s_assoc_crdi ON CMR_SERVICE_ASSOCIATIONS (concept_id, revision_id, deleted)")
+  (h/sql "CREATE INDEX s_assoc_acari ON CMR_SERVICE_ASSOCIATIONS (associated_concept_id, associated_revision_id)"))
+
+(defn- create-service-associations-sequence
+  []
+  (h/sql "CREATE SEQUENCE CMR_SERVICE_ASSOCIATIONS_SEQ"))
+
+
+(defn up
+  "Migrates the database up to version 83."
+  []
+  (println "cmr.metadata-db.migrations.083-migrate-service-associations up...")
+
+  ;; rename CMR_SERVICE_ASSOCIATIONS table to prevent new entries from being inserted.
+  (h/sql "ALTER TABLE CMR_SERVICE_ASSOCIATIONS RENAME TO CMR_SERVICE_ASSOCIATIONS_OLD")
+
+  ;; Migrate service association data to CMR_ASSOCIATIONS table.
+  ;; Note: We will need to use the CMR_ASSOCIATIONS_SEQ.nextval for the ID column to make it unique in the new table.
+  (h/sql "INSERT INTO CMR_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER,ASSOCIATION_TYPE)
+          SELECT CMR_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SERVICE_CONCEPT_ID,'SERVICE-COLLECTION'
+          FROM CMR_SERVICE_ASSOCIATIONS_OLD")
+
+  ;; Drop the old table.
+  (h/sql "DROP TABLE CMR_SERVICE_ASSOCIATIONS_OLD")
+
+  ;; Drop the sequence
+  (h/sql "DROP SEQUENCE METADATA_DB.cmr_service_associations_seq"))
+
+(defn down
+  "Migrates the database down from version 83."
+  []
+  (println "cmr.metadata-db.migrations.083-migrate-service-associations down...")
+
+  ;; rename CMR_ASSOCIATIONS table to prevent new entries from being inserted.
+  (h/sql "ALTER TABLE CMR_ASSOCIATIONS RENAME TO CMR_ASSOCIATIONS_OLD")
+
+
+  ;; Create CMR_SERVICE_ASSOCIATIONS table.
+  (create-service-associations-table)
+  (create-service-associations-indices)
+  (create-service-associations-sequence)
+
+  ;; Migrate service associations back to CMR_SERVICE_ASSOCIATIONS table.
+  ;; Note: We will now need to use the CMR_SERVICE_ASSOCIATIONS_SEQ.nextval for the ID column so that
+  ;; when new entries are inserted, all the ids will be unique.
+  (h/sql "INSERT INTO CMR_SERVICE_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SERVICE_CONCEPT_ID)
+          SELECT CMR_SERVICE_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER
+          FROM CMR_ASSOCIATIONS_OLD
+          WHERE ASSOCIATION_TYPE='SERVICE-COLLECTION'")
+
+  ;; Remove all the service associations migrated.
+  (h/sql "DELETE CMR_ASSOCIATIONS_OLD WHERE ASSOCIATION_TYPE='SERVICE-COLLECTION'")
+
+  ;; rename CMR_ASSOCIATIONS_OLD table back to CMR_ASSOCIATIONS table..
+  (h/sql "ALTER TABLE CMR_ASSOCIATIONS_OLD RENAME TO CMR_ASSOCIATIONS"))

metadata-db-app/src/cmr/metadata_db/migrations/084_migrate_tool_associations.clj

@@ -0,0 +1,92 @@
+(ns cmr.metadata-db.migrations.084-migrate-tool-associations
+  (:require
+   [config.mdb-migrate-helper :as h]))
+
+(def ^:private tool-assocs-column-sql
+  "id NUMBER,
+  concept_id VARCHAR(255) NOT NULL,
+  native_id VARCHAR(1500) NOT NULL,
+  tool_concept_id VARCHAR(255) NOT NULL,
+  metadata BLOB NOT NULL,
+  format VARCHAR(255) NOT NULL,
+  revision_id INTEGER DEFAULT 1 NOT NULL,
+  associated_concept_id VARCHAR(255) NOT NULL,
+  associated_revision_id INTEGER NULL,
+  revision_date TIMESTAMP WITH TIME ZONE DEFAULT SYSTIMESTAMP NOT NULL,
+  deleted INTEGER DEFAULT 0 NOT NULL,
+  user_id VARCHAR(30),
+  transaction_id NUMBER DEFAULT 0 NOT NULL")
+
+(def ^:private tool-assocs-constraint-sql
+  (str "CONSTRAINT t_assoc_pk PRIMARY KEY (id), "
+        ;; Unique constraint on native id and revision id
+        "CONSTRAINT t_assoc_con_rev UNIQUE (native_id, revision_id)
+         USING INDEX (create unique index t_assoc_ucr_i ON CMR_TOOL_ASSOCIATIONS (native_id, revision_id)), "
+
+        ;; Unique constraint on concept id and revision id
+        "CONSTRAINT t_assoc_cid_rev UNIQUE (concept_id, revision_id)
+         USING INDEX (create unique index t_assoc_cri ON CMR_TOOL_ASSOCIATIONS (concept_id, revision_id))"))
+
+(defn- create-tool-associations-table
+  []
+  (h/sql (format "CREATE TABLE METADATA_DB.CMR_TOOL_ASSOCIATIONS (%s, %s)"
+                 tool-assocs-column-sql tool-assocs-constraint-sql)))
+
+(defn- create-tool-associations-indices
+  []
+  (h/sql "CREATE INDEX t_assoc_crdi ON CMR_TOOL_ASSOCIATIONS (concept_id, revision_id, deleted)")
+  (h/sql "CREATE INDEX t_assoc_acari ON CMR_TOOL_ASSOCIATIONS (associated_concept_id, associated_revision_id)")
+  (h/sql "CREATE INDEX t_assoc_tcid ON CMR_TOOL_ASSOCIATIONS (tool_concept_id, revision_id)")
+  (h/sql "CREATE INDEX t_assoc_tcarid ON CMR_TOOL_ASSOCIATIONS (tool_concept_id, associated_concept_id, associated_revision_id)"))
+
+(defn- create-tool-associations-sequence
+  []
+  (h/sql "CREATE SEQUENCE CMR_TOOL_ASSOCIATIONS_SEQ"))
+
+(defn up
+  "Migrates the database up to version 84."
+  []
+  (println "cmr.metadata-db.migrations.084-migrate-tool-associations up...")
+
+  ;; rename CMR_TOOL_ASSOCIATIONS table to prevent new entries from being inserted.
+  (h/sql "ALTER TABLE CMR_TOOL_ASSOCIATIONS RENAME TO CMR_TOOL_ASSOCIATIONS_OLD")
+
+  ;; Migrate tool association data to CMR_ASSOCIATIONS table.
+  ;; Note: We will need to use the CMR_ASSOCIATIONS_SEQ.nextval for the ID column to make it unique in the new table.
+  (h/sql "INSERT INTO CMR_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER,ASSOCIATION_TYPE)
+          SELECT CMR_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,TOOL_CONCEPT_ID,'TOOL-COLLECTION'
+          FROM CMR_TOOL_ASSOCIATIONS_OLD")
+
+  ;; Drop the old table.
+  (h/sql "DROP TABLE CMR_TOOL_ASSOCIATIONS_OLD")
+
+  ;; Drop the sequence
+  (h/sql "DROP SEQUENCE METADATA_DB.cmr_tool_associations_seq"))
+
+(defn down
+  "Migrates the database down from version 84."
+  []
+  (println "cmr.metadata-db.migrations.084-migrate-tool-associations down...")
+
+  ;; rename CMR_ASSOCIATIONS table to prevent new entries from being inserted.
+  (h/sql "ALTER TABLE CMR_ASSOCIATIONS RENAME TO CMR_ASSOCIATIONS_OLD")
+
+
+  ;; Create CMR_TOOL_ASSOCIATIONS table.
+  (create-tool-associations-table)
+  (create-tool-associations-indices)
+  (create-tool-associations-sequence)
+
+  ;; Migrate tool associations back to CMR_TOOL_ASSOCIATIONS table.
+  ;; Note: We will now need to use the CMR_TOOL_ASSOCIATIONS_SEQ.nextval for the ID column so that
+  ;; when new entries are inserted, all the ids will be unique.
+  (h/sql "INSERT INTO CMR_TOOL_ASSOCIATIONS (ID,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,TOOL_CONCEPT_ID)
+          SELECT CMR_TOOL_ASSOCIATIONS_SEQ.nextval,CONCEPT_ID,NATIVE_ID,METADATA,FORMAT,REVISION_ID,ASSOCIATED_CONCEPT_ID,ASSOCIATED_REVISION_ID,REVISION_DATE,DELETED,USER_ID,TRANSACTION_ID,SOURCE_CONCEPT_IDENTIFIER
+          FROM CMR_ASSOCIATIONS_OLD
+          WHERE ASSOCIATION_TYPE='TOOL-COLLECTION'")
+
+  ;; Remove all the tool associations migrated.
+  (h/sql "DELETE CMR_ASSOCIATIONS_OLD WHERE ASSOCIATION_TYPE='TOOL-COLLECTION'")
+
+  ;; rename CMR_ASSOCIATIONS_OLD table back to CMR_ASSOCIATIONS table..
+  (h/sql "ALTER TABLE CMR_ASSOCIATIONS_OLD RENAME TO CMR_ASSOCIATIONS"))