Merge pull request #35 from AAFC-BICoE/34651_Allow_TabularDataExportG…

…enerator_to_export_id 34651 allow tabular data export generator to export
AAFC-BICoE · Aug 23, 2024 · a294f45 · a294f45
2 parents e24e5cc + 6cf2985
commit a294f45
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 21 deletions.
diff --git a/pom.xml b/pom.xml
@@ -32,7 +32,7 @@
     <maven-checkstyle-plugin.version>3.4.0</maven-checkstyle-plugin.version>
     <checkstyle.version>10.17.0</checkstyle.version>
 
-    <dina-base-api.version>0.124</dina-base-api.version>
+    <dina-base-api.version>0.128</dina-base-api.version>
     <spring-boot-maven-plugin.fork>false</spring-boot-maven-plugin.fork>
     <zxing.version>3.5.1</zxing.version>
     <openhtml.version>1.0.10</openhtml.version>
@@ -44,18 +44,6 @@
     <mockserver.version>5.15.0</mockserver.version>
   </properties>
 
-  <!--  temporary until we can migrate to a newer version of Crnk-->
-  <repositories>
-    <repository>
-      <snapshots>
-        <enabled>false</enabled>
-      </snapshots>
-      <id>jcenter central</id>
-      <name>bintray</name>
-      <url>https://jcenter.bintray.com</url>
-    </repository>
-  </repositories>
-
   <dependencies>
     <dependency>
       <groupId>io.github.aafc-bicoe</groupId>

diff --git a/src/main/java/ca/gc/aafc/dina/export/api/generator/TabularDataExportGenerator.java b/src/main/java/ca/gc/aafc/dina/export/api/generator/TabularDataExportGenerator.java
@@ -151,7 +151,7 @@ private void export(String sourceIndex, String query, DataOutput<JsonNode> outpu
     boolean pageAvailable = response.hits().hits().size() != 0;
     while (pageAvailable) {
       for (Hit<JsonNode> hit : response.hits().hits()) {
-        processRecord(hit.source(), output);
+        processRecord(hit.id(), hit.source(), output);
       }
       pageAvailable = false;
 
@@ -174,14 +174,16 @@ private void export(String sourceIndex, String query, DataOutput<JsonNode> outpu
    * @param output
    * @throws IOException
    */
-  private void processRecord(JsonNode record, DataOutput<JsonNode> output) throws IOException {
+  private void processRecord(String documentId, JsonNode record, DataOutput<JsonNode> output) throws IOException {
     if (record == null) {
       return;
     }
 
     Optional<JsonNode> attributes = atJsonPtr(record, JSONApiDocumentStructure.ATTRIBUTES_PTR);
     if (attributes.isPresent() && attributes.get() instanceof ObjectNode attributeObjNode) {
 
+      attributeObjNode.put(JSONApiDocumentStructure.ID, documentId);
+
       // handle nested maps (e.g. managed attributes)
       replaceNestedByDotNotation(attributeObjNode);
 

diff --git a/src/main/java/ca/gc/aafc/dina/export/api/service/ExpiredExportRemovalService.java b/src/main/java/ca/gc/aafc/dina/export/api/service/ExpiredExportRemovalService.java
@@ -50,10 +50,10 @@ public void removeExpired() {
         "FROM " + DataExport.class.getCanonicalName() + " de " +
         "WHERE status= :status";
 
-    List<DataExport> completedExport = baseDAO.resultListFromQuery(
-      DataExport.class,
-      sql, 0, MAX_QUERY_LIMIT,
-      List.of(Pair.of("status", DataExport.ExportStatus.COMPLETED)));
+    List<DataExport> completedExport = baseDAO.findAllByQuery(
+      DataExport.class, sql,
+      List.of(Pair.of("status", DataExport.ExportStatus.COMPLETED)),
+      MAX_QUERY_LIMIT, 0);
 
     for (DataExport de : completedExport) {
       if (isExpired(de)) {

diff --git a/src/test/java/ca/gc/aafc/dina/export/api/repository/DataExportRepositoryIT.java b/src/test/java/ca/gc/aafc/dina/export/api/repository/DataExportRepositoryIT.java
@@ -61,7 +61,7 @@ public void testESDatasource() throws IOException {
     UUID docId = UUID.randomUUID();
     ElasticSearchTestUtils.indexDocument(esClient, MAT_SAMPLE_INDEX, docId.toString(), JsonApiDocuments.getMaterialSampleDocument(docId));
     UUID docId2 = UUID.randomUUID();
-    ElasticSearchTestUtils.indexDocument(esClient, MAT_SAMPLE_INDEX, docId.toString(), JsonApiDocuments.getMaterialSampleDocument(docId2));
+    ElasticSearchTestUtils.indexDocument(esClient, MAT_SAMPLE_INDEX, docId2.toString(), JsonApiDocuments.getMaterialSampleDocument(docId2));
 
     // Do a query with no sort to ensure a default sort will be added for paging
     String query = "{\"query\": {\"match_all\": {}}}";
@@ -71,7 +71,7 @@ public void testESDatasource() throws IOException {
         .source(MAT_SAMPLE_INDEX)
         .name("my export")
         .query(query)
-        .columns(List.of("materialSampleName", "collectingEvent.dwcVerbatimLocality",
+        .columns(List.of("id", "materialSampleName", "collectingEvent.dwcVerbatimLocality",
           "dwcCatalogNumber", "dwcOtherCatalogNumbers", "managedAttributes.attribute_1",
           "collectingEvent.managedAttributes.attribute_ce_1"))
         .build());
@@ -96,6 +96,10 @@ public void testESDatasource() throws IOException {
     assertNotNull(response.getBody());
     String text = new String(response.getBody().getInputStream().readAllBytes(), StandardCharsets.UTF_8);
     List<String> lines = text.lines().toList();
+
+    // make sure id is exported, skip the header line and check if it's in 1 of the 2 exported line (order is undefined)
+    assertTrue(lines.get(1).contains(docId.toString()) || lines.get(2).contains(docId.toString()));
+
     // make sure managedAttributes are extracted to specific column(s)
     assertTrue(lines.get(0).contains("managedAttributes.attribute_1"));