fix: Include time when filtering in tracker exporter [DHIS2-16019]

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/program/hibernate/HibernateEnrollmentStore.java

@@ -30,8 +30,8 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids;
 import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString;
+import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getLongGmtDateString;
-import static org.hisp.dhis.util.DateUtils.getMediumDateString;
 import static org.hisp.dhis.util.DateUtils.nowMinusDuration;
 
 import com.google.common.collect.Lists;
@@ -156,10 +156,7 @@ private QueryWithOrderBy buildEnrollmentHql(EnrollmentQueryParams params) {
               + "'";
     } else if (params.hasLastUpdated()) {
       hql +=
-          hlp.whereAnd()
-              + "en.lastUpdated >= '"
-              + getMediumDateString(params.getLastUpdated())
-              + "'";
+          hlp.whereAnd() + "en.lastUpdated >= '" + getLongDateString(params.getLastUpdated()) + "'";
     }
 
     if (params.hasTrackedEntity()) {
@@ -212,15 +209,15 @@ private QueryWithOrderBy buildEnrollmentHql(EnrollmentQueryParams params) {
       hql +=
           hlp.whereAnd()
               + "en.enrollmentDate >= '"
-              + getMediumDateString(params.getProgramStartDate())
+              + getLongDateString(params.getProgramStartDate())
               + "'";
     }
 
     if (params.hasProgramEndDate()) {
       hql +=
           hlp.whereAnd()
               + "en.enrollmentDate <= '"
-              + getMediumDateString(params.getProgramEndDate())
+              + getLongDateString(params.getProgramEndDate())
               + "'";
     }
 

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/trackedentity/hibernate/HibernateTrackedEntityStore.java

@@ -47,8 +47,8 @@
 import static org.hisp.dhis.trackedentity.TrackedEntityQueryParams.TRACKED_ENTITY_ID;
 import static org.hisp.dhis.trackedentity.TrackedEntityQueryParams.TRACKED_ENTITY_TYPE_ID;
 import static org.hisp.dhis.util.DateUtils.addDays;
+import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getLongGmtDateString;
-import static org.hisp.dhis.util.DateUtils.getMediumDateString;
 
 import com.google.common.collect.Lists;
 import java.util.ArrayList;
@@ -625,14 +625,14 @@ private String getFromSubQueryTrackedEntityConditions(
         trackedEntity
             .append(whereAnd.whereAnd())
             .append(" TE.lastupdated >= '")
-            .append(getMediumDateString(params.getLastUpdatedStartDate()))
+            .append(getLongDateString(params.getLastUpdatedStartDate()))
             .append(SINGLE_QUOTE);
       }
       if (params.hasLastUpdatedEndDate()) {
         trackedEntity
             .append(whereAnd.whereAnd())
             .append(" TE.lastupdated < '")
-            .append(getMediumDateString(addDays(params.getLastUpdatedEndDate(), 1)))
+            .append(getLongDateString(addDays(params.getLastUpdatedEndDate(), 1)))
             .append(SINGLE_QUOTE);
       }
     }
@@ -641,7 +641,7 @@ private String getFromSubQueryTrackedEntityConditions(
       if (params.getSkipChangedBefore() != null) {
         trackedEntity
             .append(" AND TE.lastupdated >= '")
-            .append(getMediumDateString(params.getSkipChangedBefore()))
+            .append(getLongDateString(params.getSkipChangedBefore()))
             .append(SINGLE_QUOTE);
       }
     }
@@ -940,28 +940,28 @@ private String getFromSubQueryEnrollmentConditions(
     if (params.hasProgramEnrollmentStartDate()) {
       program
           .append("AND EN.enrollmentdate >= '")
-          .append(getMediumDateString(params.getProgramEnrollmentStartDate()))
+          .append(getLongDateString(params.getProgramEnrollmentStartDate()))
           .append("' ");
     }
 
     if (params.hasProgramEnrollmentEndDate()) {
       program
           .append("AND EN.enrollmentdate <= '")
-          .append(getMediumDateString(params.getProgramEnrollmentEndDate()))
+          .append(getLongDateString(params.getProgramEnrollmentEndDate()))
           .append("' ");
     }
 
     if (params.hasProgramIncidentStartDate()) {
       program
           .append("AND EN.incidentdate >= '")
-          .append(getMediumDateString(params.getProgramIncidentStartDate()))
+          .append(getLongDateString(params.getProgramIncidentStartDate()))
           .append("' ");
     }
 
     if (params.hasProgramIncidentEndDate()) {
       program
           .append("AND EN.incidentdate <= '")
-          .append(getMediumDateString(params.getProgramIncidentEndDate()))
+          .append(getLongDateString(params.getProgramIncidentEndDate()))
           .append("' ");
     }
 
@@ -999,8 +999,8 @@ private String getFromSubQueryEvent(TrackedEntityQueryParams params) {
     }
 
     if (params.hasEventStatus()) {
-      String start = getMediumDateString(params.getEventStartDate());
-      String end = getMediumDateString(params.getEventEndDate());
+      String start = getLongDateString(params.getEventStartDate());
+      String end = getLongDateString(params.getEventEndDate());
 
       if (params.isEventStatus(EventStatus.COMPLETED)) {
         events

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/enrollment/HibernateEnrollmentStore.java

@@ -29,8 +29,8 @@
 
 import static org.hisp.dhis.common.IdentifiableObjectUtils.getUids;
 import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString;
+import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getLongGmtDateString;
-import static org.hisp.dhis.util.DateUtils.getMediumDateString;
 import static org.hisp.dhis.util.DateUtils.nowMinusDuration;
 
 import java.util.List;
@@ -170,10 +170,7 @@ private QueryWithOrderBy buildEnrollmentHql(EnrollmentQueryParams params) {
               + "'";
     } else if (params.hasLastUpdated()) {
       hql +=
-          hlp.whereAnd()
-              + "en.lastUpdated >= '"
-              + getMediumDateString(params.getLastUpdated())
-              + "'";
+          hlp.whereAnd() + "en.lastUpdated >= '" + getLongDateString(params.getLastUpdated()) + "'";
     }
 
     if (params.hasTrackedEntity()) {
@@ -226,15 +223,15 @@ private QueryWithOrderBy buildEnrollmentHql(EnrollmentQueryParams params) {
       hql +=
           hlp.whereAnd()
               + "en.enrollmentDate >= '"
-              + getMediumDateString(params.getProgramStartDate())
+              + getLongDateString(params.getProgramStartDate())
               + "'";
     }
 
     if (params.hasProgramEndDate()) {
       hql +=
           hlp.whereAnd()
               + "en.enrollmentDate <= '"
-              + getMediumDateString(params.getProgramEndDate())
+              + getLongDateString(params.getProgramEndDate())
               + "'";
     }
 

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/HibernateTrackedEntityStore.java

@@ -32,9 +32,8 @@
 import static org.hisp.dhis.common.IdentifiableObjectUtils.getIdentifiers;
 import static org.hisp.dhis.commons.util.TextUtils.getCommaDelimitedString;
 import static org.hisp.dhis.commons.util.TextUtils.getQuotedCommaDelimitedString;
-import static org.hisp.dhis.util.DateUtils.addDays;
+import static org.hisp.dhis.util.DateUtils.getLongDateString;
 import static org.hisp.dhis.util.DateUtils.getLongGmtDateString;
-import static org.hisp.dhis.util.DateUtils.getMediumDateString;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -511,14 +510,14 @@ private String getFromSubQueryTrackedEntityConditions(
         trackedEntity
             .append(whereAnd.whereAnd())
             .append(" TE.lastupdated >= '")
-            .append(getMediumDateString(params.getLastUpdatedStartDate()))
+            .append(getLongDateString(params.getLastUpdatedStartDate()))
             .append(SINGLE_QUOTE);
       }
       if (params.hasLastUpdatedEndDate()) {
         trackedEntity
             .append(whereAnd.whereAnd())
-            .append(" TE.lastupdated < '")
-            .append(getMediumDateString(addDays(params.getLastUpdatedEndDate(), 1)))
+            .append(" TE.lastupdated <= '")
+            .append(getLongDateString(params.getLastUpdatedEndDate()))
             .append(SINGLE_QUOTE);
       }
     }
@@ -738,28 +737,28 @@ private String getFromSubQueryEnrollmentConditions(
     if (params.hasProgramEnrollmentStartDate()) {
       program
           .append("AND EN.enrollmentdate >= '")
-          .append(getMediumDateString(params.getProgramEnrollmentStartDate()))
+          .append(getLongDateString(params.getProgramEnrollmentStartDate()))
           .append("' ");
     }
 
     if (params.hasProgramEnrollmentEndDate()) {
       program
           .append("AND EN.enrollmentdate <= '")
-          .append(getMediumDateString(params.getProgramEnrollmentEndDate()))
+          .append(getLongDateString(params.getProgramEnrollmentEndDate()))
           .append("' ");
     }
 
     if (params.hasProgramIncidentStartDate()) {
       program
           .append("AND EN.incidentdate >= '")
-          .append(getMediumDateString(params.getProgramIncidentStartDate()))
+          .append(getLongDateString(params.getProgramIncidentStartDate()))
           .append("' ");
     }
 
     if (params.hasProgramIncidentEndDate()) {
       program
           .append("AND EN.incidentdate <= '")
-          .append(getMediumDateString(params.getProgramIncidentEndDate()))
+          .append(getLongDateString(params.getProgramIncidentEndDate()))
           .append("' ");
     }
 
@@ -796,8 +795,8 @@ private String getFromSubQueryEvent(TrackedEntityQueryParams params) {
     }
 
     if (params.hasEventStatus()) {
-      String start = getMediumDateString(params.getEventStartDate());
-      String end = getMediumDateString(params.getEventEndDate());
+      String start = getLongDateString(params.getEventStartDate());
+      String end = getLongDateString(params.getEventEndDate());
 
       if (params.isEventStatus(EventStatus.COMPLETED)) {
         events

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityOperationParams.java

@@ -149,9 +149,6 @@ public class TrackedEntityOperationParams {
   /** Indicates whether the search is for synchronization purposes (for Program Data sync job). */
   private boolean synchronizationQuery;
 
-  /** Indicates a point in the time used to decide the data that should not be synchronized */
-  private Date skipChangedBefore;
-
   /**
    * Potential Duplicate query parameter value. If null, we don't check whether a TE is a
    * potentialDuplicate or not

dhis-2/dhis-services/dhis-service-tracker/src/main/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityQueryParams.java

@@ -42,7 +42,6 @@
 import lombok.ToString;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.SetUtils;
-import org.apache.commons.lang3.time.DateUtils;
 import org.hisp.dhis.common.AssignedUserQueryParam;
 import org.hisp.dhis.common.AssignedUserSelectionMode;
 import org.hisp.dhis.common.BaseIdentifiableObject;
@@ -436,9 +435,7 @@ public TrackedEntityQueryParams setProgramEnrollmentStartDate(Date programEnroll
   }
 
   public Date getProgramEnrollmentEndDate() {
-    return programEnrollmentEndDate != null
-        ? DateUtils.addDays(programEnrollmentEndDate, 1)
-        : programEnrollmentEndDate;
+    return programEnrollmentEndDate;
   }
 
   public TrackedEntityQueryParams setProgramEnrollmentEndDate(Date programEnrollmentEndDate) {
@@ -456,9 +453,7 @@ public TrackedEntityQueryParams setProgramIncidentStartDate(Date programIncident
   }
 
   public Date getProgramIncidentEndDate() {
-    return programIncidentEndDate != null
-        ? DateUtils.addDays(programIncidentEndDate, 1)
-        : programIncidentEndDate;
+    return programIncidentEndDate;
   }
 
   public TrackedEntityQueryParams setProgramIncidentEndDate(Date programIncidentEndDate) {

dhis-2/dhis-services/dhis-service-tracker/src/test/java/org/hisp/dhis/tracker/export/trackedentity/TrackedEntityOperationParamsMapperTest.java

@@ -72,7 +72,6 @@
 import org.hisp.dhis.tracker.export.Order;
 import org.hisp.dhis.user.CurrentUserService;
 import org.hisp.dhis.user.User;
-import org.hisp.dhis.util.DateUtils;
 import org.hisp.dhis.webapi.controller.event.mapper.SortDirection;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -203,8 +202,7 @@ void testMapping() throws BadRequestException, ForbiddenException {
         params.getProgramEnrollmentStartDate(),
         is(operationParams.getProgramEnrollmentStartDate()));
     assertThat(
-        params.getProgramEnrollmentEndDate(),
-        is(DateUtils.addDays(operationParams.getProgramEnrollmentEndDate(), 1)));
+        params.getProgramEnrollmentEndDate(), is(operationParams.getProgramEnrollmentEndDate()));
     assertThat(params.getEventStatus(), is(EventStatus.COMPLETED));
     assertThat(params.getEventStartDate(), is(operationParams.getEventStartDate()));
     assertThat(params.getEventEndDate(), is(operationParams.getEventEndDate()));
@@ -252,7 +250,7 @@ void testMappingProgramEnrollmentEndDate() throws BadRequestException, Forbidden
 
     TrackedEntityQueryParams params = mapper.map(operationParams);
 
-    assertEquals(DateUtils.addDays(date, 1), params.getProgramEnrollmentEndDate());
+    assertEquals(date, params.getProgramEnrollmentEndDate());
   }
 
   @Test

dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/trackedentity/TrackedEntityServiceTest.java

@@ -27,11 +27,14 @@
  */
 package org.hisp.dhis.trackedentity;
 
+import static org.hisp.dhis.utils.Assertions.assertContainsOnly;
+import static org.hisp.dhis.utils.Assertions.assertIsEmpty;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import java.time.ZoneId;
 import java.util.Date;
 import java.util.HashSet;
 import java.util.List;
@@ -804,6 +807,60 @@ void shouldCountZeroEntitiesWhenNonePresent() {
     assertEquals(0, trackedEntitiesCounter);
   }
 
+  @Test
+  void shouldReturnTrackedEntityIfTEWasUpdatedAfterPassedDateAndTime() {
+    Date oneHourBeforeLastUpdated =
+        Date.from(
+            entityInstanceA1
+                .getLastUpdated()
+                .toInstant()
+                .atZone(ZoneId.systemDefault())
+                .toLocalDateTime()
+                .minusHours(1)
+                .atZone(ZoneId.systemDefault())
+                .toInstant());
+    injectSecurityContext(superUser);
+    entityInstanceA1.setTrackedEntityType(trackedEntityType);
+    entityInstanceService.addTrackedEntity(entityInstanceA1);
+
+    TrackedEntityQueryParams params = new TrackedEntityQueryParams();
+    params.setOrgUnits(Set.of(organisationUnit));
+    params.setTrackedEntityType(trackedEntityType);
+    params.setLastUpdatedStartDate(oneHourBeforeLastUpdated);
+
+    List<TrackedEntity> trackedEntities =
+        entityInstanceService.getTrackedEntities(params, true, true);
+
+    assertContainsOnly(List.of(entityInstanceA1), trackedEntities);
+  }
+
+  @Test
+  void shouldReturnEmptyIfTEWasUpdatedBeforePassedDateAndTime() {
+    Date oneHourAfterLastUpdated =
+        Date.from(
+            entityInstanceA1
+                .getLastUpdated()
+                .toInstant()
+                .atZone(ZoneId.systemDefault())
+                .toLocalDateTime()
+                .plusHours(1)
+                .atZone(ZoneId.systemDefault())
+                .toInstant());
+    injectSecurityContext(superUser);
+    entityInstanceA1.setTrackedEntityType(trackedEntityType);
+    entityInstanceService.addTrackedEntity(entityInstanceA1);
+
+    TrackedEntityQueryParams params = new TrackedEntityQueryParams();
+    params.setOrgUnits(Set.of(organisationUnit));
+    params.setTrackedEntityType(trackedEntityType);
+    params.setLastUpdatedStartDate(oneHourAfterLastUpdated);
+
+    List<TrackedEntity> trackedEntities =
+        entityInstanceService.getTrackedEntities(params, true, true);
+
+    assertIsEmpty(trackedEntities);
+  }
+
   private void initializeEntityInstance(TrackedEntity entityInstance) {
     entityInstance.setTrackedEntityType(trackedEntityType);
     entityInstanceService.addTrackedEntity(entityInstance);