[ALS-4957] Update createLabelsForBins() to properly display single ca…

…tegory

[ALS-5228] Update BDC to conditionally obfuscate

pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java

@@ -17,6 +17,7 @@
 import edu.harvard.dbmi.avillach.util.VisualizationUtil;
 import edu.harvard.dbmi.avillach.util.exception.ApplicationException;
 import edu.harvard.dbmi.avillach.util.exception.ProtocolException;
+import edu.harvard.hms.dbmi.avillach.service.RequestScopedHeader;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
@@ -33,7 +34,6 @@
 import java.io.IOException;
 import java.util.*;
 import java.util.function.Predicate;
-import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -53,6 +53,9 @@ public class AggregateDataSharingResourceRS implements IResourceRS {
     @Inject
     private ResourceRepository resourceRepository;
 
+    @Inject
+    RequestScopedHeader requestScopedHeader;
+
     private static final ObjectMapper objectMapper = new ObjectMapper();
 
     private final Header[] headers;
@@ -431,7 +434,8 @@ public Response queryFormat(QueryRequest queryRequest) {
     }
 
     private Map<String, String> processCrossCounts(String entityString) throws com.fasterxml.jackson.core.JsonProcessingException {
-        Map<String, String> crossCounts = objectMapper.readValue(entityString, new TypeReference<>() {});
+        Map<String, String> crossCounts = objectMapper.readValue(entityString, new TypeReference<>() {
+        });
 
         int requestVariance = generateVarianceWithCrossCounts(crossCounts);
         crossCounts = obfuscateCrossCounts(crossCounts, requestVariance);
@@ -517,8 +521,7 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons
         Map<String, String> crossCounts = objectMapper.readValue(crossCountResponse, new TypeReference<>() {
         });
         int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts);
-
-        boolean mustObfuscate = true;
+        boolean mustObfuscate = isCrossCountObfuscated(crossCounts, generatedVariance);
 
         // Handle the case where there is no visualization service UUID
         if (properties.getVisualizationResourceId() != null) {
@@ -542,25 +545,24 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons
             HttpEntity entity = httpResponse.getEntity();
             String binResponse = EntityUtils.toString(entity, "UTF-8");
 
-            Map<String, Map<String, Object>> binnedContinuousCrossCounts = objectMapper.readValue(binResponse, new TypeReference<Map<String, Map<String, Object>>>() {
+            Map<String, Map<String, Integer>> binnedContinuousCrossCounts = objectMapper.readValue(binResponse, new TypeReference<Map<String, Map<String, Integer>>>() {
             });
 
-            if (!mustObfuscate) {
-                // Ensure all inner values are Strings to be consistent in our returned data.
-                binnedContinuousCrossCounts.forEach(
-                        (key, value) -> value.forEach(
-                                (innerKey, innerValue) -> value.put(innerKey, innerValue.toString())
-                        )
-                );
-
-                return objectMapper.writeValueAsString(binnedContinuousCrossCounts);
+            // TODO: Return to this code. I would rather not need to keep converting the data back and forth from int to string.
+            // Convert binnedContinuousCrossCounts Map to a map<String, Map<String, Object>>
+            Map<String, Map<String, Object>> convertedContinuousCrossCount = new HashMap<>();
+            binnedContinuousCrossCounts.forEach((key, value) -> {
+                Map<String, Object> innerMap = new HashMap<>(value);
+                convertedContinuousCrossCount.put(key, innerMap);
+            });
+
+            if (mustObfuscate || doObfuscateCategoricalData(binnedContinuousCrossCounts)) {
+                obfuscatedCrossCount(generatedVariance, convertedContinuousCrossCount);
             }
 
-            obfuscatedCrossCount(generatedVariance, binnedContinuousCrossCounts);
             return objectMapper.writeValueAsString(binnedContinuousCrossCounts);
         } else {
             // If there is no visualization service resource id, we will simply return the continuous cross count response.
-
             if (!mustObfuscate) {
                 return continuousCrossCountResponse;
             }
@@ -598,21 +600,17 @@ protected String processCategoricalCrossCounts(String categoricalEntityString, S
             return null;
         }
 
-        Map<String, String> crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() {});
+        Map<String, String> crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() {
+        });
         int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts);
 
-        boolean mustObfuscate = true;
-        if (!mustObfuscate) {
-            return categoricalEntityString;
-        }
-
-        // This might break in the object mapper. We need to test this.
-        Map<String, Map<String, Integer>> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {});
-
+        Map<String, Map<String, Integer>> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {
+        });
         if (categoricalCrossCount == null) {
             return categoricalEntityString;
         }
 
+        // We have not obfuscated yet. We first process the data.
         for (Map.Entry<String, Map<String, Integer>> entry : categoricalCrossCount.entrySet()) {
             // skipKey is expecting an entrySet, so we need to convert the axisMap to an entrySet
             if (VisualizationUtil.skipKey(entry)) continue;
@@ -628,12 +626,55 @@ protected String processCategoricalCrossCounts(String categoricalEntityString, S
             convertedCategoricalCrossCount.put(key, innerMap);
         });
 
-        // Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance
-        obfuscatedCrossCount(generatedVariance, convertedCategoricalCrossCount);
+        if (isCrossCountObfuscated(crossCounts, generatedVariance) || doObfuscateCategoricalData(categoricalCrossCount)) {
+            // Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance
+            obfuscatedCrossCount(generatedVariance, convertedCategoricalCrossCount);
+        }
 
         return objectMapper.writeValueAsString(convertedCategoricalCrossCount);
     }
 
+    /**
+     * If the request source is open access, we need to check if there is at least one category less than threshold.
+     * If there is at least one category less than threshold and is open access, we return true.
+     *
+     * @param convertedCategoricalCrossCount The categorical cross count
+     * @return boolean based on if the categorical cross count needs to be obfuscated
+     */
+    private boolean doObfuscateCategoricalData(Map<String, Map<String, Integer>> convertedCategoricalCrossCount) {
+        String requestSource = getRequestSource();
+
+        if (!isRequestSourceOpen(requestSource)) {
+            return false;
+        }
+
+        // If the request source is open access we need to check if there is at least one category less than 10
+        // If there is at least one category less than thresh, we need to obfuscate the data.
+        for (Map.Entry<String, Map<String, Integer>> entry : convertedCategoricalCrossCount.entrySet()) {
+            Map<String, Integer> axisMap = entry.getValue();
+            for (Map.Entry<String, Integer> axisEntry : axisMap.entrySet()) {
+                if (axisEntry.getValue() < threshold) {
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    private boolean isRequestSourceOpen(String requestSource) {
+        return requestSource != null && requestSource.equalsIgnoreCase("Open");
+    }
+
+    private String getRequestSource() {
+        String requestSource = null;
+        if (requestScopedHeader != null && requestScopedHeader.getHeaders() != null) {
+            requestSource = requestScopedHeader.getHeaders().get("request-source").get(0);
+            logger.info("Request source: " + requestSource);
+        }
+        return requestSource;
+    }
+
     /**
      * This method will obfuscate the cross counts based on the generated variance. We do not have a return because
      * we are modifying the passed crossCount object. Java passes objects by reference value, so we do not need to return.

pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/HeaderFilter.java

@@ -0,0 +1,30 @@
+package edu.harvard.hms.dbmi.avillach.service;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Priority;
+import javax.inject.Inject;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.Provider;
+
+@Provider
+@Priority(1)
+public class HeaderFilter implements ContainerRequestFilter {
+
+    @Inject
+    private RequestScopedHeader headers;
+
+    private final Logger logger = LoggerFactory.getLogger(HeaderFilter.class);
+
+    @Override
+    public void filter(ContainerRequestContext containerRequestContext) {
+        if (containerRequestContext.getUriInfo().getPath().startsWith("/aggregate-data-sharing")) {
+            logger.info("HeaderFilter called for path: /aggregate-data-sharing with headers: " + containerRequestContext.getHeaders().toString());
+            MultivaluedMap<String, String> httpHeaders = containerRequestContext.getHeaders();
+            headers.setHeaders(httpHeaders);
+        }
+    }
+}

pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/RequestScopedHeader.java

@@ -0,0 +1,19 @@
+package edu.harvard.hms.dbmi.avillach.service;
+
+import javax.enterprise.context.RequestScoped;
+import javax.ws.rs.core.MultivaluedMap;
+
+@RequestScoped
+public class RequestScopedHeader {
+
+    private MultivaluedMap<String, String> headers;
+
+    public void setHeaders(MultivaluedMap<String, String> headers) {
+        this.headers = headers;
+    }
+
+    public MultivaluedMap<String, String> getHeaders() {
+        return headers;
+    }
+
+}

pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/VisualizationResource.java

@@ -77,6 +77,7 @@ public Response querySync(QueryRequest query) {
         String requestSource = null;
         if (requestScopedHeader != null && requestScopedHeader.getHeaders() != null) {
             requestSource = requestScopedHeader.getHeaders().get("request-source").get(0);
+            logger.info("resource=visualization /query/sync query=" + query.getQuery().toString() + " request-source=" + requestSource);
         }
 
         return visualizationService.handleQuerySync(query, requestSource);
@@ -99,6 +100,6 @@ public Response queryFormat(QueryRequest resultRequest) {
     @Path("/bin/continuous")
     public Response generateContinuousBin(QueryRequest continuousData) {
         logger.info("resource=visualization /bin/continuous query=" + continuousData.getQuery().toString());
-    	return visualizationService.generateContinuousBin(continuousData);
+        return visualizationService.generateContinuousBin(continuousData);
     }
 }

pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java

@@ -52,14 +52,9 @@ private List<CategoricalData> handleGetCategoricalData(Map<String, Map<String, I
         List<CategoricalData> categoricalDataList = new ArrayList<>();
 
         for (Map.Entry<String, Map<String, Integer>> entry : crossCountsMap.entrySet()) {
-            if (VisualizationUtil.skipKey(entry)) continue;
-
-            Map<String, Integer> axisMap = null;
-            if (isObfuscated) {
-                axisMap = VisualizationUtil.processResults(entry.getValue());
-            } else {
-                axisMap = new LinkedHashMap<>(entry.getValue());
-            }
+            // We should not need to obfuscate the data here as it should already be obfuscated
+            // Additionally, we no longer need to process the data here as it is already processed
+            Map<String, Integer> axisMap = new LinkedHashMap<>(entry.getValue());
 
             String title = getChartTitle(entry.getKey());
             categoricalDataList.add(new CategoricalData(
@@ -250,15 +245,17 @@ private static Map<String, Integer> createLabelsForBins(Map<Integer, Integer> re
             double minForLabel = ranges.get(bucket.getKey()).stream().min(Double::compareTo).orElse(0.0);
             double maxForLabel = ranges.get(bucket.getKey()).stream().max(Double::compareTo).orElse(0.0);
             if (minForLabel == maxForLabel) {
-                label = String.format("%.1f", minForLabel);
+                // This should only happen when the min and max are the same
+                label = String.format("%.1f", maxForLabel);
             } else {
                 label = String.format("%.1f", minForLabel) + " - " + String.format("%.1f", maxForLabel);
             }
             finalMap.put(label, bucket.getValue());
         }
         Integer lastCount = finalMap.get(label);
-        //Last label should be the min in the range with a '+' sign.
-        if (lastCount != null) {
+
+        //Last label should be the min in the range with a '+' sign. Only if there is more than one bin.
+        if (lastCount != null && results.size() > 1) {
             String newLabel = label;
             int hasDash = label.indexOf(" -");
             if (hasDash > 0) {

pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/Utilities.java

@@ -30,6 +30,12 @@ public static HttpClientContext buildHttpClientContext() {
         return httpClientContext;
     }
 
+    /**
+     * This method is used to get the request source from the request header. It is used for logging purposes.
+     *
+     * @param headers the request headers
+     * @return the request source
+     */
     public static String getRequestSourceFromHeader(HttpHeaders headers) {
         if (headers == null) return "headers are null";
         return headers.getHeaderString("request-source") == null ? "request-source header is null" : headers.getHeaderString("request-source");