From bf35bfd7a6a662158367d450e12447035c88bc24 Mon Sep 17 00:00:00 2001 From: gcolon021 Date: Tue, 31 Oct 2023 15:54:09 -0400 Subject: [PATCH] [ALS-4957] Update createLabelsForBins() to properly display single category [ALS-5228] Update BDC to conditionally obfuscate --- .../AggregateDataSharingResourceRS.java | 95 +++++++++++++------ .../dbmi/avillach/service/HeaderFilter.java | 30 ++++++ .../avillach/service/RequestScopedHeader.java | 19 ++++ .../visualization/VisualizationResource.java | 3 +- .../service/DataProcessingService.java | 19 ++-- .../harvard/dbmi/avillach/util/Utilities.java | 6 ++ 6 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/HeaderFilter.java create mode 100644 pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/RequestScopedHeader.java diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java index 0e2efafc..1fd61d58 100644 --- a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java @@ -17,6 +17,7 @@ import edu.harvard.dbmi.avillach.util.VisualizationUtil; import edu.harvard.dbmi.avillach.util.exception.ApplicationException; import edu.harvard.dbmi.avillach.util.exception.ProtocolException; +import edu.harvard.hms.dbmi.avillach.service.RequestScopedHeader; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -33,7 +34,6 @@ import java.io.IOException; import java.util.*; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -53,6 +53,9 @@ public class AggregateDataSharingResourceRS implements IResourceRS { @Inject private ResourceRepository resourceRepository; + @Inject + RequestScopedHeader requestScopedHeader; + private static final ObjectMapper objectMapper = new ObjectMapper(); private final Header[] headers; @@ -431,7 +434,8 @@ public Response queryFormat(QueryRequest queryRequest) { } private Map processCrossCounts(String entityString) throws com.fasterxml.jackson.core.JsonProcessingException { - Map crossCounts = objectMapper.readValue(entityString, new TypeReference<>() {}); + Map crossCounts = objectMapper.readValue(entityString, new TypeReference<>() { + }); int requestVariance = generateVarianceWithCrossCounts(crossCounts); crossCounts = obfuscateCrossCounts(crossCounts, requestVariance); @@ -517,8 +521,7 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons Map crossCounts = objectMapper.readValue(crossCountResponse, new TypeReference<>() { }); int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); - - boolean mustObfuscate = true; + boolean mustObfuscate = isCrossCountObfuscated(crossCounts, generatedVariance); // Handle the case where there is no visualization service UUID if (properties.getVisualizationResourceId() != null) { @@ -542,25 +545,24 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons HttpEntity entity = httpResponse.getEntity(); String binResponse = EntityUtils.toString(entity, "UTF-8"); - Map> binnedContinuousCrossCounts = objectMapper.readValue(binResponse, new TypeReference>>() { + Map> binnedContinuousCrossCounts = objectMapper.readValue(binResponse, new TypeReference>>() { }); - if (!mustObfuscate) { - // Ensure all inner values are Strings to be consistent in our returned data. - binnedContinuousCrossCounts.forEach( - (key, value) -> value.forEach( - (innerKey, innerValue) -> value.put(innerKey, innerValue.toString()) - ) - ); - - return objectMapper.writeValueAsString(binnedContinuousCrossCounts); + // TODO: Return to this code. I would rather not need to keep converting the data back and forth from int to string. + // Convert binnedContinuousCrossCounts Map to a map> + Map> convertedContinuousCrossCount = new HashMap<>(); + binnedContinuousCrossCounts.forEach((key, value) -> { + Map innerMap = new HashMap<>(value); + convertedContinuousCrossCount.put(key, innerMap); + }); + + if (mustObfuscate || doObfuscateCategoricalData(binnedContinuousCrossCounts)) { + obfuscatedCrossCount(generatedVariance, convertedContinuousCrossCount); } - obfuscatedCrossCount(generatedVariance, binnedContinuousCrossCounts); return objectMapper.writeValueAsString(binnedContinuousCrossCounts); } else { // If there is no visualization service resource id, we will simply return the continuous cross count response. - if (!mustObfuscate) { return continuousCrossCountResponse; } @@ -598,21 +600,17 @@ protected String processCategoricalCrossCounts(String categoricalEntityString, S return null; } - Map crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() {}); + Map crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() { + }); int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); - boolean mustObfuscate = true; - if (!mustObfuscate) { - return categoricalEntityString; - } - - // This might break in the object mapper. We need to test this. - Map> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {}); - + Map> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() { + }); if (categoricalCrossCount == null) { return categoricalEntityString; } + // We have not obfuscated yet. We first process the data. for (Map.Entry> entry : categoricalCrossCount.entrySet()) { // skipKey is expecting an entrySet, so we need to convert the axisMap to an entrySet if (VisualizationUtil.skipKey(entry)) continue; @@ -628,12 +626,55 @@ protected String processCategoricalCrossCounts(String categoricalEntityString, S convertedCategoricalCrossCount.put(key, innerMap); }); - // Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance - obfuscatedCrossCount(generatedVariance, convertedCategoricalCrossCount); + if (isCrossCountObfuscated(crossCounts, generatedVariance) || doObfuscateCategoricalData(categoricalCrossCount)) { + // Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance + obfuscatedCrossCount(generatedVariance, convertedCategoricalCrossCount); + } return objectMapper.writeValueAsString(convertedCategoricalCrossCount); } + /** + * If the request source is open access, we need to check if there is at least one category less than threshold. + * If there is at least one category less than threshold and is open access, we return true. + * + * @param convertedCategoricalCrossCount The categorical cross count + * @return boolean based on if the categorical cross count needs to be obfuscated + */ + private boolean doObfuscateCategoricalData(Map> convertedCategoricalCrossCount) { + String requestSource = getRequestSource(); + + if (!isRequestSourceOpen(requestSource)) { + return false; + } + + // If the request source is open access we need to check if there is at least one category less than 10 + // If there is at least one category less than thresh, we need to obfuscate the data. + for (Map.Entry> entry : convertedCategoricalCrossCount.entrySet()) { + Map axisMap = entry.getValue(); + for (Map.Entry axisEntry : axisMap.entrySet()) { + if (axisEntry.getValue() < threshold) { + return true; + } + } + } + + return false; + } + + private boolean isRequestSourceOpen(String requestSource) { + return requestSource != null && requestSource.equalsIgnoreCase("Open"); + } + + private String getRequestSource() { + String requestSource = null; + if (requestScopedHeader != null && requestScopedHeader.getHeaders() != null) { + requestSource = requestScopedHeader.getHeaders().get("request-source").get(0); + logger.info("Request source: " + requestSource); + } + return requestSource; + } + /** * This method will obfuscate the cross counts based on the generated variance. We do not have a return because * we are modifying the passed crossCount object. Java passes objects by reference value, so we do not need to return. diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/HeaderFilter.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/HeaderFilter.java new file mode 100644 index 00000000..f3f328e9 --- /dev/null +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/HeaderFilter.java @@ -0,0 +1,30 @@ +package edu.harvard.hms.dbmi.avillach.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Priority; +import javax.inject.Inject; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.Provider; + +@Provider +@Priority(1) +public class HeaderFilter implements ContainerRequestFilter { + + @Inject + private RequestScopedHeader headers; + + private final Logger logger = LoggerFactory.getLogger(HeaderFilter.class); + + @Override + public void filter(ContainerRequestContext containerRequestContext) { + if (containerRequestContext.getUriInfo().getPath().startsWith("/aggregate-data-sharing")) { + logger.info("HeaderFilter called for path: /aggregate-data-sharing with headers: " + containerRequestContext.getHeaders().toString()); + MultivaluedMap httpHeaders = containerRequestContext.getHeaders(); + headers.setHeaders(httpHeaders); + } + } +} diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/RequestScopedHeader.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/RequestScopedHeader.java new file mode 100644 index 00000000..4ffdc241 --- /dev/null +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/service/RequestScopedHeader.java @@ -0,0 +1,19 @@ +package edu.harvard.hms.dbmi.avillach.service; + +import javax.enterprise.context.RequestScoped; +import javax.ws.rs.core.MultivaluedMap; + +@RequestScoped +public class RequestScopedHeader { + + private MultivaluedMap headers; + + public void setHeaders(MultivaluedMap headers) { + this.headers = headers; + } + + public MultivaluedMap getHeaders() { + return headers; + } + +} diff --git a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/VisualizationResource.java b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/VisualizationResource.java index 5249ddd8..f67fa19d 100644 --- a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/VisualizationResource.java +++ b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/VisualizationResource.java @@ -77,6 +77,7 @@ public Response querySync(QueryRequest query) { String requestSource = null; if (requestScopedHeader != null && requestScopedHeader.getHeaders() != null) { requestSource = requestScopedHeader.getHeaders().get("request-source").get(0); + logger.info("resource=visualization /query/sync query=" + query.getQuery().toString() + " request-source=" + requestSource); } return visualizationService.handleQuerySync(query, requestSource); @@ -99,6 +100,6 @@ public Response queryFormat(QueryRequest resultRequest) { @Path("/bin/continuous") public Response generateContinuousBin(QueryRequest continuousData) { logger.info("resource=visualization /bin/continuous query=" + continuousData.getQuery().toString()); - return visualizationService.generateContinuousBin(continuousData); + return visualizationService.generateContinuousBin(continuousData); } } \ No newline at end of file diff --git a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java index 15c03a1e..6f481283 100644 --- a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java +++ b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java @@ -52,14 +52,9 @@ private List handleGetCategoricalData(Map categoricalDataList = new ArrayList<>(); for (Map.Entry> entry : crossCountsMap.entrySet()) { - if (VisualizationUtil.skipKey(entry)) continue; - - Map axisMap = null; - if (isObfuscated) { - axisMap = VisualizationUtil.processResults(entry.getValue()); - } else { - axisMap = new LinkedHashMap<>(entry.getValue()); - } + // We should not need to obfuscate the data here as it should already be obfuscated + // Additionally, we no longer need to process the data here as it is already processed + Map axisMap = new LinkedHashMap<>(entry.getValue()); String title = getChartTitle(entry.getKey()); categoricalDataList.add(new CategoricalData( @@ -250,15 +245,17 @@ private static Map createLabelsForBins(Map re double minForLabel = ranges.get(bucket.getKey()).stream().min(Double::compareTo).orElse(0.0); double maxForLabel = ranges.get(bucket.getKey()).stream().max(Double::compareTo).orElse(0.0); if (minForLabel == maxForLabel) { - label = String.format("%.1f", minForLabel); + // This should only happen when the min and max are the same + label = String.format("%.1f", maxForLabel); } else { label = String.format("%.1f", minForLabel) + " - " + String.format("%.1f", maxForLabel); } finalMap.put(label, bucket.getValue()); } Integer lastCount = finalMap.get(label); - //Last label should be the min in the range with a '+' sign. - if (lastCount != null) { + + //Last label should be the min in the range with a '+' sign. Only if there is more than one bin. + if (lastCount != null && results.size() > 1) { String newLabel = label; int hasDash = label.indexOf(" -"); if (hasDash > 0) { diff --git a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/Utilities.java b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/Utilities.java index 3561993b..5e63be45 100644 --- a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/Utilities.java +++ b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/Utilities.java @@ -30,6 +30,12 @@ public static HttpClientContext buildHttpClientContext() { return httpClientContext; } + /** + * This method is used to get the request source from the request header. It is used for logging purposes. + * + * @param headers the request headers + * @return the request source + */ public static String getRequestSourceFromHeader(HttpHeaders headers) { if (headers == null) return "headers are null"; return headers.getHeaderString("request-source") == null ? "request-source header is null" : headers.getHeaderString("request-source");