Skip to content

Commit

Permalink
[ALS-4957] Update categorical processing
Browse files Browse the repository at this point in the history
  • Loading branch information
@Gcolon021
Gcolon021 committed Nov 1, 2023
1 parent bf35bfd commit e94222f
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 65 deletions.
131 changes: 68 additions & 63 deletions ...-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,31 +166,31 @@ public QueryStatus query(QueryRequest queryRequest) {

}

@POST
@Path("/query/{resourceQueryId}/status")
@Override
public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest statusRequest) {
logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId);
checkQuery(statusRequest);
HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status");
return readObjectFromResponse(response, QueryStatus.class);
}

@POST
@Path("/query/{resourceQueryId}/result")
@Override
public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest resultRequest) {
logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId);
checkQuery(resultRequest);
HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result");
try {
return Response.ok(response.getEntity().getContent()).build();
} catch (IOException e) {
throw new ApplicationException(
"Error encoding query for resource with id " + resultRequest.getResourceUUID()
);
}
}
@POST
@Path("/query/{resourceQueryId}/status")
@Override
public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest statusRequest) {
logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId);
checkQuery(statusRequest);
HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status");
return readObjectFromResponse(response, QueryStatus.class);
}

@POST
@Path("/query/{resourceQueryId}/result")
@Override
public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest resultRequest) {
logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId);
checkQuery(resultRequest);
HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result");
try {
return Response.ok(response.getEntity().getContent()).build();
} catch (IOException e) {
throw new ApplicationException(
"Error encoding query for resource with id " + resultRequest.getResourceUUID()
);
}
}

private HttpResponse postRequest(QueryRequest statusRequest, String pathName) {
try {
Expand Down Expand Up @@ -233,6 +233,7 @@ public Response querySync(QueryRequest queryRequest) {
"COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT",
"OBSERVATION_CROSS_COUNT", "CATEGORICAL_CROSS_COUNT", "CONTINUOUS_CROSS_COUNT"
);

if (!allowedResultTypes.contains(expectedResultType)) {
logger.warn("Incorrect Result Type: " + expectedResultType);
return Response.status(Response.Status.BAD_REQUEST).build();
Expand Down Expand Up @@ -556,7 +557,7 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons
convertedContinuousCrossCount.put(key, innerMap);
});

if (mustObfuscate || doObfuscateCategoricalData(binnedContinuousCrossCounts)) {
if (mustObfuscate || doObfuscateData(convertedContinuousCrossCount)) {
obfuscatedCrossCount(generatedVariance, convertedContinuousCrossCount);
}

Expand Down Expand Up @@ -604,61 +605,59 @@ protected String processCategoricalCrossCounts(String categoricalEntityString, S
});
int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts);

Map<String, Map<String, Integer>> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {
Map<String, Map<String, Object>> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {
});

if (categoricalCrossCount == null) {
logger.info("Categorical cross count is null. Returning categoricalEntityString: {}", categoricalEntityString);
return categoricalEntityString;
}

// We have not obfuscated yet. We first process the data.
for (Map.Entry<String, Map<String, Integer>> entry : categoricalCrossCount.entrySet()) {
// skipKey is expecting an entrySet, so we need to convert the axisMap to an entrySet
if (VisualizationUtil.skipKey(entry)) continue;
processResults(categoricalCrossCount);

Map<String, Integer> axisMap = VisualizationUtil.processResults(entry.getValue());
categoricalCrossCount.put(entry.getKey(), axisMap);
if (isCrossCountObfuscated(crossCounts, generatedVariance) || doObfuscateData(categoricalCrossCount)) {
// Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance
obfuscatedCrossCount(generatedVariance, categoricalCrossCount);
}

// Convert the categoricalCrossCount Map to a map<String, Map<String, Object>>
Map<String, Map<String, Object>> convertedCategoricalCrossCount = new HashMap<>();
categoricalCrossCount.forEach((key, value) -> {
Map<String, Object> innerMap = new HashMap<>(value);
convertedCategoricalCrossCount.put(key, innerMap);
});
return objectMapper.writeValueAsString(categoricalCrossCount);
}

if (isCrossCountObfuscated(crossCounts, generatedVariance) || doObfuscateCategoricalData(categoricalCrossCount)) {
// Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance
obfuscatedCrossCount(generatedVariance, convertedCategoricalCrossCount);
private static void processResults(Map<String, Map<String, Object>> categoricalCrossCount) {
for (Map.Entry<String, Map<String, Object>> entry : categoricalCrossCount.entrySet()) {
// skipKey is expecting an entrySet, so we need to convert the axisMap to an entrySet
if (VisualizationUtil.skipKey(entry)) continue;
Map<String, Object> axisMap = VisualizationUtil.processResults(entry.getValue());
categoricalCrossCount.put(entry.getKey(), axisMap);
}

return objectMapper.writeValueAsString(convertedCategoricalCrossCount);
}

/**
* If the request source is open access, we need to check if there is at least one category less than threshold.
* If there is at least one category less than threshold and is open access, we return true.
*
* @param convertedCategoricalCrossCount The categorical cross count
*
* @param categoricalCrossCount The categorical cross count
* @return boolean based on if the categorical cross count needs to be obfuscated
*/
private boolean doObfuscateCategoricalData(Map<String, Map<String, Integer>> convertedCategoricalCrossCount) {
String requestSource = getRequestSource();

if (!isRequestSourceOpen(requestSource)) {
private boolean doObfuscateData(Map<String, Map<String, Object>> categoricalCrossCount) {
if (!isRequestSourceOpen(getRequestSource())) {
return false;
}

// If the request source is open access we need to check if there is at least one category less than 10
// If there is at least one category less than thresh, we need to obfuscate the data.
for (Map.Entry<String, Map<String, Integer>> entry : convertedCategoricalCrossCount.entrySet()) {
Map<String, Integer> axisMap = entry.getValue();
for (Map.Entry<String, Integer> axisEntry : axisMap.entrySet()) {
if (axisEntry.getValue() < threshold) {
return true;
}
return categoricalCrossCount.values().stream()
.anyMatch(this::checkForObfuscation);
}

private boolean checkForObfuscation(Map<String, Object> axisMap) {
for (Map.Entry<String, Object> axisEntry : axisMap.entrySet()) {
Object value = axisEntry.getValue();
if (value instanceof Integer && (Integer) value < threshold) {
return true;
}
if (value instanceof String && Integer.parseInt((String) value) < threshold) {
return true;
}
}

return false;
}

Expand All @@ -667,11 +666,17 @@ private boolean isRequestSourceOpen(String requestSource) {
}

private String getRequestSource() {
String requestSource = null;
if (requestScopedHeader != null && requestScopedHeader.getHeaders() != null) {
requestSource = requestScopedHeader.getHeaders().get("request-source").get(0);
logger.info("Request source: " + requestSource);
if (requestScopedHeader == null || requestScopedHeader.getHeaders() == null) {
logger.warn("Request scoped header or headers are null");
return null;
}
List<String> requestSources = requestScopedHeader.getHeaders().get("request-source");
if (requestSources == null || requestSources.isEmpty()) {
logger.warn("Request source header is missing or empty");
return null;
}
String requestSource = requestSources.get(0);
logger.info("Request source: " + requestSource);
return requestSource;
}

Expand Down
15 changes: 13 additions & 2 deletions pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/VisualizationUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,32 @@ public class VisualizationUtil {
private static final boolean LIMITED = true;
private static final int LIMIT_SIZE = 7;

public static boolean skipKey(Map.Entry<String, Map<String, Integer>> entry) {
public static boolean skipKey(Map.Entry<String, Map<String, Object>> entry) {
return entry.getKey().equals(CONSENTS_KEY) ||
entry.getKey().equals(HARMONIZED_CONSENT_KEY) ||
entry.getKey().equals(TOPMED_CONSENTS_KEY) ||
entry.getKey().equals(PARENT_CONSENTS_KEY);
}

public static Map<String, Object> processResults(Map<String, Object> axisMap) {
Map<String, Integer> convertedAxisMap = new HashMap<>();
for (Map.Entry<String, Object> entry : axisMap.entrySet()) {
if (entry.getValue() instanceof Integer) {
convertedAxisMap.put(entry.getKey(), (Integer) entry.getValue());
}
}
Map<String, Integer> stringIntegerMap = doProcessResults(convertedAxisMap);
return new HashMap<>(stringIntegerMap);
}

/**
* Sorts the map and if there is more than the LIMIT_SIZE then we also get the greatest 7 categories and then combines
* the others into an "other" category. Also replace long column names with shorter version.
*
* @param axisMap - Map of the categories and their counts
* @return Map<String, Integer> - sorted map of the categories and their counts with the "other" category added if necessary
*/
public static Map<String, Integer> processResults(Map<String, Integer> axisMap) {
private static Map<String, Integer> doProcessResults(Map<String, Integer> axisMap) {
Map<String, Integer> finalAxisMap = axisMap;
if (LIMITED && axisMap.size() > (LIMIT_SIZE + 1)) {
//Create Other bar and sort
Expand Down

0 comments on commit e94222f

Please sign in to comment.