From b5fee3bf6d57f840b4477ae745753ffb827bf21a Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:49:09 -0400 Subject: [PATCH 1/3] [ALS-4467] other study bar on var dist fix (#141) [ALS-4467] Fix other category for open stat vis --- .../AggregateDataSharingResourceRS.java | 1447 +++++++++-------- .../service/DataProcessingService.java | 102 +- .../dbmi/avillach/util/VisualizationUtil.java | 82 + 3 files changed, 837 insertions(+), 794 deletions(-) create mode 100644 pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/VisualizationUtil.java diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java index fc5a4fb3..30b1894d 100644 --- a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java @@ -1,26 +1,22 @@ package edu.harvard.hms.dbmi.avillach; -import static edu.harvard.dbmi.avillach.service.ResourceWebClient.QUERY_METADATA_FIELD; -import static edu.harvard.dbmi.avillach.util.HttpClientUtil.readObjectFromResponse; - -import java.io.IOException; -import java.util.*; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import javax.inject.Inject; -import javax.inject.Singleton; -import javax.ws.rs.*; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; - import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import edu.harvard.dbmi.avillach.data.entity.Resource; import edu.harvard.dbmi.avillach.data.repository.ResourceRepository; +import edu.harvard.dbmi.avillach.domain.QueryRequest; +import edu.harvard.dbmi.avillach.domain.QueryStatus; +import edu.harvard.dbmi.avillach.domain.ResourceInfo; +import edu.harvard.dbmi.avillach.domain.SearchResults; +import edu.harvard.dbmi.avillach.service.IResourceRS; import edu.harvard.dbmi.avillach.util.HttpClientUtil; +import edu.harvard.dbmi.avillach.util.VisualizationUtil; +import edu.harvard.dbmi.avillach.util.exception.ApplicationException; +import edu.harvard.dbmi.avillach.util.exception.ProtocolException; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -29,14 +25,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.*; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; -import edu.harvard.dbmi.avillach.domain.*; -import edu.harvard.dbmi.avillach.service.IResourceRS; -import edu.harvard.dbmi.avillach.util.exception.ApplicationException; -import edu.harvard.dbmi.avillach.util.exception.ProtocolException; +import static edu.harvard.dbmi.avillach.service.ResourceWebClient.QUERY_METADATA_FIELD; +import static edu.harvard.dbmi.avillach.util.HttpClientUtil.readObjectFromResponse; @Path("/aggregate-data-sharing") @Produces("application/json") @@ -44,695 +46,716 @@ @Singleton public class AggregateDataSharingResourceRS implements IResourceRS { - @Inject - private ApplicationProperties properties; - - @Inject - private ResourceRepository resourceRepository; - - private static final ObjectMapper objectMapper = new ObjectMapper(); - - private final Header[] headers; - - private static final String BEARER_STRING = "Bearer "; - - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - - private final int threshold; - private final int variance; - - private final String randomSalt; - - public AggregateDataSharingResourceRS() { - this(null); - } - - @Inject - public AggregateDataSharingResourceRS(ApplicationProperties applicationProperties) { - this.properties = applicationProperties; - if (applicationProperties == null) { - logger.info("initialize Aggregate Resource NO INJECTION"); - } else { - logger.info("initialize Aggregate Resource Injected " + applicationProperties); - } - - if (properties == null) { - properties = new ApplicationProperties(); - properties.init("pic-sure-aggregate-resource"); - } - - threshold = properties.getTargetPicsureObfuscationThreshold(); - variance = properties.getTargetPicsureObfuscationVariance(); - randomSalt = properties.getTargetPicsureObfuscationSalt(); - - headers = new Header[] {new BasicHeader(HttpHeaders.AUTHORIZATION, BEARER_STRING + properties.getTargetPicsureToken())}; - } - - - - @GET - @Path("/status") - public Response status() { - logger.debug("Calling Aggregate Data Sharing Resource status()"); - return Response.ok().build(); - } - - @POST - @Path("/info") - @Override - public ResourceInfo info(QueryRequest infoRequest) { - logger.debug("Calling Aggregate Data Sharing Resource info()"); - String pathName = "/info"; - - try { - QueryRequest chainRequest = new QueryRequest(); - if (infoRequest != null) { - chainRequest.setQuery(infoRequest.getQuery()); - chainRequest.setResourceCredentials(infoRequest.getResourceCredentials()); - //set a default value of the existing uuid here (can override in properties file) - chainRequest.setResourceUUID(infoRequest.getResourceUUID()); - } - if(properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) { - chainRequest.setResourceUUID(UUID.fromString(properties.getTargetResourceId())); - } - - String payload = objectMapper.writeValueAsString(chainRequest); - String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); - HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload); - if (!HttpClientUtil.is2xx(response)) { - logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); - HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); - } - - //if we are proxying an info request, we need to return our own resource ID - ResourceInfo resourceInfo = readObjectFromResponse(response, ResourceInfo.class); - if (infoRequest != null && infoRequest.getResourceUUID() != null) { - resourceInfo.setId(infoRequest.getResourceUUID()); - } - return resourceInfo; - } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + infoRequest.getResourceUUID()); - } catch (ClassCastException | IllegalArgumentException e) { - logger.error(e.getMessage()); - throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); - } - } - - @POST - @Path("/search") - @Override - public SearchResults search(QueryRequest searchRequest) { - logger.debug("Calling Aggregate Data Sharing Search"); - checkQuery(searchRequest); - HttpResponse response = postRequest(searchRequest, "/search"); - return readObjectFromResponse(response, SearchResults.class); - } - - @POST - @Path("/query") - @Override - public QueryStatus query(QueryRequest queryRequest) { - logger.debug("Calling Aggregate Data Sharing Resource query()"); - checkQuery(queryRequest); - HttpResponse response = postRequest(queryRequest, "/query"); - return readObjectFromResponse(response, QueryStatus.class); - - } - - @POST - @Path("/query/{resourceQueryId}/status") - @Override - public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusRequest) { - logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId); - checkQuery(statusRequest); - HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status"); - return readObjectFromResponse(response, QueryStatus.class); - } - - @POST - @Path("/query/{resourceQueryId}/result") - @Override - public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) { - logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId); - checkQuery(resultRequest); - HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result"); - try { - return Response.ok(response.getEntity().getContent()).build(); - } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + resultRequest.getResourceUUID() - ); - } - } - - private HttpResponse postRequest(QueryRequest statusRequest, String pathName) { - try { - QueryRequest chainRequest = createChainRequest(statusRequest); - String payload = objectMapper.writeValueAsString(chainRequest); - String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); - HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload); - if (!HttpClientUtil.is2xx(response)) { - logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); - HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); - } - return response; - } catch (IOException e) { - // Note: this shouldn't ever happen - logger.error("Error encoding search payload", e); - throw new ApplicationException( - "Error encoding search for resource with id " + statusRequest.getResourceUUID()); - } - } - - @POST - @Path("/query/sync") - @Override - public Response querySync(QueryRequest queryRequest) { - logger.debug("Calling Aggregate Data Sharing Resource querySync()"); - checkQuery(queryRequest); - - try { - Object query = queryRequest.getQuery(); - UUID resourceUUID = queryRequest.getResourceUUID(); - - JsonNode jsonNode = objectMapper.valueToTree(query); - if (!jsonNode.has("expectedResultType")) { - throw new ProtocolException(ProtocolException.MISSING_DATA); - } - String expectedResultType = jsonNode.get("expectedResultType").asText(); - - // TODO: This is authorization logic. We have an app for this. - // This splintering makes our code even harder to read - Set allowedResultTypes = Set.of( - "COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT", - "OBSERVATION_CROSS_COUNT", "CATEGORICAL_CROSS_COUNT", "CONTINUOUS_CROSS_COUNT", - "VARIANT_COUNT_FOR_QUERY", "AGGREGATE_VCF_EXCERPT" - ); - if (!allowedResultTypes.contains(expectedResultType)) { - logger.warn("Incorrect Result Type: " + expectedResultType); - return Response.status(Response.Status.BAD_REQUEST).build(); - } - - HttpResponse response = getHttpResponse(queryRequest, resourceUUID, "/query/sync", properties.getTargetPicsureUrl()); - - HttpEntity entity = response.getEntity(); - String entityString = EntityUtils.toString(entity, "UTF-8"); - String responseString = entityString; - - responseString = getExpectedResponse(expectedResultType, entityString, responseString, queryRequest); - - //propagate any metadata from the back end (e.g., resultId) - if (response.containsHeader(QUERY_METADATA_FIELD)) { - Header metadataHeader = ((Header[]) response.getHeaders(QUERY_METADATA_FIELD))[0]; - return Response.ok(responseString).header(QUERY_METADATA_FIELD, metadataHeader.getValue()).build(); - } - - return Response.ok(responseString).build(); - } catch (IOException e) { - logger.error(e.getMessage(), e); - throw new ApplicationException( - "Error encoding query for resource with id " + queryRequest.getResourceUUID()); - } catch (ClassCastException | IllegalArgumentException e) { - logger.error(e.getMessage()); - throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); - } - } - - private HttpResponse getHttpResponse(QueryRequest queryRequest, UUID resourceUUID, String pathName, String targetPicsureUrl) throws JsonProcessingException { - String queryString = objectMapper.writeValueAsString(queryRequest); - String composedURL = HttpClientUtil.composeURL(targetPicsureUrl, pathName); - - logger.debug("Aggregate Data Sharing Resource, sending query: " + queryString + ", to: " + composedURL); - HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, queryString); - if (!HttpClientUtil.is2xx(response)) { - logger.error("Not 200 status!"); - logger.error( - composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); - HttpClientUtil.throwResponseError(response, targetPicsureUrl); - } - return response; - } - - /** - * This method will process the response from the backend and return the - * expected response based on the expected result type. - * Currently, the only types that are handled are: - * COUNT, CROSS_COUNT, CATEGORICAL_CROSS_COUNT, CONTINUOUS_CROSS_COUNT - * - * @param expectedResultType The expected result type - * @param entityString The response from the backend that will be processed - * @param responseString The response that will be returned. Will return the passed entityString if - * no cases are matched. - * @return String The response that will be returned - * @throws JsonProcessingException If there is an error processing the response - */ - private String getExpectedResponse(String expectedResultType, String entityString, String responseString, QueryRequest queryRequest) throws IOException, JsonProcessingException { - String crossCountResponse; - switch (expectedResultType) { - case "COUNT": - responseString = aggregateCount(entityString).orElse(entityString); - - break; - case "CROSS_COUNT": - Map crossCounts = processCrossCounts(entityString); - responseString = objectMapper.writeValueAsString(crossCounts); - - break; - case "CATEGORICAL_CROSS_COUNT": - crossCountResponse = getCrossCountForQuery(queryRequest); - responseString = processCategoricalCrossCounts(entityString, crossCountResponse); - - break; - case "CONTINUOUS_CROSS_COUNT": - crossCountResponse = getCrossCountForQuery(queryRequest); - responseString = processContinuousCrossCounts(entityString, crossCountResponse, queryRequest); - - break; - } - return responseString; - } - - /** - * No matter what the expected result type is we will get the cross count instead. Additionally, - * it will include ALL study consents in the query. - * - * @param queryRequest The query request - * @return String The cross count for the query - */ - private String getCrossCountForQuery(QueryRequest queryRequest) throws IOException { - logger.debug("Calling Aggregate Data Sharing Resource getCrossCountForQuery()"); - - HttpResponse response = getHttpResponse(changeQueryToOpenCrossCount(queryRequest), queryRequest.getResourceUUID(), "/query/sync", properties.getTargetPicsureUrl()); - HttpEntity entity = response.getEntity(); - return EntityUtils.toString(entity, "UTF-8"); - } - - /** - * This method will add the study consents to the query. It will also set the expected result type to cross count. - * - * @param queryRequest The query request - * @return QueryRequest The query request with the study consents added and the expected result type set to cross count - */ - private QueryRequest changeQueryToOpenCrossCount(QueryRequest queryRequest) { - logger.debug("Calling Aggregate Data Sharing Resource handleAlterQueryToOpenCrossCount()"); - - Object query = queryRequest.getQuery(); - JsonNode jsonNode = objectMapper.valueToTree(query); - - JsonNode updatedExpectedResulType = setExpectedResultTypeToCrossCount(jsonNode); - JsonNode includesStudyConsents = addStudyConsentsToQuery(updatedExpectedResulType); - - LinkedHashMap rebuiltQuery = objectMapper.convertValue(includesStudyConsents, new TypeReference<>(){}); - queryRequest.setQuery(rebuiltQuery); - return queryRequest; - } - - private JsonNode setExpectedResultTypeToCrossCount(JsonNode jsonNode) { - logger.debug("Calling Aggregate Data Sharing Resource setExpectedResultTypeToCrossCount()"); - - List expectedResultTypeParents = jsonNode.findParents("expectedResultType"); - - // The reason we need to do this is that expected result type is a TextNode that is immutable. - // This is a jackson work around to replace the expectedResultType field with a new value. - for (JsonNode node : expectedResultTypeParents) { - ((ObjectNode) node).put("expectedResultType", "CROSS_COUNT"); - } - - return jsonNode; - } - - private JsonNode addStudyConsentsToQuery(JsonNode jsonNode) { - logger.debug("Calling Aggregate Data Sharing Resource addStudyConsentsToQuery()"); - - SearchResults consentResults = getAllStudyConsents(); - LinkedHashMap linkedHashMap = objectMapper.convertValue(consentResults.getResults(), new TypeReference<>() {}); - Object phenotypes = linkedHashMap.get("phenotypes"); - LinkedHashMap phenotypesLinkedHashMap = objectMapper.convertValue(phenotypes, new TypeReference<>() {}); - - // get all the keys from phenotypes - Set keys = phenotypesLinkedHashMap.keySet(); - - // create an ArrayNode to hold the keys - ArrayNode arrayNode = objectMapper.createArrayNode(); - - // add the keys to the ArrayNode - for (String key : keys) { - arrayNode.add(key); - } - - // add the ArrayNode to the query - ((ObjectNode) jsonNode).set("crossCountFields", arrayNode); - - return jsonNode; - } - - private SearchResults getAllStudyConsents() { - logger.debug("Calling Aggregate Data Sharing Resource getAllStudyConsents()"); - - QueryRequest studiesConsents = new QueryRequest(); - studiesConsents.setQuery("\\_studies_consents\\"); - return this.search(studiesConsents); - } - - @Override - @POST - @Path("/query/format") - public Response queryFormat(QueryRequest queryRequest) { - checkQuery(queryRequest); - - UUID resourceUUID = queryRequest.getResourceUUID(); - String pathName = "/query/format"; - - try { - String queryString = objectMapper.writeValueAsString(queryRequest); - String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); - HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, queryString); - if (!HttpClientUtil.is2xx(response)) { - logger.error( - composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); - HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); - } - - return Response.ok(response.getEntity().getContent()).build(); - } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + queryRequest.getResourceUUID()); - } catch (ClassCastException | IllegalArgumentException e) { - logger.error(e.getMessage()); - throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); - } - } - - private Map processCrossCounts(String entityString) throws com.fasterxml.jackson.core.JsonProcessingException { - Map crossCounts = objectMapper.readValue(entityString, new TypeReference<>(){}); - int requestVariance = generateVarianceWithCrossCounts(crossCounts); - crossCounts = obfuscateCrossCounts(crossCounts, requestVariance); - - return crossCounts; - } - - /** - * This method will appropriately process the obfuscation of the cross counts. - * - * @param crossCounts The cross counts - * @param requestVariance The variance for the request - * @return Map The obfuscated cross counts - */ - private Map obfuscateCrossCounts(Map crossCounts, int requestVariance) { - Set obfuscatedKeys = new HashSet<>(); - if(crossCounts != null) { - crossCounts.keySet().forEach(key -> { - String crossCount = crossCounts.get(key); - Optional aggregatedCount = aggregateCount(crossCount); - aggregatedCount.ifPresent((x) -> obfuscatedKeys.add(key)); - crossCounts.put(key, aggregatedCount.orElse(crossCount)); - }); - - Set obfuscatedParents = obfuscatedKeys.stream().flatMap(key -> { - return generateParents(key); - }).collect(Collectors.toSet()); - - crossCounts.keySet().forEach(key -> { - String crossCount = crossCounts.get(key); - if (!obfuscatedKeys.contains(key) && obfuscatedParents.contains(key)) { - crossCounts.put(key, randomize(crossCount, requestVariance)); - } - }); - } - - return crossCounts; - } - - /** - * This method is used to generate a variance for Cross Count queries. - * The variance is generated by taking the cross counts and sorting them by key. - * Then we generate a string with lines like consent:1\n consent:2\ consent:3\n etc. - * Then we generate a variance using the string. This is to give us a random variance that is deterministic for each - * query. - * - * @param crossCounts A map of cross counts - * @return int The variance - */ - private int generateVarianceWithCrossCounts(Map crossCounts) { - final List> entryList = new ArrayList(crossCounts.entrySet()); - - // sort the entry set. By sorting the entry set first we can ensure that the variance is the same for each run. - // This is to give us a random variance that is deterministic. - entryList.sort(Map.Entry.comparingByKey()); - - final StringBuffer crossCountsString = new StringBuffer(); - - // build a string with lines like consent:1\n consent:2\n consent:3\n etc. - entryList.forEach(entry -> crossCountsString.append(entry.getKey()).append(":").append(entry.getValue()).append("\n")); - - return generateRequestVariance(crossCountsString.toString()); - } - - /** - * This method will return an obfuscated binned count of continuous crosses. Due to the format of a continuous - * cross count, we are unable to directly obfuscate it in its original form. First, we send the continuous - * cross count data to the visualization resource to group it into bins. Once the data is binned, we assess whether - * obfuscation is necessary for this particular continuous cross count. If obfuscation is not required, we return - * the data in string format. However, if obfuscation is needed, we first obfuscate the data and then return it. - * - * @param continuousCrossCountResponse The continuous cross count response - * @param crossCountResponse The cross count response - * @param queryRequest The original query request - * @return String The obfuscated binned continuous cross count - * @throws IOException If there is an error processing the JSON - */ - protected String processContinuousCrossCounts(String continuousCrossCountResponse, String crossCountResponse, QueryRequest queryRequest) throws IOException { - logger.info("Processing continuous cross counts"); - - if (continuousCrossCountResponse == null || crossCountResponse == null) { - return null; - } - - Map crossCounts = objectMapper.readValue(crossCountResponse, new TypeReference<>(){}); - int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); - - boolean mustObfuscate = true; - - // Handle the case where there is no visualization service UUID - if (properties.getVisualizationResourceId() != null) { - // convert continuousCrossCountResponse to a map - Map> continuousCrossCounts = objectMapper.readValue(continuousCrossCountResponse, new TypeReference>>(){}); - - // Create Query for Visualization /bin/continuous - QueryRequest visualizationBinRequest = new QueryRequest(); - visualizationBinRequest.setResourceUUID(properties.getVisualizationResourceId()); - visualizationBinRequest.setQuery(continuousCrossCounts); - visualizationBinRequest.setResourceCredentials(queryRequest.getResourceCredentials()); - - Resource visResource = resourceRepository.getById(visualizationBinRequest.getResourceUUID()); - if (visResource == null) { - throw new ApplicationException("Visualization resource could not be found"); - } - - // call the binning endpoint - HttpResponse httpResponse = getHttpResponse(visualizationBinRequest, visualizationBinRequest.getResourceUUID(), "/bin/continuous", visResource.getResourceRSPath()); - HttpEntity entity = httpResponse.getEntity(); - String binResponse = EntityUtils.toString(entity, "UTF-8"); - - Map> binnedContinuousCrossCounts = objectMapper.readValue(binResponse, new TypeReference>>() {}); - - if (!mustObfuscate) { - // Ensure all inner values are Strings to be consistent in our returned data. - binnedContinuousCrossCounts.forEach( - (key, value) -> value.forEach( - (innerKey, innerValue) -> value.put(innerKey, innerValue.toString()) - ) - ); - - return objectMapper.writeValueAsString(binnedContinuousCrossCounts); - } - - obfuscatedCrossCount(generatedVariance, binnedContinuousCrossCounts); - return objectMapper.writeValueAsString(binnedContinuousCrossCounts); - } else { - // If there is no visualization service resource id, we will simply return the continuous cross count response. - - if(!mustObfuscate) { - return continuousCrossCountResponse; - } - - Map> continuousCrossCounts = objectMapper.readValue(continuousCrossCountResponse, new TypeReference>>(){}); - - // Convert continuousCrossCounts Map to a map> - Map> convertedContinuousCrossCount = new HashMap<>(); - continuousCrossCounts.forEach((key, value) -> { - Map innerMap = new HashMap<>(value); - convertedContinuousCrossCount.put(key, innerMap); - }); - - obfuscatedCrossCount(generatedVariance, convertedContinuousCrossCount); - return objectMapper.writeValueAsString(convertedContinuousCrossCount); - } - } - - /** - * This method handles the processing of categorical cross counts. It begins by determining whether the cross - * counts require obfuscation. This is accomplished by checking if any of the CROSS_COUNTS must be obfuscated. - * If obfuscation is required, the categorical cross counts will be obfuscated accordingly. Otherwise, - * if no obfuscation is needed, the method can simply return the categorical entity string. - * - * @param categoricalEntityString The categorical entity string - * @param crossCountEntityString The cross count entity string - * @return String The processed categorical entity string - * @throws JsonProcessingException If there is an error processing the JSON - */ - protected String processCategoricalCrossCounts(String categoricalEntityString, String crossCountEntityString) throws JsonProcessingException { - logger.info("Processing categorical cross counts"); - - if (categoricalEntityString == null || crossCountEntityString == null) { - return null; - } - - Map crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>(){}); - int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); - - boolean mustObfuscate = true; - if (!mustObfuscate) { - return categoricalEntityString; - } - - Map> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>(){}); - if (categoricalCrossCount == null) { - return categoricalEntityString; - } - - // Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance - obfuscatedCrossCount(generatedVariance, categoricalCrossCount); - return objectMapper.writeValueAsString(categoricalCrossCount); - } - - /** - * This method will obfuscate the cross counts based on the generated variance. We do not have a return because - * we are modifying the passed crossCount object. Java passes objects by reference value, so we do not need to return. - * - * @param generatedVariance The variance for the request - * @param crossCount The cross count that will be obfuscated - */ - private void obfuscatedCrossCount(int generatedVariance, Map> crossCount) { - crossCount.forEach((key, value) -> { - value.forEach((innerKey, innerValue) -> { - Optional aggregateCount = aggregateCountHelper(innerValue); - if (aggregateCount.isPresent()) { - value.put(innerKey, aggregateCount.get()); - } else { - value.put(innerKey, randomize(innerValue.toString(), generatedVariance)); - } - }); - }); - } - - /** - * This method will determine if the cross count needs to be obfuscated. It will return true if any of the - * cross counts are less than the threshold or if any of the cross counts have a variance. - * - * @param crossCounts - * @param generatedVariance - * @return - */ - private boolean isCrossCountObfuscated(Map crossCounts, int generatedVariance) { - String lessThanThresholdStr = "< " + this.threshold; - String varianceStr = " \u00B1" + variance; - - boolean mustObfuscate = false; - Map obfuscatedCrossCount = this.obfuscateCrossCounts(crossCounts, generatedVariance); - for (Map.Entry entry : obfuscatedCrossCount.entrySet()) { - String v = entry.getValue(); - if (v.contains(lessThanThresholdStr) || v.contains(varianceStr)) { - mustObfuscate = true; - break; - } - } - - return mustObfuscate; - } - - /** - * This method will generate a random variance for the request based on the passed entityString. The variance - * will be between -variance and +variance. The variance will be generated by adding a random salt to the - * entityString and then taking the hashcode of the result. The variance will be the hashcode mod the - * variance * 2 + 1 - variance. - * - * @return int The variance for the request - */ - private QueryRequest createChainRequest(QueryRequest queryRequest) { - QueryRequest chainRequest = new QueryRequest(); - chainRequest.setQuery(queryRequest.getQuery()); - chainRequest.setResourceCredentials(queryRequest.getResourceCredentials()); - - if(properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) { - chainRequest.setResourceUUID(UUID.fromString(properties.getTargetResourceId())); - } else { - chainRequest.setResourceUUID(queryRequest.getResourceUUID()); - } - return chainRequest; - } - - private static void checkQuery(QueryRequest searchRequest) { - if (searchRequest == null || searchRequest.getQuery() == null) { - throw new ProtocolException(ProtocolException.MISSING_DATA); - } - } - - private int generateRequestVariance(String entityString) { - return Math.abs((entityString + randomSalt).hashCode()) % (variance * 2 + 1) - variance; - } - - private String randomize(String crossCount, int requestVariance) { - return Math.max((Integer.parseInt(crossCount) + requestVariance), threshold) + " \u00B1" + variance; - } - - private Stream generateParents(String key) { - StringJoiner stringJoiner = new StringJoiner("\\", "\\", "\\"); - - String[] split = key.split("\\\\"); - if (split.length > 1) { - return Arrays.stream(Arrays.copyOfRange(split, 0, split.length - 1)) - .filter(Predicate.not(String::isEmpty)) - .map(segment -> stringJoiner.add(segment).toString()); - } - return Stream.empty(); - } - - /** - * Here's the core of this resource - make sure we do not return results with small (potentially identifiable) cohorts. - * @param actualCount - * @return - */ - private Optional aggregateCount(String actualCount) { - try { - int queryResult = Integer.parseInt(actualCount); - if (queryResult > 0 && queryResult < threshold) { - return Optional.of("< " + threshold); - } - } catch (NumberFormatException nfe) { - logger.warn("Count was not a number! " + actualCount); - } - return Optional.empty(); - } - - /** - * Helper method to handle the fact that the actualCount could be an Integer or a String. - * - * @param actualCount - * @return - */ - private Optional aggregateCountHelper(Object actualCount) { - if (actualCount instanceof Integer) { - return aggregateCount(actualCount.toString()); - } else if (actualCount instanceof String) { - return aggregateCount((String) actualCount); - } - return Optional.empty(); - } + @Inject + private ApplicationProperties properties; + + @Inject + private ResourceRepository resourceRepository; + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private final Header[] headers; + + private static final String BEARER_STRING = "Bearer "; + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final int threshold; + private final int variance; + + private final String randomSalt; + + public AggregateDataSharingResourceRS() { + this(null); + } + + @Inject + public AggregateDataSharingResourceRS(ApplicationProperties applicationProperties) { + this.properties = applicationProperties; + if (applicationProperties == null) { + logger.info("initialize Aggregate Resource NO INJECTION"); + } else { + logger.info("initialize Aggregate Resource Injected " + applicationProperties); + } + + if (properties == null) { + properties = new ApplicationProperties(); + properties.init("pic-sure-aggregate-resource"); + } + + threshold = properties.getTargetPicsureObfuscationThreshold(); + variance = properties.getTargetPicsureObfuscationVariance(); + randomSalt = properties.getTargetPicsureObfuscationSalt(); + + headers = new Header[]{new BasicHeader(HttpHeaders.AUTHORIZATION, BEARER_STRING + properties.getTargetPicsureToken())}; + } + + + @GET + @Path("/status") + public Response status() { + logger.debug("Calling Aggregate Data Sharing Resource status()"); + return Response.ok().build(); + } + + @POST + @Path("/info") + @Override + public ResourceInfo info(QueryRequest infoRequest) { + logger.debug("Calling Aggregate Data Sharing Resource info()"); + String pathName = "/info"; + + try { + QueryRequest chainRequest = new QueryRequest(); + if (infoRequest != null) { + chainRequest.setQuery(infoRequest.getQuery()); + chainRequest.setResourceCredentials(infoRequest.getResourceCredentials()); + //set a default value of the existing uuid here (can override in properties file) + chainRequest.setResourceUUID(infoRequest.getResourceUUID()); + } + if (properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) { + chainRequest.setResourceUUID(UUID.fromString(properties.getTargetResourceId())); + } + + String payload = objectMapper.writeValueAsString(chainRequest); + String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); + HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload); + if (!HttpClientUtil.is2xx(response)) { + logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); + } + + //if we are proxying an info request, we need to return our own resource ID + ResourceInfo resourceInfo = readObjectFromResponse(response, ResourceInfo.class); + if (infoRequest != null && infoRequest.getResourceUUID() != null) { + resourceInfo.setId(infoRequest.getResourceUUID()); + } + return resourceInfo; + } catch (IOException e) { + throw new ApplicationException( + "Error encoding query for resource with id " + infoRequest.getResourceUUID()); + } catch (ClassCastException | IllegalArgumentException e) { + logger.error(e.getMessage()); + throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); + } + } + + @POST + @Path("/search") + @Override + public SearchResults search(QueryRequest searchRequest) { + logger.debug("Calling Aggregate Data Sharing Search"); + checkQuery(searchRequest); + HttpResponse response = postRequest(searchRequest, "/search"); + return readObjectFromResponse(response, SearchResults.class); + } + + @POST + @Path("/query") + @Override + public QueryStatus query(QueryRequest queryRequest) { + logger.debug("Calling Aggregate Data Sharing Resource query()"); + checkQuery(queryRequest); + HttpResponse response = postRequest(queryRequest, "/query"); + return readObjectFromResponse(response, QueryStatus.class); + + } + + @POST + @Path("/query/{resourceQueryId}/status") + @Override + public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusRequest) { + logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId); + checkQuery(statusRequest); + HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status"); + return readObjectFromResponse(response, QueryStatus.class); + } + + @POST + @Path("/query/{resourceQueryId}/result") + @Override + public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) { + logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId); + checkQuery(resultRequest); + HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result"); + try { + return Response.ok(response.getEntity().getContent()).build(); + } catch (IOException e) { + throw new ApplicationException( + "Error encoding query for resource with id " + resultRequest.getResourceUUID() + ); + } + } + + private HttpResponse postRequest(QueryRequest statusRequest, String pathName) { + try { + QueryRequest chainRequest = createChainRequest(statusRequest); + String payload = objectMapper.writeValueAsString(chainRequest); + String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); + HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, payload); + if (!HttpClientUtil.is2xx(response)) { + logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); + } + return response; + } catch (IOException e) { + // Note: this shouldn't ever happen + logger.error("Error encoding search payload", e); + throw new ApplicationException( + "Error encoding search for resource with id " + statusRequest.getResourceUUID()); + } + } + + @POST + @Path("/query/sync") + @Override + public Response querySync(QueryRequest queryRequest) { + logger.debug("Calling Aggregate Data Sharing Resource querySync()"); + checkQuery(queryRequest); + + try { + Object query = queryRequest.getQuery(); + UUID resourceUUID = queryRequest.getResourceUUID(); + + JsonNode jsonNode = objectMapper.valueToTree(query); + if (!jsonNode.has("expectedResultType")) { + throw new ProtocolException(ProtocolException.MISSING_DATA); + } + String expectedResultType = jsonNode.get("expectedResultType").asText(); + + Set allowedResultTypes = Set.of( + "COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT", + "OBSERVATION_CROSS_COUNT", "CATEGORICAL_CROSS_COUNT", "CONTINUOUS_CROSS_COUNT" + ); + if (!allowedResultTypes.contains(expectedResultType)) { + logger.warn("Incorrect Result Type: " + expectedResultType); + return Response.status(Response.Status.BAD_REQUEST).build(); + } + + HttpResponse response = getHttpResponse(queryRequest, resourceUUID, "/query/sync", properties.getTargetPicsureUrl()); + + HttpEntity entity = response.getEntity(); + String entityString = EntityUtils.toString(entity, "UTF-8"); + String responseString = entityString; + + responseString = getExpectedResponse(expectedResultType, entityString, responseString, queryRequest); + + //propagate any metadata from the back end (e.g., resultId) + if (response.containsHeader(QUERY_METADATA_FIELD)) { + Header metadataHeader = ((Header[]) response.getHeaders(QUERY_METADATA_FIELD))[0]; + return Response.ok(responseString).header(QUERY_METADATA_FIELD, metadataHeader.getValue()).build(); + } + + return Response.ok(responseString).build(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + throw new ApplicationException( + "Error encoding query for resource with id " + queryRequest.getResourceUUID()); + } catch (ClassCastException | IllegalArgumentException e) { + logger.error(e.getMessage()); + throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); + } + } + + private HttpResponse getHttpResponse(QueryRequest queryRequest, UUID resourceUUID, String pathName, String targetPicsureUrl) throws JsonProcessingException { + String queryString = objectMapper.writeValueAsString(queryRequest); + String composedURL = HttpClientUtil.composeURL(targetPicsureUrl, pathName); + + logger.debug("Aggregate Data Sharing Resource, sending query: " + queryString + ", to: " + composedURL); + HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, queryString); + if (!HttpClientUtil.is2xx(response)) { + logger.error("Not 200 status!"); + logger.error( + composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + HttpClientUtil.throwResponseError(response, targetPicsureUrl); + } + return response; + } + + /** + * This method will process the response from the backend and return the + * expected response based on the expected result type. + * Currently, the only types that are handled are: + * COUNT, CROSS_COUNT, CATEGORICAL_CROSS_COUNT, CONTINUOUS_CROSS_COUNT + * + * @param expectedResultType The expected result type + * @param entityString The response from the backend that will be processed + * @param responseString The response that will be returned. Will return the passed entityString if + * no cases are matched. + * @return String The response that will be returned + * @throws JsonProcessingException If there is an error processing the response + */ + private String getExpectedResponse(String expectedResultType, String entityString, String responseString, QueryRequest queryRequest) throws IOException, JsonProcessingException { + String crossCountResponse; + switch (expectedResultType) { + case "COUNT": + responseString = aggregateCount(entityString).orElse(entityString); + + break; + case "CROSS_COUNT": + Map crossCounts = processCrossCounts(entityString); + responseString = objectMapper.writeValueAsString(crossCounts); + + break; + case "CATEGORICAL_CROSS_COUNT": + crossCountResponse = getCrossCountForQuery(queryRequest); + responseString = processCategoricalCrossCounts(entityString, crossCountResponse); + + break; + case "CONTINUOUS_CROSS_COUNT": + crossCountResponse = getCrossCountForQuery(queryRequest); + responseString = processContinuousCrossCounts(entityString, crossCountResponse, queryRequest); + + break; + } + return responseString; + } + + /** + * No matter what the expected result type is we will get the cross count instead. Additionally, + * it will include ALL study consents in the query. + * + * @param queryRequest The query request + * @return String The cross count for the query + */ + private String getCrossCountForQuery(QueryRequest queryRequest) throws IOException { + logger.debug("Calling Aggregate Data Sharing Resource getCrossCountForQuery()"); + + HttpResponse response = getHttpResponse(changeQueryToOpenCrossCount(queryRequest), queryRequest.getResourceUUID(), "/query/sync", properties.getTargetPicsureUrl()); + HttpEntity entity = response.getEntity(); + return EntityUtils.toString(entity, "UTF-8"); + } + + /** + * This method will add the study consents to the query. It will also set the expected result type to cross count. + * + * @param queryRequest The query request + * @return QueryRequest The query request with the study consents added and the expected result type set to cross count + */ + private QueryRequest changeQueryToOpenCrossCount(QueryRequest queryRequest) { + logger.debug("Calling Aggregate Data Sharing Resource handleAlterQueryToOpenCrossCount()"); + + Object query = queryRequest.getQuery(); + JsonNode jsonNode = objectMapper.valueToTree(query); + + JsonNode updatedExpectedResulType = setExpectedResultTypeToCrossCount(jsonNode); + JsonNode includesStudyConsents = addStudyConsentsToQuery(updatedExpectedResulType); + + LinkedHashMap rebuiltQuery = objectMapper.convertValue(includesStudyConsents, new TypeReference<>() { + }); + queryRequest.setQuery(rebuiltQuery); + return queryRequest; + } + + private JsonNode setExpectedResultTypeToCrossCount(JsonNode jsonNode) { + logger.debug("Calling Aggregate Data Sharing Resource setExpectedResultTypeToCrossCount()"); + + List expectedResultTypeParents = jsonNode.findParents("expectedResultType"); + + // The reason we need to do this is that expected result type is a TextNode that is immutable. + // This is a jackson work around to replace the expectedResultType field with a new value. + for (JsonNode node : expectedResultTypeParents) { + ((ObjectNode) node).put("expectedResultType", "CROSS_COUNT"); + } + + return jsonNode; + } + + private JsonNode addStudyConsentsToQuery(JsonNode jsonNode) { + logger.debug("Calling Aggregate Data Sharing Resource addStudyConsentsToQuery()"); + + SearchResults consentResults = getAllStudyConsents(); + LinkedHashMap linkedHashMap = objectMapper.convertValue(consentResults.getResults(), new TypeReference<>() { + }); + Object phenotypes = linkedHashMap.get("phenotypes"); + LinkedHashMap phenotypesLinkedHashMap = objectMapper.convertValue(phenotypes, new TypeReference<>() { + }); + + // get all the keys from phenotypes + Set keys = phenotypesLinkedHashMap.keySet(); + + // create an ArrayNode to hold the keys + ArrayNode arrayNode = objectMapper.createArrayNode(); + + // add the keys to the ArrayNode + for (String key : keys) { + arrayNode.add(key); + } + + // add the ArrayNode to the query + ((ObjectNode) jsonNode).set("crossCountFields", arrayNode); + + return jsonNode; + } + + private SearchResults getAllStudyConsents() { + logger.debug("Calling Aggregate Data Sharing Resource getAllStudyConsents()"); + + QueryRequest studiesConsents = new QueryRequest(); + studiesConsents.setQuery("\\_studies_consents\\"); + return this.search(studiesConsents); + } + + @Override + @POST + @Path("/query/format") + public Response queryFormat(QueryRequest queryRequest) { + checkQuery(queryRequest); + + UUID resourceUUID = queryRequest.getResourceUUID(); + String pathName = "/query/format"; + + try { + String queryString = objectMapper.writeValueAsString(queryRequest); + String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); + HttpResponse response = HttpClientUtil.retrievePostResponse(composedURL, headers, queryString); + if (!HttpClientUtil.is2xx(response)) { + logger.error( + composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); + } + + return Response.ok(response.getEntity().getContent()).build(); + } catch (IOException e) { + throw new ApplicationException( + "Error encoding query for resource with id " + queryRequest.getResourceUUID()); + } catch (ClassCastException | IllegalArgumentException e) { + logger.error(e.getMessage()); + throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); + } + } + + private Map processCrossCounts(String entityString) throws com.fasterxml.jackson.core.JsonProcessingException { + Map crossCounts = objectMapper.readValue(entityString, new TypeReference<>() {}); + + int requestVariance = generateVarianceWithCrossCounts(crossCounts); + crossCounts = obfuscateCrossCounts(crossCounts, requestVariance); + + return crossCounts; + } + + /** + * This method will appropriately process the obfuscation of the cross counts. + * + * @param crossCounts The cross counts + * @param requestVariance The variance for the request + * @return Map The obfuscated cross counts + */ + private Map obfuscateCrossCounts(Map crossCounts, int requestVariance) { + Set obfuscatedKeys = new HashSet<>(); + if (crossCounts != null) { + crossCounts.keySet().forEach(key -> { + String crossCount = crossCounts.get(key); + Optional aggregatedCount = aggregateCount(crossCount); + aggregatedCount.ifPresent((x) -> obfuscatedKeys.add(key)); + crossCounts.put(key, aggregatedCount.orElse(crossCount)); + }); + + Set obfuscatedParents = obfuscatedKeys.stream().flatMap(this::generateParents).collect(Collectors.toSet()); + + crossCounts.keySet().forEach(key -> { + String crossCount = crossCounts.get(key); + if (!obfuscatedKeys.contains(key) && obfuscatedParents.contains(key)) { + crossCounts.put(key, randomize(crossCount, requestVariance)); + } + }); + } + + return crossCounts; + } + + /** + * This method is used to generate a variance for Cross Count queries. + * The variance is generated by taking the cross counts and sorting them by key. + * Then we generate a string with lines like consent:1\n consent:2\ consent:3\n etc. + * Then we generate a variance using the string. This is to give us a random variance that is deterministic for each + * query. + * + * @param crossCounts A map of cross counts + * @return int The variance + */ + private int generateVarianceWithCrossCounts(Map crossCounts) { + final List> entryList = new ArrayList(crossCounts.entrySet()); + + // sort the entry set. By sorting the entry set first we can ensure that the variance is the same for each run. + // This is to give us a random variance that is deterministic. + entryList.sort(Map.Entry.comparingByKey()); + + final StringBuffer crossCountsString = new StringBuffer(); + + // build a string with lines like consent:1\n consent:2\n consent:3\n etc. + entryList.forEach(entry -> crossCountsString.append(entry.getKey()).append(":").append(entry.getValue()).append("\n")); + + return generateRequestVariance(crossCountsString.toString()); + } + + /** + * This method will return an obfuscated binned count of continuous crosses. Due to the format of a continuous + * cross count, we are unable to directly obfuscate it in its original form. First, we send the continuous + * cross count data to the visualization resource to group it into bins. Once the data is binned, we assess whether + * obfuscation is necessary for this particular continuous cross count. If obfuscation is not required, we return + * the data in string format. However, if obfuscation is needed, we first obfuscate the data and then return it. + * + * @param continuousCrossCountResponse The continuous cross count response + * @param crossCountResponse The cross count response + * @param queryRequest The original query request + * @return String The obfuscated binned continuous cross count + * @throws IOException If there is an error processing the JSON + */ + protected String processContinuousCrossCounts(String continuousCrossCountResponse, String crossCountResponse, QueryRequest queryRequest) throws IOException { + logger.info("Processing continuous cross counts"); + + if (continuousCrossCountResponse == null || crossCountResponse == null) { + return null; + } + + Map crossCounts = objectMapper.readValue(crossCountResponse, new TypeReference<>() { + }); + int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); + + boolean mustObfuscate = true; + + // Handle the case where there is no visualization service UUID + if (properties.getVisualizationResourceId() != null) { + // convert continuousCrossCountResponse to a map + Map> continuousCrossCounts = objectMapper.readValue(continuousCrossCountResponse, new TypeReference>>() { + }); + + // Create Query for Visualization /bin/continuous + QueryRequest visualizationBinRequest = new QueryRequest(); + visualizationBinRequest.setResourceUUID(properties.getVisualizationResourceId()); + visualizationBinRequest.setQuery(continuousCrossCounts); + visualizationBinRequest.setResourceCredentials(queryRequest.getResourceCredentials()); + + Resource visResource = resourceRepository.getById(visualizationBinRequest.getResourceUUID()); + if (visResource == null) { + throw new ApplicationException("Visualization resource could not be found"); + } + + // call the binning endpoint + HttpResponse httpResponse = getHttpResponse(visualizationBinRequest, visualizationBinRequest.getResourceUUID(), "/bin/continuous", visResource.getResourceRSPath()); + HttpEntity entity = httpResponse.getEntity(); + String binResponse = EntityUtils.toString(entity, "UTF-8"); + + Map> binnedContinuousCrossCounts = objectMapper.readValue(binResponse, new TypeReference>>() { + }); + + if (!mustObfuscate) { + // Ensure all inner values are Strings to be consistent in our returned data. + binnedContinuousCrossCounts.forEach( + (key, value) -> value.forEach( + (innerKey, innerValue) -> value.put(innerKey, innerValue.toString()) + ) + ); + + return objectMapper.writeValueAsString(binnedContinuousCrossCounts); + } + + obfuscatedCrossCount(generatedVariance, binnedContinuousCrossCounts); + return objectMapper.writeValueAsString(binnedContinuousCrossCounts); + } else { + // If there is no visualization service resource id, we will simply return the continuous cross count response. + + if (!mustObfuscate) { + return continuousCrossCountResponse; + } + + Map> continuousCrossCounts = objectMapper.readValue(continuousCrossCountResponse, new TypeReference>>() { + }); + + // Convert continuousCrossCounts Map to a map> + Map> convertedContinuousCrossCount = new HashMap<>(); + continuousCrossCounts.forEach((key, value) -> { + Map innerMap = new HashMap<>(value); + convertedContinuousCrossCount.put(key, innerMap); + }); + + obfuscatedCrossCount(generatedVariance, convertedContinuousCrossCount); + return objectMapper.writeValueAsString(convertedContinuousCrossCount); + } + } + + /** + * This method handles the processing of categorical cross counts. It begins by determining whether the cross + * counts require obfuscation. This is accomplished by checking if any of the CROSS_COUNTS must be obfuscated. + * If obfuscation is required, the categorical cross counts will be obfuscated accordingly. Otherwise, + * if no obfuscation is needed, the method can simply return the categorical entity string. + * + * @param categoricalEntityString The categorical entity string + * @param crossCountEntityString The cross count entity string + * @return String The processed categorical entity string + * @throws JsonProcessingException If there is an error processing the JSON + */ + protected String processCategoricalCrossCounts(String categoricalEntityString, String crossCountEntityString) throws JsonProcessingException { + logger.info("Processing categorical cross counts"); + + if (categoricalEntityString == null || crossCountEntityString == null) { + return null; + } + + Map crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() {}); + int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); + + boolean mustObfuscate = true; + if (!mustObfuscate) { + return categoricalEntityString; + } + + // This might break in the object mapper. We need to test this. + Map> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {}); + + if (categoricalCrossCount == null) { + return categoricalEntityString; + } + + for (Map.Entry> entry : categoricalCrossCount.entrySet()) { + // skipKey is expecting an entrySet, so we need to convert the axisMap to an entrySet + if (VisualizationUtil.skipKey(entry)) continue; + + Map axisMap = VisualizationUtil.processResults(entry.getValue()); + categoricalCrossCount.put(entry.getKey(), axisMap); + } + + // Convert the categoricalCrossCount Map to a map> + Map> convertedCategoricalCrossCount = new HashMap<>(); + categoricalCrossCount.forEach((key, value) -> { + Map innerMap = new HashMap<>(value); + convertedCategoricalCrossCount.put(key, innerMap); + }); + + // Now we need to obfuscate our return data. The only consideration is do we apply < threshold or variance + obfuscatedCrossCount(generatedVariance, convertedCategoricalCrossCount); + + return objectMapper.writeValueAsString(convertedCategoricalCrossCount); + } + + /** + * This method will obfuscate the cross counts based on the generated variance. We do not have a return because + * we are modifying the passed crossCount object. Java passes objects by reference value, so we do not need to return. + * + * @param generatedVariance The variance for the request + * @param crossCount The cross count that will be obfuscated + */ + private void obfuscatedCrossCount(int generatedVariance, Map> crossCount) { + crossCount.forEach((key, value) -> { + value.forEach((innerKey, innerValue) -> { + Optional aggregateCount = aggregateCountHelper(innerValue); + if (aggregateCount.isPresent()) { + value.put(innerKey, aggregateCount.get()); + } else { + value.put(innerKey, randomize(innerValue.toString(), generatedVariance)); + } + }); + }); + } + + /** + * This method will determine if the cross count needs to be obfuscated. It will return true if any of the + * cross counts are less than the threshold or if any of the cross counts have a variance. + * + * @param crossCounts + * @param generatedVariance + * @return + */ + private boolean isCrossCountObfuscated(Map crossCounts, int generatedVariance) { + String lessThanThresholdStr = "< " + this.threshold; + String varianceStr = " \u00B1" + variance; + + boolean mustObfuscate = false; + Map obfuscatedCrossCount = this.obfuscateCrossCounts(crossCounts, generatedVariance); + for (Map.Entry entry : obfuscatedCrossCount.entrySet()) { + String v = entry.getValue(); + if (v.contains(lessThanThresholdStr) || v.contains(varianceStr)) { + mustObfuscate = true; + break; + } + } + + return mustObfuscate; + } + + /** + * This method will generate a random variance for the request based on the passed entityString. The variance + * will be between -variance and +variance. The variance will be generated by adding a random salt to the + * entityString and then taking the hashcode of the result. The variance will be the hashcode mod the + * variance * 2 + 1 - variance. + * + * @return int The variance for the request + */ + private QueryRequest createChainRequest(QueryRequest queryRequest) { + QueryRequest chainRequest = new QueryRequest(); + chainRequest.setQuery(queryRequest.getQuery()); + chainRequest.setResourceCredentials(queryRequest.getResourceCredentials()); + + if (properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) { + chainRequest.setResourceUUID(UUID.fromString(properties.getTargetResourceId())); + } else { + chainRequest.setResourceUUID(queryRequest.getResourceUUID()); + } + return chainRequest; + } + + private static void checkQuery(QueryRequest searchRequest) { + if (searchRequest == null || searchRequest.getQuery() == null) { + throw new ProtocolException(ProtocolException.MISSING_DATA); + } + } + + private int generateRequestVariance(String entityString) { + return Math.abs((entityString + randomSalt).hashCode()) % (variance * 2 + 1) - variance; + } + + private String randomize(String crossCount, int requestVariance) { + return Math.max((Integer.parseInt(crossCount) + requestVariance), threshold) + " \u00B1" + variance; + } + + private Stream generateParents(String key) { + StringJoiner stringJoiner = new StringJoiner("\\", "\\", "\\"); + + String[] split = key.split("\\\\"); + if (split.length > 1) { + return Arrays.stream(Arrays.copyOfRange(split, 0, split.length - 1)) + .filter(Predicate.not(String::isEmpty)) + .map(segment -> stringJoiner.add(segment).toString()); + } + return Stream.empty(); + } + + /** + * Here's the core of this resource - make sure we do not return results with small (potentially identifiable) cohorts. + * + * @param actualCount + * @return + */ + private Optional aggregateCount(String actualCount) { + try { + int queryResult = Integer.parseInt(actualCount); + if (queryResult > 0 && queryResult < threshold) { + return Optional.of("< " + threshold); + } + } catch (NumberFormatException nfe) { + logger.warn("Count was not a number! " + actualCount); + } + return Optional.empty(); + } + + /** + * Helper method to handle the fact that the actualCount could be an Integer or a String. + * + * @param actualCount + * @return + */ + private Optional aggregateCountHelper(Object actualCount) { + if (actualCount instanceof Integer) { + return aggregateCount(actualCount.toString()); + } else if (actualCount instanceof String) { + return aggregateCount((String) actualCount); + } + return Optional.empty(); + } } diff --git a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java index 2dc36ade..15c03a1e 100644 --- a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java +++ b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/DataProcessingService.java @@ -1,6 +1,7 @@ package edu.harvard.hms.dbmi.avillach.resource.visualization.service; import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.dbmi.avillach.util.VisualizationUtil; import edu.harvard.hms.dbmi.avillach.resource.visualization.model.CategoricalData; import edu.harvard.hms.dbmi.avillach.resource.visualization.model.ContinuousData; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; @@ -18,14 +19,6 @@ public class DataProcessingService { private Logger logger = LoggerFactory.getLogger(DataProcessingService.class); - - private static final String CONSENTS_KEY = "\\_consents\\"; - private static final String HARMONIZED_CONSENT_KEY = "\\_harmonized_consent\\"; - private static final String TOPMED_CONSENTS_KEY = "\\_topmed_consents\\"; - private static final String PARENT_CONSENTS_KEY = "\\_parent_consents\\"; - private static final int MAX_X_LABEL_LINE_LENGTH = 45; - boolean LIMITED = true; - int LIMIT_SIZE = 7; private static final double THIRD = 0.3333333333333333; private RestTemplate restTemplate; @@ -59,8 +52,14 @@ private List handleGetCategoricalData(Map categoricalDataList = new ArrayList<>(); for (Map.Entry> entry : crossCountsMap.entrySet()) { - if (skipKey(entry)) continue; - Map axisMap = processResults(entry.getValue()); + if (VisualizationUtil.skipKey(entry)) continue; + + Map axisMap = null; + if (isObfuscated) { + axisMap = VisualizationUtil.processResults(entry.getValue()); + } else { + axisMap = new LinkedHashMap<>(entry.getValue()); + } String title = getChartTitle(entry.getKey()); categoricalDataList.add(new CategoricalData( @@ -75,12 +74,6 @@ private List handleGetCategoricalData(Map> entry) { - return entry.getKey().equals(CONSENTS_KEY) || - entry.getKey().equals(HARMONIZED_CONSENT_KEY) || - entry.getKey().equals(TOPMED_CONSENTS_KEY) || - entry.getKey().equals(PARENT_CONSENTS_KEY); - } /** * For each continuous cross count we create a histogram of the values. @@ -126,65 +119,9 @@ private static int calcNumBins(Map countMap) { DescriptiveStatistics da = new DescriptiveStatistics(keys); double smallestKey = da.getMin(); double largestKey = da.getMax(); - if (smallestKey == largestKey) return 1; - double binWidth = (3.5 * da.getStandardDeviation()) / Math.pow(countMap.size(),THIRD); - return (int)Math.round((largestKey - smallestKey) / binWidth); - } - - /** - * Sorts the map and if there is more than the LIMIT_SIZE then we also get the greatest 7 categories and then combines - * the others into an "other" category. Also replace long column names with shorter version. - * @param axisMap - Map of the categories and their counts - * @return Map - sorted map of the categories and their counts with the "other" category added if necessary - */ - private Map processResults(Map axisMap) { - Map finalAxisMap = axisMap; - if (LIMITED && axisMap.size() > (LIMIT_SIZE+1) ) { - //Create Other bar and sort - Supplier>> stream = () -> finalAxisMap.entrySet().stream().sorted(Collections.reverseOrder(Map.Entry.comparingByValue())); - Integer otherSum = stream.get().skip(LIMIT_SIZE).mapToInt(Map.Entry::getValue).sum(); - axisMap = stream.get().limit(LIMIT_SIZE).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e2, LinkedHashMap::new)); - axisMap = limitKeySize(axisMap).entrySet() - .stream() - .sorted(Collections.reverseOrder(Map.Entry.comparingByValue())) - .collect(Collectors.toMap( - Map.Entry::getKey, - Map.Entry::getValue, - (e1, e2) -> e2, - LinkedHashMap::new)); - axisMap.put("Other", otherSum); - } else { - axisMap = limitKeySize(finalAxisMap).entrySet() - .stream() - .sorted(Collections.reverseOrder(Map.Entry.comparingByValue())) - .collect(Collectors.toMap(Map.Entry::getKey, - Map.Entry::getValue, - (e1, e2) -> e2, - LinkedHashMap::new)); - } - return axisMap; - } - - /** - * Replaces long column names with shorter version. - * - * @param axisMap - * @return - */ - private Map limitKeySize(Map axisMap) { - List toRemove = new ArrayList<>(); - Map toAdd = new HashMap<>(); - axisMap.keySet().forEach(key -> { - if (key.length() > MAX_X_LABEL_LINE_LENGTH) { - toRemove.add(key); - toAdd.put( - key.substring(0, MAX_X_LABEL_LINE_LENGTH - 3) + "...", - axisMap.get(key)); - } - }); - toRemove.forEach(key -> axisMap.remove(key)); - axisMap.putAll(toAdd); - return axisMap; + if (smallestKey == largestKey) return 1; + double binWidth = (3.5 * da.getStandardDeviation()) / Math.pow(countMap.size(), THIRD); + return (int) Math.round((largestKey - smallestKey) / binWidth); } /** @@ -218,7 +155,7 @@ private static Map bucketData(Map originalMap) data.put(key, originalMap.get(key.toString())); } - if (data.isEmpty()) return new HashMap<>(); + if (data.isEmpty()) return new HashMap<>(); int numBins = calcNumBins(data); double min = data.keySet().stream().min(Double::compareTo).orElse(0.0); @@ -226,7 +163,7 @@ private static Map bucketData(Map originalMap) if ((min == 0.0 && max == 0.0) || numBins == 0) return new HashMap<>(); - int binSize = (int)Math.ceil((max - min) / numBins); + int binSize = (int) Math.ceil((max - min) / numBins); Map results = createBinsAndMergeCounts(data, numBins, min, binSize); @@ -249,7 +186,7 @@ private static Map bucketData(Map originalMap) //If not the last item in the map and the results map does not contain the key + 1 add a new key + 1 to the keysToAdd list if (bucket.getKey() != bucketMax && !results.containsKey(bucket.getKey() + 1)) { - keysToAdd.add(bucket.getKey()+1); + keysToAdd.add(bucket.getKey() + 1); ranges.put(bucket.getKey() + 1, new ArrayList<>()); ranges.get(bucket.getKey() + 1).add(min + ((bucket.getKey() + 1) * binSize)); ranges.get(bucket.getKey() + 1).add(min + ((bucket.getKey() + 2) * binSize) - 1); @@ -276,9 +213,9 @@ private static Map bucketData(Map originalMap) /** * Finds the bin location for each value in the data map and merges the counts for each bin. * - * @param data - Map - the data to be binned + * @param data - Map - the data to be binned * @param numBins - int - the number of bins to be created - * @param min - double - the minimum value in the data + * @param min - double - the minimum value in the data * @param binSize - int - the size of each bin * @return - Map - the new binned data */ @@ -303,7 +240,7 @@ private static Map createBinsAndMergeCounts(Map - the binned data - * @param ranges - Map> - the range of each bin + * @param ranges - Map> - the range of each bin * @return - Map - the new binned data with labels for each bin */ private static Map createLabelsForBins(Map results, Map> ranges) { @@ -335,6 +272,7 @@ private static Map createLabelsForBins(Map re /** * Creates a label for the x axis using the title of the chart. + * * @param title - String - the title of the chart * @return - String - the label for the x axis (usually the variable name) */ @@ -351,7 +289,7 @@ private String createXAxisLabel(String title) { public Map> binContinuousData(Map> continuousDataMap) { Map> continuousBucketedData = new LinkedHashMap<>(); for (Map.Entry> entry : continuousDataMap.entrySet()) { - continuousBucketedData.put(entry.getKey(), bucketData(entry.getValue())); + continuousBucketedData.put(entry.getKey(), bucketData(entry.getValue())); } return continuousBucketedData; diff --git a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/VisualizationUtil.java b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/VisualizationUtil.java new file mode 100644 index 00000000..1bb71d96 --- /dev/null +++ b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/VisualizationUtil.java @@ -0,0 +1,82 @@ +package edu.harvard.dbmi.avillach.util; + +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class VisualizationUtil { + + protected static final String CONSENTS_KEY = "\\_consents\\"; + protected static final String HARMONIZED_CONSENT_KEY = "\\_harmonized_consent\\"; + protected static final String TOPMED_CONSENTS_KEY = "\\_topmed_consents\\"; + protected static final String PARENT_CONSENTS_KEY = "\\_parent_consents\\"; + private static final int MAX_X_LABEL_LINE_LENGTH = 45; + private static final boolean LIMITED = true; + private static final int LIMIT_SIZE = 7; + + public static boolean skipKey(Map.Entry> entry) { + return entry.getKey().equals(CONSENTS_KEY) || + entry.getKey().equals(HARMONIZED_CONSENT_KEY) || + entry.getKey().equals(TOPMED_CONSENTS_KEY) || + entry.getKey().equals(PARENT_CONSENTS_KEY); + } + + /** + * Sorts the map and if there is more than the LIMIT_SIZE then we also get the greatest 7 categories and then combines + * the others into an "other" category. Also replace long column names with shorter version. + * + * @param axisMap - Map of the categories and their counts + * @return Map - sorted map of the categories and their counts with the "other" category added if necessary + */ + public static Map processResults(Map axisMap) { + Map finalAxisMap = axisMap; + if (LIMITED && axisMap.size() > (LIMIT_SIZE + 1)) { + //Create Other bar and sort + Supplier>> stream = () -> finalAxisMap.entrySet().stream().sorted(Collections.reverseOrder(Map.Entry.comparingByValue())); + Integer otherSum = stream.get().skip(LIMIT_SIZE).mapToInt(Map.Entry::getValue).sum(); + axisMap = stream.get().limit(LIMIT_SIZE).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e2, LinkedHashMap::new)); + axisMap = limitKeySize(axisMap).entrySet() + .stream() + .sorted(Collections.reverseOrder(Map.Entry.comparingByValue())) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue, + (e1, e2) -> e2, + LinkedHashMap::new)); + axisMap.put("Other", otherSum); + } else { + axisMap = limitKeySize(finalAxisMap).entrySet() + .stream() + .sorted(Collections.reverseOrder(Map.Entry.comparingByValue())) + .collect(Collectors.toMap(Map.Entry::getKey, + Map.Entry::getValue, + (e1, e2) -> e2, + LinkedHashMap::new)); + } + return axisMap; + } + + /** + * Replaces long column names with shorter version. + * + * @param axisMap + * @return + */ + private static Map limitKeySize(Map axisMap) { + List toRemove = new ArrayList<>(); + Map toAdd = new HashMap<>(); + axisMap.keySet().forEach(key -> { + if (key.length() > MAX_X_LABEL_LINE_LENGTH) { + toRemove.add(key); + toAdd.put( + key.substring(0, MAX_X_LABEL_LINE_LENGTH - 3) + "...", + axisMap.get(key)); + } + }); + toRemove.forEach(key -> axisMap.remove(key)); + axisMap.putAll(toAdd); + return axisMap; + } + +} From 4b134517d6851da5463ac1c634d2083ccc891763 Mon Sep 17 00:00:00 2001 From: Gcolon021 <34667267+Gcolon021@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:01:27 -0200 Subject: [PATCH 2/3] [ALS-4980] pic sure medium findings (#144) [ALS-4980] Add additional validation of parameters [ALS-4980] Use converter to enforce UUID --- .../AggregateDataSharingResourceRS.java | 50 +++++++++---------- ...eDataSharingResourceRSAcceptanceTests.java | 14 ++++-- .../hms/dbmi/avillach/GA4GHResourceRS.java | 4 +- .../hms/dbmi/avillach/HSAPIResourceRS.java | 4 +- .../passthru/PassThroughResourceRSTest.java | 4 +- .../dbmi/avillach/service/IResourceRS.java | 6 ++- .../service/VisualizationService.java | 7 ++- .../dbmi/avillach/util/HttpClientUtil.java | 2 +- .../util/converter/UUIDParamConverter.java | 23 +++++++++ .../converter/UUIDParamConverterProvider.java | 18 +++++++ 10 files changed, 93 insertions(+), 39 deletions(-) create mode 100644 pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverter.java create mode 100644 pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverterProvider.java diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java index 30b1894d..0e2efafc 100644 --- a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java @@ -163,31 +163,31 @@ public QueryStatus query(QueryRequest queryRequest) { } - @POST - @Path("/query/{resourceQueryId}/status") - @Override - public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusRequest) { - logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId); - checkQuery(statusRequest); - HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status"); - return readObjectFromResponse(response, QueryStatus.class); - } - - @POST - @Path("/query/{resourceQueryId}/result") - @Override - public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest resultRequest) { - logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId); - checkQuery(resultRequest); - HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result"); - try { - return Response.ok(response.getEntity().getContent()).build(); - } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + resultRequest.getResourceUUID() - ); - } - } + @POST + @Path("/query/{resourceQueryId}/status") + @Override + public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest statusRequest) { + logger.debug("Calling Aggregate Data Sharing Resource queryStatus() for query {}", queryId); + checkQuery(statusRequest); + HttpResponse response = postRequest(statusRequest, "/query/" + queryId + "/status"); + return readObjectFromResponse(response, QueryStatus.class); + } + + @POST + @Path("/query/{resourceQueryId}/result") + @Override + public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest resultRequest) { + logger.debug("Calling Aggregate Data Sharing Resource queryResult() for query {}", queryId); + checkQuery(resultRequest); + HttpResponse response = postRequest(resultRequest, "/query/" + queryId + "/result"); + try { + return Response.ok(response.getEntity().getContent()).build(); + } catch (IOException e) { + throw new ApplicationException( + "Error encoding query for resource with id " + resultRequest.getResourceUUID() + ); + } + } private HttpResponse postRequest(QueryRequest statusRequest, String pathName) { try { diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/test/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRSAcceptanceTests.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/test/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRSAcceptanceTests.java index 3899a786..81c8e169 100644 --- a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/test/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRSAcceptanceTests.java +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/test/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRSAcceptanceTests.java @@ -346,14 +346,17 @@ public void shouldPostQueryStatus() { QueryStatus expectedResponse = new QueryStatus(); expectedResponse.setResourceID(targetResourceId); + UUID randomUUID = UUID.randomUUID(); + String requestUUID = randomUUID.toString(); + ProxyPostEndpointMocker.start(wireMockRule) - .withPath("/query/aaaaaaaaaaaaaah/status") + .withPath("/query/" + requestUUID + "/status") .withRequestBody(postedRequest) .withResponseBody(expectedResponse) .withStatusCode(200) .commit(); - QueryStatus actual = subject.queryStatus("aaaaaaaaaaaaaah", originalRequest); + QueryStatus actual = subject.queryStatus(randomUUID, originalRequest); // equality isn't defined for QueryRequest, and I'm scared to define it, so let's // just compare resource IDs @@ -370,14 +373,17 @@ public void shouldPostQueryResult() { Response expectedResponse = new OutboundJaxrsResponse(Response.Status.OK, new OutboundMessageContext()); + UUID randomUUID = UUID.randomUUID(); + String requestUUID = randomUUID.toString(); + ProxyPostEndpointMocker.start(wireMockRule) - .withPath("/query/aaaaaaaaaaaaaah/result") + .withPath("/query/" + requestUUID + "/result") .withRequestBody(postedRequest) .withResponseBody(expectedResponse) .withStatusCode(200) .commit(); - Response actual = subject.queryResult("aaaaaaaaaaaaaah", originalRequest); + Response actual = subject.queryResult(randomUUID, originalRequest); assertEquals(expectedResponse.getStatus(), actual.getStatus()); } diff --git a/pic-sure-resources/pic-sure-ga4gh-dos/src/main/java/edu/harvard/hms/dbmi/avillach/GA4GHResourceRS.java b/pic-sure-resources/pic-sure-ga4gh-dos/src/main/java/edu/harvard/hms/dbmi/avillach/GA4GHResourceRS.java index e828ff71..357b1940 100644 --- a/pic-sure-resources/pic-sure-ga4gh-dos/src/main/java/edu/harvard/hms/dbmi/avillach/GA4GHResourceRS.java +++ b/pic-sure-resources/pic-sure-ga4gh-dos/src/main/java/edu/harvard/hms/dbmi/avillach/GA4GHResourceRS.java @@ -307,7 +307,7 @@ public QueryStatus query(QueryRequest queryJson) { @POST @Path("/query/{resourceQueryId}/status") @Override - public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusRequest) { + public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest statusRequest) { logger.debug("Getting status for for queryId {}", queryId); retrieveTargetUrl(statusRequest); @@ -351,7 +351,7 @@ public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, Que @POST @Path("/query/{dataObjectId}/result") @Override - public Response queryResult(@PathParam("dataObjectId") String dataObjectId, QueryRequest statusRequest) { + public Response queryResult(@PathParam("dataObjectId") UUID dataObjectId, QueryRequest statusRequest) { logger.debug("queryResult() calling dataobject/{}", dataObjectId); retrieveTargetUrl(statusRequest); diff --git a/pic-sure-resources/pic-sure-hsapi-resource/src/main/java/edu/harvard/hms/dbmi/avillach/HSAPIResourceRS.java b/pic-sure-resources/pic-sure-hsapi-resource/src/main/java/edu/harvard/hms/dbmi/avillach/HSAPIResourceRS.java index 5eab61fa..58d94262 100644 --- a/pic-sure-resources/pic-sure-hsapi-resource/src/main/java/edu/harvard/hms/dbmi/avillach/HSAPIResourceRS.java +++ b/pic-sure-resources/pic-sure-hsapi-resource/src/main/java/edu/harvard/hms/dbmi/avillach/HSAPIResourceRS.java @@ -138,7 +138,7 @@ public QueryStatus query(QueryRequest queryJson) { @POST @Path("/query/{resourceQueryId}/status") @Override - public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, QueryRequest statusQuery) { + public QueryStatus queryStatus(@PathParam("resourceQueryId") UUID queryId, QueryRequest statusQuery) { logger.debug("calling HSAPI Resource queryStatus() for query {}", queryId); throw new UnsupportedOperationException("Query status is not implemented in this resource. Please use query/sync"); @@ -147,7 +147,7 @@ public QueryStatus queryStatus(@PathParam("resourceQueryId") String queryId, Que @POST @Path("/query/{resourceQueryId}/result") @Override - public Response queryResult(@PathParam("resourceQueryId") String queryId, QueryRequest statusQuery) { + public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryRequest statusQuery) { logger.debug("calling HSAPI Resource queryResult() for query {}", queryId); throw new UnsupportedOperationException("Query result is not implemented in this resource. Please use query/sync"); } diff --git a/pic-sure-resources/pic-sure-passthrough-resource/src/test/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRSTest.java b/pic-sure-resources/pic-sure-passthrough-resource/src/test/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRSTest.java index 531c4945..578fba9e 100644 --- a/pic-sure-resources/pic-sure-passthrough-resource/src/test/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRSTest.java +++ b/pic-sure-resources/pic-sure-passthrough-resource/src/test/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRSTest.java @@ -144,7 +144,7 @@ void testQueryResult() throws Exception { when(httpClient.retrievePostResponse(anyString(), any(Header[].class), anyString())).thenReturn(httpResponse); assertThrows(ProtocolException.class, () -> { - resource.queryResult(null, null); + resource.queryResult("", null); }, "QueryID is required"); assertThrows(ProtocolException.class, () -> { @@ -185,7 +185,7 @@ void testQueryStatus() throws Exception { when(httpClient.retrievePostResponse(anyString(), any(Header[].class), anyString())).thenReturn(httpResponse); assertThrows(ProtocolException.class, () -> { - resource.queryStatus(null, null); + resource.queryStatus("", null); }, "QueryID is required"); assertThrows(ProtocolException.class, () -> { diff --git a/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/IResourceRS.java b/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/IResourceRS.java index e95a9b8c..bc5500a2 100644 --- a/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/IResourceRS.java +++ b/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/IResourceRS.java @@ -6,6 +6,8 @@ import edu.harvard.dbmi.avillach.domain.*; import io.swagger.v3.oas.annotations.Operation; +import java.util.UUID; + @Path("/pic-sure") @Produces("application/json") @Consumes("application/json") @@ -36,14 +38,14 @@ default QueryStatus query(QueryRequest queryJson) { @POST @Path("/query/{resourceQueryId}/status") @Operation(hidden = true) - default QueryStatus queryStatus(String queryId, QueryRequest statusRequest) { + default QueryStatus queryStatus(UUID queryId, QueryRequest statusRequest) { throw new NotSupportedException(); } @POST @Path("/query/{resourceQueryId}/result") @Operation(hidden = true) - default Response queryResult(String queryId, QueryRequest resultRequest) { + default Response queryResult(UUID queryId, QueryRequest resultRequest) { throw new NotSupportedException(); } diff --git a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/VisualizationService.java b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/VisualizationService.java index b00e9504..ae21d4a2 100644 --- a/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/VisualizationService.java +++ b/pic-sure-resources/pic-sure-visualization-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/visualization/service/VisualizationService.java @@ -231,8 +231,13 @@ private Map> getOpenContinuousCrossCounts(QueryReque * @return Response - the binned data */ public Response generateContinuousBin(QueryRequest continuousData) { + // validate the continuous data + if (continuousData == null || continuousData.getQuery() == null) { + return Response.status(Response.Status.BAD_REQUEST).entity("Continuous data is required.").build(); + } + logger.info("Continuous data: " + continuousData.getQuery()); - Map> continuousDataMap = mapper.convertValue(continuousData.getQuery(), new TypeReference>>() {}); + Map> continuousDataMap = mapper.convertValue(continuousData.getQuery(), new TypeReference<>() {}); Map> continuousProcessedData = dataProcessingServices.binContinuousData(continuousDataMap); return Response.ok(continuousProcessedData).build(); } diff --git a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/HttpClientUtil.java b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/HttpClientUtil.java index ebaff1f4..b912ed33 100644 --- a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/HttpClientUtil.java +++ b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/HttpClientUtil.java @@ -335,7 +335,7 @@ public static JsonNode simpleGetWithConfig( private static HttpClient getConfiguredHttpClient() { try { SSLConnectionSocketFactory.getSocketFactory(); - SSLContext sslContext = SSLContext.getInstance("TLS"); + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); sslContext.init(null, null, null); String[] defaultCiphers = sslContext.getServerSocketFactory().getDefaultCipherSuites(); diff --git a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverter.java b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverter.java new file mode 100644 index 00000000..9bc7adb8 --- /dev/null +++ b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverter.java @@ -0,0 +1,23 @@ +package edu.harvard.dbmi.avillach.util.converter; + +import edu.harvard.dbmi.avillach.util.exception.ProtocolException; + +import javax.ws.rs.ext.ParamConverter; +import java.util.UUID; + +public class UUIDParamConverter implements ParamConverter { + + @Override + public UUID fromString(String value) { + try { + return UUID.fromString(value); + } catch (IllegalArgumentException e) { + throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); + } + } + + @Override + public String toString(UUID value) { + return value.toString(); + } +} diff --git a/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverterProvider.java b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverterProvider.java new file mode 100644 index 00000000..05b80495 --- /dev/null +++ b/pic-sure-util/src/main/java/edu/harvard/dbmi/avillach/util/converter/UUIDParamConverterProvider.java @@ -0,0 +1,18 @@ +package edu.harvard.dbmi.avillach.util.converter; + +import javax.ws.rs.ext.ParamConverter; +import javax.ws.rs.ext.ParamConverterProvider; +import javax.ws.rs.ext.Provider; +import java.util.UUID; + +@Provider +public class UUIDParamConverterProvider implements ParamConverterProvider { + + @Override + public ParamConverter getConverter(Class rawType, java.lang.reflect.Type genericType, java.lang.annotation.Annotation[] annotations) { + if (rawType.equals(UUID.class)) { + return (ParamConverter) new UUIDParamConverter(); + } + return null; + } +} From f2dc837a57aaa95036dcbf4690d2bda575fd795e Mon Sep 17 00:00:00 2001 From: Samantha Date: Fri, 22 Sep 2023 14:19:37 -0400 Subject: [PATCH 3/3] [ALS-4736] Add named dataset (#146) * Add table and API for naming a dataset id (#132) * Adds named_dataset sql table for saving named dataset ids. * Adds a SecurityContext class to persist authenticated user information for the entire request. * Updates JWTFilter class to add the new SecurityContext to the context object. * Adds classes for new service, entity, and endpoints needed. * Add metadata column to named_dataset table to extend in the future. (#135) * Update migration file name. (#143) --- pic-sure-api-data/pom.xml | 5 + .../dbmi/avillach/data/entity/AuthUser.java | 69 +++ .../avillach/data/entity/NamedDataset.java | 101 ++++ .../dbmi/avillach/data/entity/User.java | 43 -- .../data/entity/convert/JsonConverter.java | 50 ++ .../repository/NamedDatasetRepository.java | 13 + .../data/request/NamedDatasetRequest.java | 64 ++ .../db/sql/V5__ADD_NAMED_DATASET_TABLE.sql | 13 + .../edu/harvard/dbmi/avillach/DataTest.java | 52 +- .../harvard/dbmi/avillach/NamedDatasetRS.java | 184 ++++++ .../security/AuthSecurityContext.java | 47 ++ .../dbmi/avillach/security/JWTFilter.java | 558 +++++++++--------- .../avillach/service/NamedDatasetService.java | 110 ++++ .../dbmi/avillach/service/SystemService.java | 4 - .../main/resources/META-INF/persistence.xml | 2 +- .../dbmi/avillach/security/JWTFilterTest.java | 38 +- .../service/NamedDatasetServiceTest.java | 367 ++++++++++++ .../avillach/service/SystemServiceTest.java | 4 +- .../main/resources/META-INF/persistence.xml | 3 +- 19 files changed, 1356 insertions(+), 371 deletions(-) create mode 100755 pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/AuthUser.java create mode 100644 pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/NamedDataset.java delete mode 100755 pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/User.java create mode 100644 pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/convert/JsonConverter.java create mode 100644 pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/repository/NamedDatasetRepository.java create mode 100644 pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/request/NamedDatasetRequest.java create mode 100644 pic-sure-api-data/src/main/resources/db/sql/V5__ADD_NAMED_DATASET_TABLE.sql create mode 100644 pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/NamedDatasetRS.java create mode 100644 pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/AuthSecurityContext.java create mode 100644 pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/NamedDatasetService.java create mode 100644 pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/NamedDatasetServiceTest.java diff --git a/pic-sure-api-data/pom.xml b/pic-sure-api-data/pom.xml index e29e6118..55b6eecd 100755 --- a/pic-sure-api-data/pom.xml +++ b/pic-sure-api-data/pom.xml @@ -37,5 +37,10 @@ org.hibernate hibernate-core + + io.swagger.core.v3 + swagger-annotations + 2.2.8 + diff --git a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/AuthUser.java b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/AuthUser.java new file mode 100755 index 00000000..629ad310 --- /dev/null +++ b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/AuthUser.java @@ -0,0 +1,69 @@ +package edu.harvard.dbmi.avillach.data.entity; + +import java.security.Principal; + +import javax.json.Json; + +/* + * This class is used to mirror the User object from the auth DB to maintain schema separation. - nc + */ +public class AuthUser extends BaseEntity implements Principal { + private String userId; + + private String subject; + + private String roles; + + private String email; + + public String getUserId() { + return userId; + } + + public AuthUser setUserId(String userId) { + this.userId = userId; + return this; + } + + public String getSubject() { + return subject; + } + + public AuthUser setSubject(String subject) { + this.subject = subject; + return this; + } + + public String getRoles() { + return roles; + } + + public AuthUser setRoles(String roles) { + this.roles = roles; + return this; + } + + public String getEmail(){ + return email; + } + + public AuthUser setEmail(String email){ + this.email = email; + return this; + } + + @Override // Principal method + public String getName() { + return getEmail(); + } + + @Override + public String toString() { + return Json.createObjectBuilder() + .add("userId", userId) + .add("subject", subject) + .add("email", email) + .add("roles", roles) + .build().toString(); + } +} diff --git a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/NamedDataset.java b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/NamedDataset.java new file mode 100644 index 00000000..23c9f762 --- /dev/null +++ b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/NamedDataset.java @@ -0,0 +1,101 @@ +package edu.harvard.dbmi.avillach.data.entity; + +import java.util.Map; + +import javax.json.Json; +import javax.persistence.Column; +import javax.persistence.Convert; +import javax.persistence.Entity; +import javax.persistence.JoinColumn; +import javax.persistence.OneToOne; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; + +import io.swagger.v3.oas.annotations.media.Schema; + +import edu.harvard.dbmi.avillach.data.entity.convert.JsonConverter; + +@Schema(description = "A NamedDataset object containing query, name, user, and archived status.") +@Entity(name = "named_dataset") +@Table(uniqueConstraints = { + @UniqueConstraint(name = "unique_queryId_user", columnNames = { "queryId", "user" }) +}) +public class NamedDataset extends BaseEntity { + @Schema(description = "The associated Query") + @OneToOne + @JoinColumn(name = "queryId") + private Query query; + + @Schema(description = "The user identifier") + @Column(length = 255) + private String user; + + @Schema(description = "The name user has assigned to this dataset") + @Column(length = 255) + private String name; + + @Schema(description = "The archived state") + private Boolean archived = false; + + @Schema(description = "A json string object containing override specific values") + @Column(length = 8192) + @Convert(converter = JsonConverter.class) + private Map metadata; + + public NamedDataset setName(String name) { + this.name = name; + return this; + } + + public String getName() { + return name; + } + + public NamedDataset setArchived(Boolean archived) { + this.archived = archived; + return this; + } + + public Boolean getArchived() { + return archived; + } + + public NamedDataset setQuery(Query query) { + this.query = query; + return this; + } + + public Query getQuery(){ + return query; + } + + public NamedDataset setUser(String user) { + this.user = user; + return this; + } + + public String getUser(){ + return user; + } + + public Map getMetadata(){ + return metadata; + } + + public NamedDataset setMetadata(Map metadata){ + this.metadata = metadata; + return this; + } + + @Override + public String toString() { + return Json.createObjectBuilder() + .add("uuid", uuid.toString()) + .add("name", name) + .add("archived", archived) + .add("queryId", query.getUuid().toString()) + .add("user", user) + .add("metadata", metadata.toString()) + .build().toString(); + } +} diff --git a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/User.java b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/User.java deleted file mode 100755 index 3c07c542..00000000 --- a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/User.java +++ /dev/null @@ -1,43 +0,0 @@ -package edu.harvard.dbmi.avillach.data.entity; - - -/* - * This class gets created as part of the pic-sure DB schema, but no objects of this type are ever persisted. - * Its use is to mirror the User object from the auth DB to maintain schema separation. - nc - * - */ -public class User extends BaseEntity { - - private String userId; - - private String subject; - - private String roles; - - public String getUserId() { - return userId; - } - - public User setUserId(String userId) { - this.userId = userId; - return this; - } - - public String getSubject() { - return subject; - } - - public User setSubject(String subject) { - this.subject = subject; - return this; - } - - public String getRoles() { - return roles; - } - - public User setRoles(String roles) { - this.roles = roles; - return this; - } -} diff --git a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/convert/JsonConverter.java b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/convert/JsonConverter.java new file mode 100644 index 00000000..77d7decc --- /dev/null +++ b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/entity/convert/JsonConverter.java @@ -0,0 +1,50 @@ +package edu.harvard.dbmi.avillach.data.entity.convert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import javax.persistence.AttributeConverter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class JsonConverter implements AttributeConverter, String> { + private final Logger logger = LoggerFactory.getLogger(JsonConverter.class); + + @Override + public String convertToDatabaseColumn(Map objectData) { + if (objectData == null) { + return "{}"; + } + + String jsonData = null; + try { + jsonData = new ObjectMapper().writeValueAsString(objectData); + } catch (final JsonProcessingException e) { + logger.error("JSON writing error", e); + } + + return jsonData; + } + + @Override + public Map convertToEntityAttribute(String jsonData) { + if (jsonData == null) { + return new HashMap(); + } + + Map objectData = null; + try { + objectData = new ObjectMapper().readValue(jsonData, new TypeReference>() {}); + } catch (final IOException e) { + logger.error("JSON reading error", e); + } + + return objectData; + } +} \ No newline at end of file diff --git a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/repository/NamedDatasetRepository.java b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/repository/NamedDatasetRepository.java new file mode 100644 index 00000000..5dd6ffa3 --- /dev/null +++ b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/repository/NamedDatasetRepository.java @@ -0,0 +1,13 @@ +package edu.harvard.dbmi.avillach.data.repository; + +import edu.harvard.dbmi.avillach.data.entity.NamedDataset; + +import javax.enterprise.context.ApplicationScoped; +import javax.transaction.Transactional; +import java.util.UUID; + +@Transactional +@ApplicationScoped +public class NamedDatasetRepository extends BaseRepository{ + protected NamedDatasetRepository() {super(NamedDataset.class);} +} diff --git a/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/request/NamedDatasetRequest.java b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/request/NamedDatasetRequest.java new file mode 100644 index 00000000..1f35dd3f --- /dev/null +++ b/pic-sure-api-data/src/main/java/edu/harvard/dbmi/avillach/data/request/NamedDatasetRequest.java @@ -0,0 +1,64 @@ +package edu.harvard.dbmi.avillach.data.request; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; + +import io.swagger.v3.oas.annotations.media.Schema; + +@Schema( + description = "Request to add or update a named dataset.", + example = "{\n" + // + " \"queryId\": \"ec780aeb-d981-432a-b72b-51d4ecb3fd53\",\n" + // + " \"name\": \"My first Query\",\n" + // + " \"archived\": false\n" + // + " \"metadata\": {}\n" + // + "}" +) +public class NamedDatasetRequest { + @NotNull + private UUID queryId; + + @NotNull + @Pattern(regexp = "^[\\w\\d \\-\\\\/?+=\\[\\].():\"']+$") + private String name; + + private Boolean archived = false; + + private Map metadata = new HashMap(); + + public UUID getQueryId() { + return queryId; + } + + public void setQueryId(UUID query) { + this.queryId = query; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Boolean getArchived(){ + return archived; + } + + public void setArchived(Boolean archived) { + this.archived = archived; + } + + public Map getMetadata(){ + return metadata; + } + + public void setMetadata(Map metadata){ + this.metadata = metadata; + } +} diff --git a/pic-sure-api-data/src/main/resources/db/sql/V5__ADD_NAMED_DATASET_TABLE.sql b/pic-sure-api-data/src/main/resources/db/sql/V5__ADD_NAMED_DATASET_TABLE.sql new file mode 100644 index 00000000..2ae56962 --- /dev/null +++ b/pic-sure-api-data/src/main/resources/db/sql/V5__ADD_NAMED_DATASET_TABLE.sql @@ -0,0 +1,13 @@ +USE `picsure`; + +CREATE TABLE `named_dataset` ( + `uuid` binary(16) NOT NULL, + `queryId` binary(16) NOT NULL, + `user` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `name` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `archived` bit(1) NOT NULL DEFAULT FALSE, + `metadata` TEXT, + PRIMARY KEY (`uuid`), + CONSTRAINT `foreign_queryId` FOREIGN KEY (`queryId`) REFERENCES `query` (`uuid`), + CONSTRAINT `unique_queryId_user` UNIQUE (`queryId`, `user`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; diff --git a/pic-sure-api-data/src/test/java/edu/harvard/dbmi/avillach/DataTest.java b/pic-sure-api-data/src/test/java/edu/harvard/dbmi/avillach/DataTest.java index c880e737..32279163 100755 --- a/pic-sure-api-data/src/test/java/edu/harvard/dbmi/avillach/DataTest.java +++ b/pic-sure-api-data/src/test/java/edu/harvard/dbmi/avillach/DataTest.java @@ -1,26 +1,26 @@ -package edu.harvard.dbmi.avillach; - -import edu.harvard.dbmi.avillach.data.entity.BaseEntity; -import edu.harvard.dbmi.avillach.data.entity.User; -import org.junit.Test; - -import java.util.UUID; - -import static org.junit.Assert.*; - -/** - * Unit test for simple App. - */ -public class DataTest { - - @Test - public void BaseEntityBasicFunctionsTest() { - BaseEntity user = new User(); - user.setUuid(UUID.fromString("6ef9387a-4cde-4253-bd47-0bdc74ff76ab")); - - BaseEntity user2 = new User(); - user2.setUuid(UUID.fromString("6ef9387a-4cde-4253-bd47-0bdc74ff76ab")); - - assertEquals(user, user2); - } -} +package edu.harvard.dbmi.avillach; + +import edu.harvard.dbmi.avillach.data.entity.BaseEntity; +import edu.harvard.dbmi.avillach.data.entity.AuthUser; +import org.junit.Test; + +import java.util.UUID; + +import static org.junit.Assert.*; + +/** + * Unit test for simple App. + */ +public class DataTest { + + @Test + public void BaseEntityBasicFunctionsTest() { + BaseEntity user = new AuthUser(); + user.setUuid(UUID.fromString("6ef9387a-4cde-4253-bd47-0bdc74ff76ab")); + + BaseEntity user2 = new AuthUser(); + user2.setUuid(UUID.fromString("6ef9387a-4cde-4253-bd47-0bdc74ff76ab")); + + assertEquals(user, user2); + } +} diff --git a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/NamedDatasetRS.java b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/NamedDatasetRS.java new file mode 100644 index 00000000..363ce129 --- /dev/null +++ b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/NamedDatasetRS.java @@ -0,0 +1,184 @@ +package edu.harvard.dbmi.avillach; + +import java.util.UUID; + +import javax.inject.Inject; + +import javax.validation.Valid; + +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.SecurityContext; + +import edu.harvard.dbmi.avillach.data.entity.NamedDataset; +import edu.harvard.dbmi.avillach.data.request.NamedDatasetRequest; +import edu.harvard.dbmi.avillach.service.NamedDatasetService; +import edu.harvard.dbmi.avillach.util.response.PICSUREResponse; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; + +@Path("/dataset/named") +@Produces("application/json") +@Consumes("application/json") +public class NamedDatasetRS { + @Inject + NamedDatasetService namedDatasetService; + + @GET + @Path("/") + @Operation( + summary = "Returns a list of named datasets saved by the authenticated user.", + tags = { "dataset" }, + operationId = "namedDataset", + responses = { + @ApiResponse( + responseCode = "200", + description = "A list of named datasets saved by the authenticated user.", + content = @Content( + schema = @Schema( + implementation = NamedDataset.class + ) + ) + ), + @ApiResponse( + responseCode = "500", + description = "Error finding any named datasets for user.", + content = @Content( + examples = {@ExampleObject( + name = "namedDatasets", + value = "{\"errorType\":\"error\",\"message\":\"Could not retrieve named datasets\"}" + )} + ) + ) + } + ) + public Response namedDatasets( + @Context SecurityContext context + ) { + String user = context.getUserPrincipal().getName(); + return namedDatasetService.getNamedDatasets(user) + .map(PICSUREResponse::success) + .orElse(PICSUREResponse.error("Could not retrieve named datasets")); + } + + @POST + @Path("/") + @Operation( + summary = "Returns a named dataset saved by the authenticated user.", + tags = { "dataset" }, + operationId = "addNamedDataset", + responses = { + @ApiResponse( + responseCode = "200", + description = "The named dataset saved by the authenticated user.", + content = @Content( + schema = @Schema( + implementation = NamedDataset.class + ) + ) + ), + @ApiResponse( + responseCode = "500", + description = "Error adding any named datasets.", + content = @Content( + examples = {@ExampleObject( + name = "error", + value = "{\"errorType\":\"error\",\"message\":\"Could not save named dataset\"}" + )} + ) + ) + } + ) + public Response addNamedDataset( + @Context SecurityContext context, + @Parameter @Valid NamedDatasetRequest request + ) { + String user = context.getUserPrincipal().getName(); + return namedDatasetService.addNamedDataset(user, request) + .map(PICSUREResponse::success) + .orElse(PICSUREResponse.error("Could not save named dataset")); + } + + @GET + @Path("/{namedDatasetId}/") + @Operation( + summary = "Returns a named dataset requested by the authenticated user.", + tags = { "dataset" }, + operationId = "getNamedDatasetById", + responses = { + @ApiResponse( + responseCode = "200", + description = "The named dataset requested by the authenticated user.", + content = @Content( + schema = @Schema( + implementation = NamedDataset.class + ) + ) + ), + @ApiResponse( + responseCode = "500", + description = "Error finding the named dataset.", + content = @Content( + examples = {@ExampleObject( + name = "error", + value = "{\"errorType\":\"error\",\"message\":\"Could not retrieve named dataset\"}" + )} + ) + ) + } + ) + public Response getNamedDatasetById( + @Context SecurityContext context, + @PathParam("namedDatasetId") UUID datasetId + ){ + String user = context.getUserPrincipal().getName(); + return namedDatasetService.getNamedDatasetById(user, datasetId) + .map(PICSUREResponse::success) + .orElse(PICSUREResponse.error("Could not retrieve named dataset")); + } + + @PUT + @Path("/{namedDatasetId}/") + @Operation( + summary = "Updates a named dataset that the authenticated user perviously saved.", + tags = { "dataset" }, + operationId = "updateNamedDataset", + responses = { + @ApiResponse( + responseCode = "200", + description = "The named dataset updated by the authenticated user.", + content = @Content( + schema = @Schema( + implementation = NamedDataset.class + ) + ) + ), + @ApiResponse( + responseCode = "500", + description = "Error updating the named dataset.", + content = @Content( + examples = {@ExampleObject( + name = "error", + value = "{\"errorType\":\"error\",\"message\":\"Could not update named dataset\"}" + )} + ) + ) + } + ) + public Response updateNamedDataset( + @Context SecurityContext context, + @PathParam("namedDatasetId") UUID datasetId, + @Parameter @Valid NamedDatasetRequest request + ){ + String user = context.getUserPrincipal().getName(); + return namedDatasetService.updateNamedDataset(user, datasetId, request) + .map(PICSUREResponse::success) + .orElse(PICSUREResponse.error("Could not update named dataset")); + } +} diff --git a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/AuthSecurityContext.java b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/AuthSecurityContext.java new file mode 100644 index 00000000..2f692fa1 --- /dev/null +++ b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/AuthSecurityContext.java @@ -0,0 +1,47 @@ +package edu.harvard.dbmi.avillach.security; + +import edu.harvard.dbmi.avillach.data.entity.AuthUser; + +import javax.json.Json; +import javax.ws.rs.core.SecurityContext; +import java.security.Principal; + +public class AuthSecurityContext implements SecurityContext { + private AuthUser user; + private String scheme; + + public AuthSecurityContext(AuthUser user, String scheme) { + this.user = user; + this.scheme = scheme; + } + + @Override + public Principal getUserPrincipal() { + return this.user; + } + + @Override + public boolean isUserInRole(String role) { + if (user.getRoles() != null) + return user.getRoles().contains(role); + return false; + } + + @Override + public boolean isSecure() { + return "https".equals(this.scheme); + } + + @Override + public String getAuthenticationScheme() { + return SecurityContext.DIGEST_AUTH; + } + + @Override + public String toString(){ + return Json.createObjectBuilder() + .add("scheme", scheme) + .add("user", user.getName()) + .build().toString(); + } +} \ No newline at end of file diff --git a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/JWTFilter.java b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/JWTFilter.java index b5d945f2..82709920 100755 --- a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/JWTFilter.java +++ b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/security/JWTFilter.java @@ -1,277 +1,281 @@ -package edu.harvard.dbmi.avillach.security; - -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import edu.harvard.dbmi.avillach.PicSureWarInit; -import edu.harvard.dbmi.avillach.data.entity.Query; -import edu.harvard.dbmi.avillach.data.entity.User; -import edu.harvard.dbmi.avillach.data.repository.QueryRepository; -import edu.harvard.dbmi.avillach.data.repository.ResourceRepository; -import edu.harvard.dbmi.avillach.domain.QueryRequest; -import edu.harvard.dbmi.avillach.service.ResourceWebClient; -import edu.harvard.dbmi.avillach.util.exception.ApplicationException; -import edu.harvard.dbmi.avillach.util.response.PICSUREResponse; -import org.apache.commons.io.IOUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Resource; -import javax.inject.Inject; -import javax.ws.rs.HttpMethod; -import javax.ws.rs.NotAuthorizedException; -import javax.ws.rs.container.ContainerRequestContext; -import javax.ws.rs.container.ContainerRequestFilter; -import javax.ws.rs.container.ResourceInfo; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.UriInfo; -import javax.ws.rs.ext.Provider; - -import java.io.*; -import java.util.*; - -import static edu.harvard.dbmi.avillach.util.Utilities.buildHttpClientContext; - -@Provider -public class JWTFilter implements ContainerRequestFilter { - - private final Logger logger = LoggerFactory.getLogger(JWTFilter.class); - - @Context - UriInfo uriInfo; - - @Context - ResourceInfo resourceInfo; - - @Inject - ResourceRepository resourceRepo; - - @Inject - ResourceWebClient resourceWebClient; - - @Resource(mappedName = "java:global/user_id_claim") - private String userIdClaim; - - ObjectMapper mapper = new ObjectMapper(); - - @Inject - PicSureWarInit picSureWarInit; - - @Inject - QueryRepository queryRepo; - - @Override - public void filter(ContainerRequestContext requestContext) throws IOException { - logger.debug("Entered jwtfilter.filter()..."); - - if (uriInfo.getPath().endsWith("/openapi.json")) { - return; - } - - if(requestContext.getUriInfo().getPath().contentEquals("/system/status") - && requestContext.getRequest().getMethod().contentEquals(HttpMethod.GET)) { - // GET calls to /system/status do not require authentication or authorization - requestContext.setProperty("username", "SYSTEM_MONITOR"); - }else { - // Everything else goes through PSAMA token introspection - String authorizationHeader = requestContext.getHeaderString(HttpHeaders.AUTHORIZATION); - if (authorizationHeader == null || authorizationHeader.isEmpty()) { - throw new NotAuthorizedException("No authorization header found."); - } - String token = authorizationHeader.substring(6).trim(); - - String userForLogging = null; - - try { - User authenticatedUser = null; - - authenticatedUser = callTokenIntroEndpoint(requestContext, token, userIdClaim); - - if (authenticatedUser == null) { - logger.error("Cannot extract a user from token: " + token); - throw new NotAuthorizedException("Cannot find or create a user"); - } - - userForLogging = authenticatedUser.getUserId(); - - //The request context wants to remember who the user is - requestContext.setProperty("username", userForLogging); - - logger.info("User - " + userForLogging + " - has just passed all the authentication and authorization layers."); - - } catch (NotAuthorizedException e) { - // the detail of this exception should be logged right before the exception thrown out - // logger.error("User - " + userForLogging + " - is not authorized. " + e.getChallenges()); - // we should show different response based on role - requestContext.abortWith(PICSUREResponse.unauthorizedError("User is not authorized. " + e.getChallenges())); - } catch (Exception e){ - // we should show different response based on role - e.printStackTrace(); - requestContext.abortWith(PICSUREResponse.applicationError("Inner application error, please contact system admin")); - } - } - } - - /** - * - * @param token - * @param userIdClaim - * @return - * @throws IOException - */ - - private User callTokenIntroEndpoint(ContainerRequestContext requestContext, String token, String userIdClaim) { - logger.debug("TokenIntrospection - extractUserFromTokenIntrospection() starting..."); - - String token_introspection_url = picSureWarInit.getToken_introspection_url(); - String token_introspection_token = picSureWarInit.getToken_introspection_token(); - - if (token_introspection_url.isEmpty()) - throw new ApplicationException("token_introspection_url is empty"); - - if (token_introspection_token.isEmpty()){ - throw new ApplicationException("token_introspection_token is empty"); - } - - ObjectMapper json = PicSureWarInit.objectMapper; - CloseableHttpClient client = PicSureWarInit.CLOSEABLE_HTTP_CLIENT; - - HttpPost post = new HttpPost(token_introspection_url); - - Map tokenMap = new HashMap<>(); - tokenMap.put("token", token); - - - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - HashMap requestMap = new HashMap(); - try { - String requestPath = requestContext.getUriInfo().getPath(); - requestMap.put("Target Service", requestPath); - - Query initialQuery = null; - //Read the query from the backing store if we are getting the results (full query may not be specified in request) - if(requestPath.startsWith("/query/") && (requestPath.endsWith("result") || requestPath.endsWith("result/"))) { - //Path: /query/{queryId}/result - String[] pathParts = requestPath.split("/"); - UUID uuid = UUID.fromString(pathParts[2]); - initialQuery = queryRepo.getById(uuid); - } - - if(initialQuery != null) { - IOUtils.copy(new ByteArrayInputStream(initialQuery.getQuery().getBytes()), buffer); - } else { - //This stream is only consumable once, so we need to save & reset it. - InputStream entityStream = requestContext.getEntityStream(); - IOUtils.copy(entityStream, buffer); - requestContext.setEntityStream(new ByteArrayInputStream(buffer.toByteArray())); - } - - if(buffer.size()>0) { - /* - * We remove the resourceCredentials from the token introspection copy of the query to prevent logging them as - * part of token introspection. These credentials are between the backing resource and the user, PIC-SURE should - * do its best to keep them confidential. - */ - Object queryObject = new ObjectMapper().readValue(new ByteArrayInputStream(buffer.toByteArray()), Object.class); - if (queryObject instanceof Collection) { - for (Object query: (Collection)queryObject) { - if (query instanceof Map) { - ((Map) query).remove("resourceCredentials"); - } - } - } else if (queryObject instanceof Map){ - ((Map) queryObject).remove("resourceCredentials"); - } - requestMap.put("query", queryObject); - - if(requestPath.startsWith("/query/")) { - - UUID resourceUUID = null; - String resourceUUIDStr = (String) ((Map)queryObject).get("resourceUUID"); - if(resourceUUIDStr != null) { - resourceUUID = UUID.fromString(resourceUUIDStr); - } - - if(resourceUUID != null) { - edu.harvard.dbmi.avillach.data.entity.Resource resource = resourceRepo.getById(resourceUUID); - //logger.info("resource obj: " + resource + " path: " + resource.getResourceRSPath()); - if (resource != null && resource.getResourceRSPath() != null){ - QueryRequest queryRequest = new QueryRequest(); - queryRequest.getResourceCredentials().put(ResourceWebClient.BEARER_TOKEN_KEY, resource.getToken()); - queryRequest.setResourceUUID(resourceUUID); - queryRequest.setQuery(((Map)queryObject).get("query")); - - Response formatResponse = resourceWebClient.queryFormat(resource.getResourceRSPath(), queryRequest); - if(formatResponse.getStatus() == 200) { - //add the formatted query if available - String formattedQuery = IOUtils.toString((InputStream)formatResponse.getEntity(), "UTF-8"); - logger.debug("Formatted response: " + formattedQuery); - requestMap.put("formattedQuery", formattedQuery); - } - } - } - } - } - tokenMap.put("request", requestMap); - } catch (JsonParseException ex) { - requestMap.put("query",buffer.toString()); - tokenMap.put("request", requestMap); - } catch (IOException e1) { - logger.error("IOException caught trying to build requestMap for auditing.", e1); - throw new NotAuthorizedException("The request could not be properly audited. If you recieve this error multiple times, please contact an administrator."); - } - StringEntity entity = null; - try { - entity = new StringEntity(json.writeValueAsString(tokenMap)); - } catch (IOException e) { - logger.error("callTokenIntroEndpoint() - " + e.getClass().getSimpleName() + " when composing post"); - return null; - } - post.setEntity(entity); - post.setHeader("Content-Type", "application/json"); - //Authorize into the token introspection endpoint - post.setHeader("Authorization", "Bearer " + token_introspection_token); - CloseableHttpResponse response = null; - try { - response = client.execute(post, buildHttpClientContext()); - if (response.getStatusLine().getStatusCode() != 200){ - logger.error("callTokenIntroEndpoint() error back from token intro host server [" - + token_introspection_url + "]: " + EntityUtils.toString(response.getEntity())); - throw new ApplicationException("Token Introspection host server return " + response.getStatusLine().getStatusCode() + - ". Please see the log"); - } - JsonNode responseContent = json.readTree(response.getEntity().getContent()); - if (!responseContent.get("active").asBoolean()){ - logger.error("callTokenIntroEndpoint() Token intro endpoint return invalid token, content: " + responseContent); - throw new NotAuthorizedException("Token invalid or expired"); - } - - String sub = responseContent.get(userIdClaim) != null ? responseContent.get(userIdClaim).asText() : null; - User user = new User().setSubject(sub).setUserId(sub); - return user; - } catch (IOException ex){ - logger.error("callTokenIntroEndpoint() IOException when hitting url: " + post - + " with exception msg: " + ex.getMessage()); - } finally { - try { - if (response != null) - response.close(); - } catch (IOException ex) { - logger.error("callTokenIntroEndpoint() IOExcpetion when closing http response: " + ex.getMessage()); - } - } - - return null; - } - - void setUserIdClaim(String userIdClaim) { - this.userIdClaim = userIdClaim; - } -} +package edu.harvard.dbmi.avillach.security; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import edu.harvard.dbmi.avillach.PicSureWarInit; +import edu.harvard.dbmi.avillach.data.entity.Query; +import edu.harvard.dbmi.avillach.data.entity.AuthUser; +import edu.harvard.dbmi.avillach.data.repository.QueryRepository; +import edu.harvard.dbmi.avillach.data.repository.ResourceRepository; +import edu.harvard.dbmi.avillach.domain.QueryRequest; +import edu.harvard.dbmi.avillach.service.ResourceWebClient; +import edu.harvard.dbmi.avillach.util.exception.ApplicationException; +import edu.harvard.dbmi.avillach.util.response.PICSUREResponse; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Resource; +import javax.inject.Inject; +import javax.ws.rs.HttpMethod; +import javax.ws.rs.NotAuthorizedException; +import javax.ws.rs.container.ContainerRequestContext; +import javax.ws.rs.container.ContainerRequestFilter; +import javax.ws.rs.container.ResourceInfo; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; +import javax.ws.rs.ext.Provider; + +import java.io.*; +import java.util.*; + +import static edu.harvard.dbmi.avillach.util.Utilities.buildHttpClientContext; + +@Provider +public class JWTFilter implements ContainerRequestFilter { + + private final Logger logger = LoggerFactory.getLogger(JWTFilter.class); + + @Context + UriInfo uriInfo; + + @Context + ResourceInfo resourceInfo; + + @Inject + ResourceRepository resourceRepo; + + @Inject + ResourceWebClient resourceWebClient; + + @Resource(mappedName = "java:global/user_id_claim") + private String userIdClaim; + + ObjectMapper mapper = new ObjectMapper(); + + @Inject + PicSureWarInit picSureWarInit; + + @Inject + QueryRepository queryRepo; + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + logger.debug("Entered jwtfilter.filter()..."); + + if (uriInfo.getPath().endsWith("/openapi.json")) { + return; + } + + if(requestContext.getUriInfo().getPath().contentEquals("/system/status") + && requestContext.getRequest().getMethod().contentEquals(HttpMethod.GET)) { + // GET calls to /system/status do not require authentication or authorization + requestContext.setProperty("username", "SYSTEM_MONITOR"); + }else { + // Everything else goes through PSAMA token introspection + String authorizationHeader = requestContext.getHeaderString(HttpHeaders.AUTHORIZATION); + if (authorizationHeader == null || authorizationHeader.isEmpty()) { + throw new NotAuthorizedException("No authorization header found."); + } + String token = authorizationHeader.substring(6).trim(); + + String userForLogging = null; + + try { + AuthUser authenticatedUser = null; + + authenticatedUser = callTokenIntroEndpoint(requestContext, token, userIdClaim); + + if (authenticatedUser == null) { + logger.error("Cannot extract a user from token: " + token); + throw new NotAuthorizedException("Cannot find or create a user"); + } + + userForLogging = authenticatedUser.getUserId(); + + //The request context wants to remember who the user is + requestContext.setProperty("username", userForLogging); + requestContext.setSecurityContext(new AuthSecurityContext(authenticatedUser, uriInfo.getRequestUri().getScheme())); + + logger.info("User - " + userForLogging + " - has just passed all the authentication and authorization layers."); + + } catch (NotAuthorizedException e) { + // the detail of this exception should be logged right before the exception thrown out + // logger.error("User - " + userForLogging + " - is not authorized. " + e.getChallenges()); + // we should show different response based on role + requestContext.abortWith(PICSUREResponse.unauthorizedError("User is not authorized. " + e.getChallenges())); + } catch (Exception e){ + // we should show different response based on role + e.printStackTrace(); + requestContext.abortWith(PICSUREResponse.applicationError("Inner application error, please contact system admin")); + } + } + } + + /** + * + * @param token + * @param userIdClaim + * @return + * @throws IOException + */ + + private AuthUser callTokenIntroEndpoint(ContainerRequestContext requestContext, String token, String userIdClaim) { + logger.debug("TokenIntrospection - extractUserFromTokenIntrospection() starting..."); + + String token_introspection_url = picSureWarInit.getToken_introspection_url(); + String token_introspection_token = picSureWarInit.getToken_introspection_token(); + + if (token_introspection_url.isEmpty()) + throw new ApplicationException("token_introspection_url is empty"); + + if (token_introspection_token.isEmpty()){ + throw new ApplicationException("token_introspection_token is empty"); + } + + ObjectMapper json = PicSureWarInit.objectMapper; + CloseableHttpClient client = PicSureWarInit.CLOSEABLE_HTTP_CLIENT; + + HttpPost post = new HttpPost(token_introspection_url); + + Map tokenMap = new HashMap<>(); + tokenMap.put("token", token); + + + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + HashMap requestMap = new HashMap(); + try { + String requestPath = requestContext.getUriInfo().getPath(); + requestMap.put("Target Service", requestPath); + + Query initialQuery = null; + //Read the query from the backing store if we are getting the results (full query may not be specified in request) + if(requestPath.startsWith("/query/") && (requestPath.endsWith("result") || requestPath.endsWith("result/"))) { + //Path: /query/{queryId}/result + String[] pathParts = requestPath.split("/"); + UUID uuid = UUID.fromString(pathParts[2]); + initialQuery = queryRepo.getById(uuid); + } + + if(initialQuery != null) { + IOUtils.copy(new ByteArrayInputStream(initialQuery.getQuery().getBytes()), buffer); + } else { + //This stream is only consumable once, so we need to save & reset it. + InputStream entityStream = requestContext.getEntityStream(); + IOUtils.copy(entityStream, buffer); + requestContext.setEntityStream(new ByteArrayInputStream(buffer.toByteArray())); + } + + if(buffer.size()>0) { + /* + * We remove the resourceCredentials from the token introspection copy of the query to prevent logging them as + * part of token introspection. These credentials are between the backing resource and the user, PIC-SURE should + * do its best to keep them confidential. + */ + Object queryObject = new ObjectMapper().readValue(new ByteArrayInputStream(buffer.toByteArray()), Object.class); + if (queryObject instanceof Collection) { + for (Object query: (Collection)queryObject) { + if (query instanceof Map) { + ((Map) query).remove("resourceCredentials"); + } + } + } else if (queryObject instanceof Map){ + ((Map) queryObject).remove("resourceCredentials"); + } + requestMap.put("query", queryObject); + + if(requestPath.startsWith("/query/")) { + + UUID resourceUUID = null; + String resourceUUIDStr = (String) ((Map)queryObject).get("resourceUUID"); + if(resourceUUIDStr != null) { + resourceUUID = UUID.fromString(resourceUUIDStr); + } + + if(resourceUUID != null) { + edu.harvard.dbmi.avillach.data.entity.Resource resource = resourceRepo.getById(resourceUUID); + //logger.info("resource obj: " + resource + " path: " + resource.getResourceRSPath()); + if (resource != null && resource.getResourceRSPath() != null){ + QueryRequest queryRequest = new QueryRequest(); + queryRequest.getResourceCredentials().put(ResourceWebClient.BEARER_TOKEN_KEY, resource.getToken()); + queryRequest.setResourceUUID(resourceUUID); + queryRequest.setQuery(((Map)queryObject).get("query")); + + Response formatResponse = resourceWebClient.queryFormat(resource.getResourceRSPath(), queryRequest); + if(formatResponse.getStatus() == 200) { + //add the formatted query if available + String formattedQuery = IOUtils.toString((InputStream)formatResponse.getEntity(), "UTF-8"); + logger.debug("Formatted response: " + formattedQuery); + requestMap.put("formattedQuery", formattedQuery); + } + } + } + } + } + tokenMap.put("request", requestMap); + } catch (JsonParseException ex) { + requestMap.put("query",buffer.toString()); + tokenMap.put("request", requestMap); + } catch (IOException e1) { + logger.error("IOException caught trying to build requestMap for auditing.", e1); + throw new NotAuthorizedException("The request could not be properly audited. If you recieve this error multiple times, please contact an administrator."); + } + StringEntity entity = null; + try { + entity = new StringEntity(json.writeValueAsString(tokenMap)); + } catch (IOException e) { + logger.error("callTokenIntroEndpoint() - " + e.getClass().getSimpleName() + " when composing post"); + return null; + } + post.setEntity(entity); + post.setHeader("Content-Type", "application/json"); + //Authorize into the token introspection endpoint + post.setHeader("Authorization", "Bearer " + token_introspection_token); + CloseableHttpResponse response = null; + try { + response = client.execute(post, buildHttpClientContext()); + if (response.getStatusLine().getStatusCode() != 200){ + logger.error("callTokenIntroEndpoint() error back from token intro host server [" + + token_introspection_url + "]: " + EntityUtils.toString(response.getEntity())); + throw new ApplicationException("Token Introspection host server return " + response.getStatusLine().getStatusCode() + + ". Please see the log"); + } + JsonNode responseContent = json.readTree(response.getEntity().getContent()); + if (!responseContent.get("active").asBoolean()){ + logger.error("callTokenIntroEndpoint() Token intro endpoint return invalid token, content: " + responseContent); + throw new NotAuthorizedException("Token invalid or expired"); + } + + String userId = responseContent.get(userIdClaim) != null ? responseContent.get(userIdClaim).asText() : null; + String sub = responseContent.get("sub") != null ? responseContent.get("sub").asText() : null; + String email = responseContent.get("email") != null ? responseContent.get("email").asText() : null; + String roles = responseContent.get("roles") != null ? responseContent.get("roles").asText() : null; + AuthUser user = new AuthUser().setUserId(userId).setSubject(sub).setEmail(email).setRoles(roles); + return user; + } catch (IOException ex){ + logger.error("callTokenIntroEndpoint() IOException when hitting url: " + post + + " with exception msg: " + ex.getMessage()); + } finally { + try { + if (response != null) + response.close(); + } catch (IOException ex) { + logger.error("callTokenIntroEndpoint() IOExcpetion when closing http response: " + ex.getMessage()); + } + } + + return null; + } + + void setUserIdClaim(String userIdClaim) { + this.userIdClaim = userIdClaim; + } +} diff --git a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/NamedDatasetService.java b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/NamedDatasetService.java new file mode 100644 index 00000000..04b916b8 --- /dev/null +++ b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/NamedDatasetService.java @@ -0,0 +1,110 @@ +package edu.harvard.dbmi.avillach.service; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import edu.harvard.dbmi.avillach.data.entity.NamedDataset; +import edu.harvard.dbmi.avillach.data.entity.Query; +import edu.harvard.dbmi.avillach.data.repository.NamedDatasetRepository; +import edu.harvard.dbmi.avillach.data.repository.QueryRepository; +import edu.harvard.dbmi.avillach.data.request.NamedDatasetRequest; +import edu.harvard.dbmi.avillach.util.exception.ProtocolException; + +public class NamedDatasetService { + private final Logger logger = LoggerFactory.getLogger(NamedDatasetService.class); + + @Inject + NamedDatasetRepository namedDatasetRepo; + + @Inject + QueryRepository queryRepo; + + public Optional> getNamedDatasets(String user){ + List queries = namedDatasetRepo.getByColumn("user", user); + return Optional.ofNullable(queries); + } + + public Optional getNamedDatasetById(String user, UUID datasetId){ + NamedDataset dataset = namedDatasetRepo.getById(datasetId); + if (dataset == null){ + logger.error("named dataset not found with id " + datasetId.toString()); + return Optional.empty(); + } + + if (!dataset.getUser().toString().equals(user)){ + logger.error("named dataset with id " + datasetId.toString() + " not able to be viewed by user " + user); + return Optional.empty(); + } + + return Optional.of(dataset); + } + + public Optional addNamedDataset(String user, NamedDatasetRequest request){ + UUID queryId = request.getQueryId(); + Query query = queryRepo.getById(queryId); + if (query == null){ + logger.error(ProtocolException.QUERY_NOT_FOUND + queryId.toString()); + return Optional.empty(); + } + + NamedDataset dataset = new NamedDataset() + .setName(request.getName()) + .setQuery(query) + .setUser(user) + .setArchived(request.getArchived()) + .setMetadata(request.getMetadata()); + + try { + namedDatasetRepo.persist(dataset); + logger.debug("persisted named dataset with query id " + queryId.toString()); + } catch (Exception exception){ + logger.error("Error persisting named dataset with query id " + queryId.toString(), exception); + return Optional.empty(); + } + + return Optional.of(dataset); + } + + public Optional updateNamedDataset(String user, UUID datasetId, NamedDatasetRequest request){ + NamedDataset dataset = namedDatasetRepo.getById(datasetId); + if (dataset == null){ + logger.error("named dataset not found with id " + datasetId.toString()); + return Optional.empty(); + } + + if (!dataset.getUser().equals(user)){ + logger.error("named dataset with id " + datasetId.toString() + " not able to be updated by user " + user); + return Optional.empty(); + } + + UUID queryId = request.getQueryId(); + if (!dataset.getQuery().getUuid().equals(queryId)){ + Query query = queryRepo.getById(queryId); + if (query == null){ + logger.error(ProtocolException.QUERY_NOT_FOUND + queryId.toString()); + return Optional.empty(); + } + dataset.setQuery(query); + } + + dataset.setName(request.getName()) + .setArchived(request.getArchived()) + .setMetadata(request.getMetadata()); + + try { + namedDatasetRepo.merge(dataset); + logger.debug("updated named dataset with id " + datasetId.toString() + " and query id " + queryId.toString()); + } catch (Exception exception){ + logger.error("Error updating named dataset with id " + datasetId.toString() + " and query id " + queryId.toString(), exception); + return Optional.empty(); + } + + return Optional.of(dataset); + } +} diff --git a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/SystemService.java b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/SystemService.java index 62fde1d8..4902448d 100755 --- a/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/SystemService.java +++ b/pic-sure-api-war/src/main/java/edu/harvard/dbmi/avillach/service/SystemService.java @@ -2,13 +2,11 @@ import static edu.harvard.dbmi.avillach.util.Utilities.buildHttpClientContext; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.util.List; import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.ws.rs.GET; -import javax.ws.rs.NotAuthorizedException; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -25,11 +23,9 @@ import edu.harvard.dbmi.avillach.PicSureWarInit; import edu.harvard.dbmi.avillach.data.entity.Resource; -import edu.harvard.dbmi.avillach.data.entity.User; import edu.harvard.dbmi.avillach.data.repository.ResourceRepository; import edu.harvard.dbmi.avillach.domain.QueryRequest; import edu.harvard.dbmi.avillach.domain.ResourceInfo; -import edu.harvard.dbmi.avillach.security.JWTFilter; import edu.harvard.dbmi.avillach.util.exception.ApplicationException; @Path("/system") diff --git a/pic-sure-api-war/src/main/resources/META-INF/persistence.xml b/pic-sure-api-war/src/main/resources/META-INF/persistence.xml index 12c75db5..72761fce 100644 --- a/pic-sure-api-war/src/main/resources/META-INF/persistence.xml +++ b/pic-sure-api-war/src/main/resources/META-INF/persistence.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/persistence http://xmlns.jcp.org/xml/ns/persistence/persistence_2_1.xsd"> java:jboss/datasources/PicsureDS - edu.harvard.dbmi.avillach.data.entity.User edu.harvard.dbmi.avillach.data.entity.Query edu.harvard.dbmi.avillach.data.entity.Resource + edu.harvard.dbmi.avillach.data.entity.NamedDataset diff --git a/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/security/JWTFilterTest.java b/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/security/JWTFilterTest.java index 71ee3639..eeef73b0 100644 --- a/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/security/JWTFilterTest.java +++ b/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/security/JWTFilterTest.java @@ -1,7 +1,6 @@ package edu.harvard.dbmi.avillach.security; import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; @@ -40,8 +39,7 @@ public class JWTFilterTest { private static final UUID RESOURCE_UUID = UUID.fromString("30ef4941-9656-4b47-af80-528f2b98cf17"); @Rule - public WireMockRule wireMockRule = new WireMockRule( - wireMockConfig().dynamicPort().dynamicHttpsPort()); + public WireMockRule wireMockRule = new WireMockRule(0); private int port; @@ -98,12 +96,21 @@ private Resource basicResource() { return resource; } - private void tokenIntrospectionStub(String tokenIntrospectionResult) { + private void tokenIntrospectionStub() { + tokenIntrospectionStub(true); + } + + private void tokenIntrospectionStub(Boolean active) { stubFor(post(urlEqualTo("/introspection_endpoint")) - .willReturn(aResponse() - .withStatus(200) - .withHeader("Content-Type", "application/json") - .withBody("{\"active\":" + tokenIntrospectionResult + ",\"sub\":\"TEST_USER\"}"))); + .willReturn(aResponse() + .withStatus(200) + .withHeader("Content-Type", "application/json") + .withBody("{" + + "\"active\":" + Boolean.toString(active) + "," + + "\"sub\":\"TEST_USER\"," + + "\"email\":\"some@email.com\"," + + "\"roles\":\"PIC_SURE_ANY_QUERY\"" + + "}"))); } @Test @@ -118,7 +125,7 @@ public void testSystemPathDoesNotRequireAuthenticationHeader() throws IOExceptio @Test public void testFilterCallsTokenIntrospectionAppropriatelyForQuerySync() throws IOException { - tokenIntrospectionStub("true"); + tokenIntrospectionStub(); ContainerRequestContext ctx = createRequestContext(); when(ctx.getUriInfo().getPath()).thenReturn("/query/sync"); @@ -141,7 +148,7 @@ public void testFilterCallsTokenIntrospectionAppropriatelyForQuerySync() throws @Test public void testFilterCallsTokenIntrospectionAppropriatelyForQuery() throws IOException { - tokenIntrospectionStub("true"); + tokenIntrospectionStub(); ContainerRequestContext ctx = createRequestContext(); when(ctx.getUriInfo().getPath()).thenReturn("/query"); @@ -164,7 +171,7 @@ public void testFilterCallsTokenIntrospectionAppropriatelyForQuery() throws IOEx @Test public void testFilterCallsTokenIntrospectionAppropriatelyForResultWithoutTrailingSlash() throws IOException { - tokenIntrospectionStub("true"); + tokenIntrospectionStub(); queryFormatStub(); @@ -193,7 +200,7 @@ public void testFilterCallsTokenIntrospectionAppropriatelyForResultWithoutTraili @Test public void testFilterCallsTokenIntrospectionAppropriatelyForResultWithTrailingSlash() throws IOException { - tokenIntrospectionStub("true"); + tokenIntrospectionStub(); queryFormatStub(); @@ -229,8 +236,7 @@ private void queryFormatStub() { @Test public void testFilterAbortsRequestIfTokenIntrospectionReturnsFalse() throws IOException { - String tokenIntrospectionResult = "false"; - tokenIntrospectionStub(tokenIntrospectionResult); + tokenIntrospectionStub(false); ContainerRequestContext ctx = createRequestContext(); when(ctx.getUriInfo().getPath()).thenReturn("/query/sync"); @@ -248,7 +254,7 @@ public void testFilterAbortsRequestIfTokenIntrospectionReturnsFalse() throws IOE @Test public void testFilterSetsUsernameIfTokenIntrospectionReturnsTrue() throws IOException { - tokenIntrospectionStub("true"); + tokenIntrospectionStub(); ContainerRequestContext ctx = createRequestContext(); when(ctx.getUriInfo().getPath()).thenReturn("/query/sync"); @@ -264,7 +270,7 @@ public void testFilterSetsUsernameIfTokenIntrospectionReturnsTrue() throws IOExc @Test public void testFilterRemovesResourceCredentialsBeforeSendingToTokenIntrospectionOrFormatter() throws IOException { - tokenIntrospectionStub("true"); + tokenIntrospectionStub(); ContainerRequestContext ctx = createRequestContext(); when(ctx.getUriInfo().getPath()).thenReturn("/query"); diff --git a/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/NamedDatasetServiceTest.java b/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/NamedDatasetServiceTest.java new file mode 100644 index 00000000..87466e3c --- /dev/null +++ b/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/NamedDatasetServiceTest.java @@ -0,0 +1,367 @@ +package edu.harvard.dbmi.avillach.service; + +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doThrow; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.HashMap; +import java.util.Optional; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import edu.harvard.dbmi.avillach.data.entity.NamedDataset; +import edu.harvard.dbmi.avillach.data.entity.Query; +import edu.harvard.dbmi.avillach.data.repository.NamedDatasetRepository; +import edu.harvard.dbmi.avillach.data.repository.QueryRepository; +import edu.harvard.dbmi.avillach.data.request.NamedDatasetRequest; + +@RunWith(MockitoJUnitRunner.class) +public class NamedDatasetServiceTest { + private String user = "test.user@email.com"; + private String testName = "test name"; + + @InjectMocks + private NamedDatasetService namedDatasetService = new NamedDatasetService(); + + @Mock + private NamedDatasetRepository datasetRepo = mock(NamedDatasetRepository.class); + + @Mock + private QueryRepository queryRepo = mock(QueryRepository.class); + + private Query makeQuery(UUID id){ + Query query = new Query(); + query.setUuid(id); + query.setQuery("{}"); + return query; + } + + private NamedDataset makeNamedDataset(UUID id, Query query){ + NamedDataset dataset = new NamedDataset(); + dataset.setUuid(id); + dataset.setUser(user); + dataset.setName(testName); + dataset.setQuery(query); + dataset.setArchived(false); + return dataset; + } + + private NamedDatasetRequest makeNamedDatasetRequest(UUID queryId){ + NamedDatasetRequest request = new NamedDatasetRequest(); + request.setName(testName); + request.setQueryId(queryId); + request.setArchived(false); + return request; + } + + @Test + public void getNamedDataset_success() { + // Given there is a saved dataset in the database for this user + Query query = makeQuery(UUID.randomUUID()); + NamedDataset dataset = makeNamedDataset(UUID.randomUUID(), query); + ArrayList datasets = new ArrayList(); + datasets.add(dataset); + when(datasetRepo.getByColumn("user", user)).thenReturn(datasets); + + // When the request is recieved + Optional> response = namedDatasetService.getNamedDatasets(user); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + } + + @Test + public void getNamedDataset_novalues() { + // Given there is no saved dataset in the database for this user + ArrayList datasets = new ArrayList(); + when(datasetRepo.getByColumn("user", user)).thenReturn(datasets); + + // When the request is recieved + Optional> response = namedDatasetService.getNamedDatasets(user); + + // Then return a non-empty optional with an empy list + assertTrue(response.isPresent()); + assertTrue(response.get().size() == 0); + } + + @Test + public void getNamedDatasetById_success() { + // Given there is a saved dataset in the database for this user + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(UUID.randomUUID()); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + // When the request is recieved + Optional response = namedDatasetService.getNamedDatasetById(user, namedDatasetId); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + } + + @Test + public void getNamedDatasetById_datasetNotFromUser() { + // Given there is a saved dataset in the database with this id but a different user + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(UUID.randomUUID()); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + dataset.setUser("other.user@email.com"); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + // When the request is recieved + Optional response = namedDatasetService.getNamedDatasetById(user, namedDatasetId); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void getNamedDatasetById_noNamedDatasetWithId() { + // Given there is no saved dataset in the database with this id + UUID namedDatasetId = UUID.randomUUID(); + when(datasetRepo.getById(namedDatasetId)).thenReturn(null); + + // When the request is recieved + Optional response = namedDatasetService.getNamedDatasetById(user, namedDatasetId); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void addNamedDataset_success() { + // Given there is a query in the database + UUID queryId = UUID.randomUUID(); + Query query = makeQuery(queryId); + when(queryRepo.getById(queryId)).thenReturn(query); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + Optional response = namedDatasetService.addNamedDataset(user, request); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + assertEquals("related user is saved", user, response.get().getUser()); + assertEquals("related name is saved", testName, response.get().getName()); + assertEquals("related query is saved", queryId, response.get().getQuery().getUuid()); + } + + @Test + public void addNamedDataset_metadataSet_success() { + // Given there is a query in the database + UUID queryId = UUID.randomUUID(); + Query query = makeQuery(queryId); + when(queryRepo.getById(queryId)).thenReturn(query); + String testKey = "test"; + String testValue = "value"; + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + HashMap metadata = new HashMap(); + metadata.put(testKey, testValue); + request.setMetadata(metadata); + Optional response = namedDatasetService.addNamedDataset(user, request); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + assertEquals("related metadata is saved", testValue, response.get().getMetadata().get(testKey)); + } + + @Test + public void addNamedDataset_noQueryWithID() { + // Given there is no query in the database with this id + UUID queryId = UUID.randomUUID(); + when(queryRepo.getById(queryId)).thenReturn(null); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + Optional response = namedDatasetService.addNamedDataset(user, request); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void addNamedDataset_cannotPersist() { + // Given there is an error saving to the named dataset table + UUID queryId = UUID.randomUUID(); + Query query = makeQuery(queryId); + when(queryRepo.getById(queryId)).thenReturn(query); + doThrow(new RuntimeException()).when(datasetRepo).persist(any(NamedDataset.class)); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + Optional response = namedDatasetService.addNamedDataset(user, request); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void updateNamedDataset_changeName_success() { + // Given there is a named dataset saved in the database with this id + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + // When the request is recieved + String newName = "new name"; + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + request.setName(newName); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + assertEquals("new name id is saved", newName, response.get().getName()); + } + + @Test + public void updateNamedDataset_changeArchiveState_success() { + // Given there is a named dataset saved in the database with this id + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + request.setArchived(true); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + assertEquals("new archive state is retained", true, response.get().getArchived()); + } + + @Test + public void updateNamedDataset_changeMetadata_success() { + // Given there is a named dataset saved in the database with this id + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + HashMap oldMetadata = new HashMap(); + oldMetadata.put("whatever", "something"); + dataset.setMetadata(oldMetadata); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + String testKey = "test"; + String testValue = "value"; + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + HashMap newMetadata = new HashMap(); + newMetadata.put(testKey, testValue); + request.setMetadata(newMetadata); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + assertEquals("new metadata is retained", testValue, response.get().getMetadata().get(testKey)); + } + + @Test + public void updateNamedDataset_changeQueryId_success() { + // Given there is a named dataset saved in the database with this id and the new query id is in the database + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + UUID newQueryId = UUID.randomUUID(); + Query newQuery = makeQuery(newQueryId); + when(queryRepo.getById(newQueryId)).thenReturn(newQuery); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(newQueryId); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return a non-empty optional + assertTrue(response.isPresent()); + assertEquals("new query id is saved", newQueryId, response.get().getQuery().getUuid()); + } + + @Test + public void updateNamedDataset_noNamedDatasetWithId() { + // Given there is no named dataset in the database with this id + UUID namedDatasetId = UUID.randomUUID(); + when(datasetRepo.getById(namedDatasetId)).thenReturn(null); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(UUID.randomUUID()); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void updateNamedDataset_datasetNotFromUser() { + // Given there is a saved dataset in the database with this id but a different user + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + dataset.setUser("other.user@email.com"); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void updateNamedDataset_changeQueryId_noQueryWithID() { + // Given there is a named dataset saved in the database with this id but no query id as passed in + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + + UUID newQueryId = UUID.randomUUID(); + when(queryRepo.getById(newQueryId)).thenReturn(null); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(newQueryId); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } + + @Test + public void updateNamedDataset_cannotPersistChanges() { + // Given there is an error saving to the named dataset table + UUID queryId = UUID.randomUUID(); + UUID namedDatasetId = UUID.randomUUID(); + Query query = makeQuery(queryId); + NamedDataset dataset = makeNamedDataset(namedDatasetId, query); + when(datasetRepo.getById(namedDatasetId)).thenReturn(dataset); + doThrow(new RuntimeException()).when(datasetRepo).merge(any(NamedDataset.class)); + + // When the request is recieved + NamedDatasetRequest request = makeNamedDatasetRequest(queryId); + Optional response = namedDatasetService.updateNamedDataset(user, namedDatasetId, request); + + // Then return an empty optional + assertTrue(response.isEmpty()); + } +} diff --git a/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/SystemServiceTest.java b/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/SystemServiceTest.java index f06767f7..354ca31e 100644 --- a/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/SystemServiceTest.java +++ b/pic-sure-api-war/src/test/java/edu/harvard/dbmi/avillach/service/SystemServiceTest.java @@ -1,7 +1,6 @@ package edu.harvard.dbmi.avillach.service; import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.*; @@ -21,8 +20,7 @@ public class SystemServiceTest { @Rule - public WireMockRule wireMockRule = new WireMockRule( - wireMockConfig().dynamicPort().dynamicHttpsPort()); + public WireMockRule wireMockRule = new WireMockRule(0); private int port; diff --git a/pic-sure-api-wildfly/src/main/resources/META-INF/persistence.xml b/pic-sure-api-wildfly/src/main/resources/META-INF/persistence.xml index 2cfbdeaa..84874384 100644 --- a/pic-sure-api-wildfly/src/main/resources/META-INF/persistence.xml +++ b/pic-sure-api-wildfly/src/main/resources/META-INF/persistence.xml @@ -3,9 +3,10 @@ org.hibernate.ejb.HibernatePersistence java:/PicsureDS - edu.harvard.dbmi.avillach.data.entity.User + edu.harvard.dbmi.avillach.data.entity.AuthUser edu.harvard.dbmi.avillach.data.entity.Resource edu.harvard.dbmi.avillach.data.entity.Query + edu.harvard.dbmi.avillach.data.entity.NamedDataset