diff --git a/checkstyle-suppressions.xml b/checkstyle-suppressions.xml index d69b31805c..7be2a7532a 100644 --- a/checkstyle-suppressions.xml +++ b/checkstyle-suppressions.xml @@ -14,4 +14,6 @@ + + diff --git a/elide-contrib/elide-dynamic-config-helpers/pom.xml b/elide-contrib/elide-dynamic-config-helpers/pom.xml new file mode 100644 index 0000000000..dc720e496b --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/pom.xml @@ -0,0 +1,165 @@ + + + + 4.0.0 + elide-dynamic-config-helpers + jar + Elide Dynamic Config Helpers + Dynamic config helpers + https://github.com/yahoo/elide + + elide-contrib-parent-pom + com.yahoo.elide + 5.0.0-pr9-SNAPSHOT + + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + + Yahoo Inc. + https://github.com/yahoo + + + + + scm:git:ssh://git@github.com/yahoo/elide.git + https://github.com/yahoo/elide.git + HEAD + + + + 3.0.0 + 2.10.2 + 4.2.0 + 2.2.12 + 1.3.0 + 1.0 + 2.6 + + + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + + + org.apache.commons + commons-lang3 + + + commons-io + commons-io + ${commons-io.version} + + + org.slf4j + slf4j-api + + + org.projectlombok + lombok + + + com.github.java-json-tools + json-schema-validator + ${json-schema-validator.version} + test + + + org.hjson + hjson + ${hjson.version} + + + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + com.github.jknack + handlebars + ${handlebars.version} + + + org.junit.platform + junit-platform-launcher + test + + + org.mdkt.compiler + InMemoryJavaCompiler + ${mdkt.compiler.version} + + + com.google.collections + google-collections + ${google.collections.version} + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + maven-assembly-plugin + + + package + + single + + + + + + jar-with-dependencies + + + + + + diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpers.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpers.java new file mode 100644 index 0000000000..33bb35fe28 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpers.java @@ -0,0 +1,169 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers; + +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.Table; +import com.yahoo.elide.contrib.dynamicconfighelpers.parser.handlebars.HandlebarsHydrator; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.commons.io.FileUtils; +import org.hjson.JsonValue; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Slf4j +/** + * Util class for Dynamic config helper module. + */ +public class DynamicConfigHelpers { + + private static final String TABLE_CONFIG_PATH = "tables" + File.separator; + private static final String SECURITY_CONFIG_PATH = "security.hjson"; + private static final String VARIABLE_CONFIG_PATH = "variables.hjson"; + private static final String NEW_LINE = "\n"; + + /** + * Checks whether input is null or empty. + * @param input : input string + * @return true or false + */ + public static boolean isNullOrEmpty(String input) { + return (input == null || input.trim().length() == 0); + } + + /** + * format config file path. + * @param basePath : path to hjson config. + * @return formatted file path. + */ + public static String formatFilePath(String basePath) { + String path = basePath; + if (!path.endsWith(File.separator)) { + path += File.separator; + } + return path; + } + + /** + * converts variable.hjson to map of variables. + * @param basePath : root path to model dir + * @return Map of variables + * @throws JsonProcessingException + */ + @SuppressWarnings("unchecked") + public static Map getVariablesPojo(String basePath) throws JsonProcessingException { + String filePath = basePath + VARIABLE_CONFIG_PATH; + File variableFile = new File(filePath); + if (variableFile.exists()) { + String jsonConfig = hjsonToJson(readConfigFile(variableFile)); + return getModelPojo(jsonConfig, Map.class); + } else { + log.info("Variables config file not found at " + filePath); + return null; + } + } + + /** + * converts all available table config to ElideTableConfig Pojo. + * @param basePath : root path to model dir + * @param variables : variables to resolve. + * @return ElideTableConfig pojo + * @throws IOException + */ + public static ElideTableConfig getElideTablePojo(String basePath, Map variables) + throws IOException { + return getElideTablePojo(basePath, variables, TABLE_CONFIG_PATH); + } + + /** + * converts all available table config to ElideTableConfig Pojo. + * @param basePath : root path to model dir + * @param variables : variables to resolve. + * @param tableDirName : dir name for table configs + * @return ElideTableConfig pojo + * @throws IOException + */ + public static ElideTableConfig getElideTablePojo(String basePath, Map variables, + String tableDirName) throws IOException { + Collection tableConfigs = FileUtils.listFiles(new File(basePath + tableDirName), + new String[] {"hjson"}, false); + Set tables = new HashSet<>(); + for (File tableConfig : tableConfigs) { + String jsonConfig = hjsonToJson(resolveVariables(readConfigFile(tableConfig), variables)); + ElideTableConfig table = getModelPojo(jsonConfig, ElideTableConfig.class); + tables.addAll(table.getTables()); + } + ElideTableConfig elideTableConfig = new ElideTableConfig(); + elideTableConfig.setTables(tables); + return elideTableConfig; + } + + /** + * converts security.hjson to ElideSecurityConfig Pojo. + * @param basePath : root path to model dir. + * @param variables : variables to resolve. + * @return ElideSecurityConfig Pojo + * @throws IOException + */ + public static ElideSecurityConfig getElideSecurityPojo(String basePath, Map variables) + throws IOException { + String filePath = basePath + SECURITY_CONFIG_PATH; + File securityFile = new File(filePath); + if (securityFile.exists()) { + String jsonConfig = hjsonToJson(resolveVariables(readConfigFile(securityFile), variables)); + return getModelPojo(jsonConfig, ElideSecurityConfig.class); + } else { + log.info("Security config file not found at " + filePath); + return null; + } + } + + /** + * resolves variables in table and security config. + * @param jsonConfig of table or security + * @param variables map from config + * @return json string with resolved variables + * @throws IOException + */ + public static String resolveVariables(String jsonConfig, Map variables) throws IOException { + HandlebarsHydrator hydrator = new HandlebarsHydrator(); + return hydrator.hydrateConfigTemplate(jsonConfig, variables); + } + + private static String hjsonToJson(String hjson) { + return JsonValue.readHjson(hjson).toString(); + } + + private static T getModelPojo(String jsonConfig, final Class configPojo) throws JsonProcessingException { + return new ObjectMapper().readValue(jsonConfig, configPojo); + } + + private static String readConfigFile(File configFile) { + StringBuffer sb = new StringBuffer(); + try { + for (String line : FileUtils.readLines(configFile, StandardCharsets.UTF_8)) { + sb.append(line); + sb.append(NEW_LINE); + } + } catch (IOException e) { + log.error("error while reading config file " + configFile.getName()); + log.error(e.getMessage()); + } + return sb.toString(); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/compile/ElideDynamicEntityCompiler.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/compile/ElideDynamicEntityCompiler.java new file mode 100644 index 0000000000..5973b3baea --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/compile/ElideDynamicEntityCompiler.java @@ -0,0 +1,150 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.compile; + +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.parser.ElideConfigParser; +import com.yahoo.elide.contrib.dynamicconfighelpers.parser.handlebars.HandlebarsHydrator; + +import com.google.common.collect.Sets; + +import org.mdkt.compiler.InMemoryJavaCompiler; + +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Compiles dynamic model pojos generated from hjson files. + * + */ +@Slf4j +public class ElideDynamicEntityCompiler { + + public static ArrayList classNames = new ArrayList(); + + public static final String PACKAGE_NAME = "dynamicconfig.models."; + private Map> compiledObjects; + + private InMemoryJavaCompiler compiler = InMemoryJavaCompiler.newInstance().ignoreWarnings(); + + private Map tableClasses = new HashMap(); + private Map securityClasses = new HashMap(); + + /** + * Parse dynamic config path. + * @param path : Dynamic config hjsons root location + * @throws Exception Exception thrown + */ + public ElideDynamicEntityCompiler(String path) throws Exception { + + ElideTableConfig tableConfig = new ElideTableConfig(); + ElideSecurityConfig securityConfig = new ElideSecurityConfig(); + ElideConfigParser elideConfigParser = new ElideConfigParser(path); + HandlebarsHydrator hydrator = new HandlebarsHydrator(); + + tableConfig = elideConfigParser.getElideTableConfig(); + securityConfig = elideConfigParser.getElideSecurityConfig(); + tableClasses = hydrator.hydrateTableTemplate(tableConfig); + securityClasses = hydrator.hydrateSecurityTemplate(securityConfig); + + for (Entry entry : tableClasses.entrySet()) { + classNames.add(PACKAGE_NAME + entry.getKey()); + } + + for (Entry entry : securityClasses.entrySet()) { + classNames.add(PACKAGE_NAME + entry.getKey()); + } + + compiler.useParentClassLoader( + new ElideDynamicInMemoryClassLoader(ClassLoader.getSystemClassLoader(), + Sets.newHashSet(classNames))); + compile(); + } + + /** + * Compile table and security model pojos. + * @throws Exception + */ + private void compile() throws Exception { + + for (Map.Entry tablePojo : tableClasses.entrySet()) { + log.debug("key: " + tablePojo.getKey() + ", value: " + tablePojo.getValue()); + compiler.addSource(PACKAGE_NAME + tablePojo.getKey(), tablePojo.getValue()); + } + + for (Map.Entry secPojo : securityClasses.entrySet()) { + log.debug("key: " + secPojo.getKey() + ", value: " + secPojo.getValue()); + compiler.addSource(PACKAGE_NAME + secPojo.getKey(), secPojo.getValue()); + } + + compiledObjects = compiler.compileAll(); + } + + /** + * Get Inmemorycompiler's classloader. + * @return ClassLoader + */ + public ClassLoader getClassLoader() { + return compiler.getClassloader(); + } + + /** + * Get the class from compiled class lists. + * @param name name of the class + * @return Class + */ + public Class getCompiled(String name) { + return compiledObjects.get(name); + } + + /** + * Find classes with a particular annotation from dynamic compiler. + * @param annotationClass Annotation to search for. + * @return Set of Classes matching the annotation. + * @throws ClassNotFoundException + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public Set> findAnnotatedClasses(Class annotationClass) + throws ClassNotFoundException { + + Set> annotatedClasses = new HashSet>(); + ArrayList dynamicClasses = classNames; + + for (String dynamicClass : dynamicClasses) { + Class classz = compiledObjects.get(dynamicClass); + if (classz.getAnnotation(annotationClass) != null) { + annotatedClasses.add(classz); + } + } + + return annotatedClasses; + } + + /** + * Find classes with a particular annotation from dynamic compiler. + * @param annotationClass Annotation to search for. + * @return Set of Classes matching the annotation. + * @throws ClassNotFoundException + */ + @SuppressWarnings({ "rawtypes" }) + public List findAnnotatedClassNames(Class annotationClass) + throws ClassNotFoundException { + + return findAnnotatedClasses(annotationClass) + .stream() + .map(Class::getName) + .collect(Collectors.toList()); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/compile/ElideDynamicInMemoryClassLoader.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/compile/ElideDynamicInMemoryClassLoader.java new file mode 100644 index 0000000000..1165f6a6ca --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/compile/ElideDynamicInMemoryClassLoader.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.compile; + +import com.google.common.collect.Sets; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Set; + +/** + * ClassLoader for dynamic configuration. + */ +@Slf4j +@Data +@AllArgsConstructor +public class ElideDynamicInMemoryClassLoader extends ClassLoader { + + private Set classNames = Sets.newHashSet(); + + public ElideDynamicInMemoryClassLoader(ClassLoader parent, Set classNames) { + super(parent); + setClassNames(classNames); + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + return super.findClass(name); + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return super.loadClass(name); + } + + @Override + protected URL findResource(String name) { + log.debug("Finding Resource " + name + " in " + classNames); + if (classNames.contains(name.replace("/", ".").replace(".class", ""))) { + try { + log.debug("Returning Resource " + "file://" + name); + return new URL("file://" + name); + } catch (MalformedURLException e) { + throw new IllegalStateException(e); + } + } + return super.findResource(name); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Dimension.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Dimension.java new file mode 100644 index 0000000000..320f24d8cf --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Dimension.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +/** + * Dimensions represent labels for measures. + * Dimensions are used to filter and group measures. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "name", + "description", + "category", + "hidden", + "readAccess", + "definition", + "type", + "grains", + "tags" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class Dimension { + + @JsonProperty("name") + private String name; + + @JsonProperty("description") + private String description; + + @JsonProperty("category") + private String category; + + + @JsonProperty("hidden") + private Boolean hidden = false; + + @JsonProperty("readAccess") + private String readAccess = "Prefab.Role.All"; + + @JsonProperty("definition") + private String definition = ""; + + @JsonProperty("type") + private Type type = Type.TEXT; + + @JsonProperty("grains") + private List grains = new ArrayList(); + + @JsonProperty("tags") + @JsonDeserialize(as = LinkedHashSet.class) + private Set tags = new LinkedHashSet(); + + /** + * Returns description of the dimension. + * If null, returns the name. + * @return description + */ + public String getDescription() { + return (this.description == null ? getName() : this.description); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/ElideSecurityConfig.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/ElideSecurityConfig.java new file mode 100644 index 0000000000..9ef99e6797 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/ElideSecurityConfig.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Elide Security POJO. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "roles", + "rules" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class ElideSecurityConfig { + + @JsonProperty("roles") + @JsonDeserialize(as = LinkedHashSet.class) + private Set roles = new LinkedHashSet(); + + @JsonProperty("rules") + @JsonDeserialize(as = LinkedHashSet.class) + private Set rules = new LinkedHashSet(); +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/ElideTableConfig.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/ElideTableConfig.java new file mode 100644 index 0000000000..ed85357d2e --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/ElideTableConfig.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.LinkedHashSet; +import java.util.Set; + +/** + * Elide Table POJO. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "tables" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class ElideTableConfig { + + @JsonProperty("tables") + @JsonDeserialize(as = LinkedHashSet.class) + private Set
tables = new LinkedHashSet
(); +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Grains.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Grains.java new file mode 100644 index 0000000000..dc2e9b801e --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Grains.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonValue; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.HashMap; +import java.util.Map; + +/** + * Grains can have SQL expressions that can substitute column + * with the dimension definition expression. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "grain", + "sql" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class Grains { + + + @JsonProperty("grain") + private Grains.Grain grain; + + @JsonProperty("sql") + private String sql; + + public enum Grain { + + DAY("DAY"), + WEEK("WEEK"), + MONTH("MONTH"), + YEAR("YEAR"); + private final String value; + private final static Map CONSTANTS = new HashMap(); + + static { + for (Grains.Grain c: values()) { + CONSTANTS.put(c.value, c); + } + } + + private Grain(String value) { + this.value = value; + } + + @JsonValue + @Override + public String toString() { + return this.value; + } + + @JsonCreator + public static Grains.Grain fromValue(String value) { + Grains.Grain constant = CONSTANTS.get(value); + if (constant == null) { + throw new IllegalArgumentException(value); + } else { + return constant; + } + } + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Join.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Join.java new file mode 100644 index 0000000000..f5e2d2d7b5 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Join.java @@ -0,0 +1,85 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonValue; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.HashMap; +import java.util.Map; + +/** + * Joins describe the SQL expression necessary to join two physical tables. + * Joins can be used when defining dimension columns that reference other tables. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "name", + "to", + "type", + "definition" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class Join { + + + @JsonProperty("name") + private String name; + + @JsonProperty("to") + private String to; + + @JsonProperty("type") + private Join.Type type; + + @JsonProperty("definition") + private String definition; + + public enum Type { + + TO_ONE("toOne"), + TO_MANY("toMany"); + private final String value; + private final static Map CONSTANTS = new HashMap(); + + static { + for (Join.Type c: values()) { + CONSTANTS.put(c.value, c); + } + } + + private Type(String value) { + this.value = value; + } + + @JsonValue + @Override + public String toString() { + return this.value; + } + + @JsonCreator + public static Join.Type fromValue(String value) { + Join.Type constant = CONSTANTS.get(value); + if (constant == null) { + throw new IllegalArgumentException(value); + } else { + return constant; + } + } + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Measure.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Measure.java new file mode 100644 index 0000000000..b95174a171 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Measure.java @@ -0,0 +1,65 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * Measures represent metrics that can be aggregated at query time. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "name", + "description", + "category", + "hidden", + "readAccess", + "definition", + "type" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class Measure { + + @JsonProperty("name") + private String name; + + @JsonProperty("description") + private String description; + + @JsonProperty("category") + private String category; + + @JsonProperty("hidden") + private Boolean hidden = false; + + @JsonProperty("readAccess") + private String readAccess = "Prefab.Role.All"; + + @JsonProperty("definition") + private String definition; + + @JsonProperty("type") + private Type type = Type.INTEGER; + + /** + * Returns description of the measure. + * If null, returns the name. + * @return description + */ + public String getDescription() { + return (this.description == null ? getName() : this.description); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Rule.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Rule.java new file mode 100644 index 0000000000..7e97311532 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Rule.java @@ -0,0 +1,74 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonValue; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * Rules are a list of RSQL filter expression templates that + * support property expansion on the principal object. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "type", + "filter", + "name" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class Rule { + + @JsonProperty("type") + private Rule.Type type; + + @JsonProperty("filter") + private String filter; + + @JsonProperty("name") + private String name; + + public enum Type { + + FILTER("filter"); + private final String value; + + private Type(String value) { + this.value = value; + } + + @JsonValue + @Override + public String toString() { + return this.value; + } + } + + public enum Filter { + + FILTER("filter"); + private final String value; + + private Filter(String value) { + this.value = value; + } + + @JsonValue + @Override + public String toString() { + return this.value; + } + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Table.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Table.java new file mode 100644 index 0000000000..f610cc4ce6 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Table.java @@ -0,0 +1,137 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonValue; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Table Model JSON. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonPropertyOrder({ + "name", + "schema", + "hidden", + "description", + "cardinality", + "readAccess", + "joins", + "measures", + "dimensions", + "tags", + "extends", + "sql", + "table" +}) +@Data +@EqualsAndHashCode() +@AllArgsConstructor +@NoArgsConstructor +public class Table { + + @JsonProperty("name") + private String name; + + @JsonProperty("schema") + private String schema = ""; + + @JsonProperty("hidden") + private Boolean hidden = false; + + @JsonProperty("description") + private String description; + + @JsonProperty("cardinality") + private Table.Cardinality cardinality = Table.Cardinality.fromValue("tiny"); + + @JsonProperty("readAccess") + private String readAccess = "Prefab.Role.All"; + + @JsonProperty("joins") + private List joins = new ArrayList(); + + @JsonProperty("measures") + private List measures = new ArrayList(); + + @JsonProperty("dimensions") + private List dimensions = new ArrayList(); + + @JsonProperty("tags") + @JsonDeserialize(as = LinkedHashSet.class) + private Set tags = new LinkedHashSet(); + + @JsonProperty("extends") + private String extend = ""; + + @JsonProperty("sql") + private String sql = ""; + + @JsonProperty("table") + private String table = ""; + + /** + * Returns description of the table object. + * If null, returns the name. + * @return description + */ + public String getDescription() { + return (this.description == null ? getName() : this.description); + } + + public enum Cardinality { + + TINY("tiny"), + SMALL("small"), + MEDIUM("medium"), + LARGE("large"), + HUGE("huge"); + private final String value; + private final static Map CONSTANTS = new HashMap(); + + static { + for (Table.Cardinality c: values()) { + CONSTANTS.put(c.value, c); + } + } + + private Cardinality(String value) { + this.value = value; + } + + @JsonValue + @Override + public String toString() { + return this.value; + } + + @JsonCreator + public static Table.Cardinality fromValue(String value) { + Table.Cardinality constant = CONSTANTS.get(value); + if (constant == null) { + throw new IllegalArgumentException(value); + } else { + return constant; + } + } + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Type.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Type.java new file mode 100644 index 0000000000..703fa50c23 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/model/Type.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.model; + +/** + * Data Type of the field. + */ +public enum Type { + + TIME("TIME"), + INTEGER("INTEGER"), + DECIMAL("DECIMAL"), + MONEY("MONEY"), + TEXT("TEXT"), + COORDINATE("COORDINATE"), + BOOLEAN("BOOLEAN"); + + private final String value; + + private Type(String value) { + this.value = value; + } + + @Override + public String toString() { + return this.value; + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/ElideConfigParser.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/ElideConfigParser.java new file mode 100644 index 0000000000..9117383518 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/ElideConfigParser.java @@ -0,0 +1,52 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.parser; + +import com.yahoo.elide.contrib.dynamicconfighelpers.DynamicConfigHelpers; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.Map; + +@Slf4j +/** + * Parses Hjson configuration from local file path and initializes Dynamic Model POJOs + */ +@Data +public class ElideConfigParser { + + private ElideTableConfig elideTableConfig; + private ElideSecurityConfig elideSecurityConfig; + private Map variables; + + /** + * Initialize Dynamic config objects. + * @param localFilePath : Path to dynamic model config dir. + * @throws IllegalArgumentException + */ + public ElideConfigParser(String localFilePath) { + + if (DynamicConfigHelpers.isNullOrEmpty(localFilePath)) { + throw new IllegalArgumentException("Config path is null"); + } + try { + String basePath = DynamicConfigHelpers.formatFilePath(localFilePath); + + this.variables = DynamicConfigHelpers.getVariablesPojo(basePath); + this.elideTableConfig = DynamicConfigHelpers.getElideTablePojo(basePath, this.variables); + this.elideSecurityConfig = DynamicConfigHelpers.getElideSecurityPojo(basePath, this.variables); + + } catch (IOException e) { + log.error("Error while parsing dynamic config at location " + localFilePath); + log.error(e.getMessage()); + throw new IllegalStateException(e); + } + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHelper.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHelper.java new file mode 100644 index 0000000000..285ec7dabb --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHelper.java @@ -0,0 +1,125 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.parser.handlebars; + +import com.yahoo.elide.contrib.dynamicconfighelpers.model.Type; +import com.github.jknack.handlebars.Options; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; +import java.util.stream.Collectors; + +/** + * Helper class for handlebar template hydration. + */ +public class HandlebarsHelper { + + private static final String EMPTY_STRING = ""; + private static final String STRING = "String"; + private static final String DATE = "Date"; + private static final String BIGDECIMAL = "BigDecimal"; + private static final String LONG = "Long"; + private static final String BOOLEAN = "Boolean"; + private static final String WHITESPACE_REGEX = "\\s+"; + + /** + * Capitalize first letter of the string. + * @param str string to capitalize first letter + * @return string with first letter capitalized + */ + public String capitalizeFirstLetter(String str) { + + return (str == null || str.length() == 0) ? str : str.substring(0, 1).toUpperCase(Locale.ENGLISH) + + str.substring(1); + } + + /** + * LowerCase first letter of the string. + * @param str string to lower case first letter + * @return string with first letter lower cased + */ + public String lowerCaseFirstLetter(String str) { + + return (str == null || str.length() == 0) ? str : str.substring(0, 1).toLowerCase(Locale.ENGLISH) + + str.substring(1); + } + + /** + * Transform string to capitalize first character of each word, change other + * characters to lower case and remove spaces. + * @param str String to be transformed + * @return Capitalize First Letter of Each Word and remove spaces + */ + public String titleCaseRemoveSpaces(String str) { + + return (str == null || str.length() == 0) ? str + : String.join(EMPTY_STRING, Arrays.asList(str.trim().split(WHITESPACE_REGEX)).stream().map( + s -> toUpperCase(s.substring(0, 1)) + toLowerCase(s.substring(1))) + .collect(Collectors.toList())); + } + + /** + * Transform string to upper case. + * @param obj Object representation of the string + * @return string converted to upper case + */ + public String toUpperCase(Object obj) { + + return (obj == null) ? EMPTY_STRING : obj.toString().toUpperCase(Locale.ENGLISH); + } + + /** + * Transform string to lower case. + * @param obj Object representation of the string + * @return string converted to lower case + */ + public String toLowerCase(Object obj) { + + return (obj == null) ? EMPTY_STRING : obj.toString().toLowerCase(Locale.ENGLISH); + } + + /** + * If type matches passed value. + * @param type Elide model type object + * @param options options object with type/string to match + * @return template if matched + * @throws IOException IOException + */ + public CharSequence ifTypeMatches(Object type, Options options) throws IOException { + + String inputType = type.toString(); + String typeToMatch = options.param(0, null); + return inputType.equals(typeToMatch) ? options.fn() : options.inverse(); + } + + /** + * Get java type name corresponding to the Elide model type. + * @param type Elide model type object + * @return The corresponding java type name + */ + public String getJavaType(Type type) { + + switch (type) { + case BOOLEAN: + return BOOLEAN; + case COORDINATE: + return STRING; + case INTEGER: + return LONG; + case TEXT: + return STRING; + case TIME: + return DATE; + case DECIMAL: + return BIGDECIMAL; + case MONEY: + return BIGDECIMAL; + default: + return STRING; + } + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHydrator.java b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHydrator.java new file mode 100644 index 0000000000..30fd5a0b23 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHydrator.java @@ -0,0 +1,107 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.parser.handlebars; + +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.Table; +import com.github.jknack.handlebars.Context; +import com.github.jknack.handlebars.EscapingStrategy; +import com.github.jknack.handlebars.EscapingStrategy.Hbs; +import com.github.jknack.handlebars.Handlebars; +import com.github.jknack.handlebars.Template; +import com.github.jknack.handlebars.helper.ConditionalHelpers; +import com.github.jknack.handlebars.io.ClassPathTemplateLoader; +import com.github.jknack.handlebars.io.TemplateLoader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Class for handlebars hydration. + */ +public class HandlebarsHydrator { + + public static final String SECURITY_CLASS_PREFIX = "DynamicConfigOperationChecksPrincipalIs"; + public static final String HANDLEBAR_START_DELIMITER = "<%"; + public static final String HANDLEBAR_END_DELIMITER = "%>"; + public static final EscapingStrategy MY_ESCAPING_STRATEGY = new Hbs(new String[][]{ + {"<", "<" }, + {">", ">" }, + {"\"", """ }, + {"`", "`" }, + {"&", "&" } + }); + + /** + * Method to hydrate the Table template. + * @param table ElideTable object + * @return map with key as table java class name and value as table java class definition + * @throws IOException IOException + */ + public Map hydrateTableTemplate(ElideTableConfig table) throws IOException { + + Map tableClasses = new HashMap<>(); + + TemplateLoader loader = new ClassPathTemplateLoader("/templates"); + Handlebars handlebars = new Handlebars(loader).with(MY_ESCAPING_STRATEGY); + HandlebarsHelper helper = new HandlebarsHelper(); + handlebars.registerHelpers(ConditionalHelpers.class); + handlebars.registerHelpers(helper); + Template template = handlebars.compile("table", HANDLEBAR_START_DELIMITER, HANDLEBAR_END_DELIMITER); + + for (Table t : table.getTables()) { + tableClasses.put(helper.capitalizeFirstLetter(t.getName()), template.apply(t)); + } + + return tableClasses; + } + + /** + * Method to replace variables in hjson config. + * @param config hjson config string + * @param replacements Map of variable key value pairs + * @return hjson config string with variables replaced + * @throws IOException IOException + */ + public String hydrateConfigTemplate(String config, Map replacements) throws IOException { + + Context context = Context.newBuilder(replacements).build(); + Handlebars handlebars = new Handlebars(); + Template template = handlebars.compileInline(config, HANDLEBAR_START_DELIMITER, HANDLEBAR_END_DELIMITER); + + return template.apply(context); + } + + /** + * Method to hydrate the Security template. + * @param security ElideSecurity Object + * @return security java class string + * @throws IOException IOException + */ + public Map hydrateSecurityTemplate(ElideSecurityConfig security) throws IOException { + + Map securityClasses = new HashMap<>(); + + if (security == null) { + return securityClasses; + } + + TemplateLoader loader = new ClassPathTemplateLoader("/templates"); + Handlebars handlebars = new Handlebars(loader).with(MY_ESCAPING_STRATEGY); + HandlebarsHelper helper = new HandlebarsHelper(); + handlebars.registerHelpers(ConditionalHelpers.class); + handlebars.registerHelpers(helper); + Template template = handlebars.compile("security", HANDLEBAR_START_DELIMITER, HANDLEBAR_END_DELIMITER); + + for (String role : security.getRoles()) { + securityClasses.put(SECURITY_CLASS_PREFIX + helper.titleCaseRemoveSpaces(role), template.apply(role)); + } + + return securityClasses; + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideSecuritySchema.json b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideSecuritySchema.json new file mode 100644 index 0000000000..1f89dff286 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideSecuritySchema.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft-06/schema#", + "$id": "https://elide.io/schemas/security_schema_v1#", + "description": "Elide Security config json/hjson schema", + "type": "object", + "properties": { + "roles": { + "title": "Security Roles", + "description": "List of Roles that will map to security checks", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + } + }, + "rules": { + "title": "Security Rules", + "description": "List of RSQL filter expression templates", + "type": "array", + "uniqueItems": true, + "items": { + "properties": { + "type": { + "title": "Rule Type", + "description": "Type of security rule", + "type": "string", + "enum": [ + "filter" + ] + }, + "filter": { + "title": "Rule Filter", + "description": "Rule filter expression", + "type": "string", + "enum": [ + "filter" + ] + }, + "name": { + "title": "Rule Name", + "description": "Name of the security rule", + "type": "string" + } + }, + "required": [ + "filter", + "name" + ], + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideTableSchema.json b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideTableSchema.json new file mode 100644 index 0000000000..3342f4edc3 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideTableSchema.json @@ -0,0 +1,406 @@ +{ + "$schema": "https://json-schema.org/draft-06/schema#", + "$id": "https://elide.io/schemas/table_schema_v1#", + "description": "Elide Table config json/hjson schema", + "definitions": { + "grain": { + "title": "Grains", + "description": "Grains can have SQL expressions that can substitute column with the dimension definition expression", + "type": "object", + "properties": { + "grain": { + "title": "Time granularity", + "description": "Indicates grain time granularity", + "type": "string", + "enum": [ + "DAY", + "WEEK", + "MONTH", + "YEAR" + ] + }, + "sql": { + "title": "Grain SQL", + "description": "Grain SQL query", + "type": "string" + } + }, + "required": [ + "grain", + "sql" + ], + "additionalProperties": false + }, + "join": { + "title": "Join", + "description": "Joins describe the SQL expression necessary to join two physical tables. Joins can be used when defining dimension columns that reference other tables.", + "type": "object", + "properties": { + "name": { + "title": "Join name", + "description": "The name of the join relationship.", + "type": "string" + }, + "to": { + "title": "Join table name", + "description": "The name of the table that is being joined to", + "type": "string", + "pattern": "^[A-Za-z]([0-9A-Za-z]*_?[0-9A-Za-z]*)*$" + }, + "type": { + "title": "Type of Join", + "description": "Type of the join - toOne or toMany", + "type": "string", + "enum": [ + "toOne", + "toMany" + ] + }, + "definition": { + "title": "Join definition SQL", + "description": "Templated SQL expression that represents the ON clause of the join", + "type": "string" + } + }, + "required": [ + "name", + "to", + "type", + "definition" + ], + "additionalProperties": false + }, + "enumtype": { + "title": "Dimension field type", + "description": "The data type of the dimension field", + "type": "string", + "enum": [ + "INTEGER", + "DECIMAL", + "MONEY", + "TEXT", + "COORDINATE", + "BOOLEAN" + ] + }, + "measure": { + "title": "Measure", + "description": "Metric definitions are extensible objects that contain a type field and one or more additional attributes. Each type is tied to logic in Elide that generates a metric function.", + "type": "object", + "properties": { + "name": { + "title": "Metric name", + "description": "The name of the metric. This will be the same as the POJO field name.", + "type": "string", + "pattern": "^[A-Za-z]([0-9A-Za-z]*_?[0-9A-Za-z]*)*$" + }, + "description": { + "title": "Metric description", + "description": "A long description of the metric.", + "type": "string" + }, + "category": { + "title": "Measure group category", + "description": "Category for grouping", + "type": "string" + }, + "hidden": { + "title": "Hide/Show measure", + "description": "Whether this metric is exposed via API metadata", + "type": "boolean", + "default": false + }, + "readAccess": { + "title": "Measure read access", + "description": "Read permission for the metric.", + "type": "string", + "default": "Prefab.Role.All" + }, + "definition": { + "title": "Metric definition", + "description": "The definition of the metric", + "type": "string" + }, + "type": { + "oneOf":[ + {"$ref": "#/definitions/enumtype"}, + {"enum": [ + "TIME" + ]} + ], + "default": "INTEGER" + } + }, + "required": [ + "name", + "definition" + ], + "additionalProperties": false + }, + "dimensionRef": { + "title": "Dimension", + "description": "Dimensions represent labels for measures. Dimensions are used to filter and group measures.", + "type": "object", + "properties": { + "name": { + "title": "Dimension name", + "description": "The name of the dimension. This will be the same as the POJO field name.", + "type": "string", + "pattern": "^[A-Za-z]([0-9A-Za-z]*_?[0-9A-Za-z]*)*$" + }, + "description": { + "title": "Dimension description", + "description": "A long description of the dimension.", + "type": "string" + }, + "category": { + "title": "Dimension group category", + "description": "Category for grouping dimension", + "type": "string" + }, + "hidden": { + "title": "Hide/Show dimension", + "description": "Whether this dimension is exposed via API metadata", + "type": "boolean", + "default": false + }, + "readAccess": { + "title": "Dimension read access", + "description": "Read permission for the dimension.", + "type": "string", + "default": "Prefab.Role.All" + }, + "definition": { + "title": "Dimension definition", + "description": "The definition of the dimension", + "type": "string", + "default": "" + }, + "tags": { + "title": "Dimension tags", + "description": "An array of string based tags for dimensions", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "default": [] + } + } + }, + "dimension": { + "title": "Dimension", + "description": "Dimensions represent labels for measures. Dimensions are used to filter and group measures.", + "type": "object", + "allOf": [ + { "$ref": "#/definitions/dimensionRef" }, + { + "properties": { + "type": { + "oneOf":[ + {"$ref": "#/definitions/enumtype"}, + {"enum": [ + "RELATIONSHIP","ID" + ]} + ], + "default": "TEXT" + } + } + } + ], + "required": [ + "name", + "type", + "definition" + ] + }, + "timeDimension": { + "title": "Time Dimension", + "description": "Time Dimensions represent labels for measures. Dimensions are used to filter and group measures.", + "type": "object", + "allOf": [ + { "$ref": "#/definitions/dimensionRef" }, + { + "properties": { + "type": { + "title": "Dimension field type", + "description": "The data type of the dimension field", + "type": "string", + "enum": [ + "TIME" + ], + "default": "TIME" + }, + "grains": { + "title": "Time Dimension grains", + "description": "Time Dimension granularity and Sqls", + "type": "array", + "items": { + "$ref": "#/definitions/grain" + }, + "default": [] + } + } + } + ], + "required": [ + "name", + "type", + "definition", + "grains" + ] + } + }, + "type": "object", + "properties": { + "tables": { + "title": "Elide Table Models", + "description": "Array of elide Table Models", + "type": "array", + "uniqueItems": true, + "items": { + "title": "Elide Table Models", + "description": "Array of elide Table Models", + "type": "object", + "properties": { + "name": { + "title": "Table Model Name", + "description": "The name of the model. This will be the same as the POJO class name.", + "type": "string", + "pattern": "^[A-Z][0-9A-Za-z]*$" + }, + "schema": { + "title": "Table Schema", + "description": "The database or schema where the model lives.", + "type": "string", + "pattern": "^[A-Za-z]([0-9A-Za-z]*_?[0-9A-Za-z]*)*$", + "default": "" + }, + "hidden": { + "title": "Hide/Show Table", + "description": "Whether this table is exposed via API metadata", + "type": "boolean", + "default": false + }, + "description": { + "title": "Table Model description", + "description": "A long description of the model.", + "type": "string" + }, + "cardinality": { + "title": "Model cardinality", + "description": "The number of rows in the table: (tiny, small, medium, large, huge). The relative sizes are decided by the table designer(s).", + "type": "string", + "enum": [ + "tiny", + "small", + "medium", + "large", + "huge" + ], + "default": "tiny" + }, + "readAccess": { + "title": "Table read access", + "description": "Read permission for the table.", + "type": "string", + "default": "Prefab.Role.All" + }, + "joins": { + "title": "Table joins", + "description": "Describes SQL joins to other tables for column references.", + "type": "array", + "items": { + "$ref": "#/definitions/join" + }, + "default": [] + }, + "measures": { + "title": "Table measures", + "description": "Zero or more metric definitions.", + "type": "array", + "items": { + "$ref": "#/definitions/measure" + }, + "default": [] + }, + "dimensions": { + "title": "Table dimensions", + "description": "One or more dimension definitions.", + "type": "array", + "items": { + "anyOf": [ + {"$ref": "#/definitions/dimension"} , + {"$ref": "#/definitions/timeDimension"} + ] + }, + "default": [] + }, + "tags": { + "title": "Table tags", + "description": "An array of string based tags", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string" + }, + "default": [] + } + }, + "oneOf": [ + { + "properties": { + "sql": { + "title": "Table SQL", + "description": "SQL query which is used to populate the table.", + "type": "string", + "default": "" + } + }, + "required": [ + "name", + "sql", + "dimensions" + ] + } , + { + "properties": { + "table": { + "title": "Table name", + "description": "The physical table name where the model lives.", + "type": "string", + "pattern": "^[A-Za-z]([0-9A-Za-z]*_?[0-9A-Za-z]*)*$", + "default": "" + } + }, + "required": [ + "name", + "table", + "dimensions" + ] + }, + { + "properties": { + "extends": { + "title": "Table Extends", + "description": "Extends another logical table.", + "type": "string", + "default": "" + } + }, + "required": [ + "name", + "extends", + "dimensions" + ] + } + ] + } + } + }, + "minProperties": 1, + "required": [ + "tables" + ], + "additionalProperties": false +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideVariableSchema.json b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideVariableSchema.json new file mode 100644 index 0000000000..1e8c38bb1a --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/elideVariableSchema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft-06/schema#", + "$id": "https://elide.io/schemas/variable_schema_v1#", + "description": "Elide Variable config json/hjson schema", + "type": "object", + "patternProperties": { + "^([A-Za-z]*_?[A-Za-z]*)*$": { + "anyOf": [ + {"type": "string"}, + {"type": "array"}, + {"type": "object"}, + {"type": "null"} + ] + } + }, + "additionalProperties": false, + "minProperties": 1 +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/resources/templates/security.hbs b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/templates/security.hbs new file mode 100644 index 0000000000..d9f3ee8afd --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/templates/security.hbs @@ -0,0 +1,18 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package dynamicconfig.models; + +import com.yahoo.elide.annotation.SecurityCheck; +import com.yahoo.elide.security.checks.prefab.Role.RoleMemberCheck; + +@SecurityCheck(DynamicConfigOperationChecksPrincipalIs<%#titleCaseRemoveSpaces this%><%/titleCaseRemoveSpaces%>.PRINCIPAL_IS_<%#toUpperCase this%><%/toUpperCase%>) +public class DynamicConfigOperationChecksPrincipalIs<%#titleCaseRemoveSpaces this%><%/titleCaseRemoveSpaces%> extends RoleMemberCheck { + + public static final String PRINCIPAL_IS_<%#toUpperCase this%><%/toUpperCase%> = "Principal is <%this%>"; + public DynamicConfigOperationChecksPrincipalIs<%#titleCaseRemoveSpaces this%><%/titleCaseRemoveSpaces%>() { + super("<%this%>"); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/main/resources/templates/table.hbs b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/templates/table.hbs new file mode 100644 index 0000000000..5a2f6f7f75 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/main/resources/templates/table.hbs @@ -0,0 +1,90 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package dynamicconfig.models; + +import com.yahoo.elide.annotation.DeletePermission; +import com.yahoo.elide.annotation.Include; +import com.yahoo.elide.annotation.Exclude; +import com.yahoo.elide.annotation.ReadPermission; +import com.yahoo.elide.annotation.UpdatePermission; +import com.yahoo.elide.datastores.aggregation.annotation.Cardinality; +import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize; +import com.yahoo.elide.datastores.aggregation.annotation.DimensionFormula; +import com.yahoo.elide.datastores.aggregation.annotation.MetricFormula; +import com.yahoo.elide.datastores.aggregation.annotation.Join; +import com.yahoo.elide.datastores.aggregation.annotation.Meta; +import com.yahoo.elide.datastores.aggregation.annotation.Temporal; +import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition; +import com.yahoo.elide.datastores.aggregation.metadata.enums.TimeGrain; +import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery; +import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable; + +import lombok.EqualsAndHashCode; +import lombok.ToString; +import lombok.Data; + +import java.util.Date; +import javax.persistence.Column; +import javax.persistence.Id; + +/** + * A root level entity for testing AggregationDataStore. + */ +@Cardinality(size = CardinalitySize.<%#toUpperCase cardinality%><%/toUpperCase%>) +@EqualsAndHashCode +@ToString +@Data +<%#if table%>@FromTable(name = "<%#if schema%><%schema%>.<%/if%><%table%>") +<%else if sql%>@FromSubquery(sql = "<%sql%>") +<%/if%> +<%#if readAccess%>@ReadPermission(expression = "<%readAccess%>")<%/if%> +<%#if description%>@Meta(description = "<%description%>")<%/if%> +<%#if hidden%>@Exclude<%else%>@Include(rootLevel = true, type = "<%#lowerCaseFirstLetter name%><%/lowerCaseFirstLetter%>")<%/if%> +public class <%#capitalizeFirstLetter name%><%/capitalizeFirstLetter%> <%#if extend%>extends <%#capitalizeFirstLetter extend%><%/capitalizeFirstLetter%><%/if%>{ + + @Id + private String id; + +<%#each dimensions%> + +<%#ifTypeMatches type "TIME"%> + @Temporal(grains = { + <%#each grains%> + @TimeGrainDefinition(grain = TimeGrain.<%grain%>, expression = "<%sql%>")<%#if @last%><%else%>, <%/if%> + <%/each%> + }, timeZone = "UTC") +<%/ifTypeMatches%> + + <%#if readAccess%>@ReadPermission(expression = "<%readAccess%>")<%/if%> + <%#if description%>@Meta(description = "<%description%>")<%/if%> + <%#if hidden%>@Exclude<%/if%> + @DimensionFormula("<%definition%>") + private <%#getJavaType type%><%/getJavaType%> <%name%>; + +<%/each%> + + +<%#each joins%> + + @Join("<% definition %>") +<%#ifTypeMatches type "toMany"%> + private Set<<%#capitalizeFirstLetter to%><%/capitalizeFirstLetter%>> <%name%>; +<%else%> + private <%#capitalizeFirstLetter to%><%/capitalizeFirstLetter%> <%name%>; +<%/ifTypeMatches%> + +<%/each%> + +<%#each measures%> + + @MetricFormula("<%definition%>") + <%#if readAccess%>@ReadPermission(expression = "<%readAccess%>")<%/if%> + <%#if description%>@Meta(description = "<%description%>")<%/if%> + <%#if hidden%>@Exclude<%/if%> + private <%#getJavaType type%><%/getJavaType%> <%name%>; + +<%/each%> +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpersTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpersTest.java new file mode 100644 index 0000000000..0ab9b555a2 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/DynamicConfigHelpersTest.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig; + +import com.fasterxml.jackson.core.JsonProcessingException; + +import org.junit.jupiter.api.Test; + +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +@Slf4j +public class DynamicConfigHelpersTest { + + @Test + public void testValidSecuritySchema() throws IOException { + String path = "src/test/resources/security/valid"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + Map vars = DynamicConfigHelpers.getVariablesPojo( + DynamicConfigHelpers.formatFilePath(absolutePath)); + ElideSecurityConfig config = DynamicConfigHelpers.getElideSecurityPojo( + DynamicConfigHelpers.formatFilePath(absolutePath), vars); + assertNotNull(config); + } + + @Test + public void testValidVariableSchema() throws JsonProcessingException { + String path = "src/test/resources/variables/valid"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + Map config = DynamicConfigHelpers.getVariablesPojo( + DynamicConfigHelpers.formatFilePath(absolutePath)); + assertNotNull(config); + } + + @Test + public void testValidTableSchema() throws IOException { + String path = "src/test/resources/tables"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + Map vars = DynamicConfigHelpers.getVariablesPojo( + DynamicConfigHelpers.formatFilePath(absolutePath)); + ElideTableConfig config = DynamicConfigHelpers.getElideTablePojo( + DynamicConfigHelpers.formatFilePath(absolutePath), vars, "valid/"); + assertNotNull(config); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/SchemaTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/SchemaTest.java new file mode 100644 index 0000000000..0b92b173bb --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/SchemaTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.hjson.JsonValue; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; + +public class SchemaTest { + + private InputStream loadStreamFromClasspath(String resource) throws Exception { + return TableSchemaValidationTest.class.getResourceAsStream(resource); + } + + private Reader loadReaderFromClasspath(String resource) throws Exception { + return new InputStreamReader(loadStreamFromClasspath(resource)); + } + + protected JsonNode loadJsonFromClasspath(String resource, boolean translate) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + + Reader reader = loadReaderFromClasspath(resource); + + if (translate) { + String jsonText = JsonValue.readHjson(reader).toString(); + return objectMapper.readTree(jsonText); + } + + return objectMapper.readTree(reader); + } + + protected JsonNode loadJsonFromClasspath(String resource) throws Exception { + return loadJsonFromClasspath(resource, false); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/SecuritySchemaValidationTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/SecuritySchemaValidationTest.java new file mode 100644 index 0000000000..8455aa5bd3 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/SecuritySchemaValidationTest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.github.fge.jsonschema.core.report.ProcessingReport; +import com.github.fge.jsonschema.main.JsonSchema; +import com.github.fge.jsonschema.main.JsonSchemaFactory; +import org.junit.jupiter.api.Test; + +/** + * Security Schema functional test. + */ +public class SecuritySchemaValidationTest extends SchemaTest { + + private final JsonSchema schema; + + public SecuritySchemaValidationTest() throws Exception { + JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); + schema = factory.getJsonSchema("resource:/elideSecuritySchema.json"); + } + + @Test + public void testValidSecuritySchema() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/security/valid/security.json"); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testInValidSecuritySchema() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/security/invalid/security.json"); + ProcessingReport results = schema.validate(testNode); + assertFalse(results.isSuccess()); + } + + @Test + public void testValidSecurityHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/security/valid/security.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testInvalidSecurityHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/security/invalid/security.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertFalse(results.isSuccess()); + } + + @Test + public void testModelecurityHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/models/security.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/TableSchemaValidationTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/TableSchemaValidationTest.java new file mode 100644 index 0000000000..a2bdc9f77b --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/TableSchemaValidationTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.github.fge.jsonschema.core.report.ProcessingReport; +import com.github.fge.jsonschema.main.JsonSchema; +import com.github.fge.jsonschema.main.JsonSchemaFactory; +import org.junit.jupiter.api.Test; + +/** + * Security Schema functional test. + */ +public class TableSchemaValidationTest extends SchemaTest { + + private final JsonSchema schema; + + public TableSchemaValidationTest() throws Exception { + JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); + schema = factory.getJsonSchema("resource:/elideTableSchema.json"); + } + + @Test + public void testValidTableSchema() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/tables/valid/table.json"); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testInvalidTableSchema() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/tables/invalid/table.json"); + ProcessingReport results = schema.validate(testNode); + assertFalse(results.isSuccess()); + } + + @Test + public void testValidTableHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/tables/valid/table.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testInvalidTableHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/tables/invalid/table.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertFalse(results.isSuccess()); + } + + @Test + public void testModelsTable1HJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/models/tables/table1.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testModelsTable2HJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/models/tables/table2.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testModelsTable3HJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/models_missing/tables/table1.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/VariableSchemaValidationTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/VariableSchemaValidationTest.java new file mode 100644 index 0000000000..3cc47d8c37 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/VariableSchemaValidationTest.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.github.fge.jsonschema.core.report.ProcessingReport; +import com.github.fge.jsonschema.main.JsonSchema; +import com.github.fge.jsonschema.main.JsonSchemaFactory; +import org.junit.jupiter.api.Test; + +/** + * Security Schema functional test. + */ +public class VariableSchemaValidationTest extends SchemaTest { + + private final JsonSchema schema; + + public VariableSchemaValidationTest() throws Exception { + JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); + schema = factory.getJsonSchema("resource:/elideVariableSchema.json"); + } + + @Test + public void testValidVariableSchema() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/variables/valid/variables.json"); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testInValidVariableSchema() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/variables/invalid/variables.json"); + ProcessingReport results = schema.validate(testNode); + assertFalse(results.isSuccess()); + } + + @Test + public void testValidVariableHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/variables/valid/variables.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } + + @Test + public void testInvalidVariableHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/variables/invalid/variables.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertFalse(results.isSuccess()); + } + + @Test + public void testModelsVariableHJson() throws Exception { + JsonNode testNode = loadJsonFromClasspath("/models/variables.hjson", true); + ProcessingReport results = schema.validate(testNode); + assertTrue(results.isSuccess()); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/ElideConfigParserTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/ElideConfigParserTest.java new file mode 100644 index 0000000000..1a267f8bd3 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/ElideConfigParserTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.parser; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideSecurityConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.ElideTableConfig; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.Table; +import com.yahoo.elide.contrib.dynamicconfighelpers.model.Type; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.Map; + +public class ElideConfigParserTest { + + @Test + public void testValidateVariablePath() throws Exception { + + String path = "src/test/resources/models"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + ElideConfigParser testClass = new ElideConfigParser(absolutePath); + + Map variable = testClass.getVariables(); + assertEquals(6, variable.size()); + assertEquals("blah", variable.get("bar")); + + ElideSecurityConfig security = testClass.getElideSecurityConfig(); + assertEquals(3, security.getRoles().size()); + + ElideTableConfig tables = testClass.getElideTableConfig(); + assertEquals(2, tables.getTables().size()); + for (Table t : tables.getTables()) { + assertEquals(t.getMeasures().get(0).getName() , t.getMeasures().get(0).getDescription()); + assertEquals("MAX(score)", t.getMeasures().get(0).getDefinition()); + assertEquals(Table.Cardinality.LARGE, t.getCardinality()); + // test hydration, variable substitution + assertEquals(Type.INTEGER, t.getMeasures().get(0).getType()); + } + } + + @Test + public void testNullConfig() { + try { + new ElideConfigParser(null); + } catch (IllegalArgumentException e) { + assertEquals("Config path is null", e.getMessage()); + } + } + + @Test + public void testMissingConfig() { + String path = "src/test/resources/models_missing"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + ElideConfigParser testClass = new ElideConfigParser(absolutePath); + + assertNull(testClass.getVariables()); + assertNull(testClass.getElideSecurityConfig()); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHydratorTest.java b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHydratorTest.java new file mode 100644 index 0000000000..d7baefa88d --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/java/com/yahoo/elide/contrib/dynamicconfighelpers/parser/handlebars/HandlebarsHydratorTest.java @@ -0,0 +1,286 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.contrib.dynamicconfighelpers.parser.handlebars; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.yahoo.elide.contrib.dynamicconfighelpers.parser.ElideConfigParser; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; + +public class HandlebarsHydratorTest { + + private static final String VALID_TABLE_WITH_VARIABLES = "{\n" + + " tables: [{\n" + + " name: <% name %>\n" + + " table: <% table %>\n" + + " schema: gamedb\n" + + " description:\n" + + " '''\n" + + " A long description\n" + + " '''\n" + + " cardinality : large\n" + + " hidden : false\n" + + " readAccess : A user is admin or is a player in the game\n" + + " joins: [\n" + + " {\n" + + " name: playerCountry\n" + + " to: country\n" + + " type: toOne\n" + + " definition: '${to}.id = ${from}.country_id'\n" + + " },\n" + + " {\n" + + " name: playerTeam\n" + + " to: team\n" + + " type: toMany\n" + + " definition: '${to}.id = ${from}.team_id'\n" + + " }\n" + + " ]\n" + + "\n" + + " measures : [\n" + + " {\n" + + " name : highScore\n" + + " type : INTEGER\n" + + " definition: 'MAX(score)'\n" + + " }\n" + + " ]\n" + + " dimensions : [\n" + + " {\n" + + " name : countryIsoCode\n" + + " type : TEXT\n" + + " definition : '{{playerCountry.isoCode}}'\n" + + " },\n" + + " {\n" + + " name : createdOn\n" + + " type : TIME\n" + + " definition : create_on\n" + + " grains: [\n" + + " {\n" + + " grain : DAY\n" + + " sql : '''\n" + + " PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-dd'), 'yyyy-MM-dd')\n" + + " '''\n" + + " },\n" + + " {\n" + + " grain : MONTH\n" + + " sql : '''\n" + + " PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd')\n" + + " '''\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }]\n" + + "}\n"; + + private static final String VALID_TABLE_JAVA_NAME = "PlayerStats"; + + private static final String VALID_TABLE_JAVA = "/*\n" + + " * Copyright 2020, Yahoo Inc.\n" + + " * Licensed under the Apache License, Version 2.0\n" + + " * See LICENSE file in project root for terms.\n" + + " */\n" + + "package dynamicconfig.models;\n" + + "\n" + + "import com.yahoo.elide.annotation.DeletePermission;\n" + + "import com.yahoo.elide.annotation.Include;\n" + + "import com.yahoo.elide.annotation.Exclude;\n" + + "import com.yahoo.elide.annotation.ReadPermission;\n" + + "import com.yahoo.elide.annotation.UpdatePermission;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.Cardinality;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.CardinalitySize;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.DimensionFormula;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.MetricFormula;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.Join;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.Meta;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.Temporal;\n" + + "import com.yahoo.elide.datastores.aggregation.annotation.TimeGrainDefinition;\n" + + "import com.yahoo.elide.datastores.aggregation.metadata.enums.TimeGrain;\n" + + "import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery;\n" + + "import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable;\n" + + "\n" + + "import lombok.EqualsAndHashCode;\n" + + "import lombok.ToString;\n" + + "import lombok.Data;\n" + + "\n" + + "import java.util.Date;\n" + + "import javax.persistence.Column;\n" + + "import javax.persistence.Id;\n" + + "\n" + + "/**\n" + + " * A root level entity for testing AggregationDataStore.\n" + + " */\n" + + "@Cardinality(size = CardinalitySize.LARGE)\n" + + "@EqualsAndHashCode\n" + + "@ToString\n" + + "@Data\n" + + "@FromTable(name = \"gamedb.player_stats\")\n" + + "\n" + + "@ReadPermission(expression = \"A user is admin or is a player in the game\")\n" + + "@Meta(description = \"A long description\")\n" + + "@Include(rootLevel = true, type = \"playerStats\")\n" + + "public class PlayerStats {\n" + + "\n" + + " @Id\n" + + " private String id;\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + " @ReadPermission(expression = \"Prefab.Role.All\")\n" + + " @Meta(description = \"countryIsoCode\")\n" + + " \n" + + " @DimensionFormula(\"{{playerCountry.isoCode}}\")\n" + + " private String countryIsoCode;\n" + + "\n" + + "\n" + + "\n" + + "\n" + + " @Temporal(grains = {\n" + + " \n" + + " @TimeGrainDefinition(grain = TimeGrain.DAY, expression = \"PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-dd'), 'yyyy-MM-dd')\"), \n" + + " \n" + + " @TimeGrainDefinition(grain = TimeGrain.MONTH, expression = \"PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd')\")\n" + + " \n" + + " }, timeZone = \"UTC\")\n" + + "\n" + + "\n" + + " @ReadPermission(expression = \"Prefab.Role.All\")\n" + + " @Meta(description = \"createdOn\")\n" + + " \n" + + " @DimensionFormula(\"create_on\")\n" + + " private Date createdOn;\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + " @Join(\"${to}.id = ${from}.country_id\")\n" + + "\n" + + " private Country playerCountry;\n" + + "\n" + + "\n" + + "\n" + + "\n" + + " @Join(\"${to}.id = ${from}.team_id\")\n" + + "\n" + + " private Set playerTeam;\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + " @MetricFormula(\"MAX(score)\")\n" + + " @ReadPermission(expression = \"Prefab.Role.All\")\n" + + " @Meta(description = \"highScore\")\n" + + " \n" + + " private Long highScore;\n" + + "\n" + + "\n" + + "}\n"; + + + private static final String VALID_SECURITY_ADMIN_JAVA_NAME = "DynamicConfigOperationChecksPrincipalIsAdmin"; + private static final String VALID_SECURITY_GUEST_JAVA_NAME = "DynamicConfigOperationChecksPrincipalIsGuest"; + + private static final String VALID_SECURITY_ADMIN_JAVA = "/*\n" + + " * Copyright 2020, Yahoo Inc.\n" + + " * Licensed under the Apache License, Version 2.0\n" + + " * See LICENSE file in project root for terms.\n" + + " */\n" + + "package dynamicconfig.models;\n" + + "\n" + + "import com.yahoo.elide.annotation.SecurityCheck;\n" + + "import com.yahoo.elide.security.checks.prefab.Role.RoleMemberCheck;\n" + + "\n" + + "@SecurityCheck(DynamicConfigOperationChecksPrincipalIsAdmin.PRINCIPAL_IS_ADMIN)\n" + + "public class DynamicConfigOperationChecksPrincipalIsAdmin extends RoleMemberCheck {\n" + + "\n" + + " public static final String PRINCIPAL_IS_ADMIN = \"Principal is admin\";\n" + + " public DynamicConfigOperationChecksPrincipalIsAdmin() {\n" + + " super(\"admin\");\n" + + " }\n" + + "}\n"; + + private static final String VALID_SECURITY_GUEST_JAVA = "/*\n" + + " * Copyright 2020, Yahoo Inc.\n" + + " * Licensed under the Apache License, Version 2.0\n" + + " * See LICENSE file in project root for terms.\n" + + " */\n" + + "package dynamicconfig.models;\n" + + "\n" + + "import com.yahoo.elide.annotation.SecurityCheck;\n" + + "import com.yahoo.elide.security.checks.prefab.Role.RoleMemberCheck;\n" + + "\n" + + "@SecurityCheck(DynamicConfigOperationChecksPrincipalIsGuest.PRINCIPAL_IS_GUEST)\n" + + "public class DynamicConfigOperationChecksPrincipalIsGuest extends RoleMemberCheck {\n" + + "\n" + + " public static final String PRINCIPAL_IS_GUEST = \"Principal is guest\";\n" + + " public DynamicConfigOperationChecksPrincipalIsGuest() {\n" + + " super(\"guest\");\n" + + " }\n" + + "}\n"; + + @Test + public void testConfigHydration() throws IOException { + + HandlebarsHydrator obj = new HandlebarsHydrator(); + String path = "src/test/resources/models"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + String hjsonPath = absolutePath + "/tables/table1.hjson"; + + ElideConfigParser testClass = new ElideConfigParser(absolutePath); + + Map map = testClass.getVariables(); + + String content = new String (Files.readAllBytes(Paths.get(hjsonPath))); + + assertEquals(content, obj.hydrateConfigTemplate(VALID_TABLE_WITH_VARIABLES, map)); + } + + @Test + public void testTableHydration() throws IOException { + + HandlebarsHydrator obj = new HandlebarsHydrator(); + String path = "src/test/resources/models"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + + ElideConfigParser testClass = new ElideConfigParser(absolutePath); + + Map tableClasses = obj.hydrateTableTemplate(testClass.getElideTableConfig()); + + assertEquals(true, tableClasses.keySet().contains(VALID_TABLE_JAVA_NAME)); + assertEquals(VALID_TABLE_JAVA, tableClasses.get(VALID_TABLE_JAVA_NAME)); + } + + @Test + public void testSecurityHydration() throws IOException { + HandlebarsHydrator obj = new HandlebarsHydrator(); + String path = "src/test/resources/models"; + File file = new File(path); + String absolutePath = file.getAbsolutePath(); + + ElideConfigParser testClass = new ElideConfigParser(absolutePath); + + Map securityClasses = obj.hydrateSecurityTemplate(testClass.getElideSecurityConfig()); + + assertEquals(true, securityClasses.keySet().contains(VALID_SECURITY_ADMIN_JAVA_NAME)); + assertEquals(true, securityClasses.keySet().contains(VALID_SECURITY_GUEST_JAVA_NAME)); + assertEquals(VALID_SECURITY_ADMIN_JAVA, securityClasses.get(VALID_SECURITY_ADMIN_JAVA_NAME)); + assertEquals(VALID_SECURITY_GUEST_JAVA, securityClasses.get(VALID_SECURITY_GUEST_JAVA_NAME)); + } +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/security.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/security.hjson new file mode 100644 index 0000000000..effa2da850 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/security.hjson @@ -0,0 +1,18 @@ +{ + roles : [ + admin + guest + member + ] + rules: [ + { + type: filter + filter: filter + name: User belongs to company + }, + { + filter: filter + name: Principal is owner + }, + ] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/tables/table1.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/tables/table1.hjson new file mode 100644 index 0000000000..90cdaf69d7 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/tables/table1.hjson @@ -0,0 +1,62 @@ +{ + tables: [{ + name: PlayerStats + table: player_stats + schema: gamedb + description: + ''' + A long description + ''' + cardinality : large + hidden : false + readAccess : A user is admin or is a player in the game + joins: [ + { + name: playerCountry + to: country + type: toOne + definition: '${to}.id = ${from}.country_id' + }, + { + name: playerTeam + to: team + type: toMany + definition: '${to}.id = ${from}.team_id' + } + ] + + measures : [ + { + name : highScore + type : INTEGER + definition: 'MAX(score)' + } + ] + dimensions : [ + { + name : countryIsoCode + type : TEXT + definition : '{{playerCountry.isoCode}}' + }, + { + name : createdOn + type : TIME + definition : create_on + grains: [ + { + grain : DAY + sql : ''' + PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-dd'), 'yyyy-MM-dd') + ''' + }, + { + grain : MONTH + sql : ''' + PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd') + ''' + } + ] + } + ] + }] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/tables/table2.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/tables/table2.hjson new file mode 100644 index 0000000000..848bab22f5 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/tables/table2.hjson @@ -0,0 +1,48 @@ +{ + tables: [{ + name: Player + table: player + schema: playerdb + description: + ''' + A long description + ''' + cardinality : large + readAccess : A user is admin or is a player in the game + joins: [ + { + name: playerCountry + to: country + type: toOne + definition: '${to}.id = ${from}.country_id' + } + ] + measures : [ + { + name : highScore + type : "INTEGER" + definition: '<%measure_type%>(score)' + } + ] + dimensions : [ + { + name : countryCode + type : TEXT + definition : '{{playerCountry.isoCode}}' + }, + { + name : createdOn + type : TIME + definition : create_on + grains: [ + { + grain : MONTH + sql : ''' + PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd') + ''' + } + ] + } + ] + }] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/variables.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/variables.hjson new file mode 100644 index 0000000000..6d534ed7b7 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models/variables.hjson @@ -0,0 +1,8 @@ +{ + foo: [1, 2, 3] + bar: blah + hour: hour_replace + measure_type: MAX + name: PlayerStats + table: player_stats +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models_missing/tables/table1.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models_missing/tables/table1.hjson new file mode 100644 index 0000000000..8f0b8014d1 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/models_missing/tables/table1.hjson @@ -0,0 +1,48 @@ +{ + tables: [{ + name: PlayerStats + table: player_stats + schema: gamedb + description: + ''' + A long description + ''' + cardinality : large + readAccess : A user is admin or is a player in the game + joins: [ + { + name: playerCountry + to: country + type: toOne + definition: '${to}.id = ${from}.country_id' + } + ] + measures : [ + { + name : highScore + type : INTEGER + definition: 'MAX(score)' + } + ] + dimensions : [ + { + name : countryCode + type : TEXT + definition : playerCountry.isoCode + }, + { + name : createdOn + type : TIME + definition : create_on + grains: [ + { + grain : MONTH + sql : ''' + PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd') + ''' + } + ] + } + ] + }] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/invalid/security.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/invalid/security.hjson new file mode 100644 index 0000000000..d166716f97 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/invalid/security.hjson @@ -0,0 +1,11 @@ +{ + //Comment + name : book + table : book + schema$ : [123] + description : + ''' + valid schema for a book + ''' + cardinality : small +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/invalid/security.json b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/invalid/security.json new file mode 100644 index 0000000000..c1e1422253 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/invalid/security.json @@ -0,0 +1,8 @@ + +{ + "name!" : "book", + "table" : "book", + "schema$" : "testdb", + "description" : "valid schema for a book", + "cardinality" : "invalid" +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/valid/security.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/valid/security.hjson new file mode 100644 index 0000000000..ad33e38836 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/valid/security.hjson @@ -0,0 +1,14 @@ +{ + roles : ["admin", "guest", "member"] + rules: [ + { + type: filter + filter: filter + name: User belongs to company + }, + { + filter: filter + name: Principal is owner + } + ] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/valid/security.json b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/valid/security.json new file mode 100644 index 0000000000..e3a52f6e1c --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/security/valid/security.json @@ -0,0 +1,14 @@ +{ + "roles" : ["admin", "guest", "member"], + "rules": [ + { + "type": "filter", + "filter": "filter", + "name": "User belongs to company" + }, + { + "filter": "filter", + "name": "Principal is owner" + } + ] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/invalid/table.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/invalid/table.hjson new file mode 100644 index 0000000000..d166716f97 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/invalid/table.hjson @@ -0,0 +1,11 @@ +{ + //Comment + name : book + table : book + schema$ : [123] + description : + ''' + valid schema for a book + ''' + cardinality : small +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/invalid/table.json b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/invalid/table.json new file mode 100644 index 0000000000..0c3f97c1bd --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/invalid/table.json @@ -0,0 +1,7 @@ +{ + "fname" : "book", + "ftable" : "book", + "fschema$" : "testdb", + "description" : "valid schema for a book", + "cardinality" : "invalid" +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/valid/table.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/valid/table.hjson new file mode 100644 index 0000000000..8f0b8014d1 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/valid/table.hjson @@ -0,0 +1,48 @@ +{ + tables: [{ + name: PlayerStats + table: player_stats + schema: gamedb + description: + ''' + A long description + ''' + cardinality : large + readAccess : A user is admin or is a player in the game + joins: [ + { + name: playerCountry + to: country + type: toOne + definition: '${to}.id = ${from}.country_id' + } + ] + measures : [ + { + name : highScore + type : INTEGER + definition: 'MAX(score)' + } + ] + dimensions : [ + { + name : countryCode + type : TEXT + definition : playerCountry.isoCode + }, + { + name : createdOn + type : TIME + definition : create_on + grains: [ + { + grain : MONTH + sql : ''' + PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd') + ''' + } + ] + } + ] + }] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/valid/table.json b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/valid/table.json new file mode 100644 index 0000000000..309f59ae45 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/tables/valid/table.json @@ -0,0 +1,63 @@ +{ + "tables": [ + { + "name": "PlayerStats", + "table": "player_stats", + "schema": "gamedb", + "cardinality" : "large", + "readAccess" : "A user is admin or is a player in the game", + "joins": [ + { + "name": "playerCountry", + "to": "country", + "type": "toOne", + "definition": "${to}.id = ${from}.country_id" + } + ], + "measures" : [ + { + "name" : "highScore", + "type" : "INTEGER", + "definition": "MAX(score)" + }, + { + "name" : "highScoreCoord", + "type" : "COORDINATE", + "definition": "MAX(score)" + } + ], + "dimensions" : [ + { + "name" : "countryCode", + "type" : "RELATIONSHIP", + "definition" : "playerCountry.isoCode" + }, + { + "name" : "countryCode", + "type" : "TEXT", + "definition" : "playerCountry.isoCode" + }, + { + "name" : "createdOn", + "type" : "TIME", + "definition" : "create_on", + "grains":[{ + + "grain": "MONTH", + "sql": "PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd')" + }] + }, + { + "name" : "createdOn", + "type" : "TIME", + "definition" : "create_on", + "grains":[{ + + "grain": "MONTH", + "sql": "PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd')" + }] + } + ] + } + ] +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/invalid/variables.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/invalid/variables.hjson new file mode 100644 index 0000000000..d166716f97 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/invalid/variables.hjson @@ -0,0 +1,11 @@ +{ + //Comment + name : book + table : book + schema$ : [123] + description : + ''' + valid schema for a book + ''' + cardinality : small +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/invalid/variables.json b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/invalid/variables.json new file mode 100644 index 0000000000..c1e1422253 --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/invalid/variables.json @@ -0,0 +1,8 @@ + +{ + "name!" : "book", + "table" : "book", + "schema$" : "testdb", + "description" : "valid schema for a book", + "cardinality" : "invalid" +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/valid/variables.hjson b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/valid/variables.hjson new file mode 100644 index 0000000000..e0358bfc8b --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/valid/variables.hjson @@ -0,0 +1,9 @@ +{ + desc_blah: blah blah blah + def_on: create_on + grain_dmy: ["{{day}}", "{{month}}", "{{year}}"] + grain_hd: ["{{hour}}", "{{day}}"] + foo: [1, 2, 3] + foobar: "[1, 2, 3]" + nullCheck: null +} diff --git a/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/valid/variables.json b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/valid/variables.json new file mode 100644 index 0000000000..0d0b85e3fe --- /dev/null +++ b/elide-contrib/elide-dynamic-config-helpers/src/test/resources/variables/valid/variables.json @@ -0,0 +1,11 @@ +{ + "desc_blah": "blah blah blah", + "def_on_test": "create_on", + "grain_dmy": "[{{day}}, {{month}}, {{year}}]", + "grain_hd": ["{{hour}}", "{{day}}"], + "nullCheck": null, + "grainVariable":[{ + "grain": "MONTH", + "sql": "PARSEDATETIME(FORMATDATETIME(${column}, 'yyyy-MM-01'), 'yyyy-MM-dd')" + }] +} diff --git a/elide-contrib/pom.xml b/elide-contrib/pom.xml index dcceac4347..a4da51c46d 100644 --- a/elide-contrib/pom.xml +++ b/elide-contrib/pom.xml @@ -46,6 +46,7 @@ elide-swagger elide-test-helpers + elide-dynamic-config-helpers diff --git a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java index f9f530356e..5c4274c016 100644 --- a/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java +++ b/elide-core/src/main/java/com/yahoo/elide/core/EntityDictionary.java @@ -1175,7 +1175,7 @@ private boolean isClassBound(Class objClass) { } /** - * Check whether a class is a JPA entity + * Check whether a class is a JPA entity. * * @param objClass class * @return True if it is a JPA entity @@ -1256,7 +1256,22 @@ public void scanForSecurityChecks() { // /elide-spring-boot-autoconfigure/src/main/java/org/illyasviel/elide // /spring/boot/autoconfigure/ElideAutoConfiguration.java - for (Class cls : ClassScanner.getAnnotatedClasses(SecurityCheck.class)) { + Set> classes = ClassScanner.getAnnotatedClasses(SecurityCheck.class); + + addSecurityChecks(classes); + } + + /** + * Add security checks and bind them to the dictionary. + * @param classes Security check classes. + */ + public void addSecurityChecks(Set> classes) { + + if (classes == null && classes.size() == 0) { + return; + } + + for (Class cls : classes) { if (Check.class.isAssignableFrom(cls)) { SecurityCheck securityCheckMeta = cls.getAnnotation(SecurityCheck.class); log.debug("Register Elide Check [{}] with expression [{}]", diff --git a/elide-datastore/elide-datastore-aggregation/pom.xml b/elide-datastore/elide-datastore-aggregation/pom.xml index 163e9a7991..7ce98f61fd 100644 --- a/elide-datastore/elide-datastore-aggregation/pom.xml +++ b/elide-datastore/elide-datastore-aggregation/pom.xml @@ -60,11 +60,18 @@ elide-graphql 5.0.0-pr9-SNAPSHOT + com.yahoo.elide elide-datastore-multiplex 5.0.0-pr9-SNAPSHOT + + + com.yahoo.elide + elide-dynamic-config-helpers + 5.0.0-pr9-SNAPSHOT + org.projectlombok diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java index 920337c807..12de280989 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/AggregationDataStore.java @@ -20,12 +20,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Set; /** * DataStore that supports Aggregation. Uses {@link QueryEngine} to return results. */ public class AggregationDataStore implements DataStore { private QueryEngine queryEngine; + private Set> dynamicCompiledClasses; /** * These are the classes the Aggregation Store manages. @@ -37,12 +39,23 @@ public AggregationDataStore(QueryEngine queryEngine) { this.queryEngine = queryEngine; } + public AggregationDataStore(QueryEngine queryEngine, Set> dynamicCompiledClasses) { + this.queryEngine = queryEngine; + this.dynamicCompiledClasses = dynamicCompiledClasses; + } + /** * Populate an {@link EntityDictionary} and use this dictionary to construct a {@link QueryEngine}. * @param dictionary the dictionary */ @Override public void populateEntityDictionary(EntityDictionary dictionary) { + + if (dynamicCompiledClasses != null && dynamicCompiledClasses.size() != 0) { + dynamicCompiledClasses.forEach(dynamicLoadedClass -> dictionary.bindEntity(dynamicLoadedClass, + Collections.singleton(Join.class))); + } + for (Class annotation : AGGREGATION_STORE_CLASSES) { // bind non-jpa entity tables ClassScanner.getAnnotatedClasses(annotation) diff --git a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java index d8bc27b4a1..53bad91a17 100644 --- a/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java +++ b/elide-datastore/elide-datastore-aggregation/src/main/java/com/yahoo/elide/datastores/aggregation/metadata/MetaDataStore.java @@ -5,6 +5,7 @@ */ package com.yahoo.elide.datastores.aggregation.metadata; +import com.yahoo.elide.contrib.dynamicconfighelpers.compile.ElideDynamicEntityCompiler; import com.yahoo.elide.core.EntityDictionary; import com.yahoo.elide.core.Path; import com.yahoo.elide.core.datastore.inmemory.HashMapDataStore; @@ -55,6 +56,20 @@ public MetaDataStore() { this(ClassScanner.getAnnotatedClasses(METADATA_STORE_ANNOTATIONS)); } + public MetaDataStore(ElideDynamicEntityCompiler compiler) throws ClassNotFoundException { + this(); + + Set> dynamicCompiledClasses = compiler.findAnnotatedClasses(FromTable.class); + dynamicCompiledClasses.addAll(compiler.findAnnotatedClasses(FromSubquery.class)); + + if (dynamicCompiledClasses != null && dynamicCompiledClasses.size() != 0) { + dynamicCompiledClasses.forEach(dynamicCompiledClass -> { + this.dictionary.bindEntity(dynamicCompiledClass, Collections.singleton(Join.class)); + this.modelsToBind.add(dynamicCompiledClass); + }); + } + } + /** * Construct MetaDataStore with data models. * diff --git a/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/PersistenceUnitInfoImpl.java b/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/PersistenceUnitInfoImpl.java new file mode 100644 index 0000000000..e66d8b215d --- /dev/null +++ b/elide-datastore/elide-datastore-jpa/src/main/java/com/yahoo/elide/datastores/jpa/PersistenceUnitInfoImpl.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.datastores.jpa; + +import lombok.Data; + +import java.net.URL; +import java.util.List; +import java.util.Properties; + +import javax.persistence.SharedCacheMode; +import javax.persistence.ValidationMode; +import javax.persistence.spi.ClassTransformer; +import javax.persistence.spi.PersistenceUnitInfo; +import javax.persistence.spi.PersistenceUnitTransactionType; +import javax.sql.DataSource; + +/** + * Persistent Unit implementation for Dynamic Configuration. + */ +@Data +public class PersistenceUnitInfoImpl implements PersistenceUnitInfo { + + private String persistenceUnitName; + private String persistenceProviderClassName; + private PersistenceUnitTransactionType transactionType; + private DataSource jtaDataSource; + private DataSource nonJtaDataSource; + private List mappingFileNames; + private List jarFileUrls; + private URL persistenceUnitRootUrl; + private List managedClassNames; + private SharedCacheMode sharedCacheMode; + private ValidationMode validationMode; + private Properties properties; + private String persistenceXMLSchemaVersion; + private ClassLoader classLoader; + private ClassLoader newTempClassLoader; + + public PersistenceUnitInfoImpl( + String persistenceUnitName, + List managedClassNames, + Properties properties, + ClassLoader loader) { + this.persistenceUnitName = persistenceUnitName; + this.managedClassNames = managedClassNames; + this.properties = properties; + this.classLoader = loader; + this.newTempClassLoader = loader; + } + + public PersistenceUnitInfoImpl(String persistenceUnitName, List managedClassNames, Properties properties) { + this.persistenceUnitName = persistenceUnitName; + this.managedClassNames = managedClassNames; + this.properties = properties; + } + + + @Override + public boolean excludeUnlistedClasses() { + return false; + } + + @Override + public void addTransformer(ClassTransformer classTransformer) { + //Not implemented + } +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/pom.xml b/elide-spring/elide-spring-boot-autoconfigure/pom.xml index 18f73f767b..58010ba467 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/pom.xml +++ b/elide-spring/elide-spring-boot-autoconfigure/pom.xml @@ -99,6 +99,13 @@ 5.0.0-pr9-SNAPSHOT true + + + com.yahoo.elide + elide-dynamic-config-helpers + 5.0.0-pr9-SNAPSHOT + true + org.projectlombok @@ -181,6 +188,12 @@ spring-boot-starter-test ${spring.boot.version} test + + + com.vaadin.external.google + android-json + + diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/DynamicConfigProperties.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/DynamicConfigProperties.java new file mode 100644 index 0000000000..e7e8b3e064 --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/DynamicConfigProperties.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.spring.config; + +import lombok.Data; + +/** + * Extra properties for setting up dynamic model config. + */ +@Data +public class DynamicConfigProperties { + + /** + * Whether or not dynamic model config is enabled. + */ + private boolean enabled = false; + + /** + * The path where the config hjsons are stored. + */ + private String path = "/"; + +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java index fd5260692e..529d219709 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAsyncConfiguration.java @@ -41,6 +41,8 @@ public class ElideAsyncConfiguration { * Configure the AsyncExecutorService used for submitting async query requests. * @param elide elideObject. * @param settings Elide settings. + * @param asyncQueryDao AsyncDao object. + * @param dictionary EntityDictionary. * @return a AsyncExecutorService. */ @Bean @@ -64,6 +66,7 @@ public AsyncExecutorService buildAsyncExecutorService(Elide elide, ElideConfigPr * Configure the AsyncCleanerService used for cleaning up async query requests. * @param elide elideObject. * @param settings Elide settings. + * @param asyncQueryDao AsyncDao object. * @return a AsyncCleanerService. */ @Bean diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java index 505ad44283..1a87c33db8 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideAutoConfiguration.java @@ -8,7 +8,9 @@ import com.yahoo.elide.Elide; import com.yahoo.elide.ElideSettingsBuilder; import com.yahoo.elide.Injector; +import com.yahoo.elide.annotation.SecurityCheck; import com.yahoo.elide.audit.Slf4jLogger; +import com.yahoo.elide.contrib.dynamicconfighelpers.compile.ElideDynamicEntityCompiler; import com.yahoo.elide.contrib.swagger.SwaggerBuilder; import com.yahoo.elide.core.DataStore; import com.yahoo.elide.core.EntityDictionary; @@ -17,10 +19,13 @@ import com.yahoo.elide.datastores.aggregation.QueryEngine; import com.yahoo.elide.datastores.aggregation.metadata.MetaDataStore; import com.yahoo.elide.datastores.aggregation.queryengines.sql.SQLQueryEngine; +import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromSubquery; +import com.yahoo.elide.datastores.aggregation.queryengines.sql.annotation.FromTable; import com.yahoo.elide.datastores.jpa.JpaDataStore; import com.yahoo.elide.datastores.jpa.transaction.NonJtaTransaction; import com.yahoo.elide.datastores.multiplex.MultiplexManager; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.config.AutowireCapableBeanFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -29,8 +34,10 @@ import io.swagger.models.Info; import io.swagger.models.Swagger; +import lombok.extern.slf4j.Slf4j; import java.util.HashMap; +import java.util.Set; import java.util.TimeZone; import javax.persistence.EntityManagerFactory; @@ -40,8 +47,27 @@ */ @Configuration @EnableConfigurationProperties(ElideConfigProperties.class) +@Slf4j public class ElideAutoConfiguration { + /** + * Creates a entity compiler for compiling dynamic config classes. + * @param settings Config Settings. + * @return An instance of ElideDynamicEntityCompiler. + * @throws Exception Exception thrown. + */ + @Bean + @ConditionalOnMissingBean + public ElideDynamicEntityCompiler buildElideDynamicEntityCompiler(ElideConfigProperties settings) throws Exception { + + ElideDynamicEntityCompiler compiler = null; + + if (settings.getDynamicConfig().isEnabled()) { + compiler = new ElideDynamicEntityCompiler(settings.getDynamicConfig().getPath()); + } + return compiler; + } + /** * Creates the Elide instance with standard settings. * @param dictionary Stores the static metadata about Elide models. @@ -52,8 +78,7 @@ public class ElideAutoConfiguration { @Bean @ConditionalOnMissingBean public Elide initializeElide(EntityDictionary dictionary, - DataStore dataStore, - ElideConfigProperties settings) { + DataStore dataStore, ElideConfigProperties settings) { ElideSettingsBuilder builder = new ElideSettingsBuilder(dataStore) .withEntityDictionary(dictionary) @@ -71,11 +96,16 @@ public Elide initializeElide(EntityDictionary dictionary, * Creates the entity dictionary for Elide which contains static metadata about Elide models. * Override to load check classes or life cycle hooks. * @param beanFactory Injector to inject Elide models. + * @param dynamicCompiler An instance of objectprovider for ElideDynamicEntityCompiler. + * @param settings Elide configuration settings. * @return a newly configured EntityDictionary. + * @throws ClassNotFoundException Exception thrown. */ @Bean @ConditionalOnMissingBean - public EntityDictionary buildDictionary(AutowireCapableBeanFactory beanFactory) { + public EntityDictionary buildDictionary(AutowireCapableBeanFactory beanFactory, + ObjectProvider dynamicCompiler, ElideConfigProperties settings) + throws ClassNotFoundException { EntityDictionary dictionary = new EntityDictionary(new HashMap<>(), new Injector() { @Override @@ -90,18 +120,37 @@ public T instantiate(Class cls) { }); dictionary.scanForSecurityChecks(); + + if (settings.getDynamicConfig().isEnabled()) { + ElideDynamicEntityCompiler compiler = dynamicCompiler.getIfAvailable(); + Set> annotatedClass = compiler.findAnnotatedClasses(SecurityCheck.class); + dictionary.addSecurityChecks(annotatedClass); + } + return dictionary; } /** * Create a QueryEngine instance for aggregation data store to use. * @param entityManagerFactory The JPA factory which creates entity managers. + * @param dynamicCompiler An instance of objectprovider for ElideDynamicEntityCompiler. + * @param settings Elide configuration settings. * @return An instance of a QueryEngine + * @throws ClassNotFoundException Exception thrown. */ @Bean @ConditionalOnMissingBean - public QueryEngine buildQueryEngine(EntityManagerFactory entityManagerFactory) { - MetaDataStore metaDataStore = new MetaDataStore(); + public QueryEngine buildQueryEngine(EntityManagerFactory entityManagerFactory, + ObjectProvider dynamicCompiler, ElideConfigProperties settings) + throws ClassNotFoundException { + + MetaDataStore metaDataStore = null; + + if (settings.getDynamicConfig().isEnabled()) { + metaDataStore = new MetaDataStore(dynamicCompiler.getIfAvailable()); + } else { + metaDataStore = new MetaDataStore(); + } return new SQLQueryEngine(metaDataStore, entityManagerFactory, null); } @@ -109,13 +158,27 @@ public QueryEngine buildQueryEngine(EntityManagerFactory entityManagerFactory) { /** * Creates the DataStore Elide. Override to use a different store. * @param entityManagerFactory The JPA factory which creates entity managers. - * @param queryEngine QueryEngine instance for aggregation data store + * @param queryEngine QueryEngine instance for aggregation data store. + * @param dynamicCompiler An instance of objectprovider for ElideDynamicEntityCompiler. + * @param settings Elide configuration settings. * @return An instance of a JPA DataStore. + * @throws ClassNotFoundException Exception thrown. */ @Bean @ConditionalOnMissingBean - public DataStore buildDataStore(EntityManagerFactory entityManagerFactory, QueryEngine queryEngine) { - AggregationDataStore aggregationDataStore = new AggregationDataStore(queryEngine); + public DataStore buildDataStore(EntityManagerFactory entityManagerFactory, QueryEngine queryEngine, + ObjectProvider dynamicCompiler, ElideConfigProperties settings) + throws ClassNotFoundException { + AggregationDataStore aggregationDataStore = null; + + if (settings.getDynamicConfig().isEnabled()) { + ElideDynamicEntityCompiler compiler = dynamicCompiler.getIfAvailable(); + Set> annotatedClass = compiler.findAnnotatedClasses(FromTable.class); + annotatedClass.addAll(compiler.findAnnotatedClasses(FromSubquery.class)); + aggregationDataStore = new AggregationDataStore(queryEngine, annotatedClass); + } else { + aggregationDataStore = new AggregationDataStore(queryEngine); + } JpaDataStore jpaDataStore = new JpaDataStore( () -> { return entityManagerFactory.createEntityManager(); }, diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideConfigProperties.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideConfigProperties.java index 38d9aced4b..ba034ebfd4 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideConfigProperties.java +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideConfigProperties.java @@ -36,6 +36,11 @@ public class ElideConfigProperties { */ private AsyncProperties async; + /** + * Settings for the Dynamic Configuration. + */ + private DynamicConfigProperties dynamicConfig; + /** * Default pagination size for collections if the client doesn't paginate. */ diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideDynamicConfiguration.java b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideDynamicConfiguration.java new file mode 100644 index 0000000000..dd069e70a1 --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/java/com/yahoo/elide/spring/config/ElideDynamicConfiguration.java @@ -0,0 +1,158 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package com.yahoo.elide.spring.config; + +import com.yahoo.elide.contrib.dynamicconfighelpers.compile.ElideDynamicEntityCompiler; +import com.yahoo.elide.datastores.jpa.PersistenceUnitInfoImpl; +import com.yahoo.elide.utils.ClassScanner; + +import org.hibernate.cfg.AvailableSettings; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateProperties; +import org.springframework.boot.autoconfigure.orm.jpa.HibernateProperties.Naming; +import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean; +import org.springframework.orm.jpa.persistenceunit.PersistenceUnitManager; +import org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter; + +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import javax.persistence.Entity; +import javax.persistence.spi.PersistenceUnitInfo; +import javax.sql.DataSource; + +/** + * Dynamic Configuration For Elide Services. Override any of the beans (by + * defining your own) and setting flags to disable in properties to change the + * default behavior. + */ + +@Slf4j +@Configuration +@EnableConfigurationProperties(ElideConfigProperties.class) +@ConditionalOnExpression("${elide.dynamic-config.enabled:false}") +public class ElideDynamicConfiguration { + + public static final String HIBERNATE_DDL_AUTO = "hibernate.hbm2ddl.auto"; + public static final String HIBERNATE_PHYSICAL_NAMING = "hibernate.physical_naming_strategy"; + public static final String HIBERNATE_IMPLICIT_NAMING = "hibernate.implicit_naming_strategy"; + public static final String HIBERNATE_ID_GEN_MAPPING = "hibernate.use-new-id-generator-mappings"; + + /** + * Configure factory bean to create EntityManagerFactory for Dynamic Configuration. + * @param source :DataSource for JPA + * @param jpaProperties : JPA Config Properties + * @param hibernateProperties : Hibernate Config Properties + * @param dynamicCompiler : ElideDynamicEntityCompiler + * @return LocalContainerEntityManagerFactoryBean bean + */ + @Bean + public LocalContainerEntityManagerFactoryBean entityManagerFactory ( + DataSource source, + JpaProperties jpaProperties, + HibernateProperties hibernateProperties, + ObjectProvider dynamicCompiler) { + + //Map for Persistent Unit properties + Map puiPropertyMap = new HashMap<>(); + + //Bind entity classes from classpath to Persistence Unit + ArrayList bindClasses = new ArrayList<>(); + bindClasses.addAll(ClassScanner.getAnnotatedClasses(Entity.class)); + + //Map of JPA Properties to be be passed to EntityManager + Map jpaPropMap = jpaProperties.getProperties(); + + String hibernateGetDDLAuto = hibernateProperties.getDdlAuto(); + Naming hibernateGetNaming = hibernateProperties.getNaming(); + String hibernateImplicitStrategy = hibernateGetNaming.getImplicitStrategy(); + String hibernatePhysicalStrategy = hibernateGetNaming.getPhysicalStrategy(); + Boolean hibernateGetIdenGen = hibernateProperties.isUseNewIdGeneratorMappings(); + + //Set the relevant property in JPA corresponding to Hibernate Property Value + hibernateJPAPropertyOverride(jpaPropMap, HIBERNATE_DDL_AUTO, hibernateGetDDLAuto); + hibernateJPAPropertyOverride(jpaPropMap, HIBERNATE_PHYSICAL_NAMING, hibernatePhysicalStrategy); + hibernateJPAPropertyOverride(jpaPropMap, HIBERNATE_IMPLICIT_NAMING, hibernateImplicitStrategy); + hibernateJPAPropertyOverride(jpaPropMap, HIBERNATE_ID_GEN_MAPPING, + (hibernateGetIdenGen != null) ? hibernateGetIdenGen.toString() : null); + + ElideDynamicEntityCompiler compiler = dynamicCompiler.getIfAvailable(); + + Collection classLoaders = new ArrayList<>(); + classLoaders.add(compiler.getClassLoader()); + + //Add dynamic classes to Pui Map + puiPropertyMap.put(AvailableSettings.CLASSLOADERS, classLoaders); + //Add classpath entity model classes to Pui Map + puiPropertyMap.put(AvailableSettings.LOADED_CLASSES, bindClasses); + + //pui properties from pui map + Properties puiProps = new Properties(); + puiProps.putAll(puiPropertyMap); + + //Create Elide dynamic Persistence Unit + PersistenceUnitInfoImpl elideDynamicPersistenceUnit = + new PersistenceUnitInfoImpl("dynamic", compiler.classNames, puiProps, + compiler.getClassLoader()); + elideDynamicPersistenceUnit.setNonJtaDataSource(source); + elideDynamicPersistenceUnit.setJtaDataSource(source); + + HibernateJpaVendorAdapter vendorAdapter = new HibernateJpaVendorAdapter(); + vendorAdapter.setShowSql(jpaProperties.isShowSql()); + vendorAdapter.setGenerateDdl(jpaProperties.isGenerateDdl()); + if (jpaProperties.getDatabase() != null) { + vendorAdapter.setDatabase(jpaProperties.getDatabase()); + } + if (jpaProperties.getDatabasePlatform() != null) { + vendorAdapter.setDatabasePlatform(jpaProperties.getDatabasePlatform()); + } + + LocalContainerEntityManagerFactoryBean bean = new LocalContainerEntityManagerFactoryBean(); + bean.setJpaVendorAdapter(vendorAdapter); + + //Add JPA Properties from Application.yaml + bean.setJpaPropertyMap(jpaPropMap); + + //Add Classes + bean.setJpaPropertyMap(puiPropertyMap); + + bean.setPersistenceUnitManager(new PersistenceUnitManager() { + @Override + public PersistenceUnitInfo obtainDefaultPersistenceUnitInfo() throws IllegalStateException { + return elideDynamicPersistenceUnit; + } + + @Override + public PersistenceUnitInfo obtainPersistenceUnitInfo(String persistenceUnitName) + throws IllegalArgumentException, IllegalStateException { + return elideDynamicPersistenceUnit; + } + }); + + return bean; + } + + /** + * Override Hibernate properties in application.yaml with jpa hibernate properties. + */ + private void hibernateJPAPropertyOverride(Map jpaPropMap, + String jpaPropertyName, String hibernateProperty) { + if (jpaPropMap.get(jpaPropertyName) == null && hibernateProperty != null) { + jpaPropMap.put(jpaPropertyName, hibernateProperty); + } + + } +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories b/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories index beaec82177..89e2683d18 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories +++ b/elide-spring/elide-spring-boot-autoconfigure/src/main/resources/META-INF/spring.factories @@ -1,6 +1,7 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ com.yahoo.elide.spring.config.ElideAutoConfiguration, \ com.yahoo.elide.spring.config.ElideAsyncConfiguration, \ + com.yahoo.elide.spring.config.ElideDynamicConfiguration, \ com.yahoo.elide.spring.controllers.JsonApiController, \ com.yahoo.elide.spring.controllers.GraphqlController, \ com.yahoo.elide.spring.controllers.SwaggerController diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java new file mode 100644 index 0000000000..366ab2f3bb --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/java/example/tests/DynamicConfigTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020, Yahoo Inc. + * Licensed under the Apache License, Version 2.0 + * See LICENSE file in project root for terms. + */ +package example.tests; + +import static com.yahoo.elide.contrib.testhelpers.jsonapi.JsonApiDSL.attr; +import static com.yahoo.elide.contrib.testhelpers.jsonapi.JsonApiDSL.attributes; +import static com.yahoo.elide.contrib.testhelpers.jsonapi.JsonApiDSL.data; +import static com.yahoo.elide.contrib.testhelpers.jsonapi.JsonApiDSL.id; +import static com.yahoo.elide.contrib.testhelpers.jsonapi.JsonApiDSL.resource; +import static com.yahoo.elide.contrib.testhelpers.jsonapi.JsonApiDSL.type; +import static io.restassured.RestAssured.when; +import static org.hamcrest.CoreMatchers.hasItems; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.yahoo.elide.core.HttpStatus; + +import org.junit.jupiter.api.Test; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlMergeMode; + +/** + * Dynamic Configuration functional test. + */ +@SqlMergeMode(SqlMergeMode.MergeMode.MERGE) +@Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + statements = "CREATE TABLE PlayerStats (name varchar(255) not null," + + "\t\t countryId varchar(255), createdOn timestamp, " + + "\t\t highScore bigint, primary key (name));" + + "CREATE TABLE PlayerCountry (id varchar(255) not null," + + "\t\t isoCode varchar(255), primary key (id));" + + "INSERT INTO PlayerStats (name,countryId,createdOn) VALUES\n" + + "\t\t('SerenaWilliams','1','2000-10-01');" + + "INSERT INTO PlayerCountry (id,isoCode) VALUES\n" + + "\t\t('2','IND');" + + "INSERT INTO PlayerCountry (id,isoCode) VALUES\n" + + "\t\t('1','USA');") +@Sql(executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD, +statements = "DROP TABLE PlayerStats; DROP TABLE PlayerCountry;") +public class DynamicConfigTest extends IntegrationTest { + /** + * This test demonstrates an example test using the JSON-API DSL. + * @throws InterruptedException + */ + + @Test + public void jsonApiGetTest() { + String apiGetViewRequest = when() + .get("/json/playerStats") + .then() + .body(equalTo( + data( + resource( + type("playerStats"), + id("0"), + attributes( + attr("countryCode", "USA"), + attr("createdOn", "2000-10-01T04:00Z"), + attr("highScore", null), + attr("name", "SerenaWilliams") + ) + ) + ).toJSON()) + ) + .statusCode(HttpStatus.SC_OK).extract().response().asString(); + String apiGetViewExpected = "{\"data\":[{\"type\":\"playerStats\",\"id\":\"0\",\"attributes\":{\"countryCode\":\"USA\",\"createdOn\":\"2000-10-01T04:00Z\",\"highScore\":null,\"name\":\"SerenaWilliams\"}}]}"; + assertEquals(apiGetViewRequest, apiGetViewExpected); + } + + @SqlMergeMode(SqlMergeMode.MergeMode.MERGE) + @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, + statements = "INSERT INTO PlayerStats (name,countryId,createdOn) VALUES\n" + + "\t\t('SaniaMirza','2','2000-10-01');") + @Test + public void jsonApiGetMultiTest() { + when() + .get("/json/playerStats") + .then() + .body("data.id", hasItems("1")) + .body("data.attributes.name", hasItems("SaniaMirza", "SerenaWilliams")) + .body("data.attributes.countryCode", hasItems("USA", "IND")) + .statusCode(HttpStatus.SC_OK); + } +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml index d8464654f9..0403f64509 100644 --- a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/application.yaml @@ -20,10 +20,12 @@ elide: cleanupEnabled: true queryCleanupDays: 7 defaultAsyncQueryDAO: true - + dynamic-config: + path: src/test/resources/models + enabled: true spring: jpa: - show_sql: true + show-sql: true properties: hibernate: dialect: 'org.hibernate.dialect.H2Dialect' diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/models/tables/playerCountry.hjson b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/models/tables/playerCountry.hjson new file mode 100644 index 0000000000..79440bd20b --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/models/tables/playerCountry.hjson @@ -0,0 +1,19 @@ +{ + tables: [{ + name: PlayerCountry + table: PlayerCountry + description: + ''' + A long description + ''' + cardinality : small + readAccess : Prefab.Role.All + dimensions : [ + { + name : isoCode + type : TEXT + definition : isoCode + } + ] + }] +} diff --git a/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/models/tables/playerStats.hjson b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/models/tables/playerStats.hjson new file mode 100644 index 0000000000..32fc6ff8f6 --- /dev/null +++ b/elide-spring/elide-spring-boot-autoconfigure/src/test/resources/models/tables/playerStats.hjson @@ -0,0 +1,52 @@ +{ + tables: [{ + name: PlayerStats + table: PlayerStats + description: + ''' + A long description + ''' + cardinality : large + readAccess : Prefab.Role.All + joins: [ + { + name: playerCountry + to: PlayerCountry + type: toOne + definition: '%join.id = %from.countryId' + } + ] + measures : [ + { + name : highScore + type : INTEGER + definition: 'MAX(highScore)' + } + ] + dimensions : [ + { + name : name + type : TEXT + definition : name + }, + { + name : countryCode + type : TEXT + definition : '{{playerCountry.isoCode}}' + }, + { + name : createdOn + type : TIME + definition : createdOn + grains: [ + { + grain : MONTH + sql : ''' + PARSEDATETIME(FORMATDATETIME(%s, 'yyyy-MM-01'), 'yyyy-MM-dd') + ''' + } + ] + } + ] + }] +} diff --git a/pom.xml b/pom.xml index d3a3431e02..794c2990fb 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,7 @@ 2.10.3 2.30.1 5.6.2 + 1.6.0 3.6.10.Final 8.0.19 5.4.2.Final @@ -213,6 +214,13 @@ test + + org.junit.platform + junit-platform-launcher + ${version.junit.platform} + test + + com.h2database h2