diff --git a/common/src/main/java/org/opensearch/ml/common/CommonValue.java b/common/src/main/java/org/opensearch/ml/common/CommonValue.java index e06e552536..60d4dbad0e 100644 --- a/common/src/main/java/org/opensearch/ml/common/CommonValue.java +++ b/common/src/main/java/org/opensearch/ml/common/CommonValue.java @@ -44,15 +44,15 @@ public class CommonValue { public static final Set stopWordsIndices = ImmutableSet.of(".plugins-ml-stop-words"); // Index mapping paths - public static final String ML_MODEL_GROUP_INDEX_MAPPING_PATH = "index-mappings/ml-model-group.json"; - public static final String ML_MODEL_INDEX_MAPPING_PATH = "index-mappings/ml-model.json"; - public static final String ML_TASK_INDEX_MAPPING_PATH = "index-mappings/ml-task.json"; - public static final String ML_CONNECTOR_INDEX_MAPPING_PATH = "index-mappings/ml-connector.json"; - public static final String ML_CONFIG_INDEX_MAPPING_PATH = "index-mappings/ml-config.json"; - public static final String ML_CONTROLLER_INDEX_MAPPING_PATH = "index-mappings/ml-controller.json"; - public static final String ML_AGENT_INDEX_MAPPING_PATH = "index-mappings/ml-agent.json"; - public static final String ML_MEMORY_META_INDEX_MAPPING_PATH = "index-mappings/ml-memory-meta.json"; - public static final String ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH = "index-mappings/ml-memory-message.json"; + public static final String ML_MODEL_GROUP_INDEX_MAPPING_PATH = "index-mappings/ml_model_group.json"; + public static final String ML_MODEL_INDEX_MAPPING_PATH = "index-mappings/ml_model.json"; + public static final String ML_TASK_INDEX_MAPPING_PATH = "index-mappings/ml_task.json"; + public static final String ML_CONNECTOR_INDEX_MAPPING_PATH = "index-mappings/ml_connector.json"; + public static final String ML_CONFIG_INDEX_MAPPING_PATH = "index-mappings/ml_config.json"; + public static final String ML_CONTROLLER_INDEX_MAPPING_PATH = "index-mappings/ml_controller.json"; + public static final String ML_AGENT_INDEX_MAPPING_PATH = "index-mappings/ml_agent.json"; + public static final String ML_MEMORY_META_INDEX_MAPPING_PATH = "index-mappings/ml_memory_meta.json"; + public static final String ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH = "index-mappings/ml_memory_message.json"; // Calculate Versions independently of OpenSearch core version public static final Version VERSION_2_11_0 = Version.fromString("2.11.0"); diff --git a/common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java b/common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java index bec1771f68..05aa9e791a 100644 --- a/common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java +++ b/common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.net.URL; +import java.util.HashMap; import java.util.Map; import com.google.common.base.Charsets; @@ -73,34 +74,46 @@ public static String replacePlaceholders(String mapping) throws IOException { throw new IllegalArgumentException("Mapping cannot be null or empty"); } + // Preload resources into memory to avoid redundant I/O + Map loadedPlaceholders = new HashMap<>(); for (Map.Entry placeholder : MAPPING_PLACEHOLDERS.entrySet()) { URL url = IndexUtils.class.getClassLoader().getResource(placeholder.getValue()); if (url == null) { throw new IOException("Resource not found: " + placeholder.getValue()); } - String placeholderMapping = Resources.toString(url, Charsets.UTF_8); - mapping = mapping.replace(placeholder.getKey(), placeholderMapping); + loadedPlaceholders.put(placeholder.getKey(), Resources.toString(url, Charsets.UTF_8)); } - return mapping; + StringBuilder result = new StringBuilder(mapping); + for (Map.Entry entry : loadedPlaceholders.entrySet()) { + String placeholder = entry.getKey(); + String replacement = entry.getValue(); + + // Replace all occurrences of the placeholder + int index; + while ((index = result.indexOf(placeholder)) != -1) { + result.replace(index, index + placeholder.length(), replacement); + } + } + + return result.toString(); } /** - - Checks if mapping is a valid json - - Validates mapping against a schema found in mappings/schema.json - - Schema validates the following: - - Below fields are present: - - "_meta" - - "_meta.schema_version" - - "properties" - - No additional fields at root level - - No additional fields in "_meta" object - - "properties" is an object type - - "_meta" is an object type - - "_meta_.schema_version" provided type is integer - - Note: Validation can be made more strict if a specific schema is defined for each index. + * Checks if mapping is a valid json + * Validates mapping against a schema found in mappings/schema.json + * Schema validates the following: + * Below fields are present: + * "_meta" + * "_meta.schema_version" + * "properties" + * No additional fields at root level + * No additional fields in "_meta" object + * "properties" is an object type + * "_meta" is an object type + * "_meta_.schema_version" provided type is integer + * Note: Validation can be made more strict if a specific schema is defined for each index. */ public static void validateMapping(String mapping) throws IOException { if (mapping.isBlank() || !StringUtils.isJson(mapping)) { diff --git a/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java index ce377e482b..8879306773 100644 --- a/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/ml/common/utils/StringUtils.java @@ -5,14 +5,12 @@ package org.opensearch.ml.common.utils; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -20,6 +18,7 @@ import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.commons.lang3.BooleanUtils; import org.json.JSONArray; @@ -27,6 +26,7 @@ import org.json.JSONObject; import org.opensearch.OpenSearchParseException; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -347,25 +347,28 @@ public static JsonObject getJsonObjectFromString(String jsonString) { return JsonParser.parseString(jsonString).getAsJsonObject(); } - public static void validateSchema(String schemaString, String instanceString) throws IOException { - // parse the schema JSON as string - JsonNode schemaNode = MAPPER.readTree(schemaString); - JsonSchema schema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaNode); - - // JSON data to validate - JsonNode jsonNode = MAPPER.readTree(instanceString); - - // Validate JSON node against the schema - Set errors = schema.validate(jsonNode); - if (!errors.isEmpty()) { - throw new OpenSearchParseException( - "Validation failed: " - + Arrays.toString(errors.toArray(new ValidationMessage[0])) - + " for instance: " - + instanceString - + " with schema: " - + schemaString - ); + public static void validateSchema(String schemaString, String instanceString) { + try { + // parse the schema JSON as string + JsonNode schemaNode = MAPPER.readTree(schemaString); + JsonSchema schema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V202012).getSchema(schemaNode); + + // JSON data to validate + JsonNode jsonNode = MAPPER.readTree(instanceString); + + // Validate JSON node against the schema + Set errors = schema.validate(jsonNode); + if (!errors.isEmpty()) { + String errorMessage = errors.stream().map(ValidationMessage::getMessage).collect(Collectors.joining(", ")); + + throw new OpenSearchParseException( + "Validation failed: " + errorMessage + " for instance: " + instanceString + " with schema: " + schemaString + ); + } + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("Invalid JSON format: " + e.getMessage(), e); + } catch (Exception e) { + throw new OpenSearchParseException("Schema validation failed: " + e.getMessage(), e); } } } diff --git a/common/src/main/resources/index-mappings/ml-agent.json b/common/src/main/resources/index-mappings/ml_agent.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-agent.json rename to common/src/main/resources/index-mappings/ml_agent.json diff --git a/common/src/main/resources/index-mappings/ml-config.json b/common/src/main/resources/index-mappings/ml_config.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-config.json rename to common/src/main/resources/index-mappings/ml_config.json diff --git a/common/src/main/resources/index-mappings/ml-connector.json b/common/src/main/resources/index-mappings/ml_connector.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-connector.json rename to common/src/main/resources/index-mappings/ml_connector.json diff --git a/common/src/main/resources/index-mappings/ml-controller.json b/common/src/main/resources/index-mappings/ml_controller.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-controller.json rename to common/src/main/resources/index-mappings/ml_controller.json diff --git a/common/src/main/resources/index-mappings/ml-memory-message.json b/common/src/main/resources/index-mappings/ml_memory_message.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-memory-message.json rename to common/src/main/resources/index-mappings/ml_memory_message.json diff --git a/common/src/main/resources/index-mappings/ml-memory-meta.json b/common/src/main/resources/index-mappings/ml_memory_meta.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-memory-meta.json rename to common/src/main/resources/index-mappings/ml_memory_meta.json diff --git a/common/src/main/resources/index-mappings/ml-model.json b/common/src/main/resources/index-mappings/ml_model.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-model.json rename to common/src/main/resources/index-mappings/ml_model.json diff --git a/common/src/main/resources/index-mappings/ml-model-group.json b/common/src/main/resources/index-mappings/ml_model_group.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-model-group.json rename to common/src/main/resources/index-mappings/ml_model_group.json diff --git a/common/src/main/resources/index-mappings/ml-task.json b/common/src/main/resources/index-mappings/ml_task.json similarity index 100% rename from common/src/main/resources/index-mappings/ml-task.json rename to common/src/main/resources/index-mappings/ml_task.json diff --git a/common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java b/common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java index f9334db729..d55ac37fc1 100644 --- a/common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java +++ b/common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java @@ -67,23 +67,23 @@ public void testGetMappingFromFile() { + " }\n" + "}\n"; try { - String actualMapping = IndexUtils.getMappingFromFile("index-mappings/test-mapping.json"); + String actualMapping = IndexUtils.getMappingFromFile("index-mappings/test_mapping.json"); // comparing JsonObjects to avoid issues caused by eol character in different OS assertEquals(StringUtils.getJsonObjectFromString(expectedMapping), StringUtils.getJsonObjectFromString(actualMapping)); } catch (IOException e) { - throw new RuntimeException("Failed to read file at path: index-mappings/test-mapping.json"); + throw new RuntimeException("Failed to read file at path: index-mappings/test_mapping.json"); } } @Test public void testGetMappingFromFileFileNotFound() { - String path = "index-mappings/test-mapping-not-found.json"; + String path = "index-mappings/test_mapping_not_found.json"; IOException e = assertThrows(IOException.class, () -> IndexUtils.getMappingFromFile(path)); } @Test public void testGetMappingFromFilesMalformedJson() { - String path = "index-mappings/test-mapping-malformed.json"; + String path = "index-mappings/test_mapping_malformed.json"; IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> IndexUtils.getMappingFromFile(path)); } diff --git a/common/src/test/resources/index-mappings/test-mapping.json b/common/src/test/resources/index-mappings/test_mapping.json similarity index 100% rename from common/src/test/resources/index-mappings/test-mapping.json rename to common/src/test/resources/index-mappings/test_mapping.json diff --git a/common/src/test/resources/index-mappings/test-mapping-malformed.json b/common/src/test/resources/index-mappings/test_mapping_malformed.json similarity index 100% rename from common/src/test/resources/index-mappings/test-mapping-malformed.json rename to common/src/test/resources/index-mappings/test_mapping_malformed.json