From ee9641595d33d222d03406204d8bdc8140876a67 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 1 Mar 2024 08:10:57 -0500 Subject: [PATCH 01/37] #67 Working toward a unique field name. --- .../org/opensearch/ubi/SettingsConstants.java | 3 ++ .../UserBehaviorInsightsRestHandler.java | 4 ++- .../org/opensearch/ubi/backends/Backend.java | 2 +- .../ubi/backends/OpenSearchBackend.java | 35 ++++++++++++------- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/SettingsConstants.java b/src/main/java/org/opensearch/ubi/SettingsConstants.java index d4811e9..9bfa625 100644 --- a/src/main/java/org/opensearch/ubi/SettingsConstants.java +++ b/src/main/java/org/opensearch/ubi/SettingsConstants.java @@ -13,4 +13,7 @@ public class SettingsConstants { public static final String INDEX_NAMES = "plugins.ubi.indices"; public static final String VERSION_SETTING = "index.ubistore.version"; + public static final String INDEX = "ubi.store.index"; + public static final String ID_FIELD = "ubi.store.id_field"; + } diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java index 24f7bb9..38731b6 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java @@ -60,6 +60,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod if (request.method() == PUT) { final String storeName = request.param("store"); + final String index = request.param("index"); + final String idField = request.param("id_field"); // Validate the store name. if(!backend.validateStoreName(storeName)) { @@ -72,7 +74,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod /*if(backend.exists(storeName)) { channel.sendResponse(new BytesRestResponse(RestStatus.CONFLICT, "already exists")); } else {*/ - backend.initialize(storeName); + backend.initialize(storeName, index, idField); channel.sendResponse(new BytesRestResponse(RestStatus.OK, "created")); //} }; diff --git a/src/main/java/org/opensearch/ubi/backends/Backend.java b/src/main/java/org/opensearch/ubi/backends/Backend.java index b11016c..6640156 100644 --- a/src/main/java/org/opensearch/ubi/backends/Backend.java +++ b/src/main/java/org/opensearch/ubi/backends/Backend.java @@ -15,7 +15,7 @@ public interface Backend { - void initialize(final String storeName); + void initialize(final String storeName, final String index, final String idField); void delete(final String storeName); diff --git a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java index 7a7428e..46fd047 100644 --- a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java +++ b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java @@ -51,25 +51,44 @@ public OpenSearchBackend(final Client client) { } @Override - public void initialize(final String storeName) { + public void initialize(final String storeName, final String index, final String idField) { LOGGER.info("Creating search relevance store {}", storeName); // Create the events index. final String eventsIndexName = getEventsIndexName(storeName); + final Settings eventsIndexSettings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") + .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + .put(SettingsConstants.INDEX, index) + .put(SettingsConstants.ID_FIELD, index) + .build(); + final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndexName) .mapping(getResourceFile(EVENTS_MAPPING_FILE)) - .settings(getIndexSettings()); + .settings(eventsIndexSettings); client.admin().indices().create(createEventsIndexRequest); // Create the queries index. final String queriesIndexName = getQueriesIndexName(storeName); + final Settings queriesIndexSettings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") + .put(SettingsConstants.VERSION_SETTING, VERSION) + .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + .put(SettingsConstants.INDEX, index) + .put(SettingsConstants.ID_FIELD, index) + .build(); + final CreateIndexRequest createQueryIndexRequest = new CreateIndexRequest(queriesIndexName) .mapping(getResourceFile(QUERIES_MAPPING_FILE)) - .settings(getIndexSettings()); + .settings(queriesIndexSettings); client.admin().indices().create(createQueryIndexRequest); @@ -189,14 +208,4 @@ private String getResourceFile(final String fileName) { } } - private Settings getIndexSettings() { - return Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") - .put(SettingsConstants.VERSION_SETTING, VERSION) - .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .build(); - } - } From c187835d735375732072de48f41038cc6c9e66fe Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 1 Mar 2024 08:11:33 -0500 Subject: [PATCH 02/37] #67 Working toward a unique field name. --- .../opensearch/ubi/action/UserBehaviorInsightsActionFilter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index e6f7084..ab20f51 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -92,6 +92,7 @@ public void onResponse(Response response) { final SearchResponse searchResponse = (SearchResponse) response; // Add each hit to the list of query responses. + // TODO: Get the name of the id_field from the settings. searchResponse.getHits().forEach(hit -> queryResponseHitIds.add(String.valueOf(hit.docId()))); try { From ef4309972c9bf80d6b339b669a2f09a44af5e686 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 1 Mar 2024 08:16:00 -0500 Subject: [PATCH 03/37] #67 Removing duplicate code. --- .../ubi/backends/OpenSearchBackend.java | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java index 46fd047..538b808 100644 --- a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java +++ b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java @@ -55,40 +55,31 @@ public void initialize(final String storeName, final String index, final String LOGGER.info("Creating search relevance store {}", storeName); - // Create the events index. - final String eventsIndexName = getEventsIndexName(storeName); - - final Settings eventsIndexSettings = Settings.builder() + final Settings indexSettings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) .put(SettingsConstants.INDEX, index) .put(SettingsConstants.ID_FIELD, index) + .put(SettingsConstants.VERSION_SETTING, VERSION) .build(); + // Create the events index. + final String eventsIndexName = getEventsIndexName(storeName); + final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndexName) .mapping(getResourceFile(EVENTS_MAPPING_FILE)) - .settings(eventsIndexSettings); + .settings(indexSettings); client.admin().indices().create(createEventsIndexRequest); // Create the queries index. final String queriesIndexName = getQueriesIndexName(storeName); - final Settings queriesIndexSettings = Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") - .put(SettingsConstants.VERSION_SETTING, VERSION) - .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .put(SettingsConstants.INDEX, index) - .put(SettingsConstants.ID_FIELD, index) - .build(); - final CreateIndexRequest createQueryIndexRequest = new CreateIndexRequest(queriesIndexName) .mapping(getResourceFile(QUERIES_MAPPING_FILE)) - .settings(queriesIndexSettings); + .settings(indexSettings); client.admin().indices().create(createQueryIndexRequest); From 084bfda8a1431a339a9518f9af6def3174bd33f7 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 4 Mar 2024 09:17:36 -0500 Subject: [PATCH 04/37] #67 Keeping the store's index and id_field in a settings map. --- .../ubi/UserBehaviorInsightsPlugin.java | 6 +- .../UserBehaviorInsightsActionFilter.java | 65 ++++++++++++------- .../ubi/backends/OpenSearchBackend.java | 8 +++ 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 1f5744c..c7b2813 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -35,10 +35,7 @@ import org.opensearch.ubi.events.OpenSearchEventManager; import org.opensearch.watcher.ResourceWatcherService; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -50,6 +47,7 @@ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private Backend backend; private ActionFilter userBehaviorLoggingFilter; + public static Map> storeSettings = new HashMap<>(); @Override public Collection getRestHeaders() { diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index ab20f51..e9b2f1d 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -18,13 +18,17 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; +import org.opensearch.index.reindex.ScrollableHitSource; +import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.ubi.HeaderConstants; +import org.opensearch.ubi.UserBehaviorInsightsPlugin; import org.opensearch.ubi.backends.Backend; import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.UUID; @@ -82,37 +86,54 @@ public void onResponse(Response response) { final String userId = getHeaderValue(HeaderConstants.USER_ID_HEADER, "", task); final String sessionId = getHeaderValue(HeaderConstants.SESSION_ID_HEADER, "", task); - // The query will be empty when there is no query, e.g. /_search - final String query = searchRequest.source().toString(); + // Get the store settings. + final String index = UserBehaviorInsightsPlugin.storeSettings.get(eventStore).get("index"); - // Create a UUID for this search response. - final String queryResponseId = UUID.randomUUID().toString(); + // Only consider this search if the index being searched matches the store's index setting. + if (Arrays.asList(searchRequest.indices()).contains(index)) { - final List queryResponseHitIds = new LinkedList<>(); - final SearchResponse searchResponse = (SearchResponse) response; + // The query will be empty when there is no query, e.g. /_search + final String query = searchRequest.source().toString(); - // Add each hit to the list of query responses. - // TODO: Get the name of the id_field from the settings. - searchResponse.getHits().forEach(hit -> queryResponseHitIds.add(String.valueOf(hit.docId()))); + // Create a UUID for this search response. + final String queryResponseId = UUID.randomUUID().toString(); - try { + final List queryResponseHitIds = new LinkedList<>(); + final SearchResponse searchResponse = (SearchResponse) response; - // Persist the query to the backend. - backend.persistQuery(eventStore, - new QueryRequest(queryId, query, userId, sessionId), - new QueryResponse(queryId, queryResponseId, queryResponseHitIds)); + // Get the id_field to use for each result's unique identifier. + final String idField = UserBehaviorInsightsPlugin.storeSettings.get(eventStore).getOrDefault("id_field", ""); - } catch (Exception ex) { - // TODO: Handle this. - LOGGER.error("Unable to persist query.", ex); - } + // Add each hit to the list of query responses. + for (final SearchHit hit : searchResponse.getHits()) { + if ("".equals(idField)) { + // Use the _id since there is no id_field setting for this index. + queryResponseHitIds.add(String.valueOf(hit.docId())); + } else { + queryResponseHitIds.add(String.valueOf(hit.field(idField))); + } + } + + try { + + // Persist the query to the backend. + backend.persistQuery(eventStore, + new QueryRequest(queryId, query, userId, sessionId), + new QueryResponse(queryId, queryResponseId, queryResponseHitIds)); - threadPool.getThreadContext().addResponseHeader("query_id", queryId); + } catch (Exception ex) { + // TODO: Handle this. + LOGGER.error("Unable to persist query.", ex); + } - //} + threadPool.getThreadContext().addResponseHeader("query_id", queryId); - final long elapsedTime = System.currentTimeMillis() - startTime; - LOGGER.info("UBI search request filter took {} ms", elapsedTime); + //} + + final long elapsedTime = System.currentTimeMillis() - startTime; + LOGGER.info("UBI search request filter took {} ms", elapsedTime); + + } } diff --git a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java index 538b808..aa695cb 100644 --- a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java +++ b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java @@ -21,6 +21,7 @@ import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ubi.SettingsConstants; +import org.opensearch.ubi.UserBehaviorInsightsPlugin; import org.opensearch.ubi.events.Event; import org.opensearch.ubi.events.OpenSearchEventManager; import org.opensearch.ubi.model.QueryRequest; @@ -83,6 +84,12 @@ public void initialize(final String storeName, final String index, final String client.admin().indices().create(createQueryIndexRequest); + // Store the settings. + final Map settings = new HashMap<>(); + settings.put("index", index); + settings.put("id_field", idField); + UserBehaviorInsightsPlugin.storeSettings.put(storeName, settings); + } @Override @@ -95,6 +102,7 @@ public void delete(String storeName) { final DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(eventsIndexName, queriesIndexName); client.admin().indices().delete(deleteIndexRequest); + UserBehaviorInsightsPlugin.storeSettings.remove(storeName); } From 0b2dd6e29c5f925fe061d1e7f4d4a2cbb9e8bac5 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 4 Mar 2024 14:18:53 -0500 Subject: [PATCH 05/37] #67 Working on event threading. --- .../ubi/UserBehaviorInsightsPlugin.java | 19 +-- .../UserBehaviorInsightsActionFilter.java | 6 +- .../UserBehaviorInsightsRestHandler.java | 160 +++++++++++++++--- .../org/opensearch/ubi/backends/Backend.java | 3 + .../ubi/backends/OpenSearchBackend.java | 36 +++- .../rest-api-spec/api/ubi.get_stores.json | 15 ++ .../test/_plugins.ubi/10_manage.yml | 41 +++-- 7 files changed, 218 insertions(+), 62 deletions(-) create mode 100644 src/yamlRestTest/resources/rest-api-spec/api/ubi.get_stores.json diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index c7b2813..6627300 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -47,7 +47,6 @@ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private Backend backend; private ActionFilter userBehaviorLoggingFilter; - public static Map> storeSettings = new HashMap<>(); @Override public Collection getRestHeaders() { @@ -97,7 +96,6 @@ public List> getSettings() { @Override public List getActionFilters() { - // LOGGER.info("Index name: {}", settings.get(ConfigConstants.INDEX_NAME)); return singletonList(userBehaviorLoggingFilter); } @@ -117,24 +115,21 @@ public Collection createComponents( ) { this.backend = new OpenSearchBackend(client); - this.userBehaviorLoggingFilter = new UserBehaviorInsightsActionFilter(backend, environment.settings(), threadPool); - LOGGER.info("Creating scheduled task"); - - // TODO: Only start this if an OpenSearch store is already initialized. + // TODO Only start this if an OpenSearch store is already initialized. // Otherwise, start it when a store is initialized. + + LOGGER.info("Creating UBI scheduled task to persist events."); + // TODO: Allow these time parameters to be configurable. threadPool.scheduler().scheduleAtFixedRate(() -> { OpenSearchEventManager.getInstance(client).process(); }, 0, 2000, TimeUnit.MILLISECONDS); + // Initialize the action filter. + this.userBehaviorLoggingFilter = new UserBehaviorInsightsActionFilter(backend, environment.settings(), threadPool); + return Collections.emptyList(); } -// @Override -// public void close() { -// LOGGER.info("Stopping scheduled runnable."); -// FutureUtils.cancel(scheduled); -// } - } diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index e9b2f1d..2b55199 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -18,12 +18,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; -import org.opensearch.index.reindex.ScrollableHitSource; import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.ubi.HeaderConstants; -import org.opensearch.ubi.UserBehaviorInsightsPlugin; import org.opensearch.ubi.backends.Backend; import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; @@ -87,7 +85,7 @@ public void onResponse(Response response) { final String sessionId = getHeaderValue(HeaderConstants.SESSION_ID_HEADER, "", task); // Get the store settings. - final String index = UserBehaviorInsightsPlugin.storeSettings.get(eventStore).get("index"); + final String index = backend.getStoreSettings().get(eventStore).get("index"); // Only consider this search if the index being searched matches the store's index setting. if (Arrays.asList(searchRequest.indices()).contains(index)) { @@ -102,7 +100,7 @@ public void onResponse(Response response) { final SearchResponse searchResponse = (SearchResponse) response; // Get the id_field to use for each result's unique identifier. - final String idField = UserBehaviorInsightsPlugin.storeSettings.get(eventStore).getOrDefault("id_field", ""); + final String idField = backend.getStoreSettings().get(eventStore).getOrDefault("id_field", ""); // Add each hit to the list of query responses. for (final SearchHit hit : searchResponse.getHits()) { diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java index 38731b6..99b84c5 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java @@ -15,16 +15,29 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.client.node.NodeClient; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.util.io.Streams; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; +import org.opensearch.ubi.SettingsConstants; import org.opensearch.ubi.backends.Backend; +import org.opensearch.ubi.backends.OpenSearchBackend; +import org.opensearch.ubi.events.Event; +import org.opensearch.ubi.events.OpenSearchEventManager; -import java.util.List; -import java.util.Objects; -import java.util.Set; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.*; import static org.opensearch.rest.RestRequest.Method.*; @@ -32,6 +45,11 @@ public class UserBehaviorInsightsRestHandler extends BaseRestHandler { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsRestHandler.class); + private static final String EVENTS_MAPPING_FILE = "events-mapping.json"; + private static final String QUERIES_MAPPING_FILE = "queries-mapping.json"; + public static final int VERSION = 1; + private Map> storeSettings = new HashMap<>(); + private final Backend backend; public UserBehaviorInsightsRestHandler(final Backend backend) { @@ -60,24 +78,64 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod if (request.method() == PUT) { final String storeName = request.param("store"); - final String index = request.param("index"); - final String idField = request.param("id_field"); + //final String index = request.param("index"); + //final String idField = request.param("id_field"); // Validate the store name. - if(!backend.validateStoreName(storeName)) { + /*if(!backend.validateStoreName(storeName)) { return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "missing store name")); - } + } else {*/ - LOGGER.info("Creating UBL store {}", storeName); + return (channel) -> { + LOGGER.info("Creating UBL store {}", storeName); - return (channel) -> { /*if(backend.exists(storeName)) { channel.sendResponse(new BytesRestResponse(RestStatus.CONFLICT, "already exists")); } else {*/ - backend.initialize(storeName, index, idField); - channel.sendResponse(new BytesRestResponse(RestStatus.OK, "created")); - //} - }; + //backend.initialize(storeName, index, idField); + + //channel.sendResponse(new BytesRestResponse(RestStatus.OK, "created")); + //} + + LOGGER.info("Creating search relevance store {}", storeName); + + final Settings indexSettings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") + .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + //.put(SettingsConstants.INDEX, index) + // .put(SettingsConstants.ID_FIELD, index) + .put(SettingsConstants.VERSION_SETTING, VERSION) + .build(); + + // Create the events index. + final String eventsIndexName = getEventsIndexName(storeName); + + final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndexName) + .mapping(getResourceFile(EVENTS_MAPPING_FILE)) + .settings(indexSettings); + + nodeClient.admin().indices().create(createEventsIndexRequest); + + // Create the queries index. + final String queriesIndexName = getQueriesIndexName(storeName); + + final CreateIndexRequest createQueryIndexRequest = new CreateIndexRequest(queriesIndexName) + .mapping(getResourceFile(QUERIES_MAPPING_FILE)) + .settings(indexSettings); + + nodeClient.admin().indices().create(createQueryIndexRequest); + + // Store the settings. + /*final Map settings = new HashMap<>(); + settings.put("index", index); + settings.put("id_field", idField); + storeSettings.put(storeName, settings);*/ + + }; + + // } } else if (request.method() == DELETE) { @@ -91,8 +149,18 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod LOGGER.info("Deleting UBL store {}", storeName); return (channel) -> { - backend.delete(storeName); - channel.sendResponse(new BytesRestResponse(RestStatus.OK, "created")); + //backend.delete(storeName); + + LOGGER.info("Deleting search relevance store {}", storeName); + + final String eventsIndexName = getEventsIndexName(storeName); + final String queriesIndexName = getQueriesIndexName(storeName); + final DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(eventsIndexName, queriesIndexName); + + nodeClient.admin().indices().delete(deleteIndexRequest); + //storeSettings.remove(storeName); + + channel.sendResponse(new BytesRestResponse(RestStatus.OK, "deleted")); }; } else if (request.method() == POST) { @@ -113,8 +181,16 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - backend.persistEvent(storeName, eventJsonWithTimestamp); - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); + //backend.persistEvent(storeName, eventJsonWithTimestamp); + //return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); + + // Add the event for indexing. + LOGGER.info("Indexing event into {}", storeName); + final String eventsIndexName = getEventsIndexName(storeName); + + //return (channel) -> client.index(indexRequest, new RestToXContentListener<>(channel)); + final Event event = new Event(eventsIndexName, eventJson); + OpenSearchEventManager.getInstance(nodeClient).add(event); } catch (JsonProcessingException ex) { LOGGER.error("Unable to get/set timestamp on event.", ex); @@ -127,10 +203,29 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nod } else if (request.method() == GET) { - final Set stores = backend.get(); - final String s = String.join(",", stores); + //final Set stores = backend.get(); + //final String s = String.join(",", stores); + + //return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, s)); - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, s)); + return (channel) -> { + + final GetIndexRequest getIndexRequest = new GetIndexRequest(); + final GetIndexResponse getIndexResponse = nodeClient.admin().indices().getIndex(getIndexRequest).actionGet(); + final String[] indexes = getIndexResponse.indices(); + final Set stores = new HashSet<>(); + + for (final String index : indexes) { + LOGGER.info("Index name: " + index); + if (index.startsWith(".") && index.endsWith("_queries")) { + stores.add(index); + } + } + + //return stores; + channel.sendResponse(new BytesRestResponse(RestStatus.OK, String.join(",", stores))); + + }; } @@ -154,4 +249,29 @@ private String setEventTimestamp(final String eventJson) throws JsonProcessingEx } + private boolean validateStoreName(final String storeName) { + + // Validate the store name. + return storeName != null && !storeName.isEmpty(); + + } + + private String getEventsIndexName(final String storeName) { + return "." + storeName + "_events"; + } + + private String getQueriesIndexName(final String storeName) { + return "." + storeName + "_queries"; + } + + private String getResourceFile(final String fileName) { + try (InputStream is = OpenSearchBackend.class.getResourceAsStream(fileName)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + return out.toString(StandardCharsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException("Unable to create index with resource [" + fileName + "]", e); + } + } + } diff --git a/src/main/java/org/opensearch/ubi/backends/Backend.java b/src/main/java/org/opensearch/ubi/backends/Backend.java index 6640156..3f57290 100644 --- a/src/main/java/org/opensearch/ubi/backends/Backend.java +++ b/src/main/java/org/opensearch/ubi/backends/Backend.java @@ -11,6 +11,7 @@ import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; +import java.util.Map; import java.util.Set; public interface Backend { @@ -29,4 +30,6 @@ public interface Backend { boolean validateStoreName(final String storeName); + Map> getStoreSettings(); + } diff --git a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java index aa695cb..ceccf13 100644 --- a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java +++ b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java @@ -14,6 +14,8 @@ import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.opensearch.action.admin.indices.get.GetIndexRequest; +import org.opensearch.action.admin.indices.get.GetIndexResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; @@ -21,7 +23,6 @@ import org.opensearch.common.util.io.Streams; import org.opensearch.common.xcontent.XContentType; import org.opensearch.ubi.SettingsConstants; -import org.opensearch.ubi.UserBehaviorInsightsPlugin; import org.opensearch.ubi.events.Event; import org.opensearch.ubi.events.OpenSearchEventManager; import org.opensearch.ubi.model.QueryRequest; @@ -31,8 +32,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -47,10 +48,30 @@ public class OpenSearchBackend implements Backend { private final Client client; + private Map> storeSettings = new HashMap<>(); + public OpenSearchBackend(final Client client) { this.client = client; } + @Override + public Map> getStoreSettings() { + + final Map> settings = new HashMap<>(); + + // TODO: Get each index for the store. + final Set stores = get(); + + for(final String store : stores) { + // TODO: Get this index. + // TODO: Get the settings for this index. + } + + // TODO: Get the settings for the store. + return settings; + + } + @Override public void initialize(final String storeName, final String index, final String idField) { @@ -88,7 +109,7 @@ public void initialize(final String storeName, final String index, final String final Map settings = new HashMap<>(); settings.put("index", index); settings.put("id_field", idField); - UserBehaviorInsightsPlugin.storeSettings.put(storeName, settings); + storeSettings.put(storeName, settings); } @@ -102,7 +123,7 @@ public void delete(String storeName) { final DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(eventsIndexName, queriesIndexName); client.admin().indices().delete(deleteIndexRequest); - UserBehaviorInsightsPlugin.storeSettings.remove(storeName); + storeSettings.remove(storeName); } @@ -149,9 +170,8 @@ public void persistQuery(final String storeName, final QueryRequest queryRequest @Override public Set get() { - /*final GetIndexRequest getIndexRequest = new GetIndexRequest(); + final GetIndexRequest getIndexRequest = new GetIndexRequest(); final GetIndexResponse getIndexResponse = client.admin().indices().getIndex(getIndexRequest).actionGet(); - final String[] indexes = getIndexResponse.indices(); final Set stores = new HashSet<>(); @@ -162,9 +182,9 @@ public Set get() { } } - return stores;*/ + return stores; - return Collections.emptySet(); + //return Collections.emptySet(); } diff --git a/src/yamlRestTest/resources/rest-api-spec/api/ubi.get_stores.json b/src/yamlRestTest/resources/rest-api-spec/api/ubi.get_stores.json new file mode 100644 index 0000000..5327f5b --- /dev/null +++ b/src/yamlRestTest/resources/rest-api-spec/api/ubi.get_stores.json @@ -0,0 +1,15 @@ +{ + "ubi.get_stores": { + "stability": "stable", + "url": { + "paths": [ + { + "path": "/_plugins/ubi", + "methods": [ + "GET" + ] + } + ] + } + } +} \ No newline at end of file diff --git a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml index 28dcc82..cfc60b1 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml +++ b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml @@ -1,21 +1,26 @@ --- "Create and delete a store": - - do: - ubi.create_store: - store: mystore +# - do: +# ubi.create_store: +# store: mystore - - do: - indices.exists: - index: .mystore_events - - - match: { created } - - - do: - ubi.delete_store: - store: mystore - - - do: - indices.exists: - index: .mystore_events - - - is_false: '' +# - do: +# indices.exists: +# index: .mystore_events +# +# - match: { created } +# +# - do: +# ubi.get_stores: {} +# +# - match: { mystore } +# +# - do: +# ubi.delete_store: +# store: mystore +# +# - do: +# indices.exists: +# index: .mystore_events +# +# - is_false: '' From cc257f414f4c9d8e1192e55d82993c6398218952 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 12:24:28 -0500 Subject: [PATCH 06/37] #67: Working on simplifying the backend structure. --- documentation.md | 4 +- .../ubi/UserBehaviorInsightsPlugin.java | 11 +- .../UserBehaviorInsightsActionFilter.java | 55 +++- .../UserBehaviorInsightsRestHandler.java | 256 ++++++------------ .../org/opensearch/ubi/backends/Backend.java | 35 --- .../ubi/backends/OpenSearchBackend.java | 230 ---------------- .../ubi/{ => model}/HeaderConstants.java | 2 +- .../ubi/{ => model}/SettingsConstants.java | 2 +- .../StoreInitializationResponse.java | 33 +++ .../org/opensearch/ubi/utils/UbiUtils.java | 42 +++ .../ubi/backends => }/events-mapping.json | 0 .../ubi/backends => }/queries-mapping.json | 0 .../test/_plugins.ubi/10_manage.yml | 14 +- 13 files changed, 219 insertions(+), 465 deletions(-) delete mode 100644 src/main/java/org/opensearch/ubi/backends/Backend.java delete mode 100644 src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java rename src/main/java/org/opensearch/ubi/{ => model}/HeaderConstants.java (95%) rename src/main/java/org/opensearch/ubi/{ => model}/SettingsConstants.java (93%) create mode 100644 src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java create mode 100644 src/main/java/org/opensearch/ubi/utils/UbiUtils.java rename src/main/resources/{org/opensearch/ubi/backends => }/events-mapping.json (100%) rename src/main/resources/{org/opensearch/ubi/backends => }/queries-mapping.json (100%) diff --git a/documentation.md b/documentation.md index 0f499b3..5e4e34a 100644 --- a/documentation.md +++ b/documentation.md @@ -16,8 +16,8 @@ index is used to store events, and the other index is for storing queries. #### OpenSearch Data Mappings -* The current event mappings file can be found [here](https://github.com/o19s/opensearch-ubi/blob/main/src/main/resources/org/opensearch/ubi/backends/events-mapping.json). -* The current query mappings file can be found [here](https://github.com/o19s/opensearch-ubi/blob/main/src/main/resources/org/opensearch/ubi/backends/query-mapping.json). +* The current event mappings file can be found [here](https://github.com/o19s/opensearch-ubi/blob/main/src/main/resources/events-mapping.json). +* The current query mappings file can be found [here](https://github.com/o19s/opensearch-ubi/blob/main/src/main/resources/query-mapping.json). Schema for events: diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 6627300..72079c4 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -30,9 +30,9 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.ubi.action.UserBehaviorInsightsActionFilter; import org.opensearch.ubi.action.UserBehaviorInsightsRestHandler; -import org.opensearch.ubi.backends.Backend; -import org.opensearch.ubi.backends.OpenSearchBackend; import org.opensearch.ubi.events.OpenSearchEventManager; +import org.opensearch.ubi.model.HeaderConstants; +import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; import java.util.*; @@ -45,7 +45,6 @@ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsPlugin.class); - private Backend backend; private ActionFilter userBehaviorLoggingFilter; @Override @@ -77,7 +76,7 @@ public List getRestHandlers(final Settings settings, final IndexNameExpressionResolver indexNameExpressionResolver, final Supplier nodesInCluster) { - return singletonList(new UserBehaviorInsightsRestHandler(backend)); + return singletonList(new UserBehaviorInsightsRestHandler()); } @@ -114,8 +113,6 @@ public Collection createComponents( Supplier repositoriesServiceSupplier ) { - this.backend = new OpenSearchBackend(client); - // TODO Only start this if an OpenSearch store is already initialized. // Otherwise, start it when a store is initialized. @@ -126,7 +123,7 @@ public Collection createComponents( }, 0, 2000, TimeUnit.MILLISECONDS); // Initialize the action filter. - this.userBehaviorLoggingFilter = new UserBehaviorInsightsActionFilter(backend, environment.settings(), threadPool); + this.userBehaviorLoggingFilter = new UserBehaviorInsightsActionFilter(client, threadPool); return Collections.emptyList(); diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 2b55199..796080c 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -11,37 +11,34 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; +import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; -import org.opensearch.common.settings.Settings; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.ubi.HeaderConstants; -import org.opensearch.ubi.backends.Backend; +import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; +import org.opensearch.ubi.utils.UbiUtils; -import java.util.Arrays; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; +import java.util.*; public class UserBehaviorInsightsActionFilter implements ActionFilter { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsActionFilter.class); - private final Backend backend; - private final Settings settings; + private final Client client; private final ThreadPool threadPool; - public UserBehaviorInsightsActionFilter(final Backend backend, final Settings settings, ThreadPool threadPool) { - this.backend = backend; - this.settings = settings; + public UserBehaviorInsightsActionFilter(Client client, ThreadPool threadPool) { + this.client = client; this.threadPool = threadPool; } @@ -85,7 +82,8 @@ public void onResponse(Response response) { final String sessionId = getHeaderValue(HeaderConstants.SESSION_ID_HEADER, "", task); // Get the store settings. - final String index = backend.getStoreSettings().get(eventStore).get("index"); + final String index = "ecommerce"; + //final String index = backend.getStoreSettings().get(eventStore).get("index"); // Only consider this search if the index being searched matches the store's index setting. if (Arrays.asList(searchRequest.indices()).contains(index)) { @@ -100,7 +98,8 @@ public void onResponse(Response response) { final SearchResponse searchResponse = (SearchResponse) response; // Get the id_field to use for each result's unique identifier. - final String idField = backend.getStoreSettings().get(eventStore).getOrDefault("id_field", ""); + final String idField = ""; + //final String idField = backend.getStoreSettings().get(eventStore).getOrDefault("id_field", ""); // Add each hit to the list of query responses. for (final SearchHit hit : searchResponse.getHits()) { @@ -115,7 +114,7 @@ public void onResponse(Response response) { try { // Persist the query to the backend. - backend.persistQuery(eventStore, + persistQuery(eventStore, new QueryRequest(queryId, query, userId, sessionId), new QueryResponse(queryId, queryResponseId, queryResponseHitIds)); @@ -160,4 +159,30 @@ private String getHeaderValue(final HeaderConstants header, final String default } + public void persistQuery(final String storeName, final QueryRequest queryRequest, QueryResponse queryResponse) { + + LOGGER.info("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryResponse.getQueryResponseId()); + + // What will be indexed - adheres to the queries-mapping.json + final Map source = new HashMap<>(); + source.put("timestamp", queryRequest.getTimestamp()); + source.put("query_id", queryRequest.getQueryId()); + source.put("query", queryRequest.getQuery()); + source.put("query_response_id", queryResponse.getQueryResponseId()); + source.put("query_response_hit_ids", queryResponse.getQueryResponseHitIds()); + source.put("user_id", queryRequest.getUserId()); + source.put("session_id", queryRequest.getSessionId()); + + // Get the name of the queries. + final String queriesIndexName = UbiUtils.getQueriesIndexName(storeName); + + // Build the index request. + final IndexRequest indexRequest = new IndexRequest(queriesIndexName) + .source(source, XContentType.JSON); + + // TODO: Move this to the queue, too. + client.index(indexRequest); + + } + } diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java index 99b84c5..d3cd2ed 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.create.CreateIndexRequest; @@ -22,21 +21,16 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.io.Streams; import org.opensearch.core.rest.RestStatus; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.ubi.SettingsConstants; -import org.opensearch.ubi.backends.Backend; -import org.opensearch.ubi.backends.OpenSearchBackend; +import org.opensearch.rest.action.RestToXContentListener; +import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.events.Event; import org.opensearch.ubi.events.OpenSearchEventManager; +import org.opensearch.ubi.utils.UbiUtils; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.*; import static org.opensearch.rest.RestRequest.Method.*; @@ -45,16 +39,9 @@ public class UserBehaviorInsightsRestHandler extends BaseRestHandler { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsRestHandler.class); - private static final String EVENTS_MAPPING_FILE = "events-mapping.json"; - private static final String QUERIES_MAPPING_FILE = "queries-mapping.json"; + private static final String EVENTS_MAPPING_FILE = "/events-mapping.json"; + private static final String QUERIES_MAPPING_FILE = "/queries-mapping.json"; public static final int VERSION = 1; - private Map> storeSettings = new HashMap<>(); - - private final Backend backend; - - public UserBehaviorInsightsRestHandler(final Backend backend) { - this.backend = backend; - } @Override public String getName() { @@ -71,166 +58,119 @@ public List routes() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient nodeClient) { + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) { - LOGGER.log(Level.INFO, "received event"); + final String storeName = restRequest.param("store"); - if (request.method() == PUT) { - - final String storeName = request.param("store"); + if (restRequest.method() == PUT) { //final String index = request.param("index"); //final String idField = request.param("id_field"); + return create(nodeClient, storeName); + } else if(restRequest.method() == DELETE) { + return delete(nodeClient, storeName); + } else if(restRequest.method() == POST) { + return post(nodeClient, storeName, restRequest); + } - // Validate the store name. - /*if(!backend.validateStoreName(storeName)) { - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "missing store name")); - } else {*/ - - return (channel) -> { - LOGGER.info("Creating UBL store {}", storeName); - - /*if(backend.exists(storeName)) { - channel.sendResponse(new BytesRestResponse(RestStatus.CONFLICT, "already exists")); - } else {*/ - //backend.initialize(storeName, index, idField); - - //channel.sendResponse(new BytesRestResponse(RestStatus.OK, "created")); - //} - - LOGGER.info("Creating search relevance store {}", storeName); - - final Settings indexSettings = Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") - .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - //.put(SettingsConstants.INDEX, index) - // .put(SettingsConstants.ID_FIELD, index) - .put(SettingsConstants.VERSION_SETTING, VERSION) - .build(); - - // Create the events index. - final String eventsIndexName = getEventsIndexName(storeName); - - final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndexName) - .mapping(getResourceFile(EVENTS_MAPPING_FILE)) - .settings(indexSettings); - - nodeClient.admin().indices().create(createEventsIndexRequest); - - // Create the queries index. - final String queriesIndexName = getQueriesIndexName(storeName); - - final CreateIndexRequest createQueryIndexRequest = new CreateIndexRequest(queriesIndexName) - .mapping(getResourceFile(QUERIES_MAPPING_FILE)) - .settings(indexSettings); - - nodeClient.admin().indices().create(createQueryIndexRequest); - - // Store the settings. - /*final Map settings = new HashMap<>(); - settings.put("index", index); - settings.put("id_field", idField); - storeSettings.put(storeName, settings);*/ + return get(nodeClient); - }; + } - // } + private RestChannelConsumer get(final NodeClient nodeClient) { - } else if (request.method() == DELETE) { + return (channel) -> { - final String storeName = request.param("store"); + final GetIndexRequest getIndexRequest = new GetIndexRequest(); + final GetIndexResponse getIndexResponse = nodeClient.admin().indices().getIndex(getIndexRequest).actionGet(); + final String[] indexes = getIndexResponse.indices(); + final Set stores = new HashSet<>(); - // Validate the store name. - if(!backend.validateStoreName(storeName)) { - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "missing store name")); + for (final String index : indexes) { + LOGGER.info("Index name: " + index); + if (index.startsWith(".") && index.endsWith("_queries")) { + stores.add(index); + } } - LOGGER.info("Deleting UBL store {}", storeName); - - return (channel) -> { - //backend.delete(storeName); - - LOGGER.info("Deleting search relevance store {}", storeName); - - final String eventsIndexName = getEventsIndexName(storeName); - final String queriesIndexName = getQueriesIndexName(storeName); - final DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(eventsIndexName, queriesIndexName); + //return stores; + channel.sendResponse(new BytesRestResponse(RestStatus.OK, String.join(",", stores))); - nodeClient.admin().indices().delete(deleteIndexRequest); - //storeSettings.remove(storeName); + }; - channel.sendResponse(new BytesRestResponse(RestStatus.OK, "deleted")); - }; - - } else if (request.method() == POST) { - - if (request.hasContent()) { - - final String storeName = request.param("store"); - - // Make sure the store exists. - /*if(!backend.exists(storeName)) { - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.NOT_FOUND, "store not found")); - }*/ - - LOGGER.info("Queuing event for storage into UBL store {}", storeName); - final String eventJson = request.content().utf8ToString(); - - try { + } - final String eventJsonWithTimestamp = setEventTimestamp(eventJson); + private RestChannelConsumer create(final NodeClient nodeClient, final String storeName) { + return (channel) -> { + + final Settings indexSettings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") + .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) + .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) + //.put(SettingsConstants.INDEX, index) + // .put(SettingsConstants.ID_FIELD, index) + .put(SettingsConstants.VERSION_SETTING, VERSION) + .build(); + + // Create the events index. + final String eventsIndex = UbiUtils.getEventsIndexName(storeName); + final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndex) + .mapping(UbiUtils.getResourceFile(EVENTS_MAPPING_FILE)) + .settings(indexSettings); + + nodeClient.admin().indices().create(createEventsIndexRequest, new RestToXContentListener<>(channel)); + +// // Create the queries index. +// final String queriesIndex = getEventsIndexName(storeName); +// final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(eventsIndex) +// .mapping(getResourceFile(EVENTS_MAPPING_FILE)) +// .settings(indexSettings); +// +// nodeClient.admin().indices().create(createEventsIndexRequest, new RestToXContentListener<>(channel)); + + }; + } - //backend.persistEvent(storeName, eventJsonWithTimestamp); - //return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); + private RestChannelConsumer post(final NodeClient nodeClient, final String storeName, final RestRequest restRequest) { - // Add the event for indexing. - LOGGER.info("Indexing event into {}", storeName); - final String eventsIndexName = getEventsIndexName(storeName); + try { - //return (channel) -> client.index(indexRequest, new RestToXContentListener<>(channel)); - final Event event = new Event(eventsIndexName, eventJson); - OpenSearchEventManager.getInstance(nodeClient).add(event); + final String eventJson = restRequest.content().utf8ToString(); + final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - } catch (JsonProcessingException ex) { - LOGGER.error("Unable to get/set timestamp on event.", ex); - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "unable to set event timestamp")); - } - - } else { - throw new IllegalArgumentException("Missing event content"); - } + //backend.persistEvent(storeName, eventJsonWithTimestamp); + //return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); - } else if (request.method() == GET) { + // Add the event for indexing. + LOGGER.info("Indexing event into {}", storeName); + final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); - //final Set stores = backend.get(); - //final String s = String.join(",", stores); + //return (channel) -> client.index(indexRequest, new RestToXContentListener<>(channel)); + final Event event = new Event(eventsIndexName, eventJson); + OpenSearchEventManager.getInstance(nodeClient).add(event); - //return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, s)); + } catch (JsonProcessingException ex) { + LOGGER.error("Unable to get/set timestamp on event.", ex); + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "unable to set event timestamp")); + } - return (channel) -> { + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); - final GetIndexRequest getIndexRequest = new GetIndexRequest(); - final GetIndexResponse getIndexResponse = nodeClient.admin().indices().getIndex(getIndexRequest).actionGet(); - final String[] indexes = getIndexResponse.indices(); - final Set stores = new HashSet<>(); + } - for (final String index : indexes) { - LOGGER.info("Index name: " + index); - if (index.startsWith(".") && index.endsWith("_queries")) { - stores.add(index); - } - } + private RestChannelConsumer delete(final NodeClient nodeClient, final String storeName) { - //return stores; - channel.sendResponse(new BytesRestResponse(RestStatus.OK, String.join(",", stores))); + return (channel) -> { - }; + // Delete the events index. + final DeleteIndexRequest deleteEventsIndexRequest = new DeleteIndexRequest(UbiUtils.getEventsIndexName(storeName)); + nodeClient.admin().indices().delete(deleteEventsIndexRequest, new RestToXContentListener<>(channel)); - } + // Delete the queries index. + final DeleteIndexRequest deleteQueriesIndexRequest = new DeleteIndexRequest(UbiUtils.getQueriesIndexName(storeName)); + nodeClient.admin().indices().delete(deleteQueriesIndexRequest, new RestToXContentListener<>(channel)); - // TODO: Return a list names of all search_relevance stores. - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "ok")); + }; } @@ -256,22 +196,4 @@ private boolean validateStoreName(final String storeName) { } - private String getEventsIndexName(final String storeName) { - return "." + storeName + "_events"; - } - - private String getQueriesIndexName(final String storeName) { - return "." + storeName + "_queries"; - } - - private String getResourceFile(final String fileName) { - try (InputStream is = OpenSearchBackend.class.getResourceAsStream(fileName)) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Streams.copy(is, out); - return out.toString(StandardCharsets.UTF_8); - } catch (IOException e) { - throw new IllegalStateException("Unable to create index with resource [" + fileName + "]", e); - } - } - } diff --git a/src/main/java/org/opensearch/ubi/backends/Backend.java b/src/main/java/org/opensearch/ubi/backends/Backend.java deleted file mode 100644 index 3f57290..0000000 --- a/src/main/java/org/opensearch/ubi/backends/Backend.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ubi.backends; - -import org.opensearch.ubi.model.QueryRequest; -import org.opensearch.ubi.model.QueryResponse; - -import java.util.Map; -import java.util.Set; - -public interface Backend { - - void initialize(final String storeName, final String index, final String idField); - - void delete(final String storeName); - - void persistEvent(final String storeName, String eventJson); - - void persistQuery(final String storeName, QueryRequest queryRequest, QueryResponse queryResponse) throws Exception; - - Set get(); - - boolean exists(final String storeName); - - boolean validateStoreName(final String storeName); - - Map> getStoreSettings(); - -} diff --git a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java b/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java deleted file mode 100644 index ceccf13..0000000 --- a/src/main/java/org/opensearch/ubi/backends/OpenSearchBackend.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ubi.backends; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.opensearch.action.admin.indices.get.GetIndexRequest; -import org.opensearch.action.admin.indices.get.GetIndexResponse; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.client.Client; -import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.util.io.Streams; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.ubi.SettingsConstants; -import org.opensearch.ubi.events.Event; -import org.opensearch.ubi.events.OpenSearchEventManager; -import org.opensearch.ubi.model.QueryRequest; -import org.opensearch.ubi.model.QueryResponse; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class OpenSearchBackend implements Backend { - - private static final Logger LOGGER = LogManager.getLogger(OpenSearchBackend.class); - - private static final String EVENTS_MAPPING_FILE = "events-mapping.json"; - private static final String QUERIES_MAPPING_FILE = "queries-mapping.json"; - - public static final int VERSION = 1; - - private final Client client; - - private Map> storeSettings = new HashMap<>(); - - public OpenSearchBackend(final Client client) { - this.client = client; - } - - @Override - public Map> getStoreSettings() { - - final Map> settings = new HashMap<>(); - - // TODO: Get each index for the store. - final Set stores = get(); - - for(final String store : stores) { - // TODO: Get this index. - // TODO: Get the settings for this index. - } - - // TODO: Get the settings for the store. - return settings; - - } - - @Override - public void initialize(final String storeName, final String index, final String idField) { - - LOGGER.info("Creating search relevance store {}", storeName); - - final Settings indexSettings = Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") - .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .put(SettingsConstants.INDEX, index) - .put(SettingsConstants.ID_FIELD, index) - .put(SettingsConstants.VERSION_SETTING, VERSION) - .build(); - - // Create the events index. - final String eventsIndexName = getEventsIndexName(storeName); - - final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndexName) - .mapping(getResourceFile(EVENTS_MAPPING_FILE)) - .settings(indexSettings); - - client.admin().indices().create(createEventsIndexRequest); - - // Create the queries index. - final String queriesIndexName = getQueriesIndexName(storeName); - - final CreateIndexRequest createQueryIndexRequest = new CreateIndexRequest(queriesIndexName) - .mapping(getResourceFile(QUERIES_MAPPING_FILE)) - .settings(indexSettings); - - client.admin().indices().create(createQueryIndexRequest); - - // Store the settings. - final Map settings = new HashMap<>(); - settings.put("index", index); - settings.put("id_field", idField); - storeSettings.put(storeName, settings); - - } - - @Override - public void delete(String storeName) { - - LOGGER.info("Deleting search relevance store {}", storeName); - - final String eventsIndexName = getEventsIndexName(storeName); - final String queriesIndexName = getQueriesIndexName(storeName); - final DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(eventsIndexName, queriesIndexName); - - client.admin().indices().delete(deleteIndexRequest); - storeSettings.remove(storeName); - - } - - @Override - public void persistEvent(String storeName, String eventJson) { - - // Add the event for indexing. - LOGGER.info("Indexing event into {}", storeName); - final String eventsIndexName = getEventsIndexName(storeName); - - //return (channel) -> client.index(indexRequest, new RestToXContentListener<>(channel)); - final Event event = new Event(eventsIndexName, eventJson); - OpenSearchEventManager.getInstance(client).add(event); - - } - - @Override - public void persistQuery(final String storeName, final QueryRequest queryRequest, QueryResponse queryResponse) { - - LOGGER.info("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryResponse.getQueryResponseId()); - - // What will be indexed - adheres to the queries-mapping.json - final Map source = new HashMap<>(); - source.put("timestamp", queryRequest.getTimestamp()); - source.put("query_id", queryRequest.getQueryId()); - source.put("query", queryRequest.getQuery()); - source.put("query_response_id", queryResponse.getQueryResponseId()); - source.put("query_response_hit_ids", queryResponse.getQueryResponseHitIds()); - source.put("user_id", queryRequest.getUserId()); - source.put("session_id", queryRequest.getSessionId()); - - // Get the name of the queries. - final String queriesIndexName = getQueriesIndexName(storeName); - - // Build the index request. - final IndexRequest indexRequest = new IndexRequest(queriesIndexName) - .source(source, XContentType.JSON); - - // TODO: Move this to the queue, too. - client.index(indexRequest); - - } - - @Override - public Set get() { - - final GetIndexRequest getIndexRequest = new GetIndexRequest(); - final GetIndexResponse getIndexResponse = client.admin().indices().getIndex(getIndexRequest).actionGet(); - final String[] indexes = getIndexResponse.indices(); - final Set stores = new HashSet<>(); - - for(final String index : indexes) { - LOGGER.info("Index name: " + index); - if(index.startsWith(".") && index.endsWith("_queries")) { - stores.add(index); - } - } - - return stores; - - //return Collections.emptySet(); - - } - - @Override - public boolean exists(final String storeName) { - - final String indexName = getEventsIndexName(storeName); - - // TODO: This has to run on a non-blocking thread. - final IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(indexName); - final IndicesExistsResponse indicesExistsResponse = client.admin().indices().exists(indicesExistsRequest).actionGet(); - - return indicesExistsResponse.isExists(); - - } - - @Override - public boolean validateStoreName(final String storeName) { - - // Validate the store name. - return storeName != null && !storeName.isEmpty(); - - } - - private String getEventsIndexName(final String storeName) { - return "." + storeName + "_events"; - } - - private String getQueriesIndexName(final String storeName) { - return "." + storeName + "_queries"; - } - - private String getResourceFile(final String fileName) { - try (InputStream is = OpenSearchBackend.class.getResourceAsStream(fileName)) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Streams.copy(is, out); - return out.toString(StandardCharsets.UTF_8); - } catch (IOException e) { - throw new IllegalStateException("Unable to create index with resource [" + OpenSearchBackend.EVENTS_MAPPING_FILE + "]", e); - } - } - -} diff --git a/src/main/java/org/opensearch/ubi/HeaderConstants.java b/src/main/java/org/opensearch/ubi/model/HeaderConstants.java similarity index 95% rename from src/main/java/org/opensearch/ubi/HeaderConstants.java rename to src/main/java/org/opensearch/ubi/model/HeaderConstants.java index e8b7d03..cd106c0 100644 --- a/src/main/java/org/opensearch/ubi/HeaderConstants.java +++ b/src/main/java/org/opensearch/ubi/model/HeaderConstants.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi; +package org.opensearch.ubi.model; public enum HeaderConstants { diff --git a/src/main/java/org/opensearch/ubi/SettingsConstants.java b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java similarity index 93% rename from src/main/java/org/opensearch/ubi/SettingsConstants.java rename to src/main/java/org/opensearch/ubi/model/SettingsConstants.java index 9bfa625..03ed0af 100644 --- a/src/main/java/org/opensearch/ubi/SettingsConstants.java +++ b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi; +package org.opensearch.ubi.model; public class SettingsConstants { diff --git a/src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java b/src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java new file mode 100644 index 0000000..1065a49 --- /dev/null +++ b/src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java @@ -0,0 +1,33 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ubi.model.responses; + +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; + +public class StoreInitializationResponse implements ToXContent { + + private final String status; + + public StoreInitializationResponse(final String status) { + this.status = status; + } + + @Override + public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { + return xContentBuilder.field("status", status); + } + + public String getStatus() { + return status; + } + +} diff --git a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java b/src/main/java/org/opensearch/ubi/utils/UbiUtils.java new file mode 100644 index 0000000..181c69e --- /dev/null +++ b/src/main/java/org/opensearch/ubi/utils/UbiUtils.java @@ -0,0 +1,42 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ubi.utils; + +import org.opensearch.common.util.io.Streams; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class UbiUtils { + + private UbiUtils() { + + } + + public static String getQueriesIndexName(final String storeName) { + return "." + storeName + "_queries"; + } + + public static String getEventsIndexName(final String storeName) { + return "." + storeName + "_events"; + } + + public static String getResourceFile(final String fileName) { + try (InputStream is = UbiUtils.class.getResourceAsStream(fileName)) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + Streams.copy(is, out); + return out.toString(StandardCharsets.UTF_8); + } catch (IOException e) { + throw new IllegalStateException("Unable to create index with resource [" + fileName + "]", e); + } + } + +} diff --git a/src/main/resources/org/opensearch/ubi/backends/events-mapping.json b/src/main/resources/events-mapping.json similarity index 100% rename from src/main/resources/org/opensearch/ubi/backends/events-mapping.json rename to src/main/resources/events-mapping.json diff --git a/src/main/resources/org/opensearch/ubi/backends/queries-mapping.json b/src/main/resources/queries-mapping.json similarity index 100% rename from src/main/resources/org/opensearch/ubi/backends/queries-mapping.json rename to src/main/resources/queries-mapping.json diff --git a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml index cfc60b1..6abdec3 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml +++ b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml @@ -1,13 +1,13 @@ --- "Create and delete a store": -# - do: -# ubi.create_store: -# store: mystore + - do: + ubi.create_store: + store: mystore + + - do: + indices.exists: + index: .mystore_events -# - do: -# indices.exists: -# index: .mystore_events -# # - match: { created } # # - do: From dfcbb6209d68beb25a444c58687919f3861f26a8 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 12:38:56 -0500 Subject: [PATCH 07/37] #67 Exposing the index settings. --- README.md | 6 ++++ .../ubi/UserBehaviorInsightsPlugin.java | 9 +++-- .../UserBehaviorInsightsRestHandler.java | 35 +++++++++---------- .../ubi/model/SettingsConstants.java | 7 ++-- .../rest-api-spec/api/ubi.create_store.json | 15 ++++++-- .../test/_plugins.ubi/10_manage.yml | 1 + 6 files changed, 46 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 293e0ce..f3726a8 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ Delete the store: curl -X DELETE http://localhost:9200/_plugins/ubi/awesome ``` +Get the stores: + +``` +curl http://localhost:9200/_plugins/ubi +``` + ## Load Test The `load-test` directory contains a basic load testing example. The purpose of the files under `load-test` are to provide a means of testing the plugin's ability to receive and store a large number of events over time. To use the load test, first start OpenSearch on `localhost:9200`, and then: diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 72079c4..f45a39e 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -35,7 +35,10 @@ import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -84,10 +87,10 @@ public List getRestHandlers(final Settings settings, public List> getSettings() { final List> settings = new ArrayList<>(); - settings.add(Setting.simpleString(SettingsConstants.INDEX_NAMES, "", Setting.Property.NodeScope)); - // The version of the index mapping. settings.add(Setting.intSetting(SettingsConstants.VERSION_SETTING, 1, -1, Integer.MAX_VALUE, Setting.Property.IndexScope)); + settings.add(Setting.simpleString(SettingsConstants.INDEX, "", Setting.Property.IndexScope)); + settings.add(Setting.simpleString(SettingsConstants.ID_FIELD, "", Setting.Property.IndexScope)); return settings; diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java index d3cd2ed..4819c6b 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java @@ -26,12 +26,15 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; -import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.events.Event; import org.opensearch.ubi.events.OpenSearchEventManager; +import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.utils.UbiUtils; -import java.util.*; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; import static org.opensearch.rest.RestRequest.Method.*; @@ -63,9 +66,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient final String storeName = restRequest.param("store"); if (restRequest.method() == PUT) { - //final String index = request.param("index"); - //final String idField = request.param("id_field"); - return create(nodeClient, storeName); + final String index = restRequest.param("index"); + final String idField = restRequest.param("id_field"); + return create(nodeClient, storeName, index, idField); } else if(restRequest.method() == DELETE) { return delete(nodeClient, storeName); } else if(restRequest.method() == POST) { @@ -87,19 +90,18 @@ private RestChannelConsumer get(final NodeClient nodeClient) { for (final String index : indexes) { LOGGER.info("Index name: " + index); - if (index.startsWith(".") && index.endsWith("_queries")) { + if (index.startsWith(".") && index.endsWith("_events")) { stores.add(index); } } - //return stores; channel.sendResponse(new BytesRestResponse(RestStatus.OK, String.join(",", stores))); }; } - private RestChannelConsumer create(final NodeClient nodeClient, final String storeName) { + private RestChannelConsumer create(final NodeClient nodeClient, final String storeName, final String index, final String idField) { return (channel) -> { final Settings indexSettings = Settings.builder() @@ -107,8 +109,8 @@ private RestChannelConsumer create(final NodeClient nodeClient, final String sto .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - //.put(SettingsConstants.INDEX, index) - // .put(SettingsConstants.ID_FIELD, index) + .put(SettingsConstants.INDEX, index) + .put(SettingsConstants.ID_FIELD, idField) .put(SettingsConstants.VERSION_SETTING, VERSION) .build(); @@ -121,12 +123,12 @@ private RestChannelConsumer create(final NodeClient nodeClient, final String sto nodeClient.admin().indices().create(createEventsIndexRequest, new RestToXContentListener<>(channel)); // // Create the queries index. -// final String queriesIndex = getEventsIndexName(storeName); -// final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(eventsIndex) -// .mapping(getResourceFile(EVENTS_MAPPING_FILE)) +// final String queriesIndex = UbiUtils.getEventsIndexName(storeName); +// final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(queriesIndex) +// .mapping(UbiUtils.getResourceFile(QUERIES_MAPPING_FILE)) // .settings(indexSettings); // -// nodeClient.admin().indices().create(createEventsIndexRequest, new RestToXContentListener<>(channel)); +// nodeClient.admin().indices().create(createQueriesIndexRequest, new RestToXContentListener<>(channel)); }; } @@ -138,15 +140,12 @@ private RestChannelConsumer post(final NodeClient nodeClient, final String store final String eventJson = restRequest.content().utf8ToString(); final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - //backend.persistEvent(storeName, eventJsonWithTimestamp); - //return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); - // Add the event for indexing. LOGGER.info("Indexing event into {}", storeName); final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); //return (channel) -> client.index(indexRequest, new RestToXContentListener<>(channel)); - final Event event = new Event(eventsIndexName, eventJson); + final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); OpenSearchEventManager.getInstance(nodeClient).add(event); } catch (JsonProcessingException ex) { diff --git a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java index 03ed0af..91ea8ec 100644 --- a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java +++ b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java @@ -10,10 +10,9 @@ public class SettingsConstants { - public static final String INDEX_NAMES = "plugins.ubi.indices"; - public static final String VERSION_SETTING = "index.ubistore.version"; + public static final String VERSION_SETTING = "index.ubi.version"; - public static final String INDEX = "ubi.store.index"; - public static final String ID_FIELD = "ubi.store.id_field"; + public static final String INDEX = "index.ubi.index"; + public static final String ID_FIELD = "index.ubi.id_field"; } diff --git a/src/yamlRestTest/resources/rest-api-spec/api/ubi.create_store.json b/src/yamlRestTest/resources/rest-api-spec/api/ubi.create_store.json index 8082522..bfa909d 100644 --- a/src/yamlRestTest/resources/rest-api-spec/api/ubi.create_store.json +++ b/src/yamlRestTest/resources/rest-api-spec/api/ubi.create_store.json @@ -7,7 +7,7 @@ "path": "/_plugins/ubi/{store}", "parts": { "store": { - "required": false, + "required": true, "type": "string", "description": "The name of the store" } @@ -18,6 +18,17 @@ } ] }, - "body": null + "params": { + "index": { + "required": true, + "type": "string", + "description": "The name of the index being searched" + }, + "id_field": { + "required": false, + "type": "string", + "description": "The name of the field to use for the doc ID field" + } + } } } \ No newline at end of file diff --git a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml index 6abdec3..12fa708 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml +++ b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml @@ -3,6 +3,7 @@ - do: ubi.create_store: store: mystore + index: ecommerce - do: indices.exists: From 95833257441bd87110b080b8e266e8b4a9e4854c Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 12:59:43 -0500 Subject: [PATCH 08/37] #67 Adding json responses. --- docker-compose.yaml | 1 - .../UserBehaviorInsightsRestHandler.java | 117 ++++++++++-------- .../StoreInitializationResponse.java | 33 ----- 3 files changed, 67 insertions(+), 84 deletions(-) delete mode 100644 src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java diff --git a/docker-compose.yaml b/docker-compose.yaml index 89201b6..6e1a13a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -16,7 +16,6 @@ services: http.cors.allow-methods: OPTIONS,HEAD,GET,POST,PUT,DELETE http.cors.allow-credentials: true http.cors.allow-headers: X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization - plugins.ubi.indices: "awesome" logger.level: info OPENSEARCH_INITIAL_ADMIN_PASSWORD: SuperSecretPassword_123 ulimits: diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java index 4819c6b..107873c 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java @@ -21,16 +21,18 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.rest.action.RestToXContentListener; import org.opensearch.ubi.events.Event; import org.opensearch.ubi.events.OpenSearchEventManager; import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.utils.UbiUtils; +import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -61,7 +63,7 @@ public List routes() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) { + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) throws IOException { final String storeName = restRequest.param("store"); @@ -89,87 +91,102 @@ private RestChannelConsumer get(final NodeClient nodeClient) { final Set stores = new HashSet<>(); for (final String index : indexes) { - LOGGER.info("Index name: " + index); if (index.startsWith(".") && index.endsWith("_events")) { - stores.add(index); + stores.add(index.substring(1, index.length() - 7)); } } - channel.sendResponse(new BytesRestResponse(RestStatus.OK, String.join(",", stores))); + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("stores", stores); + builder.endObject(); + + channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); }; } - private RestChannelConsumer create(final NodeClient nodeClient, final String storeName, final String index, final String idField) { - return (channel) -> { + private RestChannelConsumer create(final NodeClient nodeClient, final String storeName, final String index, final String idField) throws IOException { - final Settings indexSettings = Settings.builder() - .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) - .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") - .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) - .put(IndexMetadata.SETTING_INDEX_HIDDEN, true) - .put(SettingsConstants.INDEX, index) - .put(SettingsConstants.ID_FIELD, idField) - .put(SettingsConstants.VERSION_SETTING, VERSION) - .build(); - - // Create the events index. - final String eventsIndex = UbiUtils.getEventsIndexName(storeName); - final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndex) - .mapping(UbiUtils.getResourceFile(EVENTS_MAPPING_FILE)) - .settings(indexSettings); - - nodeClient.admin().indices().create(createEventsIndexRequest, new RestToXContentListener<>(channel)); - -// // Create the queries index. -// final String queriesIndex = UbiUtils.getEventsIndexName(storeName); -// final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(queriesIndex) -// .mapping(UbiUtils.getResourceFile(QUERIES_MAPPING_FILE)) -// .settings(indexSettings); -// -// nodeClient.admin().indices().create(createQueriesIndexRequest, new RestToXContentListener<>(channel)); + final Settings indexSettings = Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) + .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") + .put(IndexMetadata.SETTING_PRIORITY, Integer.MAX_VALUE) + .put(SettingsConstants.INDEX, index) + .put(SettingsConstants.ID_FIELD, idField) + .put(SettingsConstants.VERSION_SETTING, VERSION) + .build(); + + // Create the events index. + final String eventsIndex = UbiUtils.getEventsIndexName(storeName); + final CreateIndexRequest createEventsIndexRequest = new CreateIndexRequest(eventsIndex) + .mapping(UbiUtils.getResourceFile(EVENTS_MAPPING_FILE)) + .settings(indexSettings); + + nodeClient.admin().indices().create(createEventsIndexRequest); + + // Create the queries index. + final String queriesIndex = UbiUtils.getEventsIndexName(storeName); + final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(queriesIndex) + .mapping(UbiUtils.getResourceFile(QUERIES_MAPPING_FILE)) + .settings(indexSettings); + + nodeClient.admin().indices().create(createQueriesIndexRequest); + + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("status", "initialized"); + builder.endObject(); + + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); - }; } - private RestChannelConsumer post(final NodeClient nodeClient, final String storeName, final RestRequest restRequest) { + private RestChannelConsumer post(final NodeClient nodeClient, final String storeName, final RestRequest restRequest) throws IOException { try { final String eventJson = restRequest.content().utf8ToString(); final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - // Add the event for indexing. - LOGGER.info("Indexing event into {}", storeName); + LOGGER.info("Indexing UBI event into store {}", storeName); final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); - //return (channel) -> client.index(indexRequest, new RestToXContentListener<>(channel)); final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); OpenSearchEventManager.getInstance(nodeClient).add(event); } catch (JsonProcessingException ex) { - LOGGER.error("Unable to get/set timestamp on event.", ex); - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "unable to set event timestamp")); + LOGGER.error("Unable to get/set timestamp on UBI event.", ex); + + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("error", "unable to set event timestamp"); + builder.endObject(); + + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); } - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, "event received")); + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("status", "received"); + builder.endObject(); + + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); } - private RestChannelConsumer delete(final NodeClient nodeClient, final String storeName) { + private RestChannelConsumer delete(final NodeClient nodeClient, final String storeName) throws IOException { - return (channel) -> { + // Delete the events index. + final DeleteIndexRequest deleteEventsIndexRequest = new DeleteIndexRequest(UbiUtils.getEventsIndexName(storeName)); + nodeClient.admin().indices().delete(deleteEventsIndexRequest); - // Delete the events index. - final DeleteIndexRequest deleteEventsIndexRequest = new DeleteIndexRequest(UbiUtils.getEventsIndexName(storeName)); - nodeClient.admin().indices().delete(deleteEventsIndexRequest, new RestToXContentListener<>(channel)); + // Delete the queries index. + final DeleteIndexRequest deleteQueriesIndexRequest = new DeleteIndexRequest(UbiUtils.getQueriesIndexName(storeName)); + nodeClient.admin().indices().delete(deleteQueriesIndexRequest); - // Delete the queries index. - final DeleteIndexRequest deleteQueriesIndexRequest = new DeleteIndexRequest(UbiUtils.getQueriesIndexName(storeName)); - nodeClient.admin().indices().delete(deleteQueriesIndexRequest, new RestToXContentListener<>(channel)); + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("status", "deleted"); + builder.endObject(); - }; + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); } @@ -177,7 +194,7 @@ private String setEventTimestamp(final String eventJson) throws JsonProcessingEx final JsonNode rootNode = new ObjectMapper().readTree(eventJson); - ObjectNode target = (ObjectNode) rootNode; + final ObjectNode target = (ObjectNode) rootNode; // If there is already a timestamp don't overwrite it. if(target.get("timestamp") == null || Objects.equals(target.get("timestamp").asText(), "")) { diff --git a/src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java b/src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java deleted file mode 100644 index 1065a49..0000000 --- a/src/main/java/org/opensearch/ubi/model/responses/StoreInitializationResponse.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ubi.model.responses; - -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.core.xcontent.XContentBuilder; - -import java.io.IOException; - -public class StoreInitializationResponse implements ToXContent { - - private final String status; - - public StoreInitializationResponse(final String status) { - this.status = status; - } - - @Override - public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params) throws IOException { - return xContentBuilder.field("status", status); - } - - public String getStatus() { - return status; - } - -} From 6b9ccbb1a5d2b0830fc35fb261f8f31aa1d46694 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 13:34:14 -0500 Subject: [PATCH 09/37] #67 Addign script to index Chorus data. Working on custom field. --- README.md | 6 ++++++ index-chorus-data.sh | 14 ++++++++++++++ .../UserBehaviorInsightsActionFilter.java | 18 ++++++++---------- .../ubi/model/SettingsConstants.java | 2 +- 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100755 index-chorus-data.sh diff --git a/README.md b/README.md index f3726a8..8c41542 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,12 @@ curl -s http://localhost:9200/.awesome_events/_search | jq curl -s http://localhost:9200/.awesome_events/_search -H 'Content-Type: application/json' -d '{"query": {"term": {"type": "instant-search"}}}' | jq ``` +Do a search of the `ecommerce` index: + +``` +curl -s http://localhost:9200/ecommerce/_search -H "X-ubi-store: awesome" | jq +``` + Get queries: ``` diff --git a/index-chorus-data.sh b/index-chorus-data.sh new file mode 100755 index 0000000..b85ecb9 --- /dev/null +++ b/index-chorus-data.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +CHORUS_HOME=`realpath ../chorus-opensearch-edition` +echo "Using CHORUS_HOME = ${CHORUS_HOME}" + +echo "Deleting index" +curl -s -X DELETE "localhost:9200/ecommerce" + +echo "Creating index" +curl -s -X PUT "localhost:9200/ecommerce" -H "Content-Type: application/json" --data-binary @${CHORUS_HOME}/opensearch/schema.json +curl -s -X PUT "localhost:9200/ecommerce/_settings" -H "Content-Type: application/json" -d '{"index.mapping.total_fields.limit": 20000}' + +echo "Indexing data" +curl -s -X POST "localhost:9200/ecommerce/_bulk?pretty" -H "Content-Type: application/json" --data-binary @${CHORUS_HOME}/transformed_data.json diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 796080c..fb09fa6 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -11,12 +11,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; +import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.client.Client; +import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; @@ -26,8 +29,10 @@ import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; +import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.utils.UbiUtils; +import java.io.IOException; import java.util.*; public class UserBehaviorInsightsActionFilter implements ActionFilter { @@ -67,11 +72,6 @@ public void onResponse(Response response) { // Get the search itself. final SearchRequest searchRequest = (SearchRequest) request; - // TODO: Restrict logging to only queries of certain indices specified in the settings. - //final List indices = Arrays.asList(searchRequest.indices()); - //final Set indicesToLog = new HashSet<>(Arrays.asList(settings.get(SettingsConstants.INDEX_NAMES).split(","))); - //if(indicesToLog.containsAll(indices)) { - // Get all search hits from the response. if (response instanceof SearchResponse) { @@ -81,9 +81,7 @@ public void onResponse(Response response) { final String userId = getHeaderValue(HeaderConstants.USER_ID_HEADER, "", task); final String sessionId = getHeaderValue(HeaderConstants.SESSION_ID_HEADER, "", task); - // Get the store settings. final String index = "ecommerce"; - //final String index = backend.getStoreSettings().get(eventStore).get("index"); // Only consider this search if the index being searched matches the store's index setting. if (Arrays.asList(searchRequest.indices()).contains(index)) { @@ -98,8 +96,7 @@ public void onResponse(Response response) { final SearchResponse searchResponse = (SearchResponse) response; // Get the id_field to use for each result's unique identifier. - final String idField = ""; - //final String idField = backend.getStoreSettings().get(eventStore).getOrDefault("id_field", ""); + final String idField = "name"; // Add each hit to the list of query responses. for (final SearchHit hit : searchResponse.getHits()) { @@ -161,7 +158,8 @@ private String getHeaderValue(final HeaderConstants header, final String default public void persistQuery(final String storeName, final QueryRequest queryRequest, QueryResponse queryResponse) { - LOGGER.info("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryResponse.getQueryResponseId()); + LOGGER.info("Writing query ID {} with response ID {}", + queryRequest.getQueryId(), queryResponse.getQueryResponseId()); // What will be indexed - adheres to the queries-mapping.json final Map source = new HashMap<>(); diff --git a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java index 91ea8ec..907b63a 100644 --- a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java +++ b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java @@ -12,7 +12,7 @@ public class SettingsConstants { public static final String VERSION_SETTING = "index.ubi.version"; - public static final String INDEX = "index.ubi.index"; + public static final String INDEX = "index.ubi.store"; public static final String ID_FIELD = "index.ubi.id_field"; } From 8aee641f706b7c64be194fdee93362ae4dd24f3e Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 13:45:37 -0500 Subject: [PATCH 10/37] #67 Updating script to index a few documents. Working on getting set field. --- index-chorus-data.sh | 5 ++++- .../action/UserBehaviorInsightsActionFilter.java | 13 ++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/index-chorus-data.sh b/index-chorus-data.sh index b85ecb9..edf7f8d 100755 --- a/index-chorus-data.sh +++ b/index-chorus-data.sh @@ -3,6 +3,9 @@ CHORUS_HOME=`realpath ../chorus-opensearch-edition` echo "Using CHORUS_HOME = ${CHORUS_HOME}" +TEMP_FILE=`mktemp` +head -n 100 ${CHORUS_HOME}/transformed_data.json > ${TEMP_FILE} + echo "Deleting index" curl -s -X DELETE "localhost:9200/ecommerce" @@ -11,4 +14,4 @@ curl -s -X PUT "localhost:9200/ecommerce" -H "Content-Type: application/json" -- curl -s -X PUT "localhost:9200/ecommerce/_settings" -H "Content-Type: application/json" -d '{"index.mapping.total_fields.limit": 20000}' echo "Indexing data" -curl -s -X POST "localhost:9200/ecommerce/_bulk?pretty" -H "Content-Type: application/json" --data-binary @${CHORUS_HOME}/transformed_data.json +curl -s -X POST "localhost:9200/ecommerce/_bulk?pretty" -H "Content-Type: application/json" --data-binary @${TEMP_FILE} diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index fb09fa6..479a3e3 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -19,6 +19,7 @@ import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.client.Client; +import org.opensearch.common.document.DocumentField; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; @@ -100,12 +101,18 @@ public void onResponse(Response response) { // Add each hit to the list of query responses. for (final SearchHit hit : searchResponse.getHits()) { - if ("".equals(idField)) { + LOGGER.info("HIT FIELD LENGTH = " + hit.getFields().size()); + + for(final String f : hit.getFields().keySet()) { + LOGGER.info("FIELD = " + f + ", VALUE = " + hit.field(f).toString()); + } + + /*if ("".equals(idField)) { // Use the _id since there is no id_field setting for this index. queryResponseHitIds.add(String.valueOf(hit.docId())); } else { - queryResponseHitIds.add(String.valueOf(hit.field(idField))); - } + queryResponseHitIds.add(hit.field(idField).getValue()); + }*/ } try { From cc2a10e2446436a9faa8fbe94f13ea3ecf2b9144 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 14:19:28 -0500 Subject: [PATCH 11/37] #67 Getting the field from the source. --- README.md | 2 +- .../ubi/UserBehaviorInsightsPlugin.java | 2 +- .../action/UserBehaviorInsightsActionFilter.java | 15 ++++++++------- .../UserBehaviorInsightsRestHandler.java | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) rename src/main/java/org/opensearch/ubi/{action => rest}/UserBehaviorInsightsRestHandler.java (99%) diff --git a/README.md b/README.md index 8c41542..4d30f1d 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ curl -s http://localhost:9200/ecommerce/_search -H "X-ubi-store: awesome" | jq Get queries: ``` -curl -s http://localhost:9200/.awesome_queries/_search -H "X-ubi-store: awesome" | jq +curl -s http://localhost:9200/.awesome_queries/_search | jq ``` Delete the store: diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index f45a39e..2eb10dd 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -29,7 +29,7 @@ import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.ubi.action.UserBehaviorInsightsActionFilter; -import org.opensearch.ubi.action.UserBehaviorInsightsRestHandler; +import org.opensearch.ubi.rest.UserBehaviorInsightsRestHandler; import org.opensearch.ubi.events.OpenSearchEventManager; import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.SettingsConstants; diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 479a3e3..a76ea2e 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -101,18 +101,19 @@ public void onResponse(Response response) { // Add each hit to the list of query responses. for (final SearchHit hit : searchResponse.getHits()) { - LOGGER.info("HIT FIELD LENGTH = " + hit.getFields().size()); - for(final String f : hit.getFields().keySet()) { - LOGGER.info("FIELD = " + f + ", VALUE = " + hit.field(f).toString()); - } + if ("".equals(idField)) { - /*if ("".equals(idField)) { // Use the _id since there is no id_field setting for this index. queryResponseHitIds.add(String.valueOf(hit.docId())); + } else { - queryResponseHitIds.add(hit.field(idField).getValue()); - }*/ + + final Map source = hit.getSourceAsMap(); + queryResponseHitIds.add((String) source.get(idField)); + + } + } try { diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java similarity index 99% rename from src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java rename to src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java index 107873c..6dbb9f4 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.action; +package org.opensearch.ubi.rest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; From 92c0cb1897cbdf6e57fcff3484ad2cf01c33d8df Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 5 Mar 2024 14:35:30 -0500 Subject: [PATCH 12/37] #67 Looking at index and id_field settings from the store index. --- README.md | 2 +- .../UserBehaviorInsightsActionFilter.java | 81 ++++++++++--------- .../rest/UserBehaviorInsightsRestHandler.java | 4 +- 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 4d30f1d..c7d9b61 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Start the containers: Initialize the `awesome` UBL store: ``` -curl -X PUT http://localhost:9200/_plugins/ubi/awesome +curl -X PUT "http://localhost:9200/_plugins/ubi/awesome?index=ecommerce&id_field=name" ``` Send an event to the `awesome` store: diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index a76ea2e..c0bc7b4 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -19,8 +19,6 @@ import org.opensearch.action.support.ActionFilter; import org.opensearch.action.support.ActionFilterChain; import org.opensearch.client.Client; -import org.opensearch.common.document.DocumentField; -import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; import org.opensearch.core.action.ActionResponse; @@ -33,7 +31,6 @@ import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.utils.UbiUtils; -import java.io.IOException; import java.util.*; public class UserBehaviorInsightsActionFilter implements ActionFilter { @@ -78,62 +75,72 @@ public void onResponse(Response response) { // Get info from the headers. final String queryId = getHeaderValue(HeaderConstants.QUERY_ID_HEADER, UUID.randomUUID().toString(), task); - final String eventStore = getHeaderValue(HeaderConstants.EVENT_STORE_HEADER, "default", task); + final String eventStore = getHeaderValue(HeaderConstants.EVENT_STORE_HEADER, "", task); final String userId = getHeaderValue(HeaderConstants.USER_ID_HEADER, "", task); final String sessionId = getHeaderValue(HeaderConstants.SESSION_ID_HEADER, "", task); - final String index = "ecommerce"; + // If there is no event store header, ignore this search. + if(!"".equals(eventStore)) { - // Only consider this search if the index being searched matches the store's index setting. - if (Arrays.asList(searchRequest.indices()).contains(index)) { + // Get the id_field to use for each result's unique identifier. + final String queriesIndexName = UbiUtils.getQueriesIndexName(eventStore); + final GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices(queriesIndexName); - // The query will be empty when there is no query, e.g. /_search - final String query = searchRequest.source().toString(); + final GetSettingsResponse getSettingsResponse = client.admin().indices().getSettings(getSettingsRequest).actionGet(); + final String idField = getSettingsResponse.getSetting(queriesIndexName, SettingsConstants.ID_FIELD); + final String index = getSettingsResponse.getSetting(queriesIndexName, SettingsConstants.INDEX); - // Create a UUID for this search response. - final String queryResponseId = UUID.randomUUID().toString(); + LOGGER.info("Using id_field [{}] of index [{}] for UBI query.", idField, index); - final List queryResponseHitIds = new LinkedList<>(); - final SearchResponse searchResponse = (SearchResponse) response; + // Only consider this search if the index being searched matches the store's index setting. + if (Arrays.asList(searchRequest.indices()).contains(index)) { - // Get the id_field to use for each result's unique identifier. - final String idField = "name"; + // The query will be empty when there is no query, e.g. /_search + final String query = searchRequest.source().toString(); + + // Create a UUID for this search response. + final String queryResponseId = UUID.randomUUID().toString(); + + final List queryResponseHitIds = new LinkedList<>(); + final SearchResponse searchResponse = (SearchResponse) response; + + // Add each hit to the list of query responses. + for (final SearchHit hit : searchResponse.getHits()) { - // Add each hit to the list of query responses. - for (final SearchHit hit : searchResponse.getHits()) { + if ("".equals(idField)) { - if ("".equals(idField)) { + // Use the _id since there is no id_field setting for this index. + queryResponseHitIds.add(String.valueOf(hit.docId())); - // Use the _id since there is no id_field setting for this index. - queryResponseHitIds.add(String.valueOf(hit.docId())); + } else { - } else { + final Map source = hit.getSourceAsMap(); + queryResponseHitIds.add((String) source.get(idField)); - final Map source = hit.getSourceAsMap(); - queryResponseHitIds.add((String) source.get(idField)); + } } - } + try { - try { + // Persist the query to the backend. + persistQuery(eventStore, + new QueryRequest(queryId, query, userId, sessionId), + new QueryResponse(queryId, queryResponseId, queryResponseHitIds)); - // Persist the query to the backend. - persistQuery(eventStore, - new QueryRequest(queryId, query, userId, sessionId), - new QueryResponse(queryId, queryResponseId, queryResponseHitIds)); + } catch (Exception ex) { + // TODO: Handle this. + LOGGER.error("Unable to persist query.", ex); + } - } catch (Exception ex) { - // TODO: Handle this. - LOGGER.error("Unable to persist query.", ex); - } + threadPool.getThreadContext().addResponseHeader("query_id", queryId); - threadPool.getThreadContext().addResponseHeader("query_id", queryId); + //} - //} + final long elapsedTime = System.currentTimeMillis() - startTime; + LOGGER.info("UBI search request filter took {} ms", elapsedTime); - final long elapsedTime = System.currentTimeMillis() - startTime; - LOGGER.info("UBI search request filter took {} ms", elapsedTime); + } } diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java index 6dbb9f4..670f0e8 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -108,6 +108,8 @@ private RestChannelConsumer get(final NodeClient nodeClient) { private RestChannelConsumer create(final NodeClient nodeClient, final String storeName, final String index, final String idField) throws IOException { + LOGGER.info("Creating UBI store [{}] for index [{}] using field [{}]", storeName, index, idField); + final Settings indexSettings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_AUTO_EXPAND_REPLICAS_SETTING.getKey(), "0-2") @@ -126,7 +128,7 @@ private RestChannelConsumer create(final NodeClient nodeClient, final String sto nodeClient.admin().indices().create(createEventsIndexRequest); // Create the queries index. - final String queriesIndex = UbiUtils.getEventsIndexName(storeName); + final String queriesIndex = UbiUtils.getQueriesIndexName(storeName); final CreateIndexRequest createQueriesIndexRequest = new CreateIndexRequest(queriesIndex) .mapping(UbiUtils.getResourceFile(QUERIES_MAPPING_FILE)) .settings(indexSettings); From bb033ffc1983cbb2ad9326f1ff3bba336cf6cf24 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 6 Mar 2024 14:47:59 -0500 Subject: [PATCH 13/37] changes from following the README; --- Dockerfile | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index cd7c708..05b82c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM opensearchproject/opensearch:2.12.0 -COPY ./build/distributions/opensearch-ubi.zip /tmp/ +COPY ./build/distributions/opensearch-ubi-0.0.8-os2.12.0.zip /tmp/ -RUN /usr/share/opensearch/bin/opensearch-plugin install file:/tmp/opensearch-ubi.zip +RUN /usr/share/opensearch/bin/opensearch-plugin install file:/tmp/opensearch-ubi-0.0.8-os2.12.0.zip diff --git a/README.md b/README.md index c7d9b61..5e9bd27 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Start the containers: `docker compose up` -Initialize the `awesome` UBL store: +Initialize the `awesome` UBI store: ``` curl -X PUT "http://localhost:9200/_plugins/ubi/awesome?index=ecommerce&id_field=name" @@ -86,4 +86,4 @@ POST /_plugins/ubi/mystore Found 8 indexed ``` -This shows 8 total requests made by locust, and 8 events are in the index. The idea being we can assert that the number of events sent matches the events stored in the index. \ No newline at end of file +This shows 8 total requests made by locust, and 8 events are in the index. The idea being we can assert that the number of events sent matches the events stored in the index. From 6cc588fa98bf550d334a2a22ad64070ef220d14c Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Thu, 7 Mar 2024 09:52:40 -0500 Subject: [PATCH 14/37] #79 Adding javadocs. --- .../ubi/UserBehaviorInsightsPlugin.java | 3 ++ .../UserBehaviorInsightsActionFilter.java | 15 ++++++++++ .../ubi/events/AbstractEventManager.java | 16 ++++++++++ .../java/org/opensearch/ubi/events/Event.java | 23 ++++++++++++++ .../ubi/events/OpenSearchEventManager.java | 8 +++++ .../ubi/events/queues/EventQueue.java | 18 +++++++++++ .../ubi/events/queues/InternalQueue.java | 3 ++ .../opensearch/ubi/model/HeaderConstants.java | 22 ++++++++++++++ .../opensearch/ubi/model/QueryRequest.java | 30 +++++++++++++++++++ .../opensearch/ubi/model/QueryResponse.java | 21 +++++++++++++ .../ubi/model/SettingsConstants.java | 13 ++++++++ .../rest/UserBehaviorInsightsRestHandler.java | 6 +++- .../org/opensearch/ubi/utils/UbiUtils.java | 21 +++++++++++++ 13 files changed, 198 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 2eb10dd..530ed9e 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -44,6 +44,9 @@ import static java.util.Collections.singletonList; +/** + * OpenSearch User Behavior Insights Plugin + */ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsPlugin.class); diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index f189aee..2366e40 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -33,6 +33,10 @@ import java.util.*; +/** + * An implementation of {@link ActionFilter} that passively listens for OpenSearch + * queries and persists the queries to the UBI store. + */ public class UserBehaviorInsightsActionFilter implements ActionFilter { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsActionFilter.class); @@ -40,6 +44,11 @@ public class UserBehaviorInsightsActionFilter implements ActionFilter { private final Client client; private final ThreadPool threadPool; + /** + * Creates a new filter. + * @param client An OpenSearch {@link Client}. + * @param threadPool The OpenSearch {@link ThreadPool}. + */ public UserBehaviorInsightsActionFilter(Client client, ThreadPool threadPool) { this.client = client; this.threadPool = threadPool; @@ -181,6 +190,12 @@ private String getHeaderValue(final HeaderConstants header, final String default } + /** + * Persist the query to the UBI store. + * @param storeName The name of the UBI store. + * @param queryRequest The {@link QueryRequest} that initiated the query. + * @param queryResponse The {@link QueryResponse} that resulted from the query. + */ public void persistQuery(final String storeName, final QueryRequest queryRequest, QueryResponse queryResponse) { LOGGER.info("Writing query ID {} with response ID {}", diff --git a/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java b/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java index 28cdbba..219a33a 100644 --- a/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java +++ b/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java @@ -13,19 +13,35 @@ import org.opensearch.ubi.events.queues.EventQueue; import org.opensearch.ubi.events.queues.InternalQueue; +/** + * Base class for managing client-side events. + */ public abstract class AbstractEventManager { @SuppressWarnings("unused") private final Logger LOGGER = LogManager.getLogger(AbstractEventManager.class); + /** + * The {@link EventQueue queue} that stores the client-side events. + */ protected final EventQueue eventQueue; + /** + * Initialize the base client-side event manager. + */ public AbstractEventManager() { this.eventQueue = new InternalQueue(); } + /** + * Process the items on the queue by writing them to persistent storage. + */ public abstract void process(); + /** + * Add an event to the queue. + * @param event A client-side {@link Event event} to be persisted. + */ public abstract void add(Event event); } diff --git a/src/main/java/org/opensearch/ubi/events/Event.java b/src/main/java/org/opensearch/ubi/events/Event.java index 56fff82..6895583 100644 --- a/src/main/java/org/opensearch/ubi/events/Event.java +++ b/src/main/java/org/opensearch/ubi/events/Event.java @@ -8,20 +8,43 @@ package org.opensearch.ubi.events; +/** + * A client-side event. + */ public class Event { + /** + * The name of the OpenSearch index where this event will be stored. + */ private final String indexName; + + /** + * The event (a JSON string). + */ private final String event; + /** + * Create a new event. + * @param indexName The name of the index where this event will be stored. + * @param event The event (a JSON string). + */ public Event(String indexName, String event) { this.indexName = indexName; this.event = event; } + /** + * Gets the name of the index where this event is to be stored. + * @return The name of the index where this event is to be stored. + */ public String getIndexName() { return indexName; } + /** + * Gets the event. + * @return The event. + */ public String getEvent() { return event; } diff --git a/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java b/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java index d1972a9..99a88b7 100644 --- a/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java +++ b/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java @@ -15,6 +15,9 @@ import org.opensearch.client.Client; import org.opensearch.common.xcontent.XContentType; +/** + * An event manager that inserts events into an OpenSearch index. + */ public class OpenSearchEventManager extends AbstractEventManager { private static final Logger LOGGER = LogManager.getLogger(OpenSearchEventManager.class); @@ -55,6 +58,11 @@ public void add(Event event) { eventQueue.add(event); } + /** + * Gets a singleton instance of the manager. + * @param client An OpenSearch {@link Client}. + * @return An instance of {@link OpenSearchEventManager}. + */ public static OpenSearchEventManager getInstance(Client client) { if(openSearchEventManager == null) { openSearchEventManager = new OpenSearchEventManager(client); diff --git a/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java b/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java index e26fa7f..bbd8541 100644 --- a/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java +++ b/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java @@ -12,14 +12,32 @@ import java.util.List; +/** + * A queue that stores events prior to being indexed. + */ public interface EventQueue { + /** + * Add an {@link Event event} to the queue. + * @param event The {@link Event event} to add to the queue. + */ void add(Event event); + /** + * Remove all events from the queue. + */ void clear(); + /** + * Get a list of items in the queue. + * @return A list of items in the queue. + */ List get(); + /** + * Gets the count of items on the queue. + * @return The count of items on the queue. + */ int size(); } diff --git a/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java b/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java index 62a8a45..c621beb 100644 --- a/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java +++ b/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java @@ -13,6 +13,9 @@ import java.util.LinkedList; import java.util.List; +/** + * An implementation of {@link EventQueue} that uses an in-memory list. + */ public class InternalQueue implements EventQueue { private static final List indexRequests = new LinkedList<>(); diff --git a/src/main/java/org/opensearch/ubi/model/HeaderConstants.java b/src/main/java/org/opensearch/ubi/model/HeaderConstants.java index cd106c0..958f3de 100644 --- a/src/main/java/org/opensearch/ubi/model/HeaderConstants.java +++ b/src/main/java/org/opensearch/ubi/model/HeaderConstants.java @@ -8,11 +8,29 @@ package org.opensearch.ubi.model; +/** + * HTTP headers used by the plugin. + */ public enum HeaderConstants { + /** + * The plugin-assigned ID of the query. + */ QUERY_ID_HEADER("X-ubi-query-id"), + + /** + * The name of the UBI store associated with a query. + */ EVENT_STORE_HEADER("X-ubi-store"), + + /** + * The ID of a user performing a query. + */ USER_ID_HEADER("X-ubi-user-id"), + + /** + * A session ID corresponding to the query. + */ SESSION_ID_HEADER("X-ubi-session-id"); private final String header; @@ -21,6 +39,10 @@ private HeaderConstants(String header) { this.header = header; } + /** + * Gets the string value of the header. + * @return The string value of the header. + */ public String getHeader() { return header; } diff --git a/src/main/java/org/opensearch/ubi/model/QueryRequest.java b/src/main/java/org/opensearch/ubi/model/QueryRequest.java index 3b89b05..ba4db66 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryRequest.java +++ b/src/main/java/org/opensearch/ubi/model/QueryRequest.java @@ -8,6 +8,9 @@ package org.opensearch.ubi.model; +/** + * A query received by OpenSearch. + */ public class QueryRequest { private final long timestamp; @@ -16,6 +19,13 @@ public class QueryRequest { private final String userId; private final String sessionId; + /** + * Creates a query request. + * @param queryId The ID of the query. + * @param query The query run by OpenSearch. + * @param userId The ID of the user that initiated the query. + * @param sessionId The ID of the session under which the query was run. + */ public QueryRequest(final String queryId, final String query, final String userId, final String sessionId) { this.timestamp = System.currentTimeMillis(); this.queryId = queryId; @@ -24,22 +34,42 @@ public QueryRequest(final String queryId, final String query, final String userI this.sessionId = sessionId; } + /** + * Gets the timestamp. + * @return The timestamp. + */ public long getTimestamp() { return timestamp; } + /** + * Gets the query ID. + * @return The query ID. + */ public String getQueryId() { return queryId; } + /** + * Gets the query. + * @return The query. + */ public String getQuery() { return query; } + /** + * Gets the user ID. + * @return The user ID. + */ public String getUserId() { return userId; } + /** + * Gets the session ID. + * @return The session ID. + */ public String getSessionId() { return sessionId; } diff --git a/src/main/java/org/opensearch/ubi/model/QueryResponse.java b/src/main/java/org/opensearch/ubi/model/QueryResponse.java index 4fe2bc6..8392178 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryResponse.java +++ b/src/main/java/org/opensearch/ubi/model/QueryResponse.java @@ -10,26 +10,47 @@ import java.util.List; +/** + * A query response. + */ public class QueryResponse { private final String queryId; private final String queryResponseId; private final List queryResponseHitIds; + /** + * Creates a query response. + * @param queryId The ID of the query. + * @param queryResponseId The ID of the query response. + * @param queryResponseHitIds A list of IDs for the hits in the query. + */ public QueryResponse(final String queryId, final String queryResponseId, final List queryResponseHitIds) { this.queryId = queryId; this.queryResponseId = queryResponseId; this.queryResponseHitIds = queryResponseHitIds; } + /** + * Gets the query ID. + * @return The query ID. + */ public String getQueryId() { return queryId; } + /** + * Gets the query response ID. + * @return The query response ID. + */ public String getQueryResponseId() { return queryResponseId; } + /** + * Gets the list of query response hit IDs. + * @return A list of query response hit IDs. + */ public List getQueryResponseHitIds() { return queryResponseHitIds; } diff --git a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java index 907b63a..1b5f323 100644 --- a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java +++ b/src/main/java/org/opensearch/ubi/model/SettingsConstants.java @@ -8,11 +8,24 @@ package org.opensearch.ubi.model; +/** + * Settings constants used by the plugin. + */ public class SettingsConstants { + /** + * The schema version. + */ public static final String VERSION_SETTING = "index.ubi.version"; + /** + * The name of the UBI store. + */ public static final String INDEX = "index.ubi.store"; + + /** + * The field in an index's mapping that will be used as the unique identifier for a query result item. + */ public static final String ID_FIELD = "index.ubi.id_field"; } diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java index 309cf59..a49871f 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -38,13 +38,17 @@ import static org.opensearch.rest.RestRequest.Method.*; +/** + * The REST handler for User Behavior Insights. The handler provides the + * REST interface for interacting with UBI stores and for storing client-side events. + */ public class UserBehaviorInsightsRestHandler extends BaseRestHandler { private static final Logger LOGGER = LogManager.getLogger(UserBehaviorInsightsRestHandler.class); private static final String EVENTS_MAPPING_FILE = "/events-mapping.json"; private static final String QUERIES_MAPPING_FILE = "/queries-mapping.json"; - public static final int VERSION = 1; + private static final int VERSION = 1; @Override public String getName() { diff --git a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java b/src/main/java/org/opensearch/ubi/utils/UbiUtils.java index 181c69e..fd034b7 100644 --- a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java +++ b/src/main/java/org/opensearch/ubi/utils/UbiUtils.java @@ -15,20 +15,41 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; +/** + * A utility class used by the plugin. + */ public class UbiUtils { + /** + * This is a static utility class. + */ private UbiUtils() { } + /** + * Gets the formatted name of the queries index. + * @param storeName The name of the UBI store. + * @return The formatted name of the queries index. + */ public static String getQueriesIndexName(final String storeName) { return "." + storeName + "_queries"; } + /** + * Gets the formatted name of the events index. + * @param storeName The name of the UBI store. + * @return The formatted name of the events index. + */ public static String getEventsIndexName(final String storeName) { return "." + storeName + "_events"; } + /** + * Gets the content of a resource file. + * @param fileName The file name to open and read. + * @return The content of the given filename. + */ public static String getResourceFile(final String fileName) { try (InputStream is = UbiUtils.class.getResourceAsStream(fileName)) { ByteArrayOutputStream out = new ByteArrayOutputStream(); From 92115272e6f2b31e81ce5dc347bf6781e0795a2d Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Thu, 7 Mar 2024 10:54:58 -0500 Subject: [PATCH 15/37] #67 Addign null check for id field setting. --- index-chorus-data.sh | 2 +- .../opensearch/ubi/action/UserBehaviorInsightsActionFilter.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/index-chorus-data.sh b/index-chorus-data.sh index edf7f8d..a885990 100755 --- a/index-chorus-data.sh +++ b/index-chorus-data.sh @@ -4,7 +4,7 @@ CHORUS_HOME=`realpath ../chorus-opensearch-edition` echo "Using CHORUS_HOME = ${CHORUS_HOME}" TEMP_FILE=`mktemp` -head -n 100 ${CHORUS_HOME}/transformed_data.json > ${TEMP_FILE} +head -n 50 ${CHORUS_HOME}/transformed_data.json > ${TEMP_FILE} echo "Deleting index" curl -s -X DELETE "localhost:9200/ecommerce" diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index f189aee..88b84ee 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -107,7 +107,7 @@ public void onResponse(Response response) { // Add each hit to the list of query responses. for (final SearchHit hit : searchResponse.getHits()) { - if ("".equals(idField)) { + if (idField == null || "".equals(idField) || idField.equals("null")) { // Use the _id since there is no id_field setting for this index. queryResponseHitIds.add(String.valueOf(hit.docId())); From c3a149baffa8e0b39ff2e8927777adff084be28a Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Thu, 7 Mar 2024 13:24:26 -0500 Subject: [PATCH 16/37] #92 Caching index settings. --- .../ubi/UserBehaviorInsightsPlugin.java | 7 ++-- .../UserBehaviorInsightsActionFilter.java | 42 +++++++++++++++---- .../rest/UserBehaviorInsightsRestHandler.java | 4 ++ 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 530ed9e..7c229ed 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -35,10 +35,7 @@ import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -53,6 +50,8 @@ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private ActionFilter userBehaviorLoggingFilter; + public static final Map storeSettings = new HashMap<>(); + @Override public Collection getRestHeaders() { return List.of( diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 0be29c6..02471bd 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -25,6 +25,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.ubi.UserBehaviorInsightsPlugin; import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; @@ -91,13 +92,8 @@ public void onResponse(Response response) { // If there is no event store header, ignore this search. if(!"".equals(eventStore)) { - // Get the id_field to use for each result's unique identifier. - final String queriesIndexName = UbiUtils.getQueriesIndexName(eventStore); - final GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices(queriesIndexName); - - final GetSettingsResponse getSettingsResponse = client.admin().indices().getSettings(getSettingsRequest).actionGet(); - final String idField = getSettingsResponse.getSetting(queriesIndexName, SettingsConstants.ID_FIELD); - final String index = getSettingsResponse.getSetting(queriesIndexName, SettingsConstants.INDEX); + final String index = getStoreSettings(eventStore, SettingsConstants.INDEX); + final String idField = getStoreSettings(eventStore, SettingsConstants.ID_FIELD); LOGGER.info("Using id_field [{}] of index [{}] for UBI query.", idField, index); @@ -178,6 +174,36 @@ public void onFailure(Exception ex) { } + private String getStoreSettings(final String storeName, final String setting) { + + final String key = storeName + "." + setting; + final String value; + + if(UserBehaviorInsightsPlugin.storeSettings.containsKey(key)) { + + LOGGER.info("Getting setting " + setting + " for store " + storeName + " from the cache."); + value = UserBehaviorInsightsPlugin.storeSettings.get(key); + + } else{ + + LOGGER.info("Getting setting " + setting + " for store " + storeName + " from the index."); + + // Get the id_field to use for each result's unique identifier. + final String queriesIndexName = UbiUtils.getQueriesIndexName(storeName); + final GetSettingsRequest getSettingsRequest = new GetSettingsRequest().indices(queriesIndexName); + + final GetSettingsResponse getSettingsResponse = client.admin().indices().getSettings(getSettingsRequest).actionGet(); + final String settingResponse = getSettingsResponse.getSetting(queriesIndexName, setting); + + UserBehaviorInsightsPlugin.storeSettings.put(key, settingResponse); + value = settingResponse; + + } + + return value; + + } + private String getHeaderValue(final HeaderConstants header, final String defaultValue, final Task task) { final String value = task.getHeader(header.getHeader()); @@ -196,7 +222,7 @@ private String getHeaderValue(final HeaderConstants header, final String default * @param queryRequest The {@link QueryRequest} that initiated the query. * @param queryResponse The {@link QueryResponse} that resulted from the query. */ - public void persistQuery(final String storeName, final QueryRequest queryRequest, QueryResponse queryResponse) { + private void persistQuery(final String storeName, final QueryRequest queryRequest, QueryResponse queryResponse) { LOGGER.info("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryResponse.getQueryResponseId()); diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java index a49871f..9b9bdef 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -27,6 +27,7 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; +import org.opensearch.ubi.UserBehaviorInsightsPlugin; import org.opensearch.ubi.events.Event; import org.opensearch.ubi.events.OpenSearchEventManager; import org.opensearch.ubi.model.HeaderConstants; @@ -220,6 +221,9 @@ private RestChannelConsumer delete(final NodeClient nodeClient, final String sto builder.startObject().field("status", "deleted"); builder.endObject(); + // Remove this store's settings from the settings map. + UserBehaviorInsightsPlugin.storeSettings.entrySet().removeIf(entry -> entry.getKey().startsWith(storeName + ".")); + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); } From 215d9543942895d55c01eed73b4518dea3aad2cf Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Thu, 7 Mar 2024 13:26:45 -0500 Subject: [PATCH 17/37] #92 Changing info to debug log. --- .../ubi/action/UserBehaviorInsightsActionFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 02471bd..a7b9cae 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -181,12 +181,12 @@ private String getStoreSettings(final String storeName, final String setting) { if(UserBehaviorInsightsPlugin.storeSettings.containsKey(key)) { - LOGGER.info("Getting setting " + setting + " for store " + storeName + " from the cache."); + LOGGER.debug("Getting setting " + setting + " for store " + storeName + " from the cache."); value = UserBehaviorInsightsPlugin.storeSettings.get(key); } else{ - LOGGER.info("Getting setting " + setting + " for store " + storeName + " from the index."); + LOGGER.debug("Getting setting " + setting + " for store " + storeName + " from the index."); // Get the id_field to use for each result's unique identifier. final String queriesIndexName = UbiUtils.getQueriesIndexName(storeName); From a7acee04d06aa47452e24d5b287abf7159b64b8a Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Thu, 7 Mar 2024 15:51:28 -0500 Subject: [PATCH 18/37] #92 Removing grouped imports. --- .../ubi/UserBehaviorInsightsPlugin.java | 13 +++++++++++-- .../action/UserBehaviorInsightsActionFilter.java | 7 ++++++- .../ubi/rest/UserBehaviorInsightsRestHandler.java | 15 ++++++++++++--- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 7c229ed..02dbb3d 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -15,7 +15,11 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.*; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.settings.SettingsFilter; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; @@ -35,7 +39,12 @@ import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index a7b9cae..6ba5ccc 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -32,7 +32,12 @@ import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.utils.UbiUtils; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; /** * An implementation of {@link ActionFilter} that passively listens for OpenSearch diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java index 9b9bdef..6886f1f 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -35,9 +35,18 @@ import org.opensearch.ubi.utils.UbiUtils; import java.io.IOException; -import java.util.*; - -import static org.opensearch.rest.RestRequest.Method.*; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +import static org.opensearch.rest.RestRequest.Method.DELETE; +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.rest.RestRequest.Method.PUT; +import static org.opensearch.rest.RestRequest.Method.TRACE; /** * The REST handler for User Behavior Insights. The handler provides the From d62df3ba658e286e22a7d6af39a9a98502ef7052 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 8 Mar 2024 08:35:01 -0500 Subject: [PATCH 19/37] #101 Changing group name to be com.o19s. --- build.gradle | 6 +++--- .../o19s}/ubi/UserBehaviorInsightsPlugin.java | 15 +++++++++------ .../action/UserBehaviorInsightsActionFilter.java | 16 ++++++++-------- .../o19s}/ubi/events/AbstractEventManager.java | 6 +++--- .../o19s}/ubi/events/Event.java | 2 +- .../o19s}/ubi/events/OpenSearchEventManager.java | 2 +- .../o19s}/ubi/events/queues/EventQueue.java | 4 ++-- .../o19s}/ubi/events/queues/InternalQueue.java | 4 ++-- .../o19s}/ubi/model/HeaderConstants.java | 2 +- .../o19s}/ubi/model/QueryRequest.java | 2 +- .../o19s}/ubi/model/QueryResponse.java | 2 +- .../o19s}/ubi/model/SettingsConstants.java | 2 +- .../rest/UserBehaviorInsightsRestHandler.java | 14 +++++++------- .../o19s}/ubi/utils/UbiUtils.java | 2 +- 14 files changed, 41 insertions(+), 38 deletions(-) rename src/main/java/{org/opensearch => com/o19s}/ubi/UserBehaviorInsightsPlugin.java (93%) rename src/main/java/{org/opensearch => com/o19s}/ubi/action/UserBehaviorInsightsActionFilter.java (96%) rename src/main/java/{org/opensearch => com/o19s}/ubi/events/AbstractEventManager.java (88%) rename src/main/java/{org/opensearch => com/o19s}/ubi/events/Event.java (96%) rename src/main/java/{org/opensearch => com/o19s}/ubi/events/OpenSearchEventManager.java (98%) rename src/main/java/{org/opensearch => com/o19s}/ubi/events/queues/EventQueue.java (90%) rename src/main/java/{org/opensearch => com/o19s}/ubi/events/queues/InternalQueue.java (90%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/HeaderConstants.java (96%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/QueryRequest.java (98%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/QueryResponse.java (97%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/SettingsConstants.java (95%) rename src/main/java/{org/opensearch => com/o19s}/ubi/rest/UserBehaviorInsightsRestHandler.java (97%) rename src/main/java/{org/opensearch => com/o19s}/ubi/utils/UbiUtils.java (98%) diff --git a/build.gradle b/build.gradle index 915ee5c..2eb099d 100644 --- a/build.gradle +++ b/build.gradle @@ -7,12 +7,12 @@ apply plugin: 'maven-publish' opensearchplugin { name 'opensearch-ubi' description 'OpenSearch User Behavior Insights Plugin' - classname 'org.opensearch.ubi.UserBehaviorInsightsPlugin' + classname 'com.o19s.ubi.UserBehaviorInsightsPlugin' licenseFile rootProject.file('LICENSE.txt') noticeFile rootProject.file('NOTICE.txt') } -group = 'org.opensearch' +group = 'com.o19s' version = "${ubiVersion}-os${opensearchVersion}" // disabling some unnecessary validations for this plugin @@ -61,7 +61,7 @@ publishing { pom { name = "opensearch-ubi" description = "Provides User Behavior Insights for OpenSearch" - groupId = "org.opensearch" + groupId = "com.o19s" licenses { license { name = "The Apache License, Version 2.0" diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java similarity index 93% rename from src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java rename to src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java index 02dbb3d..9c4cc38 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi; +package com.o19s.ubi; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,11 +32,11 @@ import org.opensearch.rest.RestHeaderDefinition; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.ubi.action.UserBehaviorInsightsActionFilter; -import org.opensearch.ubi.rest.UserBehaviorInsightsRestHandler; -import org.opensearch.ubi.events.OpenSearchEventManager; -import org.opensearch.ubi.model.HeaderConstants; -import org.opensearch.ubi.model.SettingsConstants; +import com.o19s.ubi.action.UserBehaviorInsightsActionFilter; +import com.o19s.ubi.rest.UserBehaviorInsightsRestHandler; +import com.o19s.ubi.events.OpenSearchEventManager; +import com.o19s.ubi.model.HeaderConstants; +import com.o19s.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; import java.util.ArrayList; @@ -59,6 +59,9 @@ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private ActionFilter userBehaviorLoggingFilter; + /** + * A map that caches store settings to avoid round-trip calls to the index. + */ public static final Map storeSettings = new HashMap<>(); @Override diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java similarity index 96% rename from src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java rename to src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java index 6ba5ccc..be2ac87 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java @@ -6,8 +6,14 @@ * compatible open source license. */ -package org.opensearch.ubi.action; - +package com.o19s.ubi.action; + +import com.o19s.ubi.UserBehaviorInsightsPlugin; +import com.o19s.ubi.model.HeaderConstants; +import com.o19s.ubi.model.QueryRequest; +import com.o19s.ubi.model.QueryResponse; +import com.o19s.ubi.model.SettingsConstants; +import com.o19s.ubi.utils.UbiUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; @@ -25,12 +31,6 @@ import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.ubi.UserBehaviorInsightsPlugin; -import org.opensearch.ubi.model.HeaderConstants; -import org.opensearch.ubi.model.QueryRequest; -import org.opensearch.ubi.model.QueryResponse; -import org.opensearch.ubi.model.SettingsConstants; -import org.opensearch.ubi.utils.UbiUtils; import java.util.Arrays; import java.util.HashMap; diff --git a/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java b/src/main/java/com/o19s/ubi/events/AbstractEventManager.java similarity index 88% rename from src/main/java/org/opensearch/ubi/events/AbstractEventManager.java rename to src/main/java/com/o19s/ubi/events/AbstractEventManager.java index 219a33a..2f3a9b2 100644 --- a/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java +++ b/src/main/java/com/o19s/ubi/events/AbstractEventManager.java @@ -6,12 +6,12 @@ * compatible open source license. */ -package org.opensearch.ubi.events; +package com.o19s.ubi.events; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.ubi.events.queues.EventQueue; -import org.opensearch.ubi.events.queues.InternalQueue; +import com.o19s.ubi.events.queues.EventQueue; +import com.o19s.ubi.events.queues.InternalQueue; /** * Base class for managing client-side events. diff --git a/src/main/java/org/opensearch/ubi/events/Event.java b/src/main/java/com/o19s/ubi/events/Event.java similarity index 96% rename from src/main/java/org/opensearch/ubi/events/Event.java rename to src/main/java/com/o19s/ubi/events/Event.java index 6895583..5f4c250 100644 --- a/src/main/java/org/opensearch/ubi/events/Event.java +++ b/src/main/java/com/o19s/ubi/events/Event.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.events; +package com.o19s.ubi.events; /** * A client-side event. diff --git a/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java b/src/main/java/com/o19s/ubi/events/OpenSearchEventManager.java similarity index 98% rename from src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java rename to src/main/java/com/o19s/ubi/events/OpenSearchEventManager.java index 99a88b7..9c6aaf5 100644 --- a/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java +++ b/src/main/java/com/o19s/ubi/events/OpenSearchEventManager.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.events; +package com.o19s.ubi.events; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java b/src/main/java/com/o19s/ubi/events/queues/EventQueue.java similarity index 90% rename from src/main/java/org/opensearch/ubi/events/queues/EventQueue.java rename to src/main/java/com/o19s/ubi/events/queues/EventQueue.java index bbd8541..fc6e802 100644 --- a/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java +++ b/src/main/java/com/o19s/ubi/events/queues/EventQueue.java @@ -6,9 +6,9 @@ * compatible open source license. */ -package org.opensearch.ubi.events.queues; +package com.o19s.ubi.events.queues; -import org.opensearch.ubi.events.Event; +import com.o19s.ubi.events.Event; import java.util.List; diff --git a/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java b/src/main/java/com/o19s/ubi/events/queues/InternalQueue.java similarity index 90% rename from src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java rename to src/main/java/com/o19s/ubi/events/queues/InternalQueue.java index c621beb..dcb7996 100644 --- a/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java +++ b/src/main/java/com/o19s/ubi/events/queues/InternalQueue.java @@ -6,9 +6,9 @@ * compatible open source license. */ -package org.opensearch.ubi.events.queues; +package com.o19s.ubi.events.queues; -import org.opensearch.ubi.events.Event; +import com.o19s.ubi.events.Event; import java.util.LinkedList; import java.util.List; diff --git a/src/main/java/org/opensearch/ubi/model/HeaderConstants.java b/src/main/java/com/o19s/ubi/model/HeaderConstants.java similarity index 96% rename from src/main/java/org/opensearch/ubi/model/HeaderConstants.java rename to src/main/java/com/o19s/ubi/model/HeaderConstants.java index 958f3de..e098aed 100644 --- a/src/main/java/org/opensearch/ubi/model/HeaderConstants.java +++ b/src/main/java/com/o19s/ubi/model/HeaderConstants.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; /** * HTTP headers used by the plugin. diff --git a/src/main/java/org/opensearch/ubi/model/QueryRequest.java b/src/main/java/com/o19s/ubi/model/QueryRequest.java similarity index 98% rename from src/main/java/org/opensearch/ubi/model/QueryRequest.java rename to src/main/java/com/o19s/ubi/model/QueryRequest.java index ba4db66..71ab8a7 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryRequest.java +++ b/src/main/java/com/o19s/ubi/model/QueryRequest.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; /** * A query received by OpenSearch. diff --git a/src/main/java/org/opensearch/ubi/model/QueryResponse.java b/src/main/java/com/o19s/ubi/model/QueryResponse.java similarity index 97% rename from src/main/java/org/opensearch/ubi/model/QueryResponse.java rename to src/main/java/com/o19s/ubi/model/QueryResponse.java index 8392178..7ac1af0 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryResponse.java +++ b/src/main/java/com/o19s/ubi/model/QueryResponse.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; import java.util.List; diff --git a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java b/src/main/java/com/o19s/ubi/model/SettingsConstants.java similarity index 95% rename from src/main/java/org/opensearch/ubi/model/SettingsConstants.java rename to src/main/java/com/o19s/ubi/model/SettingsConstants.java index 1b5f323..469d830 100644 --- a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java +++ b/src/main/java/com/o19s/ubi/model/SettingsConstants.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; /** * Settings constants used by the plugin. diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java similarity index 97% rename from src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java rename to src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java index 6886f1f..949f150 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -6,12 +6,16 @@ * compatible open source license. */ -package org.opensearch.ubi.rest; +package com.o19s.ubi.rest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.o19s.ubi.UserBehaviorInsightsPlugin; +import com.o19s.ubi.events.Event; +import com.o19s.ubi.model.HeaderConstants; +import com.o19s.ubi.model.SettingsConstants; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.create.CreateIndexRequest; @@ -27,12 +31,8 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.ubi.UserBehaviorInsightsPlugin; -import org.opensearch.ubi.events.Event; -import org.opensearch.ubi.events.OpenSearchEventManager; -import org.opensearch.ubi.model.HeaderConstants; -import org.opensearch.ubi.model.SettingsConstants; -import org.opensearch.ubi.utils.UbiUtils; +import com.o19s.ubi.events.OpenSearchEventManager; +import com.o19s.ubi.utils.UbiUtils; import java.io.IOException; import java.util.HashSet; diff --git a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java b/src/main/java/com/o19s/ubi/utils/UbiUtils.java similarity index 98% rename from src/main/java/org/opensearch/ubi/utils/UbiUtils.java rename to src/main/java/com/o19s/ubi/utils/UbiUtils.java index fd034b7..8b4620c 100644 --- a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java +++ b/src/main/java/com/o19s/ubi/utils/UbiUtils.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.utils; +package com.o19s.ubi.utils; import org.opensearch.common.util.io.Streams; From a84320f5e36eb7fcf03503c2ef792eb6a4ada519 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Fri, 8 Mar 2024 16:31:03 -0500 Subject: [PATCH 20/37] #106 Adding literal 'null' check. --- .../opensearch/ubi/action/UserBehaviorInsightsActionFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 6ba5ccc..4c6abca 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -213,7 +213,7 @@ private String getHeaderValue(final HeaderConstants header, final String default final String value = task.getHeader(header.getHeader()); - if(value == null || value.trim().isEmpty()) { + if(value == null || value.trim().isEmpty() || value.equals("null")) { return defaultValue; } else { return value; From 0dfb65dd183b2ea5639e3135cedec04f168465b0 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 10:59:07 -0400 Subject: [PATCH 21/37] #103 Adding release workflow. --- .github/ISSUE_TEMPLATE/BUG_TEMPLATE.md | 24 +++++++++++ .../FEATURE_REQUEST_TEMPLATE.md | 18 ++++++++ .github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md | 40 +++++++++++++++++ .github/workflows/release.yml | 43 +++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/BUG_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md create mode 100644 .github/workflows/release.yml diff --git a/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md b/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md new file mode 100644 index 0000000..823be5e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md @@ -0,0 +1,24 @@ +--- +name: � Bug report +about: Create a report to help us improve +title: '[BUG]' +labels: 'bug, untriaged' +assignees: '' +--- +### What is the bug? +_A clear and concise description of the bug._ + +### How can one reproduce the bug? +_Steps to reproduce the behavior._ + +### What is the expected behavior? +_A clear and concise description of what you expected to happen._ + +### What is your host/environment? +_Operating system, version._ + +### Do you have any screenshots? +_If applicable, add screenshots to help explain your problem._ + +### Do you have any additional context? +_Add any other context about the problem._ \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..0836c34 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +--- +name: � Feature request +about: Request a feature in this project +title: '[FEATURE]' +labels: 'enhancement, untriaged' +assignees: '' +--- +### Is your feature request related to a problem? +_A clear and concise description of what the problem is, e.g. I'm always frustrated when [...]._ + +### What solution would you like? +_A clear and concise description of what you want to happen._ + +### What alternatives have you considered? +_A clear and concise description of any alternative solutions or features you've considered._ + +### Do you have any additional context? +_Add any other context or screenshots about the feature request here._ \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md b/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md new file mode 100644 index 0000000..0043ce5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md @@ -0,0 +1,40 @@ +--- +name: � Proposal +about: Suggest an idea for a specific feature you wish to propose to the community for comment +title: '[PROPOSAL]' +labels: proposal +assignees: '' +--- +## What/Why +### What are you proposing? +_In a few sentences, describe the feature and its core capabilities._ + +### What users have asked for this feature? +_Highlight any research, proposals, requests or anecdotes that signal this is the right thing to build. Include links to GitHub Issues, Forums, Stack Overflow, Twitter, Etc_ + +### What problems are you trying to solve? +_Summarize the core use cases and user problems and needs you are trying to solve. Describe the most important user needs, pain points and jobs as expressed by the user asks above. Template: When \ , a \ wants to \, so they can \. (Example: When **searching by postal code**, **a buyer** wants to **be required to enter a valid code** so they **don’t waste time searching for a clearly invalid postal code.**)_ + +### What is the developer experience going to be? +_Does this have a REST API? If so, please describe the API and any impact it may have to existing APIs. In a brief summary (not a spec), highlight what new REST APIs or changes to REST APIs are planned. as well as any other API, CLI or Configuration changes that are planned as part of this feature._ + +#### Are there any security considerations? +_Describe if the feature has any security considerations or impact. What is the security model of the new APIs? Features should be integrated into the OpenSearch security suite and so if they are not, we should highlight the reasons here._ + +#### Are there any breaking changes to the API +_If this feature will require breaking changes to any APIs, ouline what those are and why they are needed. What is the path to minimizing impact? (example, add new API and deprecate the old one)_ + +### What is the user experience going to be? +_Describe the feature requirements and or user stories. You may include low-fidelity sketches, wireframes, APIs stubs, or other examples of how a user would use the feature via CLI, OpenSearch Dashboards, REST API, etc. Using a bulleted list or simple diagrams to outline features is okay. If this is net new functionality, call this out as well._ + +#### Are there breaking changes to the User Experience? +_Will this change the existing user experience? Will this be a breaking change from a user flow or user experience perspective?_ + +### Why should it be built? Any reason not to? +_Describe the value that this feature will bring to the OpenSearch community, as well as what impact it has if it isn't built, or new risks if it is. Highlight opportunities for additional research._ + +### What will it take to execute? +_Describe what it will take to build this feature. Are there any assumptions you may be making that could limit scope or add limitations? Are there performance, cost, or technical constraints that may impact the user experience? Does this feature depend on other feature work? What additional risks are there?_ + +### Any remaining open questions? +_What are known enhancements to this feature? Any enhancements that may be out of scope but that we will want to track long term? List any other open questions that may need to be answered before proceeding with an implementation._ \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d870f87 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Tag and publish a release + +on: + push: + tags: + - 'v*.*.*' + branches: [ test-release ] + +jobs: + release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v2 + - name: Set release version Name + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up JDK 17.0 + uses: actions/setup-java@v1 + with: + java-version: 17.0 + - name: Grant execute permission for gradlew + run: chmod +x gradlew + - name: Build with Gradle + run: ./gradlew -Dtests.security.manager=false build + - name: Rename build assets + run: cp ./build/distributions/opensearch-ubi.zip ./opensearch-ubi-plugin-${{ env.RELEASE_VERSION }}.zip + - name: Create Release + id: create_release + uses: ncipollo/release-action@v1 + with: + artifacts: "./opensearch-ubi-plugin-${{ env.RELEASE_VERSION }}.zip" + token: ${{ secrets.GITHUB_TOKEN }} + tag: "release-${{ env.RELEASE_VERSION }}" + - name: Upload Release Asset + id: upload-release-asset + uses: softprops/action-gh-release@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + tag_name: "release-${{ env.RELEASE_VERSION }}" + #upload_url: ${{ steps.create_release.outputs.upload_url }} + files: ./opensearch-ubi-plugin-${{ env.RELEASE_VERSION }}.zip + name: ${{ env.RELEASE_VERSION }} From c2aed66e5b9f4ee8272928ad5f20a80ccc7192d4 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 11:02:12 -0400 Subject: [PATCH 22/37] #103 Removing special characters. --- .github/ISSUE_TEMPLATE/BUG_TEMPLATE.md | 2 +- .github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md | 2 +- .github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md b/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md index 823be5e..c8fe373 100644 --- a/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md @@ -1,5 +1,5 @@ --- -name: � Bug report +name: Bug report about: Create a report to help us improve title: '[BUG]' labels: 'bug, untriaged' diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md index 0836c34..21b62ae 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST_TEMPLATE.md @@ -1,5 +1,5 @@ --- -name: � Feature request +name: Feature request about: Request a feature in this project title: '[FEATURE]' labels: 'enhancement, untriaged' diff --git a/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md b/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md index 0043ce5..08a6bb7 100644 --- a/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/PROPOSAL_TEMPLATE.md @@ -1,5 +1,5 @@ --- -name: � Proposal +name: Proposal about: Suggest an idea for a specific feature you wish to propose to the community for comment title: '[PROPOSAL]' labels: proposal From c340ae4831c841c19068a76c140731c8ce0b3ff1 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 11:05:26 -0400 Subject: [PATCH 23/37] #103 Changing UBI version. --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 915ee5c..7fae8f4 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,8 @@ opensearchplugin { } group = 'org.opensearch' -version = "${ubiVersion}-os${opensearchVersion}" +//version = "${ubiVersion}-os${opensearchVersion}" +version = "${ubiVersion}" // disabling some unnecessary validations for this plugin testingConventions.enabled = false From feb911f862ac9783911b810c3fbfa9072af32ab0 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 11:10:06 -0400 Subject: [PATCH 24/37] #103 Changing UBI version and copy. --- .github/workflows/release.yml | 2 +- build.gradle | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d870f87..f8c1b37 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: - name: Build with Gradle run: ./gradlew -Dtests.security.manager=false build - name: Rename build assets - run: cp ./build/distributions/opensearch-ubi.zip ./opensearch-ubi-plugin-${{ env.RELEASE_VERSION }}.zip + run: cp ./build/distributions/opensearch-ubi-*.zip ./opensearch-ubi-plugin-${{ env.RELEASE_VERSION }}.zip - name: Create Release id: create_release uses: ncipollo/release-action@v1 diff --git a/build.gradle b/build.gradle index 7fae8f4..915ee5c 100644 --- a/build.gradle +++ b/build.gradle @@ -13,8 +13,7 @@ opensearchplugin { } group = 'org.opensearch' -//version = "${ubiVersion}-os${opensearchVersion}" -version = "${ubiVersion}" +version = "${ubiVersion}-os${opensearchVersion}" // disabling some unnecessary validations for this plugin testingConventions.enabled = false From 4804243b30b0ab8f80dd4cae580aa34b28073966 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 11:15:30 -0400 Subject: [PATCH 25/37] #103 Adding script to tag. --- tag.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 tag.sh diff --git a/tag.sh b/tag.sh new file mode 100755 index 0000000..77ae161 --- /dev/null +++ b/tag.sh @@ -0,0 +1,18 @@ +#!/bin/bash -e + +GRADLE_PROPERTIES_FILE=gradle.properties + +function getProperty { + PROP_KEY=$1 + PROP_VALUE=`cat $GRADLE_PROPERTIES_FILE | grep "$PROP_KEY" | cut -d'=' -f2` + echo $PROP_VALUE +} + +OPENSEARCH_VERSION=$(getProperty "opensearchVersion") +UBI_VERSION=$(getProperty "ubiVersion") + +TAG_VERSION="v${UBI_VERSION}-os${OPENSEARCH_VERSION}" +echo "Tagging as ${TAG_VERSION}" + +git tag -a "${TAG_VERSION}" -m "${TAG_VERSION}" +git push --tags From 0db65b3d0a00d6917069dace382d9cbeb0c633c0 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 11:16:34 -0400 Subject: [PATCH 26/37] #103 Renaming script. --- tag.sh => tag-and-release.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tag.sh => tag-and-release.sh (100%) diff --git a/tag.sh b/tag-and-release.sh similarity index 100% rename from tag.sh rename to tag-and-release.sh From c13b4aa399c71ac5fa26e52fd10ce40b7308a9ce Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 12:21:36 -0400 Subject: [PATCH 27/37] #97 Writing queries to a queue prior to indexing. --- index-chorus-data.sh | 2 +- load-test/load-test.py | 9 ++ load-test/run.sh | 19 ++- .../ubi/UserBehaviorInsightsPlugin.java | 2 +- .../UserBehaviorInsightsActionFilter.java | 42 ++++--- .../ubi/events/AbstractEventManager.java | 47 ------- .../ubi/events/OpenSearchEventManager.java | 73 ----------- .../opensearch/ubi/model/QueryRequest.java | 25 +++- .../ubi/{ => model}/events/Event.java | 2 +- .../ubi/model/events/EventManager.java | 60 +++++++++ .../model/events/OpenSearchEventManager.java | 119 ++++++++++++++++++ .../events/queues/InternalQueue.java | 14 +-- .../events/queues/Queue.java} | 10 +- .../rest/UserBehaviorInsightsRestHandler.java | 4 +- 14 files changed, 264 insertions(+), 164 deletions(-) delete mode 100644 src/main/java/org/opensearch/ubi/events/AbstractEventManager.java delete mode 100644 src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java rename src/main/java/org/opensearch/ubi/{ => model}/events/Event.java (96%) create mode 100644 src/main/java/org/opensearch/ubi/model/events/EventManager.java create mode 100644 src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java rename src/main/java/org/opensearch/ubi/{ => model}/events/queues/InternalQueue.java (60%) rename src/main/java/org/opensearch/ubi/{events/queues/EventQueue.java => model/events/queues/Queue.java} (81%) diff --git a/index-chorus-data.sh b/index-chorus-data.sh index a885990..44d34e6 100755 --- a/index-chorus-data.sh +++ b/index-chorus-data.sh @@ -1,6 +1,6 @@ #!/bin/bash -e -CHORUS_HOME=`realpath ../chorus-opensearch-edition` +CHORUS_HOME=${1:-`realpath ../chorus-opensearch-edition`} echo "Using CHORUS_HOME = ${CHORUS_HOME}" TEMP_FILE=`mktemp` diff --git a/load-test/load-test.py b/load-test/load-test.py index d818ad3..780c359 100644 --- a/load-test/load-test.py +++ b/load-test/load-test.py @@ -22,3 +22,12 @@ def event_task(self): } self.client.post("/_plugins/ubi/mystore", headers=headers, json=data) + + @task + def queries_task(self): + headers = { + "Content-Type": "application/json", + "X-ubi-store": "mystore" + } + + self.client.get("/ecommerce/_search", headers=headers) \ No newline at end of file diff --git a/load-test/run.sh b/load-test/run.sh index fc36a81..09851e4 100755 --- a/load-test/run.sh +++ b/load-test/run.sh @@ -1,17 +1,24 @@ #!/bin/bash -e # Delete the store if it exists. -curl -X DELETE http://localhost:9200/_plugins/ubi/mystore +curl -X DELETE "http://localhost:9200/_plugins/ubi/mystore" -# Create the store -curl -X PUT http://localhost:9200/_plugins/ubi/mystore +# Create the store. +curl -X PUT "http://localhost:9200/_plugins/ubi/mystore?index=ecommerce" -# Insert events +# Index subset of Chorus data. +../index-chorus-data.sh `realpath ../../chorus-opensearch-edition` + +# Insert events and queries. locust -f load-test.py --headless -u 1 -r 1 --run-time 10s --host http://localhost:9200 # Let events index. -sleep 2 +sleep 5 # Get count of indexed events. EVENTS=`curl -s http://localhost:9200/.mystore_events/_count | jq .count` -echo "Found $EVENTS indexed" +echo "Found $EVENTS events" + +# Get count of indexed queries. +QUERIES=`curl -s http://localhost:9200/.mystore_queries/_count | jq .count` +echo "Found $QUERIES queries" \ No newline at end of file diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java index 02dbb3d..e82d0ff 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java @@ -34,7 +34,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.ubi.action.UserBehaviorInsightsActionFilter; import org.opensearch.ubi.rest.UserBehaviorInsightsRestHandler; -import org.opensearch.ubi.events.OpenSearchEventManager; +import org.opensearch.ubi.model.events.OpenSearchEventManager; import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java index 6ba5ccc..fb10ecb 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java @@ -26,6 +26,8 @@ import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.ubi.UserBehaviorInsightsPlugin; +import org.opensearch.ubi.model.events.EventManager; +import org.opensearch.ubi.model.events.OpenSearchEventManager; import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.QueryRequest; import org.opensearch.ubi.model.QueryResponse; @@ -49,6 +51,7 @@ public class UserBehaviorInsightsActionFilter implements ActionFilter { private final Client client; private final ThreadPool threadPool; + private final EventManager eventManager; /** * Creates a new filter. @@ -58,6 +61,7 @@ public class UserBehaviorInsightsActionFilter implements ActionFilter { public UserBehaviorInsightsActionFilter(Client client, ThreadPool threadPool) { this.client = client; this.threadPool = threadPool; + this.eventManager = OpenSearchEventManager.getInstance(client); } @Override @@ -90,15 +94,15 @@ public void onResponse(Response response) { // Get info from the headers. final String queryId = getHeaderValue(HeaderConstants.QUERY_ID_HEADER, UUID.randomUUID().toString(), task); - final String eventStore = getHeaderValue(HeaderConstants.EVENT_STORE_HEADER, "", task); + final String storeName = getHeaderValue(HeaderConstants.EVENT_STORE_HEADER, "", task); final String userId = getHeaderValue(HeaderConstants.USER_ID_HEADER, "", task); final String sessionId = getHeaderValue(HeaderConstants.SESSION_ID_HEADER, "", task); // If there is no event store header, ignore this search. - if(!"".equals(eventStore)) { + if(!"".equals(storeName)) { - final String index = getStoreSettings(eventStore, SettingsConstants.INDEX); - final String idField = getStoreSettings(eventStore, SettingsConstants.ID_FIELD); + final String index = getStoreSettings(storeName, SettingsConstants.INDEX); + final String idField = getStoreSettings(storeName, SettingsConstants.ID_FIELD); LOGGER.info("Using id_field [{}] of index [{}] for UBI query.", idField, index); @@ -133,10 +137,12 @@ public void onResponse(Response response) { try { - // Persist the query to the backend. - persistQuery(eventStore, - new QueryRequest(queryId, query, userId, sessionId), - new QueryResponse(queryId, queryResponseId, queryResponseHitIds)); + final QueryResponse queryResponse = new QueryResponse(queryId, queryResponseId, queryResponseHitIds); + final QueryRequest queryRequest = new QueryRequest(storeName, queryId, query, userId, sessionId, queryResponse); + + // Queue this for writing to the UBI store. + LOGGER.info("Queueing query request"); + eventManager.add(queryRequest); } catch (Exception ex) { // TODO: Handle this. @@ -145,24 +151,22 @@ public void onResponse(Response response) { threadPool.getThreadContext().addResponseHeader("query_id", queryId); - //} - final long elapsedTime = System.currentTimeMillis() - startTime; LOGGER.info("UBI search request filter took {} ms", elapsedTime); } - } + LOGGER.info("Setting and exposing query_id {}", queryId); + //HACK: this should be set in the OpenSearch config (to send to the client code just once), + // and not on every single search response, + // but that server setting doesn't appear to be exposed. + threadPool.getThreadContext().addResponseHeader("Access-Control-Expose-Headers", "query_id"); + threadPool.getThreadContext().addResponseHeader("query_id", queryId); - LOGGER.info("Setting and exposing query_id {}", queryId); - //HACK: this should be set in the OpenSearch config (to send to the client code just once), - // and not on every single search response, - // but that server setting doesn't appear to be exposed. - threadPool.getThreadContext().addResponseHeader("Access-Control-Expose-Headers", "query_id"); - threadPool.getThreadContext().addResponseHeader("query_id", queryId); + final long elapsedTime = System.currentTimeMillis() - startTime; + LOGGER.info("UBI search request filter took {} ms", elapsedTime); - final long elapsedTime = System.currentTimeMillis() - startTime; - LOGGER.info("UBI search request filter took {} ms", elapsedTime); + } } diff --git a/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java b/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java deleted file mode 100644 index 219a33a..0000000 --- a/src/main/java/org/opensearch/ubi/events/AbstractEventManager.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ubi.events; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.ubi.events.queues.EventQueue; -import org.opensearch.ubi.events.queues.InternalQueue; - -/** - * Base class for managing client-side events. - */ -public abstract class AbstractEventManager { - - @SuppressWarnings("unused") - private final Logger LOGGER = LogManager.getLogger(AbstractEventManager.class); - - /** - * The {@link EventQueue queue} that stores the client-side events. - */ - protected final EventQueue eventQueue; - - /** - * Initialize the base client-side event manager. - */ - public AbstractEventManager() { - this.eventQueue = new InternalQueue(); - } - - /** - * Process the items on the queue by writing them to persistent storage. - */ - public abstract void process(); - - /** - * Add an event to the queue. - * @param event A client-side {@link Event event} to be persisted. - */ - public abstract void add(Event event); - -} diff --git a/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java b/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java deleted file mode 100644 index 99a88b7..0000000 --- a/src/main/java/org/opensearch/ubi/events/OpenSearchEventManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.ubi.events; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.client.Client; -import org.opensearch.common.xcontent.XContentType; - -/** - * An event manager that inserts events into an OpenSearch index. - */ -public class OpenSearchEventManager extends AbstractEventManager { - - private static final Logger LOGGER = LogManager.getLogger(OpenSearchEventManager.class); - - private final Client client; - private static OpenSearchEventManager openSearchEventManager; - - private OpenSearchEventManager(Client client) { - this.client = client; - } - - @Override - public void process() { - - if(eventQueue.size() > 0) { - - final BulkRequest bulkRequest = new BulkRequest(); - LOGGER.info("Bulk inserting " + eventQueue.size() + " UBI events"); - - for (final Event event : eventQueue.get()) { - - final IndexRequest indexRequest = new IndexRequest(event.getIndexName()) - .source(event.getEvent(), XContentType.JSON); - - bulkRequest.add(indexRequest); - - } - - eventQueue.clear(); - client.bulk(bulkRequest); - - } - - } - - @Override - public void add(Event event) { - eventQueue.add(event); - } - - /** - * Gets a singleton instance of the manager. - * @param client An OpenSearch {@link Client}. - * @return An instance of {@link OpenSearchEventManager}. - */ - public static OpenSearchEventManager getInstance(Client client) { - if(openSearchEventManager == null) { - openSearchEventManager = new OpenSearchEventManager(client); - } - return openSearchEventManager; - } - -} diff --git a/src/main/java/org/opensearch/ubi/model/QueryRequest.java b/src/main/java/org/opensearch/ubi/model/QueryRequest.java index ba4db66..82e8a21 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryRequest.java +++ b/src/main/java/org/opensearch/ubi/model/QueryRequest.java @@ -13,25 +13,40 @@ */ public class QueryRequest { + private final String storeName; private final long timestamp; private final String queryId; private final String query; private final String userId; private final String sessionId; + private final QueryResponse queryResponse; /** * Creates a query request. + * @param storeName The name of the UBI store to hold this query request. * @param queryId The ID of the query. * @param query The query run by OpenSearch. * @param userId The ID of the user that initiated the query. * @param sessionId The ID of the session under which the query was run. + * @param queryResponse The {@link QueryResponse} for this query request. */ - public QueryRequest(final String queryId, final String query, final String userId, final String sessionId) { + public QueryRequest(final String storeName, final String queryId, final String query, + final String userId, final String sessionId, final QueryResponse queryResponse) { + this.storeName = storeName; this.timestamp = System.currentTimeMillis(); this.queryId = queryId; this.query = query; this.userId = userId; this.sessionId = sessionId; + this.queryResponse = queryResponse; + } + + /** + * Gets the name of the UBI store. + * @return The name of the UBI store. + */ + public String getStoreName() { + return storeName; } /** @@ -74,4 +89,12 @@ public String getSessionId() { return sessionId; } + /** + * Gets the query response for this query request. + * @return The {@link QueryResponse} for this query request. + */ + public QueryResponse getQueryResponse() { + return queryResponse; + } + } diff --git a/src/main/java/org/opensearch/ubi/events/Event.java b/src/main/java/org/opensearch/ubi/model/events/Event.java similarity index 96% rename from src/main/java/org/opensearch/ubi/events/Event.java rename to src/main/java/org/opensearch/ubi/model/events/Event.java index 6895583..5f9779f 100644 --- a/src/main/java/org/opensearch/ubi/events/Event.java +++ b/src/main/java/org/opensearch/ubi/model/events/Event.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.events; +package org.opensearch.ubi.model.events; /** * A client-side event. diff --git a/src/main/java/org/opensearch/ubi/model/events/EventManager.java b/src/main/java/org/opensearch/ubi/model/events/EventManager.java new file mode 100644 index 0000000..827eddb --- /dev/null +++ b/src/main/java/org/opensearch/ubi/model/events/EventManager.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ubi.model.events; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.ubi.model.events.queues.Queue; +import org.opensearch.ubi.model.events.queues.InternalQueue; +import org.opensearch.ubi.model.QueryRequest; + +/** + * Base class for managing client-side events. + */ +public abstract class EventManager { + + @SuppressWarnings("unused") + private final Logger LOGGER = LogManager.getLogger(EventManager.class); + + /** + * The {@link Queue queue} that stores the client-side events. + */ + protected final Queue eventsQueue; + + /** + * The {@link Queue queue} that stores the query responses. + */ + protected final Queue queryRequestQueue; + + /** + * Initialize the base client-side event manager. + */ + public EventManager() { + this.eventsQueue = new InternalQueue<>(); + this.queryRequestQueue = new InternalQueue<>(); + } + + /** + * Process the items on the queue by writing them to persistent storage. + */ + public abstract void process(); + + /** + * Add an event to the queue. + * @param event A client-side {@link Event event} to be persisted. + */ + public abstract void add(Event event); + + /** + * Add a query request to the queue + * @param queryRequest A {@link QueryRequest} to be persisted. + */ + public abstract void add(QueryRequest queryRequest); + +} diff --git a/src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java b/src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java new file mode 100644 index 0000000..9196158 --- /dev/null +++ b/src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java @@ -0,0 +1,119 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.ubi.model.events; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.ubi.model.QueryRequest; +import org.opensearch.ubi.utils.UbiUtils; + +import java.util.HashMap; +import java.util.Map; + +/** + * An event manager that inserts events into an OpenSearch index. + */ +public class OpenSearchEventManager extends EventManager { + + private static final Logger LOGGER = LogManager.getLogger(OpenSearchEventManager.class); + + private final Client client; + private static OpenSearchEventManager openSearchEventManager; + + private OpenSearchEventManager(Client client) { + this.client = client; + } + + @Override + public void process() { + + if(eventsQueue.size() > 0) { + + final BulkRequest eventsBulkRequest = new BulkRequest(); + LOGGER.info("Bulk inserting " + eventsQueue.size() + " UBI events"); + + for (final Event event : eventsQueue.get()) { + + final IndexRequest indexRequest = new IndexRequest(event.getIndexName()) + .source(event.getEvent(), XContentType.JSON); + + eventsBulkRequest.add(indexRequest); + + } + + eventsQueue.clear(); + client.bulk(eventsBulkRequest); + + } + + if(queryRequestQueue.size() > 0) { + + final BulkRequest queryRequestsBulkRequest = new BulkRequest(); + LOGGER.info("Bulk inserting " + queryRequestQueue.size() + " UBI queries"); + + for(final QueryRequest queryRequest : queryRequestQueue.get()) { + + LOGGER.info("Writing query ID {} with response ID {}", + queryRequest.getQueryId(), queryRequest.getQueryResponse().getQueryResponseId()); + + // What will be indexed - adheres to the queries-mapping.json + final Map source = new HashMap<>(); + source.put("timestamp", queryRequest.getTimestamp()); + source.put("query_id", queryRequest.getQueryId()); + source.put("query", queryRequest.getQuery()); + source.put("query_response_id", queryRequest.getQueryResponse().getQueryResponseId()); + source.put("query_response_hit_ids", queryRequest.getQueryResponse().getQueryResponseHitIds()); + source.put("user_id", queryRequest.getUserId()); + source.put("session_id", queryRequest.getSessionId()); + + // Get the name of the queries. + final String queriesIndexName = UbiUtils.getQueriesIndexName(queryRequest.getStoreName()); + + // Build the index request. + final IndexRequest indexRequest = new IndexRequest(queriesIndexName) + .source(source, XContentType.JSON); + + queryRequestsBulkRequest.add(indexRequest); + + } + + queryRequestQueue.clear(); + client.bulk(queryRequestsBulkRequest); + + } + + } + + @Override + public void add(final Event event) { + eventsQueue.add(event); + } + + @Override + public void add(final QueryRequest queryRequest) { + queryRequestQueue.add(queryRequest); + } + + /** + * Gets a singleton instance of the manager. + * @param client An OpenSearch {@link Client}. + * @return An instance of {@link OpenSearchEventManager}. + */ + public static OpenSearchEventManager getInstance(Client client) { + if(openSearchEventManager == null) { + openSearchEventManager = new OpenSearchEventManager(client); + } + return openSearchEventManager; + } + +} diff --git a/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java b/src/main/java/org/opensearch/ubi/model/events/queues/InternalQueue.java similarity index 60% rename from src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java rename to src/main/java/org/opensearch/ubi/model/events/queues/InternalQueue.java index c621beb..05ff217 100644 --- a/src/main/java/org/opensearch/ubi/events/queues/InternalQueue.java +++ b/src/main/java/org/opensearch/ubi/model/events/queues/InternalQueue.java @@ -6,22 +6,20 @@ * compatible open source license. */ -package org.opensearch.ubi.events.queues; - -import org.opensearch.ubi.events.Event; +package org.opensearch.ubi.model.events.queues; import java.util.LinkedList; import java.util.List; /** - * An implementation of {@link EventQueue} that uses an in-memory list. + * An implementation of {@link Queue} that uses an in-memory list. */ -public class InternalQueue implements EventQueue { +public class InternalQueue implements Queue { - private static final List indexRequests = new LinkedList<>(); + private final List indexRequests = new LinkedList<>(); @Override - public void add(Event event) { + public void add(T event) { indexRequests.add(event); } @@ -31,7 +29,7 @@ public void clear() { } @Override - public List get() { + public List get() { return indexRequests; } diff --git a/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java b/src/main/java/org/opensearch/ubi/model/events/queues/Queue.java similarity index 81% rename from src/main/java/org/opensearch/ubi/events/queues/EventQueue.java rename to src/main/java/org/opensearch/ubi/model/events/queues/Queue.java index bbd8541..b14b0c0 100644 --- a/src/main/java/org/opensearch/ubi/events/queues/EventQueue.java +++ b/src/main/java/org/opensearch/ubi/model/events/queues/Queue.java @@ -6,22 +6,22 @@ * compatible open source license. */ -package org.opensearch.ubi.events.queues; +package org.opensearch.ubi.model.events.queues; -import org.opensearch.ubi.events.Event; +import org.opensearch.ubi.model.events.Event; import java.util.List; /** * A queue that stores events prior to being indexed. */ -public interface EventQueue { +public interface Queue { /** * Add an {@link Event event} to the queue. * @param event The {@link Event event} to add to the queue. */ - void add(Event event); + void add(T event); /** * Remove all events from the queue. @@ -32,7 +32,7 @@ public interface EventQueue { * Get a list of items in the queue. * @return A list of items in the queue. */ - List get(); + List get(); /** * Gets the count of items on the queue. diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java index 6886f1f..123b3c3 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -28,8 +28,8 @@ import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; import org.opensearch.ubi.UserBehaviorInsightsPlugin; -import org.opensearch.ubi.events.Event; -import org.opensearch.ubi.events.OpenSearchEventManager; +import org.opensearch.ubi.model.events.Event; +import org.opensearch.ubi.model.events.OpenSearchEventManager; import org.opensearch.ubi.model.HeaderConstants; import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.ubi.utils.UbiUtils; From b0a4b846dd5c8e72e16a972ac4e081bb3da3cee8 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 13:38:23 -0400 Subject: [PATCH 28/37] Resolving merge conflicts. --- build.gradle | 6 ++-- load-test/run.sh | 6 ++-- .../o19s/ubi}/OpenSearchEventManager.java | 8 +++-- .../o19s}/ubi/UserBehaviorInsightsPlugin.java | 27 ++++++--------- .../UserBehaviorInsightsActionFilter.java | 34 ++++++++----------- .../o19s}/ubi/model/HeaderConstants.java | 2 +- .../o19s}/ubi/model/QueryRequest.java | 2 +- .../o19s}/ubi/model/QueryResponse.java | 2 +- .../o19s}/ubi/model/SettingsConstants.java | 2 +- .../o19s}/ubi/model/events/Event.java | 2 +- .../o19s}/ubi/model/events/EventManager.java | 13 +++---- .../model/events/queues/InternalQueue.java | 2 +- .../o19s}/ubi/model/events/queues/Queue.java | 8 ++--- .../rest/UserBehaviorInsightsRestHandler.java | 29 ++++++---------- .../o19s}/ubi/utils/UbiUtils.java | 2 +- ...rBehaviorLoggingClientYamlTestSuiteIT.java | 2 +- 16 files changed, 60 insertions(+), 87 deletions(-) rename src/main/java/{org/opensearch/ubi/model/events => com/o19s/ubi}/OpenSearchEventManager.java (95%) rename src/main/java/{org/opensearch => com/o19s}/ubi/UserBehaviorInsightsPlugin.java (87%) rename src/main/java/{org/opensearch => com/o19s}/ubi/action/UserBehaviorInsightsActionFilter.java (92%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/HeaderConstants.java (96%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/QueryRequest.java (98%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/QueryResponse.java (97%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/SettingsConstants.java (95%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/events/Event.java (96%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/events/EventManager.java (76%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/events/queues/InternalQueue.java (94%) rename src/main/java/{org/opensearch => com/o19s}/ubi/model/events/queues/Queue.java (76%) rename src/main/java/{org/opensearch => com/o19s}/ubi/rest/UserBehaviorInsightsRestHandler.java (93%) rename src/main/java/{org/opensearch => com/o19s}/ubi/utils/UbiUtils.java (98%) rename src/yamlRestTest/java/{org/opensearch => com/o19s/ubi}/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java (96%) diff --git a/build.gradle b/build.gradle index 915ee5c..2eb099d 100644 --- a/build.gradle +++ b/build.gradle @@ -7,12 +7,12 @@ apply plugin: 'maven-publish' opensearchplugin { name 'opensearch-ubi' description 'OpenSearch User Behavior Insights Plugin' - classname 'org.opensearch.ubi.UserBehaviorInsightsPlugin' + classname 'com.o19s.ubi.UserBehaviorInsightsPlugin' licenseFile rootProject.file('LICENSE.txt') noticeFile rootProject.file('NOTICE.txt') } -group = 'org.opensearch' +group = 'com.o19s' version = "${ubiVersion}-os${opensearchVersion}" // disabling some unnecessary validations for this plugin @@ -61,7 +61,7 @@ publishing { pom { name = "opensearch-ubi" description = "Provides User Behavior Insights for OpenSearch" - groupId = "org.opensearch" + groupId = "com.o19s" licenses { license { name = "The Apache License, Version 2.0" diff --git a/load-test/run.sh b/load-test/run.sh index 09851e4..246d0ce 100755 --- a/load-test/run.sh +++ b/load-test/run.sh @@ -10,10 +10,10 @@ curl -X PUT "http://localhost:9200/_plugins/ubi/mystore?index=ecommerce" ../index-chorus-data.sh `realpath ../../chorus-opensearch-edition` # Insert events and queries. -locust -f load-test.py --headless -u 1 -r 1 --run-time 10s --host http://localhost:9200 +locust -f load-test.py --headless -u 1 -r 1 --run-time 30s --host http://localhost:9200 # Let events index. -sleep 5 +sleep 10 # Get count of indexed events. EVENTS=`curl -s http://localhost:9200/.mystore_events/_count | jq .count` @@ -21,4 +21,4 @@ echo "Found $EVENTS events" # Get count of indexed queries. QUERIES=`curl -s http://localhost:9200/.mystore_queries/_count | jq .count` -echo "Found $QUERIES queries" \ No newline at end of file +echo "Found $QUERIES queries" diff --git a/src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java similarity index 95% rename from src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java rename to src/main/java/com/o19s/ubi/OpenSearchEventManager.java index 9196158..279351a 100644 --- a/src/main/java/org/opensearch/ubi/model/events/OpenSearchEventManager.java +++ b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java @@ -6,16 +6,18 @@ * compatible open source license. */ -package org.opensearch.ubi.model.events; +package com.o19s.ubi; +import com.o19s.ubi.model.QueryRequest; +import com.o19s.ubi.model.events.Event; +import com.o19s.ubi.model.events.EventManager; +import com.o19s.ubi.utils.UbiUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.index.IndexRequest; import org.opensearch.client.Client; import org.opensearch.common.xcontent.XContentType; -import org.opensearch.ubi.model.QueryRequest; -import org.opensearch.ubi.utils.UbiUtils; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java similarity index 87% rename from src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java rename to src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java index e82d0ff..580f8cc 100644 --- a/src/main/java/org/opensearch/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java @@ -6,8 +6,12 @@ * compatible open source license. */ -package org.opensearch.ubi; +package com.o19s.ubi; +import com.o19s.ubi.action.UserBehaviorInsightsActionFilter; +import com.o19s.ubi.model.HeaderConstants; +import com.o19s.ubi.model.SettingsConstants; +import com.o19s.ubi.rest.UserBehaviorInsightsRestHandler; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.support.ActionFilter; @@ -15,11 +19,7 @@ import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.settings.IndexScopedSettings; -import org.opensearch.common.settings.Setting; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsFilter; +import org.opensearch.common.settings.*; import org.opensearch.core.common.io.stream.NamedWriteableRegistry; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.env.Environment; @@ -32,19 +32,9 @@ import org.opensearch.rest.RestHeaderDefinition; import org.opensearch.script.ScriptService; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.ubi.action.UserBehaviorInsightsActionFilter; -import org.opensearch.ubi.rest.UserBehaviorInsightsRestHandler; -import org.opensearch.ubi.model.events.OpenSearchEventManager; -import org.opensearch.ubi.model.HeaderConstants; -import org.opensearch.ubi.model.SettingsConstants; import org.opensearch.watcher.ResourceWatcherService; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -59,6 +49,9 @@ public class UserBehaviorInsightsPlugin extends Plugin implements ActionPlugin { private ActionFilter userBehaviorLoggingFilter; + /** + * A map that caches store settings to avoid round-trip calls to the index. + */ public static final Map storeSettings = new HashMap<>(); @Override diff --git a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java similarity index 92% rename from src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java rename to src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java index fb10ecb..fb666f7 100644 --- a/src/main/java/org/opensearch/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java @@ -6,8 +6,14 @@ * compatible open source license. */ -package org.opensearch.ubi.action; - +package com.o19s.ubi.action; + +import com.o19s.ubi.UserBehaviorInsightsPlugin; +import com.o19s.ubi.model.HeaderConstants; +import com.o19s.ubi.model.QueryRequest; +import com.o19s.ubi.model.QueryResponse; +import com.o19s.ubi.model.SettingsConstants; +import com.o19s.ubi.utils.UbiUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.ActionRequest; @@ -25,21 +31,10 @@ import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; -import org.opensearch.ubi.UserBehaviorInsightsPlugin; -import org.opensearch.ubi.model.events.EventManager; -import org.opensearch.ubi.model.events.OpenSearchEventManager; -import org.opensearch.ubi.model.HeaderConstants; -import org.opensearch.ubi.model.QueryRequest; -import org.opensearch.ubi.model.QueryResponse; -import org.opensearch.ubi.model.SettingsConstants; -import org.opensearch.ubi.utils.UbiUtils; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import com.o19s.ubi.model.events.EventManager; +import com.o19s.ubi.OpenSearchEventManager; + +import java.util.*; /** * An implementation of {@link ActionFilter} that passively listens for OpenSearch @@ -104,7 +99,7 @@ public void onResponse(Response response) { final String index = getStoreSettings(storeName, SettingsConstants.INDEX); final String idField = getStoreSettings(storeName, SettingsConstants.ID_FIELD); - LOGGER.info("Using id_field [{}] of index [{}] for UBI query.", idField, index); + LOGGER.debug("Using id_field [{}] of index [{}] for UBI query.", idField, index); // Only consider this search if the index being searched matches the store's index setting. if (Arrays.asList(searchRequest.indices()).contains(index)) { @@ -141,7 +136,6 @@ public void onResponse(Response response) { final QueryRequest queryRequest = new QueryRequest(storeName, queryId, query, userId, sessionId, queryResponse); // Queue this for writing to the UBI store. - LOGGER.info("Queueing query request"); eventManager.add(queryRequest); } catch (Exception ex) { @@ -217,7 +211,7 @@ private String getHeaderValue(final HeaderConstants header, final String default final String value = task.getHeader(header.getHeader()); - if(value == null || value.trim().isEmpty()) { + if(value == null || value.trim().isEmpty() || value.equals("null")) { return defaultValue; } else { return value; diff --git a/src/main/java/org/opensearch/ubi/model/HeaderConstants.java b/src/main/java/com/o19s/ubi/model/HeaderConstants.java similarity index 96% rename from src/main/java/org/opensearch/ubi/model/HeaderConstants.java rename to src/main/java/com/o19s/ubi/model/HeaderConstants.java index 958f3de..e098aed 100644 --- a/src/main/java/org/opensearch/ubi/model/HeaderConstants.java +++ b/src/main/java/com/o19s/ubi/model/HeaderConstants.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; /** * HTTP headers used by the plugin. diff --git a/src/main/java/org/opensearch/ubi/model/QueryRequest.java b/src/main/java/com/o19s/ubi/model/QueryRequest.java similarity index 98% rename from src/main/java/org/opensearch/ubi/model/QueryRequest.java rename to src/main/java/com/o19s/ubi/model/QueryRequest.java index 82e8a21..5b65ac5 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryRequest.java +++ b/src/main/java/com/o19s/ubi/model/QueryRequest.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; /** * A query received by OpenSearch. diff --git a/src/main/java/org/opensearch/ubi/model/QueryResponse.java b/src/main/java/com/o19s/ubi/model/QueryResponse.java similarity index 97% rename from src/main/java/org/opensearch/ubi/model/QueryResponse.java rename to src/main/java/com/o19s/ubi/model/QueryResponse.java index 8392178..7ac1af0 100644 --- a/src/main/java/org/opensearch/ubi/model/QueryResponse.java +++ b/src/main/java/com/o19s/ubi/model/QueryResponse.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; import java.util.List; diff --git a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java b/src/main/java/com/o19s/ubi/model/SettingsConstants.java similarity index 95% rename from src/main/java/org/opensearch/ubi/model/SettingsConstants.java rename to src/main/java/com/o19s/ubi/model/SettingsConstants.java index 1b5f323..469d830 100644 --- a/src/main/java/org/opensearch/ubi/model/SettingsConstants.java +++ b/src/main/java/com/o19s/ubi/model/SettingsConstants.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model; +package com.o19s.ubi.model; /** * Settings constants used by the plugin. diff --git a/src/main/java/org/opensearch/ubi/model/events/Event.java b/src/main/java/com/o19s/ubi/model/events/Event.java similarity index 96% rename from src/main/java/org/opensearch/ubi/model/events/Event.java rename to src/main/java/com/o19s/ubi/model/events/Event.java index 5f9779f..8e4cd83 100644 --- a/src/main/java/org/opensearch/ubi/model/events/Event.java +++ b/src/main/java/com/o19s/ubi/model/events/Event.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model.events; +package com.o19s.ubi.model.events; /** * A client-side event. diff --git a/src/main/java/org/opensearch/ubi/model/events/EventManager.java b/src/main/java/com/o19s/ubi/model/events/EventManager.java similarity index 76% rename from src/main/java/org/opensearch/ubi/model/events/EventManager.java rename to src/main/java/com/o19s/ubi/model/events/EventManager.java index 827eddb..cd987aa 100644 --- a/src/main/java/org/opensearch/ubi/model/events/EventManager.java +++ b/src/main/java/com/o19s/ubi/model/events/EventManager.java @@ -6,22 +6,17 @@ * compatible open source license. */ -package org.opensearch.ubi.model.events; +package com.o19s.ubi.model.events; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.ubi.model.events.queues.Queue; -import org.opensearch.ubi.model.events.queues.InternalQueue; -import org.opensearch.ubi.model.QueryRequest; +import com.o19s.ubi.model.QueryRequest; +import com.o19s.ubi.model.events.queues.InternalQueue; +import com.o19s.ubi.model.events.queues.Queue; /** * Base class for managing client-side events. */ public abstract class EventManager { - @SuppressWarnings("unused") - private final Logger LOGGER = LogManager.getLogger(EventManager.class); - /** * The {@link Queue queue} that stores the client-side events. */ diff --git a/src/main/java/org/opensearch/ubi/model/events/queues/InternalQueue.java b/src/main/java/com/o19s/ubi/model/events/queues/InternalQueue.java similarity index 94% rename from src/main/java/org/opensearch/ubi/model/events/queues/InternalQueue.java rename to src/main/java/com/o19s/ubi/model/events/queues/InternalQueue.java index 05ff217..7403ece 100644 --- a/src/main/java/org/opensearch/ubi/model/events/queues/InternalQueue.java +++ b/src/main/java/com/o19s/ubi/model/events/queues/InternalQueue.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model.events.queues; +package com.o19s.ubi.model.events.queues; import java.util.LinkedList; import java.util.List; diff --git a/src/main/java/org/opensearch/ubi/model/events/queues/Queue.java b/src/main/java/com/o19s/ubi/model/events/queues/Queue.java similarity index 76% rename from src/main/java/org/opensearch/ubi/model/events/queues/Queue.java rename to src/main/java/com/o19s/ubi/model/events/queues/Queue.java index b14b0c0..78c526b 100644 --- a/src/main/java/org/opensearch/ubi/model/events/queues/Queue.java +++ b/src/main/java/com/o19s/ubi/model/events/queues/Queue.java @@ -6,9 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.model.events.queues; - -import org.opensearch.ubi.model.events.Event; +package com.o19s.ubi.model.events.queues; import java.util.List; @@ -18,8 +16,8 @@ public interface Queue { /** - * Add an {@link Event event} to the queue. - * @param event The {@link Event event} to add to the queue. + * Add an object to the queue. + * @param event The object to add to the queue. */ void add(T event); diff --git a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java similarity index 93% rename from src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java rename to src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java index 123b3c3..2d61ce6 100644 --- a/src/main/java/org/opensearch/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -6,12 +6,16 @@ * compatible open source license. */ -package org.opensearch.ubi.rest; +package com.o19s.ubi.rest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.o19s.ubi.UserBehaviorInsightsPlugin; +import com.o19s.ubi.model.HeaderConstants; +import com.o19s.ubi.model.SettingsConstants; +import com.o19s.ubi.utils.UbiUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.indices.create.CreateIndexRequest; @@ -27,26 +31,13 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; -import org.opensearch.ubi.UserBehaviorInsightsPlugin; -import org.opensearch.ubi.model.events.Event; -import org.opensearch.ubi.model.events.OpenSearchEventManager; -import org.opensearch.ubi.model.HeaderConstants; -import org.opensearch.ubi.model.SettingsConstants; -import org.opensearch.ubi.utils.UbiUtils; +import com.o19s.ubi.model.events.Event; +import com.o19s.ubi.OpenSearchEventManager; import java.io.IOException; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; - -import static org.opensearch.rest.RestRequest.Method.DELETE; -import static org.opensearch.rest.RestRequest.Method.GET; -import static org.opensearch.rest.RestRequest.Method.POST; -import static org.opensearch.rest.RestRequest.Method.PUT; -import static org.opensearch.rest.RestRequest.Method.TRACE; +import java.util.*; + +import static org.opensearch.rest.RestRequest.Method.*; /** * The REST handler for User Behavior Insights. The handler provides the diff --git a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java b/src/main/java/com/o19s/ubi/utils/UbiUtils.java similarity index 98% rename from src/main/java/org/opensearch/ubi/utils/UbiUtils.java rename to src/main/java/com/o19s/ubi/utils/UbiUtils.java index fd034b7..8b4620c 100644 --- a/src/main/java/org/opensearch/ubi/utils/UbiUtils.java +++ b/src/main/java/com/o19s/ubi/utils/UbiUtils.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.ubi.utils; +package com.o19s.ubi.utils; import org.opensearch.common.util.io.Streams; diff --git a/src/yamlRestTest/java/org/opensearch/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java b/src/yamlRestTest/java/com/o19s/ubi/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java similarity index 96% rename from src/yamlRestTest/java/org/opensearch/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java rename to src/yamlRestTest/java/com/o19s/ubi/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java index cd9165f..ab75960 100644 --- a/src/yamlRestTest/java/org/opensearch/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java +++ b/src/yamlRestTest/java/com/o19s/ubi/rest/action/UserBehaviorLoggingClientYamlTestSuiteIT.java @@ -5,7 +5,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.rest.action; +package com.o19s.ubi.rest.action; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; From deb5822785d5be5eceaf8ff4c9077cfdaa70ca29 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 13:56:52 -0400 Subject: [PATCH 29/37] Increasing run time duration. --- load-test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/load-test/run.sh b/load-test/run.sh index edbb59c..246d0ce 100755 --- a/load-test/run.sh +++ b/load-test/run.sh @@ -10,7 +10,7 @@ curl -X PUT "http://localhost:9200/_plugins/ubi/mystore?index=ecommerce" ../index-chorus-data.sh `realpath ../../chorus-opensearch-edition` # Insert events and queries. -locust -f load-test.py --headless -u 1 -r 1 --run-time 10s --host http://localhost:9200 +locust -f load-test.py --headless -u 1 -r 1 --run-time 30s --host http://localhost:9200 # Let events index. sleep 10 From 3b4e43a3bf33e81d6fbe892373b2ecaa46f81839 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 18:23:52 -0400 Subject: [PATCH 30/37] Improving the getting of indexes. --- .../rest/UserBehaviorInsightsRestHandler.java | 102 ++++++++++++------ .../test/_plugins.ubi/10_manage.yml | 32 +++--- 2 files changed, 88 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java index 6a35b19..fd572a2 100644 --- a/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.o19s.ubi.OpenSearchEventManager; import com.o19s.ubi.UserBehaviorInsightsPlugin; import com.o19s.ubi.events.Event; import com.o19s.ubi.model.HeaderConstants; @@ -32,13 +33,21 @@ import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest; - -import com.o19s.ubi.OpenSearchEventManager; +import org.opensearch.rest.action.RestActionListener; import java.io.IOException; -import java.util.*; - -import static org.opensearch.rest.RestRequest.Method.*; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; + +import static org.opensearch.rest.RestRequest.Method.DELETE; +import static org.opensearch.rest.RestRequest.Method.GET; +import static org.opensearch.rest.RestRequest.Method.POST; +import static org.opensearch.rest.RestRequest.Method.PUT; +import static org.opensearch.rest.RestRequest.Method.TRACE; /** * The REST handler for User Behavior Insights. The handler provides the @@ -70,16 +79,24 @@ public List routes() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient nodeClient) throws IOException { - final String storeName = restRequest.param("store"); - if (restRequest.method() == PUT) { + + final String storeName = restRequest.param("store"); final String index = restRequest.param("index"); final String idField = restRequest.param("id_field"); + return create(nodeClient, storeName, index, idField); + } else if(restRequest.method() == DELETE) { + + final String storeName = restRequest.param("store"); return delete(nodeClient, storeName); + } else if(restRequest.method() == POST) { + + final String storeName = restRequest.param("store"); return post(nodeClient, storeName, restRequest); + } else if(restRequest.method() == TRACE) { return trace(nodeClient, restRequest); } @@ -127,14 +144,27 @@ private RestChannelConsumer get(final NodeClient nodeClient) { return (channel) -> { final GetIndexRequest getIndexRequest = new GetIndexRequest(); - final GetIndexResponse getIndexResponse = nodeClient.admin().indices().getIndex(getIndexRequest).actionGet(); - final Set stores = getStoreNames(getIndexResponse.indices()); - final XContentBuilder builder = XContentType.JSON.contentBuilder(); - builder.startObject().field("stores", stores); - builder.endObject(); + nodeClient.admin().indices().getIndex(getIndexRequest, new RestActionListener<>(channel) { + @Override + public void processResponse(final GetIndexResponse getIndexResponse) throws Exception { + + final Set stores = new HashSet<>(); + + for(final String index : getIndexResponse.indices()) { + if(index.startsWith(".") && index.endsWith("_events")) { + stores.add(index.substring(1, index.length() - 7)); + } + } + + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("stores", stores); + builder.endObject(); - channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + + } + }); }; @@ -179,32 +209,44 @@ private RestChannelConsumer create(final NodeClient nodeClient, final String sto private RestChannelConsumer post(final NodeClient nodeClient, final String storeName, final RestRequest restRequest) throws IOException { - try { + if(storeName == null || storeName.isEmpty()) { + + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("error", "a store name is required"); + builder.endObject(); + + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); + + } else { + + try { + + final String eventJson = restRequest.content().utf8ToString(); + final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - final String eventJson = restRequest.content().utf8ToString(); - final String eventJsonWithTimestamp = setEventTimestamp(eventJson); + LOGGER.info("Indexing UBI event into store {}", storeName); + final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); - LOGGER.info("Indexing UBI event into store {}", storeName); - final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); + final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); + OpenSearchEventManager.getInstance(nodeClient).add(event); - final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); - OpenSearchEventManager.getInstance(nodeClient).add(event); + } catch (JsonProcessingException ex) { + LOGGER.error("Unable to get/set timestamp on UBI event.", ex); - } catch (JsonProcessingException ex) { - LOGGER.error("Unable to get/set timestamp on UBI event.", ex); + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("error", "unable to set event timestamp"); + builder.endObject(); + + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); + } final XContentBuilder builder = XContentType.JSON.contentBuilder(); - builder.startObject().field("error", "unable to set event timestamp"); + builder.startObject().field("status", "received"); builder.endObject(); - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); - } + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); - final XContentBuilder builder = XContentType.JSON.contentBuilder(); - builder.startObject().field("status", "received"); - builder.endObject(); - - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + } } diff --git a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml index 12fa708..46c7882 100644 --- a/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml +++ b/src/yamlRestTest/resources/rest-api-spec/test/_plugins.ubi/10_manage.yml @@ -9,19 +9,19 @@ indices.exists: index: .mystore_events -# - match: { created } -# -# - do: -# ubi.get_stores: {} -# -# - match: { mystore } -# -# - do: -# ubi.delete_store: -# store: mystore -# -# - do: -# indices.exists: -# index: .mystore_events -# -# - is_false: '' + - match: { created } + + - do: + ubi.get_stores: {} + + - match: { mystore } + + - do: + ubi.delete_store: + store: mystore + + - do: + indices.exists: + index: .mystore_events + + - is_false: '' From a9165f44fb40d595abf38b077768502bb4e6078a Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 18:31:27 -0400 Subject: [PATCH 31/37] Removing store param check. --- .../rest/UserBehaviorInsightsRestHandler.java | 46 +++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java index fd572a2..7de29ff 100644 --- a/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -209,45 +209,33 @@ private RestChannelConsumer create(final NodeClient nodeClient, final String sto private RestChannelConsumer post(final NodeClient nodeClient, final String storeName, final RestRequest restRequest) throws IOException { - if(storeName == null || storeName.isEmpty()) { + try { - final XContentBuilder builder = XContentType.JSON.contentBuilder(); - builder.startObject().field("error", "a store name is required"); - builder.endObject(); - - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); - - } else { + final String eventJson = restRequest.content().utf8ToString(); + final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - try { + LOGGER.info("Indexing UBI event into store {}", storeName); + final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); - final String eventJson = restRequest.content().utf8ToString(); - final String eventJsonWithTimestamp = setEventTimestamp(eventJson); + final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); + OpenSearchEventManager.getInstance(nodeClient).add(event); - LOGGER.info("Indexing UBI event into store {}", storeName); - final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); - - final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); - OpenSearchEventManager.getInstance(nodeClient).add(event); - - } catch (JsonProcessingException ex) { - LOGGER.error("Unable to get/set timestamp on UBI event.", ex); - - final XContentBuilder builder = XContentType.JSON.contentBuilder(); - builder.startObject().field("error", "unable to set event timestamp"); - builder.endObject(); - - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); - } + } catch (JsonProcessingException ex) { + LOGGER.error("Unable to get/set timestamp on UBI event.", ex); final XContentBuilder builder = XContentType.JSON.contentBuilder(); - builder.startObject().field("status", "received"); + builder.startObject().field("error", "unable to set event timestamp"); builder.endObject(); - return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); - + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, builder)); } + final XContentBuilder builder = XContentType.JSON.contentBuilder(); + builder.startObject().field("status", "received"); + builder.endObject(); + + return (channel) -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, builder)); + } private RestChannelConsumer delete(final NodeClient nodeClient, final String storeName) throws IOException { From d004164e60e3492da20aa9845607ad4d3c346b6a Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Mon, 11 Mar 2024 19:30:23 -0400 Subject: [PATCH 32/37] Changing scheduler to fixed delay and separating events and queries tasks. --- load-test/run.sh | 2 +- .../com/o19s/ubi/OpenSearchEventManager.java | 8 +++++++- .../o19s/ubi/UserBehaviorInsightsPlugin.java | 13 ++++++++----- .../UserBehaviorInsightsActionFilter.java | 18 +++++++++++------- .../java/com/o19s/ubi/events/EventManager.java | 9 +++++++-- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/load-test/run.sh b/load-test/run.sh index 246d0ce..288d621 100755 --- a/load-test/run.sh +++ b/load-test/run.sh @@ -10,7 +10,7 @@ curl -X PUT "http://localhost:9200/_plugins/ubi/mystore?index=ecommerce" ../index-chorus-data.sh `realpath ../../chorus-opensearch-edition` # Insert events and queries. -locust -f load-test.py --headless -u 1 -r 1 --run-time 30s --host http://localhost:9200 +locust -f load-test.py --headless -u 1 -r 1 --run-time 600s --host http://localhost:9200 # Let events index. sleep 10 diff --git a/src/main/java/com/o19s/ubi/OpenSearchEventManager.java b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java index 0fb06e0..6428530 100644 --- a/src/main/java/com/o19s/ubi/OpenSearchEventManager.java +++ b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java @@ -37,7 +37,7 @@ private OpenSearchEventManager(Client client) { } @Override - public void process() { + public void processEvents() { if(eventsQueue.size() > 0) { @@ -58,6 +58,11 @@ public void process() { } + } + + @Override + public void processQueries() { + if(queryRequestsQueue.size() > 0) { final BulkRequest queryRequestsBulkRequest = new BulkRequest(); @@ -104,6 +109,7 @@ public void add(final Event event) { @Override public void add(final QueryRequest queryRequest) { queryRequestsQueue.add(queryRequest); + LOGGER.info("Size of queryRequestsQueue " + queryRequestsQueue.size()); } /** diff --git a/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java index 580f8cc..11cf127 100644 --- a/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java @@ -120,13 +120,16 @@ public Collection createComponents( Supplier repositoriesServiceSupplier ) { - // TODO Only start this if an OpenSearch store is already initialized. - // Otherwise, start it when a store is initialized. + // TODO: Allow the parameters of the scheduled tasks to be configurable. LOGGER.info("Creating UBI scheduled task to persist events."); - // TODO: Allow these time parameters to be configurable. - threadPool.scheduler().scheduleAtFixedRate(() -> { - OpenSearchEventManager.getInstance(client).process(); + threadPool.scheduler().scheduleWithFixedDelay(() -> { + OpenSearchEventManager.getInstance(client).processEvents(); + }, 0, 2000, TimeUnit.MILLISECONDS); + + LOGGER.info("Creating UBI scheduled task to persist queries."); + threadPool.scheduler().scheduleWithFixedDelay(() -> { + OpenSearchEventManager.getInstance(client).processQueries(); }, 0, 2000, TimeUnit.MILLISECONDS); // Initialize the action filter. diff --git a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java index 425fd05..6758d30 100644 --- a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java @@ -134,6 +134,7 @@ public void onResponse(Response response) { final QueryRequest queryRequest = new QueryRequest(storeName, queryId, query, userId, sessionId, queryResponse); // Queue this for writing to the UBI store. + LOGGER.trace("Queueing queryRequest for write"); eventManager.add(queryRequest); // Add the query_id to the response headers. @@ -142,18 +143,21 @@ public void onResponse(Response response) { final long elapsedTime = System.currentTimeMillis() - startTime; LOGGER.info("UBI search request filter took {} ms", elapsedTime); - } + LOGGER.debug("Setting and exposing query_id {}", queryId); + //HACK: this should be set in the OpenSearch config (to send to the client code just once), + // and not on every single search response, + // but that server setting doesn't appear to be exposed. + threadPool.getThreadContext().addResponseHeader("Access-Control-Expose-Headers", "query_id"); - LOGGER.info("Setting and exposing query_id {}", queryId); - //HACK: this should be set in the OpenSearch config (to send to the client code just once), - // and not on every single search response, - // but that server setting doesn't appear to be exposed. - threadPool.getThreadContext().addResponseHeader("Access-Control-Expose-Headers", "query_id"); - threadPool.getThreadContext().addResponseHeader("query_id", queryId); + } else { + LOGGER.trace("Discarding query for UBI due to index name mismatch."); + } final long elapsedTime = System.currentTimeMillis() - startTime; LOGGER.info("UBI search request filter took {} ms", elapsedTime); + } else { + LOGGER.trace("Discarding query for UBI due to missing store name."); } } diff --git a/src/main/java/com/o19s/ubi/events/EventManager.java b/src/main/java/com/o19s/ubi/events/EventManager.java index 2ce64d7..4c7fdd8 100644 --- a/src/main/java/com/o19s/ubi/events/EventManager.java +++ b/src/main/java/com/o19s/ubi/events/EventManager.java @@ -41,9 +41,14 @@ public EventManager() { } /** - * Process the items on the queue by writing them to persistent storage. + * Process the events on the queue by writing them to persistent storage. */ - public abstract void process(); + public abstract void processEvents(); + + /** + * Process the queries on the queue by writing them to persistent storage. + */ + public abstract void processQueries(); /** * Add an event to the queue. From 5466a20e97c3ea378b74f29c8e7f4891d74fb647 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 12 Mar 2024 11:42:36 -0400 Subject: [PATCH 33/37] Removing and lowering logging level. --- src/main/java/com/o19s/ubi/OpenSearchEventManager.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/o19s/ubi/OpenSearchEventManager.java b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java index 6428530..fc3cb65 100644 --- a/src/main/java/com/o19s/ubi/OpenSearchEventManager.java +++ b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java @@ -42,7 +42,7 @@ public void processEvents() { if(eventsQueue.size() > 0) { final BulkRequest eventsBulkRequest = new BulkRequest(); - LOGGER.info("Bulk inserting " + eventsQueue.size() + " UBI events"); + LOGGER.debug("Bulk inserting " + eventsQueue.size() + " UBI events"); for (final Event event : eventsQueue.get()) { @@ -66,11 +66,11 @@ public void processQueries() { if(queryRequestsQueue.size() > 0) { final BulkRequest queryRequestsBulkRequest = new BulkRequest(); - LOGGER.info("Bulk inserting " + queryRequestsQueue.size() + " UBI queries"); + LOGGER.debug("Bulk inserting " + queryRequestsQueue.size() + " UBI queries"); for(final QueryRequest queryRequest : queryRequestsQueue.get()) { - LOGGER.info("Writing query ID {} with response ID {}", + LOGGER.debug("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryRequest.getQueryResponse().getQueryResponseId()); // What will be indexed - adheres to the queries-mapping.json @@ -109,7 +109,6 @@ public void add(final Event event) { @Override public void add(final QueryRequest queryRequest) { queryRequestsQueue.add(queryRequest); - LOGGER.info("Size of queryRequestsQueue " + queryRequestsQueue.size()); } /** From 34ea169261c27370d05f1a0b654164dcf9c338bf Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 12 Mar 2024 13:42:00 -0400 Subject: [PATCH 34/37] Changing to a threadsafe queue. --- load-test/run.sh | 4 +- .../com/o19s/ubi/OpenSearchEventManager.java | 126 ------------------ .../o19s/ubi/UserBehaviorInsightsPlugin.java | 9 +- .../UserBehaviorInsightsActionFilter.java | 17 +-- .../DataManager.java} | 28 ++-- .../o19s/ubi/data/OpenSearchDataManager.java | 122 +++++++++++++++++ .../o19s/ubi/events/queues/InternalQueue.java | 43 ------ .../com/o19s/ubi/events/queues/Queue.java | 41 ------ .../com/o19s/ubi/{events => model}/Event.java | 2 +- .../rest/UserBehaviorInsightsRestHandler.java | 8 +- 10 files changed, 155 insertions(+), 245 deletions(-) delete mode 100644 src/main/java/com/o19s/ubi/OpenSearchEventManager.java rename src/main/java/com/o19s/ubi/{events/EventManager.java => data/DataManager.java} (61%) create mode 100644 src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java delete mode 100644 src/main/java/com/o19s/ubi/events/queues/InternalQueue.java delete mode 100644 src/main/java/com/o19s/ubi/events/queues/Queue.java rename src/main/java/com/o19s/ubi/{events => model}/Event.java (97%) diff --git a/load-test/run.sh b/load-test/run.sh index 288d621..0f2df5a 100755 --- a/load-test/run.sh +++ b/load-test/run.sh @@ -10,10 +10,10 @@ curl -X PUT "http://localhost:9200/_plugins/ubi/mystore?index=ecommerce" ../index-chorus-data.sh `realpath ../../chorus-opensearch-edition` # Insert events and queries. -locust -f load-test.py --headless -u 1 -r 1 --run-time 600s --host http://localhost:9200 +locust -f load-test.py --headless -u 50 -r 10 --run-time 300s --host http://localhost:9200 # Let events index. -sleep 10 +sleep 30 # Get count of indexed events. EVENTS=`curl -s http://localhost:9200/.mystore_events/_count | jq .count` diff --git a/src/main/java/com/o19s/ubi/OpenSearchEventManager.java b/src/main/java/com/o19s/ubi/OpenSearchEventManager.java deleted file mode 100644 index fc3cb65..0000000 --- a/src/main/java/com/o19s/ubi/OpenSearchEventManager.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package com.o19s.ubi; - -import com.o19s.ubi.events.Event; -import com.o19s.ubi.events.EventManager; -import com.o19s.ubi.model.QueryRequest; -import com.o19s.ubi.utils.UbiUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.action.bulk.BulkRequest; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.client.Client; -import org.opensearch.common.xcontent.XContentType; - -import java.util.HashMap; -import java.util.Map; - -/** - * An event manager that inserts events into an OpenSearch index. - */ -public class OpenSearchEventManager extends EventManager { - - private static final Logger LOGGER = LogManager.getLogger(OpenSearchEventManager.class); - - private final Client client; - private static OpenSearchEventManager openSearchEventManager; - - private OpenSearchEventManager(Client client) { - this.client = client; - } - - @Override - public void processEvents() { - - if(eventsQueue.size() > 0) { - - final BulkRequest eventsBulkRequest = new BulkRequest(); - LOGGER.debug("Bulk inserting " + eventsQueue.size() + " UBI events"); - - for (final Event event : eventsQueue.get()) { - - final IndexRequest indexRequest = new IndexRequest(event.getIndexName()) - .source(event.getEvent(), XContentType.JSON); - - eventsBulkRequest.add(indexRequest); - - } - - eventsQueue.clear(); - client.bulk(eventsBulkRequest); - - } - - } - - @Override - public void processQueries() { - - if(queryRequestsQueue.size() > 0) { - - final BulkRequest queryRequestsBulkRequest = new BulkRequest(); - LOGGER.debug("Bulk inserting " + queryRequestsQueue.size() + " UBI queries"); - - for(final QueryRequest queryRequest : queryRequestsQueue.get()) { - - LOGGER.debug("Writing query ID {} with response ID {}", - queryRequest.getQueryId(), queryRequest.getQueryResponse().getQueryResponseId()); - - // What will be indexed - adheres to the queries-mapping.json - final Map source = new HashMap<>(); - source.put("timestamp", queryRequest.getTimestamp()); - source.put("query_id", queryRequest.getQueryId()); - source.put("query", queryRequest.getQuery()); - source.put("query_response_id", queryRequest.getQueryResponse().getQueryResponseId()); - source.put("query_response_hit_ids", queryRequest.getQueryResponse().getQueryResponseHitIds()); - source.put("user_id", queryRequest.getUserId()); - source.put("session_id", queryRequest.getSessionId()); - - // Get the name of the queries. - final String queriesIndexName = UbiUtils.getQueriesIndexName(queryRequest.getStoreName()); - - // Build the index request. - final IndexRequest indexRequest = new IndexRequest(queriesIndexName) - .source(source, XContentType.JSON); - - queryRequestsBulkRequest.add(indexRequest); - - } - - queryRequestsQueue.clear(); - client.bulk(queryRequestsBulkRequest); - - } - - } - - @Override - public void add(final Event event) { - eventsQueue.add(event); - } - - @Override - public void add(final QueryRequest queryRequest) { - queryRequestsQueue.add(queryRequest); - } - - /** - * Gets a singleton instance of the manager. - * @param client An OpenSearch {@link Client}. - * @return An instance of {@link OpenSearchEventManager}. - */ - public static OpenSearchEventManager getInstance(Client client) { - if(openSearchEventManager == null) { - openSearchEventManager = new OpenSearchEventManager(client); - } - return openSearchEventManager; - } - -} diff --git a/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java index 11cf127..3cd8ff1 100644 --- a/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java @@ -9,6 +9,7 @@ package com.o19s.ubi; import com.o19s.ubi.action.UserBehaviorInsightsActionFilter; +import com.o19s.ubi.data.OpenSearchDataManager; import com.o19s.ubi.model.HeaderConstants; import com.o19s.ubi.model.SettingsConstants; import com.o19s.ubi.rest.UserBehaviorInsightsRestHandler; @@ -122,14 +123,14 @@ public Collection createComponents( // TODO: Allow the parameters of the scheduled tasks to be configurable. - LOGGER.info("Creating UBI scheduled task to persist events."); + LOGGER.info("Starting UBI scheduled task to persist events."); threadPool.scheduler().scheduleWithFixedDelay(() -> { - OpenSearchEventManager.getInstance(client).processEvents(); + OpenSearchDataManager.getInstance(client).processEvents(); }, 0, 2000, TimeUnit.MILLISECONDS); - LOGGER.info("Creating UBI scheduled task to persist queries."); + LOGGER.info("Starting UBI scheduled task to persist queries."); threadPool.scheduler().scheduleWithFixedDelay(() -> { - OpenSearchEventManager.getInstance(client).processQueries(); + OpenSearchDataManager.getInstance(client).processQueries(); }, 0, 2000, TimeUnit.MILLISECONDS); // Initialize the action filter. diff --git a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java index 6758d30..4e92cd7 100644 --- a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java @@ -9,7 +9,7 @@ package com.o19s.ubi.action; import com.o19s.ubi.UserBehaviorInsightsPlugin; -import com.o19s.ubi.events.EventManager; +import com.o19s.ubi.data.DataManager; import com.o19s.ubi.model.HeaderConstants; import com.o19s.ubi.model.QueryRequest; import com.o19s.ubi.model.QueryResponse; @@ -32,7 +32,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; -import com.o19s.ubi.OpenSearchEventManager; +import com.o19s.ubi.data.OpenSearchDataManager; import java.util.*; @@ -46,7 +46,7 @@ public class UserBehaviorInsightsActionFilter implements ActionFilter { private final Client client; private final ThreadPool threadPool; - private final EventManager eventManager; + private final DataManager dataManager; /** * Creates a new filter. @@ -56,7 +56,7 @@ public class UserBehaviorInsightsActionFilter implements ActionFilter { public UserBehaviorInsightsActionFilter(Client client, ThreadPool threadPool) { this.client = client; this.threadPool = threadPool; - this.eventManager = OpenSearchEventManager.getInstance(client); + this.dataManager = OpenSearchDataManager.getInstance(client); } @Override @@ -134,16 +134,11 @@ public void onResponse(Response response) { final QueryRequest queryRequest = new QueryRequest(storeName, queryId, query, userId, sessionId, queryResponse); // Queue this for writing to the UBI store. - LOGGER.trace("Queueing queryRequest for write"); - eventManager.add(queryRequest); + dataManager.add(queryRequest); // Add the query_id to the response headers. threadPool.getThreadContext().addResponseHeader("query_id", queryId); - final long elapsedTime = System.currentTimeMillis() - startTime; - LOGGER.info("UBI search request filter took {} ms", elapsedTime); - - LOGGER.debug("Setting and exposing query_id {}", queryId); //HACK: this should be set in the OpenSearch config (to send to the client code just once), // and not on every single search response, // but that server setting doesn't appear to be exposed. @@ -154,7 +149,7 @@ public void onResponse(Response response) { } final long elapsedTime = System.currentTimeMillis() - startTime; - LOGGER.info("UBI search request filter took {} ms", elapsedTime); + LOGGER.trace("UBI search request filter took {} ms", elapsedTime); } else { LOGGER.trace("Discarding query for UBI due to missing store name."); diff --git a/src/main/java/com/o19s/ubi/events/EventManager.java b/src/main/java/com/o19s/ubi/data/DataManager.java similarity index 61% rename from src/main/java/com/o19s/ubi/events/EventManager.java rename to src/main/java/com/o19s/ubi/data/DataManager.java index 4c7fdd8..e381d98 100644 --- a/src/main/java/com/o19s/ubi/events/EventManager.java +++ b/src/main/java/com/o19s/ubi/data/DataManager.java @@ -6,38 +6,40 @@ * compatible open source license. */ -package com.o19s.ubi.events; +package com.o19s.ubi.data; +import com.o19s.ubi.model.Event; import com.o19s.ubi.model.QueryRequest; -import com.o19s.ubi.model.events.queues.Queue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.o19s.ubi.events.queues.InternalQueue; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; /** - * Base class for managing client-side events. + * Base class for managing client-side events and queries. */ -public abstract class EventManager { +public abstract class DataManager { @SuppressWarnings("unused") - private final Logger LOGGER = LogManager.getLogger(EventManager.class); + private final Logger LOGGER = LogManager.getLogger(DataManager.class); /** - * The {@link Queue queue} that stores the client-side events. + * The queue that stores the client-side events. */ - protected final Queue eventsQueue; + protected final BlockingQueue eventsQueue; /** - * The {@link Queue queue} that stores the query reqeusts. + * The queue that stores the query requests. */ - protected final Queue queryRequestsQueue; + protected final BlockingQueue queryRequestsQueue; /** * Initialize the base client-side event manager. */ - public EventManager() { - this.eventsQueue = new InternalQueue<>(); - this.queryRequestsQueue = new InternalQueue<>(); + public DataManager() { + this.eventsQueue = new LinkedBlockingDeque<>(); + this.queryRequestsQueue = new LinkedBlockingDeque<>(); } /** diff --git a/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java b/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java new file mode 100644 index 0000000..718d069 --- /dev/null +++ b/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java @@ -0,0 +1,122 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package com.o19s.ubi.data; + +import com.o19s.ubi.model.Event; +import com.o19s.ubi.model.QueryRequest; +import com.o19s.ubi.utils.UbiUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.action.bulk.BulkRequest; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.client.Client; +import org.opensearch.common.xcontent.XContentType; + +import java.util.HashMap; +import java.util.Map; + +/** + * An event manager that inserts events into an OpenSearch index. + */ +public class OpenSearchDataManager extends DataManager { + + private static final Logger LOGGER = LogManager.getLogger(OpenSearchDataManager.class); + + private final Client client; + private final int max_items_batch = 1000; + private static OpenSearchDataManager openSearchEventManager; + + private OpenSearchDataManager(Client client) { + this.client = client; + } + + @Override + public void processEvents() { + + final BulkRequest eventsBulkRequest = new BulkRequest(); + + while(eventsQueue.peek() != null && eventsBulkRequest.numberOfActions() <= max_items_batch) { + + final Event event = eventsQueue.remove(); + + final IndexRequest indexRequest = new IndexRequest(event.getIndexName()) + .source(event.getEvent(), XContentType.JSON); + + eventsBulkRequest.add(indexRequest); + + } + + if(eventsBulkRequest.numberOfActions() > 0) { + client.bulk(eventsBulkRequest); + } + + } + + @Override + public void processQueries() { + + final BulkRequest queryRequestsBulkRequest = new BulkRequest(); + + while(queryRequestsQueue.peek() != null && queryRequestsBulkRequest.numberOfActions() <= max_items_batch) { + + final QueryRequest queryRequest = queryRequestsQueue.remove(); + + LOGGER.trace("Writing query ID {} with response ID {}", + queryRequest.getQueryId(), queryRequest.getQueryResponse().getQueryResponseId()); + + // What will be indexed - adheres to the queries-mapping.json + final Map source = new HashMap<>(); + source.put("timestamp", queryRequest.getTimestamp()); + source.put("query_id", queryRequest.getQueryId()); + source.put("query", queryRequest.getQuery()); + source.put("query_response_id", queryRequest.getQueryResponse().getQueryResponseId()); + source.put("query_response_hit_ids", queryRequest.getQueryResponse().getQueryResponseHitIds()); + source.put("user_id", queryRequest.getUserId()); + source.put("session_id", queryRequest.getSessionId()); + + // Get the name of the queries. + final String queriesIndexName = UbiUtils.getQueriesIndexName(queryRequest.getStoreName()); + + // Build the index request. + final IndexRequest indexRequest = new IndexRequest(queriesIndexName) + .source(source, XContentType.JSON); + + queryRequestsBulkRequest.add(indexRequest); + + } + + if(queryRequestsBulkRequest.numberOfActions() > 0) { + client.bulk(queryRequestsBulkRequest); + } + + } + + @Override + public void add(final Event event) { + eventsQueue.add(event); + } + + @Override + public void add(final QueryRequest queryRequest) { + queryRequestsQueue.add(queryRequest); + } + + /** + * Gets a singleton instance of the manager. + * @param client An OpenSearch {@link Client}. + * @return An instance of {@link OpenSearchDataManager}. + */ + public static OpenSearchDataManager getInstance(Client client) { + if(openSearchEventManager == null) { + openSearchEventManager = new OpenSearchDataManager(client); + } + return openSearchEventManager; + } + +} diff --git a/src/main/java/com/o19s/ubi/events/queues/InternalQueue.java b/src/main/java/com/o19s/ubi/events/queues/InternalQueue.java deleted file mode 100644 index 6a50fd4..0000000 --- a/src/main/java/com/o19s/ubi/events/queues/InternalQueue.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package com.o19s.ubi.events.queues; - -import com.o19s.ubi.model.events.queues.Queue; - -import java.util.LinkedList; -import java.util.List; - -/** - * An implementation of {@link Queue} that uses an in-memory list. - */ -public class InternalQueue implements Queue { - - private final List indexRequests = new LinkedList<>(); - - @Override - public void add(T event) { - indexRequests.add(event); - } - - @Override - public void clear() { - indexRequests.clear(); - } - - @Override - public List get() { - return indexRequests; - } - - @Override - public int size() { - return indexRequests.size(); - } - -} diff --git a/src/main/java/com/o19s/ubi/events/queues/Queue.java b/src/main/java/com/o19s/ubi/events/queues/Queue.java deleted file mode 100644 index 78c526b..0000000 --- a/src/main/java/com/o19s/ubi/events/queues/Queue.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package com.o19s.ubi.model.events.queues; - -import java.util.List; - -/** - * A queue that stores events prior to being indexed. - */ -public interface Queue { - - /** - * Add an object to the queue. - * @param event The object to add to the queue. - */ - void add(T event); - - /** - * Remove all events from the queue. - */ - void clear(); - - /** - * Get a list of items in the queue. - * @return A list of items in the queue. - */ - List get(); - - /** - * Gets the count of items on the queue. - * @return The count of items on the queue. - */ - int size(); - -} diff --git a/src/main/java/com/o19s/ubi/events/Event.java b/src/main/java/com/o19s/ubi/model/Event.java similarity index 97% rename from src/main/java/com/o19s/ubi/events/Event.java rename to src/main/java/com/o19s/ubi/model/Event.java index 5f4c250..020fe81 100644 --- a/src/main/java/com/o19s/ubi/events/Event.java +++ b/src/main/java/com/o19s/ubi/model/Event.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package com.o19s.ubi.events; +package com.o19s.ubi.model; /** * A client-side event. diff --git a/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java index 7de29ff..c09997c 100644 --- a/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java +++ b/src/main/java/com/o19s/ubi/rest/UserBehaviorInsightsRestHandler.java @@ -12,9 +12,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.o19s.ubi.OpenSearchEventManager; +import com.o19s.ubi.data.OpenSearchDataManager; import com.o19s.ubi.UserBehaviorInsightsPlugin; -import com.o19s.ubi.events.Event; +import com.o19s.ubi.model.Event; import com.o19s.ubi.model.HeaderConstants; import com.o19s.ubi.model.SettingsConstants; import com.o19s.ubi.utils.UbiUtils; @@ -214,11 +214,11 @@ private RestChannelConsumer post(final NodeClient nodeClient, final String store final String eventJson = restRequest.content().utf8ToString(); final String eventJsonWithTimestamp = setEventTimestamp(eventJson); - LOGGER.info("Indexing UBI event into store {}", storeName); + LOGGER.trace("Indexing UBI event into store {}", storeName); final String eventsIndexName = UbiUtils.getEventsIndexName(storeName); final Event event = new Event(eventsIndexName, eventJsonWithTimestamp); - OpenSearchEventManager.getInstance(nodeClient).add(event); + OpenSearchDataManager.getInstance(nodeClient).add(event); } catch (JsonProcessingException ex) { LOGGER.error("Unable to get/set timestamp on UBI event.", ex); From 376bc65ab719443c8ee212a51588e01701c044fd Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 12 Mar 2024 13:57:11 -0400 Subject: [PATCH 35/37] Simplifying and changing to a blocking queue. --- .../UserBehaviorInsightsActionFilter.java | 1 + .../java/com/o19s/ubi/data/DataManager.java | 21 ---------------- .../o19s/ubi/data/OpenSearchDataManager.java | 24 ++++++++++++++++++- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java index 4e92cd7..61c191f 100644 --- a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java @@ -134,6 +134,7 @@ public void onResponse(Response response) { final QueryRequest queryRequest = new QueryRequest(storeName, queryId, query, userId, sessionId, queryResponse); // Queue this for writing to the UBI store. + LOGGER.info("Queuing queryrequest"); dataManager.add(queryRequest); // Add the query_id to the response headers. diff --git a/src/main/java/com/o19s/ubi/data/DataManager.java b/src/main/java/com/o19s/ubi/data/DataManager.java index e381d98..84c13c6 100644 --- a/src/main/java/com/o19s/ubi/data/DataManager.java +++ b/src/main/java/com/o19s/ubi/data/DataManager.java @@ -13,9 +13,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingDeque; - /** * Base class for managing client-side events and queries. */ @@ -24,24 +21,6 @@ public abstract class DataManager { @SuppressWarnings("unused") private final Logger LOGGER = LogManager.getLogger(DataManager.class); - /** - * The queue that stores the client-side events. - */ - protected final BlockingQueue eventsQueue; - - /** - * The queue that stores the query requests. - */ - protected final BlockingQueue queryRequestsQueue; - - /** - * Initialize the base client-side event manager. - */ - public DataManager() { - this.eventsQueue = new LinkedBlockingDeque<>(); - this.queryRequestsQueue = new LinkedBlockingDeque<>(); - } - /** * Process the events on the queue by writing them to persistent storage. */ diff --git a/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java b/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java index 718d069..aea5dda 100644 --- a/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java +++ b/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java @@ -20,6 +20,8 @@ import java.util.HashMap; import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; /** * An event manager that inserts events into an OpenSearch index. @@ -32,8 +34,21 @@ public class OpenSearchDataManager extends DataManager { private final int max_items_batch = 1000; private static OpenSearchDataManager openSearchEventManager; + /** + * The queue that stores the client-side events. + */ + protected final BlockingQueue eventsQueue; + + /** + * The queue that stores the query requests. + */ + protected final BlockingQueue queryRequestsQueue; + + private OpenSearchDataManager(Client client) { this.client = client; + this.eventsQueue = new LinkedBlockingQueue<>(); + this.queryRequestsQueue = new LinkedBlockingQueue<>(); } @Override @@ -67,7 +82,7 @@ public void processQueries() { final QueryRequest queryRequest = queryRequestsQueue.remove(); - LOGGER.trace("Writing query ID {} with response ID {}", + LOGGER.info("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryRequest.getQueryResponse().getQueryResponseId()); // What will be indexed - adheres to the queries-mapping.json @@ -83,6 +98,8 @@ public void processQueries() { // Get the name of the queries. final String queriesIndexName = UbiUtils.getQueriesIndexName(queryRequest.getStoreName()); + LOGGER.info("Going to index into " + queriesIndexName); + // Build the index request. final IndexRequest indexRequest = new IndexRequest(queriesIndexName) .source(source, XContentType.JSON); @@ -91,10 +108,14 @@ public void processQueries() { } + LOGGER.info("Indexing " + queryRequestsBulkRequest.numberOfActions() + " queries"); + if(queryRequestsBulkRequest.numberOfActions() > 0) { client.bulk(queryRequestsBulkRequest); } + LOGGER.info("Done processing queries"); + } @Override @@ -104,6 +125,7 @@ public void add(final Event event) { @Override public void add(final QueryRequest queryRequest) { + LOGGER.info("Adding query request to queue: " + queryRequestsQueue.size()); queryRequestsQueue.add(queryRequest); } From a29f6820129cc0654d95c2dae4d56c9c4f533128 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 12 Mar 2024 14:06:31 -0400 Subject: [PATCH 36/37] Lowering logger level. --- .../o19s/ubi/UserBehaviorInsightsPlugin.java | 4 +-- .../o19s/ubi/data/OpenSearchDataManager.java | 32 ++++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java index 3cd8ff1..3b451a4 100644 --- a/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java +++ b/src/main/java/com/o19s/ubi/UserBehaviorInsightsPlugin.java @@ -126,12 +126,12 @@ public Collection createComponents( LOGGER.info("Starting UBI scheduled task to persist events."); threadPool.scheduler().scheduleWithFixedDelay(() -> { OpenSearchDataManager.getInstance(client).processEvents(); - }, 0, 2000, TimeUnit.MILLISECONDS); + }, 5000, 2000, TimeUnit.MILLISECONDS); LOGGER.info("Starting UBI scheduled task to persist queries."); threadPool.scheduler().scheduleWithFixedDelay(() -> { OpenSearchDataManager.getInstance(client).processQueries(); - }, 0, 2000, TimeUnit.MILLISECONDS); + }, 5000, 2000, TimeUnit.MILLISECONDS); // Initialize the action filter. this.userBehaviorLoggingFilter = new UserBehaviorInsightsActionFilter(client, threadPool); diff --git a/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java b/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java index aea5dda..c81337e 100644 --- a/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java +++ b/src/main/java/com/o19s/ubi/data/OpenSearchDataManager.java @@ -44,6 +44,17 @@ public class OpenSearchDataManager extends DataManager { */ protected final BlockingQueue queryRequestsQueue; + /** + * Gets a singleton instance of the manager. + * @param client An OpenSearch {@link Client}. + * @return An instance of {@link OpenSearchDataManager}. + */ + public static OpenSearchDataManager getInstance(Client client) { + if(openSearchEventManager == null) { + openSearchEventManager = new OpenSearchDataManager(client); + } + return openSearchEventManager; + } private OpenSearchDataManager(Client client) { this.client = client; @@ -82,7 +93,7 @@ public void processQueries() { final QueryRequest queryRequest = queryRequestsQueue.remove(); - LOGGER.info("Writing query ID {} with response ID {}", + LOGGER.trace("Writing query ID {} with response ID {}", queryRequest.getQueryId(), queryRequest.getQueryResponse().getQueryResponseId()); // What will be indexed - adheres to the queries-mapping.json @@ -98,8 +109,6 @@ public void processQueries() { // Get the name of the queries. final String queriesIndexName = UbiUtils.getQueriesIndexName(queryRequest.getStoreName()); - LOGGER.info("Going to index into " + queriesIndexName); - // Build the index request. final IndexRequest indexRequest = new IndexRequest(queriesIndexName) .source(source, XContentType.JSON); @@ -108,14 +117,12 @@ public void processQueries() { } - LOGGER.info("Indexing " + queryRequestsBulkRequest.numberOfActions() + " queries"); + LOGGER.trace("Indexing " + queryRequestsBulkRequest.numberOfActions() + " queries"); if(queryRequestsBulkRequest.numberOfActions() > 0) { client.bulk(queryRequestsBulkRequest); } - LOGGER.info("Done processing queries"); - } @Override @@ -125,20 +132,7 @@ public void add(final Event event) { @Override public void add(final QueryRequest queryRequest) { - LOGGER.info("Adding query request to queue: " + queryRequestsQueue.size()); queryRequestsQueue.add(queryRequest); } - /** - * Gets a singleton instance of the manager. - * @param client An OpenSearch {@link Client}. - * @return An instance of {@link OpenSearchDataManager}. - */ - public static OpenSearchDataManager getInstance(Client client) { - if(openSearchEventManager == null) { - openSearchEventManager = new OpenSearchDataManager(client); - } - return openSearchEventManager; - } - } From c3bfed24b69301adaff41b3f12907ae263a85a80 Mon Sep 17 00:00:00 2001 From: jzonthemtn Date: Tue, 12 Mar 2024 14:08:05 -0400 Subject: [PATCH 37/37] Removing logger. --- .../com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java index 61c191f..4e92cd7 100644 --- a/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java +++ b/src/main/java/com/o19s/ubi/action/UserBehaviorInsightsActionFilter.java @@ -134,7 +134,6 @@ public void onResponse(Response response) { final QueryRequest queryRequest = new QueryRequest(storeName, queryId, query, userId, sessionId, queryResponse); // Queue this for writing to the UBI store. - LOGGER.info("Queuing queryrequest"); dataManager.add(queryRequest); // Add the query_id to the response headers.