Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check before delete #3209

Open
wants to merge 43 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
432d81f
add logic to detect agent before deleting
xinyual Nov 8, 2024
a273f30
add logic to detect agent before deleting
xinyual Nov 8, 2024
a839db4
add logic to detect pipelines before delete model
xinyual Nov 8, 2024
9691e96
check pipeline before deleting
xinyual Nov 11, 2024
0c100fe
apply spotless
xinyual Nov 11, 2024
aafe91f
remove useless file
xinyual Nov 11, 2024
d93928d
rename functions
xinyual Nov 12, 2024
9f7947e
fix failure test
xinyual Nov 12, 2024
8fea510
add UT
xinyual Nov 12, 2024
ca41ec4
apply spotless
xinyual Nov 12, 2024
56ec29f
renam
xinyual Nov 18, 2024
88d5fd5
refactor to parallel check
xinyual Nov 18, 2024
e6c6bfc
concate error message
xinyual Nov 19, 2024
ba587fd
move logic after user access check
xinyual Nov 19, 2024
de464e9
change agent model searcher map to set
xinyual Nov 19, 2024
ef68097
rename and remove useless method
xinyual Nov 19, 2024
525926f
fix bug to fetch all pipelines
xinyual Nov 19, 2024
5c1b3da
apply spotless
xinyual Nov 19, 2024
7cc0a08
apply spotless
xinyual Nov 19, 2024
b4adbc7
remove and add comment
xinyual Nov 20, 2024
b963587
rename and add more UTs
xinyual Nov 21, 2024
9c831ba
use correct key
xinyual Nov 21, 2024
f53a656
simplify function
xinyual Nov 21, 2024
542de96
change to a better class
xinyual Nov 21, 2024
266d3fa
apply spotless
xinyual Nov 21, 2024
1df7947
change compareAndSet to set
xinyual Nov 25, 2024
cde3934
apply comment
xinyual Nov 25, 2024
6bb7c63
change name and reformat logic
xinyual Dec 4, 2024
b709839
change name
xinyual Dec 4, 2024
642b022
remove useless line
xinyual Dec 4, 2024
eb33bd2
rebase
xinyual Dec 4, 2024
6a38c2b
change to a better method
xinyual Dec 4, 2024
e04d9d6
change name
xinyual Dec 9, 2024
411dc73
apply spotless
xinyual Dec 9, 2024
e29c011
add java doc for function
xinyual Dec 9, 2024
b777398
add another interface
xinyual Dec 13, 2024
273af49
apply java spotless
xinyual Dec 13, 2024
3b142c3
change interface to with model
xinyual Dec 16, 2024
42d2d6f
apply spot less
xinyual Dec 16, 2024
b472292
add settings
xinyual Dec 17, 2024
8ef451e
apply spot less
xinyual Dec 17, 2024
d9e0925
add test for cluster setting
xinyual Dec 18, 2024
c83bfac
apply spotless
xinyual Dec 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ public class CommonValue {
public static final String ML_STOP_WORDS_INDEX = ".plugins-ml-stop-words";
public static final Set<String> stopWordsIndices = ImmutableSet.of(".plugins-ml-stop-words");
public static final Integer ML_MEMORY_MESSAGE_INDEX_SCHEMA_VERSION = 1;
public static final String TOOL_MODEL_RELATED_FIELD_PREFIX = "tools.parameters.";
xinyual marked this conversation as resolved.
Show resolved Hide resolved
public static final String USER_FIELD_MAPPING = " \""
+ CommonValue.USER
+ "\": {\n"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package org.opensearch.ml.engine.tools;

import static org.opensearch.ml.common.CommonValue.ML_AGENT_INDEX;
import static org.opensearch.ml.common.CommonValue.TOOL_MODEL_RELATED_FIELD_PREFIX;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.opensearch.action.search.SearchRequest;
import org.opensearch.index.query.BoolQueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.ml.common.spi.tools.Tool;
import org.opensearch.search.builder.SearchSourceBuilder;

public class AgentModelsSearcher {
xinyual marked this conversation as resolved.
Show resolved Hide resolved
private final Set<String> relatedModelIdSet;

public AgentModelsSearcher(Map<String, Tool.Factory> toolFactories) {
xinyual marked this conversation as resolved.
Show resolved Hide resolved
relatedModelIdSet = new HashSet<>();
for (Map.Entry<String, Tool.Factory> entry : toolFactories.entrySet()) {
Tool.Factory toolFactory = entry.getValue();
relatedModelIdSet.addAll(toolFactory.getAllModelKeys());
}
}

public SearchRequest constructQueryRequest(String candidateModelId) {
SearchRequest searchRequest = new SearchRequest(ML_AGENT_INDEX);
BoolQueryBuilder shouldQuery = QueryBuilders.boolQuery();
for (String keyField : relatedModelIdSet) {
shouldQuery.should(QueryBuilders.termsQuery(TOOL_MODEL_RELATED_FIELD_PREFIX + keyField, candidateModelId));
}
searchRequest.source(new SearchSourceBuilder().query(shouldQuery));
return searchRequest;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import static org.opensearch.ml.common.utils.StringUtils.gson;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.opensearch.action.ActionRequest;
Expand Down Expand Up @@ -138,6 +139,11 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}

private Map<String, String> extractInputParameters(Map<String, String> parameters) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,11 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}

private Table getTableWithHeader() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,5 +144,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
xinyual marked this conversation as resolved.
Show resolved Hide resolved
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -206,5 +206,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -172,5 +172,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of(MODEL_ID_FIELD);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
Expand Down Expand Up @@ -195,5 +196,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.opensearch.ml.engine.tools;

import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -174,5 +175,10 @@ public String getDefaultType() {
public String getDefaultVersion() {
return null;
}

@Override
public List<String> getAllModelKeys() {
return List.of();
}
}
}
Loading
Loading