Skip to content

Commit

Permalink
Make backwards compatible with SolrJ
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlbiscoc
mlbiscoc committed Dec 24, 2024
1 parent f6e77cd commit 43f30fa
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 47 deletions.
3 changes: 1 addition & 2 deletions solr/core/src/java/org/apache/solr/cli/StatusTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,8 +353,7 @@ protected Map<String, String> getCloudStatus(SolrClient solrClient, String zkHos
cloudStatus.put("liveNodes", String.valueOf(liveNodes.size()));

// TODO get this as a metric from the metrics API instead, or something else.
Map<String, Object> collections =
(Map<String, Object>) json.findRecursive("cluster", "collections");
var collections = (NamedList<Object>) json.findRecursive("cluster", "collections");
cloudStatus.put("collections", String.valueOf(collections.size()));

return cloudStatus;
Expand Down
45 changes: 30 additions & 15 deletions solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
Expand Down Expand Up @@ -205,21 +206,35 @@ private void fetchClusterStatusForCollOrAlias(
}
}

MapWriter collectionPropsWriter =
ew -> {
collectionStream.forEach(
(collectionState) -> {
ew.putNoEx(
collectionState.getName(),
buildResponseForCollection(
collectionState,
collectionVsAliases,
routeKey,
liveNodes,
requestedShards));
});
};
clusterStatus.add("collections", collectionPropsWriter);
// Because of back-compat for SolrJ, create the whole response into a NamedList
// Otherwise stream with MapWriter to save memory
if (CommonParams.JAVABIN.equals(solrParams.get(CommonParams.WT))) {
NamedList<Object> collectionProps = new SimpleOrderedMap<>();
collectionStream.forEach(
collectionState -> {
collectionProps.add(
collectionState.getName(),
buildResponseForCollection(
collectionState, collectionVsAliases, routeKey, liveNodes, requestedShards));
});
clusterStatus.add("collections", collectionProps);
} else {
MapWriter collectionPropsWriter =
ew -> {
collectionStream.forEach(
(collectionState) -> {
ew.putNoEx(
collectionState.getName(),
buildResponseForCollection(
collectionState,
collectionVsAliases,
routeKey,
liveNodes,
requestedShards));
});
};
clusterStatus.add("collections", collectionPropsWriter);
}
}

private void addAliasMap(Aliases aliases, NamedList<Object> clusterStatus) {
Expand Down
71 changes: 51 additions & 20 deletions solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionAPI.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package org.apache.solr.cloud.api.collections;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
Expand All @@ -28,6 +29,7 @@
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.NoOpResponseParser;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
Expand Down Expand Up @@ -81,7 +83,6 @@ public void test() throws Exception {
client.request(req);
createCollection(null, COLLECTION_NAME1, 1, 1, client, null, "conf1");
}

waitForCollection(ZkStateReader.from(cloudClient), COLLECTION_NAME, 2);
waitForCollection(ZkStateReader.from(cloudClient), COLLECTION_NAME1, 1);
waitForRecoveriesToFinish(COLLECTION_NAME, false);
Expand All @@ -91,6 +92,7 @@ public void test() throws Exception {
clusterStatusNoCollection();
clusterStatusWithCollection();
clusterStatusWithCollectionAndShard();
clusterStatusWithCollectionAndShardJSON();
clusterStatusWithCollectionAndMultipleShards();
clusterStatusWithCollectionHealthState();
clusterStatusWithRouteKey();
Expand Down Expand Up @@ -129,12 +131,10 @@ private void testModifyCollection() throws Exception {
.getResponse();
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertEquals(1, collections.size());
Map<?, ?> collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);
collectionProperties.get("replicationFactor");
assertEquals("25", collectionProperties.get("replicationFactor"));
assertEquals("25", collections._getStr(List.of(COLLECTION_NAME, "replicationFactor"), null));

params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
Expand All @@ -153,12 +153,10 @@ private void testModifyCollection() throws Exception {
System.out.println(rsp);
cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
collections = (Map<?, ?>) cluster.get("collections");
collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertEquals(1, collections.size());
collectionProperties = (Map<?, ?>) collections.get(COLLECTION_NAME);
collectionProperties.get("replicationFactor");
assertNull(collectionProperties.get("replicationFactor"));
assertNull(collections._getStr(List.of(COLLECTION_NAME, "replicationFactor"), null));

params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.MODIFYCOLLECTION.toString());
Expand Down Expand Up @@ -257,7 +255,7 @@ private void testNoConfigset() throws Exception {
NamedList<?> rsp = client.request(req);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(
"Testing to insure collections are returned", collections.get(COLLECTION_NAME1));
Expand All @@ -284,7 +282,7 @@ private void assertCountsForRepFactorAndNrtReplicas(CloudSolrClient client, Stri
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertEquals(1, collections.size());
@SuppressWarnings({"unchecked"})
Expand All @@ -306,7 +304,7 @@ private void clusterStatusWithCollectionAndShard() throws IOException, SolrServe
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
Expand All @@ -332,7 +330,7 @@ private void clusterStatusWithCollectionAndMultipleShards()
NamedList<Object> rsp = request.process(client).getResponse();
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
Expand Down Expand Up @@ -467,7 +465,7 @@ private void clusterStatusNoCollection() throws Exception {
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(COLLECTION_NAME1));
assertEquals(4, collections.size());
Expand All @@ -489,7 +487,7 @@ private void clusterStatusWithCollection() throws IOException, SolrServerExcepti
NamedList<Object> rsp = client.request(request);
NamedList<?> cluster = (NamedList<?>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<?> collections = (NamedList<?>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertEquals(1, collections.size());
@SuppressWarnings({"unchecked"})
Expand Down Expand Up @@ -519,7 +517,7 @@ private void clusterStatusZNodeVersion() throws Exception {
NamedList<Object> rsp = client.request(request);
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertEquals(1, collections.size());
Map<String, Object> collection = (Map<String, Object>) collections.get(cname);
Expand All @@ -535,7 +533,7 @@ private void clusterStatusZNodeVersion() throws Exception {

rsp = client.request(request);
cluster = (NamedList<Object>) rsp.get("cluster");
collections = (Map<?, ?>) cluster.get("collections");
collections = (NamedList<Object>) cluster.get("collections");
collection = (Map<String, Object>) collections.get(cname);
Integer newVersion = (Integer) collection.get("znodeVersion");
assertNotNull(newVersion);
Expand All @@ -562,7 +560,7 @@ private void clusterStatusWithRouteKey() throws IOException, SolrServerException
NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
@SuppressWarnings({"unchecked"})
Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
assertEquals(1, collections.size());
Expand Down Expand Up @@ -609,7 +607,7 @@ private void clusterStatusAliasTest() throws Exception {
DEFAULT_COLLECTION + "," + COLLECTION_NAME,
aliases.get("myalias"));

Map<?, ?> collections = (Map<?, ?>) cluster.get("collections");
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
Map<String, Object> collection = (Map<String, Object>) collections.get(DEFAULT_COLLECTION);
Expand All @@ -629,7 +627,7 @@ private void clusterStatusAliasTest() throws Exception {

cluster = (NamedList<Object>) rsp.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
collections = (Map<?, ?>) cluster.get("collections");
collections = (NamedList<Object>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(DEFAULT_COLLECTION));
assertNotNull(collections.get(COLLECTION_NAME));
Expand All @@ -652,6 +650,39 @@ private void clusterStatusAliasTest() throws Exception {
}
}

@SuppressWarnings("unchecked")
private void clusterStatusWithCollectionAndShardJSON() throws IOException, SolrServerException {

try (CloudSolrClient client = createCloudClient(null)) {
ObjectMapper mapper = new ObjectMapper();

ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
params.set("collection", COLLECTION_NAME);
params.set("shard", SHARD1);
params.set("wt", "json");
QueryRequest request = new QueryRequest(params);
request.setResponseParser(new NoOpResponseParser("json"));
request.setPath("/admin/collections");
NamedList<Object> rsp = client.request(request);
String actualResponse = (String) rsp.get("response");

Map<String, Object> result = mapper.readValue(actualResponse, Map.class);

var cluster = (Map<String, Object>) result.get("cluster");
assertNotNull("Cluster state should not be null", cluster);
var collections = (Map<String, Object>) cluster.get("collections");
assertNotNull("Collections should not be null in cluster state", collections);
assertNotNull(collections.get(COLLECTION_NAME));
assertEquals(1, collections.size());
var collection = (Map<String, Object>) collections.get(COLLECTION_NAME);
var shardStatus = (Map<String, Object>) collection.get("shards");
assertEquals(1, shardStatus.size());
Map<String, Object> selectedShardStatus = (Map<String, Object>) shardStatus.get(SHARD1);
assertNotNull(selectedShardStatus);
}
}

private void clusterStatusRolesTest() throws Exception {
try (CloudSolrClient client = createCloudClient(null)) {
client.connect();
Expand Down
16 changes: 7 additions & 9 deletions solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.CollectionUtil;
import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -138,16 +139,13 @@ private ClusterState fetchClusterState(SolrClient client)
this.liveNodes = Set.copyOf(liveNodesList);
liveNodesTimestamp = System.nanoTime();
}

var collectionsMap = (Map<String, Map<String, Object>>) cluster.get("collections");

var collectionsNl = (NamedList<Map<String, Object>>) cluster.get("collections");
Map<String, DocCollection> collStateByName =
CollectionUtil.newLinkedHashMap(collectionsMap.size());
for (Entry<String, Map<String, Object>> entry : collectionsMap.entrySet()) {
CollectionUtil.newLinkedHashMap(collectionsNl.size());
for (Entry<String, Map<String, Object>> entry : collectionsNl) {
collStateByName.put(
entry.getKey(), getDocCollectionFromObjects(entry.getKey(), entry.getValue()));
}

return new ClusterState(this.liveNodes, collStateByName);
}

Expand Down Expand Up @@ -184,9 +182,9 @@ private DocCollection fetchCollectionState(SolrClient client, String collection)
SimpleOrderedMap<?> cluster =
submitClusterStateRequest(client, collection, ClusterStateRequestType.FETCH_COLLECTION);

var collState = (Map<String, Object>) cluster.findRecursive("collections");
var collStateMap = (Map<String, Object>) collState.get(collection);

var collStateMap =
(Map<String, Object>)
cluster.findRecursive("collections", collection); // SOLRJ IS ALWAYS JAVABIN
if (collStateMap == null) {
throw new NotACollectionException(); // probably an alias
}
Expand Down
2 changes: 1 addition & 1 deletion solr/solrj/src/test/org/apache/solr/client/solrj/impl/ClusterStateProviderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ private Instant getCreationTimeFromClusterStatus(String collectionName)
NamedList<Object> response = clusterStatusResponse.getResponse();

NamedList<Object> cluster = (NamedList<Object>) response.get("cluster");
Map<String, Object> collections = (Map<String, Object>) cluster.get("collections");
NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
Map<String, Object> collection = (Map<String, Object>) collections.get(collectionName);
return Instant.ofEpochMilli((long) collection.get("creationTimeMillis"));
}
Expand Down

0 comments on commit 43f30fa

Please sign in to comment.