From 88990d640a89091a8f7b0b2493377ac24118afe8 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Thu, 10 Aug 2023 12:51:53 -0400 Subject: [PATCH 001/160] SOLR-16920: Add Test full package lifecycle (#1826), fix parsing of repository.json mimetype Github is a common place to store the repository.json file used by Solr Packages. While the file is .json, the mimetype from Github is text/plain, so this fixes the client being used to load the repository.json to accept text/plain as well as application/json mimetypes. --- solr/CHANGES.txt | 2 ++ .../DefaultPackageRepository.java | 20 ++++++++++++++-- solr/packaging/test/test_packages.bats | 23 ++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index d5d12bf7cf8..e961fdfa2ec 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -112,6 +112,8 @@ Bug Fixes * SOLR-16360: Atomic update on boolean fields doesn't reflect when value starts with "1", "t" or "T" (Rahul Goswami, Justin Sweeney, David Smiley) +* PR#1826: Allow looking up Solr Package repo when that URL references a raw repository.json hosted on Github when the file is JSON but the mimetype used is text/plain. (Eric Pugh) + Dependency Upgrades --------------------- (No changes) diff --git a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java index 32533625e60..820391804e1 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/DefaultPackageRepository.java @@ -23,7 +23,9 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collection; import java.util.Map; +import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.solr.client.solrj.SolrRequest; @@ -106,13 +108,15 @@ public Path download(String artifactName) throws SolrException, IOException { private void initPackages() { // We need http 1.1 protocol here because we are talking to the repository server and not to - // solr actually. + // an actual Solr server. // We use an Http2SolrClient so that we do not need a raw jetty http client for this GET. + // We may get a text/plain mimetype for the repository.json (for instance when looking up a repo + // on Github), so use custom ResponseParser. try (Http2SolrClient client = new Http2SolrClient.Builder(repositoryURL).useHttp1_1(true).build()) { GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, "/repository.json"); - request.setResponseParser(new JsonMapResponseParser()); + request.setResponseParser(new TalkToRepoResponseParser()); NamedList resp = client.request(request); SolrPackage[] items = PackageUtils.getMapper().readValue("[" + resp.jsonStr() + "]", SolrPackage[].class); @@ -128,4 +132,16 @@ private void initPackages() { log.debug("Found {} packages in repository '{}'", packages.size(), name); } } + + /** + * Github links for repository.json are returned in JSON format but with text/plain mimetype, so + * this works around that issue. + */ + private static class TalkToRepoResponseParser extends JsonMapResponseParser { + + @Override + public Collection getContentTypes() { + return Set.of("application/json", "text/plain"); + } + } } diff --git a/solr/packaging/test/test_packages.bats b/solr/packaging/test/test_packages.bats index fd7410b5dd8..3dda0e718e8 100644 --- a/solr/packaging/test/test_packages.bats +++ b/solr/packaging/test/test_packages.bats @@ -47,7 +47,7 @@ teardown() { assert_output --partial "Available packages:" } -@test "deploying and undeploying of packages" { +@test "deploying and undeploying a collection level package" { run solr start -c -Denable.packages=true solr create -c foo-1.2 @@ -65,3 +65,24 @@ teardown() { refute_output --partial "Invalid collection" assert_output --partial "Package PACKAGE_NAME not deployed on collection foo-1.2" } + +# This test is useful if you are debugging/working with packages. +# We have commented it out for now since it depends on a live internet +# connection to run. This could be updated with a local Repo server if we had +# a package that is part of the Solr project to use. +# @test "deploying and undeploying a cluster level package" { +# run solr start -c -Denable.packages=true + +# run solr package add-repo splainer "https://raw.githubusercontent.com/o19s/splainer/main/solr-splainer-package/repo" +# assert_output --partial "Added repository: splainer" + +# run solr package list-available +# assert_output --partial "solr-splainer Splainer for Solr" +# run solr package install solr-splainer +# assert_output --partial "solr-splainer installed." + +# run solr package deploy solr-splainer -y -cluster +# assert_output --partial "Deployment successful" + +# run -0 curl --fail http://localhost:8983/v2/splainer/index.html +# } From 8724b1d093633d0363b256384cfa75a9e5197f80 Mon Sep 17 00:00:00 2001 From: Alex D Date: Fri, 11 Aug 2023 15:15:19 -0700 Subject: [PATCH 002/160] SOLR-14667 Make zkClientTimeout consistent and based on a system property (#1831) --- solr/CHANGES.txt | 2 + .../java/org/apache/solr/cli/AuthTool.java | 13 ++-- .../solr/cli/ConfigSetDownloadTool.java | 3 +- .../apache/solr/cli/ConfigSetUploadTool.java | 3 +- .../java/org/apache/solr/cli/ZkCpTool.java | 3 +- .../java/org/apache/solr/cli/ZkLsTool.java | 3 +- .../org/apache/solr/cli/ZkMkrootTool.java | 3 +- .../java/org/apache/solr/cli/ZkMvTool.java | 3 +- .../java/org/apache/solr/cli/ZkRmTool.java | 3 +- .../src/java/org/apache/solr/cloud/ZkCLI.java | 6 +- .../org/apache/solr/cloud/ZkController.java | 13 ++-- .../org/apache/solr/core/CloudConfig.java | 4 +- .../org/apache/solr/core/ZkContainer.java | 3 +- .../solr/packagemanager/PackageManager.java | 3 +- .../schema/ManagedIndexSchemaFactory.java | 2 +- .../impl/ZkClientClusterStateProvider.java | 10 ++- .../solr/common/cloud/SolrZkClient.java | 27 +++++--- .../solrj/impl/CloudHttp2SolrClient.java | 66 +++++++++++-------- .../solrj/impl/CloudLegacySolrClient.java | 61 +++++++++-------- .../solrj/impl/SolrZkClientTimeout.java | 29 ++++++++ .../org/apache/solr/util/TestHarness.java | 3 +- 21 files changed, 168 insertions(+), 95 deletions(-) create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrZkClientTimeout.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e961fdfa2ec..9951772eecc 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -89,6 +89,8 @@ Improvements circumstances as they are generated and more likely to remain up-to-date with future API changes. (Jason Gerlowski, David Smiley, Houston Putman) +* SOLR-14667: Make zkClientTimeout consistent and based on a system property. The defauls values are stored in a single place referenced everywhere and they are based on system properties (Alex Deparvu) + Optimizations --------------------- diff --git a/solr/core/src/java/org/apache/solr/cli/AuthTool.java b/solr/core/src/java/org/apache/solr/cli/AuthTool.java index a502fa77cef..318f9feaced 100644 --- a/solr/core/src/java/org/apache/solr/cli/AuthTool.java +++ b/solr/core/src/java/org/apache/solr/cli/AuthTool.java @@ -34,6 +34,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.lucene.util.Constants; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.util.StrUtils; import org.apache.solr.security.Sha256AuthenticationProvider; @@ -187,7 +188,8 @@ private int handleKerberos(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) + .withTimeout( + SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { checkSecurityJsonExists(zkClient); } catch (Exception ex) { @@ -206,7 +208,8 @@ private int handleKerberos(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) + .withTimeout( + SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { zkClient.setData( "/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); @@ -321,7 +324,7 @@ private int handleBasicAuth(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { checkSecurityJsonExists(zkClient); } @@ -384,7 +387,7 @@ private int handleBasicAuth(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { zkClient.setData("/security.json", securityJson.getBytes(StandardCharsets.UTF_8), true); } @@ -479,7 +482,7 @@ private void clearSecurityJson(CommandLine cli, boolean updateIncludeFileOnly) t try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(10000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { zkClient.setData("/security.json", "{}".getBytes(StandardCharsets.UTF_8), true); } diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java index 8603af39513..c11e0e70a79 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetDownloadTool.java @@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +80,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); String confName = cli.getOptionValue("confname"); diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java index a36370784dd..5fd4a538bd7 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.core.ConfigSetService; @@ -86,7 +87,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); Path confPath = diff --git a/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java b/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java index 571db1351c2..02379d5748a 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkCpTool.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +79,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); String src = cli.getOptionValue("src"); diff --git a/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java b/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java index 5f4d6d83fca..24f1c80144e 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkLsTool.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -72,7 +73,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); diff --git a/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java b/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java index 8a9b7d769d8..e41187839f4 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkMkrootTool.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,7 +72,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); diff --git a/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java b/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java index 6c147c85594..6506aa9176e 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkMvTool.java @@ -24,6 +24,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +79,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli); String src = cli.getOptionValue("src"); diff --git a/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java b/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java index 0ca4694ac89..dad5bdbbe94 100644 --- a/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ZkRmTool.java @@ -24,6 +24,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.cloud.SolrZkClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,7 +85,7 @@ public void runImpl(CommandLine cli) throws Exception { try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { if (!recurse && zkClient.getChildren(znode, null, true).size() != 0) { throw new SolrServerException( diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index ef2f29cb62a..c1f9b826529 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -42,6 +42,7 @@ import org.apache.commons.cli.ParseException; import org.apache.commons.cli.PosixParser; import org.apache.solr.cli.CLIO; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.SolrZkClient; @@ -299,8 +300,9 @@ public static void main(String[] args) try (SolrZkClient zkClient = new SolrZkClient.Builder() .withUrl(zkServerAddress) - .withTimeout(30000, TimeUnit.MILLISECONDS) - .withConnTimeOut(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) + .withConnTimeOut( + SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS) .withReconnectListener(() -> {}) .withCompressor(compressor) .build()) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index c812b7767cb..c93a7dfef99 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -62,6 +62,7 @@ import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.client.solrj.impl.SolrClientCloudManager; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState; import org.apache.solr.cloud.overseer.ClusterStateMutator; @@ -233,8 +234,6 @@ public String toString() { private boolean genericCoreNodeNames; - private int clientTimeout; - private volatile boolean isClosed; private final ConcurrentHashMap replicasMetTragicEvent = @@ -331,7 +330,7 @@ public ZkController( this.leaderVoteWait = cloudConfig.getLeaderVoteWait(); this.leaderConflictResolveWait = cloudConfig.getLeaderConflictResolveWait(); - this.clientTimeout = cloudConfig.getZkClientTimeout(); + int clientTimeout = cloudConfig.getZkClientTimeout(); String connectionStrategy = System.getProperty("solr.zookeeper.connectionStrategy"); ZkClientConnectionStrategy strat = @@ -1188,8 +1187,8 @@ public static boolean checkChrootPath(String zkHost, boolean create) SolrZkClient tmpClient = new SolrZkClient.Builder() .withUrl(zkHost.substring(0, zkHost.indexOf('/'))) - .withTimeout(60000, TimeUnit.MILLISECONDS) - .withConnTimeOut(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) + .withConnTimeOut(SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT, TimeUnit.MILLISECONDS) .build(); boolean exists = tmpClient.exists(chrootPath, true); if (!exists && create) { @@ -2339,10 +2338,6 @@ public boolean clearAsyncId(String asyncId) throws KeeperException { } } - public int getClientTimeout() { - return clientTimeout; - } - public Overseer getOverseer() { return overseer; } diff --git a/solr/core/src/java/org/apache/solr/core/CloudConfig.java b/solr/core/src/java/org/apache/solr/core/CloudConfig.java index 5cc3e58e37e..790aa333012 100644 --- a/solr/core/src/java/org/apache/solr/core/CloudConfig.java +++ b/solr/core/src/java/org/apache/solr/core/CloudConfig.java @@ -16,6 +16,7 @@ */ package org.apache.solr.core; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.SolrException; public class CloudConfig { @@ -179,7 +180,6 @@ public String getStateCompressorClass() { public static class CloudConfigBuilder { - private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 45000; private static final int DEFAULT_LEADER_VOTE_WAIT = 180000; // 3 minutes private static final int DEFAULT_LEADER_CONFLICT_RESOLVE_WAIT = 180000; private static final int DEFAULT_CREATE_COLLECTION_ACTIVE_WAIT = 45; // 45 seconds @@ -188,7 +188,7 @@ public static class CloudConfigBuilder { -1; // By default compression for state is disabled private String zkHost; - private int zkClientTimeout = Integer.getInteger("zkClientTimeout", DEFAULT_ZK_CLIENT_TIMEOUT); + private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; private final int hostPort; private final String hostName; private boolean useGenericCoreNames; diff --git a/solr/core/src/java/org/apache/solr/core/ZkContainer.java b/solr/core/src/java/org/apache/solr/core/ZkContainer.java index 23be2346099..f7ea012b249 100644 --- a/solr/core/src/java/org/apache/solr/core/ZkContainer.java +++ b/solr/core/src/java/org/apache/solr/core/ZkContainer.java @@ -29,6 +29,7 @@ import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.cloud.SolrZkServer; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.AlreadyClosedException; @@ -111,7 +112,7 @@ public void initZooKeeper(final CoreContainer cc, CloudConfig config) { } } - int zkClientConnectTimeout = 30000; + int zkClientConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; if (zookeeperHost != null) { diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index a8387b4c669..79b9fa1c9c3 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -42,6 +42,7 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.request.beans.PackagePayload; @@ -83,7 +84,7 @@ public PackageManager(SolrClient solrClient, String solrUrl, String zkHost) { this.zkClient = new SolrZkClient.Builder() .withUrl(zkHost) - .withTimeout(30000, TimeUnit.MILLISECONDS) + .withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS) .build(); log.info("Done initializing a zkClient instance..."); } diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java index 1cc5903ba4e..9008b3e443e 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java @@ -485,7 +485,7 @@ private void zkUgradeToManagedSchema() { // Rename the non-managed schema znode in ZooKeeper final String nonManagedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + resourceName; try { - ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(zkController.getClientTimeout()); + ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout()); if (zkController.pathExists(nonManagedSchemaPath)) { // First, copy the non-managed schema znode content to the upgraded schema znode byte[] bytes = zkController.getZkClient().getData(nonManagedSchemaPath, null, null, true); diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java index ff805fced17..075c1e4d3de 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout.SolrZkClientTimeoutAware; import org.apache.solr.common.AlreadyClosedException; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; @@ -40,14 +41,15 @@ import org.slf4j.LoggerFactory; /** Retrieves cluster state from Zookeeper */ -public class ZkClientClusterStateProvider implements ClusterStateProvider { +public class ZkClientClusterStateProvider + implements ClusterStateProvider, SolrZkClientTimeoutAware { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); volatile ZkStateReader zkStateReader; private boolean closeZkStateReader = true; private final String zkHost; - private int zkConnectTimeout = 15000; - private int zkClientTimeout = 45000; + private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; + private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; private volatile boolean isClosed = false; @@ -306,6 +308,7 @@ public int getZkConnectTimeout() { } /** Set the connect timeout to the zookeeper ensemble in ms */ + @Override public void setZkConnectTimeout(int zkConnectTimeout) { this.zkConnectTimeout = zkConnectTimeout; } @@ -315,6 +318,7 @@ public int getZkClientTimeout() { } /** Set the timeout to the zookeeper ensemble in ms */ + @Override public void setZkClientTimeout(int zkClientTimeout) { this.zkClientTimeout = zkClientTimeout; } diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java index b7aeec05b7d..ddbd70d375f 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java @@ -34,6 +34,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.annotation.JsonProperty; @@ -70,8 +71,6 @@ public class SolrZkClient implements Closeable { static final String NEWL = System.getProperty("line.separator"); - static final int DEFAULT_CLIENT_CONNECT_TIMEOUT = 30000; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private ConnectionManager connManager; @@ -1127,8 +1126,8 @@ public NodeData(Stat stat, byte[] data) { public static class Builder { public String zkServerAddress; - public int zkClientTimeout = DEFAULT_CLIENT_CONNECT_TIMEOUT; - public int zkClientConnectTimeout = DEFAULT_CLIENT_CONNECT_TIMEOUT; + public int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; + public int zkClientConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; public OnReconnect onReconnect; public BeforeReconnect beforeReconnect; public ZkClientConnectionStrategy connectionStrategy; @@ -1143,13 +1142,25 @@ public Builder withUrl(String server) { return this; } - public Builder withTimeout(int i, TimeUnit unit) { - this.zkClientTimeout = (int) unit.toMillis(i); + /** + * Sets the Zk client session timeout + * + * @param zkClientTimeout timeout value + * @param unit time unit + */ + public Builder withTimeout(int zkClientTimeout, TimeUnit unit) { + this.zkClientTimeout = Math.toIntExact(unit.toMillis(zkClientTimeout)); return this; } - public Builder withConnTimeOut(int i, TimeUnit unit) { - this.zkClientConnectTimeout = (int) unit.toMillis(i); + /** + * Sets the Zk connection timeout + * + * @param zkConnectTimeout timeout value + * @param unit time unit + */ + public Builder withConnTimeOut(int zkConnectTimeout, TimeUnit unit) { + this.zkClientConnectTimeout = Math.toIntExact(unit.toMillis(zkConnectTimeout)); return this; } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java index 5222275c069..6311acf1b29 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java @@ -24,6 +24,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout.SolrZkClientTimeoutAware; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrException; @@ -73,31 +74,7 @@ protected CloudHttp2SolrClient(Builder builder) { if (builder.responseParser != null) { this.myClient.setParser(builder.responseParser); } - if (builder.stateProvider == null) { - if (builder.zkHosts != null && builder.solrUrls != null) { - throw new IllegalArgumentException( - "Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); - } - if (builder.zkHosts != null) { - this.stateProvider = - ClusterStateProvider.newZkClusterStateProvider(builder.zkHosts, builder.zkChroot); - } else if (builder.solrUrls != null && !builder.solrUrls.isEmpty()) { - try { - this.stateProvider = new Http2ClusterStateProvider(builder.solrUrls, builder.httpClient); - } catch (Exception e) { - throw new RuntimeException( - "Couldn't initialize a HttpClusterStateProvider (is/are the " - + "Solr server(s), " - + builder.solrUrls - + ", down?)", - e); - } - } else { - throw new IllegalArgumentException("Both zkHosts and solrUrl cannot be null."); - } - } else { - this.stateProvider = builder.stateProvider; - } + this.stateProvider = builder.stateProvider; this.collectionStateCache.timeToLiveMs = TimeUnit.MILLISECONDS.convert(builder.timeToLiveSeconds, TimeUnit.SECONDS); @@ -159,6 +136,8 @@ public static class Builder { private String defaultCollection; private long timeToLiveSeconds = 60; private int parallelCacheRefreshesLocks = 3; + private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; + private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; /** * Provide a series of Solr URLs to be used when configuring {@link CloudHttp2SolrClient} @@ -389,13 +368,42 @@ public Builder withInternalClientBuilder(Http2SolrClient.Builder internalClientB return this; } + /** + * Sets the Zk connection timeout + * + * @param zkConnectTimeout timeout value + * @param unit time unit + */ + public Builder withZkConnectTimeout(int zkConnectTimeout, TimeUnit unit) { + this.zkConnectTimeout = Math.toIntExact(unit.toMillis(zkConnectTimeout)); + return this; + } + + /** + * Sets the Zk client session timeout + * + * @param zkClientTimeout timeout value + * @param unit time unit + */ + public Builder withZkClientTimeout(int zkClientTimeout, TimeUnit unit) { + this.zkClientTimeout = Math.toIntExact(unit.toMillis(zkClientTimeout)); + return this; + } + /** Create a {@link CloudHttp2SolrClient} based on the provided configuration. */ public CloudHttp2SolrClient build() { if (stateProvider == null) { - if (!zkHosts.isEmpty()) { - stateProvider = - ClusterStateProvider.newZkClusterStateProvider(zkHosts, Builder.this.zkChroot); - } else if (!this.solrUrls.isEmpty()) { + if (!zkHosts.isEmpty() && !solrUrls.isEmpty()) { + throw new IllegalArgumentException( + "Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); + } else if (!zkHosts.isEmpty()) { + stateProvider = ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot); + if (stateProvider instanceof SolrZkClientTimeoutAware) { + var timeoutAware = (SolrZkClientTimeoutAware) stateProvider; + timeoutAware.setZkClientTimeout(zkClientTimeout); + timeoutAware.setZkConnectTimeout(zkConnectTimeout); + } + } else if (!solrUrls.isEmpty()) { try { stateProvider = new Http2ClusterStateProvider(solrUrls, httpClient); } catch (Exception e) { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java index 4a94359fca5..c1bc6811411 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java @@ -26,6 +26,7 @@ import org.apache.http.NoHttpResponseException; import org.apache.http.client.HttpClient; import org.apache.http.conn.ConnectTimeoutException; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout.SolrZkClientTimeoutAware; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.DocCollection; @@ -61,31 +62,7 @@ public class CloudLegacySolrClient extends CloudSolrClient { */ protected CloudLegacySolrClient(Builder builder) { super(builder.shardLeadersOnly, builder.parallelUpdates, builder.directUpdatesToLeadersOnly); - if (builder.stateProvider == null) { - if (builder.zkHosts != null && builder.solrUrls != null) { - throw new IllegalArgumentException( - "Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); - } - if (builder.zkHosts != null) { - this.stateProvider = - ClusterStateProvider.newZkClusterStateProvider(builder.zkHosts, builder.zkChroot); - } else if (builder.solrUrls != null && !builder.solrUrls.isEmpty()) { - try { - this.stateProvider = new HttpClusterStateProvider(builder.solrUrls, builder.httpClient); - } catch (Exception e) { - throw new RuntimeException( - "Couldn't initialize a HttpClusterStateProvider (is/are the " - + "Solr server(s), " - + builder.solrUrls - + ", down?)", - e); - } - } else { - throw new IllegalArgumentException("Both zkHosts and solrUrl cannot be null."); - } - } else { - this.stateProvider = builder.stateProvider; - } + this.stateProvider = builder.stateProvider; this.retryExpiryTimeNano = builder.retryExpiryTimeNano; this.defaultCollection = builder.defaultCollection; this.collectionStateCache.timeToLiveMs = @@ -194,6 +171,8 @@ public static class Builder extends SolrClientBuilder { protected long retryExpiryTimeNano = TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); // 3 seconds or 3 million nanos protected ClusterStateProvider stateProvider; + private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; + private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; /** Constructor for use by subclasses. This constructor was public prior to version 9.0 */ protected Builder() {} @@ -370,11 +349,41 @@ public Builder withDefaultCollection(String collection) { return this; } + /** + * Sets the Zk connection timeout + * + * @param zkConnectTimeout timeout value + * @param unit time unit + */ + public Builder withZkConnectTimeout(int zkConnectTimeout, TimeUnit unit) { + this.zkConnectTimeout = Math.toIntExact(unit.toMillis(zkConnectTimeout)); + return this; + } + + /** + * Sets the Zk client session timeout + * + * @param zkClientTimeout timeout value + * @param unit time unit + */ + public Builder withZkClientTimeout(int zkClientTimeout, TimeUnit unit) { + this.zkClientTimeout = Math.toIntExact(unit.toMillis(zkClientTimeout)); + return this; + } + /** Create a {@link CloudLegacySolrClient} based on the provided configuration. */ public CloudLegacySolrClient build() { if (stateProvider == null) { - if (!zkHosts.isEmpty()) { + if (!zkHosts.isEmpty() && !solrUrls.isEmpty()) { + throw new IllegalArgumentException( + "Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); + } else if (!zkHosts.isEmpty()) { this.stateProvider = ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot); + if (stateProvider instanceof SolrZkClientTimeoutAware) { + var timeoutAware = (SolrZkClientTimeoutAware) stateProvider; + timeoutAware.setZkClientTimeout(zkClientTimeout); + timeoutAware.setZkConnectTimeout(zkConnectTimeout); + } } else if (!this.solrUrls.isEmpty()) { try { stateProvider = new HttpClusterStateProvider(solrUrls, httpClient); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrZkClientTimeout.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrZkClientTimeout.java new file mode 100644 index 00000000000..0a0cfd7fdad --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrZkClientTimeout.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.impl; + +public class SolrZkClientTimeout { + public static final int DEFAULT_ZK_CLIENT_TIMEOUT = Integer.getInteger("zkClientTimeout", 30000); + public static final int DEFAULT_ZK_CONNECT_TIMEOUT = + Integer.getInteger("zkConnectTimeout", 15000); + + interface SolrZkClientTimeoutAware { + void setZkClientTimeout(int zkClientTimeout); + + void setZkConnectTimeout(int zkConnectTimeout); + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java index 3384a03d0be..157f0a935e2 100644 --- a/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java +++ b/solr/test-framework/src/java/org/apache/solr/util/TestHarness.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; @@ -190,7 +191,7 @@ public static NodeConfig buildTestNodeConfig(Path solrHome) { ? null : new CloudConfig.CloudConfigBuilder( System.getProperty("host"), Integer.getInteger("hostPort", 8983)) - .setZkClientTimeout(Integer.getInteger("zkClientTimeout", 30000)) + .setZkClientTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT) .setZkHost(System.getProperty("zkHost")) .build(); From 3b2b1b7f790986c84671c505e8055afb4dc59c76 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Mon, 14 Aug 2023 11:17:13 -0400 Subject: [PATCH 003/160] SOLR-16926: Give override for EmbeddedZK bind host (#1836) --- solr/CHANGES.txt | 4 ++ solr/bin/solr | 4 ++ solr/bin/solr.cmd | 4 ++ solr/bin/solr.in.cmd | 2 + solr/bin/solr.in.sh | 2 + .../org/apache/solr/cloud/SolrZkServer.java | 41 ++++++----- .../org/apache/solr/cloud/ZkController.java | 34 +-------- .../org/apache/solr/util/AddressUtils.java | 64 +++++++++++++++++ .../docker/templates/Dockerfile.body.template | 3 +- .../cloud_multi_node_embedded_zk/test.sh | 70 +++++++++++++++++++ .../cases/prometheus-exporter-cloud/test.sh | 51 ++++++++++++++ .../test.sh | 0 solr/server/solr/zoo.cfg | 3 - .../deployment-guide/pages/securing-solr.adoc | 13 ++++ .../pages/major-changes-in-solr-9.adoc | 4 ++ 15 files changed, 242 insertions(+), 57 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/util/AddressUtils.java create mode 100755 solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh create mode 100755 solr/docker/tests/cases/prometheus-exporter-cloud/test.sh rename solr/docker/tests/cases/{prometheus-exporter => prometheus-exporter-standalone}/test.sh (100%) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9951772eecc..a0ff03bc0bc 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -91,6 +91,10 @@ Improvements * SOLR-14667: Make zkClientTimeout consistent and based on a system property. The defauls values are stored in a single place referenced everywhere and they are based on system properties (Alex Deparvu) +* SOLR-16926: The embedded Zookeeper's bind host can now be overridden, but still defaults to "127.0.0.1". + This is useful when using the ZkCli on a remote Solr using the embedded ZK, or Solr running in a Docker container. + The SOLR_ZK_EMBEDDED_HOST envVar or -Dsolr.zk.embedded.host sysProp control this bind address. (Houston Putman) + Optimizations --------------------- diff --git a/solr/bin/solr b/solr/bin/solr index 3ff7832037a..bae4b722432 100644 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -1494,6 +1494,10 @@ if [ -n "${SOLR_JETTY_HOST:-}" ]; then SOLR_OPTS+=("-Dsolr.jetty.host=$SOLR_JETTY_HOST") fi +if [ -n "${SOLR_ZK_EMBEDDED_HOST:-}" ]; then + SOLR_OPTS+=("-Dsolr.zk.embedded.host=$SOLR_ZK_EMBEDDED_HOST") +fi + : "${STOP_PORT:=$((SOLR_PORT - 1000))}" if [ "$SCRIPT_CMD" == "start" ] || [ "$SCRIPT_CMD" == "restart" ] ; then diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 61414f807da..4055bad6d00 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -955,6 +955,10 @@ IF DEFINED SOLR_JETTY_HOST ( set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.jetty.host=%SOLR_JETTY_HOST%" ) +IF DEFINED SOLR_ZK_EMBEDDED_HOST ( + set "SOLR_OPTS=%SOLR_OPTS% -Dsolr.zk.embedded.host=%SOLR_ZK_EMBEDDED_HOST%" +) + IF "%SCRIPT_CMD%"=="start" ( REM see if Solr is already running using netstat For /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^| find ":%SOLR_PORT% "') do ( diff --git a/solr/bin/solr.in.cmd b/solr/bin/solr.in.cmd index 0ca0c12c3d5..5f982cb16b5 100755 --- a/solr/bin/solr.in.cmd +++ b/solr/bin/solr.in.cmd @@ -130,6 +130,8 @@ REM set this value as narrowly as required before going to production. In REM environments where security is not a concern, 0.0.0.0 can be used to allow REM Solr to accept connections on all network interfaces. REM set SOLR_JETTY_HOST=127.0.0.1 +REM Sets the network interface the Embedded ZK binds to. +REM set SOLR_ZK_EMBEDDED_HOST=127.0.0.1 REM Restrict access to solr by IP address. REM Specify a comma-separated list of addresses or networks, for example: diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh index b1dc6ab5ab4..117ef1761a9 100644 --- a/solr/bin/solr.in.sh +++ b/solr/bin/solr.in.sh @@ -154,6 +154,8 @@ # environments where security is not a concern, 0.0.0.0 can be used to allow # Solr to accept connections on all network interfaces. #SOLR_JETTY_HOST="127.0.0.1" +# Sets the network interface the Embedded ZK binds to. +#SOLR_ZK_EMBEDDED_HOST="127.0.0.1" # Enables HTTPS. It is implictly true if you set SOLR_SSL_KEY_STORE. Use this config # to enable https module with custom jetty configuration. diff --git a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java index f340f4d6c4d..ee7a14733ed 100644 --- a/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java +++ b/solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java @@ -20,9 +20,7 @@ import java.io.IOException; import java.io.Reader; import java.lang.invoke.MethodHandles; -import java.net.InetAddress; import java.net.InetSocketAddress; -import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -30,6 +28,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.solr.common.SolrException; import org.apache.solr.servlet.SolrDispatchFilter; +import org.apache.solr.util.AddressUtils; import org.apache.zookeeper.server.ServerConfig; import org.apache.zookeeper.server.ZooKeeperServerMain; import org.apache.zookeeper.server.quorum.QuorumPeer; @@ -72,7 +71,15 @@ public String getClientString() { if (zkRun == null) return null; InetSocketAddress addr = zkProps.getClientPortAddress(); - return addr.getHostString() + ":" + addr.getPort(); + String hostName; + // We cannot advertise 0.0.0.0, so choose the best host to advertise + // (the same that the Solr Node defaults to) + if (addr.getAddress().isAnyLocalAddress()) { + hostName = AddressUtils.getHostToAdvertise(); + } else { + hostName = addr.getAddress().getHostAddress(); + } + return hostName + ":" + addr.getPort(); } public void parseConfig() { @@ -101,6 +108,10 @@ public void parseConfig() { try { props = SolrZkServerProps.getProperties(zooCfgPath); SolrZkServerProps.injectServers(props, zkRun, zkHost); + // This is the address that the embedded Zookeeper will bind to. Like Solr, it defaults to + // "127.0.0.1". + props.setProperty( + "clientPortAddress", System.getProperty("solr.zk.embedded.host", "127.0.0.1")); if (props.getProperty("clientPort") == null) { props.setProperty("clientPort", Integer.toString(solrPort + 1000)); } @@ -145,14 +156,16 @@ public void start() { if (zkProps.getServers().size() > 1) { if (log.isInfoEnabled()) { log.info( - "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}", - zkProps.getClientPortAddress().getPort()); + "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening on host {}", + zkProps.getClientPortAddress().getPort(), + zkProps.getClientPortAddress().getAddress().getHostAddress()); } } else { if (log.isInfoEnabled()) { log.info( - "STARTING EMBEDDED STANDALONE ZOOKEEPER SERVER at port {}", - zkProps.getClientPortAddress().getPort()); + "STARTING EMBEDDED ENSEMBLE ZOOKEEPER SERVER at port {}, listening on host {}", + zkProps.getClientPortAddress().getPort(), + zkProps.getClientPortAddress().getAddress().getHostAddress()); } } @@ -268,20 +281,6 @@ public void setDataDir(File dataDir) { this.dataDir = dataDir; } - public void setClientPort(int clientPort) { - if (clientPortAddress != null) { - try { - this.clientPortAddress = - new InetSocketAddress( - InetAddress.getByName(clientPortAddress.getHostName()), clientPort); - } catch (UnknownHostException e) { - throw new RuntimeException(e); - } - } else { - this.clientPortAddress = new InetSocketAddress(clientPort); - } - } - /** * Parse config from a Properties. * diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index c93a7dfef99..25f097cc8c8 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -31,15 +31,11 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.Array; -import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -121,6 +117,7 @@ import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.update.UpdateLog; +import org.apache.solr.util.AddressUtils; import org.apache.solr.util.RTimer; import org.apache.solr.util.RefCounted; import org.apache.zookeeper.CreateMode; @@ -875,35 +872,8 @@ public SolrCloudManager getSolrCloudManager() { // normalize host removing any url scheme. // input can be null, host, or url_prefix://host private String normalizeHostName(String host) { - if (host == null || host.length() == 0) { - String hostaddress; - try { - hostaddress = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - hostaddress = "127.0.0.1"; // cannot resolve system hostname, fall through - } - // Re-get the IP again for "127.0.0.1", the other case we trust the hosts - // file is right. - if ("127.0.0.1".equals(hostaddress)) { - Enumeration netInterfaces = null; - try { - netInterfaces = NetworkInterface.getNetworkInterfaces(); - while (netInterfaces.hasMoreElements()) { - NetworkInterface ni = netInterfaces.nextElement(); - Enumeration ips = ni.getInetAddresses(); - while (ips.hasMoreElements()) { - InetAddress ip = ips.nextElement(); - if (ip.isSiteLocalAddress()) { - hostaddress = ip.getHostAddress(); - } - } - } - } catch (Exception e) { - log.error("Error while looking for a better host name than 127.0.0.1", e); - } - } - host = hostaddress; + host = AddressUtils.getHostToAdvertise(); } else { if (URLUtil.hasScheme(host)) { host = URLUtil.removeScheme(host); diff --git a/solr/core/src/java/org/apache/solr/util/AddressUtils.java b/solr/core/src/java/org/apache/solr/util/AddressUtils.java new file mode 100644 index 00000000000..ca03e26cf85 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/AddressUtils.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util; + +import java.lang.invoke.MethodHandles; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.UnknownHostException; +import java.util.Enumeration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Simple utilities for working Hostname/IP Addresses */ +public final class AddressUtils { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + // normalize host removing any url scheme. + // input can be null, host, or url_prefix://host + public static String getHostToAdvertise() { + String hostaddress; + try { + hostaddress = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + hostaddress = + InetAddress.getLoopbackAddress() + .getHostAddress(); // cannot resolve system hostname, fall through + } + // Re-get the IP again for "127.0.0.1", the other case we trust the hosts + // file is right. + if ("127.0.0.1".equals(hostaddress)) { + try { + var netInterfaces = NetworkInterface.getNetworkInterfaces(); + while (netInterfaces.hasMoreElements()) { + NetworkInterface ni = netInterfaces.nextElement(); + Enumeration ips = ni.getInetAddresses(); + while (ips.hasMoreElements()) { + InetAddress ip = ips.nextElement(); + if (ip.isSiteLocalAddress()) { + hostaddress = ip.getHostAddress(); + } + } + } + } catch (Exception e) { + log.error("Error while looking for a better host name than 127.0.0.1", e); + } + } + return hostaddress; + } +} diff --git a/solr/docker/templates/Dockerfile.body.template b/solr/docker/templates/Dockerfile.body.template index 7282642a73f..d754b9d7e42 100644 --- a/solr/docker/templates/Dockerfile.body.template +++ b/solr/docker/templates/Dockerfile.body.template @@ -45,7 +45,8 @@ ENV SOLR_USER="solr" \ SOLR_PID_DIR=/var/solr \ SOLR_LOGS_DIR=/var/solr/logs \ LOG4J_PROPS=/var/solr/log4j2.xml \ - SOLR_JETTY_HOST="0.0.0.0" + SOLR_JETTY_HOST="0.0.0.0" \ + SOLR_ZK_EMBEDDED_HOST="0.0.0.0" RUN set -ex; \ groupadd -r --gid "$SOLR_GID" "$SOLR_GROUP"; \ diff --git a/solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh b/solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh new file mode 100755 index 00000000000..2ee47103264 --- /dev/null +++ b/solr/docker/tests/cases/cloud_multi_node_embedded_zk/test.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +TEST_DIR="${TEST_DIR:-$(dirname -- "${BASH_SOURCE[0]}")}" +source "${TEST_DIR}/../../shared.sh" + +echo "Running base solr node w/embeddedZk - $container_name" +docker run --name "${container_name}" -d "$tag" solr-fg -c + +wait_for_container_and_solr "${container_name}" + +solr_ip=$(docker inspect --format="{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" "${container_name}") + + +container_cleanup "${container_name}-2" + +echo "Running additional solr node - $container_name-2" +docker run --name "$container_name-2" -d \ + --env "ZK_HOST=${solr_ip}:9983" \ + "$tag" solr-fg -c + +wait_for_container_and_solr "${container_name}-2" + +echo "Check live nodes" +data=$(docker exec --user=solr "${container_name}-2" wget -q -O - 'http://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS') + +if ! grep -q "${solr_ip}:8983" <<<"$data"; then + echo "Test $TEST_NAME $tag failed; could not find first solr node in cluster state of second node" + echo "$data" + exit 1 +fi + +echo "Creating distributed collection" +data=$(docker exec --user=solr "$container_name" solr create -c test -rf 1 -s 2) + +if ! grep -q "Created collection 'test'" <<<"$data"; then + echo "Test $TEST_NAME $tag failed; could not create distributed collection" + echo "$data" + exit 1 +fi + +echo "Submitting Solr query" +data=$(docker exec --user=solr "${container_name}-2" wget -q -O - 'http://localhost:8983/solr/test/select?q=*:*') + +if ! grep -q '"numFound":0' <<<"$data"; then + echo "Test $TEST_NAME $tag failed; could not query distributed collection" + echo "$data" + exit 1 +fi + + +container_cleanup "${container_name}-2" +container_cleanup "$container_name" + +echo "Test $TEST_NAME $tag succeeded" diff --git a/solr/docker/tests/cases/prometheus-exporter-cloud/test.sh b/solr/docker/tests/cases/prometheus-exporter-cloud/test.sh new file mode 100755 index 00000000000..6407dfeb272 --- /dev/null +++ b/solr/docker/tests/cases/prometheus-exporter-cloud/test.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +TEST_DIR="${TEST_DIR:-$(dirname -- "${BASH_SOURCE[0]}")}" +source "${TEST_DIR}/../../shared.sh" + +container_cleanup "${container_name}-solr" + +echo "Running $container_name" +docker run --name "${container_name}-solr" -d "$tag" solr-fg -c + +wait_for_container_and_solr "${container_name}-solr" + +solr_ip=$(docker inspect --format="{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" "${container_name}-solr") + +docker run --name "$container_name" -d \ + --env "ZK_HOST=${solr_ip}:9983" \ + --env "SCRAPE_INTERVAL=1" \ + --env "CLUSTER_ID=myCluster" \ + "$tag" "solr-exporter" + +wait_for_container_and_solr_exporter "${container_name}" + +echo "Checking prometheus data" +data=$(docker exec --user=solr "$container_name" wget -q -O - 'http://localhost:8989/metrics') + +if ! grep -E -q "solr_collections_live_nodes{zk_host=\"${solr_ip}:9983\",cluster_id=\"myCluster\",} 1.0" <<<"$data"; then + echo "Test $TEST_NAME $tag failed; did not find correct data" + echo "$data" + exit 1 +fi + +container_cleanup "${container_name}-solr" +container_cleanup "$container_name" + +echo "Test $TEST_NAME $tag succeeded" diff --git a/solr/docker/tests/cases/prometheus-exporter/test.sh b/solr/docker/tests/cases/prometheus-exporter-standalone/test.sh similarity index 100% rename from solr/docker/tests/cases/prometheus-exporter/test.sh rename to solr/docker/tests/cases/prometheus-exporter-standalone/test.sh diff --git a/solr/server/solr/zoo.cfg b/solr/server/solr/zoo.cfg index 4ef8dcea2ae..7c32425e718 100644 --- a/solr/server/solr/zoo.cfg +++ b/solr/server/solr/zoo.cfg @@ -11,9 +11,6 @@ syncLimit=5 # dataDir=/opt/zookeeper/data # NOTE: Solr defaults the dataDir to /zoo_data -# the address that embedded zookeeper will bind to -clientPortAddress=127.0.0.1 - # the port at which the clients will connect # clientPort=2181 # NOTE: Solr sets this based on zkRun / zkHost params diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc index 3732cea17cb..9a4fdea3898 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/securing-solr.adoc @@ -115,4 +115,17 @@ This can be done by setting a `SOLR_JETTY_HOST` value in your environment's "inc ---- SOLR_JETTY_HOST="0.0.0.0" ---- + +The same setting is also available as the `-Dsolr.jetty.host` System Property. + +The same is true for the embedded Zookeeper, if it is run with Solr. +By default, the embedded Zookeeper only listens on the loopback interface ("127.0.0.1") +The bind host is controlled via the `SOLR_ZK_EMBEDDED_HOST` value in your environment's "include script" (`solr.in.sh` or `solr.in.cmd`): + +[source,bash] + ---- + SOLR_ZK_EMBEDDED_HOST="0.0.0.0" + ---- + +The same setting is also available as the `-Dsolr.zk.embedded.host` System Property. // end::security-network-binding-1[] diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc index 750919eabef..2e670ca9f06 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc @@ -76,6 +76,10 @@ Upgrading existing clouds and use-cases that have custom configSets will not be * The default `minimalFreeDiskGB` value has been lowered from `20GB` to `5GB` when using the xref:configuration-guide:replica-placement-plugins.adoc#affinityplacementfactory[AffinityPlacementPlugin]. Therefore, when using the default settings, nodes that were previously excluded from Replica placements due to low available disk space may be selected after upgrading. +=== Embedded Zookeeper +* The Embedded Zookeeper can now be configured to listen to (or bind to) more hosts than just `localhost`, +see the xref:deployment-guide:securing-solr.adoc#network-configuration[Network Configuration documentation] for more information. + == Solr 9.3 === Binary Releases From b0d3085b83994b1636e455f95e6858aab7039e29 Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Tue, 15 Aug 2023 20:52:07 +0200 Subject: [PATCH 004/160] Update io.netty:* to v4.1.96.Final (#1813) --- .../netty-buffer-4.1.94.Final.jar.sha1 | 1 - .../netty-buffer-4.1.96.Final.jar.sha1 | 1 + .../netty-codec-4.1.94.Final.jar.sha1 | 1 - .../netty-codec-4.1.96.Final.jar.sha1 | 1 + .../netty-codec-http-4.1.94.Final.jar.sha1 | 1 - .../netty-codec-http-4.1.96.Final.jar.sha1 | 1 + .../netty-codec-http2-4.1.94.Final.jar.sha1 | 1 - .../netty-codec-http2-4.1.96.Final.jar.sha1 | 1 + .../netty-codec-socks-4.1.94.Final.jar.sha1 | 1 - .../netty-codec-socks-4.1.96.Final.jar.sha1 | 1 + .../netty-common-4.1.94.Final.jar.sha1 | 1 - .../netty-common-4.1.96.Final.jar.sha1 | 1 + .../netty-handler-4.1.94.Final.jar.sha1 | 1 - .../netty-handler-4.1.96.Final.jar.sha1 | 1 + .../netty-handler-proxy-4.1.94.Final.jar.sha1 | 1 - .../netty-handler-proxy-4.1.96.Final.jar.sha1 | 1 + .../netty-resolver-4.1.94.Final.jar.sha1 | 1 - .../netty-resolver-4.1.96.Final.jar.sha1 | 1 + .../netty-transport-4.1.94.Final.jar.sha1 | 1 - .../netty-transport-4.1.96.Final.jar.sha1 | 1 + ...nsport-classes-epoll-4.1.94.Final.jar.sha1 | 1 - ...nsport-classes-epoll-4.1.96.Final.jar.sha1 | 1 + ...ansport-native-epoll-4.1.94.Final.jar.sha1 | 1 - ...ansport-native-epoll-4.1.96.Final.jar.sha1 | 1 + ...t-native-unix-common-4.1.94.Final.jar.sha1 | 1 - ...t-native-unix-common-4.1.96.Final.jar.sha1 | 1 + versions.lock | 26 +++++++++---------- versions.props | 2 +- 28 files changed, 27 insertions(+), 27 deletions(-) delete mode 100644 solr/licenses/netty-buffer-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-buffer-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-http-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-http-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-http2-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-http2-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-codec-socks-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-codec-socks-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-common-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-common-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-handler-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-handler-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-handler-proxy-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-handler-proxy-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-resolver-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-resolver-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-classes-epoll-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-classes-epoll-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-native-epoll-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-native-unix-common-4.1.94.Final.jar.sha1 create mode 100644 solr/licenses/netty-transport-native-unix-common-4.1.96.Final.jar.sha1 diff --git a/solr/licenses/netty-buffer-4.1.94.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.94.Final.jar.sha1 deleted file mode 100644 index 5ab0cfb858d..00000000000 --- a/solr/licenses/netty-buffer-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -eec248b26f16e888688e5bb37b7eeda76b78d2f7 diff --git a/solr/licenses/netty-buffer-4.1.96.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..c2e6b557029 --- /dev/null +++ b/solr/licenses/netty-buffer-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +4b80fffbe77485b457bf844289bf1801f61b9e91 diff --git a/solr/licenses/netty-codec-4.1.94.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.94.Final.jar.sha1 deleted file mode 100644 index 863ee90d58c..00000000000 --- a/solr/licenses/netty-codec-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c70ef20ca338558147887df60f46341bc47f6900 diff --git a/solr/licenses/netty-codec-4.1.96.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..2b1de703195 --- /dev/null +++ b/solr/licenses/netty-codec-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +9cfe430f8b14e7ba86969d8e1126aa0aae4d18f0 diff --git a/solr/licenses/netty-codec-http-4.1.94.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.94.Final.jar.sha1 deleted file mode 100644 index cdc34be07ff..00000000000 --- a/solr/licenses/netty-codec-http-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9e5404764092c1f6305ad5719078f46ab228d587 diff --git a/solr/licenses/netty-codec-http-4.1.96.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..dbcc61d860e --- /dev/null +++ b/solr/licenses/netty-codec-http-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +a4d0d95df5026965c454902ef3d6d84b81f89626 diff --git a/solr/licenses/netty-codec-http2-4.1.94.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.94.Final.jar.sha1 deleted file mode 100644 index 957bf6c2893..00000000000 --- a/solr/licenses/netty-codec-http2-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f651595784d6cca4cbca6a8ad74c48fceed6cea8 diff --git a/solr/licenses/netty-codec-http2-4.1.96.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..226c1b7d206 --- /dev/null +++ b/solr/licenses/netty-codec-http2-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +cc8baf4ff67c1bcc0cde60bc5c2bb9447d92d9e6 diff --git a/solr/licenses/netty-codec-socks-4.1.94.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.94.Final.jar.sha1 deleted file mode 100644 index 24cf1720029..00000000000 --- a/solr/licenses/netty-codec-socks-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b9192c7cda295d75f236a13a0b1f5a008f05d516 diff --git a/solr/licenses/netty-codec-socks-4.1.96.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..7928c87eeb5 --- /dev/null +++ b/solr/licenses/netty-codec-socks-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +f53c52dbddaa4a02a51430405792d3f30a89b147 diff --git a/solr/licenses/netty-common-4.1.94.Final.jar.sha1 b/solr/licenses/netty-common-4.1.94.Final.jar.sha1 deleted file mode 100644 index 532256b817b..00000000000 --- a/solr/licenses/netty-common-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ad4ecf779ebc794cd351f57792f56ea01387b868 diff --git a/solr/licenses/netty-common-4.1.96.Final.jar.sha1 b/solr/licenses/netty-common-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..cb5590c8416 --- /dev/null +++ b/solr/licenses/netty-common-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +d10c167623cbc471753f950846df241d1021655c diff --git a/solr/licenses/netty-handler-4.1.94.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.94.Final.jar.sha1 deleted file mode 100644 index e05fe7897d1..00000000000 --- a/solr/licenses/netty-handler-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cd9121ce24d6d3f2898946d04b0ef3ec548b00b4 diff --git a/solr/licenses/netty-handler-4.1.96.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..a1f0cd55210 --- /dev/null +++ b/solr/licenses/netty-handler-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +7840d7523d709e02961b647546f9d9dde1699306 diff --git a/solr/licenses/netty-handler-proxy-4.1.94.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.94.Final.jar.sha1 deleted file mode 100644 index bbaaa1fab18..00000000000 --- a/solr/licenses/netty-handler-proxy-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -26ba9d30b8f7b095155b9ac63378d6d9386d85c3 diff --git a/solr/licenses/netty-handler-proxy-4.1.96.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..aa3d220477a --- /dev/null +++ b/solr/licenses/netty-handler-proxy-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +dcabd63f4aaec2b4cad7588bfdd4cd2c82287e38 diff --git a/solr/licenses/netty-resolver-4.1.94.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.94.Final.jar.sha1 deleted file mode 100644 index edc89616fce..00000000000 --- a/solr/licenses/netty-resolver-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e96f649e8e9dcb29a1f8e95328b99c9eb6cf76c2 diff --git a/solr/licenses/netty-resolver-4.1.96.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..9f0e6ac60de --- /dev/null +++ b/solr/licenses/netty-resolver-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +0e51db5568a881e0f9b013b35617c597dc32f130 diff --git a/solr/licenses/netty-transport-4.1.94.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.94.Final.jar.sha1 deleted file mode 100644 index 0934b80923d..00000000000 --- a/solr/licenses/netty-transport-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ec783a737f96991a87b1d5794e2f9eb2024d708a diff --git a/solr/licenses/netty-transport-4.1.96.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..7d0cc889525 --- /dev/null +++ b/solr/licenses/netty-transport-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +dbd15ca244be28e1a98ed29b9d755edbfa737e02 diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.94.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.94.Final.jar.sha1 deleted file mode 100644 index 120f42fad3b..00000000000 --- a/solr/licenses/netty-transport-classes-epoll-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -240e36cd5c2ffaf655913f8857f2d58b26394679 diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.96.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..a05cc215dcc --- /dev/null +++ b/solr/licenses/netty-transport-classes-epoll-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +b0369501645f6e71f89ff7f77b5c5f52510a2e31 diff --git a/solr/licenses/netty-transport-native-epoll-4.1.94.Final.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.94.Final.jar.sha1 deleted file mode 100644 index b9e9bfe13df..00000000000 --- a/solr/licenses/netty-transport-native-epoll-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1f77c54f1877d9ed51f207eb64a5634bdcc7efa1 diff --git a/solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..21a85fa4799 --- /dev/null +++ b/solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +9faf365396c933f1b39b60d129391c6c6c43fb86 diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.94.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.94.Final.jar.sha1 deleted file mode 100644 index 95d73ae8259..00000000000 --- a/solr/licenses/netty-transport-native-unix-common-4.1.94.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3fa5f9d04b6b782d869d6e0657d896eeadca5866 diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.96.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.96.Final.jar.sha1 new file mode 100644 index 00000000000..72071232dfb --- /dev/null +++ b/solr/licenses/netty-transport-native-unix-common-4.1.96.Final.jar.sha1 @@ -0,0 +1 @@ +daf8578cade63a01525ee9d70371fa78e6e91094 diff --git a/versions.lock b/versions.lock index ee128a6f95d..97a8faa8d31 100644 --- a/versions.lock +++ b/versions.lock @@ -108,19 +108,19 @@ io.grpc:grpc-rls:1.56.0 (1 constraints: 20100fa6) io.grpc:grpc-services:1.56.0 (1 constraints: 20100fa6) io.grpc:grpc-stub:1.56.0 (2 constraints: 5d159cd7) io.grpc:grpc-xds:1.56.0 (1 constraints: 20100fa6) -io.netty:netty-buffer:4.1.94.Final (10 constraints: ac994cf0) -io.netty:netty-codec:4.1.94.Final (5 constraints: 60455680) -io.netty:netty-codec-http:4.1.94.Final (3 constraints: 522408ed) -io.netty:netty-codec-http2:4.1.94.Final (1 constraints: ed0a94cb) -io.netty:netty-codec-socks:4.1.94.Final (1 constraints: 140fab6c) -io.netty:netty-common:4.1.94.Final (12 constraints: 24b37164) -io.netty:netty-handler:4.1.94.Final (3 constraints: 6f2b2d3c) -io.netty:netty-handler-proxy:4.1.94.Final (1 constraints: ed0a94cb) -io.netty:netty-resolver:4.1.94.Final (2 constraints: 581a1732) -io.netty:netty-transport:4.1.94.Final (9 constraints: f98bf530) -io.netty:netty-transport-classes-epoll:4.1.94.Final (1 constraints: b112f81e) -io.netty:netty-transport-native-epoll:4.1.94.Final (1 constraints: db0f188f) -io.netty:netty-transport-native-unix-common:4.1.94.Final (4 constraints: 583daf95) +io.netty:netty-buffer:4.1.96.Final (10 constraints: c0990f02) +io.netty:netty-codec:4.1.96.Final (5 constraints: 6a45dc83) +io.netty:netty-codec-http:4.1.96.Final (3 constraints: 582436ee) +io.netty:netty-codec-http2:4.1.96.Final (1 constraints: ed0a94cb) +io.netty:netty-codec-socks:4.1.96.Final (1 constraints: 160fb96c) +io.netty:netty-common:4.1.96.Final (12 constraints: 3cb3817d) +io.netty:netty-handler:4.1.96.Final (3 constraints: 732b553d) +io.netty:netty-handler-proxy:4.1.96.Final (1 constraints: ed0a94cb) +io.netty:netty-resolver:4.1.96.Final (2 constraints: 5c1a8332) +io.netty:netty-transport:4.1.96.Final (9 constraints: 0b8c6f3f) +io.netty:netty-transport-classes-epoll:4.1.96.Final (1 constraints: b312061f) +io.netty:netty-transport-native-epoll:4.1.96.Final (1 constraints: db0f188f) +io.netty:netty-transport-native-unix-common:4.1.96.Final (4 constraints: 5e3d1997) io.opencensus:opencensus-api:0.31.1 (5 constraints: 924d4692) io.opencensus:opencensus-contrib-http-util:0.31.1 (3 constraints: 7232a9fc) io.opencensus:opencensus-proto:0.2.0 (1 constraints: e60fd595) diff --git a/versions.props b/versions.props index afad823b7aa..9f7b6e400a5 100644 --- a/versions.props +++ b/versions.props @@ -22,7 +22,7 @@ commons-collections:commons-collections=3.2.2 commons-io:commons-io=2.11.0 io.dropwizard.metrics:*=4.2.19 io.grpc:grpc-*=1.56.0 -io.netty:*=4.1.94.Final +io.netty:*=4.1.96.Final io.opentelemetry:opentelemetry-bom-alpha=1.21.0-alpha io.opentracing:*=0.33.0 io.prometheus:*=0.16.0 From 194b2050c0b62094e746097c046f792682bf471d Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Tue, 15 Aug 2023 22:50:12 +0200 Subject: [PATCH 005/160] Update dependency org.apache.commons:commons-lang3 to v3.13.0 (#1814) --- solr/licenses/commons-lang3-3.12.0.jar.sha1 | 1 - solr/licenses/commons-lang3-3.13.0.jar.sha1 | 1 + versions.lock | 2 +- versions.props | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 solr/licenses/commons-lang3-3.12.0.jar.sha1 create mode 100644 solr/licenses/commons-lang3-3.13.0.jar.sha1 diff --git a/solr/licenses/commons-lang3-3.12.0.jar.sha1 b/solr/licenses/commons-lang3-3.12.0.jar.sha1 deleted file mode 100644 index 8846a21387e..00000000000 --- a/solr/licenses/commons-lang3-3.12.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c6842c86792ff03b9f1d1fe2aab8dc23aa6c6f0e diff --git a/solr/licenses/commons-lang3-3.13.0.jar.sha1 b/solr/licenses/commons-lang3-3.13.0.jar.sha1 new file mode 100644 index 00000000000..e2d4d379fe0 --- /dev/null +++ b/solr/licenses/commons-lang3-3.13.0.jar.sha1 @@ -0,0 +1 @@ +b7263237aa89c1f99b327197c41d0669707a462e diff --git a/versions.lock b/versions.lock index 97a8faa8d31..9556f8c0982 100644 --- a/versions.lock +++ b/versions.lock @@ -174,7 +174,7 @@ org.apache.commons:commons-compress:1.23.0 (3 constraints: fd1bd8d1) org.apache.commons:commons-configuration2:2.9.0 (1 constraints: 0d051036) org.apache.commons:commons-csv:1.9.0 (1 constraints: 610cfc01) org.apache.commons:commons-exec:1.3 (2 constraints: a41056b8) -org.apache.commons:commons-lang3:3.12.0 (5 constraints: 9b3e0e36) +org.apache.commons:commons-lang3:3.13.0 (5 constraints: 9c3eb936) org.apache.commons:commons-math3:3.6.1 (5 constraints: f431a634) org.apache.commons:commons-text:1.10.0 (1 constraints: d911adf8) org.apache.curator:curator-client:4.3.0 (2 constraints: e214cba2) diff --git a/versions.props b/versions.props index 9f7b6e400a5..71c8f2671d0 100644 --- a/versions.props +++ b/versions.props @@ -38,7 +38,7 @@ org.apache.commons:commons-collections4=4.4 org.apache.commons:commons-compress=1.23.0 org.apache.commons:commons-configuration2=2.9.0 org.apache.commons:commons-exec=1.3 -org.apache.commons:commons-lang3=3.12.0 +org.apache.commons:commons-lang3=3.13.0 org.apache.commons:commons-math3=3.6.1 org.apache.curator:*=4.3.0 org.apache.hadoop.thirdparty:*=1.1.1 From ef3e4498689a4bb5ffbfd05ef6854e12cfa895e9 Mon Sep 17 00:00:00 2001 From: Jonathon Henderson <8791010+jrhenderson1988@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:35:30 +0100 Subject: [PATCH 006/160] SOLR-16928: Add missing unit tests for RegexFileFilter and StrUtils.toLower (#1837) * SOLR-16928: Add missing unit tests for RegexFileFilter and StrUtils.toLower * SOLR-16928: Add license to RegexFileFilterTest --- .../apache/solr/util/RegexFileFilterTest.java | 49 +++++++++++++++++++ .../test/org/apache/solr/util/TestUtils.java | 7 +++ 2 files changed, 56 insertions(+) create mode 100644 solr/core/src/test/org/apache/solr/util/RegexFileFilterTest.java diff --git a/solr/core/src/test/org/apache/solr/util/RegexFileFilterTest.java b/solr/core/src/test/org/apache/solr/util/RegexFileFilterTest.java new file mode 100644 index 00000000000..8e69018c941 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/RegexFileFilterTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util; + +import java.io.File; +import org.apache.solr.SolrTestCase; +import org.junit.Test; + +public class RegexFileFilterTest extends SolrTestCase { + + private final RegexFileFilter endsWithDotTxt = new RegexFileFilter(".*\\.txt$"); + private final RegexFileFilter alphaWithTxtOrPdfExt = new RegexFileFilter("^[a-z]+\\.(txt|pdf)$"); + + @Test + public void testAcceptTrue() { + assertTrue(endsWithDotTxt.accept(new File("/foo/bar/baz.txt"))); + assertTrue(endsWithDotTxt.accept(new File("/baz.txt"))); + assertTrue(endsWithDotTxt.accept(new File("~/baz.txt"))); + assertTrue(endsWithDotTxt.accept(new File("~/1234-abc_.txt"))); + assertTrue(endsWithDotTxt.accept(new File(".txt"))); + assertTrue(alphaWithTxtOrPdfExt.accept(new File("/foo/bar.txt"))); + assertTrue(alphaWithTxtOrPdfExt.accept(new File("/foo/baz.pdf"))); + } + + @Test + public void testAcceptFalse() { + assertFalse(endsWithDotTxt.accept(new File("/foo/bar.tx"))); + assertFalse(endsWithDotTxt.accept(new File("/foo/bar.txts"))); + assertFalse(endsWithDotTxt.accept(new File("/foo/bar.exe"))); + assertFalse(alphaWithTxtOrPdfExt.accept(new File("/foo/bar/b4z.txt"))); + assertFalse(alphaWithTxtOrPdfExt.accept(new File("/foo/bar/baz.jpg"))); + assertFalse(alphaWithTxtOrPdfExt.accept(new File("~/foo-bar.txt"))); + assertFalse(alphaWithTxtOrPdfExt.accept(new File("~/foobar123.txt"))); + } +} diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.java index c2a0f85d146..90b118b7f1c 100644 --- a/solr/core/src/test/org/apache/solr/util/TestUtils.java +++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java @@ -109,6 +109,13 @@ public void testSplitEscaping() { assertEquals("/h/s", arr.get(0)); } + public void testToLower() { + assertEquals(List.of(), StrUtils.toLower(List.of())); + assertEquals(List.of(""), StrUtils.toLower(List.of(""))); + assertEquals(List.of("foo"), StrUtils.toLower(List.of("foo"))); + assertEquals(List.of("bar", "baz-123"), StrUtils.toLower(List.of("BAR", "Baz-123"))); + } + public void testNamedLists() { SimpleOrderedMap map = new SimpleOrderedMap<>(); map.add("test", 10); From 21740acc3392645ffc08b4dd07e26e8235a03880 Mon Sep 17 00:00:00 2001 From: Alex D Date: Wed, 16 Aug 2023 11:04:51 -0700 Subject: [PATCH 007/160] SOLR-16859 Missing Proxy support for Http2SolrClient (#1779) --- solr/CHANGES.txt | 2 + .../client/solrj/impl/Http2SolrClient.java | 52 ++++++++++ .../solrj/impl/Http2SolrClientProxyTest.java | 95 +++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientProxyTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a0ff03bc0bc..b7606af3ba7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -120,6 +120,8 @@ Bug Fixes * PR#1826: Allow looking up Solr Package repo when that URL references a raw repository.json hosted on Github when the file is JSON but the mimetype used is text/plain. (Eric Pugh) +* SOLR-16859: Missing Proxy support for Http2SolrClient (Alex Deparvu) + Dependency Upgrades --------------------- (No changes) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java index 648504a816a..25513a74dd4 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java @@ -76,7 +76,12 @@ import org.apache.solr.common.util.Utils; import org.eclipse.jetty.client.HttpClient; import org.eclipse.jetty.client.HttpClientTransport; +import org.eclipse.jetty.client.HttpProxy; +import org.eclipse.jetty.client.Origin.Address; +import org.eclipse.jetty.client.Origin.Protocol; import org.eclipse.jetty.client.ProtocolHandlers; +import org.eclipse.jetty.client.ProxyConfiguration; +import org.eclipse.jetty.client.Socks4Proxy; import org.eclipse.jetty.client.api.AuthenticationStore; import org.eclipse.jetty.client.api.Request; import org.eclipse.jetty.client.api.Response; @@ -281,6 +286,8 @@ private HttpClient createHttpClient(Builder builder) { if (builder.connectionTimeoutMillis != null) httpClient.setConnectTimeout(builder.connectionTimeoutMillis); + setupProxy(builder, httpClient); + try { httpClient.start(); } catch (Exception e) { @@ -291,6 +298,29 @@ private HttpClient createHttpClient(Builder builder) { return httpClient; } + private void setupProxy(Builder builder, HttpClient httpClient) { + if (builder.proxyHost == null) { + return; + } + Address address = new Address(builder.proxyHost, builder.proxyPort); + + final ProxyConfiguration.Proxy proxy; + if (builder.proxyIsSocks4) { + proxy = new Socks4Proxy(address, builder.proxyIsSecure); + } else { + final Protocol protocol; + if (builder.useHttp1_1) { + protocol = HttpClientTransportOverHTTP.HTTP11; + } else { + // see HttpClientTransportOverHTTP2#newOrigin + String protocolName = builder.proxyIsSecure ? "h2" : "h2c"; + protocol = new Protocol(List.of(protocolName), false); + } + proxy = new HttpProxy(address, builder.proxyIsSecure, protocol); + } + httpClient.getProxyConfiguration().addProxy(proxy); + } + @Override public void close() { // we wait for async requests, so far devs don't want to give sugar for this @@ -1018,6 +1048,10 @@ public static class Builder { protected ResponseParser responseParser; private Set urlParamNames; private CookieStore cookieStore = getDefaultCookieStore(); + private String proxyHost; + private int proxyPort; + private boolean proxyIsSocks4; + private boolean proxyIsSecure; public Builder() {} @@ -1241,6 +1275,24 @@ public Builder withCookieStore(CookieStore cookieStore) { this.cookieStore = cookieStore; return this; } + + /** + * Setup a proxy + * + * @param host The proxy host + * @param port The proxy port + * @param isSocks4 If true creates an SOCKS 4 proxy, otherwise creates an HTTP proxy + * @param isSecure If true enables the secure flag on the proxy + * @return this Builder + */ + public Builder withProxyConfiguration( + String host, int port, boolean isSocks4, boolean isSecure) { + this.proxyHost = host; + this.proxyPort = port; + this.proxyIsSocks4 = isSocks4; + this.proxyIsSecure = isSecure; + return this; + } } public Set getUrlParamNames() { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientProxyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientProxyTest.java new file mode 100644 index 00000000000..09f3601c47b --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/Http2SolrClientProxyTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.impl; + +import com.carrotsearch.randomizedtesting.RandomizedTest; +import java.nio.file.Path; +import java.util.Properties; +import org.apache.solr.SolrJettyTestBase; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.embedded.JettyConfig; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.BeforeClass; +import org.junit.Test; + +public class Http2SolrClientProxyTest extends SolrJettyTestBase { + + // TODO add SSL test + + @BeforeClass + public static void beforeTest() throws Exception { + RandomizedTest.assumeFalse(sslConfig.isSSLMode()); + + JettyConfig jettyConfig = + JettyConfig.builder().withSSLConfig(sslConfig.buildServerSSLConfig()).build(); + createAndStartJettyWithProxy(legacyExampleCollection1SolrHome(), new Properties(), jettyConfig); + } + + public static JettySolrRunner createAndStartJettyWithProxy( + String solrHome, Properties nodeProperties, JettyConfig jettyConfig) throws Exception { + + initCore(null, null, solrHome); + + Path coresDir = createTempDir().resolve("cores"); + + Properties props = new Properties(); + props.setProperty("name", DEFAULT_TEST_CORENAME); + props.setProperty("configSet", "collection1"); + props.setProperty("config", "${solrconfig:solrconfig.xml}"); + props.setProperty("schema", "${schema:schema.xml}"); + + writeCoreProperties(coresDir.resolve("core"), props, "RestTestBase"); + + Properties nodeProps = new Properties(nodeProperties); + nodeProps.setProperty("coreRootDirectory", coresDir.toString()); + nodeProps.setProperty("configSetBaseDir", solrHome); + + jetty = new JettySolrRunner(solrHome, nodeProps, jettyConfig, true); + jetty.start(); + port = jetty.getLocalPort(); + return jetty; + } + + /** Setup a simple http proxy and verify a request works */ + @Test + public void testProxy() throws Exception { + var proxy = jetty.getProxy(); + assertNotNull(proxy); + + String host = proxy.getUrl().getHost(); + String url = "http://" + host + ":" + (proxy.getUrl().getPort() + 10) + "/solr"; + + var builder = + new Http2SolrClient.Builder(url) + .withProxyConfiguration(host, proxy.getListenPort(), false, false); + + try (Http2SolrClient client = builder.build()) { + String id = "1234"; + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", id); + client.add(DEFAULT_TEST_COLLECTION_NAME, doc); + client.commit(DEFAULT_TEST_COLLECTION_NAME); + assertEquals( + 1, + client + .query(DEFAULT_TEST_COLLECTION_NAME, new SolrQuery("id:" + id)) + .getResults() + .getNumFound()); + } + } +} From c3aef6e8551bd757605fc308c4b19d9c784cfe68 Mon Sep 17 00:00:00 2001 From: Alex D Date: Wed, 16 Aug 2023 11:34:39 -0700 Subject: [PATCH 008/160] Refactor ZkCmdExecutor, move all ensureExists methods to ZkMaintenanceUtils (#1840) --- .../org/apache/solr/cloud/DistributedMap.java | 5 +- .../org/apache/solr/cloud/LeaderElector.java | 11 +-- .../solr/cloud/OverseerElectionContext.java | 5 +- .../cloud/ShardLeaderElectionContextBase.java | 4 +- .../org/apache/solr/cloud/ZkController.java | 17 ++-- .../apache/solr/cloud/ZkDistributedQueue.java | 6 +- .../admin/api/CreateCollectionAPI.java | 11 +-- .../schema/ManagedIndexSchemaFactory.java | 5 +- .../apache/solr/cloud/ZkSolrClientTest.java | 10 +- .../solr/common/cloud/ZkCmdExecutor.java | 60 ------------ .../solr/common/cloud/ZkMaintenanceUtils.java | 94 +++++++++++++++---- 11 files changed, 108 insertions(+), 120 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java b/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java index ece2589310a..323c378f71d 100644 --- a/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java +++ b/solr/core/src/java/org/apache/solr/cloud/DistributedMap.java @@ -22,7 +22,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NodeExistsException; @@ -42,9 +42,8 @@ public class DistributedMap { public DistributedMap(SolrZkClient zookeeper, String dir) { this.dir = dir; - ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zookeeper.getZkClientTimeout()); try { - cmdExecutor.ensureExists(dir, zookeeper); + ZkMaintenanceUtils.ensureExists(dir, zookeeper); } catch (KeeperException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java index 7b0de72eca5..6a003b19955 100644 --- a/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java +++ b/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java @@ -28,7 +28,7 @@ import org.apache.solr.common.AlreadyClosedException; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -61,8 +61,6 @@ public class LeaderElector { protected SolrZkClient zkClient; - private ZkCmdExecutor zkCmdExecutor; - private volatile ElectionContext context; private ElectionWatcher watcher; @@ -72,13 +70,11 @@ public class LeaderElector { public LeaderElector(SolrZkClient zkClient) { this.zkClient = zkClient; - zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout()); } public LeaderElector( SolrZkClient zkClient, ContextKey key, Map electionContexts) { this.zkClient = zkClient; - zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout()); this.electionContexts = electionContexts; this.contextKey = key; } @@ -366,10 +362,11 @@ public void process(WatchedEvent event) { public void setup(final ElectionContext context) throws InterruptedException, KeeperException { String electZKPath = context.electionPath + LeaderElector.ELECTION_NODE; if (context instanceof OverseerElectionContext) { - zkCmdExecutor.ensureExists(electZKPath, zkClient); + ZkMaintenanceUtils.ensureExists(electZKPath, zkClient); } else { // we use 2 param so that replica won't create /collection/{collection} if it doesn't exist - zkCmdExecutor.ensureExists(electZKPath, (byte[]) null, CreateMode.PERSISTENT, zkClient, 2); + ZkMaintenanceUtils.ensureExists( + electZKPath, (byte[]) null, CreateMode.PERSISTENT, zkClient, 2); } this.context = context; diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java index 2d2669af2ce..8eb8a6d1386 100644 --- a/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java +++ b/solr/core/src/java/org/apache/solr/cloud/OverseerElectionContext.java @@ -23,7 +23,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.util.Utils; import org.apache.zookeeper.CreateMode; @@ -43,8 +43,7 @@ public OverseerElectionContext( this.overseer = overseer; this.zkClient = zkClient; try { - new ZkCmdExecutor(zkClient.getZkClientTimeout()) - .ensureExists(Overseer.OVERSEER_ELECT, zkClient); + ZkMaintenanceUtils.ensureExists(Overseer.OVERSEER_ELECT, zkClient); } catch (KeeperException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java index c6ac7421d95..32706698d8b 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java +++ b/solr/core/src/java/org/apache/solr/cloud/ShardLeaderElectionContextBase.java @@ -27,7 +27,6 @@ import org.apache.solr.common.cloud.PerReplicaStatesOps; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; @@ -79,11 +78,10 @@ public ShardLeaderElectionContextBase( this.collection = collection; String parent = ZkMaintenanceUtils.getZkParent(leaderPath); - ZkCmdExecutor zcmd = new ZkCmdExecutor(30000); // only if /collections/{collection} exists already do we succeed in creating this path log.info("make sure parent is created {}", parent); try { - zcmd.ensureExists(parent, (byte[]) null, CreateMode.PERSISTENT, zkClient, 2); + ZkMaintenanceUtils.ensureExists(parent, (byte[]) null, CreateMode.PERSISTENT, zkClient, 2); } catch (KeeperException e) { throw new RuntimeException(e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java index 25f097cc8c8..c27e79c8274 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java @@ -87,10 +87,10 @@ import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkACLProvider; import org.apache.solr.common.cloud.ZkClientConnectionStrategy; -import org.apache.solr.common.cloud.ZkCmdExecutor; import org.apache.solr.common.cloud.ZkCoreNodeProps; import org.apache.solr.common.cloud.ZkCredentialsInjector; import org.apache.solr.common.cloud.ZkCredentialsProvider; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.cloud.ZooKeeperException; @@ -915,20 +915,19 @@ boolean isClosed() { */ public static void createClusterZkNodes(SolrZkClient zkClient) throws KeeperException, InterruptedException, IOException { - ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout()); - cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient); - cmdExecutor.ensureExists(ZkStateReader.NODE_ROLES, zkClient); + ZkMaintenanceUtils.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient); + ZkMaintenanceUtils.ensureExists(ZkStateReader.NODE_ROLES, zkClient); for (NodeRoles.Role role : NodeRoles.Role.values()) { - cmdExecutor.ensureExists(NodeRoles.getZNodeForRole(role), zkClient); + ZkMaintenanceUtils.ensureExists(NodeRoles.getZNodeForRole(role), zkClient); for (String mode : role.supportedModes()) { - cmdExecutor.ensureExists(NodeRoles.getZNodeForRoleMode(role, mode), zkClient); + ZkMaintenanceUtils.ensureExists(NodeRoles.getZNodeForRoleMode(role, mode), zkClient); } } - cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient); - cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient); + ZkMaintenanceUtils.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient); + ZkMaintenanceUtils.ensureExists(ZkStateReader.ALIASES, zkClient); byte[] emptyJson = "{}".getBytes(StandardCharsets.UTF_8); - cmdExecutor.ensureExists(ZkStateReader.SOLR_SECURITY_CONF_PATH, emptyJson, zkClient); + ZkMaintenanceUtils.ensureExists(ZkStateReader.SOLR_SECURITY_CONF_PATH, emptyJson, zkClient); repairSecurityJson(zkClient); } diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java index 39a1de01feb..57db14740c5 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkDistributedQueue.java @@ -37,7 +37,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.ConnectionManager.IsClosed; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.util.Pair; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -134,10 +134,8 @@ public ZkDistributedQueue( IsClosed higherLevelIsClosed) { this.dir = dir; - ZkCmdExecutor cmdExecutor = - new ZkCmdExecutor(zookeeper.getZkClientTimeout(), higherLevelIsClosed); try { - cmdExecutor.ensureExists(dir, zookeeper); + ZkMaintenanceUtils.ensureExists(dir, zookeeper); } catch (KeeperException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java index a33c4a51f3b..b8ae2dca46f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/CreateCollectionAPI.java @@ -68,7 +68,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; @@ -265,9 +265,8 @@ private static Integer readIntegerDefaultFromClusterProp( private static void createSysConfigSet(CoreContainer coreContainer) throws KeeperException, InterruptedException { SolrZkClient zk = coreContainer.getZkController().getZkStateReader().getZkClient(); - ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zk.getZkClientTimeout()); - cmdExecutor.ensureExists(ZkStateReader.CONFIGS_ZKNODE, zk); - cmdExecutor.ensureExists( + ZkMaintenanceUtils.ensureExists(ZkStateReader.CONFIGS_ZKNODE, zk); + ZkMaintenanceUtils.ensureExists( ZkStateReader.CONFIGS_ZKNODE + "/" + CollectionAdminParams.SYSTEM_COLL, zk); try { @@ -280,7 +279,7 @@ private static void createSysConfigSet(CoreContainer coreContainer) data = inputStream.readAllBytes(); } assert data != null && data.length > 0; - cmdExecutor.ensureExists(path, data, CreateMode.PERSISTENT, zk); + ZkMaintenanceUtils.ensureExists(path, data, CreateMode.PERSISTENT, zk); path = ZkStateReader.CONFIGS_ZKNODE + "/" @@ -292,7 +291,7 @@ private static void createSysConfigSet(CoreContainer coreContainer) data = inputStream.readAllBytes(); } assert data != null && data.length > 0; - cmdExecutor.ensureExists(path, data, CreateMode.PERSISTENT, zk); + ZkMaintenanceUtils.ensureExists(path, data, CreateMode.PERSISTENT, zk); } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java index 9008b3e443e..73187383bd8 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java @@ -32,7 +32,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.NamedList; @@ -485,12 +485,11 @@ private void zkUgradeToManagedSchema() { // Rename the non-managed schema znode in ZooKeeper final String nonManagedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + resourceName; try { - ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout()); if (zkController.pathExists(nonManagedSchemaPath)) { // First, copy the non-managed schema znode content to the upgraded schema znode byte[] bytes = zkController.getZkClient().getData(nonManagedSchemaPath, null, null, true); final String upgradedSchemaPath = nonManagedSchemaPath + UPGRADED_SCHEMA_EXTENSION; - zkCmdExecutor.ensureExists(upgradedSchemaPath, zkController.getZkClient()); + ZkMaintenanceUtils.ensureExists(upgradedSchemaPath, zkController.getZkClient()); zkController.getZkClient().setData(upgradedSchemaPath, bytes, true); // Then delete the non-managed schema znode if (zkController.getZkClient().exists(nonManagedSchemaPath, true)) { diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java index a9ed66f70ff..4503c862d50 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java @@ -30,6 +30,7 @@ import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkCmdExecutor; +import org.apache.solr.common.cloud.ZkMaintenanceUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.Op; @@ -394,12 +395,11 @@ public void testSkipPathPartsOnMakePath() throws Exception { zkClient.clean("/"); - ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(30000); expectThrows( KeeperException.NoNodeException.class, "We should not be able to create this path", () -> - zkCmdExecutor.ensureExists( + ZkMaintenanceUtils.ensureExists( "/collection/collection/leader", (byte[]) null, CreateMode.PERSISTENT, @@ -412,7 +412,7 @@ public void testSkipPathPartsOnMakePath() throws Exception { KeeperException.NoNodeException.class, "We should not be able to create this path", () -> - zkCmdExecutor.ensureExists( + ZkMaintenanceUtils.ensureExists( "/collections/collection/leader", (byte[]) null, CreateMode.PERSISTENT, @@ -421,7 +421,7 @@ public void testSkipPathPartsOnMakePath() throws Exception { zkClient.makePath("/collection/collection", true); byte[] bytes = new byte[10]; - zkCmdExecutor.ensureExists( + ZkMaintenanceUtils.ensureExists( "/collection/collection", bytes, CreateMode.PERSISTENT, zkClient, 2); byte[] returnedBytes = zkClient.getData("/collection/collection", null, null, true); @@ -430,7 +430,7 @@ public void testSkipPathPartsOnMakePath() throws Exception { zkClient.makePath("/collection/collection/leader", true); - zkCmdExecutor.ensureExists( + ZkMaintenanceUtils.ensureExists( "/collection/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2); } } diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java index 06134f1fbae..01924b3c044 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkCmdExecutor.java @@ -19,9 +19,7 @@ import java.lang.invoke.MethodHandles; import org.apache.solr.common.AlreadyClosedException; import org.apache.solr.common.cloud.ConnectionManager.IsClosed; -import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.KeeperException.NodeExistsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,64 +92,6 @@ private boolean isClosed() { return isClosed != null && isClosed.isClosed(); } - /** - * Create a persistent znode with no data if it does not already exist - * - * @see #ensureExists(String, byte[], CreateMode, SolrZkClient, int) - */ - public void ensureExists(String path, final SolrZkClient zkClient) - throws KeeperException, InterruptedException { - ensureExists(path, null, CreateMode.PERSISTENT, zkClient, 0); - } - - /** - * Create a persistent znode with the given data if it does not already exist - * - * @see #ensureExists(String, byte[], CreateMode, SolrZkClient, int) - */ - public void ensureExists(String path, final byte[] data, final SolrZkClient zkClient) - throws KeeperException, InterruptedException { - ensureExists(path, data, CreateMode.PERSISTENT, zkClient, 0); - } - - /** - * Create a znode with the given mode and data if it does not already exist - * - * @see #ensureExists(String, byte[], CreateMode, SolrZkClient, int) - */ - public void ensureExists( - String path, final byte[] data, CreateMode createMode, final SolrZkClient zkClient) - throws KeeperException, InterruptedException { - ensureExists(path, data, createMode, zkClient, 0); - } - - /** - * Create a node if it does not exist - * - * @param path the path at which to create the znode - * @param data the optional data to set on the znode - * @param createMode the mode with which to create the znode - * @param zkClient the client to use to check and create - * @param skipPathParts how many path elements to skip - */ - public void ensureExists( - final String path, - final byte[] data, - CreateMode createMode, - final SolrZkClient zkClient, - int skipPathParts) - throws KeeperException, InterruptedException { - - if (zkClient.exists(path, true)) { - return; - } - try { - zkClient.makePath(path, data, createMode, null, true, true, skipPathParts); - } catch (NodeExistsException ignored) { - // it's okay if another beats us creating the node - } - } - /** * Performs a retry delay if this is not the first attempt * diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java index 7ade32ebf3d..34d770c7a12 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java @@ -37,6 +37,7 @@ import org.apache.solr.common.util.StrUtils; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -589,28 +590,87 @@ public static boolean isFileForbiddenInConfigSets(String filePath) { int lastDot = filePath.lastIndexOf('.'); return lastDot >= 0 && USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1)); } -} -class ZkCopier implements ZkMaintenanceUtils.ZkVisitor { + /** + * Create a persistent znode with no data if it does not already exist + * + * @see #ensureExists(String, byte[], CreateMode, SolrZkClient, int) + */ + public static void ensureExists(String path, final SolrZkClient zkClient) + throws KeeperException, InterruptedException { + ensureExists(path, null, CreateMode.PERSISTENT, zkClient, 0); + } + + /** + * Create a persistent znode with the given data if it does not already exist + * + * @see #ensureExists(String, byte[], CreateMode, SolrZkClient, int) + */ + public static void ensureExists(String path, final byte[] data, final SolrZkClient zkClient) + throws KeeperException, InterruptedException { + ensureExists(path, data, CreateMode.PERSISTENT, zkClient, 0); + } + + /** + * Create a znode with the given mode and data if it does not already exist + * + * @see #ensureExists(String, byte[], CreateMode, SolrZkClient, int) + */ + public static void ensureExists( + String path, final byte[] data, CreateMode createMode, final SolrZkClient zkClient) + throws KeeperException, InterruptedException { + ensureExists(path, data, createMode, zkClient, 0); + } - String source; - String dest; - SolrZkClient zkClient; + /** + * Create a node if it does not exist + * + * @param path the path at which to create the znode + * @param data the optional data to set on the znode + * @param createMode the mode with which to create the znode + * @param zkClient the client to use to check and create + * @param skipPathParts how many path elements to skip + */ + public static void ensureExists( + final String path, + final byte[] data, + CreateMode createMode, + final SolrZkClient zkClient, + int skipPathParts) + throws KeeperException, InterruptedException { - ZkCopier(SolrZkClient zkClient, String source, String dest) { - this.source = source; - this.dest = dest; - if (dest.endsWith("/")) { - this.dest = dest.substring(0, dest.length() - 1); + if (zkClient.exists(path, true)) { + return; + } + try { + zkClient.makePath(path, data, createMode, null, true, true, skipPathParts); + } catch (NodeExistsException ignored) { + // it's okay if another beats us creating the node } - this.zkClient = zkClient; } - @Override - public void visit(String path) throws InterruptedException, KeeperException { - String finalDestination = dest; - if (path.equals(source) == false) finalDestination += "/" + path.substring(source.length() + 1); - zkClient.makePath(finalDestination, false, true); - zkClient.setData(finalDestination, zkClient.getData(path, null, null, true), true); + static class ZkCopier implements ZkMaintenanceUtils.ZkVisitor { + + String source; + String dest; + SolrZkClient zkClient; + + ZkCopier(SolrZkClient zkClient, String source, String dest) { + this.source = source; + this.dest = dest; + if (dest.endsWith("/")) { + this.dest = dest.substring(0, dest.length() - 1); + } + this.zkClient = zkClient; + } + + @Override + public void visit(String path) throws InterruptedException, KeeperException { + String finalDestination = dest; + if (path.equals(source) == false) + finalDestination += "/" + path.substring(source.length() + 1); + zkClient.makePath(finalDestination, false, true); + zkClient.setData(finalDestination, zkClient.getData(path, null, null, true), true); + } } } From 7f5d98a2749980f967896c9424d18bdfcdb1d0dd Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Wed, 16 Aug 2023 20:49:53 +0200 Subject: [PATCH 009/160] Update dependency org.openjdk.jmh:jmh-core to v1.37 (#1838) --- solr/licenses/jmh-core-1.36.jar.sha1 | 1 - solr/licenses/jmh-core-1.37.jar.sha1 | 1 + versions.lock | 4 ++-- versions.props | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 solr/licenses/jmh-core-1.36.jar.sha1 create mode 100644 solr/licenses/jmh-core-1.37.jar.sha1 diff --git a/solr/licenses/jmh-core-1.36.jar.sha1 b/solr/licenses/jmh-core-1.36.jar.sha1 deleted file mode 100644 index 14878efb422..00000000000 --- a/solr/licenses/jmh-core-1.36.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5a69117788322630fc5f228bc804771335d41b1b diff --git a/solr/licenses/jmh-core-1.37.jar.sha1 b/solr/licenses/jmh-core-1.37.jar.sha1 new file mode 100644 index 00000000000..12dc78b52de --- /dev/null +++ b/solr/licenses/jmh-core-1.37.jar.sha1 @@ -0,0 +1 @@ +896f27e49105b35ea1964319c83d12082e7a79ef diff --git a/versions.lock b/versions.lock index 9556f8c0982..f0320208225 100644 --- a/versions.lock +++ b/versions.lock @@ -175,7 +175,7 @@ org.apache.commons:commons-configuration2:2.9.0 (1 constraints: 0d051036) org.apache.commons:commons-csv:1.9.0 (1 constraints: 610cfc01) org.apache.commons:commons-exec:1.3 (2 constraints: a41056b8) org.apache.commons:commons-lang3:3.13.0 (5 constraints: 9c3eb936) -org.apache.commons:commons-math3:3.6.1 (5 constraints: f431a634) +org.apache.commons:commons-math3:3.6.1 (5 constraints: 57322799) org.apache.commons:commons-text:1.10.0 (1 constraints: d911adf8) org.apache.curator:curator-client:4.3.0 (2 constraints: e214cba2) org.apache.curator:curator-framework:4.3.0 (2 constraints: ff13b474) @@ -334,7 +334,7 @@ org.locationtech.jts.io:jts-io-common:1.19.0 (1 constraints: 930d513a) org.locationtech.proj4j:proj4j:1.2.2 (1 constraints: 5d0daf2c) org.locationtech.spatial4j:spatial4j:0.8 (1 constraints: 59105498) org.opengis:geoapi:3.0.1 (7 constraints: 1361d046) -org.openjdk.jmh:jmh-core:1.36 (1 constraints: de04fb30) +org.openjdk.jmh:jmh-core:1.37 (1 constraints: df04fc30) org.osgi:org.osgi.resource:1.0.0 (1 constraints: e60f2999) org.osgi:org.osgi.service.serviceloader:1.0.0 (1 constraints: e60f2999) org.osgi:osgi.annotation:8.1.0 (1 constraints: 0b051636) diff --git a/versions.props b/versions.props index 71c8f2671d0..4ba9d7b879f 100644 --- a/versions.props +++ b/versions.props @@ -64,7 +64,7 @@ org.hsqldb:hsqldb=2.7.1 org.immutables:value-annotations=2.9.3 org.jctools:jctools-core=4.0.1 org.mockito:mockito*=5.3.1 -org.openjdk.jmh:*=1.36 +org.openjdk.jmh:*=1.37 org.osgi:osgi.annotation=8.1.0 org.quicktheories:quicktheories=0.26 org.semver4j:semver4j=4.3.0 From 0c2e39bbbfb081fba8a24df903027d7f3d174b79 Mon Sep 17 00:00:00 2001 From: Alex D Date: Wed, 16 Aug 2023 21:13:16 -0700 Subject: [PATCH 010/160] Update io.opentelemetry to 1.29.0 (#1846) --- .../kotlin-stdlib-jdk7-1.6.0.jar.sha1 | 1 - .../kotlin-stdlib-jdk7-1.6.20.jar.sha1 | 1 + .../kotlin-stdlib-jdk8-1.6.0.jar.sha1 | 1 - .../kotlin-stdlib-jdk8-1.6.20.jar.sha1 | 1 + solr/licenses/okhttp-4.10.0.jar.sha1 | 1 - solr/licenses/okhttp-4.11.0.jar.sha1 | 1 + solr/licenses/okio-jvm-3.0.0.jar.sha1 | 1 - solr/licenses/okio-jvm-3.2.0.jar.sha1 | 1 + .../opentelemetry-api-1.21.0.jar.sha1 | 1 - .../opentelemetry-api-1.29.0.jar.sha1 | 1 + ...telemetry-api-events-1.29.0-alpha.jar.sha1 | 1 + ...entelemetry-api-logs-1.21.0-alpha.jar.sha1 | 1 - .../opentelemetry-context-1.21.0.jar.sha1 | 1 - .../opentelemetry-context-1.29.0.jar.sha1 | 1 + ...ntelemetry-exporter-common-1.21.0.jar.sha1 | 1 - ...ntelemetry-exporter-common-1.29.0.jar.sha1 | 1 + ...pentelemetry-exporter-otlp-1.21.0.jar.sha1 | 1 - ...pentelemetry-exporter-otlp-1.29.0.jar.sha1 | 1 + ...metry-exporter-otlp-common-1.21.0.jar.sha1 | 1 - ...metry-exporter-otlp-common-1.29.0.jar.sha1 | 1 + ...try-exporter-sender-okhttp-1.29.0.jar.sha1 | 1 + ...-extension-incubator-1.29.0-alpha.jar.sha1 | 1 + ...try-opentracing-shim-1.21.0-alpha.jar.sha1 | 1 - ...telemetry-opentracing-shim-1.29.0.jar.sha1 | 1 + .../opentelemetry-sdk-1.21.0.jar.sha1 | 1 - .../opentelemetry-sdk-1.29.0.jar.sha1 | 1 + .../opentelemetry-sdk-common-1.21.0.jar.sha1 | 1 - .../opentelemetry-sdk-common-1.29.0.jar.sha1 | 1 + ...ension-autoconfigure-1.21.0-alpha.jar.sha1 | 1 - ...dk-extension-autoconfigure-1.29.0.jar.sha1 | 1 + ...xtension-autoconfigure-spi-1.21.0.jar.sha1 | 1 - ...xtension-autoconfigure-spi-1.29.0.jar.sha1 | 1 + ...entelemetry-sdk-logs-1.21.0-alpha.jar.sha1 | 1 - .../opentelemetry-sdk-logs-1.29.0.jar.sha1 | 1 + .../opentelemetry-sdk-metrics-1.21.0.jar.sha1 | 1 - .../opentelemetry-sdk-metrics-1.29.0.jar.sha1 | 1 + .../opentelemetry-sdk-trace-1.21.0.jar.sha1 | 1 - .../opentelemetry-sdk-trace-1.29.0.jar.sha1 | 1 + ...pentelemetry-semconv-1.21.0-alpha.jar.sha1 | 1 - ...pentelemetry-semconv-1.29.0-alpha.jar.sha1 | 1 + solr/modules/opentelemetry/build.gradle | 2 +- versions.lock | 49 ++++++++++--------- versions.props | 2 +- 43 files changed, 48 insertions(+), 45 deletions(-) delete mode 100644 solr/licenses/kotlin-stdlib-jdk7-1.6.0.jar.sha1 create mode 100644 solr/licenses/kotlin-stdlib-jdk7-1.6.20.jar.sha1 delete mode 100644 solr/licenses/kotlin-stdlib-jdk8-1.6.0.jar.sha1 create mode 100644 solr/licenses/kotlin-stdlib-jdk8-1.6.20.jar.sha1 delete mode 100644 solr/licenses/okhttp-4.10.0.jar.sha1 create mode 100644 solr/licenses/okhttp-4.11.0.jar.sha1 delete mode 100644 solr/licenses/okio-jvm-3.0.0.jar.sha1 create mode 100644 solr/licenses/okio-jvm-3.2.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-api-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-api-1.29.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-api-events-1.29.0-alpha.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-api-logs-1.21.0-alpha.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-context-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-context-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-exporter-common-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-exporter-common-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-exporter-otlp-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-exporter-otlp-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-exporter-otlp-common-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-exporter-otlp-common-1.29.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-exporter-sender-okhttp-1.29.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-extension-incubator-1.29.0-alpha.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-opentracing-shim-1.21.0-alpha.jar.sha1 create mode 100644 solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-common-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-common-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.21.0-alpha.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-logs-1.21.0-alpha.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-logs-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-metrics-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-metrics-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-sdk-trace-1.21.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-trace-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1 create mode 100644 solr/licenses/opentelemetry-semconv-1.29.0-alpha.jar.sha1 diff --git a/solr/licenses/kotlin-stdlib-jdk7-1.6.0.jar.sha1 b/solr/licenses/kotlin-stdlib-jdk7-1.6.0.jar.sha1 deleted file mode 100644 index a48893479ee..00000000000 --- a/solr/licenses/kotlin-stdlib-jdk7-1.6.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -da6bdc87391322974a43ccc00a25536ae74dad51 diff --git a/solr/licenses/kotlin-stdlib-jdk7-1.6.20.jar.sha1 b/solr/licenses/kotlin-stdlib-jdk7-1.6.20.jar.sha1 new file mode 100644 index 00000000000..e649f57208c --- /dev/null +++ b/solr/licenses/kotlin-stdlib-jdk7-1.6.20.jar.sha1 @@ -0,0 +1 @@ +f8629f336bad4001c89e9cffa5ef3d4b5d0f5e22 diff --git a/solr/licenses/kotlin-stdlib-jdk8-1.6.0.jar.sha1 b/solr/licenses/kotlin-stdlib-jdk8-1.6.0.jar.sha1 deleted file mode 100644 index 50cd854fef6..00000000000 --- a/solr/licenses/kotlin-stdlib-jdk8-1.6.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -baf82c475e9372c25407f3d132439e4aa803b8b8 diff --git a/solr/licenses/kotlin-stdlib-jdk8-1.6.20.jar.sha1 b/solr/licenses/kotlin-stdlib-jdk8-1.6.20.jar.sha1 new file mode 100644 index 00000000000..dcea4bda07a --- /dev/null +++ b/solr/licenses/kotlin-stdlib-jdk8-1.6.20.jar.sha1 @@ -0,0 +1 @@ +dab8089bca6ac0e394c37281ea8cff2f99acd421 diff --git a/solr/licenses/okhttp-4.10.0.jar.sha1 b/solr/licenses/okhttp-4.10.0.jar.sha1 deleted file mode 100644 index 16cf73975de..00000000000 --- a/solr/licenses/okhttp-4.10.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cd63657ac15770ed1420647154c9f44645533bef diff --git a/solr/licenses/okhttp-4.11.0.jar.sha1 b/solr/licenses/okhttp-4.11.0.jar.sha1 new file mode 100644 index 00000000000..443d3088eec --- /dev/null +++ b/solr/licenses/okhttp-4.11.0.jar.sha1 @@ -0,0 +1 @@ +436932d695b2c43f2c86b8111c596179cd133d56 diff --git a/solr/licenses/okio-jvm-3.0.0.jar.sha1 b/solr/licenses/okio-jvm-3.0.0.jar.sha1 deleted file mode 100644 index 663b07539a3..00000000000 --- a/solr/licenses/okio-jvm-3.0.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0ab5a73fa2ccb4a36b0b5c69fe10b16d0255bcf8 diff --git a/solr/licenses/okio-jvm-3.2.0.jar.sha1 b/solr/licenses/okio-jvm-3.2.0.jar.sha1 new file mode 100644 index 00000000000..e410b57207f --- /dev/null +++ b/solr/licenses/okio-jvm-3.2.0.jar.sha1 @@ -0,0 +1 @@ +332d1c5dc82b0241cb1d35bb0901d28470cc89ca diff --git a/solr/licenses/opentelemetry-api-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-api-1.21.0.jar.sha1 deleted file mode 100644 index a745b7e36c0..00000000000 --- a/solr/licenses/opentelemetry-api-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1e3d45bf10061c31179b0d287da83699090b399d diff --git a/solr/licenses/opentelemetry-api-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-api-1.29.0.jar.sha1 new file mode 100644 index 00000000000..7425920b87a --- /dev/null +++ b/solr/licenses/opentelemetry-api-1.29.0.jar.sha1 @@ -0,0 +1 @@ +45010687a1181dc886fd12403e48cf76e94c65b1 diff --git a/solr/licenses/opentelemetry-api-events-1.29.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-api-events-1.29.0-alpha.jar.sha1 new file mode 100644 index 00000000000..0b08d51c6f2 --- /dev/null +++ b/solr/licenses/opentelemetry-api-events-1.29.0-alpha.jar.sha1 @@ -0,0 +1 @@ +cdc6b637c6374d4ae4be5cad381f6a566fb4310e diff --git a/solr/licenses/opentelemetry-api-logs-1.21.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-api-logs-1.21.0-alpha.jar.sha1 deleted file mode 100644 index 1db5505ceb6..00000000000 --- a/solr/licenses/opentelemetry-api-logs-1.21.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -74cf69b29832c134c1ac539ef1fc047f96c39fca diff --git a/solr/licenses/opentelemetry-context-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-context-1.21.0.jar.sha1 deleted file mode 100644 index 87cc9b62d69..00000000000 --- a/solr/licenses/opentelemetry-context-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3b26607d39336dfd092e5d4dc148c9e1f3d8043b diff --git a/solr/licenses/opentelemetry-context-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-context-1.29.0.jar.sha1 new file mode 100644 index 00000000000..7f8f3d28249 --- /dev/null +++ b/solr/licenses/opentelemetry-context-1.29.0.jar.sha1 @@ -0,0 +1 @@ +a4cf6857f268b9637ea330fffc70c1e6421d1d55 diff --git a/solr/licenses/opentelemetry-exporter-common-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-common-1.21.0.jar.sha1 deleted file mode 100644 index 8282877f6a2..00000000000 --- a/solr/licenses/opentelemetry-exporter-common-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d92b4897c6b07f5a452d7a9b6d0f7b010596c31f diff --git a/solr/licenses/opentelemetry-exporter-common-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-common-1.29.0.jar.sha1 new file mode 100644 index 00000000000..2f7556109de --- /dev/null +++ b/solr/licenses/opentelemetry-exporter-common-1.29.0.jar.sha1 @@ -0,0 +1 @@ +7fa947205c2e85383c5b24139325f36a9e474067 diff --git a/solr/licenses/opentelemetry-exporter-otlp-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-otlp-1.21.0.jar.sha1 deleted file mode 100644 index 9b028264554..00000000000 --- a/solr/licenses/opentelemetry-exporter-otlp-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -10fae1d61b7901f4d50baf3f9bcc959ad08d7142 diff --git a/solr/licenses/opentelemetry-exporter-otlp-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-otlp-1.29.0.jar.sha1 new file mode 100644 index 00000000000..7146545d1ea --- /dev/null +++ b/solr/licenses/opentelemetry-exporter-otlp-1.29.0.jar.sha1 @@ -0,0 +1 @@ +471956f2773b5409355a09da6ef4cb8976c82db2 diff --git a/solr/licenses/opentelemetry-exporter-otlp-common-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-otlp-common-1.21.0.jar.sha1 deleted file mode 100644 index 774691534ce..00000000000 --- a/solr/licenses/opentelemetry-exporter-otlp-common-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d378982284bcb440d2529341ce66e18a4d4cd6ff diff --git a/solr/licenses/opentelemetry-exporter-otlp-common-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-otlp-common-1.29.0.jar.sha1 new file mode 100644 index 00000000000..fe86cf5eb9e --- /dev/null +++ b/solr/licenses/opentelemetry-exporter-otlp-common-1.29.0.jar.sha1 @@ -0,0 +1 @@ +992edd4ba5d473abcd6b4ca126f11149b2bdba0c diff --git a/solr/licenses/opentelemetry-exporter-sender-okhttp-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-exporter-sender-okhttp-1.29.0.jar.sha1 new file mode 100644 index 00000000000..1ea79b62365 --- /dev/null +++ b/solr/licenses/opentelemetry-exporter-sender-okhttp-1.29.0.jar.sha1 @@ -0,0 +1 @@ +d3a2848644687cfe541c8b2b4b05f5d4e58c2709 diff --git a/solr/licenses/opentelemetry-extension-incubator-1.29.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-extension-incubator-1.29.0-alpha.jar.sha1 new file mode 100644 index 00000000000..e781a342e63 --- /dev/null +++ b/solr/licenses/opentelemetry-extension-incubator-1.29.0-alpha.jar.sha1 @@ -0,0 +1 @@ +4af0adab426582e9acea130d39b11791ccf0b513 diff --git a/solr/licenses/opentelemetry-opentracing-shim-1.21.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-opentracing-shim-1.21.0-alpha.jar.sha1 deleted file mode 100644 index a156a8c7526..00000000000 --- a/solr/licenses/opentelemetry-opentracing-shim-1.21.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -321f54fc7a0ad717a8eda466d682e78fa140957b diff --git a/solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 new file mode 100644 index 00000000000..1d378f65bc4 --- /dev/null +++ b/solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 @@ -0,0 +1 @@ +cbe64aea8ea3811911bf6e572f96ec0bc745948d diff --git a/solr/licenses/opentelemetry-sdk-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-1.21.0.jar.sha1 deleted file mode 100644 index e68cdab696d..00000000000 --- a/solr/licenses/opentelemetry-sdk-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -881e035b0bac54a2b1f73a226cc96267065347d9 diff --git a/solr/licenses/opentelemetry-sdk-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-1.29.0.jar.sha1 new file mode 100644 index 00000000000..23cae568fb8 --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-1.29.0.jar.sha1 @@ -0,0 +1 @@ +5d3a0e83ab6a2849c5b4123d539f994a721ddcd5 diff --git a/solr/licenses/opentelemetry-sdk-common-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-common-1.21.0.jar.sha1 deleted file mode 100644 index 12f69983134..00000000000 --- a/solr/licenses/opentelemetry-sdk-common-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3b04a4f6e54aa30b02b86f60b4346a99750fb73b diff --git a/solr/licenses/opentelemetry-sdk-common-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-common-1.29.0.jar.sha1 new file mode 100644 index 00000000000..4033aa6f1bf --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-common-1.29.0.jar.sha1 @@ -0,0 +1 @@ +a4a84b83c226c91a54bd0e9244d49908875cea3a diff --git a/solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.21.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.21.0-alpha.jar.sha1 deleted file mode 100644 index 94eb09c3eb0..00000000000 --- a/solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.21.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b36c7b9009f060b355b301d31ff6ecd68490d8d3 diff --git a/solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.29.0.jar.sha1 new file mode 100644 index 00000000000..635e3eada28 --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-extension-autoconfigure-1.29.0.jar.sha1 @@ -0,0 +1 @@ +36bd08550cbdcc4f58a4949e0f97f97443220a2d diff --git a/solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.21.0.jar.sha1 deleted file mode 100644 index f68de258d2d..00000000000 --- a/solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7264cbab013612395043985293ca63acd9d00e4a diff --git a/solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.29.0.jar.sha1 new file mode 100644 index 00000000000..299eb7083b6 --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-extension-autoconfigure-spi-1.29.0.jar.sha1 @@ -0,0 +1 @@ +b810dedc1fc8b28892eca5fa945e1c82cb95be4f diff --git a/solr/licenses/opentelemetry-sdk-logs-1.21.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-sdk-logs-1.21.0-alpha.jar.sha1 deleted file mode 100644 index 94257a0b2a8..00000000000 --- a/solr/licenses/opentelemetry-sdk-logs-1.21.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -afa30fc84dae4d46b473de5977574d719a6cadd1 diff --git a/solr/licenses/opentelemetry-sdk-logs-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-logs-1.29.0.jar.sha1 new file mode 100644 index 00000000000..ff6a4151576 --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-logs-1.29.0.jar.sha1 @@ -0,0 +1 @@ +68151d4465c12183db8edc4f9d8f0a878bada16b diff --git a/solr/licenses/opentelemetry-sdk-metrics-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-metrics-1.21.0.jar.sha1 deleted file mode 100644 index 36a3e08b603..00000000000 --- a/solr/licenses/opentelemetry-sdk-metrics-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b1c254f525c442e7725c0ab6836a5c3df10fda3f diff --git a/solr/licenses/opentelemetry-sdk-metrics-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-metrics-1.29.0.jar.sha1 new file mode 100644 index 00000000000..813a65b66fc --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-metrics-1.29.0.jar.sha1 @@ -0,0 +1 @@ +6bb59616f1180286bc2ccf40e34d636984581ba9 diff --git a/solr/licenses/opentelemetry-sdk-trace-1.21.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-trace-1.21.0.jar.sha1 deleted file mode 100644 index fc78bc73977..00000000000 --- a/solr/licenses/opentelemetry-sdk-trace-1.21.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -83a6f03af6dbab00409ecc96087269bbfa9a8cc7 diff --git a/solr/licenses/opentelemetry-sdk-trace-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-trace-1.29.0.jar.sha1 new file mode 100644 index 00000000000..1a9ce5eb65d --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-trace-1.29.0.jar.sha1 @@ -0,0 +1 @@ +1008bf3794f6fc10238b1f63d0546ae817ad1783 diff --git a/solr/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1 deleted file mode 100644 index a8fa3b33d98..00000000000 --- a/solr/licenses/opentelemetry-semconv-1.21.0-alpha.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1c89d57f157d4da3a2306b23bf6366d036a70f05 diff --git a/solr/licenses/opentelemetry-semconv-1.29.0-alpha.jar.sha1 b/solr/licenses/opentelemetry-semconv-1.29.0-alpha.jar.sha1 new file mode 100644 index 00000000000..aa0f7608da5 --- /dev/null +++ b/solr/licenses/opentelemetry-semconv-1.29.0-alpha.jar.sha1 @@ -0,0 +1 @@ +8ee51f51d9c1c959b537c8dba67d7a524204b974 diff --git a/solr/modules/opentelemetry/build.gradle b/solr/modules/opentelemetry/build.gradle index 0018a710c6b..444d3c6ffb4 100644 --- a/solr/modules/opentelemetry/build.gradle +++ b/solr/modules/opentelemetry/build.gradle @@ -22,7 +22,7 @@ description = 'Open Telemetry (OTEL) tracer' dependencies { implementation project(':solr:core') - implementation platform('io.opentelemetry:opentelemetry-bom-alpha') + implementation platform('io.opentelemetry:opentelemetry-bom') implementation 'io.opentracing:opentracing-api' implementation 'org.slf4j:slf4j-api' diff --git a/versions.lock b/versions.lock index f0320208225..c78c403b7d4 100644 --- a/versions.lock +++ b/versions.lock @@ -69,9 +69,9 @@ com.mchange:mchange-commons-java:0.2.19 (1 constraints: 84075b75) com.pff:java-libpst:0.9.3 (1 constraints: 630cfa01) com.rometools:rome:1.18.0 (1 constraints: 910c870e) com.rometools:rome-utils:1.18.0 (1 constraints: 10095d96) -com.squareup.okhttp3:okhttp:4.10.0 (2 constraints: ad259100) -com.squareup.okio:okio:3.0.0 (1 constraints: 4d0c26fd) -com.squareup.okio:okio-jvm:3.0.0 (1 constraints: 4a0ac1b9) +com.squareup.okhttp3:okhttp:4.11.0 (2 constraints: 8126a451) +com.squareup.okio:okio:3.2.0 (1 constraints: 4f0c2cfd) +com.squareup.okio:okio-jvm:3.2.0 (1 constraints: 4c0ac7b9) com.sun.activation:jakarta.activation:1.2.2 (1 constraints: ba0dac35) com.sun.istack:istack-commons-runtime:3.0.12 (1 constraints: eb0d9a43) com.tdunning:t-digest:3.1 (1 constraints: a804212c) @@ -124,23 +124,24 @@ io.netty:netty-transport-native-unix-common:4.1.96.Final (4 constraints: 5e3d199 io.opencensus:opencensus-api:0.31.1 (5 constraints: 924d4692) io.opencensus:opencensus-contrib-http-util:0.31.1 (3 constraints: 7232a9fc) io.opencensus:opencensus-proto:0.2.0 (1 constraints: e60fd595) -io.opentelemetry:opentelemetry-api:1.21.0 (9 constraints: fda10c96) -io.opentelemetry:opentelemetry-api-logs:1.21.0-alpha (2 constraints: a0279d00) -io.opentelemetry:opentelemetry-bom:1.21.0 (1 constraints: c411a7f3) -io.opentelemetry:opentelemetry-bom-alpha:1.21.0-alpha (1 constraints: 69079861) -io.opentelemetry:opentelemetry-context:1.21.0 (2 constraints: 1d1fd2b3) -io.opentelemetry:opentelemetry-exporter-common:1.21.0 (3 constraints: 443f09e9) -io.opentelemetry:opentelemetry-exporter-otlp:1.21.0 (1 constraints: 910fe183) -io.opentelemetry:opentelemetry-exporter-otlp-common:1.21.0 (2 constraints: 48230b1b) -io.opentelemetry:opentelemetry-opentracing-shim:1.21.0-alpha (1 constraints: f7137265) -io.opentelemetry:opentelemetry-sdk:1.21.0 (3 constraints: 96438ee0) -io.opentelemetry:opentelemetry-sdk-common:1.21.0 (5 constraints: 2355c8a6) -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.21.0-alpha (1 constraints: f7137265) -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.21.0 (2 constraints: d7280b4d) -io.opentelemetry:opentelemetry-sdk-logs:1.21.0-alpha (4 constraints: 295e5301) -io.opentelemetry:opentelemetry-sdk-metrics:1.21.0 (5 constraints: e1669a5a) -io.opentelemetry:opentelemetry-sdk-trace:1.21.0 (3 constraints: dc32bdaf) -io.opentelemetry:opentelemetry-semconv:1.21.0-alpha (5 constraints: ed6e375b) +io.opentelemetry:opentelemetry-api:1.29.0 (11 constraints: ceca6768) +io.opentelemetry:opentelemetry-api-events:1.29.0-alpha (2 constraints: 332f2380) +io.opentelemetry:opentelemetry-bom:1.29.0 (1 constraints: 3e05483b) +io.opentelemetry:opentelemetry-context:1.29.0 (2 constraints: 2d1f62b5) +io.opentelemetry:opentelemetry-exporter-common:1.29.0 (3 constraints: 563d51ae) +io.opentelemetry:opentelemetry-exporter-otlp:1.29.0 (1 constraints: 990ff983) +io.opentelemetry:opentelemetry-exporter-otlp-common:1.29.0 (2 constraints: 5823eb1c) +io.opentelemetry:opentelemetry-exporter-sender-okhttp:1.29.0 (2 constraints: 5823eb1c) +io.opentelemetry:opentelemetry-extension-incubator:1.29.0-alpha (1 constraints: f414ee96) +io.opentelemetry:opentelemetry-opentracing-shim:1.29.0 (1 constraints: 990ff983) +io.opentelemetry:opentelemetry-sdk:1.29.0 (3 constraints: ae4376e7) +io.opentelemetry:opentelemetry-sdk-common:1.29.0 (6 constraints: 936c396e) +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.29.0 (1 constraints: 990ff983) +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.29.0 (3 constraints: a63c5427) +io.opentelemetry:opentelemetry-sdk-logs:1.29.0 (3 constraints: f43275b4) +io.opentelemetry:opentelemetry-sdk-metrics:1.29.0 (3 constraints: f43275b4) +io.opentelemetry:opentelemetry-sdk-trace:1.29.0 (3 constraints: f43275b4) +io.opentelemetry:opentelemetry-semconv:1.29.0-alpha (4 constraints: 175b2583) io.opentracing:opentracing-api:0.33.0 (5 constraints: 7244331d) io.opentracing:opentracing-noop:0.33.0 (4 constraints: 48361e21) io.opentracing:opentracing-util:0.33.0 (2 constraints: 5013be5a) @@ -324,10 +325,10 @@ org.javassist:javassist:3.25.0-GA (1 constraints: 2a110ef1) org.jctools:jctools-core:4.0.1 (1 constraints: 07050036) org.jdom:jdom2:2.0.6.1 (1 constraints: be0c371b) org.jetbrains:annotations:13.0 (1 constraints: df0e795c) -org.jetbrains.kotlin:kotlin-stdlib:1.6.20 (5 constraints: 6c4c1dcd) -org.jetbrains.kotlin:kotlin-stdlib-common:1.6.20 (3 constraints: 352cbb71) -org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.6.0 (1 constraints: ae1013c2) -org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.0 (3 constraints: 582c9186) +org.jetbrains.kotlin:kotlin-stdlib:1.6.20 (5 constraints: d04c515f) +org.jetbrains.kotlin:kotlin-stdlib-common:1.6.20 (3 constraints: 342c6071) +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.6.20 (1 constraints: e010f5d2) +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.20 (4 constraints: d9389ae4) org.junit:junit-bom:5.9.1 (1 constraints: c8116ede) org.locationtech.jts:jts-core:1.19.0 (2 constraints: a31de760) org.locationtech.jts.io:jts-io-common:1.19.0 (1 constraints: 930d513a) diff --git a/versions.props b/versions.props index 4ba9d7b879f..45ffbbad5d4 100644 --- a/versions.props +++ b/versions.props @@ -23,7 +23,7 @@ commons-io:commons-io=2.11.0 io.dropwizard.metrics:*=4.2.19 io.grpc:grpc-*=1.56.0 io.netty:*=4.1.96.Final -io.opentelemetry:opentelemetry-bom-alpha=1.21.0-alpha +io.opentelemetry:opentelemetry-bom=1.29.0 io.opentracing:*=0.33.0 io.prometheus:*=0.16.0 io.swagger.core.v3:*=2.2.12 From e36023c16dafaad815b24503f5f05e9e40c48cec Mon Sep 17 00:00:00 2001 From: Alex D Date: Thu, 17 Aug 2023 05:31:15 -0700 Subject: [PATCH 011/160] CHANGES.txt update for io.opentelemetry update (#1847) --- solr/CHANGES.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b7606af3ba7..b82508e9e6b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -124,7 +124,8 @@ Bug Fixes Dependency Upgrades --------------------- -(No changes) + +* PR#1846: Update io.opentelemetry to 1.29.0 (Alex Deparvu) Other Changes --------------------- From 92dede18504518a5542ddd35b0d9cc2887566934 Mon Sep 17 00:00:00 2001 From: Alex D Date: Thu, 17 Aug 2023 05:54:52 -0700 Subject: [PATCH 012/160] SOLR-16536 Replace OpenTracing instrumentation with OpenTelemetry (#1841) OpenTracing libraries were completely removed and replaced with OpenTelemetry libraries --- solr/CHANGES.txt | 2 + solr/core/build.gradle | 6 +- .../org/apache/solr/api/AnnotatedApi.java | 2 +- .../src/java/org/apache/solr/api/Api.java | 2 +- .../java/org/apache/solr/api/V2HttpCall.java | 10 +- .../org/apache/solr/core/CoreContainer.java | 10 +- .../apache/solr/core/TracerConfigurator.java | 80 +------ .../admin/api/RestoreCollectionAPI.java | 2 +- .../handler/component/HttpShardHandler.java | 12 +- .../solr/handler/component/ShardRequest.java | 5 - .../request/DelegatingSolrQueryRequest.java | 8 +- .../apache/solr/request/SolrQueryRequest.java | 22 +- .../solr/request/SolrQueryRequestBase.java | 32 --- .../org/apache/solr/servlet/HttpSolrCall.java | 22 +- .../org/apache/solr/servlet/ServletUtils.java | 49 +---- .../solr/servlet/SolrDispatchFilter.java | 27 +-- .../solr/update/SolrCmdDistributor.java | 13 +- .../org/apache/solr/update/UpdateCommand.java | 16 +- .../solr/util/tracing/HttpServletCarrier.java | 92 -------- .../tracing/HttpServletRequestGetter.java | 46 ++++ ...estCarrier.java => SolrRequestSetter.java} | 25 +-- .../apache/solr/util/tracing/TraceUtils.java | 116 +++++++++- .../admin/api/AddReplicaPropertyAPITest.java | 4 +- .../admin/api/CoreReplicationAPITest.java | 4 +- .../handler/admin/api/ListAliasesAPITest.java | 4 +- ...java => TestHttpServletRequestGetter.java} | 50 ++--- ...telemetry-opentracing-shim-1.29.0.jar.sha1 | 1 - .../opentelemetry-sdk-testing-1.29.0.jar.sha1 | 1 + solr/licenses/opentracing-api-0.33.0.jar.sha1 | 1 - solr/licenses/opentracing-api-LICENSE-ASL.txt | 201 ------------------ solr/licenses/opentracing-api-NOTICE.txt | 0 .../licenses/opentracing-mock-0.33.0.jar.sha1 | 1 - .../licenses/opentracing-mock-LICENSE-ASL.txt | 201 ------------------ solr/licenses/opentracing-mock-NOTICE.txt | 0 .../licenses/opentracing-noop-0.33.0.jar.sha1 | 1 - .../licenses/opentracing-noop-LICENSE-ASL.txt | 201 ------------------ solr/licenses/opentracing-noop-NOTICE.txt | 0 .../licenses/opentracing-util-0.33.0.jar.sha1 | 1 - .../licenses/opentracing-util-LICENSE-ASL.txt | 201 ------------------ solr/licenses/opentracing-util-NOTICE.txt | 0 solr/modules/opentelemetry/build.gradle | 14 +- .../opentelemetry/ClosableTracerShim.java | 85 -------- .../opentelemetry/OtelTracerConfigurator.java | 38 ++-- .../src/test-files/solr/solr.xml | 2 +- .../CustomTestOtelTracerConfigurator.java | 104 +++++++++ .../OtelTracerConfiguratorTest.java | 44 +--- .../TestDistributedTracing.java | 141 ++++++------ .../pages/major-changes-in-solr-10.adoc | 2 + solr/test-framework/build.gradle | 2 - .../java/org/apache/solr/SolrTestCaseJ4.java | 36 +--- versions.lock | 12 +- versions.props | 1 - 52 files changed, 496 insertions(+), 1456 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/util/tracing/HttpServletCarrier.java create mode 100644 solr/core/src/java/org/apache/solr/util/tracing/HttpServletRequestGetter.java rename solr/core/src/java/org/apache/solr/util/tracing/{SolrRequestCarrier.java => SolrRequestSetter.java} (60%) rename solr/core/src/test/org/apache/solr/util/tracing/{TestHttpServletCarrier.java => TestHttpServletRequestGetter.java} (54%) delete mode 100644 solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 create mode 100644 solr/licenses/opentelemetry-sdk-testing-1.29.0.jar.sha1 delete mode 100644 solr/licenses/opentracing-api-0.33.0.jar.sha1 delete mode 100644 solr/licenses/opentracing-api-LICENSE-ASL.txt delete mode 100644 solr/licenses/opentracing-api-NOTICE.txt delete mode 100644 solr/licenses/opentracing-mock-0.33.0.jar.sha1 delete mode 100644 solr/licenses/opentracing-mock-LICENSE-ASL.txt delete mode 100644 solr/licenses/opentracing-mock-NOTICE.txt delete mode 100644 solr/licenses/opentracing-noop-0.33.0.jar.sha1 delete mode 100644 solr/licenses/opentracing-noop-LICENSE-ASL.txt delete mode 100644 solr/licenses/opentracing-noop-NOTICE.txt delete mode 100644 solr/licenses/opentracing-util-0.33.0.jar.sha1 delete mode 100644 solr/licenses/opentracing-util-LICENSE-ASL.txt delete mode 100644 solr/licenses/opentracing-util-NOTICE.txt delete mode 100644 solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/ClosableTracerShim.java create mode 100644 solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java rename solr/{core/src/test/org/apache/solr/util/tracing => modules/opentelemetry/src/test/org/apache/solr/opentelemetry}/TestDistributedTracing.java (56%) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index b82508e9e6b..098080ed7c2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -17,6 +17,8 @@ Improvements * SOLR-16800: Solr CLI commands that use -solrUrl now work with a bare url like http://localhost:8983, you no longer need to add the /solr at the end. (Eric Pugh) +* SOLR-16536: Replace OpenTracing instrumentation with OpenTelemetry (Alex Deparvu, janhoy) + Optimizations --------------------- (No changes) diff --git a/solr/core/build.gradle b/solr/core/build.gradle index d868164cd75..61ecd1713af 100644 --- a/solr/core/build.gradle +++ b/solr/core/build.gradle @@ -141,10 +141,8 @@ dependencies { implementation 'com.tdunning:t-digest' // Distributed Tracing - api 'io.opentracing:opentracing-api' // Tracer is exposed on some methods - implementation 'io.opentracing:opentracing-noop' - implementation 'io.opentracing:opentracing-util' - testImplementation 'io.opentracing:opentracing-mock' + api 'io.opentelemetry:opentelemetry-api' // Tracer is exposed on some methods + implementation 'io.opentelemetry:opentelemetry-context' implementation 'org.apache.commons:commons-exec' diff --git a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java index 7ec6ab6a919..bfe35c47108 100644 --- a/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java +++ b/solr/core/src/java/org/apache/solr/api/AnnotatedApi.java @@ -215,7 +215,7 @@ public void call(SolrQueryRequest req, SolrQueryResponse rsp) { } for (CommandOperation cmd : cmds) { - TraceUtils.ifNotNoop(req.getSpan(), (span) -> span.log("Command: " + cmd.name)); + TraceUtils.ifNotNoop(req.getSpan(), (span) -> span.addEvent("Command: " + cmd.name)); commands.get(cmd.name).invoke(req, rsp, cmd); } diff --git a/solr/core/src/java/org/apache/solr/api/Api.java b/solr/core/src/java/org/apache/solr/api/Api.java index 54010912ae2..8e881b7f74e 100644 --- a/solr/core/src/java/org/apache/solr/api/Api.java +++ b/solr/core/src/java/org/apache/solr/api/Api.java @@ -41,7 +41,7 @@ public Map getCommandSchema() { if (commandSchema == null) { synchronized (this) { if (commandSchema == null) { - ValidatingJsonMap commands = getSpec().getMap("commands", null); + ValidatingJsonMap commands = spec != null ? getSpec().getMap("commands", null) : null; commandSchema = commands != null ? Map.copyOf(ApiBag.getParsedSchema(commands)) : Map.of(); } diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java index 46c66e403df..92008542a60 100644 --- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java +++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java @@ -23,8 +23,7 @@ import static org.apache.solr.servlet.SolrDispatchFilter.Action.PROCESS; import static org.apache.solr.servlet.SolrDispatchFilter.Action.REMOTEQUERY; -import io.opentracing.Span; -import io.opentracing.tag.Tags; +import io.opentelemetry.api.trace.Span; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.ArrayList; @@ -65,6 +64,7 @@ import org.apache.solr.servlet.SolrDispatchFilter; import org.apache.solr.servlet.SolrRequestParsers; import org.apache.solr.servlet.cache.Method; +import org.apache.solr.util.tracing.TraceUtils; import org.glassfish.jersey.server.ApplicationHandler; import org.glassfish.jersey.server.ContainerRequest; import org.slf4j.Logger; @@ -499,9 +499,7 @@ protected void populateTracingSpan(Span span) { coreOrColName = pathTemplateValues.get("core"); } } - if (coreOrColName != null) { - span.setTag(Tags.DB_INSTANCE, coreOrColName); - } + TraceUtils.setDbInstance(span, coreOrColName); // Get the templatize-ed path, ex: "/c/{collection}" final String path = computeEndpointPath(); @@ -521,7 +519,7 @@ protected void populateTracingSpan(Span span) { verb = req.getMethod().toLowerCase(Locale.ROOT); } - span.setOperationName(verb + ":" + path); + span.updateName(verb + ":" + path); } /** Example: /c/collection1/ and template map collection->collection1 produces /c/{collection}. */ diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index f223f41e36f..5606d93dc85 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -31,9 +31,7 @@ import com.github.benmanes.caffeine.cache.Interner; import com.google.common.annotations.VisibleForTesting; -import io.opentracing.Tracer; -import io.opentracing.noop.NoopTracer; -import io.opentracing.noop.NoopTracerFactory; +import io.opentelemetry.api.trace.Tracer; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.nio.file.Path; @@ -268,7 +266,7 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() { protected volatile SolrMetricsContext solrMetricsContext; - protected volatile Tracer tracer = NoopTracerFactory.create(); + protected volatile Tracer tracer; protected MetricsHandler metricsHandler; @@ -695,7 +693,7 @@ public MetricsHandler getMetricsHandler() { return metricsHandler; } - /** Never null but may implement {@link NoopTracer}. */ + /** Never null */ public Tracer getTracer() { return tracer; } @@ -1375,8 +1373,6 @@ public void shutdown() { org.apache.lucene.util.IOUtils.closeWhileHandlingException(packageLoader); } org.apache.lucene.util.IOUtils.closeWhileHandlingException(loader); // best effort - - tracer.close(); } public void cancelCoreRecoveries() { diff --git a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java index ba36ed127ec..bc1798863f7 100644 --- a/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java +++ b/solr/core/src/java/org/apache/solr/core/TracerConfigurator.java @@ -17,83 +17,23 @@ package org.apache.solr.core; -import io.opentracing.Scope; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.util.GlobalTracer; -import java.lang.invoke.MethodHandles; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.solr.common.util.ExecutorUtil; +import io.opentelemetry.api.trace.Tracer; import org.apache.solr.util.plugin.NamedListInitializedPlugin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.solr.util.tracing.TraceUtils; -/** Produces an OpenTracing {@link Tracer} from configuration. */ +/** Produces a {@link Tracer} from configuration. */ public abstract class TracerConfigurator implements NamedListInitializedPlugin { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public abstract Tracer getTracer(); public static Tracer loadTracer(SolrResourceLoader loader, PluginInfo info) { - // In "normal" Solr operation, loadTracer is called once in Solr's lifetime. - // In test mode we run many test suites, sometimes multiple servers per test suite. - // It's important that the tracing config not change throughout a test suite because of the - // static singleton pattern and assumptions based on this. - if (info != null && info.isEnabled()) { - GlobalTracer.registerIfAbsent( - () -> { - TracerConfigurator configurator = - loader.newInstance(info.className, TracerConfigurator.class); - configurator.init(info.initArgs); - return configurator.getTracer(); - }); - } - if (GlobalTracer.isRegistered()) { - // ideally we would furthermore check that it's not a no-op impl either but - // GlobalTracer.get() always returns a GlobalTracer implementing Tracer that delegates - // to the real Tracer (that may or may not be a No-Op impl). - ExecutorUtil.addThreadLocalProvider(new TracerConfigurator.SpanThreadLocalProvider()); + TracerConfigurator configurator = + loader.newInstance(info.className, TracerConfigurator.class); + configurator.init(info.initArgs); + return configurator.getTracer(); + } else { + return TraceUtils.noop(); } - - return GlobalTracer.get(); } - /** - * Propagate the active span across threads. New spans are not created, which means that we're - * possibly exposing a Span to a thread that may find that it has already finished, depending on - * how the instrumented thread pool is used. It's probably fine to create new spans related to a - * finished span? It's not okay to otherwise touch a finished span (e.g. to log or tag). - * - *

This strategy is also used by {@code - * brave.propagation.CurrentTraceContext#wrap(java.lang.Runnable)}. - */ - private static class SpanThreadLocalProvider - implements ExecutorUtil.InheritableThreadLocalProvider { - private final Tracer tracer = GlobalTracer.get(); - - @Override - public void store(AtomicReference ctx) { - assert tracer == GlobalTracer.get() : "Tracer changed; not supported!"; - ctx.set(tracer.scopeManager().activeSpan()); - } - - @Override - public void set(AtomicReference ctx) { - final Span span = (Span) ctx.get(); - if (span != null) { - log.trace("Thread received span to do async work: {}", span); - final Scope scope = tracer.scopeManager().activate(span); - ctx.set(scope); - } - } - - @Override - public void clean(AtomicReference ctx) { - Scope scope = (Scope) ctx.get(); - if (scope != null) { - scope.close(); - } - } - } + protected abstract Tracer getTracer(); } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java index 18caed1933a..1809d83f6b9 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/RestoreCollectionAPI.java @@ -108,8 +108,8 @@ public SubResponseAccumulatingJerseyResponse restoreCollection( } final String collectionName = requestBody.collection; - recordCollectionForLogAndTracing(collectionName, solrQueryRequest); SolrIdentifierValidator.validateCollectionName(collectionName); + recordCollectionForLogAndTracing(collectionName, solrQueryRequest); if (coreContainer.getAliases().hasAlias(collectionName)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, diff --git a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java index 0ad76294335..17fb78a86c6 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java @@ -16,9 +16,6 @@ */ package org.apache.solr.handler.component; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.propagation.Format; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,7 +47,7 @@ import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.security.AllowListUrlChecker; -import org.apache.solr.util.tracing.SolrRequestCarrier; +import org.apache.solr.util.tracing.TraceUtils; @NotThreadSafe public class HttpShardHandler extends ShardHandler { @@ -128,8 +125,6 @@ public void submit( final ShardRequest sreq, final String shard, final ModifiableSolrParams params) { // do this outside of the callable for thread safety reasons final List urls = getURLs(shard); - final Tracer tracer = sreq.tracer; // not null - final Span span = tracer.activeSpan(); // probably not null? params.remove(CommonParams.WT); // use default (currently javabin) params.remove(CommonParams.VERSION); @@ -171,10 +166,7 @@ public void submit( @Override public void onStart() { - if (span != null) { - tracer.inject( - span.context(), Format.Builtin.HTTP_HEADERS, new SolrRequestCarrier(req)); - } + TraceUtils.injectContextIntoRequest(req); SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo(); if (requestInfo != null) req.setUserPrincipal(requestInfo.getReq().getUserPrincipal()); diff --git a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java index 03b660ff9be..d7d7da04f22 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ShardRequest.java @@ -16,8 +16,6 @@ */ package org.apache.solr.handler.component; -import io.opentracing.Tracer; -import io.opentracing.util.GlobalTracer; import java.util.ArrayList; import java.util.List; import org.apache.solr.common.params.ModifiableSolrParams; @@ -58,9 +56,6 @@ public class ShardRequest { /** may be null */ public String nodeName; - /** Not null but may implement {@link io.opentracing.noop.NoopTracer}. */ - public final Tracer tracer = GlobalTracer.get(); - // TODO: one could store a list of numbers to correlate where returned docs // go in the top-level response rather than looking up by id... // this would work well if we ever transitioned to using internal ids and diff --git a/solr/core/src/java/org/apache/solr/request/DelegatingSolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/DelegatingSolrQueryRequest.java index 2c9c20d4f06..158f02cdff3 100644 --- a/solr/core/src/java/org/apache/solr/request/DelegatingSolrQueryRequest.java +++ b/solr/core/src/java/org/apache/solr/request/DelegatingSolrQueryRequest.java @@ -16,8 +16,7 @@ */ package org.apache.solr.request; -import io.opentracing.Span; -import io.opentracing.Tracer; +import io.opentelemetry.api.trace.Span; import java.security.Principal; import java.util.List; import java.util.Map; @@ -150,11 +149,6 @@ public HttpSolrCall getHttpSolrCall() { return delegate.getHttpSolrCall(); } - @Override - public Tracer getTracer() { - return delegate.getTracer(); - } - @Override public Span getSpan() { return delegate.getSpan(); diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java index 613ae31d438..113aeff6440 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequest.java @@ -16,10 +16,7 @@ */ package org.apache.solr.request; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.noop.NoopSpan; -import io.opentracing.util.GlobalTracer; +import io.opentelemetry.api.trace.Span; import java.security.Principal; import java.util.Collections; import java.util.List; @@ -136,21 +133,16 @@ default HttpSolrCall getHttpSolrCall() { return null; } - /** - * Distributed tracing Tracer. Never null but might implement {@link - * io.opentracing.noop.NoopTracer}. - */ - default Tracer getTracer() { - return GlobalTracer.get(); // default impl is only for some tests - } - /** * The distributed tracing Span for the request itself; never null. This is useful for adding tags - * or updating the operation name of the request span. If you need the current span, which might - * not necessarily be the request span, do this instead: {@code tracer.activeSpan()}. + * or updating the operation name of the request span. Not null. */ default Span getSpan() { - return NoopSpan.INSTANCE; + final HttpSolrCall call = getHttpSolrCall(); + if (call != null) { + return call.getSpan(); + } + return Span.getInvalid(); } default CoreContainer getCoreContainer() { diff --git a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java index f1034b00bec..363022c3979 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java +++ b/solr/core/src/java/org/apache/solr/request/SolrQueryRequestBase.java @@ -16,10 +16,6 @@ */ package org.apache.solr.request; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.noop.NoopSpan; -import io.opentracing.util.GlobalTracer; import java.io.Closeable; import java.security.Principal; import java.util.Collections; @@ -38,7 +34,6 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.schema.IndexSchema; import org.apache.solr.search.SolrIndexSearcher; -import org.apache.solr.servlet.HttpSolrCall; import org.apache.solr.util.RTimerTree; import org.apache.solr.util.RefCounted; @@ -145,33 +140,6 @@ public IndexSchema getSchema() { return schema; } - @Override - public Tracer getTracer() { - final HttpSolrCall call = getHttpSolrCall(); - if (call != null) { - final Tracer tracer = (Tracer) call.getReq().getAttribute(Tracer.class.getName()); - if (tracer != null) { - return tracer; - } - } - if (core != null) { - return core.getCoreContainer().getTracer(); - } - return GlobalTracer.get(); // this way is not ideal (particularly in tests) but it's okay - } - - @Override - public Span getSpan() { - final HttpSolrCall call = getHttpSolrCall(); - if (call != null) { - final Span span = (Span) call.getReq().getAttribute(Span.class.getName()); - if (span != null) { - return span; - } - } - return NoopSpan.INSTANCE; - } - @Override public void updateSchemaToLatest() { schema = core.getLatestSchema(); diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java index dbee25ec32f..f1e5a25693c 100644 --- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java +++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java @@ -32,8 +32,7 @@ import static org.apache.solr.servlet.SolrDispatchFilter.Action.RETRY; import static org.apache.solr.servlet.SolrDispatchFilter.Action.RETURN; -import io.opentracing.Span; -import io.opentracing.tag.Tags; +import io.opentelemetry.api.trace.Span; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; @@ -50,7 +49,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -626,10 +624,14 @@ protected void handleAdminOrRemoteRequest() throws IOException { "handleOrForwardRequest should not be invoked when serving v1 requests."); } - /** Get the span for this request. Not null. */ - protected Span getSpan() { - // Span was put into the request by SolrDispatchFilter - return (Span) Objects.requireNonNull(req.getAttribute(Span.class.getName())); + /** Get the Span for this request. Not null. */ + public Span getSpan() { + var s = TraceUtils.getSpan(req); + if (s != null) { + return s; + } else { + return Span.getInvalid(); + } } // called after init(). @@ -639,9 +641,7 @@ protected void populateTracingSpan(Span span) { if (coreOrColName == null && getCore() != null) { coreOrColName = getCore().getName(); } - if (coreOrColName != null) { - span.setTag(Tags.DB_INSTANCE, coreOrColName); - } + TraceUtils.setDbInstance(span, coreOrColName); // Set operation name. String path = getPath(); @@ -655,7 +655,7 @@ protected void populateTracingSpan(Span span) { } String verb = getQueryParams().get(CoreAdminParams.ACTION, req.getMethod()).toLowerCase(Locale.ROOT); - span.setOperationName(verb + ":" + path); + span.updateName(verb + ":" + path); } public boolean shouldAudit() { diff --git a/solr/core/src/java/org/apache/solr/servlet/ServletUtils.java b/solr/core/src/java/org/apache/solr/servlet/ServletUtils.java index a286078dafb..5297fe44401 100644 --- a/solr/core/src/java/org/apache/solr/servlet/ServletUtils.java +++ b/solr/core/src/java/org/apache/solr/servlet/ServletUtils.java @@ -17,12 +17,8 @@ package org.apache.solr.servlet; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.noop.NoopSpan; -import io.opentracing.noop.NoopTracer; -import io.opentracing.propagation.Format; -import io.opentracing.tag.Tags; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -45,7 +41,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.logging.MDCLoggingContext; -import org.apache.solr.util.tracing.HttpServletCarrier; +import org.apache.solr.util.tracing.TraceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -234,15 +230,14 @@ static void rateLimitRequest( private static void traceHttpRequestExecution2( HttpServletRequest request, HttpServletResponse response, Runnable tracedExecution) throws ServletException, IOException { - Tracer tracer = getTracer(request); - Span span = buildSpan(tracer, request); - - request.setAttribute(SolrDispatchFilter.ATTR_TRACING_SPAN, span); - try (var scope = tracer.scopeManager().activate(span)) { + Context context = TraceUtils.extractContext(request); + Span span = TraceUtils.startHttpRequestSpan(request, context); + try (var scope = context.with(span).makeCurrent()) { assert scope != null; // prevent javac warning about scope being unused - MDCLoggingContext.setTracerId(span.context().toTraceId()); // handles empty string - + TraceUtils.setSpan(request, span); + TraceUtils.ifNotNoop( + span, s -> MDCLoggingContext.setTracerId(s.getSpanContext().getTraceId())); tracedExecution.run(); } catch (ExceptionWhileTracing e) { if (e.e instanceof SolrAuthenticationException) { @@ -261,31 +256,9 @@ private static void traceHttpRequestExecution2( throw new RuntimeException(e.e); } } finally { - span.setTag(Tags.HTTP_STATUS, response.getStatus()); - span.finish(); - } - } - - private static Tracer getTracer(HttpServletRequest req) { - return (Tracer) req.getAttribute(SolrDispatchFilter.ATTR_TRACING_TRACER); - } - - protected static Span buildSpan(Tracer tracer, HttpServletRequest request) { - if (tracer instanceof NoopTracer) { - return NoopSpan.INSTANCE; - } - Tracer.SpanBuilder spanBuilder = - tracer - .buildSpan("http.request") // will be changed later - .asChildOf(tracer.extract(Format.Builtin.HTTP_HEADERS, new HttpServletCarrier(request))) - .withTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER) - .withTag(Tags.HTTP_METHOD, request.getMethod()) - .withTag(Tags.HTTP_URL, request.getRequestURL().toString()); - if (request.getQueryString() != null) { - spanBuilder.withTag("http.params", request.getQueryString()); + TraceUtils.setHttpStatus(span, response.getStatus()); + span.end(); } - spanBuilder.withTag(Tags.DB_TYPE, "solr"); - return spanBuilder.start(); } // we make sure we read the full client request so that the client does diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index cfd9836c4ed..7b9c2b7e90c 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -16,16 +16,13 @@ */ package org.apache.solr.servlet; -import static org.apache.solr.security.AuditEvent.EventType; import static org.apache.solr.servlet.ServletUtils.closeShield; import static org.apache.solr.servlet.ServletUtils.configExcludes; import static org.apache.solr.servlet.ServletUtils.excludedPath; +import static org.apache.solr.util.tracing.TraceUtils.getSpan; +import static org.apache.solr.util.tracing.TraceUtils.setTracer; import com.google.common.annotations.VisibleForTesting; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.tag.Tags; -import io.opentracing.util.GlobalTracer; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.List; @@ -54,10 +51,12 @@ import org.apache.solr.logging.MDCSnapshot; import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.security.AuditEvent; +import org.apache.solr.security.AuditEvent.EventType; import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.PublicKeyHandler; import org.apache.solr.servlet.CoreContainerProvider.ServiceHolder; +import org.apache.solr.util.tracing.TraceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,8 +72,6 @@ // things like CoreContainer can be requested. (or better yet injected) public class SolrDispatchFilter extends BaseSolrFilter implements PathExcluder { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ATTR_TRACING_SPAN = Span.class.getName(); - public static final String ATTR_TRACING_TRACER = Tracer.class.getName(); // TODO: see if we can get rid of the holder here (Servlet spec actually guarantees // ContextListeners run before filter init, but JettySolrRunner that we use for tests is @@ -206,8 +203,7 @@ public void doFilter( if (excludedPath(excludePatterns, request, response, chain)) { return; } - Tracer t = getCores() == null ? GlobalTracer.get() : getCores().getTracer(); - request.setAttribute(ATTR_TRACING_TRACER, t); + setTracer(request, getCores().getTracer()); RateLimitManager rateLimitManager = coreService.getService().getRateLimitManager(); try { ServletUtils.rateLimitRequest( @@ -228,10 +224,6 @@ public void doFilter( } } - private static Span getSpan(HttpServletRequest req) { - return (Span) req.getAttribute(ATTR_TRACING_SPAN); - } - private void dispatch( FilterChain chain, HttpServletRequest request, HttpServletResponse response, boolean retry) throws IOException, ServletException, SolrAuthenticationException { @@ -242,11 +234,12 @@ private void dispatch( request = wrappedRequest.get(); } + var span = getSpan(request); if (getCores().getAuthenticationPlugin() != null) { if (log.isDebugEnabled()) { log.debug("User principal: {}", request.getUserPrincipal()); } - getSpan(request).setTag(Tags.DB_USER, String.valueOf(request.getUserPrincipal())); + TraceUtils.setUser(span, String.valueOf(request.getUserPrincipal())); } HttpSolrCall call = getHttpSolrCall(request, response, retry); @@ -255,15 +248,15 @@ private void dispatch( Action result = call.call(); switch (result) { case PASSTHROUGH: - getSpan(request).log("SolrDispatchFilter PASSTHROUGH"); + span.addEvent("SolrDispatchFilter PASSTHROUGH"); chain.doFilter(request, response); break; case RETRY: - getSpan(request).log("SolrDispatchFilter RETRY"); + span.addEvent("SolrDispatchFilter RETRY"); doFilter(request, response, chain, true); // RECURSION break; case FORWARD: - getSpan(request).log("SolrDispatchFilter FORWARD"); + span.addEvent("SolrDispatchFilter FORWARD"); request.getRequestDispatcher(call.getPath()).forward(request, response); break; case ADMIN: diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java index b7687d4eb1b..b40c79a166f 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java +++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java @@ -16,9 +16,6 @@ */ package org.apache.solr.update; -import io.opentracing.Span; -import io.opentracing.Tracer; -import io.opentracing.propagation.Format; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -52,7 +49,7 @@ import org.apache.solr.update.processor.DistributedUpdateProcessor; import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker; import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker; -import org.apache.solr.util.tracing.SolrRequestCarrier; +import org.apache.solr.util.tracing.TraceUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -314,13 +311,7 @@ private void submit(final Req req, boolean isCommit) throws IOException { if (SolrRequestInfo.getRequestInfo() != null) { req.uReq.setUserPrincipal(SolrRequestInfo.getRequestInfo().getReq().getUserPrincipal()); } - - Tracer tracer = req.cmd.getTracer(); - Span parentSpan = tracer.activeSpan(); - if (parentSpan != null) { - tracer.inject( - parentSpan.context(), Format.Builtin.HTTP_HEADERS, new SolrRequestCarrier(req.uReq)); - } + TraceUtils.injectContextIntoRequest(req.uReq); if (req.synchronous) { blockAndDoRetries(); diff --git a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java index 9714d712dae..79bbda1f3d3 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateCommand.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateCommand.java @@ -16,13 +16,11 @@ */ package org.apache.solr.update; -import io.opentracing.Tracer; -import io.opentracing.util.GlobalTracer; import org.apache.solr.request.SolrQueryRequest; /** An index update command encapsulated in an object (Command pattern) */ public abstract class UpdateCommand implements Cloneable { - protected SolrQueryRequest req; + protected final SolrQueryRequest req; protected long version; protected String route; protected int flags; @@ -91,18 +89,6 @@ public SolrQueryRequest getReq() { return req; } - public void setReq(SolrQueryRequest req) { - this.req = req; - } - - /** - * Distributed tracing Tracer. Never null but might implement {@link - * io.opentracing.noop.NoopTracer}. - */ - public Tracer getTracer() { - return (req != null) ? req.getTracer() : GlobalTracer.get(); - } - @Override public UpdateCommand clone() { try { diff --git a/solr/core/src/java/org/apache/solr/util/tracing/HttpServletCarrier.java b/solr/core/src/java/org/apache/solr/util/tracing/HttpServletCarrier.java deleted file mode 100644 index 21ce05e29d1..00000000000 --- a/solr/core/src/java/org/apache/solr/util/tracing/HttpServletCarrier.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.util.tracing; - -import io.opentracing.propagation.TextMap; -import java.util.Enumeration; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import javax.servlet.http.HttpServletRequest; - -/** A Carrier for extract Span context out of request headers */ -public class HttpServletCarrier implements TextMap { - private Iterator> it; - - public HttpServletCarrier(HttpServletRequest request) { - this.it = - new Iterator<>() { - - Enumeration headerNameIt = request.getHeaderNames(); - String headerName = null; - Enumeration headerValue = null; - - @Override - public boolean hasNext() { - if (headerValue != null && headerValue.hasMoreElements()) { - return true; - } - - return headerNameIt.hasMoreElements(); - } - - @Override - public Map.Entry next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - - if (headerValue == null || !headerValue.hasMoreElements()) { - headerName = headerNameIt.nextElement(); - headerValue = request.getHeaders(headerName); - } - - String key = headerName; - String val = headerValue.nextElement(); - - return new Map.Entry<>() { - @Override - public String getKey() { - return key; - } - - @Override - public String getValue() { - return val; - } - - @Override - public String setValue(String value) { - throw new UnsupportedOperationException(); - } - }; - } - }; - } - - @Override - public Iterator> iterator() { - return it; - } - - @Override - public void put(String key, String value) { - throw new UnsupportedOperationException( - "HttpServletCarrier should only be used with Tracer.extract()"); - } -} diff --git a/solr/core/src/java/org/apache/solr/util/tracing/HttpServletRequestGetter.java b/solr/core/src/java/org/apache/solr/util/tracing/HttpServletRequestGetter.java new file mode 100644 index 00000000000..f2756566ed9 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/util/tracing/HttpServletRequestGetter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util.tracing; + +import io.opentelemetry.context.propagation.TextMapGetter; +import java.util.Iterator; +import javax.servlet.http.HttpServletRequest; + +/** + * {@code HttpServletRequest} aware {@code TextMapGetter} that allows header data to be extracted + * from a request + */ +public class HttpServletRequestGetter implements TextMapGetter { + + @Override + public Iterable keys(HttpServletRequest carrier) { + return new Iterable() { + @Override + public Iterator iterator() { + return carrier.getHeaderNames().asIterator(); + } + }; + } + + @Override + public String get(HttpServletRequest carrier, String key) { + if (carrier == null) { + return null; + } + return carrier.getHeader(key); + } +} diff --git a/solr/core/src/java/org/apache/solr/util/tracing/SolrRequestCarrier.java b/solr/core/src/java/org/apache/solr/util/tracing/SolrRequestSetter.java similarity index 60% rename from solr/core/src/java/org/apache/solr/util/tracing/SolrRequestCarrier.java rename to solr/core/src/java/org/apache/solr/util/tracing/SolrRequestSetter.java index 6f4917e1b8f..0923d9db014 100644 --- a/solr/core/src/java/org/apache/solr/util/tracing/SolrRequestCarrier.java +++ b/solr/core/src/java/org/apache/solr/util/tracing/SolrRequestSetter.java @@ -17,27 +17,16 @@ package org.apache.solr.util.tracing; -import io.opentracing.propagation.TextMap; -import java.util.Iterator; -import java.util.Map; +import io.opentelemetry.context.propagation.TextMapSetter; import org.apache.solr.client.solrj.SolrRequest; -/** An OpenTracing Carrier for injecting Span context through SolrRequest */ -public class SolrRequestCarrier implements TextMap { - - private final SolrRequest solrRequest; - - public SolrRequestCarrier(SolrRequest solrRequest) { - this.solrRequest = solrRequest; - } - - @Override - public Iterator> iterator() { - throw new UnsupportedOperationException("carrier is write-only"); - } +/** + * {@code SolrRequest} aware {@code TextMapSetter} that allows header data to be added to a request + */ +public class SolrRequestSetter implements TextMapSetter> { @Override - public void put(String key, String value) { - solrRequest.addHeader(key, value); + public void set(SolrRequest request, String key, String value) { + request.addHeader(key, value); } } diff --git a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java index c7d2f0f0fd9..3505f4daac6 100644 --- a/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java +++ b/solr/core/src/java/org/apache/solr/util/tracing/TraceUtils.java @@ -16,24 +16,128 @@ */ package org.apache.solr.util.tracing; -import io.opentracing.Span; -import io.opentracing.noop.NoopSpan; -import io.opentracing.tag.Tags; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.propagation.TextMapPropagator; import java.util.function.Consumer; +import javax.servlet.http.HttpServletRequest; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.request.SolrQueryRequest; /** Utilities for distributed tracing. */ public class TraceUtils { + private static final String REQ_ATTR_TRACING_SPAN = Span.class.getName(); + private static final String REQ_ATTR_TRACING_TRACER = Tracer.class.getName(); + private static final Tracer NOOP_TRACER = TracerProvider.noop().get(null); + + public static final String DEFAULT_SPAN_NAME = "http.request"; + public static final String WRITE_QUERY_RESPONSE_SPAN_NAME = "writeQueryResponse"; + + public static final AttributeKey TAG_DB = AttributeKey.stringKey("db.instance"); + public static final AttributeKey TAG_DB_TYPE = AttributeKey.stringKey("db.type"); + public static final AttributeKey TAG_USER = AttributeKey.stringKey("db.user"); + public static final AttributeKey TAG_HTTP_STATUS = + AttributeKey.longKey("http.response.status_code"); + public static final AttributeKey TAG_HTTP_METHOD = + AttributeKey.stringKey("http.request.method"); + public static final AttributeKey TAG_HTTP_URL = AttributeKey.stringKey("http.url"); + public static final AttributeKey TAG_HTTP_PARAMS = AttributeKey.stringKey("http.params"); + public static final AttributeKey TAG_RESPONSE_WRITER = + AttributeKey.stringKey("responseWriter"); + public static final AttributeKey TAG_CONTENT_TYPE = AttributeKey.stringKey("contentType"); + + @Deprecated + private static final AttributeKey TAG_HTTP_METHOD_DEP = + AttributeKey.stringKey("http.method"); + + @Deprecated + private static final AttributeKey TAG_HTTP_STATUS_DEP = + AttributeKey.longKey("http.status_code"); + + public static final String TAG_DB_TYPE_SOLR = "solr"; + + public static Tracer noop() { + return NOOP_TRACER; + } + + public static TextMapPropagator getTextMapPropagator() { + return GlobalOpenTelemetry.getPropagators().getTextMapPropagator(); + } + public static void setDbInstance(SolrQueryRequest req, String coreOrColl) { - if (req != null && coreOrColl != null) { - ifNotNoop(req.getSpan(), (span) -> span.setTag(Tags.DB_INSTANCE, coreOrColl)); + if (req != null) { + setDbInstance(req.getSpan(), coreOrColl); + } + } + + public static void setDbInstance(Span span, String coreOrColName) { + if (coreOrColName != null) { + span.setAttribute(TAG_DB, coreOrColName); } } + public static void setUser(Span span, String user) { + span.setAttribute(TAG_USER, user); + } + + public static void setHttpStatus(Span span, int httpStatus) { + span.setAttribute(TAG_HTTP_STATUS, httpStatus); + span.setAttribute(TAG_HTTP_STATUS_DEP, httpStatus); + } + public static void ifNotNoop(Span span, Consumer consumer) { - if (span != null && !(span instanceof NoopSpan)) { + if (span.isRecording()) { consumer.accept(span); } } + + public static void setSpan(HttpServletRequest req, Span span) { + req.setAttribute(REQ_ATTR_TRACING_SPAN, span); + } + + public static Span getSpan(HttpServletRequest req) { + return (Span) req.getAttribute(REQ_ATTR_TRACING_SPAN); + } + + public static void setTracer(HttpServletRequest req, Tracer t) { + req.setAttribute(REQ_ATTR_TRACING_TRACER, t); + } + + public static Tracer getTracer(HttpServletRequest req) { + return (Tracer) req.getAttribute(REQ_ATTR_TRACING_TRACER); + } + + public static Context extractContext(HttpServletRequest req) { + TextMapPropagator textMapPropagator = getTextMapPropagator(); + return textMapPropagator.extract(Context.current(), req, new HttpServletRequestGetter()); + } + + public static void injectContextIntoRequest(SolrRequest req) { + TextMapPropagator textMapPropagator = getTextMapPropagator(); + textMapPropagator.inject(Context.current(), req, new SolrRequestSetter()); + } + + public static Span startHttpRequestSpan(HttpServletRequest request, Context context) { + Tracer tracer = getTracer(request); + SpanBuilder spanBuilder = + tracer + .spanBuilder(DEFAULT_SPAN_NAME) // will be changed later + .setParent(context) + .setSpanKind(SpanKind.SERVER) + .setAttribute(TAG_HTTP_METHOD, request.getMethod()) + .setAttribute(TAG_HTTP_METHOD_DEP, request.getMethod()) + .setAttribute(TAG_HTTP_URL, request.getRequestURL().toString()); + if (request.getQueryString() != null) { + spanBuilder.setAttribute(TAG_HTTP_PARAMS, request.getQueryString()); + } + spanBuilder.setAttribute(TAG_DB_TYPE, TAG_DB_TYPE_SOLR); + return spanBuilder.startSpan(); + } } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java index 501d22b130c..6de9c823536 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/AddReplicaPropertyAPITest.java @@ -24,7 +24,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import io.opentracing.noop.NoopSpan; +import io.opentelemetry.api.trace.Span; import java.util.Map; import java.util.Optional; import org.apache.solr.SolrTestCaseJ4; @@ -73,7 +73,7 @@ public void setUp() throws Exception { when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong())) .thenReturn(new OverseerSolrResponse(new NamedList<>())); mockQueryRequest = mock(SolrQueryRequest.class); - when(mockQueryRequest.getSpan()).thenReturn(NoopSpan.INSTANCE); + when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); queryResponse = new SolrQueryResponse(); messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class); diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java index 19fd1ea96e2..e6fe9edad0b 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/CoreReplicationAPITest.java @@ -20,7 +20,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import io.opentracing.noop.NoopSpan; +import io.opentelemetry.api.trace.Span; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -53,7 +53,7 @@ public void setUp() throws Exception { super.setUp(); setUpMocks(); mockQueryRequest = mock(SolrQueryRequest.class); - when(mockQueryRequest.getSpan()).thenReturn(NoopSpan.INSTANCE); + when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); queryResponse = new SolrQueryResponse(); coreReplicationAPI = new CoreReplicationAPIMock(mockCore, mockQueryRequest, queryResponse); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ListAliasesAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ListAliasesAPITest.java index 4cb4d065e36..2a6607c40b6 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/api/ListAliasesAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ListAliasesAPITest.java @@ -19,7 +19,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import io.opentracing.noop.NoopSpan; +import io.opentelemetry.api.trace.Span; import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; @@ -67,7 +67,7 @@ public void setUp() throws Exception { when(mockCoreContainer.getZkController()).thenReturn(zkController); mockQueryRequest = mock(SolrQueryRequest.class); - when(mockQueryRequest.getSpan()).thenReturn(NoopSpan.INSTANCE); + when(mockQueryRequest.getSpan()).thenReturn(Span.getInvalid()); queryResponse = new SolrQueryResponse(); getAliasesAPI = new ListAliasesAPI(mockCoreContainer, mockQueryRequest, queryResponse); diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestHttpServletCarrier.java b/solr/core/src/test/org/apache/solr/util/tracing/TestHttpServletRequestGetter.java similarity index 54% rename from solr/core/src/test/org/apache/solr/util/tracing/TestHttpServletCarrier.java rename to solr/core/src/test/org/apache/solr/util/tracing/TestHttpServletRequestGetter.java index 4001af36b1c..d2b5fdf9bbb 100644 --- a/solr/core/src/test/org/apache/solr/util/tracing/TestHttpServletCarrier.java +++ b/solr/core/src/test/org/apache/solr/util/tracing/TestHttpServletRequestGetter.java @@ -17,10 +17,6 @@ package org.apache.solr.util.tracing; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; @@ -29,37 +25,43 @@ import java.util.Map; import java.util.Set; import javax.servlet.http.HttpServletRequest; -import org.apache.solr.SolrTestCaseJ4; +import javax.servlet.http.HttpServletRequestWrapper; +import org.apache.solr.SolrTestCase; +import org.eclipse.jetty.server.Request; import org.junit.Test; -import org.mockito.stubbing.Answer; -public class TestHttpServletCarrier extends SolrTestCaseJ4 { +public class TestHttpServletRequestGetter extends SolrTestCase { @Test public void test() { - SolrTestCaseJ4.assumeWorkingMockito(); - HttpServletRequest req = mock(HttpServletRequest.class); Map> headers = Map.of( - "a", Set.of("a", "b", "c"), - "b", Set.of("a", "b"), - "c", Set.of("a")); + "a", Set.of("0"), + "b", Set.of("1"), + "c", Set.of("2")); + + HttpServletRequest req = + new HttpServletRequestWrapper(new Request(null, null)) { + + @Override + public String getHeader(String name) { + return headers.get(name).iterator().next(); + } - when(req.getHeaderNames()).thenReturn(Collections.enumeration(headers.keySet())); - when(req.getHeaders(anyString())) - .thenAnswer( - (Answer>) - inv -> { - String key = inv.getArgument(0); - return Collections.enumeration(headers.get(key)); - }); + @Override + public Enumeration getHeaderNames() { + return Collections.enumeration(headers.keySet()); + } + }; - HttpServletCarrier servletCarrier = new HttpServletCarrier(req); - Iterator> it = servletCarrier.iterator(); + HttpServletRequestGetter httpServletRequestGetter = new HttpServletRequestGetter(); + Iterator it = httpServletRequestGetter.keys(req).iterator(); Map> resultBack = new HashMap<>(); while (it.hasNext()) { - Map.Entry entry = it.next(); - resultBack.computeIfAbsent(entry.getKey(), k -> new HashSet<>()).add(entry.getValue()); + String key = it.next(); + resultBack + .computeIfAbsent(key, k -> new HashSet<>()) + .add(httpServletRequestGetter.get(req, key)); } assertEquals(headers, resultBack); } diff --git a/solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 deleted file mode 100644 index 1d378f65bc4..00000000000 --- a/solr/licenses/opentelemetry-opentracing-shim-1.29.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cbe64aea8ea3811911bf6e572f96ec0bc745948d diff --git a/solr/licenses/opentelemetry-sdk-testing-1.29.0.jar.sha1 b/solr/licenses/opentelemetry-sdk-testing-1.29.0.jar.sha1 new file mode 100644 index 00000000000..87fa435be2c --- /dev/null +++ b/solr/licenses/opentelemetry-sdk-testing-1.29.0.jar.sha1 @@ -0,0 +1 @@ +3f36cd924e7631a6888d2280a3b368b3dce3acfc diff --git a/solr/licenses/opentracing-api-0.33.0.jar.sha1 b/solr/licenses/opentracing-api-0.33.0.jar.sha1 deleted file mode 100644 index c7b2aafb365..00000000000 --- a/solr/licenses/opentracing-api-0.33.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67336cfb9d93779c02e1fda4c87801d352720eda diff --git a/solr/licenses/opentracing-api-LICENSE-ASL.txt b/solr/licenses/opentracing-api-LICENSE-ASL.txt deleted file mode 100644 index 261eeb9e9f8..00000000000 --- a/solr/licenses/opentracing-api-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/opentracing-api-NOTICE.txt b/solr/licenses/opentracing-api-NOTICE.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/solr/licenses/opentracing-mock-0.33.0.jar.sha1 b/solr/licenses/opentracing-mock-0.33.0.jar.sha1 deleted file mode 100644 index 684a6c3d912..00000000000 --- a/solr/licenses/opentracing-mock-0.33.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -68f907743a5b355a8e975fdd3df0d2e106bbad6c diff --git a/solr/licenses/opentracing-mock-LICENSE-ASL.txt b/solr/licenses/opentracing-mock-LICENSE-ASL.txt deleted file mode 100644 index 261eeb9e9f8..00000000000 --- a/solr/licenses/opentracing-mock-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/opentracing-mock-NOTICE.txt b/solr/licenses/opentracing-mock-NOTICE.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/solr/licenses/opentracing-noop-0.33.0.jar.sha1 b/solr/licenses/opentracing-noop-0.33.0.jar.sha1 deleted file mode 100644 index ea4e70775a2..00000000000 --- a/solr/licenses/opentracing-noop-0.33.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -074b9950a587f53fbdb48c3f1f84f1ece8c10592 diff --git a/solr/licenses/opentracing-noop-LICENSE-ASL.txt b/solr/licenses/opentracing-noop-LICENSE-ASL.txt deleted file mode 100644 index 261eeb9e9f8..00000000000 --- a/solr/licenses/opentracing-noop-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/opentracing-noop-NOTICE.txt b/solr/licenses/opentracing-noop-NOTICE.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/solr/licenses/opentracing-util-0.33.0.jar.sha1 b/solr/licenses/opentracing-util-0.33.0.jar.sha1 deleted file mode 100644 index b0c42dcc5bf..00000000000 --- a/solr/licenses/opentracing-util-0.33.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -132630f17e198a1748f23ce33597efdf4a807fb9 diff --git a/solr/licenses/opentracing-util-LICENSE-ASL.txt b/solr/licenses/opentracing-util-LICENSE-ASL.txt deleted file mode 100644 index 261eeb9e9f8..00000000000 --- a/solr/licenses/opentracing-util-LICENSE-ASL.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/opentracing-util-NOTICE.txt b/solr/licenses/opentracing-util-NOTICE.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/solr/modules/opentelemetry/build.gradle b/solr/modules/opentelemetry/build.gradle index 444d3c6ffb4..4426f67649e 100644 --- a/solr/modules/opentelemetry/build.gradle +++ b/solr/modules/opentelemetry/build.gradle @@ -21,17 +21,13 @@ description = 'Open Telemetry (OTEL) tracer' dependencies { implementation project(':solr:core') + implementation project(':solr:solrj') implementation platform('io.opentelemetry:opentelemetry-bom') - - implementation 'io.opentracing:opentracing-api' implementation 'org.slf4j:slf4j-api' - implementation 'io.opentelemetry:opentelemetry-sdk' - implementation 'io.opentelemetry:opentelemetry-sdk-trace' + implementation 'io.opentelemetry:opentelemetry-api' implementation 'io.opentelemetry:opentelemetry-sdk-extension-autoconfigure' - implementation 'io.opentelemetry:opentelemetry-opentracing-shim' - runtimeOnly 'io.opentelemetry:opentelemetry-semconv' runtimeOnly 'io.opentelemetry:opentelemetry-exporter-otlp' // End users must recompile with jaeger exporter and/or zipkin exporter if they need these @@ -46,8 +42,8 @@ dependencies { compileOnly 'org.apache.tomcat:annotations-api' testImplementation project(':solr:test-framework') - testImplementation project(':solr:solrj') - testImplementation 'com.carrotsearch.randomizedtesting:randomizedtesting-runner' testImplementation 'junit:junit' - testImplementation 'io.opentracing:opentracing-util' + testImplementation 'io.opentelemetry:opentelemetry-sdk' + testImplementation 'io.opentelemetry:opentelemetry-sdk-trace' + testImplementation 'io.opentelemetry:opentelemetry-sdk-testing' } diff --git a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/ClosableTracerShim.java b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/ClosableTracerShim.java deleted file mode 100644 index ee7fb962ce9..00000000000 --- a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/ClosableTracerShim.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.opentelemetry; - -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentracing.Scope; -import io.opentracing.ScopeManager; -import io.opentracing.Span; -import io.opentracing.SpanContext; -import io.opentracing.Tracer; -import io.opentracing.propagation.Format; -import java.lang.invoke.MethodHandles; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Delegate shim that forwards all calls to the actual {@link - * io.opentelemetry.opentracingshim.OpenTracingShim}, and in addition calls {@link - * SdkTracerProvider#close()} to really close the OTEL SDK tracer when the OT shim is closed. - * - *

TODO: This can be removed once we migrate Solr instrumentation from OpenTracing to - * OpenTelemetry - */ -public class ClosableTracerShim implements Tracer { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final Tracer shim; - private final SdkTracerProvider sdkTracerProvider; - - public ClosableTracerShim(Tracer shim, SdkTracerProvider sdkTracerProvider) { - this.shim = shim; - this.sdkTracerProvider = sdkTracerProvider; - } - - @Override - public ScopeManager scopeManager() { - return shim.scopeManager(); - } - - @Override - public Span activeSpan() { - return shim.activeSpan(); - } - - @Override - public Scope activateSpan(Span span) { - return shim.activateSpan(span); - } - - @Override - public SpanBuilder buildSpan(String operationName) { - return shim.buildSpan(operationName); - } - - @Override - public void inject(SpanContext spanContext, Format format, C carrier) { - shim.inject(spanContext, format, carrier); - } - - @Override - public SpanContext extract(Format format, C carrier) { - return shim.extract(format, carrier); - } - - @Override - public void close() { - shim.close(); - log.debug("Closing wrapped OTEL tracer instance."); - sdkTracerProvider.forceFlush(); - sdkTracerProvider.close(); - } -} diff --git a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java index 6b876c7c1b6..25d819718dc 100644 --- a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java +++ b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java @@ -16,10 +16,9 @@ */ package org.apache.solr.opentelemetry; -import io.opentelemetry.opentracingshim.OpenTracingShim; -import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; -import io.opentracing.Tracer; import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.HashMap; @@ -27,26 +26,45 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.TracerConfigurator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * OpenTracing TracerConfigurator implementation which exports spans to OpenTelemetry in OTLP - * format. This impl re-uses the existing OpenTracing instrumentation through a shim, and takes care - * of properly closing the backing Tracer when Solr shuts down. + * Tracing TracerConfigurator implementation which exports spans to OpenTelemetry in OTLP format. */ public class OtelTracerConfigurator extends TracerConfigurator { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - // Copy of environment. Can be overridden by tests - Map currentEnv = System.getenv(); + + private final Map currentEnv; + + public OtelTracerConfigurator() { + this(System.getenv()); + } + + OtelTracerConfigurator(Map currentEnv) { + this.currentEnv = currentEnv; + } @Override public Tracer getTracer() { + // TODO remove reliance on global + return GlobalOpenTelemetry.getTracer("solr"); + } + + @Override + public void init(NamedList args) { + prepareConfiguration(); + AutoConfiguredOpenTelemetrySdk.initialize(); + } + + void prepareConfiguration() { setDefaultIfNotConfigured("OTEL_SERVICE_NAME", "solr"); setDefaultIfNotConfigured("OTEL_TRACES_EXPORTER", "otlp"); setDefaultIfNotConfigured("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"); setDefaultIfNotConfigured("OTEL_TRACES_SAMPLER", "parentbased_always_on"); + setDefaultIfNotConfigured("OTEL_PROPAGATORS", "tracecontext,baggage"); addOtelResourceAttributes(Map.of("host.name", System.getProperty("host"))); final String currentConfig = getCurrentOtelConfigAsString(); @@ -62,10 +80,6 @@ public Tracer getTracer() { } System.setProperty("otel.metrics.exporter", "none"); System.setProperty("otel.logs.exporter", "none"); - - OpenTelemetrySdk otelSdk = AutoConfiguredOpenTelemetrySdk.initialize().getOpenTelemetrySdk(); - Tracer shim = OpenTracingShim.createTracerShim(otelSdk); - return new ClosableTracerShim(shim, otelSdk.getSdkTracerProvider()); } /** diff --git a/solr/modules/opentelemetry/src/test-files/solr/solr.xml b/solr/modules/opentelemetry/src/test-files/solr/solr.xml index 6ad63be5e84..76926369f0d 100644 --- a/solr/modules/opentelemetry/src/test-files/solr/solr.xml +++ b/solr/modules/opentelemetry/src/test-files/solr/solr.xml @@ -34,7 +34,7 @@ ${connTimeout:15000} - + 127.0.0.1 diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java new file mode 100644 index 00000000000..bad6cce7d9f --- /dev/null +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/CustomTestOtelTracerConfigurator.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.opentelemetry; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.lang.invoke.MethodHandles; +import org.apache.solr.common.util.NamedList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CustomTestOtelTracerConfigurator extends OtelTracerConfigurator { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + static { + if (System.getProperty("host") == null) { + System.setProperty("host", "localhost"); + } + } + + private static InMemorySpanExporter exporter; + private static volatile boolean isRegistered = false; + private static OpenTelemetrySdk otelSdk = null; + + @Override + public synchronized Tracer getTracer() { + if (!isRegistered) { + throw new IllegalStateException( + "Tracer is not initialized. you need to call #prepareForTest for correct setup"); + } + return super.getTracer(); + } + + @Override + public void init(NamedList args) { + // prevent parent from init otel + } + + public static synchronized void prepareForTest() { + CustomTestOtelTracerConfigurator.resetForTest(); + isRegistered = true; + System.setProperty("otel.traces.exporter", "none"); + + // force early init + CustomTestOtelTracerConfigurator tracer = new CustomTestOtelTracerConfigurator(); + tracer.prepareConfiguration(); + + bootOtel(); + } + + private static void bootOtel() { + try { + exporter = InMemorySpanExporter.create(); + otelSdk = + AutoConfiguredOpenTelemetrySdk.builder() + .setResultAsGlobal() + .addTracerProviderCustomizer( + (builder, props) -> + builder.addSpanProcessor(SimpleSpanProcessor.create(exporter))) + .build() + .getOpenTelemetrySdk(); + } catch (RuntimeException e) { + log.error("Error on OTEL init ", e); + } + } + + public static InMemorySpanExporter getInMemorySpanExporter() { + return exporter; + } + + public static synchronized void resetForTest() { + if (isRegistered) { + isRegistered = false; + if (otelSdk != null) { + otelSdk.close(); + } + if (exporter != null) { + exporter.close(); + exporter = null; + } + System.clearProperty("otel.traces.exporter"); + GlobalOpenTelemetry.resetForTest(); + } + } +} diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java index 5aef12103da..b4825c4e7fd 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java @@ -16,18 +16,13 @@ */ package org.apache.solr.opentelemetry; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import io.opentracing.util.GlobalTracer; import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.MiniSolrCloudCluster; -import org.apache.solr.common.util.ExecutorUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; -@ThreadLeakLingering(linger = 10000) public class OtelTracerConfiguratorTest extends SolrTestCaseJ4 { private OtelTracerConfigurator instance; @@ -35,8 +30,7 @@ public class OtelTracerConfiguratorTest extends SolrTestCaseJ4 { @Before public void setUp() throws Exception { super.setUp(); - instance = new OtelTracerConfigurator(); - instance.currentEnv = + Map currentEnv = Map.of( "OTELNOTHERE", "foo", "OTEL_K1", "env-k1", @@ -44,10 +38,8 @@ public void setUp() throws Exception { System.setProperty("otelnothere", "bar"); System.setProperty("otel.k1", "prop-k1"); System.setProperty("otel.k3", "prop-k3"); - - // to be safe because this test tests tracing. - resetGlobalTracer(); - ExecutorUtil.resetThreadLocalProviders(); + System.setProperty("host", "my.solr.host"); + instance = new OtelTracerConfigurator(currentEnv); } @Override @@ -57,7 +49,8 @@ public void tearDown() throws Exception { System.clearProperty("otelnothere"); System.clearProperty("otel.k1"); System.clearProperty("otel.k3"); - System.clearProperty("otel.bsp.export.timeout"); + System.clearProperty("host"); + System.clearProperty("otel.resource.attributes"); } @Test @@ -93,28 +86,11 @@ public void testSetDefaultIfNotConfigured() { } @Test - public void testInjected() throws Exception { + public void testResourceAttributes() throws Exception { System.setProperty("otel.resource.attributes", "foo=bar,ILLEGAL-LACKS-VALUE,"); - System.setProperty("host", "my.solr.host"); - // Make sure the batch exporter times out before our thread lingering time of 10s - instance.setDefaultIfNotConfigured("OTEL_BSP_SCHEDULE_DELAY", "1000"); - instance.setDefaultIfNotConfigured("OTEL_BSP_EXPORT_TIMEOUT", "2000"); - MiniSolrCloudCluster cluster = - new MiniSolrCloudCluster.Builder(2, createTempDir()) - .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")) - .withSolrXml(getFile("solr/solr.xml").toPath()) - .build(); - try { - assertTrue( - "Tracer shim not registered with GlobalTracer", - GlobalTracer.get().toString().contains("ClosableTracerShim")); - assertEquals( - List.of("host.name=my.solr.host", "foo=bar"), - List.of(System.getProperty("otel.resource.attributes").split(","))); - } finally { - cluster.shutdown(); - System.clearProperty("otel.resource.attributes"); - System.clearProperty("host"); - } + instance.prepareConfiguration(); + assertEquals( + List.of("host.name=my.solr.host", "foo=bar"), + List.of(System.getProperty("otel.resource.attributes").split(","))); } } diff --git a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java similarity index 56% rename from solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java rename to solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java index b28f15e18bd..dea832e87f6 100644 --- a/solr/core/src/test/org/apache/solr/util/tracing/TestDistributedTracing.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/TestDistributedTracing.java @@ -15,12 +15,15 @@ * limitations under the License. */ -package org.apache.solr.util.tracing; +package org.apache.solr.opentelemetry; -import io.opentracing.mock.MockSpan; -import io.opentracing.mock.MockTracer; -import io.opentracing.util.GlobalTracer; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -34,56 +37,60 @@ import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.util.LogLevel; -import org.hamcrest.MatcherAssert; -import org.hamcrest.Matchers; +import org.apache.solr.util.tracing.TraceUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -@LogLevel("org.apache.solr.core.TracerConfigurator=trace") public class TestDistributedTracing extends SolrCloudTestCase { - private static final String COLLECTION = "collection1"; - static MockTracer tracer; + private static final String COLLECTION = "collection1"; @BeforeClass - public static void beforeTest() throws Exception { - tracer = new MockTracer(); - assertTrue(GlobalTracer.registerIfAbsent(tracer)); + public static void setupCluster() throws Exception { + // force early init + CustomTestOtelTracerConfigurator.prepareForTest(); configureCluster(4) - .addConfig( - "config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .addConfig("config", TEST_PATH().resolve("collection1").resolve("conf")) + .withSolrXml(TEST_PATH().resolve("solr.xml")) .configure(); + + assertNotEquals( + "Expecting active otel, not noop impl", + TracerProvider.noop(), + GlobalOpenTelemetry.get().getTracerProvider()); + CollectionAdminRequest.createCollection(COLLECTION, "config", 2, 2) - .setPerReplicaState(SolrCloudTestCase.USE_PER_REPLICA_STATE) .process(cluster.getSolrClient()); cluster.waitForActiveCollection(COLLECTION, 2, 4); } @AfterClass - public static void afterTest() { - tracer = null; + public static void afterClass() { + CustomTestOtelTracerConfigurator.resetForTest(); } @Test public void test() throws IOException, SolrServerException { // TODO it would be clearer if we could compare the complete Span tree between reality - // and what we assert it looks like in a structured visual way. + // and what we assert it looks like in a structured visual way. + getAndClearSpans(); // reset CloudSolrClient cloudClient = cluster.getSolrClient(); - List finishedSpans; // Indexing cloudClient.add(COLLECTION, sdoc("id", "1")); - finishedSpans = getAndClearSpans(); - finishedSpans.removeIf(x -> !x.tags().get("http.url").toString().endsWith("/update")); + var finishedSpans = getAndClearSpans(); + finishedSpans.removeIf( + span -> + span.getAttributes().get(TraceUtils.TAG_HTTP_URL) == null + || !span.getAttributes().get(TraceUtils.TAG_HTTP_URL).endsWith("/update")); assertEquals(2, finishedSpans.size()); assertOneSpanIsChildOfAnother(finishedSpans); // core because cloudClient routes to core - assertEquals("post:/{core}/update", finishedSpans.get(0).operationName()); - assertDbInstanceCore(finishedSpans.get(0)); + assertEquals("post:/{core}/update", finishedSpans.get(0).getName()); + assertCoreName(finishedSpans.get(0)); cloudClient.add(COLLECTION, sdoc("id", "2")); cloudClient.add(COLLECTION, sdoc("id", "3")); @@ -94,54 +101,58 @@ public void test() throws IOException, SolrServerException { // Searching cloudClient.query(COLLECTION, new SolrQuery("*:*")); finishedSpans = getAndClearSpans(); - finishedSpans.removeIf(x -> !x.tags().get("http.url").toString().endsWith("/select")); + finishedSpans.removeIf( + span -> + span.getAttributes().get(TraceUtils.TAG_HTTP_URL) == null + || !span.getAttributes().get(TraceUtils.TAG_HTTP_URL).endsWith("/select")); // one from client to server, 2 for execute query, 2 for fetching documents assertEquals(5, finishedSpans.size()); - assertEquals(1, finishedSpans.stream().filter(s -> s.parentId() == 0).count()); - long parentId = + assertEquals(1, finishedSpans.stream().filter(TestDistributedTracing::isRootSpan).count()); + var parentTraceId = finishedSpans.stream() - .filter(s -> s.parentId() == 0) + .filter(TestDistributedTracing::isRootSpan) .collect(Collectors.toList()) .get(0) - .context() - .spanId(); - for (MockSpan span : finishedSpans) { - if (span.parentId() != 0 && parentId != span.parentId()) { - fail("All spans must belong to single span, but:" + finishedSpans); + .getSpanContext() + .getTraceId(); + for (var span : finishedSpans) { + if (isRootSpan(span)) { + continue; } + assertEquals(span.getParentSpanContext().getTraceId(), parentTraceId); + assertEquals(span.getTraceId(), parentTraceId); } - assertEquals("get:/{core}/select", finishedSpans.get(0).operationName()); - assertDbInstanceCore(finishedSpans.get(0)); + assertEquals("get:/{core}/select", finishedSpans.get(0).getName()); + assertCoreName(finishedSpans.get(0)); } @Test public void testAdminApi() throws Exception { + getAndClearSpans(); // reset CloudSolrClient cloudClient = cluster.getSolrClient(); - List finishedSpans; - // Admin API call cloudClient.request(new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/metrics")); - finishedSpans = getAndClearSpans(); - assertEquals("get:/admin/metrics", finishedSpans.get(0).operationName()); + var finishedSpans = getAndClearSpans(); + assertEquals("get:/admin/metrics", finishedSpans.get(0).getName()); CollectionAdminRequest.listCollections(cloudClient); finishedSpans = getAndClearSpans(); - assertEquals("list:/admin/collections", finishedSpans.get(0).operationName()); + assertEquals("list:/admin/collections", finishedSpans.get(0).getName()); } @Test public void testV2Api() throws Exception { + getAndClearSpans(); // reset CloudSolrClient cloudClient = cluster.getSolrClient(); - List finishedSpans; new V2Request.Builder("/collections/" + COLLECTION + "/reload") .withMethod(SolrRequest.METHOD.POST) .withPayload("{}") .build() .process(cloudClient); - finishedSpans = getAndClearSpans(); - assertEquals("post:/collections/{collection}/reload", finishedSpans.get(0).operationName()); - assertDbInstanceColl(finishedSpans.get(0)); + var finishedSpans = getAndClearSpans(); + assertEquals("post:/collections/{collection}/reload", finishedSpans.get(0).getName()); + assertCollectionName(finishedSpans.get(0)); new V2Request.Builder("/c/" + COLLECTION + "/update/json") .withMethod(SolrRequest.METHOD.POST) @@ -150,8 +161,8 @@ public void testV2Api() throws Exception { .build() .process(cloudClient); finishedSpans = getAndClearSpans(); - assertEquals("post:/c/{collection}/update/json", finishedSpans.get(0).operationName()); - assertDbInstanceColl(finishedSpans.get(0)); + assertEquals("post:/c/{collection}/update/json", finishedSpans.get(0).getName()); + assertCollectionName(finishedSpans.get(0)); final V2Response v2Response = new V2Request.Builder("/c/" + COLLECTION + "/select") @@ -160,36 +171,40 @@ public void testV2Api() throws Exception { .build() .process(cloudClient); finishedSpans = getAndClearSpans(); - assertEquals("get:/c/{collection}/select", finishedSpans.get(0).operationName()); - assertDbInstanceColl(finishedSpans.get(0)); + assertEquals("get:/c/{collection}/select", finishedSpans.get(0).getName()); + assertCollectionName(finishedSpans.get(0)); assertEquals(1, ((SolrDocumentList) v2Response.getResponse().get("response")).getNumFound()); } - private void assertDbInstanceColl(MockSpan mockSpan) { - MatcherAssert.assertThat(mockSpan.tags().get("db.instance"), Matchers.equalTo("collection1")); + private static boolean isRootSpan(SpanData span) { + return span.getParentSpanContext() == SpanContext.getInvalid(); } - private void assertDbInstanceCore(MockSpan mockSpan) { - MatcherAssert.assertThat( - (String) mockSpan.tags().get("db.instance"), Matchers.startsWith("collection1_")); + private static void assertCollectionName(SpanData span) { + assertEquals(COLLECTION, span.getAttributes().get(TraceUtils.TAG_DB)); } - private void assertOneSpanIsChildOfAnother(List finishedSpans) { - MockSpan child = finishedSpans.get(0); - MockSpan parent = finishedSpans.get(1); - if (child.parentId() == 0) { - MockSpan temp = parent; + private static void assertCoreName(SpanData span) { + assertTrue(span.getAttributes().get(TraceUtils.TAG_DB).startsWith(COLLECTION + "_")); + } + + private void assertOneSpanIsChildOfAnother(List finishedSpans) { + SpanData child = finishedSpans.get(0); + SpanData parent = finishedSpans.get(1); + if (isRootSpan(child)) { + var temp = parent; parent = child; child = temp; } - - assertEquals(child.parentId(), parent.context().spanId()); + assertEquals(child.getParentSpanContext().getTraceId(), parent.getTraceId()); + assertEquals(child.getTraceId(), parent.getTraceId()); } - private List getAndClearSpans() { - List result = tracer.finishedSpans(); // returns a mutable copy + private List getAndClearSpans() { + InMemorySpanExporter exporter = CustomTestOtelTracerConfigurator.getInMemorySpanExporter(); + List result = new ArrayList<>(exporter.getFinishedSpanItems()); Collections.reverse(result); // nicer to see spans chronologically - tracer.reset(); + exporter.reset(); return result; } } diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc index 33e386b6b46..fed3ed14110 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc @@ -34,6 +34,8 @@ Starting in 10, the Maven POM for SolrJ does not refer to SolrJ modules like Zoo * The `jaegertracer-configurator` module, which was deprecated in 9.2, is removed. Users should migrate to the `opentelemetry` module. +* `OpenTracing` libraries were removed and replaced with `OpenTelemetry` libraries. Any Java agents providing `OpenTracing` tracers will no longer work. Telemetry tags `http.status_code` and `http.method` have been deprecated, newer version of the tags have been added to the span data: `http.response.status_code`, `http.request.method`. + * The `analytics` module, which was deprecated in 9.2, is removed. * The sysProp `-Dsolr.redaction.system.pattern`, which allows users to provide a pattern to match sysProps that should be redacted for sensitive information, diff --git a/solr/test-framework/build.gradle b/solr/test-framework/build.gradle index c6589155072..6c772aac512 100644 --- a/solr/test-framework/build.gradle +++ b/solr/test-framework/build.gradle @@ -52,8 +52,6 @@ dependencies { implementation 'org.slf4j:slf4j-api' implementation 'org.apache.logging.log4j:log4j-api' implementation 'org.apache.logging.log4j:log4j-core' - implementation 'io.opentracing:opentracing-noop' - implementation 'io.opentracing:opentracing-util' implementation 'io.dropwizard.metrics:metrics-core' implementation 'io.dropwizard.metrics:metrics-jetty10' implementation 'commons-cli:commons-cli' diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index ba96e729562..fc4a6ac27fb 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -20,15 +20,12 @@ import static org.apache.solr.cloud.SolrZkServer.ZK_WHITELIST_PROPERTY; import static org.apache.solr.common.cloud.ZkStateReader.HTTPS; import static org.apache.solr.common.cloud.ZkStateReader.URL_SCHEME; -import static org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase; import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM; import static org.hamcrest.core.StringContains.containsString; import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; -import io.opentracing.noop.NoopTracerFactory; -import io.opentracing.util.GlobalTracer; import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; @@ -43,7 +40,6 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.lang.invoke.MethodHandles; -import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; @@ -52,8 +48,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.security.SecureRandom; import java.time.Instant; import java.util.ArrayList; @@ -138,6 +132,7 @@ import org.apache.solr.security.AllowListUrlChecker; import org.apache.solr.servlet.DirectSolrConnection; import org.apache.solr.update.processor.DistributedUpdateProcessor; +import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase; import org.apache.solr.update.processor.DistributedZkUpdateProcessor; import org.apache.solr.update.processor.UpdateRequestProcessor; import org.apache.solr.util.BaseTestHarness; @@ -313,38 +308,9 @@ public static void setupTestCases() { System.setProperty(URL_SCHEME, HTTPS); } - resetGlobalTracer(); ExecutorUtil.resetThreadLocalProviders(); } - /** - * GlobalTracer is initialized by org.apache.solr.core.TracerConfigurator by - * org.apache.solr.core.CoreContainer. Tests may need to reset it in the beginning of a test if it - * might have differing configuration from other tests in the same suite. It's also important to - * call {@link ExecutorUtil#resetThreadLocalProviders()}. - */ - @SuppressForbidden(reason = "Hack to reset internal state of GlobalTracer") - public static void resetGlobalTracer() { - AccessController.doPrivileged( - (PrivilegedAction) - () -> { - try { - final Class globalTracerClass = GlobalTracer.class; - final Field isRegistered = globalTracerClass.getDeclaredField("isRegistered"); - isRegistered.setAccessible(true); - isRegistered.setBoolean(null, false); - final Field tracer = globalTracerClass.getDeclaredField("tracer"); - tracer.setAccessible(true); - tracer.set(null, NoopTracerFactory.create()); - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } - return null; - }); - - assert GlobalTracer.isRegistered() == false; - } - @AfterClass public static void teardownTestCases() throws Exception { TestInjection.notifyPauseForeverDone(); diff --git a/versions.lock b/versions.lock index c78c403b7d4..f79f9dba576 100644 --- a/versions.lock +++ b/versions.lock @@ -124,7 +124,7 @@ io.netty:netty-transport-native-unix-common:4.1.96.Final (4 constraints: 5e3d199 io.opencensus:opencensus-api:0.31.1 (5 constraints: 924d4692) io.opencensus:opencensus-contrib-http-util:0.31.1 (3 constraints: 7232a9fc) io.opencensus:opencensus-proto:0.2.0 (1 constraints: e60fd595) -io.opentelemetry:opentelemetry-api:1.29.0 (11 constraints: ceca6768) +io.opentelemetry:opentelemetry-api:1.29.0 (11 constraints: c3c8daff) io.opentelemetry:opentelemetry-api-events:1.29.0-alpha (2 constraints: 332f2380) io.opentelemetry:opentelemetry-bom:1.29.0 (1 constraints: 3e05483b) io.opentelemetry:opentelemetry-context:1.29.0 (2 constraints: 2d1f62b5) @@ -133,18 +133,14 @@ io.opentelemetry:opentelemetry-exporter-otlp:1.29.0 (1 constraints: 990ff983) io.opentelemetry:opentelemetry-exporter-otlp-common:1.29.0 (2 constraints: 5823eb1c) io.opentelemetry:opentelemetry-exporter-sender-okhttp:1.29.0 (2 constraints: 5823eb1c) io.opentelemetry:opentelemetry-extension-incubator:1.29.0-alpha (1 constraints: f414ee96) -io.opentelemetry:opentelemetry-opentracing-shim:1.29.0 (1 constraints: 990ff983) -io.opentelemetry:opentelemetry-sdk:1.29.0 (3 constraints: ae4376e7) +io.opentelemetry:opentelemetry-sdk:1.29.0 (4 constraints: 755634c7) io.opentelemetry:opentelemetry-sdk-common:1.29.0 (6 constraints: 936c396e) io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.29.0 (1 constraints: 990ff983) io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.29.0 (3 constraints: a63c5427) io.opentelemetry:opentelemetry-sdk-logs:1.29.0 (3 constraints: f43275b4) io.opentelemetry:opentelemetry-sdk-metrics:1.29.0 (3 constraints: f43275b4) io.opentelemetry:opentelemetry-sdk-trace:1.29.0 (3 constraints: f43275b4) -io.opentelemetry:opentelemetry-semconv:1.29.0-alpha (4 constraints: 175b2583) -io.opentracing:opentracing-api:0.33.0 (5 constraints: 7244331d) -io.opentracing:opentracing-noop:0.33.0 (4 constraints: 48361e21) -io.opentracing:opentracing-util:0.33.0 (2 constraints: 5013be5a) +io.opentelemetry:opentelemetry-semconv:1.29.0-alpha (4 constraints: 0c5920ad) io.perfmark:perfmark-api:0.26.0 (3 constraints: 21212b16) io.prometheus:simpleclient:0.16.0 (3 constraints: 9d257513) io.prometheus:simpleclient_common:0.16.0 (1 constraints: 1a1139c0) @@ -403,7 +399,7 @@ com.squareup.okhttp3:mockwebserver:4.9.3 (1 constraints: c60ebf62) io.github.microutils:kotlin-logging:2.1.21 (1 constraints: ec0e8871) io.github.microutils:kotlin-logging-jvm:2.1.21 (1 constraints: af0f358c) io.micrometer:micrometer-core:1.9.11 (1 constraints: fd162819) -io.opentracing:opentracing-mock:0.33.0 (1 constraints: 3805343b) +io.opentelemetry:opentelemetry-sdk-testing:1.29.0 (1 constraints: 990ff983) jakarta.servlet:jakarta.servlet-api:4.0.4 (4 constraints: 586e1f6a) jakarta.websocket:jakarta.websocket-api:1.1.2 (1 constraints: 92155ab9) javax.inject:javax.inject:1 (1 constraints: 7a0df617) diff --git a/versions.props b/versions.props index 45ffbbad5d4..41be40abeeb 100644 --- a/versions.props +++ b/versions.props @@ -24,7 +24,6 @@ io.dropwizard.metrics:*=4.2.19 io.grpc:grpc-*=1.56.0 io.netty:*=4.1.96.Final io.opentelemetry:opentelemetry-bom=1.29.0 -io.opentracing:*=0.33.0 io.prometheus:*=0.16.0 io.swagger.core.v3:*=2.2.12 jakarta.ws.rs:jakarta.ws.rs-api=2.1.6 From ac2beb340ac6efc300ff699f6313519710135223 Mon Sep 17 00:00:00 2001 From: Noble Paul Date: Fri, 18 Aug 2023 00:14:21 +1000 Subject: [PATCH 013/160] SOLR-16939: CBOR format should support nested documents (#1845) --- .../solr/handler/loader/CborLoader.java | 5 ++ .../cloud-managed/conf/managed-schema.xml | 2 + .../apache/solr/util/TestCborDataFormat.java | 50 +++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java index ab6dfaf8fa0..3f0f6067625 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java @@ -123,6 +123,11 @@ private Object readVal(JsonToken t, CBORParser p) throws IOException { if (t == JsonToken.VALUE_NUMBER_INT || t == JsonToken.VALUE_NUMBER_FLOAT) { return p.getNumberValue(); } + if (t == JsonToken.START_OBJECT) { + Object[] returnVal = new Object[1]; + new CborLoader(cborFactory, d -> returnVal[0] = d).handleDoc(p); + return returnVal[0]; + } throw new RuntimeException("Unknown type :" + t); } diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml index 14e5b94fb67..a3176d2d412 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml @@ -39,6 +39,8 @@ + + diff --git a/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java index 90b61853bfb..b24e649ce28 100644 --- a/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java +++ b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -42,6 +43,7 @@ import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.RequestWriter; import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.MapSolrParams; @@ -95,6 +97,15 @@ public void testRoundTrip() throws Exception { Object o = objectMapper.readValue(b, Object.class); List l = (List) Utils.getObjectByPath(o, false, "response/docs"); assertEquals(1100, l.size()); + client.deleteByQuery(testCollection, "*:*").getStatus(); + index( + testCollection, + client, + createCborReq(getNestedDocs().getBytes(StandardCharsets.UTF_8)), + false); + SolrQuery q = new SolrQuery("*:*").setRows(100); + QueryResponse result = new QueryRequest(q).process(client, testCollection); + assertEquals(6, result.getResults().size()); } finally { System.clearProperty("managed.schema.mutable"); cluster.shutdown(); @@ -227,4 +238,43 @@ private byte[] serializeToCbor(byte[] is) throws IOException { jsonGenerator.close(); return baos.toByteArray(); } + + private String getNestedDocs() throws IOException { + return "[{ \"id\": \"P11!prod\",\n" + + " \"name_s\": \"Swingline Stapler\",\n" + + " \"type_s\": \"PRODUCT\",\n" + + " \"description_s\": \"The Cadillac of office staplers ...\",\n" + + " \"skus\": [\n" + + " { \"id\": \"P11!S21\",\n" + + " \"type_s\": \"SKU\",\n" + + " \"color_s\": \"RED\",\n" + + " \"price_i\": 42,\n" + + " \"manuals\": [\n" + + " { \"id\": \"P11!D41\",\n" + + " \"type_s\": \"MANUAL\",\n" + + " \"name_s\": \"Red Swingline Brochure\",\n" + + " \"pages_i\":1,\n" + + " \"content_s\": \"...\"\n" + + " } ]\n" + + " },\n" + + " { \"id\": \"P11!S31\",\n" + + " \"type_s\": \"SKU\",\n" + + " \"color_s\": \"BLACK\",\n" + + " \"price_i\": 3\n" + + " },\n" + + " { \"id\": \"P11!D51\",\n" + + " \"type_s\": \"MANUAL\",\n" + + " \"name_s\": \"Quick Reference Guide\",\n" + + " \"pages_i\":1,\n" + + " \"content_s\": \"How to use your stapler ...\"\n" + + " },\n" + + " { \"id\": \"P11!D61\",\n" + + " \"type_s\": \"MANUAL\",\n" + + " \"name_s\": \"Warranty Details\",\n" + + " \"pages_i\":42,\n" + + " \"content_s\": \"... lifetime guarantee ...\"\n" + + " }\n" + + " ]\n" + + "} ]"; + } } From 61d6a1347749c9a09ce22668af7ab0148ed6cbb4 Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Fri, 18 Aug 2023 09:46:34 +0200 Subject: [PATCH 014/160] Update dependency commons-codec:commons-codec to v1.16.0 (#1719) --- solr/licenses/commons-codec-1.15.jar.sha1 | 1 - solr/licenses/commons-codec-1.16.0.jar.sha1 | 1 + versions.lock | 2 +- versions.props | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 solr/licenses/commons-codec-1.15.jar.sha1 create mode 100644 solr/licenses/commons-codec-1.16.0.jar.sha1 diff --git a/solr/licenses/commons-codec-1.15.jar.sha1 b/solr/licenses/commons-codec-1.15.jar.sha1 deleted file mode 100644 index 00ebbbf2720..00000000000 --- a/solr/licenses/commons-codec-1.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d diff --git a/solr/licenses/commons-codec-1.16.0.jar.sha1 b/solr/licenses/commons-codec-1.16.0.jar.sha1 new file mode 100644 index 00000000000..23acaadb432 --- /dev/null +++ b/solr/licenses/commons-codec-1.16.0.jar.sha1 @@ -0,0 +1 @@ +4e3eb3d79888d76b54e28b350915b5dc3919c9de diff --git a/versions.lock b/versions.lock index f79f9dba576..f6cfd5ecc34 100644 --- a/versions.lock +++ b/versions.lock @@ -77,7 +77,7 @@ com.sun.istack:istack-commons-runtime:3.0.12 (1 constraints: eb0d9a43) com.tdunning:t-digest:3.1 (1 constraints: a804212c) com.zaxxer:SparseBitSet:1.2 (1 constraints: 0d081e75) commons-cli:commons-cli:1.5.0 (1 constraints: 0805ff35) -commons-codec:commons-codec:1.15 (11 constraints: 43972235) +commons-codec:commons-codec:1.16.0 (11 constraints: a297a2db) commons-collections:commons-collections:3.2.2 (1 constraints: 09050236) commons-io:commons-io:2.11.0 (9 constraints: 9d65151d) de.l3s.boilerpipe:boilerpipe:1.1.0 (1 constraints: 590ce401) diff --git a/versions.props b/versions.props index 41be40abeeb..f96ebda77ca 100644 --- a/versions.props +++ b/versions.props @@ -17,7 +17,7 @@ com.jayway.jsonpath:json-path=2.8.0 com.lmax:disruptor=3.4.4 com.tdunning:t-digest=3.1 commons-cli:commons-cli=1.5.0 -commons-codec:commons-codec=1.15 +commons-codec:commons-codec=1.16.0 commons-collections:commons-collections=3.2.2 commons-io:commons-io=2.11.0 io.dropwizard.metrics:*=4.2.19 From 560f93a67d3eb04dc1dae5eb3afd29da6a897e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 18 Aug 2023 15:54:40 +0200 Subject: [PATCH 015/160] Update solr/README (#1850) --- solr/README.adoc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/solr/README.adoc b/solr/README.adoc index d1387439cd8..61d0b8d96e6 100644 --- a/solr/README.adoc +++ b/solr/README.adoc @@ -17,11 +17,14 @@ == Solr Source Directories +link:api/[]:: +Root package for interfaces and POJOs used to describe Solr's v2 APIs. + link:benchmark/[]:: Benchmarking module for Solr. link:bin/[]:: -Scripts to startup, manage and interact with Solr instances. +Scripts to start up, manage and interact with Solr instances. link:core/[]:: Base Solr code. @@ -40,7 +43,7 @@ link:example/[]:: Contains example documents and an alternative Solr home directory containing various examples. -link:license/[]:: +link:licenses/[]:: Licenses, notice files and signatures for Solr dependencies. link:modules/[]:: @@ -59,7 +62,13 @@ link:solr-ref-guide/[]:: The Solr Reference Guide source files including Java examples. link:solrj/[]:: -Solr client code for Java applications. +Solr client code for Java applications - common. + +link:solrj-streaming/[]:: +SolrJ additional API for Streaming Aggregation. + +link:solrj-zookeeper/[]:: +SolrJ additional API for pulling cluster state from Zookeeper. link:test-framework/[]:: Solr test framework for internal tests and external tests integrating Solr. From fc01a3d5f88a81125ae746759323cac021a2eb34 Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:01:40 +0200 Subject: [PATCH 016/160] Update org.apache.zookeeper:* to v3.9.0 (#1807) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update org.apache.zookeeper:* to v3.9.0 Uses shared LICENSE and NOTICE files for all netty-* jars Fix OverseerTest that broke due to 3.9 changes Co-authored-by: Jan Høydahl --- .../org/apache/solr/cloud/OverseerTest.java | 9 +- ...-LICENSE-ASL.txt => netty-LICENSE-ASL.txt} | 0 ...tty-buffer-NOTICE.txt => netty-NOTICE.txt} | 0 solr/licenses/netty-codec-LICENSE-ASL.txt | 202 ---------------- solr/licenses/netty-codec-NOTICE.txt | 223 ------------------ solr/licenses/netty-common-LICENSE-ASL.txt | 202 ---------------- solr/licenses/netty-common-NOTICE.txt | 223 ------------------ solr/licenses/netty-handler-LICENSE-ASL.txt | 202 ---------------- solr/licenses/netty-handler-NOTICE.txt | 223 ------------------ solr/licenses/netty-resolver-LICENSE-ASL.txt | 202 ---------------- solr/licenses/netty-resolver-NOTICE.txt | 223 ------------------ ...ive-boringssl-static-2.0.61.Final.jar.sha1 | 1 + ...tty-tcnative-classes-2.0.61.Final.jar.sha1 | 1 + solr/licenses/netty-transport-LICENSE-ASL.txt | 202 ---------------- solr/licenses/netty-transport-NOTICE.txt | 223 ------------------ ...e-epoll-4.1.96.Final-linux-x86_64.jar.sha1 | 1 + ...ansport-native-epoll-4.1.96.Final.jar.sha1 | 1 - solr/licenses/zookeeper-3.8.1-tests.jar.sha1 | 1 - solr/licenses/zookeeper-3.8.1.jar.sha1 | 1 - solr/licenses/zookeeper-3.9.0-tests.jar.sha1 | 1 + solr/licenses/zookeeper-3.9.0.jar.sha1 | 1 + solr/licenses/zookeeper-jute-3.8.1.jar.sha1 | 1 - solr/licenses/zookeeper-jute-3.9.0.jar.sha1 | 1 + versions.lock | 10 +- versions.props | 2 +- 25 files changed, 19 insertions(+), 2137 deletions(-) rename solr/licenses/{netty-buffer-LICENSE-ASL.txt => netty-LICENSE-ASL.txt} (100%) rename solr/licenses/{netty-buffer-NOTICE.txt => netty-NOTICE.txt} (100%) delete mode 100644 solr/licenses/netty-codec-LICENSE-ASL.txt delete mode 100644 solr/licenses/netty-codec-NOTICE.txt delete mode 100644 solr/licenses/netty-common-LICENSE-ASL.txt delete mode 100644 solr/licenses/netty-common-NOTICE.txt delete mode 100644 solr/licenses/netty-handler-LICENSE-ASL.txt delete mode 100644 solr/licenses/netty-handler-NOTICE.txt delete mode 100644 solr/licenses/netty-resolver-LICENSE-ASL.txt delete mode 100644 solr/licenses/netty-resolver-NOTICE.txt create mode 100644 solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 create mode 100644 solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 delete mode 100644 solr/licenses/netty-transport-LICENSE-ASL.txt delete mode 100644 solr/licenses/netty-transport-NOTICE.txt create mode 100644 solr/licenses/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar.sha1 delete mode 100644 solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 delete mode 100644 solr/licenses/zookeeper-3.8.1-tests.jar.sha1 delete mode 100644 solr/licenses/zookeeper-3.8.1.jar.sha1 create mode 100644 solr/licenses/zookeeper-3.9.0-tests.jar.sha1 create mode 100644 solr/licenses/zookeeper-3.9.0.jar.sha1 delete mode 100644 solr/licenses/zookeeper-jute-3.8.1.jar.sha1 create mode 100644 solr/licenses/zookeeper-jute-3.9.0.jar.sha1 diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java index 1e7c8de5124..f6e108e0bf5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java @@ -17,6 +17,7 @@ package org.apache.solr.cloud; import static org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus; +import static org.apache.zookeeper.WatchedEvent.NO_ZXID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; @@ -2129,7 +2130,7 @@ public void testLatchWatcher() throws InterruptedException { assertTrue(TimeUnit.NANOSECONDS.toMillis(after - before) > 50); // Mostly to make sure the millis->nanos->millis is not broken assertTrue(TimeUnit.NANOSECONDS.toMillis(after - before) < 500); - latch1.process(new WatchedEvent(new WatcherEvent(1, 1, "/foo/bar"))); + latch1.process(new WatchedEvent(new WatcherEvent(1, 1, "/foo/bar"), NO_ZXID)); before = System.nanoTime(); latch1.await(10000); // Expecting no wait after = System.nanoTime(); @@ -2145,14 +2146,16 @@ public void testLatchWatcher() throws InterruptedException { // Process an event of a different type first, this shouldn't release the latch latch2.process( new WatchedEvent( - new WatcherEvent(Event.EventType.NodeDeleted.getIntValue(), 1, "/foo/bar"))); + new WatcherEvent(Event.EventType.NodeDeleted.getIntValue(), 1, "/foo/bar"), + NO_ZXID)); assertFalse("Latch shouldn't have been released", doneWaiting.get()); // Now process the correct type of event expectedEventProcessed.set(true); latch2.process( new WatchedEvent( - new WatcherEvent(Event.EventType.NodeCreated.getIntValue(), 1, "/foo/bar"))); + new WatcherEvent(Event.EventType.NodeCreated.getIntValue(), 1, "/foo/bar"), + NO_ZXID)); }); t.start(); before = System.nanoTime(); diff --git a/solr/licenses/netty-buffer-LICENSE-ASL.txt b/solr/licenses/netty-LICENSE-ASL.txt similarity index 100% rename from solr/licenses/netty-buffer-LICENSE-ASL.txt rename to solr/licenses/netty-LICENSE-ASL.txt diff --git a/solr/licenses/netty-buffer-NOTICE.txt b/solr/licenses/netty-NOTICE.txt similarity index 100% rename from solr/licenses/netty-buffer-NOTICE.txt rename to solr/licenses/netty-NOTICE.txt diff --git a/solr/licenses/netty-codec-LICENSE-ASL.txt b/solr/licenses/netty-codec-LICENSE-ASL.txt deleted file mode 100644 index d6456956733..00000000000 --- a/solr/licenses/netty-codec-LICENSE-ASL.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/netty-codec-NOTICE.txt b/solr/licenses/netty-codec-NOTICE.txt deleted file mode 100644 index f973663670b..00000000000 --- a/solr/licenses/netty-codec-NOTICE.txt +++ /dev/null @@ -1,223 +0,0 @@ - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * NOTICE: - * license/NOTICE.harmony.txt - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. - - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper diff --git a/solr/licenses/netty-common-LICENSE-ASL.txt b/solr/licenses/netty-common-LICENSE-ASL.txt deleted file mode 100644 index d6456956733..00000000000 --- a/solr/licenses/netty-common-LICENSE-ASL.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/netty-common-NOTICE.txt b/solr/licenses/netty-common-NOTICE.txt deleted file mode 100644 index f973663670b..00000000000 --- a/solr/licenses/netty-common-NOTICE.txt +++ /dev/null @@ -1,223 +0,0 @@ - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * NOTICE: - * license/NOTICE.harmony.txt - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. - - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper diff --git a/solr/licenses/netty-handler-LICENSE-ASL.txt b/solr/licenses/netty-handler-LICENSE-ASL.txt deleted file mode 100644 index d6456956733..00000000000 --- a/solr/licenses/netty-handler-LICENSE-ASL.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/netty-handler-NOTICE.txt b/solr/licenses/netty-handler-NOTICE.txt deleted file mode 100644 index f973663670b..00000000000 --- a/solr/licenses/netty-handler-NOTICE.txt +++ /dev/null @@ -1,223 +0,0 @@ - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * NOTICE: - * license/NOTICE.harmony.txt - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. - - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper diff --git a/solr/licenses/netty-resolver-LICENSE-ASL.txt b/solr/licenses/netty-resolver-LICENSE-ASL.txt deleted file mode 100644 index d6456956733..00000000000 --- a/solr/licenses/netty-resolver-LICENSE-ASL.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/netty-resolver-NOTICE.txt b/solr/licenses/netty-resolver-NOTICE.txt deleted file mode 100644 index f973663670b..00000000000 --- a/solr/licenses/netty-resolver-NOTICE.txt +++ /dev/null @@ -1,223 +0,0 @@ - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * NOTICE: - * license/NOTICE.harmony.txt - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. - - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper diff --git a/solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 b/solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 new file mode 100644 index 00000000000..1c3e5f7d66f --- /dev/null +++ b/solr/licenses/netty-tcnative-boringssl-static-2.0.61.Final.jar.sha1 @@ -0,0 +1 @@ +944722ba2883fe5825a0c6b38dc727d7151a9371 diff --git a/solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 b/solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 new file mode 100644 index 00000000000..ca1be9444d7 --- /dev/null +++ b/solr/licenses/netty-tcnative-classes-2.0.61.Final.jar.sha1 @@ -0,0 +1 @@ +4c6ae851ed97921bc6c6b64e019c2b039f49131a diff --git a/solr/licenses/netty-transport-LICENSE-ASL.txt b/solr/licenses/netty-transport-LICENSE-ASL.txt deleted file mode 100644 index d6456956733..00000000000 --- a/solr/licenses/netty-transport-LICENSE-ASL.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/solr/licenses/netty-transport-NOTICE.txt b/solr/licenses/netty-transport-NOTICE.txt deleted file mode 100644 index f973663670b..00000000000 --- a/solr/licenses/netty-transport-NOTICE.txt +++ /dev/null @@ -1,223 +0,0 @@ - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * NOTICE: - * license/NOTICE.harmony.txt - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://github.com/y-256/libdivsufsort - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/protobuf - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * https://github.com/google/snappy - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/google/caliper - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - - -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. - - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper diff --git a/solr/licenses/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar.sha1 new file mode 100644 index 00000000000..04eae401ac3 --- /dev/null +++ b/solr/licenses/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar.sha1 @@ -0,0 +1 @@ +3f8904e072cfc9a8d67c6fe567c39bcbce5c9c55 diff --git a/solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 deleted file mode 100644 index 21a85fa4799..00000000000 --- a/solr/licenses/netty-transport-native-epoll-4.1.96.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9faf365396c933f1b39b60d129391c6c6c43fb86 diff --git a/solr/licenses/zookeeper-3.8.1-tests.jar.sha1 b/solr/licenses/zookeeper-3.8.1-tests.jar.sha1 deleted file mode 100644 index 9f92a4f13fb..00000000000 --- a/solr/licenses/zookeeper-3.8.1-tests.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -15d43384e216f03166439f06584c78d35355a10c diff --git a/solr/licenses/zookeeper-3.8.1.jar.sha1 b/solr/licenses/zookeeper-3.8.1.jar.sha1 deleted file mode 100644 index 02c7bc328f6..00000000000 --- a/solr/licenses/zookeeper-3.8.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -edb7d70db4390b9f54a92994577a9e344ade5c9d diff --git a/solr/licenses/zookeeper-3.9.0-tests.jar.sha1 b/solr/licenses/zookeeper-3.9.0-tests.jar.sha1 new file mode 100644 index 00000000000..09eb33247b7 --- /dev/null +++ b/solr/licenses/zookeeper-3.9.0-tests.jar.sha1 @@ -0,0 +1 @@ +71c696e7390a443e4cfd9c2c5b7f25c8a0ab1852 diff --git a/solr/licenses/zookeeper-3.9.0.jar.sha1 b/solr/licenses/zookeeper-3.9.0.jar.sha1 new file mode 100644 index 00000000000..b88faa06754 --- /dev/null +++ b/solr/licenses/zookeeper-3.9.0.jar.sha1 @@ -0,0 +1 @@ +d8c7c5450c91e272aaac3b908fff96262fd52694 diff --git a/solr/licenses/zookeeper-jute-3.8.1.jar.sha1 b/solr/licenses/zookeeper-jute-3.8.1.jar.sha1 deleted file mode 100644 index 087f7233f08..00000000000 --- a/solr/licenses/zookeeper-jute-3.8.1.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -aeb9b21ed9d2b99da9e0e3d1fd24d54f85fbf840 diff --git a/solr/licenses/zookeeper-jute-3.9.0.jar.sha1 b/solr/licenses/zookeeper-jute-3.9.0.jar.sha1 new file mode 100644 index 00000000000..15b498ddfaa --- /dev/null +++ b/solr/licenses/zookeeper-jute-3.9.0.jar.sha1 @@ -0,0 +1 @@ +224f476a950054a6c84b2476a1b5216b32ea7140 diff --git a/versions.lock b/versions.lock index f6cfd5ecc34..adb9555b9d5 100644 --- a/versions.lock +++ b/versions.lock @@ -114,12 +114,14 @@ io.netty:netty-codec-http:4.1.96.Final (3 constraints: 582436ee) io.netty:netty-codec-http2:4.1.96.Final (1 constraints: ed0a94cb) io.netty:netty-codec-socks:4.1.96.Final (1 constraints: 160fb96c) io.netty:netty-common:4.1.96.Final (12 constraints: 3cb3817d) -io.netty:netty-handler:4.1.96.Final (3 constraints: 732b553d) +io.netty:netty-handler:4.1.96.Final (3 constraints: 722b4f3d) io.netty:netty-handler-proxy:4.1.96.Final (1 constraints: ed0a94cb) io.netty:netty-resolver:4.1.96.Final (2 constraints: 5c1a8332) +io.netty:netty-tcnative-boringssl-static:2.0.61.Final (1 constraints: d10fc38e) +io.netty:netty-tcnative-classes:2.0.61.Final (1 constraints: d113ea5d) io.netty:netty-transport:4.1.96.Final (9 constraints: 0b8c6f3f) io.netty:netty-transport-classes-epoll:4.1.96.Final (1 constraints: b312061f) -io.netty:netty-transport-native-epoll:4.1.96.Final (1 constraints: db0f188f) +io.netty:netty-transport-native-epoll:4.1.96.Final (1 constraints: da0f128f) io.netty:netty-transport-native-unix-common:4.1.96.Final (4 constraints: 5e3d1997) io.opencensus:opencensus-api:0.31.1 (5 constraints: 924d4692) io.opencensus:opencensus-contrib-http-util:0.31.1 (3 constraints: 7232a9fc) @@ -253,8 +255,8 @@ org.apache.tika:tika-core:1.28.5 (2 constraints: d8118f11) org.apache.tika:tika-parsers:1.28.5 (1 constraints: 42054a3b) org.apache.tomcat:annotations-api:6.0.53 (1 constraints: 40054e3b) org.apache.xmlbeans:xmlbeans:5.0.3 (2 constraints: 72173075) -org.apache.zookeeper:zookeeper:3.8.1 (2 constraints: a0134c5f) -org.apache.zookeeper:zookeeper-jute:3.8.1 (2 constraints: 99125b23) +org.apache.zookeeper:zookeeper:3.9.0 (2 constraints: a0134e5f) +org.apache.zookeeper:zookeeper-jute:3.9.0 (2 constraints: 99125f23) org.apiguardian:apiguardian-api:1.1.2 (2 constraints: 601bd5a8) org.bitbucket.b_c:jose4j:0.9.3 (1 constraints: 0e050936) org.bouncycastle:bcmail-jdk15on:1.70 (1 constraints: 310c8af5) diff --git a/versions.props b/versions.props index f96ebda77ca..17af2f9c60c 100644 --- a/versions.props +++ b/versions.props @@ -50,7 +50,7 @@ org.apache.logging.log4j:*=2.20.0 org.apache.lucene:*=9.7.0 org.apache.tika:*=1.28.5 org.apache.tomcat:annotations-api=6.0.53 -org.apache.zookeeper:*=3.8.1 +org.apache.zookeeper:*=3.9.0 org.bitbucket.b_c:jose4j=0.9.3 org.carrot2:carrot2-core=4.5.1 org.codehaus.woodstox:stax2-api=4.2.1 From 4f550d324e4a912ad5668123ae3d44554574f416 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Fri, 18 Aug 2023 11:20:15 -0400 Subject: [PATCH 017/160] SOLR-16733: Fail 'test' tasks using JDK20 on Mac (#1832) Java 20 has a JDK bug that triggers on Mac systems and crashes Solr on startup. This affects deployments running on Mac as well as test runs, which obviously need to start and stop Solr frequently. This commit adds some gradle logic to detect this combination (JDK20 w/ Mac) and fail the 'test' task when present. --- build.gradle | 1 + gradle/testing/fail-on-unsupported-jdk.gradle | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 gradle/testing/fail-on-unsupported-jdk.gradle diff --git a/build.gradle b/build.gradle index ebd9b85786a..b612fe2bf58 100644 --- a/build.gradle +++ b/build.gradle @@ -127,6 +127,7 @@ apply from: file('gradle/java/javac.gradle') apply from: file('gradle/testing/defaults-tests.gradle') apply from: file('gradle/testing/randomization.gradle') apply from: file('gradle/testing/fail-on-no-tests.gradle') +apply from: file('gradle/testing/fail-on-unsupported-jdk.gradle') apply from: file('gradle/testing/alternative-jdk-support.gradle') apply from: file('gradle/java/jar-manifest.gradle') diff --git a/gradle/testing/fail-on-unsupported-jdk.gradle b/gradle/testing/fail-on-unsupported-jdk.gradle new file mode 100644 index 00000000000..7d94b709764 --- /dev/null +++ b/gradle/testing/fail-on-unsupported-jdk.gradle @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +configure(rootProject) { + task ensureJdkSupported() { + doFirst { + if (System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("mac") && rootProject.runtimeJavaVersion == JavaVersion.VERSION_20) { + throw new GradleException("Tests cannot be run with JDK20 on Mac; see SOLR-16733 for more details.") + } + } + } + + allprojects { + tasks.withType(Test) { + dependsOn ":ensureJdkSupported" + } + } +} From 7f2f6dc6f2d38cde3211a89f8859add791d74060 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Fri, 18 Aug 2023 16:21:00 -0400 Subject: [PATCH 018/160] SOLR-16941: Use smarter default Solr URL in CLI (#1849) --- solr/CHANGES.txt | 2 ++ solr/bin/solr | 22 +++++++++----- .../java/org/apache/solr/cli/CreateTool.java | 4 +-- .../src/java/org/apache/solr/cli/SolrCLI.java | 30 ++++++++++++++----- solr/packaging/test/test_ssl.bats | 4 +-- 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 098080ed7c2..90544c4c05a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -97,6 +97,8 @@ Improvements This is useful when using the ZkCli on a remote Solr using the embedded ZK, or Solr running in a Docker container. The SOLR_ZK_EMBEDDED_HOST envVar or -Dsolr.zk.embedded.host sysProp control this bind address. (Houston Putman) +* SOLR-16941: The SolrCLI now uses a smarter default for the Solr URL if none is provided, using the same envVars used when running Solr. (Houston Putman) + Optimizations --------------------- diff --git a/solr/bin/solr b/solr/bin/solr index bae4b722432..7ee0fa3c50f 100644 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -147,6 +147,11 @@ fi : "${SOLR_STOP_WAIT:=180}" : "${SOLR_START_WAIT:=$SOLR_STOP_WAIT}" # defaulting to $SOLR_STOP_WAIT for backwards compatibility reasons +# Store whether a solr port was explicitly provided, for the "solr stop" command. +PROVIDED_SOLR_PORT="${SOLR_PORT:-}" +: "${SOLR_PORT:=8983}" +export SOLR_PORT + # test that Java exists, is executable and correct version JAVA_VER=$("$JAVA" -version 2>&1) # shellcheck disable=SC2181 @@ -278,6 +283,7 @@ if [ "$SOLR_SSL_ENABLED" == "true" ]; then else SOLR_JETTY_CONFIG+=("--module=http") fi +export SOLR_URL_SCHEME # Requestlog options if [ "${SOLR_REQUESTLOG_ENABLED:-true}" == "true" ]; then @@ -330,6 +336,7 @@ AUTHC_OPTS="${AUTHC_OPTS:-}${SOLR_AUTHENTICATION_OPTS:+ $SOLR_AUTHENTICATION_OPT # Set the SOLR_TOOL_HOST variable for use when connecting to a running Solr instance SOLR_TOOL_HOST="${SOLR_HOST:-localhost}" +export SOLR_TOOL_HOST function print_usage() { CMD="${1:-}" @@ -547,7 +554,7 @@ function print_usage() { echo "" elif [ "$CMD" == "package" ]; then echo "" - run_tool package -solrUrl "http://fake.to.allow.help.to.run" "help" + run_tool package "help" fi } # end print_usage @@ -1436,8 +1443,8 @@ fi : "${STOP_KEY:=solrrocks}" -# stop all if no port specified -if [[ "$SCRIPT_CMD" == "stop" && -z "${SOLR_PORT:-}" ]]; then +# stop all if no port specified or "all" is used +if [[ "$SCRIPT_CMD" == "stop" ]]; then if $stop_all; then none_stopped=true find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | while read PIDF @@ -1455,8 +1462,9 @@ if [[ "$SCRIPT_CMD" == "stop" && -z "${SOLR_PORT:-}" ]]; then # if $none_stopped; then # echo -e "\nNo Solr nodes found to stop.\n" # fi - else - # not stopping all and don't have a port, but if we can find the pid file for the default port 8983, then use that + exit + elif [[ -z "${PROVIDED_SOLR_PORT:-}" ]]; then + # not stopping all and don't have a port, but if we can find a single pid file for Solr, then use that none_stopped=true numSolrs=$(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | wc -l | tr -d ' ') if [ "$numSolrs" -eq 1 ]; then @@ -1480,12 +1488,10 @@ if [[ "$SCRIPT_CMD" == "stop" && -z "${SOLR_PORT:-}" ]]; then fi exit 1 fi + exit fi - exit fi -: "${SOLR_PORT:=8983}" - if [ -n "${SOLR_PORT_ADVERTISE:-}" ]; then SOLR_OPTS+=("-Dsolr.port.advertise=$SOLR_PORT_ADVERTISE") fi diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 9d2f6315469..91a2214eeb4 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -130,7 +130,7 @@ public void runImpl(CommandLine cli) throws Exception { protected void createCore(CommandLine cli, SolrClient solrClient) throws Exception { String coreName = cli.getOptionValue("name"); - String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.DEFAULT_SOLR_URL); + String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.getDefaultSolrUrl()); final String solrInstallDir = System.getProperty("solr.install.dir"); final String confDirName = cli.getOptionValue("confdir", SolrCLI.DEFAULT_CONFIG_SET); @@ -343,7 +343,7 @@ private void printDefaultConfigsetWarningIfNecessary(CommandLine cli) { if (confDirectoryName.equals("_default") && (confName.equals("") || confName.equals("_default"))) { final String collectionName = cli.getOptionValue("collection"); - final String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.DEFAULT_SOLR_URL); + final String solrUrl = cli.getOptionValue("solrUrl", SolrCLI.getDefaultSolrUrl()); final String curlCommand = String.format( Locale.ROOT, diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index c3ddd46b85b..0459892eeea 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -75,7 +75,7 @@ public class SolrCLI implements CLIO { TimeUnit.NANOSECONDS.convert(1, TimeUnit.MINUTES); private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String DEFAULT_SOLR_URL = "http://localhost:8983"; + public static final String ZK_HOST = "localhost:9983"; public static final Option OPTION_ZKHOST = @@ -97,13 +97,13 @@ public class SolrCLI implements CLIO { .required(false) .desc( "Base Solr URL, which can be used to determine the zkHost if that's not known; defaults to: " - + DEFAULT_SOLR_URL + + getDefaultSolrUrl() + '.') .build(); public static final Option OPTION_VERBOSE = Option.builder("verbose").required(false).desc("Enable more verbose command output.").build(); - // should this be boolean or just a otption? + // should this be boolean or just an option? public static final Option OPTION_RECURSE = Option.builder("r") .longOpt("recurse") @@ -187,6 +187,22 @@ public static CommandLine parseCmdLine(String toolName, String[] args, List Date: Fri, 18 Aug 2023 18:15:31 -0400 Subject: [PATCH 019/160] SOLR-16940: Add SOLR_TOOL_OPTS var for custom SolrCLI options (#1848) --- solr/CHANGES.txt | 4 ++- solr/bin/solr | 4 +-- solr/bin/solr.cmd | 26 +++++++++---------- .../pages/solr-control-script-reference.adoc | 5 ++++ 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 90544c4c05a..c3985d65a8b 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -91,7 +91,7 @@ Improvements circumstances as they are generated and more likely to remain up-to-date with future API changes. (Jason Gerlowski, David Smiley, Houston Putman) -* SOLR-14667: Make zkClientTimeout consistent and based on a system property. The defauls values are stored in a single place referenced everywhere and they are based on system properties (Alex Deparvu) +* SOLR-14667: Make zkClientTimeout consistent and based on a system property. The default values are stored in a single place referenced everywhere and they are based on system properties (Alex Deparvu) * SOLR-16926: The embedded Zookeeper's bind host can now be overridden, but still defaults to "127.0.0.1". This is useful when using the ZkCli on a remote Solr using the embedded ZK, or Solr running in a Docker container. @@ -99,6 +99,8 @@ Improvements * SOLR-16941: The SolrCLI now uses a smarter default for the Solr URL if none is provided, using the same envVars used when running Solr. (Houston Putman) +* SOLR-16940: Users can pass Java system properties to the SolrCLI via the SOLR_TOOL_OPTS environment variable. (Houston Putman) + Optimizations --------------------- diff --git a/solr/bin/solr b/solr/bin/solr index 7ee0fa3c50f..8a6d0560757 100644 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -628,7 +628,7 @@ function jetty_port() { function run_tool() { # shellcheck disable=SC2086 - "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS ${SOLR_ZK_CREDS_AND_ACLS:-} -Dsolr.install.dir="$SOLR_TIP" \ + "$JAVA" ${SOLR_TOOL_OPTS:-} $SOLR_SSL_OPTS $AUTHC_OPTS ${SOLR_ZK_CREDS_AND_ACLS:-} -Dsolr.install.dir="$SOLR_TIP" \ -Dlog4j.configurationFile="$DEFAULT_SERVER_DIR/resources/log4j2-console.xml" \ -classpath "$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*:$DEFAULT_SERVER_DIR/lib/ext/*:$DEFAULT_SERVER_DIR/lib/*" \ org.apache.solr.cli.SolrCLI "$@" @@ -742,7 +742,7 @@ function stop_solr() { if [ -n "$SOLR_PID" ]; then echo -e "Sending stop command to Solr running on port $SOLR_PORT ... waiting up to $SOLR_STOP_WAIT seconds to allow Jetty process $SOLR_PID to stop gracefully." # shellcheck disable=SC2086 - "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS -jar "$DIR/start.jar" "STOP.PORT=$THIS_STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true + "$JAVA" ${SOLR_TOOL_OPTS:-} $SOLR_SSL_OPTS $AUTHC_OPTS -jar "$DIR/start.jar" "STOP.PORT=$THIS_STOP_PORT" "STOP.KEY=$STOP_KEY" --stop || true (loops=0 while true do diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 4055bad6d00..ce37a20d20e 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -923,7 +923,7 @@ IF "%SCRIPT_CMD%"=="stop" ( set found_it=1 @echo Stopping Solr process %%N running on port %SOLR_PORT% IF "%STOP_PORT%"=="" set /A STOP_PORT=%SOLR_PORT% - 1000 - "%JAVA%" %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% -Djetty.home="%SOLR_SERVER_DIR%" -jar "%SOLR_SERVER_DIR%\start.jar" %SOLR_JETTY_CONFIG% STOP.PORT=!STOP_PORT! STOP.KEY=%STOP_KEY% --stop del "%SOLR_TIP%"\bin\solr-%SOLR_PORT%.port REM wait for the process to terminate CALL :wait_for_process_exit %%N !SOLR_STOP_WAIT! @@ -1272,7 +1272,7 @@ IF "%FG%"=="1" ( set SOLR_START_WAIT=30 ) REM now wait to see Solr come online ... - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" -Dsolr.default.confdir="%DEFAULT_CONFDIR%"^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" -Dsolr.default.confdir="%DEFAULT_CONFDIR%"^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI status -maxWaitSecs !SOLR_START_WAIT! -solrUrl !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:%SOLR_PORT%/solr @@ -1287,7 +1287,7 @@ goto done :run_example REM Run the requested example -"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ +"%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -Dsolr.install.symDir="%SOLR_TIP%" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ @@ -1311,7 +1311,7 @@ for /f "usebackq" %%i in (`dir /b "%SOLR_TIP%\bin" ^| findstr /i "^solr-.*\.port set has_info=1 echo Found Solr process %%k running on port !SOME_SOLR_PORT! REM Passing in %2 (-h or -help) directly is captured by a custom help path for usage output - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI status -solrUrl !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:!SOME_SOLR_PORT!/solr %2 @@ -1326,7 +1326,7 @@ set has_info= goto done :run_solrcli -"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ +"%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI %* @@ -1475,7 +1475,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="The -d option must be set for upconfig." goto zk_short_usage ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !ZK_HOST! %ZK_VERBOSE%^ @@ -1489,7 +1489,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="The -d option must be set for downconfig." goto zk_short_usage ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !ZK_HOST! %ZK_VERBOSE% @@ -1508,7 +1508,7 @@ IF "!ZK_OP!"=="upconfig" ( goto zk_short_usage ) ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -src !ZK_SRC! -dst !ZK_DST! -recurse !ZK_RECURSE! %ZK_VERBOSE% @@ -1521,7 +1521,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG=" must be specified for 'mv' command" goto zk_short_usage ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -src !ZK_SRC! -dst !ZK_DST! %ZK_VERBOSE% @@ -1530,7 +1530,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="Zookeeper path to remove must be specified when using the 'rm' command" goto zk_short_usage ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -path !ZK_SRC! -recurse !ZK_RECURSE! %ZK_VERBOSE% @@ -1539,7 +1539,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="Zookeeper path to remove must be specified when using the 'ls' command" goto zk_short_usage ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -path !ZK_SRC! -recurse !ZK_RECURSE! %ZK_VERBOSE% @@ -1548,7 +1548,7 @@ IF "!ZK_OP!"=="upconfig" ( set ERROR_MSG="Zookeeper path to create must be specified when using the 'mkroot' command" goto zk_short_usage ) - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ + "%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%SOLR_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI !ZK_OP! -zkHost !ZK_HOST! -path !ZK_SRC! %ZK_VERBOSE% @@ -1609,7 +1609,7 @@ if "!AUTH_PORT!"=="" ( ) ) ) -"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ +"%JAVA%" %SOLR_TOOL_OPTS% %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^ -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ org.apache.solr.cli.SolrCLI auth %AUTH_PARAMS% -solrIncludeFile "%SOLR_INCLUDE%" -authConfDir "%SOLR_HOME%" ^ diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc index 4897a1a2c34..7d2a164ea28 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc @@ -301,6 +301,11 @@ For example, to set the auto soft-commit frequency to 3 seconds, you can do: `bin/solr start -Dsolr.autoSoftCommit.maxTime=3000` +The `SOLR_OPTS` environment variable is also available to set additional System Properties for Solr. + +In order to set custom System Properties when running any Solr utility other than `start` (e.g. `stop`, `create`, `auth`, `status`, `api`), +the `SOLR_TOOL_OPTS` environment variable should be used. + ==== SolrCloud Mode The `-c` and `-cloud` options are equivalent: From e0462e75a5cf848b1a3ae83ed6c95243785e21d6 Mon Sep 17 00:00:00 2001 From: Lamine <104593675+laminelam@users.noreply.github.com> Date: Mon, 21 Aug 2023 06:33:50 -0500 Subject: [PATCH 020/160] Add/fix wrong/missing "solr.home" to zkcli scripts (#1798) --------- Co-authored-by: Lamine Idjeraoui --- .../src/java/org/apache/solr/cloud/ZkCLI.java | 9 ++++++++- solr/packaging/test/test_zk.bats | 18 ++++++++++++++++++ solr/server/scripts/cloud-scripts/zkcli.bat | 6 ++++-- solr/server/scripts/cloud-scripts/zkcli.sh | 4 ++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java index c1f9b826529..07c66805068 100644 --- a/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java +++ b/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java @@ -86,6 +86,7 @@ public class ZkCLI implements CLIO { private static final String CMD = "cmd"; private static final String CLUSTERPROP = "clusterprop"; private static final String UPDATEACLS = "updateacls"; + private static final String VERBOSE = "verbose"; @VisibleForTesting public static void setStdout(PrintStream stdout) { @@ -174,12 +175,14 @@ public static void main(String[] args) options.addOption("h", HELP, false, "bring up this help page"); options.addOption(NAME, true, "name of the cluster property to set"); options.addOption(VALUE_LONG, true, "value of the cluster to set"); + options.addOption("v", VERBOSE, false, "enable verbose mode"); try { // parse the command line arguments CommandLine line = parser.parse(options, args); - if (line.hasOption(HELP) || !line.hasOption(ZKHOST) || !line.hasOption(CMD)) { + if ((line.hasOption(HELP) || !line.hasOption(ZKHOST) || !line.hasOption(CMD)) + && !line.hasOption(VERBOSE)) { // automatically generate the help statement HelpFormatter formatter = new HelpFormatter(); formatter.printHelp(ZK_CLI_NAME, options); @@ -243,6 +246,10 @@ public static void main(String[] args) if (StrUtils.isNullOrEmpty(solrHome)) { solrHome = System.getProperty("solr.home"); } + if (line.hasOption(VERBOSE)) { + stdout.println("Using " + SOLRHOME + "=" + solrHome); + return; + } String solrPort = null; if (line.hasOption(RUNZK)) { diff --git a/solr/packaging/test/test_zk.bats b/solr/packaging/test/test_zk.bats index 4e9a8bdd047..686437d914e 100644 --- a/solr/packaging/test/test_zk.bats +++ b/solr/packaging/test/test_zk.bats @@ -75,3 +75,21 @@ teardown() { assert_output --partial '"configSets":["_default","techproducts2"]' } + +@test "zkcli.sh gets 'solrhome' from 'solr.home' system property" { + sleep 1 + run "${SOLR_TIP}/server/scripts/cloud-scripts/zkcli.sh" -v + local extracted_solrhome=$(echo "$output" | grep -oE "solrhome=[^ ]+") + # remove 'solrhome=' + local path_value=${extracted_solrhome#*=} + [[ $path_value == *"/server/scripts/cloud-scripts/../../solr" ]] || [[ $path_value == *"/server/solr" ]] +} + +@test "zkcli.sh gets 'solrhome' from 'solrhome' command line option" { + sleep 1 + run "${SOLR_TIP}/server/scripts/cloud-scripts/zkcli.sh" -v -s /path/to/solr/home + local extracted_solrhome=$(echo "$output" | grep -oE "solrhome=[^ ]+") + # remove 'solrhome=' + local path_value=${extracted_solrhome#*=} + [[ $path_value == "/path/to/solr/home" ]] +} diff --git a/solr/server/scripts/cloud-scripts/zkcli.bat b/solr/server/scripts/cloud-scripts/zkcli.bat index 63c0eb108e7..0eb40faa8ea 100644 --- a/solr/server/scripts/cloud-scripts/zkcli.bat +++ b/solr/server/scripts/cloud-scripts/zkcli.bat @@ -1,6 +1,6 @@ @echo off REM You can override pass the following parameters to this script: -REM +REM set JVM=java @@ -8,6 +8,8 @@ REM Find location of this script set SDIR=%~dp0 if "%SDIR:~-1%"=="\" set SDIR=%SDIR:~0,-1% +set SOLR_INSTALL_DIR=%SDIR%\..\..\.. +set SOLR_HOME=%SOLR_INSTALL_DIR%\server\solr set "LOG4J_CONFIG=file:///%SDIR%\..\..\resources\log4j2-console.xml" @@ -21,5 +23,5 @@ REM optionally, you can use using a a Java properties file 'zkDigestCredentialsF REM ... REM -DzkDigestCredentialsFile=/path/to/zkDigestCredentialsFile.properties REM ... -"%JVM%" %SOLR_ZK_CREDS_AND_ACLS% %ZKCLI_JVM_FLAGS% -Dlog4j.configurationFile="%LOG4J_CONFIG%" ^ +"%JVM%" %SOLR_ZK_CREDS_AND_ACLS% %ZKCLI_JVM_FLAGS% -Dlog4j.configurationFile="%LOG4J_CONFIG%" -Dsolr.home=%SOLR_HOME% ^ -classpath "%SDIR%\..\..\solr-webapp\webapp\WEB-INF\lib\*;%SDIR%\..\..\lib\ext\*;%SDIR%\..\..\lib\*" org.apache.solr.cloud.ZkCLI %* diff --git a/solr/server/scripts/cloud-scripts/zkcli.sh b/solr/server/scripts/cloud-scripts/zkcli.sh index 4477df48634..4986d21c029 100755 --- a/solr/server/scripts/cloud-scripts/zkcli.sh +++ b/solr/server/scripts/cloud-scripts/zkcli.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # You can override pass the following parameters to this script: -# +# JVM="java" @@ -23,6 +23,6 @@ solr_home="$sdir/../../solr" #... # -DzkDigestCredentialsFile=/path/to/zkDigestCredentialsFile.properties #... -PATH=$JAVA_HOME/bin:$PATH $JVM $SOLR_ZK_CREDS_AND_ACLS $ZKCLI_JVM_FLAGS -Dlog4j.configurationFile=$log4j_config -DsolrHome=$solr_home \ +PATH=$JAVA_HOME/bin:$PATH $JVM $SOLR_ZK_CREDS_AND_ACLS $ZKCLI_JVM_FLAGS -Dlog4j.configurationFile=$log4j_config -Dsolr.home=$solr_home \ -classpath "$sdir/../../solr-webapp/webapp/WEB-INF/lib/*:$sdir/../../lib/ext/*:$sdir/../../lib/*" org.apache.solr.cloud.ZkCLI ${1+"$@"} From f6ef54a757cab2e3f4ee9785e234bff0bc5a22b0 Mon Sep 17 00:00:00 2001 From: Alex D Date: Mon, 21 Aug 2023 05:45:24 -0700 Subject: [PATCH 021/160] SOLR-16927 Allow SolrClientCache clients to use Jetty HTTP2 clients (#1835) --- solr/CHANGES.txt | 2 + solr/benchmark/build.gradle | 3 +- .../apache/solr/bench/MiniClusterState.java | 3 + .../solr/bench/search/StreamingSearch.java | 154 ++++++++++++++++++ .../solr/client/solrj/io/SolrClientCache.java | 153 ++++++++++------- .../solrj/io/stream/CloudSolrStream.java | 1 - .../client/solrj/io/stream/CommitStream.java | 12 +- .../solrj/io/stream/DeepRandomStream.java | 2 - .../client/solrj/io/stream/DeleteStream.java | 3 +- .../client/solrj/io/stream/Facet2DStream.java | 30 ++-- .../client/solrj/io/stream/FacetStream.java | 29 ++-- .../io/stream/FeaturesSelectionStream.java | 63 ++++--- .../client/solrj/io/stream/HavingStream.java | 4 +- .../client/solrj/io/stream/KnnStream.java | 23 ++- .../client/solrj/io/stream/LetStream.java | 2 +- .../client/solrj/io/stream/ModelStream.java | 3 - .../client/solrj/io/stream/RandomStream.java | 29 ++-- .../solrj/io/stream/ScoreNodesStream.java | 14 +- .../client/solrj/io/stream/SearchStream.java | 24 ++- .../io/stream/SignificantTermsStream.java | 54 +++--- .../client/solrj/io/stream/SolrStream.java | 42 +++-- .../client/solrj/io/stream/SqlStream.java | 2 - .../client/solrj/io/stream/StatsStream.java | 28 +++- .../client/solrj/io/stream/StreamContext.java | 3 +- .../solrj/io/stream/TextLogitStream.java | 44 +++-- .../solrj/io/stream/TimeSeriesStream.java | 23 ++- .../client/solrj/io/stream/TopicStream.java | 103 ++++++------ .../client/solrj/io/stream/UpdateStream.java | 41 ++--- .../solrj/impl/CloudHttp2SolrClient.java | 8 + .../client/solrj/impl/Http2SolrClient.java | 25 ++- .../client/solrj/impl/HttpSolrClient.java | 1 + .../client/solrj/impl/SolrClientBuilder.java | 8 + 32 files changed, 596 insertions(+), 340 deletions(-) create mode 100644 solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c3985d65a8b..e09b1849394 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -101,6 +101,8 @@ Improvements * SOLR-16940: Users can pass Java system properties to the SolrCLI via the SOLR_TOOL_OPTS environment variable. (Houston Putman) +* SOLR-16927: Allow SolrClientCache clients to use Jetty HTTP2 clients (Alex Deparvu, David Smiley) + Optimizations --------------------- diff --git a/solr/benchmark/build.gradle b/solr/benchmark/build.gradle index be99ea3ee03..63cea7af01f 100644 --- a/solr/benchmark/build.gradle +++ b/solr/benchmark/build.gradle @@ -46,9 +46,10 @@ task echoCp { dependencies { implementation project(':solr:test-framework') implementation project(':solr:solrj') + implementation project(':solr:solrj-streaming') implementation 'org.apache.lucene:lucene-core' - + implementation 'org.apache.httpcomponents:httpclient' implementation 'commons-io:commons-io' implementation 'io.dropwizard.metrics:metrics-core' implementation 'org.apache.commons:commons-math3' diff --git a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java index a35dd07bbcc..dd1c0afe639 100755 --- a/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java @@ -86,6 +86,8 @@ public static class MiniClusterBenchState { /** The Nodes. */ public List nodes; + public String zkHost; + /** The Cluster. */ MiniSolrCloudCluster cluster; @@ -277,6 +279,7 @@ public void startMiniCluster(int nodeCount) { for (JettySolrRunner runner : jetties) { nodes.add(runner.getBaseUrl().toString()); } + zkHost = cluster.getZkServer().getZkAddress(); client = new Http2SolrClient.Builder().useHttp1_1(useHttp1).build(); diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java new file mode 100644 index 00000000000..14046644c46 --- /dev/null +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/StreamingSearch.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.search; + +import static org.apache.solr.bench.Docs.docs; +import static org.apache.solr.bench.generators.SourceDSL.integers; +import static org.apache.solr.bench.generators.SourceDSL.strings; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.solr.bench.Docs; +import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.bench.MiniClusterState.MiniClusterBenchState; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.io.SolrClientCache; +import org.apache.solr.client.solrj.io.Tuple; +import org.apache.solr.client.solrj.io.stream.CloudSolrStream; +import org.apache.solr.client.solrj.io.stream.StreamContext; +import org.apache.solr.client.solrj.io.stream.TupleStream; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +@Fork(value = 1) +@BenchmarkMode(Mode.Throughput) +@Warmup(time = 5, iterations = 1) +@Measurement(time = 30, iterations = 4) +@Threads(value = 1) +public class StreamingSearch { + + private static final String collection = "benchStreamingSearch"; + + @State(Scope.Benchmark) + public static class BenchState { + + @Param({"false", "true"}) + boolean useHttp1; + + private int docs = 1000; + private String zkHost; + private ModifiableSolrParams params; + private StreamContext streamContext; + private Http2SolrClient http2SolrClient; + + @Setup(Level.Trial) + public void setup(MiniClusterBenchState miniClusterState) throws Exception { + + miniClusterState.startMiniCluster(3); + miniClusterState.createCollection(collection, 3, 1); + Docs docGen = + docs() + .field("id", integers().incrementing()) + .field("text2_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)) + .field("text3_ts", strings().basicLatinAlphabet().multi(312).ofLengthBetween(30, 64)) + .field("int1_i_dv", integers().all()); + miniClusterState.index(collection, docGen, docs); + miniClusterState.waitForMerges(collection); + + zkHost = miniClusterState.zkHost; + + params = new ModifiableSolrParams(); + params.set(CommonParams.Q, "*:*"); + params.set(CommonParams.FL, "id,text2_ts,text3_ts,int1_i_dv"); + params.set(CommonParams.SORT, "id asc,int1_i_dv asc"); + params.set(CommonParams.ROWS, docs); + } + + @Setup(Level.Iteration) + public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + SolrClientCache solrClientCache; + if (useHttp1) { + var httpClient = HttpClientUtil.createClient(null); // TODO tune params? + solrClientCache = new SolrClientCache(httpClient); + } else { + http2SolrClient = newHttp2SolrClient(); + solrClientCache = new SolrClientCache(http2SolrClient); + } + + streamContext = new StreamContext(); + streamContext.setSolrClientCache(solrClientCache); + } + + @TearDown(Level.Iteration) + public void teardownIt() { + streamContext.getSolrClientCache().close(); + if (http2SolrClient != null) { + http2SolrClient.close(); + } + } + } + + @Benchmark + public Object stream( + BenchState benchState, MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + CloudSolrStream stream = new CloudSolrStream(benchState.zkHost, collection, benchState.params); + stream.setStreamContext(benchState.streamContext); + return getTuples(stream); + } + + private static List getTuples(TupleStream tupleStream) throws IOException { + List tuples = new ArrayList<>(); + try { + tupleStream.open(); + while (true) { + Tuple t = tupleStream.read(); + if (t.EOF) { + break; + } else { + tuples.add(t); + } + } + return tuples; + } finally { + tupleStream.close(); + } + } + + public static Http2SolrClient newHttp2SolrClient() { + // TODO tune params? + var builder = new Http2SolrClient.Builder(); + return builder.build(); + } +} diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java index ff2963c89fb..02621784d00 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java @@ -16,10 +16,9 @@ */ package org.apache.solr.client.solrj.io; +import java.io.Closeable; import java.io.IOException; -import java.io.Serializable; import java.lang.invoke.MethodHandles; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,97 +27,137 @@ import java.util.concurrent.TimeUnit; import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.SolrClientBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * The SolrClientCache caches SolrClients so they can be reused by different TupleStreams. - * - *

TODO: Cut this over to using Solr's new Http2 clients - */ -public class SolrClientCache implements Serializable { +/** The SolrClientCache caches SolrClients so they can be reused by different TupleStreams. */ +public class SolrClientCache implements Closeable { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private final Map solrClients = new HashMap<>(); - private final HttpClient httpClient; // Set the floor for timeouts to 60 seconds. // Timeouts cans be increased by setting the system properties defined below. - private static final int conTimeout = + private static final int MIN_TIMEOUT = 60000; + private static final int minConnTimeout = Math.max( - Integer.parseInt(System.getProperty(HttpClientUtil.PROP_CONNECTION_TIMEOUT, "60000")), - 60000); - private static final int socketTimeout = - Math.max( - Integer.parseInt(System.getProperty(HttpClientUtil.PROP_SO_TIMEOUT, "60000")), 60000); + Integer.getInteger(HttpClientUtil.PROP_CONNECTION_TIMEOUT, MIN_TIMEOUT), MIN_TIMEOUT); + private static final int minSocketTimeout = + Math.max(Integer.getInteger(HttpClientUtil.PROP_SO_TIMEOUT, MIN_TIMEOUT), MIN_TIMEOUT); + + private final Map solrClients = new HashMap<>(); + private final HttpClient apacheHttpClient; + private final Http2SolrClient http2SolrClient; public SolrClientCache() { - httpClient = null; + this.apacheHttpClient = null; + this.http2SolrClient = null; } @Deprecated(since = "9.0") - public SolrClientCache(HttpClient httpClient) { - this.httpClient = httpClient; + public SolrClientCache(HttpClient apacheHttpClient) { + this.apacheHttpClient = apacheHttpClient; + this.http2SolrClient = null; } - @Deprecated(since = "9.0") - public synchronized CloudSolrClient getCloudSolrClient(String zkHost) { - - // Timeouts should never be lower then 60000 but they can be set higher - assert (conTimeout >= 60000); - assert (socketTimeout >= 60000); - - if (log.isDebugEnabled()) { - log.debug("SolrClientCache.conTimeout: {}", conTimeout); - log.debug("SolrClientCache.socketTimeout: {}", socketTimeout); - } + public SolrClientCache(Http2SolrClient http2SolrClient) { + this.apacheHttpClient = null; + this.http2SolrClient = http2SolrClient; + } + public synchronized CloudSolrClient getCloudSolrClient(String zkHost) { Objects.requireNonNull(zkHost, "ZooKeeper host cannot be null!"); - CloudSolrClient client; if (solrClients.containsKey(zkHost)) { - client = (CloudSolrClient) solrClients.get(zkHost); + return (CloudSolrClient) solrClients.get(zkHost); + } + final CloudSolrClient client; + if (apacheHttpClient != null) { + client = newCloudLegacySolrClient(zkHost, apacheHttpClient); } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - var builder = - new CloudLegacySolrClient.Builder(hosts, Optional.empty()) - .withSocketTimeout(socketTimeout, TimeUnit.MILLISECONDS) - .withConnectionTimeout(conTimeout, TimeUnit.MILLISECONDS); - if (httpClient != null) { - builder = builder.withHttpClient(httpClient); - } - - client = builder.build(); - client.connect(); - solrClients.put(zkHost, client); + client = newCloudHttp2SolrClient(zkHost, http2SolrClient); } + solrClients.put(zkHost, client); + return client; + } + @Deprecated + private static CloudSolrClient newCloudLegacySolrClient(String zkHost, HttpClient httpClient) { + final List hosts = List.of(zkHost); + var builder = new CloudLegacySolrClient.Builder(hosts, Optional.empty()); + adjustTimeouts(builder, httpClient); + var client = builder.build(); + client.connect(); + return client; + } + + private static CloudHttp2SolrClient newCloudHttp2SolrClient( + String zkHost, Http2SolrClient http2SolrClient) { + final List hosts = List.of(zkHost); + var builder = new CloudHttp2SolrClient.Builder(hosts, Optional.empty()); + // using internal builder to ensure the internal client gets closed + builder = builder.withInternalClientBuilder(newHttp2SolrClientBuilder(null, http2SolrClient)); + var client = builder.build(); + client.connect(); return client; } - @Deprecated(since = "9.0") public synchronized SolrClient getHttpSolrClient(String baseUrl) { - SolrClient client; + Objects.requireNonNull(baseUrl, "Url cannot be null!"); if (solrClients.containsKey(baseUrl)) { - client = solrClients.get(baseUrl); + return solrClients.get(baseUrl); + } + final SolrClient client; + if (apacheHttpClient != null) { + client = newHttpSolrClient(baseUrl, apacheHttpClient); } else { - HttpSolrClient.Builder builder = - new HttpSolrClient.Builder(baseUrl) - .withSocketTimeout(socketTimeout, TimeUnit.MILLISECONDS) - .withConnectionTimeout(conTimeout, TimeUnit.MILLISECONDS); - if (httpClient != null) { - builder = builder.withHttpClient(httpClient); - } - client = builder.build(); - solrClients.put(baseUrl, client); + client = newHttp2SolrClientBuilder(baseUrl, http2SolrClient).build(); } + solrClients.put(baseUrl, client); return client; } + @Deprecated + private static SolrClient newHttpSolrClient(String baseUrl, HttpClient httpClient) { + HttpSolrClient.Builder builder = new HttpSolrClient.Builder(baseUrl); + adjustTimeouts(builder, httpClient); + return builder.build(); + } + + @Deprecated + private static void adjustTimeouts(SolrClientBuilder builder, HttpClient httpClient) { + builder.withHttpClient(httpClient); + int socketTimeout = Math.max(minSocketTimeout, builder.getSocketTimeoutMillis()); + builder.withSocketTimeout(socketTimeout, TimeUnit.MILLISECONDS); + int connTimeout = Math.max(minConnTimeout, builder.getConnectionTimeoutMillis()); + builder.withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS); + } + + private static Http2SolrClient.Builder newHttp2SolrClientBuilder( + String baseUrl, Http2SolrClient http2SolrClient) { + var builder = new Http2SolrClient.Builder(baseUrl); + if (http2SolrClient != null) { + builder = builder.withHttpClient(http2SolrClient); + } + long idleTimeout = minSocketTimeout; + if (builder.getIdleTimeoutMillis() != null) { + idleTimeout = Math.max(idleTimeout, builder.getIdleTimeoutMillis()); + } + builder.withIdleTimeout(idleTimeout, TimeUnit.MILLISECONDS); + long connTimeout = minConnTimeout; + if (builder.getConnectionTimeout() != null) { + connTimeout = Math.max(idleTimeout, builder.getConnectionTimeout()); + } + builder.withConnectionTimeout(connTimeout, TimeUnit.MILLISECONDS); + return builder; + } + + @Override public synchronized void close() { for (Map.Entry entry : solrClients.entrySet()) { try { diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java index 6cef1b2c9b6..8b2bd54aef7 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CloudSolrStream.java @@ -75,7 +75,6 @@ public class CloudSolrStream extends TupleStream implements Expressible { protected StreamComparator comp; private boolean trace; protected transient Map eofTuples; - protected transient CloudSolrClient cloudSolrClient; protected transient List solrStreams; protected transient TreeSet tuples; protected transient StreamContext streamContext; diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CommitStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CommitStream.java index 1121ab12421..79f8abfab23 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CommitStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/CommitStream.java @@ -55,6 +55,7 @@ public class CommitStream extends TupleStream implements Expressible { private TupleStream tupleSource; private transient SolrClientCache clientCache; + private transient boolean doCloseCache; private long docsSinceCommit; public CommitStream(StreamExpression expression, StreamFactory factory) throws IOException { @@ -150,7 +151,12 @@ private void init( @Override public void open() throws IOException { tupleSource.open(); - clientCache = new SolrClientCache(); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); + } else { + doCloseCache = false; + } docsSinceCommit = 0; } @@ -193,7 +199,9 @@ private boolean isInteger(String string) { @Override public void close() throws IOException { - clientCache.close(); + if (doCloseCache) { + clientCache.close(); + } tupleSource.close(); } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeepRandomStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeepRandomStream.java index 02bb701d5d3..2e32dc26f09 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeepRandomStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeepRandomStream.java @@ -37,7 +37,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.stream.Collectors; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; import org.apache.solr.client.solrj.io.stream.expr.Explanation; @@ -71,7 +70,6 @@ public class DeepRandomStream extends TupleStream implements Expressible { protected StreamComparator comp; private boolean trace; protected transient Map eofTuples; - protected transient CloudSolrClient cloudSolrClient; protected transient List solrStreams; protected transient Deque tuples; protected transient StreamContext streamContext; diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeleteStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeleteStream.java index e78941c80a8..d00908792ed 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeleteStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/DeleteStream.java @@ -79,7 +79,8 @@ protected void uploadBatchToCollection(List documentBatch) th final Long version = getVersion(doc); req.deleteById(id, version); } - req.process(getCloudSolrClient(), getCollectionName()); + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); + req.process(cloudSolrClient, getCollectionName()); } catch (SolrServerException | NumberFormatException | IOException e) { log.warn("Unable to delete documents from collection due to unexpected error.", e); String className = e.getClass().getName(); diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/Facet2DStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/Facet2DStream.java index c362257387c..878dee75af4 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/Facet2DStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/Facet2DStream.java @@ -24,12 +24,8 @@ import java.util.List; import java.util.Locale; import java.util.Map.Entry; -import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -68,8 +64,8 @@ public class Facet2DStream extends TupleStream implements Expressible { private int dimensionY; private FieldComparator bucketSort; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; + protected transient SolrClientCache clientCache; + private transient boolean doCloseCache; public Facet2DStream( String zkHost, @@ -312,7 +308,7 @@ public StreamExpressionParameter toExpression(StreamFactory factory) throws IOEx @Override public void setStreamContext(StreamContext context) { - cache = context.getSolrClientCache(); + clientCache = context.getSolrClientCache(); } @Override @@ -322,16 +318,11 @@ public List children() { @Override public void open() throws IOException { - if (cache != null) { - cloudSolrClient = cache.getCloudSolrClient(zkHost); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - cloudSolrClient = - new CloudLegacySolrClient.Builder(hosts, Optional.empty()) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .build(); + doCloseCache = false; } FieldComparator[] adjustedSorts = adjustSorts(x, y, bucketSort); @@ -344,6 +335,7 @@ public void open() throws IOException { QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST); try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); NamedList response = cloudSolrClient.request(request, collection); getTuples(response, x, y, metric); this.out = tuples.iterator(); @@ -366,10 +358,8 @@ public Tuple read() throws IOException { @Override public void close() throws IOException { - if (cache == null) { - if (cloudSolrClient != null) { - cloudSolrClient.close(); - } + if (doCloseCache) { + clientCache.close(); } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java index 3b0b785cbc9..759c038d7a0 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java @@ -24,11 +24,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.ClusterStateProvider; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -88,8 +85,8 @@ public class FacetStream extends TupleStream implements Expressible, ParallelMet private boolean resortNeeded; private boolean serializeBucketSizeLimit; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; + protected transient SolrClientCache clientCache; + private transient boolean doCloseCache; protected transient TupleStream parallelizedStream; protected transient StreamContext context; @@ -647,7 +644,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { this.context = context; - cache = context.getSolrClientCache(); + this.clientCache = context.getSolrClientCache(); } @Override @@ -657,17 +654,13 @@ public List children() { @Override public void open() throws IOException { - if (cache != null) { - cloudSolrClient = cache.getCloudSolrClient(zkHost); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - cloudSolrClient = - new CloudLegacySolrClient.Builder(hosts, Optional.empty()) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .build(); + doCloseCache = false; } + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); // Parallelize the facet expression across multiple collections for an alias using plist if // possible @@ -761,10 +754,8 @@ private boolean expectedJson(String json) { @Override public void close() throws IOException { - if (cache == null) { - if (cloudSolrClient != null) { - cloudSolrClient.close(); - } + if (doCloseCache) { + clientCache.close(); } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FeaturesSelectionStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FeaturesSelectionStream.java index d4990800cd4..3a761ceaee4 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FeaturesSelectionStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/FeaturesSelectionStream.java @@ -37,7 +37,6 @@ import java.util.concurrent.Future; import java.util.stream.Stream; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -76,12 +75,9 @@ public class FeaturesSelectionStream extends TupleStream implements Expressible protected int positiveLabel; protected int numTerms; - protected transient SolrClientCache cache; - protected transient boolean isCloseCache; - protected transient CloudSolrClient cloudSolrClient; - - protected transient StreamContext streamContext; - protected ExecutorService executorService; + protected transient SolrClientCache clientCache; + private transient boolean doCloseCache; + protected transient ExecutorService executorService; public FeaturesSelectionStream( String zkHost, @@ -254,21 +250,19 @@ private void init( @Override public void setStreamContext(StreamContext context) { - this.cache = context.getSolrClientCache(); - this.streamContext = context; + this.clientCache = context.getSolrClientCache(); } /** Opens the CloudSolrStream */ @Override public void open() throws IOException { - if (cache == null) { - isCloseCache = true; - cache = new SolrClientCache(); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - isCloseCache = false; + doCloseCache = false; } - this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost); this.executorService = ExecutorUtil.newMDCAwareCachedThreadPool( new SolrNamedThreadFactory("FeaturesSelectionStream")); @@ -281,6 +275,7 @@ public List children() { private List getShardUrls() throws IOException { try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); Slice[] slices = CloudSolrStream.getSlices(this.collection, cloudSolrClient, false); Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); @@ -314,7 +309,14 @@ private List>> callShards(List baseUrls) throws IOEx List>> futures = new ArrayList<>(); for (String baseUrl : baseUrls) { FeaturesSelectionCall lc = - new FeaturesSelectionCall(baseUrl, this.params, this.field, this.outcome); + new FeaturesSelectionCall( + baseUrl, + this.params, + this.field, + this.outcome, + this.positiveLabel, + this.numTerms, + this.clientCache); Future> future = executorService.submit(lc); futures.add(future); @@ -325,8 +327,8 @@ private List>> callShards(List baseUrls) throws IOEx @Override public void close() throws IOException { - if (isCloseCache && cache != null) { - cache.close(); + if (doCloseCache) { + clientCache.close(); } if (executorService != null) { @@ -417,26 +419,37 @@ private > Map sortByValue(Map map return result; } - protected class FeaturesSelectionCall implements Callable> { + protected static class FeaturesSelectionCall implements Callable> { - private String baseUrl; - private String outcome; - private String field; - private Map paramsMap; + private final String baseUrl; + private final String outcome; + private final String field; + private final Map paramsMap; + private final int positiveLabel; + private final int numTerms; + private final SolrClientCache clientCache; public FeaturesSelectionCall( - String baseUrl, Map paramsMap, String field, String outcome) { - + String baseUrl, + Map paramsMap, + String field, + String outcome, + int positiveLabel, + int numTerms, + SolrClientCache clientCache) { this.baseUrl = baseUrl; this.outcome = outcome; this.field = field; this.paramsMap = paramsMap; + this.positiveLabel = positiveLabel; + this.numTerms = numTerms; + this.clientCache = clientCache; } @Override public NamedList call() throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); - SolrClient solrClient = cache.getHttpSolrClient(baseUrl); + SolrClient solrClient = clientCache.getHttpSolrClient(baseUrl); params.add(DISTRIB, "false"); params.add("fq", "{!igain}"); diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/HavingStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/HavingStream.java index e151aaa19f7..988a3dbc73e 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/HavingStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/HavingStream.java @@ -44,9 +44,7 @@ public class HavingStream extends TupleStream implements Expressible { private TupleStream stream; private RecursiveBooleanEvaluator evaluator; - private StreamContext streamContext; - - private transient Tuple currentGroupHead; + private transient StreamContext streamContext; public HavingStream(TupleStream stream, RecursiveBooleanEvaluator evaluator) throws IOException { init(stream, evaluator); diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java index fb1d2ba6e63..41cd08d18bf 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/KnnStream.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.stream.Collectors; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -60,8 +59,8 @@ public class KnnStream extends TupleStream implements Expressible { private String zkHost; private Map props; private String collection; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; private Iterator documentIterator; private String id; @@ -193,7 +192,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { - cache = context.getSolrClientCache(); + clientCache = context.getSolrClientCache(); } @Override @@ -204,7 +203,13 @@ public List children() { @Override public void open() throws IOException { - cloudSolrClient = cache.getCloudSolrClient(zkHost); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); + } else { + doCloseCache = false; + } + ModifiableSolrParams params = getParams(this.props); StringBuilder builder = new StringBuilder(); @@ -227,7 +232,7 @@ public void open() throws IOException { QueryRequest request = new QueryRequest(params); try { - QueryResponse response = request.process(cloudSolrClient, collection); + QueryResponse response = request.process(clientCache.getCloudSolrClient(zkHost), collection); SolrDocumentList docs = response.getResults(); documentIterator = docs.iterator(); } catch (Exception e) { @@ -236,7 +241,11 @@ public void open() throws IOException { } @Override - public void close() throws IOException {} + public void close() throws IOException { + if (doCloseCache) { + clientCache.close(); + } + } @Override public Tuple read() throws IOException { diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java index e5a6c4113ab..9576cf9658e 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/LetStream.java @@ -44,7 +44,7 @@ public class LetStream extends TupleStream implements Expressible { private static final long serialVersionUID = 1; private TupleStream stream; - private StreamContext streamContext; + private transient StreamContext streamContext; private Map letParams = new LinkedHashMap<>(); public LetStream(StreamExpression expression, StreamFactory factory) throws IOException { diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ModelStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ModelStream.java index 727df5560a5..096f4a55e9c 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ModelStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ModelStream.java @@ -26,7 +26,6 @@ import java.util.Locale; import java.util.Map; import org.apache.solr.client.solrj.io.ModelCache; -import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; import org.apache.solr.client.solrj.io.stream.expr.Explanation; @@ -53,7 +52,6 @@ public class ModelStream extends TupleStream implements Expressible { protected String collection; protected String modelID; protected ModelCache modelCache; - protected SolrClientCache solrClientCache; protected Tuple model; protected long cacheMillis; @@ -162,7 +160,6 @@ private void init(String collectionName, String zkHost, String modelID, long cac @Override public void setStreamContext(StreamContext context) { - this.solrClientCache = context.getSolrClientCache(); this.modelCache = context.getModelCache(); } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java index aa0087bf8b0..c802fd870ee 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java @@ -27,13 +27,9 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Random; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -64,12 +60,13 @@ public class RandomStream extends TupleStream implements Expressible { private String zkHost; private Map props; private String collection; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; private Iterator documentIterator; private int x; private boolean outputX; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + public RandomStream() { // Used by the RandomFacade } @@ -200,7 +197,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { - cache = context.getSolrClientCache(); + clientCache = context.getSolrClientCache(); } @Override @@ -211,16 +208,11 @@ public List children() { @Override public void open() throws IOException { - if (cache != null) { - cloudSolrClient = cache.getCloudSolrClient(zkHost); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - cloudSolrClient = - new CloudLegacySolrClient.Builder(hosts, Optional.empty()) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .build(); + doCloseCache = false; } ModifiableSolrParams params = getParams(this.props); @@ -235,6 +227,7 @@ public void open() throws IOException { QueryRequest request = new QueryRequest(params, SolrRequest.METHOD.POST); try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); QueryResponse response = request.process(cloudSolrClient, collection); SolrDocumentList docs = response.getResults(); documentIterator = docs.iterator(); @@ -245,8 +238,8 @@ public void open() throws IOException { @Override public void close() throws IOException { - if (cache == null) { - cloudSolrClient.close(); + if (doCloseCache) { + clientCache.close(); } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ScoreNodesStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ScoreNodesStream.java index ea10185f730..01be8e309c4 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ScoreNodesStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/ScoreNodesStream.java @@ -59,7 +59,6 @@ public class ScoreNodesStream extends TupleStream implements Expressible { protected String zkHost; private TupleStream stream; - private transient SolrClientCache clientCache; private Map nodes = new HashMap<>(); private Iterator tuples; private String termFreq; @@ -68,6 +67,9 @@ public class ScoreNodesStream extends TupleStream implements Expressible { private String bucket; private String facetCollection; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + public ScoreNodesStream(TupleStream tupleStream, String nodeFreqField) throws IOException { init(tupleStream, nodeFreqField); } @@ -176,6 +178,13 @@ public List children() { @Override public void open() throws IOException { + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); + } else { + doCloseCache = false; + } + stream.open(); Tuple node = null; StringBuilder builder = new StringBuilder(); @@ -261,6 +270,9 @@ public void open() throws IOException { @Override public void close() throws IOException { + if (doCloseCache) { + clientCache.close(); + } stream.close(); } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java index 1e6226b7536..f1e6c33c2fb 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java @@ -24,10 +24,7 @@ import java.util.List; import java.util.Locale; import java.util.Map.Entry; -import java.util.Optional; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; @@ -55,11 +52,12 @@ public class SearchStream extends TupleStream implements Expressible { private String zkHost; private ModifiableSolrParams params; private String collection; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; private Iterator documentIterator; protected StreamComparator comp; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + public SearchStream() {} public SearchStream(StreamExpression expression, StreamFactory factory) throws IOException { @@ -184,7 +182,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { - cache = context.getSolrClientCache(); + clientCache = context.getSolrClientCache(); } @Override @@ -195,16 +193,16 @@ public List children() { @Override public void open() throws IOException { - if (cache != null) { - cloudSolrClient = cache.getCloudSolrClient(zkHost); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - cloudSolrClient = new CloudLegacySolrClient.Builder(hosts, Optional.empty()).build(); + doCloseCache = false; } QueryRequest request = new QueryRequest(params, SolrRequest.METHOD.POST); try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); QueryResponse response = request.process(cloudSolrClient, collection); SolrDocumentList docs = response.getResults(); documentIterator = docs.iterator(); @@ -215,8 +213,8 @@ public void open() throws IOException { @Override public void close() throws IOException { - if (cache == null) { - cloudSolrClient.close(); + if (doCloseCache) { + clientCache.close(); } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java index 7e47003fa63..22d7c8bc73c 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java @@ -68,10 +68,10 @@ public class SignificantTermsStream extends TupleStream implements Expressible { protected float maxDocFreq; protected int minTermLength; - protected transient SolrClientCache cache; - protected transient boolean isCloseCache; - protected transient StreamContext streamContext; - protected ExecutorService executorService; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + private transient StreamContext streamContext; + private transient ExecutorService executorService; public SignificantTermsStream( String zkHost, @@ -239,17 +239,17 @@ private void init( @Override public void setStreamContext(StreamContext context) { - this.cache = context.getSolrClientCache(); + this.clientCache = context.getSolrClientCache(); this.streamContext = context; } @Override public void open() throws IOException { - if (cache == null) { - isCloseCache = true; - cache = new SolrClientCache(); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - isCloseCache = false; + doCloseCache = false; } this.executorService = @@ -274,7 +274,9 @@ private List>> callShards(List baseUrls) throws IOEx this.minDocFreq, this.maxDocFreq, this.minTermLength, - this.numTerms); + this.numTerms, + streamContext.isLocal(), + clientCache); Future> future = executorService.submit(lc); futures.add(future); @@ -285,8 +287,8 @@ private List>> callShards(List baseUrls) throws IOEx @Override public void close() throws IOException { - if (isCloseCache) { - cache.close(); + if (doCloseCache) { + clientCache.close(); } executorService.shutdown(); @@ -387,15 +389,17 @@ public int compare(Map a, Map b) { } } - protected class SignificantTermsCall implements Callable> { + protected static class SignificantTermsCall implements Callable> { - private String baseUrl; - private String field; - private float minDocFreq; - private float maxDocFreq; - private int numTerms; - private int minTermLength; - private Map paramsMap; + private final String baseUrl; + private final String field; + private final float minDocFreq; + private final float maxDocFreq; + private final int numTerms; + private final int minTermLength; + private final Map paramsMap; + private final boolean isLocal; + private final SolrClientCache clientCache; public SignificantTermsCall( String baseUrl, @@ -404,7 +408,9 @@ public SignificantTermsCall( float minDocFreq, float maxDocFreq, int minTermLength, - int numTerms) { + int numTerms, + boolean isLocal, + SolrClientCache clientCache) { this.baseUrl = baseUrl; this.field = field; @@ -413,12 +419,14 @@ public SignificantTermsCall( this.paramsMap = paramsMap; this.numTerms = numTerms; this.minTermLength = minTermLength; + this.isLocal = isLocal; + this.clientCache = clientCache; } @Override public NamedList call() throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); - SolrClient solrClient = cache.getHttpSolrClient(baseUrl); + SolrClient solrClient = clientCache.getHttpSolrClient(baseUrl); params.add(DISTRIB, "false"); params.add("fq", "{!significantTerms}"); @@ -432,7 +440,7 @@ public NamedList call() throws Exception { params.add("minTermLength", Integer.toString(minTermLength)); params.add("field", field); params.add("numTerms", String.valueOf(numTerms * 5)); - if (streamContext.isLocal()) { + if (isLocal) { params.add("distrib", "false"); } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java index 8f019c36e61..f58546cb432 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java @@ -16,6 +16,7 @@ */ package org.apache.solr.client.solrj.io.stream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -25,10 +26,8 @@ import java.util.List; import java.util.Map; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.impl.InputStreamResponseParser; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -47,8 +46,6 @@ /** * Queries a single Solr instance and maps SolrDocs to a Stream of Tuples. * - *

TODO: Move this to Http2SolrClient - * * @since 5.1.0 */ public class SolrStream extends TupleStream { @@ -62,16 +59,17 @@ public class SolrStream extends TupleStream { private boolean trace; private Map fieldMappings; private transient TupleStreamParser tupleStreamParser; - private transient SolrClient client; - private transient SolrClientCache cache; private String slice; private long checkpoint = -1; - private CloseableHttpResponse closeableHttpResponse; + private Closeable closeableHttpResponse; private boolean distrib = true; private String user; private String password; private String core; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + /** * @param baseUrl Base URL of the stream. * @param params Map<String, String> of parameters @@ -104,7 +102,7 @@ public void setStreamContext(StreamContext context) { this.distrib = !context.isLocal(); this.numWorkers = context.numWorkers; this.workerID = context.workerID; - this.cache = context.getSolrClientCache(); + this.clientCache = context.getSolrClientCache(); } public void setCredentials(String user, String password) { @@ -115,12 +113,11 @@ public void setCredentials(String user, String password) { /** Opens the stream to a single Solr instance. */ @Override public void open() throws IOException { - - // Reuse the same client per node vs. having one per replica - if (cache == null) { - client = new HttpSolrClient.Builder(baseUrl).build(); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - client = cache.getHttpSolrClient(baseUrl); + doCloseCache = false; } try { @@ -193,8 +190,8 @@ public void close() throws IOException { if (closeableHttpResponse != null) { closeableHttpResponse.close(); } - if (cache == null && client != null) { - client.close(); + if (doCloseCache) { + clientCache.close(); } } @@ -303,15 +300,26 @@ private TupleStreamParser constructParser(SolrParams requestParams) query.setBasicAuthCredentials(user, password); } + var client = clientCache.getHttpSolrClient(baseUrl); NamedList genericResponse = client.request(query); InputStream stream = (InputStream) genericResponse.get("stream"); + CloseableHttpResponse httpResponse = (CloseableHttpResponse) genericResponse.get("closeableResponse"); + // still attempting to read http status from http response for backwards compatibility reasons + // since 9.4 the updated format will have a dedicated status field + final int statusCode; + if (httpResponse != null) { + statusCode = httpResponse.getStatusLine().getStatusCode(); + } else { + statusCode = (int) genericResponse.get("responseStatus"); + } - final int statusCode = httpResponse.getStatusLine().getStatusCode(); if (statusCode != 200) { String errMsg = consumeStreamAsErrorMessage(stream); - httpResponse.close(); + if (httpResponse != null) { + httpResponse.close(); + } throw new IOException( "Query to '" + query.getPath() diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SqlStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SqlStream.java index a0419567d1d..b39ab1dbdc1 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SqlStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SqlStream.java @@ -24,7 +24,6 @@ import java.util.Map.Entry; import java.util.Random; import java.util.stream.Collectors; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; import org.apache.solr.client.solrj.io.stream.expr.Explanation; @@ -49,7 +48,6 @@ public class SqlStream extends TupleStream implements Expressible { protected String zkHost; protected String collection; protected SolrParams params; - protected transient CloudSolrClient cloudSolrClient; protected transient TupleStream tupleStream; protected transient StreamContext streamContext; diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java index fb7696c8ac9..507dc3820fd 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java @@ -30,7 +30,6 @@ import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.ClusterStateProvider; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -69,9 +68,10 @@ public class StatsStream extends TupleStream implements Expressible, ParallelMet private String zkHost; private SolrParams params; private String collection; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; - private StreamContext context; + + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + private transient StreamContext context; protected transient TupleStream parallelizedStream; public StatsStream(String zkHost, String collection, SolrParams params, Metric[] metrics) @@ -216,7 +216,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { this.context = context; - cache = context.getSolrClientCache(); + this.clientCache = context.getSolrClientCache(); } @Override @@ -226,6 +226,12 @@ public List children() { @Override public void open() throws IOException { + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); + } else { + doCloseCache = false; + } @SuppressWarnings({"unchecked"}) Map> shardsMap = (Map>) context.get("shards"); @@ -233,7 +239,7 @@ public void open() throws IOException { // Parallelize the stats stream across multiple collections for an alias using plist if possible if (shardsMap == null && params.getBool(TIERED_PARAM, defaultTieredEnabled)) { ClusterStateProvider clusterStateProvider = - cache.getCloudSolrClient(zkHost).getClusterStateProvider(); + clientCache.getCloudSolrClient(zkHost).getClusterStateProvider(); final List resolved = clusterStateProvider != null ? clusterStateProvider.resolveAlias(collection) : null; if (resolved != null && resolved.size() > 1) { @@ -255,7 +261,7 @@ public void open() throws IOException { if (shardsMap == null) { QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST); - cloudSolrClient = cache.getCloudSolrClient(zkHost); + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); try { NamedList response = cloudSolrClient.request(request, collection); getTuples(response, metrics); @@ -264,7 +270,7 @@ public void open() throws IOException { } } else { List shards = shardsMap.get(collection); - SolrClient client = cache.getHttpSolrClient(shards.get(0)); + SolrClient client = clientCache.getHttpSolrClient(shards.get(0)); if (shards.size() > 1) { String shardsParam = getShardString(shards); @@ -294,7 +300,11 @@ private String getShardString(List shards) { } @Override - public void close() throws IOException {} + public void close() throws IOException { + if (doCloseCache) { + clientCache.close(); + } + } @Override public Tuple read() throws IOException { diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StreamContext.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StreamContext.java index 5feb15e46fa..2301b3d2136 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StreamContext.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/StreamContext.java @@ -16,7 +16,6 @@ */ package org.apache.solr.client.solrj.io.stream; -import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentMap; @@ -33,7 +32,7 @@ *

Note: The StreamContext contains the SolrClientCache which is used to cache SolrClients for * reuse across multiple TupleStreams. */ -public class StreamContext implements Serializable { +public class StreamContext { private Map entries = new HashMap<>(); private Map tupleContext = new HashMap<>(); diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TextLogitStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TextLogitStream.java index 1f746ef35f8..e3ff627232a 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TextLogitStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TextLogitStream.java @@ -37,7 +37,6 @@ import java.util.concurrent.Future; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.ClassificationEvaluation; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; @@ -82,12 +81,11 @@ public class TextLogitStream extends TupleStream implements Expressible { protected List idfs; protected ClassificationEvaluation evaluation; - protected transient SolrClientCache cache; - protected transient boolean isCloseCache; - protected transient CloudSolrClient cloudSolrClient; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; protected transient StreamContext streamContext; - protected ExecutorService executorService; + protected transient ExecutorService executorService; protected TupleStream termsStream; private List terms; @@ -355,7 +353,7 @@ private void init( @Override public void setStreamContext(StreamContext context) { - this.cache = context.getSolrClientCache(); + this.clientCache = context.getSolrClientCache(); this.streamContext = context; this.termsStream.setStreamContext(context); } @@ -363,14 +361,13 @@ public void setStreamContext(StreamContext context) { /** Opens the CloudSolrStream */ @Override public void open() throws IOException { - if (cache == null) { - isCloseCache = true; - cache = new SolrClientCache(); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - isCloseCache = false; + doCloseCache = false; } - this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost); this.executorService = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrNamedThreadFactory("TextLogitSolrStream")); } @@ -384,6 +381,7 @@ public List children() { protected List getShardUrls() throws IOException { try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); Slice[] slices = CloudSolrStream.getSlices(this.collection, cloudSolrClient, false); Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); @@ -426,7 +424,10 @@ private List> callShards(List baseUrls) throws IOException this.outcome, this.positiveLabel, this.learningRate, - this.iteration); + this.iteration, + this.threshold, + this.idfs, + this.clientCache); Future future = executorService.submit(lc); futures.add(future); @@ -437,8 +438,8 @@ private List> callShards(List baseUrls) throws IOException @Override public void close() throws IOException { - if (isCloseCache && cache != null) { - cache.close(); + if (doCloseCache) { + clientCache.close(); } if (executorService != null) { @@ -642,7 +643,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { } } - protected class LogitCall implements Callable { + protected static class LogitCall implements Callable { private String baseUrl; private String feature; @@ -653,6 +654,9 @@ protected class LogitCall implements Callable { private int positiveLabel; private double learningRate; private Map paramsMap; + private double threshold; + private List idfs; + private SolrClientCache clientCache; public LogitCall( String baseUrl, @@ -663,7 +667,10 @@ public LogitCall( String outcome, int positiveLabel, double learningRate, - int iteration) { + int iteration, + double threshold, + List idfs, + SolrClientCache clientCache) { this.baseUrl = baseUrl; this.feature = feature; @@ -674,12 +681,15 @@ public LogitCall( this.positiveLabel = positiveLabel; this.learningRate = learningRate; this.paramsMap = paramsMap; + this.threshold = threshold; + this.idfs = idfs; + this.clientCache = clientCache; } @Override public Tuple call() throws Exception { ModifiableSolrParams params = new ModifiableSolrParams(); - SolrClient solrClient = cache.getHttpSolrClient(baseUrl); + SolrClient solrClient = clientCache.getHttpSolrClient(baseUrl); params.add(DISTRIB, "false"); params.add("fq", "{!tlogit}"); diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java index 683cc35119c..383389d2551 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java @@ -25,11 +25,8 @@ import java.util.List; import java.util.Locale; import java.util.Map.Entry; -import java.util.Optional; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -71,8 +68,8 @@ public class TimeSeriesStream extends TupleStream implements Expressible { private String zkHost; private SolrParams params; private String collection; - protected transient SolrClientCache cache; - protected transient CloudSolrClient cloudSolrClient; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; public TimeSeriesStream( String zkHost, @@ -338,7 +335,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { - cache = context.getSolrClientCache(); + clientCache = context.getSolrClientCache(); } @Override @@ -348,12 +345,11 @@ public List children() { @Override public void open() throws IOException { - if (cache != null) { - cloudSolrClient = cache.getCloudSolrClient(zkHost); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - cloudSolrClient = new CloudLegacySolrClient.Builder(hosts, Optional.empty()).build(); + doCloseCache = false; } String json = getJsonFacetString(field, metrics, start, end, gap); @@ -364,6 +360,7 @@ public void open() throws IOException { QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST); try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); NamedList response = cloudSolrClient.request(request, collection); getTuples(response, field, metrics); } catch (Exception e) { @@ -373,8 +370,8 @@ public void open() throws IOException { @Override public void close() throws IOException { - if (cache == null) { - cloudSolrClient.close(); + if (doCloseCache) { + clientCache.close(); } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java index 54dc364fcf3..e343d56e35c 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/TopicStream.java @@ -32,7 +32,6 @@ import java.util.Locale; import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; import java.util.Random; import java.util.Set; import java.util.TreeSet; @@ -40,7 +39,7 @@ import java.util.concurrent.Future; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; +import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.ComparatorOrder; import org.apache.solr.client.solrj.io.comp.FieldComparator; @@ -79,6 +78,9 @@ public class TopicStream extends CloudSolrStream implements Expressible { private String checkpointCollection; private long initialCheckpoint = -1; + private transient SolrClientCache clientCache; + private transient boolean doCloseCache; + public TopicStream( String zkHost, String checkpointCollection, @@ -310,6 +312,12 @@ public List children() { @Override public void open() throws IOException { + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); + } else { + doCloseCache = false; + } this.tuples = new TreeSet<>(); this.solrStreams = new ArrayList<>(); this.eofTuples = Collections.synchronizedMap(new HashMap<>()); @@ -318,16 +326,6 @@ public void open() throws IOException { // Each worker must maintain its own checkpoints this.id = this.id + "_" + streamContext.workerID; } - - if (streamContext.getSolrClientCache() != null) { - cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost); - } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - cloudSolrClient = new CloudLegacySolrClient.Builder(hosts, Optional.empty()).build(); - this.cloudSolrClient.connect(); - } - if (checkpoints.size() == 0) { getPersistedCheckpoints(); if (checkpoints.size() == 0) { @@ -384,8 +382,8 @@ public void close() throws IOException { } } - if (streamContext != null && streamContext.getSolrClientCache() == null) { - cloudSolrClient.close(); + if (doCloseCache) { + clientCache.close(); } } } @@ -426,6 +424,7 @@ public int getCost() { } private void getCheckpoints() throws IOException { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); this.checkpoints = new HashMap<>(); Slice[] slices = CloudSolrStream.getSlices(this.collection, cloudSolrClient, false); Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); @@ -483,9 +482,11 @@ private long getCheckpoint(Slice slice, Set liveNodes) throws IOExceptio private void persistCheckpoints() throws IOException { - if (cloudSolrClient == null) { + if (clientCache == null) { return; } + + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); UpdateRequest request = new UpdateRequest(); request.setParam("collection", checkpointCollection); SolrInputDocument doc = new SolrInputDocument(); @@ -504,6 +505,7 @@ private void persistCheckpoints() throws IOException { } private void getPersistedCheckpoints() throws IOException { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, cloudSolrClient, false); Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); @@ -537,47 +539,50 @@ private void getPersistedCheckpoints() throws IOException { @Override protected void constructStreams() throws IOException { - try { - Slice[] slices = CloudSolrStream.getSlices(this.collection, cloudSolrClient, false); + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); + Slice[] slices = CloudSolrStream.getSlices(this.collection, cloudSolrClient, false); - ModifiableSolrParams mParams = new ModifiableSolrParams(params); - mParams.set(DISTRIB, "false"); // We are the aggregator. - String fl = mParams.get("fl"); - mParams.set(SORT, "_version_ asc"); - if (!fl.contains(VERSION_FIELD)) { - fl += ",_version_"; - } - mParams.set("fl", fl); + ModifiableSolrParams mParams = new ModifiableSolrParams(params); + mParams.set(DISTRIB, "false"); // We are the aggregator. + String fl = mParams.get("fl"); + mParams.set(SORT, "_version_ asc"); + if (!fl.contains(VERSION_FIELD)) { + fl += ",_version_"; + } + mParams.set("fl", fl); - Random random = new Random(); + Random random = new Random(); - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); + Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - for (Slice slice : slices) { - ModifiableSolrParams localParams = new ModifiableSolrParams(mParams); - long checkpoint = checkpoints.get(slice.getName()); + for (Slice slice : slices) { + ModifiableSolrParams localParams = new ModifiableSolrParams(mParams); + long checkpoint = checkpoints.get(slice.getName()); - Collection replicas = slice.getReplicas(); - List shuffler = new ArrayList<>(); - for (Replica replica : replicas) { - if (replica.getState() == Replica.State.ACTIVE - && liveNodes.contains(replica.getNodeName())) shuffler.add(replica); - } + Collection replicas = slice.getReplicas(); + List shuffler = new ArrayList<>(); + for (Replica replica : replicas) { + if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) + shuffler.add(replica); + } - Replica rep = shuffler.get(random.nextInt(shuffler.size())); - ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); - String url = zkProps.getCoreUrl(); - SolrStream solrStream = new SolrStream(url, localParams); - solrStream.setSlice(slice.getName()); - solrStream.setCheckpoint(checkpoint); - solrStream.setTrace(true); - if (streamContext != null) { - solrStream.setStreamContext(streamContext); - } - solrStreams.add(solrStream); + Replica rep = shuffler.get(random.nextInt(shuffler.size())); + ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); + String url = zkProps.getCoreUrl(); + SolrStream solrStream = new SolrStream(url, localParams); + solrStream.setSlice(slice.getName()); + solrStream.setCheckpoint(checkpoint); + solrStream.setTrace(true); + if (streamContext != null) { + solrStream.setStreamContext(streamContext); } - } catch (Exception e) { - throw new IOException(e); + solrStreams.add(solrStream); } } + + @Override + public void setStreamContext(StreamContext context) { + super.setStreamContext(context); + this.clientCache = context.getSolrClientCache(); + } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java index 03188e24005..adf1640f671 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/UpdateStream.java @@ -21,10 +21,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Optional; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -52,7 +49,7 @@ public class UpdateStream extends TupleStream implements Expressible { // field name in summary tuple for #docs updated in batch public static String BATCH_INDEXED_FIELD_NAME = "batchIndexed"; private String collection; - private String zkHost; + protected String zkHost; private int updateBatchSize; /** * Indicates if the {@link CommonParams#VERSION_FIELD} should be removed from tuples when @@ -64,11 +61,12 @@ public class UpdateStream extends TupleStream implements Expressible { private int batchNumber; private long totalDocsIndex; private PushBackStream tupleSource; - private transient SolrClientCache cache; - private transient CloudSolrClient cloudSolrClient; private List documentBatch = new ArrayList<>(); private String coreName; + protected transient SolrClientCache clientCache; + private transient boolean doCloseCache; + public UpdateStream(StreamExpression expression, StreamFactory factory) throws IOException { String collectionName = factory.getValueOperand(expression, 0); verifyCollectionName(collectionName, expression); @@ -122,7 +120,12 @@ protected String getCollectionName() { @Override public void open() throws IOException { - setCloudSolrClient(); + if (clientCache == null) { + doCloseCache = true; + clientCache = new SolrClientCache(); + } else { + doCloseCache = false; + } tupleSource.open(); } @@ -153,8 +156,8 @@ public Tuple read() throws IOException { @Override public void close() throws IOException { - if (cache == null && cloudSolrClient != null) { - cloudSolrClient.close(); + if (doCloseCache) { + clientCache.close(); } tupleSource.close(); } @@ -226,7 +229,7 @@ public Explanation toExplanation(StreamFactory factory) throws IOException { @Override public void setStreamContext(StreamContext context) { - this.cache = context.getSolrClientCache(); + this.clientCache = context.getSolrClientCache(); this.coreName = (String) context.get("core"); this.tupleSource.setStreamContext(context); } @@ -314,23 +317,6 @@ protected boolean defaultPruneVersionField() { return true; } - /** Only viable after calling {@link #open} */ - protected CloudSolrClient getCloudSolrClient() { - assert null != this.cloudSolrClient; - return this.cloudSolrClient; - } - - private void setCloudSolrClient() { - if (this.cache != null) { - this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost); - } else { - final List hosts = new ArrayList<>(); - hosts.add(zkHost); - this.cloudSolrClient = new CloudLegacySolrClient.Builder(hosts, Optional.empty()).build(); - this.cloudSolrClient.connect(); - } - } - private SolrInputDocument convertTupleToSolrDocument(Tuple tuple) { SolrInputDocument doc = new SolrInputDocument(); for (String field : tuple.getFields().keySet()) { @@ -365,6 +351,7 @@ protected void uploadBatchToCollection(List documentBatch) th } try { + var cloudSolrClient = clientCache.getCloudSolrClient(zkHost); cloudSolrClient.add(collection, documentBatch); } catch (SolrServerException | IOException e) { // TODO: it would be nice if there was an option to "skipFailedBatches" diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java index 6311acf1b29..260d9a059c4 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java @@ -342,6 +342,14 @@ public Builder withCollectionCacheTtl(long timeToLive, TimeUnit unit) { return this; } + /** + * Set the internal http client. + * + *

Note: closing the httpClient instance is at the responsibility of the caller. + * + * @param httpClient http client + * @return this + */ public Builder withHttpClient(Http2SolrClient httpClient) { if (this.internalClientBuilder != null) { throw new IllegalStateException( diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java index 25513a74dd4..63fc96f2b77 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java @@ -170,11 +170,7 @@ protected Http2SolrClient(String serverBaseUrl, Builder builder) { this.serverBaseUrl = null; } - if (builder.idleTimeoutMillis != null && builder.idleTimeoutMillis > 0) { - this.idleTimeoutMillis = builder.idleTimeoutMillis; - } else { - this.idleTimeoutMillis = HttpClientUtil.DEFAULT_SO_TIMEOUT; - } + this.idleTimeoutMillis = builder.idleTimeoutMillis; if (builder.httpClient != null) { this.httpClient = builder.httpClient; @@ -283,8 +279,7 @@ private HttpClient createHttpClient(Builder builder) { this.authenticationStore = new AuthenticationStoreHolder(); httpClient.setAuthenticationStore(this.authenticationStore); - if (builder.connectionTimeoutMillis != null) - httpClient.setConnectTimeout(builder.connectionTimeoutMillis); + httpClient.setConnectTimeout(builder.connectionTimeoutMillis); setupProxy(builder, httpClient); @@ -859,6 +854,7 @@ private NamedList processErrorsAndResponse( // no processor specified, return raw stream NamedList rsp = new NamedList<>(); rsp.add("stream", is); + rsp.add("responseStatus", httpStatus); // Only case where stream should not be closed shouldClose = false; return rsp; @@ -1060,6 +1056,13 @@ public Builder(String baseSolrUrl) { } public Http2SolrClient build() { + if (idleTimeoutMillis == null || idleTimeoutMillis <= 0) { + idleTimeoutMillis = (long) HttpClientUtil.DEFAULT_SO_TIMEOUT; + } + if (connectionTimeoutMillis == null) { + connectionTimeoutMillis = (long) HttpClientUtil.DEFAULT_CONNECT_TIMEOUT; + } + Http2SolrClient client = new Http2SolrClient(baseSolrUrl, this); try { httpClientBuilderSetup(client); @@ -1222,6 +1225,10 @@ public Builder withIdleTimeout(long idleConnectionTimeout, TimeUnit unit) { return this; } + public Long getIdleTimeoutMillis() { + return idleTimeoutMillis; + } + public Builder useHttp1_1(boolean useHttp1_1) { this.useHttp1_1 = useHttp1_1; return this; @@ -1241,6 +1248,10 @@ public Builder withConnectionTimeout(long connectionTimeout, TimeUnit unit) { return this; } + public Long getConnectionTimeout() { + return connectionTimeoutMillis; + } + /** * Set a timeout in milliseconds for requests issued by this client. * diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index 5f7b8a17ee8..c03fbc9f3f9 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -643,6 +643,7 @@ protected NamedList executeMethod( NamedList rsp = new NamedList<>(); rsp.add("stream", respBody); rsp.add("closeableResponse", response); + rsp.add("responseStatus", response.getStatusLine().getStatusCode()); // Only case where stream should not be closed shouldClose = false; return rsp; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java index 645a19040de..d0c97f8ed22 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java @@ -127,6 +127,10 @@ public B withConnectionTimeout(int connectionTimeout, TimeUnit unit) { return getThis(); } + public int getConnectionTimeoutMillis() { + return this.connectionTimeoutMillis; + } + /** * Tells {@link Builder} that created clients should set the following read timeout on all * sockets. @@ -155,4 +159,8 @@ public B withSocketTimeout(int socketTimeout, TimeUnit unit) { socketTimeoutMillisUpdate = true; return getThis(); } + + public int getSocketTimeoutMillis() { + return this.socketTimeoutMillis; + } } From 579e24bc54d4ea19510afa364a063fe6cea57fa9 Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Tue, 22 Aug 2023 15:46:19 +0200 Subject: [PATCH 022/160] Update software.amazon.awssdk:* to v2.20.128 (#1820) --- solr/licenses/annotations-2.20.128.jar.sha1 | 1 + solr/licenses/annotations-2.20.97.jar.sha1 | 1 - solr/licenses/apache-client-2.20.128.jar.sha1 | 1 + solr/licenses/apache-client-2.20.97.jar.sha1 | 1 - solr/licenses/arns-2.20.128.jar.sha1 | 1 + solr/licenses/arns-2.20.97.jar.sha1 | 1 - solr/licenses/auth-2.20.128.jar.sha1 | 1 + solr/licenses/auth-2.20.97.jar.sha1 | 1 - solr/licenses/aws-core-2.20.128.jar.sha1 | 1 + solr/licenses/aws-core-2.20.97.jar.sha1 | 1 - .../aws-query-protocol-2.20.128.jar.sha1 | 1 + .../aws-query-protocol-2.20.97.jar.sha1 | 1 - .../aws-xml-protocol-2.20.128.jar.sha1 | 1 + .../aws-xml-protocol-2.20.97.jar.sha1 | 1 - solr/licenses/crt-core-2.20.128.jar.sha1 | 1 + solr/licenses/crt-core-2.20.97.jar.sha1 | 1 - solr/licenses/endpoints-spi-2.20.128.jar.sha1 | 1 + solr/licenses/endpoints-spi-2.20.97.jar.sha1 | 1 - .../http-client-spi-2.20.128.jar.sha1 | 1 + .../licenses/http-client-spi-2.20.97.jar.sha1 | 1 - solr/licenses/json-utils-2.20.128.jar.sha1 | 1 + solr/licenses/json-utils-2.20.97.jar.sha1 | 1 - solr/licenses/metrics-spi-2.20.128.jar.sha1 | 1 + solr/licenses/metrics-spi-2.20.97.jar.sha1 | 1 - solr/licenses/profiles-2.20.128.jar.sha1 | 1 + solr/licenses/profiles-2.20.97.jar.sha1 | 1 - solr/licenses/protocol-core-2.20.128.jar.sha1 | 1 + solr/licenses/protocol-core-2.20.97.jar.sha1 | 1 - solr/licenses/regions-2.20.128.jar.sha1 | 1 + solr/licenses/regions-2.20.97.jar.sha1 | 1 - solr/licenses/s3-2.20.128.jar.sha1 | 1 + solr/licenses/s3-2.20.97.jar.sha1 | 1 - solr/licenses/sdk-core-2.20.128.jar.sha1 | 1 + solr/licenses/sdk-core-2.20.97.jar.sha1 | 1 - solr/licenses/sts-2.20.128.jar.sha1 | 1 + solr/licenses/sts-2.20.97.jar.sha1 | 1 - ...third-party-jackson-core-2.20.128.jar.sha1 | 1 + .../third-party-jackson-core-2.20.97.jar.sha1 | 1 - .../url-connection-client-2.20.128.jar.sha1 | 1 + .../url-connection-client-2.20.97.jar.sha1 | 1 - solr/licenses/utils-2.20.128.jar.sha1 | 1 + solr/licenses/utils-2.20.97.jar.sha1 | 1 - versions.lock | 44 +++++++++---------- versions.props | 2 +- 44 files changed, 44 insertions(+), 44 deletions(-) create mode 100644 solr/licenses/annotations-2.20.128.jar.sha1 delete mode 100644 solr/licenses/annotations-2.20.97.jar.sha1 create mode 100644 solr/licenses/apache-client-2.20.128.jar.sha1 delete mode 100644 solr/licenses/apache-client-2.20.97.jar.sha1 create mode 100644 solr/licenses/arns-2.20.128.jar.sha1 delete mode 100644 solr/licenses/arns-2.20.97.jar.sha1 create mode 100644 solr/licenses/auth-2.20.128.jar.sha1 delete mode 100644 solr/licenses/auth-2.20.97.jar.sha1 create mode 100644 solr/licenses/aws-core-2.20.128.jar.sha1 delete mode 100644 solr/licenses/aws-core-2.20.97.jar.sha1 create mode 100644 solr/licenses/aws-query-protocol-2.20.128.jar.sha1 delete mode 100644 solr/licenses/aws-query-protocol-2.20.97.jar.sha1 create mode 100644 solr/licenses/aws-xml-protocol-2.20.128.jar.sha1 delete mode 100644 solr/licenses/aws-xml-protocol-2.20.97.jar.sha1 create mode 100644 solr/licenses/crt-core-2.20.128.jar.sha1 delete mode 100644 solr/licenses/crt-core-2.20.97.jar.sha1 create mode 100644 solr/licenses/endpoints-spi-2.20.128.jar.sha1 delete mode 100644 solr/licenses/endpoints-spi-2.20.97.jar.sha1 create mode 100644 solr/licenses/http-client-spi-2.20.128.jar.sha1 delete mode 100644 solr/licenses/http-client-spi-2.20.97.jar.sha1 create mode 100644 solr/licenses/json-utils-2.20.128.jar.sha1 delete mode 100644 solr/licenses/json-utils-2.20.97.jar.sha1 create mode 100644 solr/licenses/metrics-spi-2.20.128.jar.sha1 delete mode 100644 solr/licenses/metrics-spi-2.20.97.jar.sha1 create mode 100644 solr/licenses/profiles-2.20.128.jar.sha1 delete mode 100644 solr/licenses/profiles-2.20.97.jar.sha1 create mode 100644 solr/licenses/protocol-core-2.20.128.jar.sha1 delete mode 100644 solr/licenses/protocol-core-2.20.97.jar.sha1 create mode 100644 solr/licenses/regions-2.20.128.jar.sha1 delete mode 100644 solr/licenses/regions-2.20.97.jar.sha1 create mode 100644 solr/licenses/s3-2.20.128.jar.sha1 delete mode 100644 solr/licenses/s3-2.20.97.jar.sha1 create mode 100644 solr/licenses/sdk-core-2.20.128.jar.sha1 delete mode 100644 solr/licenses/sdk-core-2.20.97.jar.sha1 create mode 100644 solr/licenses/sts-2.20.128.jar.sha1 delete mode 100644 solr/licenses/sts-2.20.97.jar.sha1 create mode 100644 solr/licenses/third-party-jackson-core-2.20.128.jar.sha1 delete mode 100644 solr/licenses/third-party-jackson-core-2.20.97.jar.sha1 create mode 100644 solr/licenses/url-connection-client-2.20.128.jar.sha1 delete mode 100644 solr/licenses/url-connection-client-2.20.97.jar.sha1 create mode 100644 solr/licenses/utils-2.20.128.jar.sha1 delete mode 100644 solr/licenses/utils-2.20.97.jar.sha1 diff --git a/solr/licenses/annotations-2.20.128.jar.sha1 b/solr/licenses/annotations-2.20.128.jar.sha1 new file mode 100644 index 00000000000..f669d174c70 --- /dev/null +++ b/solr/licenses/annotations-2.20.128.jar.sha1 @@ -0,0 +1 @@ +4eba3312fad5b1b621fde4cfbf1b9780591589d5 diff --git a/solr/licenses/annotations-2.20.97.jar.sha1 b/solr/licenses/annotations-2.20.97.jar.sha1 deleted file mode 100644 index 074e5ee3d44..00000000000 --- a/solr/licenses/annotations-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -41037256f5c45c951ce61e96370b1e59ed571e17 diff --git a/solr/licenses/apache-client-2.20.128.jar.sha1 b/solr/licenses/apache-client-2.20.128.jar.sha1 new file mode 100644 index 00000000000..0ef9ff55ec0 --- /dev/null +++ b/solr/licenses/apache-client-2.20.128.jar.sha1 @@ -0,0 +1 @@ +5b82f09fd66fbd3c0b27ff88f7eb2e9a78e9a080 diff --git a/solr/licenses/apache-client-2.20.97.jar.sha1 b/solr/licenses/apache-client-2.20.97.jar.sha1 deleted file mode 100644 index e96404f979c..00000000000 --- a/solr/licenses/apache-client-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -366e2a5a9824eb750ea77ea1c7581055dafd6dac diff --git a/solr/licenses/arns-2.20.128.jar.sha1 b/solr/licenses/arns-2.20.128.jar.sha1 new file mode 100644 index 00000000000..395694a27e7 --- /dev/null +++ b/solr/licenses/arns-2.20.128.jar.sha1 @@ -0,0 +1 @@ +5e809b3b87e6f2806e0b67a390d3fe07fcaf7c5e diff --git a/solr/licenses/arns-2.20.97.jar.sha1 b/solr/licenses/arns-2.20.97.jar.sha1 deleted file mode 100644 index aa4c743d9b7..00000000000 --- a/solr/licenses/arns-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3e187cb36e2ba023823f839cb6a1e2eb175b02e6 diff --git a/solr/licenses/auth-2.20.128.jar.sha1 b/solr/licenses/auth-2.20.128.jar.sha1 new file mode 100644 index 00000000000..a7481505d04 --- /dev/null +++ b/solr/licenses/auth-2.20.128.jar.sha1 @@ -0,0 +1 @@ +a27499eb27835bb21f0a8f6d6df9b7dcd2725529 diff --git a/solr/licenses/auth-2.20.97.jar.sha1 b/solr/licenses/auth-2.20.97.jar.sha1 deleted file mode 100644 index 97b18333ff8..00000000000 --- a/solr/licenses/auth-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e04a9881737fdaf84e1d11e27de64e78eb2d0f25 diff --git a/solr/licenses/aws-core-2.20.128.jar.sha1 b/solr/licenses/aws-core-2.20.128.jar.sha1 new file mode 100644 index 00000000000..7d0e6e344c4 --- /dev/null +++ b/solr/licenses/aws-core-2.20.128.jar.sha1 @@ -0,0 +1 @@ +ca2c9d71fe86609a6e69c7b770cb282f365ca054 diff --git a/solr/licenses/aws-core-2.20.97.jar.sha1 b/solr/licenses/aws-core-2.20.97.jar.sha1 deleted file mode 100644 index 8459bca4d97..00000000000 --- a/solr/licenses/aws-core-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -378334b0a5f20cce0cca4d85f5855de1e0b072cf diff --git a/solr/licenses/aws-query-protocol-2.20.128.jar.sha1 b/solr/licenses/aws-query-protocol-2.20.128.jar.sha1 new file mode 100644 index 00000000000..6c657e007e0 --- /dev/null +++ b/solr/licenses/aws-query-protocol-2.20.128.jar.sha1 @@ -0,0 +1 @@ +ab36db110c976df57e6bf81b95a24b7736ddac43 diff --git a/solr/licenses/aws-query-protocol-2.20.97.jar.sha1 b/solr/licenses/aws-query-protocol-2.20.97.jar.sha1 deleted file mode 100644 index de87d5585fb..00000000000 --- a/solr/licenses/aws-query-protocol-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0fef3e8eadb282ff3b4de1ee1fdc9dd6624fd975 diff --git a/solr/licenses/aws-xml-protocol-2.20.128.jar.sha1 b/solr/licenses/aws-xml-protocol-2.20.128.jar.sha1 new file mode 100644 index 00000000000..4c0615d9097 --- /dev/null +++ b/solr/licenses/aws-xml-protocol-2.20.128.jar.sha1 @@ -0,0 +1 @@ +ca6a04c0b1d5b492612e5b8816aedfc5895733cf diff --git a/solr/licenses/aws-xml-protocol-2.20.97.jar.sha1 b/solr/licenses/aws-xml-protocol-2.20.97.jar.sha1 deleted file mode 100644 index e68e59378aa..00000000000 --- a/solr/licenses/aws-xml-protocol-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d441f5ee009122b7294923933492aede55acaad0 diff --git a/solr/licenses/crt-core-2.20.128.jar.sha1 b/solr/licenses/crt-core-2.20.128.jar.sha1 new file mode 100644 index 00000000000..8fd66294823 --- /dev/null +++ b/solr/licenses/crt-core-2.20.128.jar.sha1 @@ -0,0 +1 @@ +fa7c6f4f1037479c021d90d14ec58c55c72f7c4f diff --git a/solr/licenses/crt-core-2.20.97.jar.sha1 b/solr/licenses/crt-core-2.20.97.jar.sha1 deleted file mode 100644 index 37b54404abf..00000000000 --- a/solr/licenses/crt-core-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4cdd43cda3bbcf3e36162853cfa60acb5bf844e6 diff --git a/solr/licenses/endpoints-spi-2.20.128.jar.sha1 b/solr/licenses/endpoints-spi-2.20.128.jar.sha1 new file mode 100644 index 00000000000..04050c7defd --- /dev/null +++ b/solr/licenses/endpoints-spi-2.20.128.jar.sha1 @@ -0,0 +1 @@ +2761b9d660eb3db814d4078eb1e7ee53515effd0 diff --git a/solr/licenses/endpoints-spi-2.20.97.jar.sha1 b/solr/licenses/endpoints-spi-2.20.97.jar.sha1 deleted file mode 100644 index 5126f671412..00000000000 --- a/solr/licenses/endpoints-spi-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2a2763c7ae68ddaaee0513af4bfda4cab936033c diff --git a/solr/licenses/http-client-spi-2.20.128.jar.sha1 b/solr/licenses/http-client-spi-2.20.128.jar.sha1 new file mode 100644 index 00000000000..b7986560ff0 --- /dev/null +++ b/solr/licenses/http-client-spi-2.20.128.jar.sha1 @@ -0,0 +1 @@ +d614e5750362fabb312ccb4f2db440d22ed95c72 diff --git a/solr/licenses/http-client-spi-2.20.97.jar.sha1 b/solr/licenses/http-client-spi-2.20.97.jar.sha1 deleted file mode 100644 index bbd40cf57cb..00000000000 --- a/solr/licenses/http-client-spi-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2835a2c7c10225d563ade9c15bce3bff5809b7b8 diff --git a/solr/licenses/json-utils-2.20.128.jar.sha1 b/solr/licenses/json-utils-2.20.128.jar.sha1 new file mode 100644 index 00000000000..3eb1f4fc2e3 --- /dev/null +++ b/solr/licenses/json-utils-2.20.128.jar.sha1 @@ -0,0 +1 @@ +0f00a80b03dde32f1d7aca472516b6e45e4a8959 diff --git a/solr/licenses/json-utils-2.20.97.jar.sha1 b/solr/licenses/json-utils-2.20.97.jar.sha1 deleted file mode 100644 index db43070cb42..00000000000 --- a/solr/licenses/json-utils-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f3cdc0572c1db2ac78c5feede45a5e266ec13084 diff --git a/solr/licenses/metrics-spi-2.20.128.jar.sha1 b/solr/licenses/metrics-spi-2.20.128.jar.sha1 new file mode 100644 index 00000000000..83880b2a032 --- /dev/null +++ b/solr/licenses/metrics-spi-2.20.128.jar.sha1 @@ -0,0 +1 @@ +d59e5e2264e03fe27cd1ab1843ccc2984636bc3c diff --git a/solr/licenses/metrics-spi-2.20.97.jar.sha1 b/solr/licenses/metrics-spi-2.20.97.jar.sha1 deleted file mode 100644 index 820a98fc66a..00000000000 --- a/solr/licenses/metrics-spi-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3845268eaaf2269cb83fb0b420ecfaec0736639a diff --git a/solr/licenses/profiles-2.20.128.jar.sha1 b/solr/licenses/profiles-2.20.128.jar.sha1 new file mode 100644 index 00000000000..1d15205ac83 --- /dev/null +++ b/solr/licenses/profiles-2.20.128.jar.sha1 @@ -0,0 +1 @@ +cbc0b7b6e98b9e168b96ef92c41f3d274b3d403b diff --git a/solr/licenses/profiles-2.20.97.jar.sha1 b/solr/licenses/profiles-2.20.97.jar.sha1 deleted file mode 100644 index 21bcd31d075..00000000000 --- a/solr/licenses/profiles-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a26bb7ce18958e71040c3bf9cf6371ab022771c6 diff --git a/solr/licenses/protocol-core-2.20.128.jar.sha1 b/solr/licenses/protocol-core-2.20.128.jar.sha1 new file mode 100644 index 00000000000..42e13ee9ab0 --- /dev/null +++ b/solr/licenses/protocol-core-2.20.128.jar.sha1 @@ -0,0 +1 @@ +a79cb2c8d2d4399ccbc1f432d23d2b6413c09396 diff --git a/solr/licenses/protocol-core-2.20.97.jar.sha1 b/solr/licenses/protocol-core-2.20.97.jar.sha1 deleted file mode 100644 index ef95d4ef40a..00000000000 --- a/solr/licenses/protocol-core-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0df4613072a99b366974e3f67be0aa955624655e diff --git a/solr/licenses/regions-2.20.128.jar.sha1 b/solr/licenses/regions-2.20.128.jar.sha1 new file mode 100644 index 00000000000..ea8ed851217 --- /dev/null +++ b/solr/licenses/regions-2.20.128.jar.sha1 @@ -0,0 +1 @@ +f9cdaff8fc49e3b3473fb741d70f6458bdd1d108 diff --git a/solr/licenses/regions-2.20.97.jar.sha1 b/solr/licenses/regions-2.20.97.jar.sha1 deleted file mode 100644 index 54e30673925..00000000000 --- a/solr/licenses/regions-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8d5e815ff83afb34666277cf1c40a1a77ee88763 diff --git a/solr/licenses/s3-2.20.128.jar.sha1 b/solr/licenses/s3-2.20.128.jar.sha1 new file mode 100644 index 00000000000..c233b710661 --- /dev/null +++ b/solr/licenses/s3-2.20.128.jar.sha1 @@ -0,0 +1 @@ +0eae09026984386b3b657c76952f1c33980cc60b diff --git a/solr/licenses/s3-2.20.97.jar.sha1 b/solr/licenses/s3-2.20.97.jar.sha1 deleted file mode 100644 index 89367f80a09..00000000000 --- a/solr/licenses/s3-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7a9f0859a8a22b4a0a77bd929746bde6d222884d diff --git a/solr/licenses/sdk-core-2.20.128.jar.sha1 b/solr/licenses/sdk-core-2.20.128.jar.sha1 new file mode 100644 index 00000000000..a7feb66351b --- /dev/null +++ b/solr/licenses/sdk-core-2.20.128.jar.sha1 @@ -0,0 +1 @@ +4d2681298b8364b10e8e448c7cfb38a0c57ea137 diff --git a/solr/licenses/sdk-core-2.20.97.jar.sha1 b/solr/licenses/sdk-core-2.20.97.jar.sha1 deleted file mode 100644 index 0a9f23dd634..00000000000 --- a/solr/licenses/sdk-core-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d2a5ab879bd75d84a582212bf04fc06474064810 diff --git a/solr/licenses/sts-2.20.128.jar.sha1 b/solr/licenses/sts-2.20.128.jar.sha1 new file mode 100644 index 00000000000..e42ac3530a8 --- /dev/null +++ b/solr/licenses/sts-2.20.128.jar.sha1 @@ -0,0 +1 @@ +5091ddfe486a878fbca7f7068e4d781109129d47 diff --git a/solr/licenses/sts-2.20.97.jar.sha1 b/solr/licenses/sts-2.20.97.jar.sha1 deleted file mode 100644 index 8675ad9c4f8..00000000000 --- a/solr/licenses/sts-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -717eb0636175d4274758390b306f7344974ddfc6 diff --git a/solr/licenses/third-party-jackson-core-2.20.128.jar.sha1 b/solr/licenses/third-party-jackson-core-2.20.128.jar.sha1 new file mode 100644 index 00000000000..886d413afca --- /dev/null +++ b/solr/licenses/third-party-jackson-core-2.20.128.jar.sha1 @@ -0,0 +1 @@ +9714093f164eac041f3f3af506a32ad61fc26e3f diff --git a/solr/licenses/third-party-jackson-core-2.20.97.jar.sha1 b/solr/licenses/third-party-jackson-core-2.20.97.jar.sha1 deleted file mode 100644 index 7af03a3d0a4..00000000000 --- a/solr/licenses/third-party-jackson-core-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b5d4cb0064b567d65f82d867ea209b2638674d34 diff --git a/solr/licenses/url-connection-client-2.20.128.jar.sha1 b/solr/licenses/url-connection-client-2.20.128.jar.sha1 new file mode 100644 index 00000000000..89ad177e745 --- /dev/null +++ b/solr/licenses/url-connection-client-2.20.128.jar.sha1 @@ -0,0 +1 @@ +f6fb7d2001efce42580c16b4ce4e55ebe556ab0b diff --git a/solr/licenses/url-connection-client-2.20.97.jar.sha1 b/solr/licenses/url-connection-client-2.20.97.jar.sha1 deleted file mode 100644 index 11f2b2b3775..00000000000 --- a/solr/licenses/url-connection-client-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -37955fd70059ebbc23673d6e66e2e0c015ea94d7 diff --git a/solr/licenses/utils-2.20.128.jar.sha1 b/solr/licenses/utils-2.20.128.jar.sha1 new file mode 100644 index 00000000000..70c3f94382d --- /dev/null +++ b/solr/licenses/utils-2.20.128.jar.sha1 @@ -0,0 +1 @@ +a63a7b5cefd69dc24fddc200d31d2f6ec613b513 diff --git a/solr/licenses/utils-2.20.97.jar.sha1 b/solr/licenses/utils-2.20.97.jar.sha1 deleted file mode 100644 index eddd78f9dec..00000000000 --- a/solr/licenses/utils-2.20.97.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a84bf8c7dbf73700740f2f0d8e4babb7bbbdf0b diff --git a/versions.lock b/versions.lock index adb9555b9d5..25d922b05a8 100644 --- a/versions.lock +++ b/versions.lock @@ -354,27 +354,27 @@ org.tallison.xmp:xmpcore-shaded:6.1.10 (1 constraints: 300e8d49) org.threeten:threetenbp:1.6.8 (4 constraints: 2433e267) org.tukaani:xz:1.9 (1 constraints: 030c5be9) org.xerial.snappy:snappy-java:1.1.10.1 (4 constraints: b1385afd) -software.amazon.awssdk:annotations:2.20.97 (20 constraints: 252b9ae9) -software.amazon.awssdk:apache-client:2.20.97 (4 constraints: 112ab2e3) -software.amazon.awssdk:arns:2.20.97 (2 constraints: 231864c1) -software.amazon.awssdk:auth:2.20.97 (5 constraints: 51382e41) -software.amazon.awssdk:aws-core:2.20.97 (6 constraints: 044ef720) -software.amazon.awssdk:aws-query-protocol:2.20.97 (3 constraints: 5e2af9dd) -software.amazon.awssdk:aws-xml-protocol:2.20.97 (2 constraints: 231864c1) -software.amazon.awssdk:bom:2.20.97 (1 constraints: 7605b240) -software.amazon.awssdk:crt-core:2.20.97 (1 constraints: c60b97f9) -software.amazon.awssdk:endpoints-spi:2.20.97 (4 constraints: b63406cd) -software.amazon.awssdk:http-client-spi:2.20.97 (11 constraints: e6a5bb82) -software.amazon.awssdk:json-utils:2.20.97 (5 constraints: 833f7e3f) -software.amazon.awssdk:metrics-spi:2.20.97 (7 constraints: 4462e915) -software.amazon.awssdk:profiles:2.20.97 (8 constraints: 6f61ca49) -software.amazon.awssdk:protocol-core:2.20.97 (5 constraints: 8f48165b) -software.amazon.awssdk:regions:2.20.97 (7 constraints: 495063af) -software.amazon.awssdk:s3:2.20.97 (3 constraints: ba240128) -software.amazon.awssdk:sdk-core:2.20.97 (10 constraints: 9287b9ec) -software.amazon.awssdk:sts:2.20.97 (2 constraints: d3114515) -software.amazon.awssdk:third-party-jackson-core:2.20.97 (2 constraints: 951b26a8) -software.amazon.awssdk:utils:2.20.97 (19 constraints: 901873a5) +software.amazon.awssdk:annotations:2.20.128 (20 constraints: 812e7fc3) +software.amazon.awssdk:apache-client:2.20.128 (4 constraints: bd2a0169) +software.amazon.awssdk:arns:2.20.128 (2 constraints: 791848ec) +software.amazon.awssdk:auth:2.20.128 (5 constraints: 28393d1f) +software.amazon.awssdk:aws-core:2.20.128 (6 constraints: 064f9593) +software.amazon.awssdk:aws-query-protocol:2.20.128 (3 constraints: df2a3b4b) +software.amazon.awssdk:aws-xml-protocol:2.20.128 (2 constraints: 791848ec) +software.amazon.awssdk:bom:2.20.128 (1 constraints: a1053e46) +software.amazon.awssdk:crt-core:2.20.128 (1 constraints: f10b8205) +software.amazon.awssdk:endpoints-spi:2.20.128 (4 constraints: 6235767a) +software.amazon.awssdk:http-client-spi:2.20.128 (11 constraints: bfa7e005) +software.amazon.awssdk:json-utils:2.20.128 (5 constraints: 5a407940) +software.amazon.awssdk:metrics-spi:2.20.128 (7 constraints: 7163ee3b) +software.amazon.awssdk:profiles:2.20.128 (8 constraints: c76252a3) +software.amazon.awssdk:protocol-core:2.20.128 (5 constraints: 66496c88) +software.amazon.awssdk:regions:2.20.128 (7 constraints: 4b518731) +software.amazon.awssdk:s3:2.20.128 (3 constraints: 1025d161) +software.amazon.awssdk:sdk-core:2.20.128 (10 constraints: 40897300) +software.amazon.awssdk:sts:2.20.128 (2 constraints: 29121c33) +software.amazon.awssdk:third-party-jackson-core:2.20.128 (2 constraints: eb1bd4d7) +software.amazon.awssdk:utils:2.20.128 (19 constraints: 961bfbe5) software.amazon.eventstream:eventstream:1.0.1 (2 constraints: 2e1ae62b) ua.net.nlp:morfologik-ukrainian-search:4.9.1 (1 constraints: d5126e1e) xerces:xercesImpl:2.12.2 (1 constraints: 8e0c7d0e) @@ -457,5 +457,5 @@ org.springframework.boot:spring-boot-starter-json:2.7.12 (1 constraints: d814669 org.springframework.boot:spring-boot-starter-logging:2.7.12 (1 constraints: 6d138b46) org.springframework.boot:spring-boot-starter-web:2.7.12 (1 constraints: f20a38d6) org.yaml:snakeyaml:1.30 (1 constraints: 0713d91f) -software.amazon.awssdk:url-connection-client:2.20.97 (2 constraints: 451f8bf6) +software.amazon.awssdk:url-connection-client:2.20.128 (2 constraints: 701ff515) software.amazon.ion:ion-java:1.0.2 (1 constraints: 720db831) diff --git a/versions.props b/versions.props index 17af2f9c60c..3e61cbe6326 100644 --- a/versions.props +++ b/versions.props @@ -69,4 +69,4 @@ org.quicktheories:quicktheories=0.26 org.semver4j:semver4j=4.3.0 org.slf4j:*=2.0.7 org.xerial.snappy:snappy-java=1.1.10.1 -software.amazon.awssdk:*=2.20.97 +software.amazon.awssdk:*=2.20.128 From c7b58302d334795e1c820f2a28903b64bce91e59 Mon Sep 17 00:00:00 2001 From: Alex D Date: Tue, 22 Aug 2023 09:27:21 -0700 Subject: [PATCH 023/160] SOLR-16929 SolrStream propagates undecoded error message (#1852) --- solr/CHANGES.txt | 2 + .../io/stream/JavabinTupleStreamParser.java | 24 ++++++++++- .../client/solrj/io/stream/SolrStream.java | 3 +- .../client/solrj/io/stream/StreamingTest.java | 40 ++++++++++++++++--- 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e09b1849394..66e12ec7384 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -130,6 +130,8 @@ Bug Fixes * SOLR-16859: Missing Proxy support for Http2SolrClient (Alex Deparvu) +* SOLR-16929: SolrStream propagates undecoded error message (Alex Deparvu) + Dependency Upgrades --------------------- diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/JavabinTupleStreamParser.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/JavabinTupleStreamParser.java index 751d833168f..5209d29e4be 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/JavabinTupleStreamParser.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/JavabinTupleStreamParser.java @@ -24,6 +24,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.solr.common.util.DataInputInputStream; import org.apache.solr.common.util.FastInputStream; import org.apache.solr.common.util.JavaBinCodec; @@ -34,12 +35,12 @@ public class JavabinTupleStreamParser extends JavaBinCodec implements TupleStrea private int arraySize = Integer.MAX_VALUE; private boolean onlyJsonTypes = false; int objectSize; + private boolean atDocs; public JavabinTupleStreamParser(InputStream is, boolean onlyJsonTypes) throws IOException { this.onlyJsonTypes = onlyJsonTypes; this.is = is; this.fis = initRead(is); - if (!readTillDocs()) arraySize = 0; } private boolean readTillDocs() throws IOException { @@ -61,6 +62,9 @@ private boolean readTillDocs() throws IOException { return true; } return false; + } else if ("error".equals(k)) { + handleError(); + return true; } else { if (readTillDocs()) return true; } @@ -91,7 +95,7 @@ private boolean isObjectType(DataInputInputStream dis) throws IOException { return tagByte == SOLRDOCLST; } - private Map readAsMap(DataInputInputStream dis) throws IOException { + private Map readAsMap(DataInputInputStream dis) throws IOException { int sz = readSize(dis); Map m = new LinkedHashMap<>(); for (int i = 0; i < sz; i++) { @@ -175,6 +179,14 @@ protected Object readObject(DataInputInputStream dis) throws IOException { @Override @SuppressWarnings({"unchecked"}) public Map next() throws IOException { + if (!atDocs) { + atDocs = true; + if (!readTillDocs()) { + arraySize = 0; + return null; + } + } + if (arraySize == 0) return null; Object o = readVal(fis); arraySize--; @@ -186,4 +198,12 @@ public Map next() throws IOException { public void close() throws IOException { is.close(); } + + private void handleError() throws IOException { + tagByte = fis.readByte(); + var error = readAsMap(fis); + var msg = + Optional.ofNullable(error.get("msg")).map(String::valueOf).orElse("Unknown Exception"); + throw new SolrStream.HandledException(msg); + } } diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java index f58546cb432..ce4d31babaa 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SolrStream.java @@ -315,7 +315,8 @@ private TupleStreamParser constructParser(SolrParams requestParams) statusCode = (int) genericResponse.get("responseStatus"); } - if (statusCode != 200) { + if (statusCode == 401 + || statusCode == 403) { // auth response comes as html, so propagate as string String errMsg = consumeStreamAsErrorMessage(stream); if (httpResponse != null) { httpResponse.close(); diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java index 295fca76f9f..d8770711876 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java @@ -627,7 +627,6 @@ public void testParallelReducerStream() throws Exception { } @Test - @Ignore public void testExceptionStream() throws Exception { helloDocsUpdateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS); @@ -635,8 +634,8 @@ public void testExceptionStream() throws Exception { StreamContext streamContext = new StreamContext(); SolrClientCache solrClientCache = new SolrClientCache(); streamContext.setSolrClientCache(solrClientCache); - // Test an error that comes originates from the /select handler try { + // Test an error that originates from the /select handler SolrParams sParamsA = params("q", "*:*", "fl", "a_s,a_i,a_f,blah", "sort", "blah asc"); CloudSolrStream stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA); ExceptionStream estream = new ExceptionStream(stream); @@ -646,16 +645,45 @@ public void testExceptionStream() throws Exception { assertTrue(t.EXCEPTION); assertTrue(t.getException().contains("sort param field can't be found: blah")); - // Test an error that comes originates from the /export handler - sParamsA = params("q", "*:*", "fl", "a_s,a_i,a_f,score", "sort", "a_s asc", "qt", "/export"); + sParamsA = params("q", "*:*", "fl", "a_s,a_i,a_f,blah", "sort", "blah asc", "wt", "javabin"); stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA); estream = new ExceptionStream(stream); estream.setStreamContext(streamContext); t = getTuple(estream); assertTrue(t.EOF); assertTrue(t.EXCEPTION); - // The /export handler will pass through a real exception. - assertTrue(t.getException().contains("undefined field:")); + assertTrue(t.getException().contains("sort param field can't be found: blah")); + + // Test an error that originates from the /export handler + sParamsA = params("q", "*:*", "fl", "a_s,a_i,a_f,blah", "sort", "blah asc", "qt", "/export"); + stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA); + estream = new ExceptionStream(stream); + estream.setStreamContext(streamContext); + t = getTuple(estream); + assertTrue(t.EOF); + assertTrue(t.EXCEPTION); + assertTrue(t.getException().contains("sort param field can't be found: blah")); + + // Test an error that originates from the /export handler + sParamsA = + params( + "q", + "*:*", + "fl", + "a_s,a_i,a_f,blah", + "sort", + "blah asc", + "qt", + "/export", + "wt", + "javabin"); + stream = new CloudSolrStream(zkHost, COLLECTIONORALIAS, sParamsA); + estream = new ExceptionStream(stream); + estream.setStreamContext(streamContext); + t = getTuple(estream); + assertTrue(t.EOF); + assertTrue(t.EXCEPTION); + assertTrue(t.getException().contains("sort param field can't be found: blah")); } finally { solrClientCache.close(); } From 59f70172167795e7e2bdb2da66b1f45c6fc8250c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 22 Aug 2023 20:04:55 +0200 Subject: [PATCH 024/160] SOLR-16944 V2 API /api/node/health should be governed by "health" permission, not "config-read" (#1858) --- solr/CHANGES.txt | 2 ++ .../java/org/apache/solr/handler/admin/api/NodeHealthAPI.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 66e12ec7384..14bd442a9c1 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -128,6 +128,8 @@ Bug Fixes * PR#1826: Allow looking up Solr Package repo when that URL references a raw repository.json hosted on Github when the file is JSON but the mimetype used is text/plain. (Eric Pugh) +* SOLR-16944: V2 API /api/node/health should be governed by "health" permission, not "config-read" (janhoy) + * SOLR-16859: Missing Proxy support for Http2SolrClient (Alex Deparvu) * SOLR-16929: SolrStream propagates undecoded error message (Alex Deparvu) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java index db5e97039b2..df5f64900f0 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/NodeHealthAPI.java @@ -18,7 +18,7 @@ package org.apache.solr.handler.admin.api; import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET; -import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM; +import static org.apache.solr.security.PermissionNameProvider.Name.HEALTH_PERM; import org.apache.solr.api.EndPoint; import org.apache.solr.handler.admin.HealthCheckHandler; @@ -41,7 +41,7 @@ public NodeHealthAPI(HealthCheckHandler handler) { @EndPoint( path = {"/node/health"}, method = GET, - permission = CONFIG_READ_PERM) + permission = HEALTH_PERM) public void getSystemInformation(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { handler.handleRequestBody(req, rsp); } From 0e4e169c0ed083a1d082f26d2b66986169243358 Mon Sep 17 00:00:00 2001 From: Mikhail Khludnev Date: Tue, 22 Aug 2023 22:36:44 +0300 Subject: [PATCH 025/160] Update v2-api.adoc: remove mention of /api/c (#1790) --- .../modules/configuration-guide/pages/v2-api.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/v2-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/v2-api.adoc index a3a9e980d06..79882093a6d 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/v2-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/v2-api.adoc @@ -39,7 +39,7 @@ Following are some v2 API URL paths and path prefixes, along with some of the op [width="100%",options="header",] |=== |Path prefix |Some Supported Operations -|`/api/collections` or equivalently: `/api/c` |Create, alias, backup, and restore a collection. +|`/api/collections` |Create, alias, backup, and restore a collection. |`/api/c/_collection-name_/update` |Update requests. |`/api/c/_collection-name_/config` |Configuration requests. |`/api/c/_collection-name_/schema` |Schema requests. From 6e508802ab8ede9809b9e76a507b064d8af7ae76 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 22 Aug 2023 15:57:01 -0400 Subject: [PATCH 026/160] SOLR-16934: Give securityManager permission for client TLS (#1857) --- solr/CHANGES.txt | 2 ++ solr/packaging/test/test_ssl.bats | 58 +++++++++++++++++++++++++++++++ solr/server/etc/security.policy | 10 +++--- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 14bd442a9c1..f3f715df647 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -134,6 +134,8 @@ Bug Fixes * SOLR-16929: SolrStream propagates undecoded error message (Alex Deparvu) +* SOLR-16934: Allow Solr to read client (javax.net.ssl.*) trustStores and keyStores via SecurityManager. (Houston Putman) + Dependency Upgrades --------------------- diff --git a/solr/packaging/test/test_ssl.bats b/solr/packaging/test/test_ssl.bats index 9f3fdfe806c..a40d3232a13 100644 --- a/solr/packaging/test/test_ssl.bats +++ b/solr/packaging/test/test_ssl.bats @@ -97,3 +97,61 @@ teardown() { run curl --http2 --cacert "$ssl_dir/solr-ssl.pem" 'https://localhost:8983/solr/test/select?q=*:*' assert_output --partial '401 require authentication' } + +@test "start solr with client truststore and security manager" { + # Make a test tmp dir, as the security policy includes TMP, so that might already contain the BATS_TEST_TMPDIR + test_tmp_dir="${BATS_TEST_TMPDIR}/tmp" + mkdir -p "${test_tmp_dir}" + test_tmp_dir="$(cd -P "${test_tmp_dir}" && pwd)" + + export SOLR_SECURITY_MANAGER_ENABLED=true + export SOLR_OPTS="-Djava.io.tmpdir=${test_tmp_dir}" + export SOLR_TOOL_OPTS="-Djava.io.tmpdir=${test_tmp_dir}" + + # Create a keystore + export ssl_dir="${BATS_TEST_TMPDIR}/ssl" + export client_ssl_dir="${ssl_dir}-client" + mkdir -p "$ssl_dir" + ( + cd "$ssl_dir" + rm -f solr-ssl.keystore.p12 + keytool -genkeypair -alias solr-ssl -keyalg RSA -keysize 2048 -keypass secret -storepass secret -validity 9999 -keystore solr-ssl.keystore.p12 -storetype PKCS12 -ext SAN=DNS:localhost,IP:127.0.0.1 -dname "CN=localhost, OU=Organizational Unit, O=Organization, L=Location, ST=State, C=Country" + ) + mkdir -p "$client_ssl_dir" + ( + cd "$client_ssl_dir" + rm -f * + keytool -export -alias solr-ssl -file solr-ssl.crt -keystore "$ssl_dir/solr-ssl.keystore.p12" -keypass secret -storepass secret + keytool -import -v -trustcacerts -alias solr-ssl -file solr-ssl.crt -storetype PKCS12 -keystore solr-ssl.truststore.p12 -keypass secret -storepass secret -noprompt + ) + cp -R "$ssl_dir" "$client_ssl_dir" + + # Set ENV_VARs so that Solr uses this keystore + export SOLR_SSL_ENABLED=true + export SOLR_SSL_KEY_STORE=$ssl_dir/solr-ssl.keystore.p12 + export SOLR_SSL_KEY_STORE_PASSWORD=secret + export SOLR_SSL_TRUST_STORE=$ssl_dir/solr-ssl.keystore.p12 + export SOLR_SSL_TRUST_STORE_PASSWORD=secret + export SOLR_SSL_CLIENT_TRUST_STORE=$client_ssl_dir/solr-ssl.truststore.p12 + export SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD=secret + export SOLR_SSL_NEED_CLIENT_AUTH=false + export SOLR_SSL_WANT_CLIENT_AUTH=true + export SOLR_SSL_CHECK_PEER_NAME=true + export SOLR_HOST=localhost + export SOLR_SECURITY_MANAGER_ENABLED=true + + run solr start -c + + export SOLR_SSL_KEY_STORE= + export SOLR_SSL_KEY_STORE_PASSWORD= + export SOLR_SSL_TRUST_STORE= + export SOLR_SSL_TRUST_STORE_PASSWORD= + + solr assert --started https://localhost:8983/solr --timeout 5000 + + run solr create -c test -s 2 + assert_output --partial "Created collection 'test'" + + run solr api -get 'https://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS' + assert_output --partial '"urlScheme":"https"' +} diff --git a/solr/server/etc/security.policy b/solr/server/etc/security.policy index 77ac99704c5..aec2e2ddcfe 100644 --- a/solr/server/etc/security.policy +++ b/solr/server/etc/security.policy @@ -185,11 +185,13 @@ grant { permission java.io.FilePermission "${hadoop.security.credential.provider.path}", "read,write,delete,readlink"; permission java.io.FilePermission "${hadoop.security.credential.provider.path}${/}-", "read,write,delete,readlink"; - permission java.io.FilePermission "${solr.jetty.keystore}", "read,write,delete,readlink"; - permission java.io.FilePermission "${solr.jetty.keystore}${/}-", "read,write,delete,readlink"; + permission java.io.FilePermission "${solr.jetty.keystore}", "read,readlink"; - permission java.io.FilePermission "${solr.jetty.truststore}", "read,write,delete,readlink"; - permission java.io.FilePermission "${solr.jetty.truststore}${/}-", "read,write,delete,readlink"; + permission java.io.FilePermission "${solr.jetty.truststore}", "read,readlink"; + + permission java.io.FilePermission "${javax.net.ssl.keyStore}", "read,readlink"; + + permission java.io.FilePermission "${javax.net.ssl.trustStore}", "read,readlink"; permission java.io.FilePermission "${solr.install.dir}", "read,write,delete,readlink"; permission java.io.FilePermission "${solr.install.dir}${/}-", "read,write,delete,readlink"; From c79c8994b5e29f69bcb5db12fcd42d332e3bba37 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 22 Aug 2023 17:28:44 -0400 Subject: [PATCH 027/160] SOLR-16265: Fix NPE for req in Http2SolrClient (#1860) --- .../solr/client/solrj/impl/Http2SolrClient.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java index 63fc96f2b77..867b5edc85d 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java @@ -528,34 +528,35 @@ public NamedList request(SolrRequest solrRequest, String collection) InputStreamResponseListener listener = new InputStreamResponseListener(); req = makeRequestAndSend(solrRequest, url, listener, false); Response response = listener.get(idleTimeoutMillis, TimeUnit.MILLISECONDS); + url = req.getURI().toString(); InputStream is = listener.getInputStream(); assert ObjectReleaseTracker.track(is); - return processErrorsAndResponse(solrRequest, response, is, req.getURI().toString()); + return processErrorsAndResponse(solrRequest, response, is, url); } catch (InterruptedException e) { Thread.currentThread().interrupt(); abortCause = e; throw new RuntimeException(e); } catch (TimeoutException e) { throw new SolrServerException( - "Timeout occurred while waiting response from server at: " + req.getURI(), e); + "Timeout occurred while waiting response from server at: " + url, e); } catch (ExecutionException e) { Throwable cause = e.getCause(); abortCause = cause; if (cause instanceof ConnectException) { - throw new SolrServerException("Server refused connection at: " + req.getURI(), cause); + throw new SolrServerException("Server refused connection at: " + url, cause); } if (cause instanceof SolrServerException) { throw (SolrServerException) cause; } else if (cause instanceof IOException) { throw new SolrServerException( - "IOException occurred when talking to server at: " + req.getURI(), cause); + "IOException occurred when talking to server at: " + url, cause); } throw new SolrServerException(cause.getMessage(), cause); } catch (SolrServerException | RuntimeException sse) { abortCause = sse; throw sse; } finally { - if (abortCause != null) { + if (abortCause != null && req != null) { req.abort(abortCause); } } From 0619f7873cfcb2f19539d6b2a69225d76e17ca66 Mon Sep 17 00:00:00 2001 From: pjmcarthur <92818583+pjmcarthur@users.noreply.github.com> Date: Wed, 23 Aug 2023 01:18:13 -0700 Subject: [PATCH 028/160] SOLR-16946: Updated Cluster Singleton plugins are stopped correctly when the Overseer is closed (#1862) --- solr/CHANGES.txt | 2 ++ .../apache/solr/core/ClusterSingletons.java | 2 +- .../solr/handler/TestContainerPlugin.java | 18 +++++++++++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index f3f715df647..e5e59103585 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -136,6 +136,8 @@ Bug Fixes * SOLR-16934: Allow Solr to read client (javax.net.ssl.*) trustStores and keyStores via SecurityManager. (Houston Putman) +* SOLR-16946: Updated Cluster Singleton plugins are stopped correctly when the Overseer is closed. (Paul McArthur) + Dependency Upgrades --------------------- diff --git a/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java b/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java index 6e2c0417818..0c71c853400 100644 --- a/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java +++ b/solr/core/src/java/org/apache/solr/core/ClusterSingletons.java @@ -99,8 +99,8 @@ public void deleted(ContainerPluginsRegistry.ApiInfo plugin) { @Override public void modified( ContainerPluginsRegistry.ApiInfo old, ContainerPluginsRegistry.ApiInfo replacement) { - added(replacement); deleted(old); + added(replacement); } }; } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 9fdb0936e88..27daaab4d9a 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -220,8 +220,11 @@ public void testApi() throws Exception { MatcherAssert.assertThat(msg, containsString("Cannot find API for the path")); // test ClusterSingleton plugin + CConfig c6Cfg = new CConfig(); + c6Cfg.strVal = "added"; plugin.name = "clusterSingleton"; plugin.klass = C6.class.getName(); + plugin.config = c6Cfg; addPlugin.process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); @@ -233,6 +236,16 @@ public void testApi() throws Exception { assertTrue("startCalled", C6.startCalled); assertFalse("stopCalled", C6.stopCalled); + // update the clusterSingleton config + c6Cfg.strVal = "updated"; + postPlugin(singletonMap("update", plugin)).process(cluster.getSolrClient()); + version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); + + assertTrue("stopCalled", C6.stopCalled); + + // Clear stopCalled, it will be verified again when the Overseer Jetty is killed + C6.stopCalled = false; + assertEquals(CConfig.class, ContainerPluginsRegistry.getConfigClass(new CC())); assertEquals(CConfig.class, ContainerPluginsRegistry.getConfigClass(new CC1())); assertEquals(CConfig.class, ContainerPluginsRegistry.getConfigClass(new CC2())); @@ -394,7 +407,7 @@ public static class CConfig implements ReflectMapWriter { @JsonProperty public Boolean boolVal; } - public static class C6 implements ClusterSingleton { + public static class C6 implements ClusterSingleton, ConfigurablePlugin { static boolean startCalled = false; static boolean stopCalled = false; static boolean ccProvided = false; @@ -430,6 +443,9 @@ public void stop() { stopCalled = true; state = State.STOPPED; } + + @Override + public void configure(CConfig cfg) {} } public static class C5 implements ResourceLoaderAware { From 473cea28b55dcf03fb2338399af321f425c90002 Mon Sep 17 00:00:00 2001 From: Renato Haeberli Date: Wed, 23 Aug 2023 13:48:27 +0200 Subject: [PATCH 029/160] Cleaning up old code to prevent warnings (#1834) --------- Co-authored-by: Renato Haeberli <> Co-authored-by: Eric Pugh --- .../java/org/apache/solr/cli/AssertTool.java | 3 -- .../org/apache/solr/cli/SimplePostTool.java | 31 +++++++++---------- .../src/java/org/apache/solr/cli/SolrCLI.java | 11 ++----- .../java/org/apache/solr/cli/StatusTool.java | 5 +-- 4 files changed, 19 insertions(+), 31 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/AssertTool.java b/solr/core/src/java/org/apache/solr/cli/AssertTool.java index 9ece33ccb8d..44b4dce1250 100644 --- a/solr/core/src/java/org/apache/solr/cli/AssertTool.java +++ b/solr/core/src/java/org/apache/solr/cli/AssertTool.java @@ -30,7 +30,6 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.HealthCheckRequest; -import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -231,8 +230,6 @@ public static int assertSolrNotRunning(String url) throws Exception { NamedList response = solrClient.request(new HealthCheckRequest()); Integer statusCode = (Integer) response.findRecursive("responseHeader", "status"); SolrCLI.checkCodeForAuthError(statusCode); - } catch (SolrException se) { - throw se; } catch (IOException | SolrServerException e) { log.debug("Opening connection to {} failed, Solr does not seem to be running", url, e); return 0; diff --git a/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java b/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java index d15650e1636..6def73ad789 100644 --- a/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SimplePostTool.java @@ -519,14 +519,7 @@ public int postFiles(String[] args, int startIndexInArgs, OutputStream out, Stri int filesPosted = 0; for (int j = startIndexInArgs; j < args.length; j++) { File srcFile = new File(args[j]); - boolean isValidPath = checkIsValidPath(srcFile); - if (isValidPath && srcFile.isDirectory() && srcFile.canRead()) { - filesPosted += postDirectory(srcFile, out, type); - } else if (isValidPath && srcFile.isFile() && srcFile.canRead()) { - filesPosted += postFiles(new File[] {srcFile}, out, type); - } else { - filesPosted += handleGlob(srcFile, out, type); - } + filesPosted = getFilesPosted(out, type, srcFile); } return filesPosted; } @@ -544,14 +537,20 @@ public int postFiles(File[] files, int startIndexInArgs, OutputStream out, Strin reset(); int filesPosted = 0; for (File srcFile : files) { - boolean isValidPath = checkIsValidPath(srcFile); - if (isValidPath && srcFile.isDirectory() && srcFile.canRead()) { - filesPosted += postDirectory(srcFile, out, type); - } else if (isValidPath && srcFile.isFile() && srcFile.canRead()) { - filesPosted += postFiles(new File[] {srcFile}, out, type); - } else { - filesPosted += handleGlob(srcFile, out, type); - } + filesPosted = getFilesPosted(out, type, srcFile); + } + return filesPosted; + } + + private int getFilesPosted(final OutputStream out, final String type, final File srcFile) { + int filesPosted = 0; + boolean isValidPath = checkIsValidPath(srcFile); + if (isValidPath && srcFile.isDirectory() && srcFile.canRead()) { + filesPosted += postDirectory(srcFile, out, type); + } else if (isValidPath && srcFile.isFile() && srcFile.canRead()) { + filesPosted += postFiles(new File[] {srcFile}, out, type); + } else { + filesPosted += handleGlob(srcFile, out, type); } return filesPosted; } diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index 0459892eeea..d7a424f6215 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -27,7 +27,6 @@ import java.io.File; import java.io.IOException; import java.lang.invoke.MethodHandles; -import java.net.ConnectException; import java.net.SocketException; import java.net.URL; import java.util.ArrayList; @@ -378,11 +377,7 @@ private static Set findClasses(String path, String packageName) throws E */ public static boolean checkCommunicationError(Exception exc) { Throwable rootCause = SolrException.getRootCause(exc); - boolean wasCommError = - (rootCause instanceof ConnectException - || rootCause instanceof SolrServerException - || rootCause instanceof SocketException); - return wasCommError; + return (rootCause instanceof SolrServerException || rootCause instanceof SocketException); } public static void checkCodeForAuthError(int code) { @@ -600,8 +595,8 @@ public static boolean safeCheckCoreExists(String solrUrl, String coreName) { } NamedList existsCheckResult = CoreAdminRequest.getStatus(coreName, solrClient).getResponse(); - NamedList status = (NamedList) existsCheckResult.get("status"); - NamedList coreStatus = (NamedList) status.get(coreName); + NamedList status = (NamedList) existsCheckResult.get("status"); + NamedList coreStatus = (NamedList) status.get(coreName); Map failureStatus = (Map) existsCheckResult.get("initFailures"); String errorMsg = (String) failureStatus.get(coreName); diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java b/solr/core/src/java/org/apache/solr/cli/StatusTool.java index 546dc742f3d..3758a60b3fe 100644 --- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java +++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; @@ -130,8 +129,6 @@ public Map waitToSeeSolrUp(String solrUrl, long maxWait, TimeUni while (System.nanoTime() < timeout) { try { return getStatus(solrUrl); - } catch (SSLPeerUnverifiedException exc) { - throw exc; } catch (Exception exc) { if (SolrCLI.exceptionIsAuthRelated(exc)) { throw exc; @@ -204,7 +201,7 @@ protected Map getCloudStatus(SolrClient solrClient, String zkHos cloudStatus.put("liveNodes", String.valueOf(liveNodes.size())); Map collections = - ((NamedList) json.findRecursive("cluster", "collections")).asMap(); + ((NamedList) json.findRecursive("cluster", "collections")).asMap(); cloudStatus.put("collections", String.valueOf(collections.size())); return cloudStatus; From f5b0d22f06363a06db25368f70e0d429a0e3b192 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Wed, 23 Aug 2023 15:40:27 -0400 Subject: [PATCH 030/160] SOLR-16933: Include full query response in API tool (#1863) --- solr/CHANGES.txt | 2 + .../src/java/org/apache/solr/cli/ApiTool.java | 53 ++++++++------- .../java/org/apache/solr/cli/CreateTool.java | 16 ++--- .../java/org/apache/solr/cli/DeleteTool.java | 16 ++--- .../test/org/apache/solr/cli/ApiToolTest.java | 64 ++++++++++++++++++- .../client/solrj/impl/NodeValueFetcher.java | 2 +- .../impl/SolrClientNodeStateProvider.java | 5 +- .../apache/solr/common/IteratorWriter.java | 2 +- .../apache/solr/common/SolrDocumentList.java | 21 +++++- .../org/apache/solr/common/util/Utils.java | 10 ++- .../solrj/src/java/org/noggit/JSONWriter.java | 21 +++++- 11 files changed, 160 insertions(+), 52 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e5e59103585..009f9bc2d44 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -138,6 +138,8 @@ Bug Fixes * SOLR-16946: Updated Cluster Singleton plugins are stopped correctly when the Overseer is closed. (Paul McArthur) +* SOLR-16933: Include the full query response when using the API tool, and fix serialization issues for SolrDocumentList. (Houston Putman) + Dependency Upgrades --------------------- diff --git a/solr/core/src/java/org/apache/solr/cli/ApiTool.java b/solr/core/src/java/org/apache/solr/cli/ApiTool.java index 6122000211b..82980ea5257 100644 --- a/solr/core/src/java/org/apache/solr/cli/ApiTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ApiTool.java @@ -23,11 +23,10 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.NoOpResponseParser; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; -import org.noggit.CharArr; -import org.noggit.JSONWriter; /** * Supports api command in the bin/solr script. @@ -61,29 +60,39 @@ public List