From 7a92bbfcb18153157c346c623b326fe8da7079e0 Mon Sep 17 00:00:00 2001 From: James Dyer Date: Mon, 11 Dec 2023 08:02:59 -0600 Subject: [PATCH 01/21] SOLR-15484 JWTAuthPluginIntegrationTest - dynamically generate the self-signed certificate using the localhost loopback address (#2099) SOLR-15484: dynamically generate the self-signed certificate using the localhost loopback address --- .../randomization/policies/solr-tests.policy | 4 + solr/modules/jwt-auth/build.gradle | 2 + .../solr/security/jwt/JWTIssuerConfig.java | 17 ++- .../jwt/JWTAuthPluginIntegrationTest.java | 17 ++- .../solr/security/jwt/KeystoreGenerator.java | 116 ++++++++++++++++++ 5 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/KeystoreGenerator.java diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy index 276532b519d..86871e72613 100644 --- a/gradle/testing/randomization/policies/solr-tests.policy +++ b/gradle/testing/randomization/policies/solr-tests.policy @@ -162,6 +162,10 @@ grant { permission javax.security.auth.AuthPermission "createLoginContext.Server"; permission javax.security.auth.AuthPermission "createLoginContext.Client"; + // Needed by BouncyCastle in jwt-auth tests + permission java.security.SecurityPermission "putProviderProperty.BC"; + permission java.security.SecurityPermission "getProperty.org.bouncycastle.x509.allow_non-der_tbscert"; + // may only be necessary with Java 7? permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read"; permission javax.security.auth.PrivateCredentialPermission "sun.security.jgss.krb5.Krb5Util$KeysFromKeyTab * \"*\"", "read"; diff --git a/solr/modules/jwt-auth/build.gradle b/solr/modules/jwt-auth/build.gradle index 1b420899e33..17886099e18 100644 --- a/solr/modules/jwt-auth/build.gradle +++ b/solr/modules/jwt-auth/build.gradle @@ -61,6 +61,8 @@ dependencies { testImplementation 'com.fasterxml.jackson.core:jackson-databind' permitTestUnusedDeclared 'com.fasterxml.jackson.core:jackson-databind' + testImplementation 'org.bouncycastle:bcpkix-jdk15on' + testImplementation 'org.bouncycastle:bcprov-jdk15on' testImplementation 'com.nimbusds:nimbus-jose-jwt' testImplementation 'com.squareup.okhttp3:mockwebserver' testImplementation 'com.squareup.okhttp3:okhttp' diff --git a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java index 947d040da8b..b7e650c1401 100644 --- a/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java +++ b/solr/modules/jwt-auth/src/java/org/apache/solr/security/jwt/JWTIssuerConfig.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; @@ -423,6 +424,14 @@ public boolean isValid() { return jwkConfigured > 0; } + private static void disableHostVerificationIfLocalhost(URL url, Get httpGet) { + InetAddress loopbackAddress = InetAddress.getLoopbackAddress(); + if (loopbackAddress.getCanonicalHostName().equals(url.getHost()) + || loopbackAddress.getHostName().equals(url.getHost())) { + httpGet.setHostnameVerifier((hostname, session) -> true); + } + } + public void setTrustedCerts(Collection trustedCerts) { this.trustedCerts = trustedCerts; } @@ -472,9 +481,7 @@ private HttpsJwks create(String url) { if (trustedCerts != null) { Get getWithCustomTrust = new Get(); getWithCustomTrust.setTrustedCertificates(trustedCerts); - if ("localhost".equals(jwksUrl.getHost())) { - getWithCustomTrust.setHostnameVerifier((hostname, session) -> true); - } + disableHostVerificationIfLocalhost(jwksUrl, getWithCustomTrust); httpsJkws.setSimpleHttpGet(getWithCustomTrust); } return httpsJkws; @@ -525,9 +532,7 @@ public static WellKnownDiscoveryConfig parse( Get httpGet = new Get(); if (trustedCerts != null) { httpGet.setTrustedCertificates(trustedCerts); - if ("localhost".equals(url.getHost())) { - httpGet.setHostnameVerifier((hostname, session) -> true); - } + disableHostVerificationIfLocalhost(url, httpGet); } SimpleResponse resp = httpGet.get(url.toString()); return parse(new ByteArrayInputStream(resp.getBody().getBytes(StandardCharsets.UTF_8))); diff --git a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java index c2613d8550c..ef7fe35d4f5 100644 --- a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java +++ b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/JWTAuthPluginIntegrationTest.java @@ -25,6 +25,7 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.net.HttpURLConnection; +import java.net.InetAddress; import java.net.URL; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -102,6 +103,14 @@ public static void beforeClass() throws Exception { pemFilePath = JWT_TEST_PATH().resolve("security").resolve("jwt_plugin_idp_cert.pem"); wrongPemFilePath = JWT_TEST_PATH().resolve("security").resolve("jwt_plugin_idp_wrongcert.pem"); + Path tempDir = Files.createTempDirectory(JWTAuthPluginIntegrationTest.class.getSimpleName()); + tempDir.toFile().deleteOnExit(); + Path modifiedP12Cert = tempDir.resolve(p12Cert.getFileName()); + new KeystoreGenerator() + .generateKeystore( + p12Cert, modifiedP12Cert, InetAddress.getLoopbackAddress().getCanonicalHostName()); + p12Cert = modifiedP12Cert; + mockOAuth2Server = createMockOAuthServer(p12Cert, "secret"); mockOAuth2Server.start(); mockOAuthToken = @@ -112,7 +121,7 @@ public static void beforeClass() throws Exception { } @AfterClass - public static void afterClass() throws Exception { + public static void afterClass() { if (mockOAuth2Server != null) { mockOAuth2Server.shutdown(); } @@ -485,11 +494,7 @@ private void executeCommand(String url, HttpClient cl, String payload, JsonWebSi * and trusting its SSL */ private static String createMockOAuthSecurityJson(Path pemFilePath) throws IOException { - String wellKnown = - mockOAuth2Server - .wellKnownUrl("default") - .toString() - .replace(".localdomain", ""); // Use only 'localhost' to match our SSL cert + String wellKnown = mockOAuth2Server.wellKnownUrl("default").toString(); String pemCert = CryptoKeys.extractCertificateFromPem(Files.readString(pemFilePath)) .replace("\\n", "\\\\n"); // Use literal \n to play well with JSON diff --git a/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/KeystoreGenerator.java b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/KeystoreGenerator.java new file mode 100644 index 00000000000..6d2d6c9c23a --- /dev/null +++ b/solr/modules/jwt-auth/src/test/org/apache/solr/security/jwt/KeystoreGenerator.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security.jwt; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.Provider; +import java.security.Security; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Date; +import org.bouncycastle.asn1.DERUTF8String; +import org.bouncycastle.asn1.x500.AttributeTypeAndValue; +import org.bouncycastle.asn1.x500.RDN; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x500.style.BCStyle; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.cert.CertIOException; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.OperatorCreationException; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; + +public class KeystoreGenerator { + + private static final String PASS_PHRASE = "secret"; + + public void generateKeystore(Path existingKeystore, Path newKeystore, String cn) { + KeyStore ks = null; + try (FileInputStream fis = new FileInputStream(existingKeystore.toFile())) { + ks = KeyStore.getInstance(KeyStore.getDefaultType()); + ks.load(fis, PASS_PHRASE.toCharArray()); + ks.setCertificateEntry(cn, selfSignCertificate(cn)); + + } catch (KeyStoreException | CertificateException | NoSuchAlgorithmException | IOException e) { + throw new RuntimeException(e); + } + try (OutputStream fos = new FileOutputStream(newKeystore.toFile())) { + ks.store(fos, PASS_PHRASE.toCharArray()); + } catch (CertificateException | KeyStoreException | IOException | NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + private X509Certificate selfSignCertificate(String commonName) { + try { + KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA"); + kpg.initialize(1024); + KeyPair kp = kpg.generateKeyPair(); + Provider bcp = new BouncyCastleProvider(); + Security.addProvider(bcp); + + X500Name cn = + new X500Name( + new RDN[] { + new RDN(new AttributeTypeAndValue(BCStyle.CN, new DERUTF8String(commonName))) + }); + X500Name issuer = + new X500Name( + new RDN[] { + new RDN(new AttributeTypeAndValue(BCStyle.CN, new DERUTF8String("Solr Root CA"))) + }); + + Instant yesterday = + LocalDate.now(ZoneOffset.UTC).minusDays(1).atStartOfDay().toInstant(ZoneOffset.UTC); + Instant oneYear = ZonedDateTime.ofInstant(yesterday, ZoneOffset.UTC).plusYears(1).toInstant(); + BigInteger serial = new BigInteger(String.valueOf(yesterday.toEpochMilli())); + + JcaX509v3CertificateBuilder certBuilder = + new JcaX509v3CertificateBuilder( + issuer, serial, Date.from(yesterday), Date.from(oneYear), cn, kp.getPublic()); + BasicConstraints basicConstraints = new BasicConstraints(true); + certBuilder.addExtension(Extension.basicConstraints, true, basicConstraints); + + ContentSigner cs = new JcaContentSignerBuilder("SHA256WithRSA").build(kp.getPrivate()); + return new JcaX509CertificateConverter() + .setProvider(bcp) + .getCertificate(certBuilder.build(cs)); + } catch (CertificateException + | CertIOException + | NoSuchAlgorithmException + | OperatorCreationException e) { + throw new RuntimeException(e); + } + } +} From 0bf936fe805fafe251333f521e53975f42ef11ca Mon Sep 17 00:00:00 2001 From: Solr Bot <125606113+solrbot@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:19:24 +0100 Subject: [PATCH 02/21] Update dependency org.bitbucket.b_c:jose4j to v0.9.4 (#2143) --- solr/licenses/jose4j-0.9.3.jar.sha1 | 1 - solr/licenses/jose4j-0.9.4.jar.sha1 | 1 + versions.lock | 4 ++-- versions.props | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 solr/licenses/jose4j-0.9.3.jar.sha1 create mode 100644 solr/licenses/jose4j-0.9.4.jar.sha1 diff --git a/solr/licenses/jose4j-0.9.3.jar.sha1 b/solr/licenses/jose4j-0.9.3.jar.sha1 deleted file mode 100644 index bae58301d21..00000000000 --- a/solr/licenses/jose4j-0.9.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9670e11587194cb6b1b2edcaa688a3fab85b4148 diff --git a/solr/licenses/jose4j-0.9.4.jar.sha1 b/solr/licenses/jose4j-0.9.4.jar.sha1 new file mode 100644 index 00000000000..837b7df47df --- /dev/null +++ b/solr/licenses/jose4j-0.9.4.jar.sha1 @@ -0,0 +1 @@ +7efe6ccf593e52a2b77f98de52238f15b4a67188 diff --git a/versions.lock b/versions.lock index aa992040b51..e95bc0f871e 100644 --- a/versions.lock +++ b/versions.lock @@ -258,7 +258,7 @@ org.apache.xmlbeans:xmlbeans:5.0.3 (2 constraints: 72173075) org.apache.zookeeper:zookeeper:3.9.1 (2 constraints: 9d13795f) org.apache.zookeeper:zookeeper-jute:3.9.1 (2 constraints: 9b128823) org.apiguardian:apiguardian-api:1.1.2 (2 constraints: 601bd5a8) -org.bitbucket.b_c:jose4j:0.9.3 (1 constraints: 0e050936) +org.bitbucket.b_c:jose4j:0.9.4 (1 constraints: 0f050a36) org.bouncycastle:bcmail-jdk15on:1.70 (1 constraints: 310c8af5) org.bouncycastle:bcpkix-jdk15on:1.70 (2 constraints: ce1b11b3) org.bouncycastle:bcprov-jdk15on:1.70 (4 constraints: 1f34ee12) @@ -345,7 +345,7 @@ org.reactivestreams:reactive-streams:1.0.4 (3 constraints: 3f2b77fd) org.semver4j:semver4j:5.2.2 (1 constraints: 0b050c36) org.slf4j:jcl-over-slf4j:2.0.9 (3 constraints: cf17cfa6) org.slf4j:jul-to-slf4j:2.0.9 (3 constraints: 29286349) -org.slf4j:slf4j-api:2.0.9 (59 constraints: dd104075) +org.slf4j:slf4j-api:2.0.9 (59 constraints: e310a790) org.tallison:isoparser:1.9.41.7 (1 constraints: fb0c5528) org.tallison:jmatio:1.5 (1 constraints: ff0b57e9) org.tallison:metadata-extractor:2.17.1.0 (1 constraints: f00c3b28) diff --git a/versions.props b/versions.props index f38263dd33b..1a2527610ab 100644 --- a/versions.props +++ b/versions.props @@ -51,7 +51,7 @@ org.apache.lucene:*=9.8.0 org.apache.tika:*=1.28.5 org.apache.tomcat:annotations-api=6.0.53 org.apache.zookeeper:*=3.9.1 -org.bitbucket.b_c:jose4j=0.9.3 +org.bitbucket.b_c:jose4j=0.9.4 org.carrot2:carrot2-core=4.5.1 org.codehaus.woodstox:stax2-api=4.2.2 org.eclipse.jetty*:*=10.0.18 From 686700b96edab4ee35199015297cd27c5ddca4a2 Mon Sep 17 00:00:00 2001 From: James Dyer Date: Tue, 12 Dec 2023 08:23:59 -0600 Subject: [PATCH 03/21] SOLR-15484: use latest version of bouncycastle libraries (#2145) --- gradle/testing/randomization/policies/solr-tests.policy | 1 + solr/licenses/bcpkix-jdk18on-1.77.jar.sha1 | 1 + solr/licenses/bcprov-jdk18on-1.77.jar.sha1 | 1 + ...il-jdk15on-LICENSE-BSD_LIKE.txt => bcutil-LICENSE-MIT.txt} | 0 solr/licenses/bcutil-jdk18on-1.77.jar.sha1 | 1 + .../{bcutil-jdk15on-NOTICE.txt => bcutil-jdk18on-NOTICE.txt} | 0 solr/modules/jwt-auth/build.gradle | 4 ++-- versions.lock | 3 +++ versions.props | 1 + 9 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 solr/licenses/bcpkix-jdk18on-1.77.jar.sha1 create mode 100644 solr/licenses/bcprov-jdk18on-1.77.jar.sha1 rename solr/licenses/{bcutil-jdk15on-LICENSE-BSD_LIKE.txt => bcutil-LICENSE-MIT.txt} (100%) create mode 100644 solr/licenses/bcutil-jdk18on-1.77.jar.sha1 rename solr/licenses/{bcutil-jdk15on-NOTICE.txt => bcutil-jdk18on-NOTICE.txt} (100%) diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy index 86871e72613..c4b07f8ac1a 100644 --- a/gradle/testing/randomization/policies/solr-tests.policy +++ b/gradle/testing/randomization/policies/solr-tests.policy @@ -164,6 +164,7 @@ grant { // Needed by BouncyCastle in jwt-auth tests permission java.security.SecurityPermission "putProviderProperty.BC"; + permission java.security.SecurityPermission "removeProviderProperty.BC"; permission java.security.SecurityPermission "getProperty.org.bouncycastle.x509.allow_non-der_tbscert"; // may only be necessary with Java 7? diff --git a/solr/licenses/bcpkix-jdk18on-1.77.jar.sha1 b/solr/licenses/bcpkix-jdk18on-1.77.jar.sha1 new file mode 100644 index 00000000000..78f704d21a8 --- /dev/null +++ b/solr/licenses/bcpkix-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +ed953791ba0229747dd0fd9911e3d76a462acfd3 diff --git a/solr/licenses/bcprov-jdk18on-1.77.jar.sha1 b/solr/licenses/bcprov-jdk18on-1.77.jar.sha1 new file mode 100644 index 00000000000..72d478f021a --- /dev/null +++ b/solr/licenses/bcprov-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +2cc971b6c20949c1ff98d1a4bc741ee848a09523 diff --git a/solr/licenses/bcutil-jdk15on-LICENSE-BSD_LIKE.txt b/solr/licenses/bcutil-LICENSE-MIT.txt similarity index 100% rename from solr/licenses/bcutil-jdk15on-LICENSE-BSD_LIKE.txt rename to solr/licenses/bcutil-LICENSE-MIT.txt diff --git a/solr/licenses/bcutil-jdk18on-1.77.jar.sha1 b/solr/licenses/bcutil-jdk18on-1.77.jar.sha1 new file mode 100644 index 00000000000..003ab86c340 --- /dev/null +++ b/solr/licenses/bcutil-jdk18on-1.77.jar.sha1 @@ -0,0 +1 @@ +de3eaef351545fe8562cf29ddff4a403a45b49b7 diff --git a/solr/licenses/bcutil-jdk15on-NOTICE.txt b/solr/licenses/bcutil-jdk18on-NOTICE.txt similarity index 100% rename from solr/licenses/bcutil-jdk15on-NOTICE.txt rename to solr/licenses/bcutil-jdk18on-NOTICE.txt diff --git a/solr/modules/jwt-auth/build.gradle b/solr/modules/jwt-auth/build.gradle index 17886099e18..c2a4990b5b3 100644 --- a/solr/modules/jwt-auth/build.gradle +++ b/solr/modules/jwt-auth/build.gradle @@ -61,8 +61,8 @@ dependencies { testImplementation 'com.fasterxml.jackson.core:jackson-databind' permitTestUnusedDeclared 'com.fasterxml.jackson.core:jackson-databind' - testImplementation 'org.bouncycastle:bcpkix-jdk15on' - testImplementation 'org.bouncycastle:bcprov-jdk15on' + testImplementation 'org.bouncycastle:bcpkix-jdk18on' + testImplementation 'org.bouncycastle:bcprov-jdk18on' testImplementation 'com.nimbusds:nimbus-jose-jwt' testImplementation 'com.squareup.okhttp3:mockwebserver' testImplementation 'com.squareup.okhttp3:okhttp' diff --git a/versions.lock b/versions.lock index e95bc0f871e..9c0f9ef53d4 100644 --- a/versions.lock +++ b/versions.lock @@ -418,6 +418,9 @@ org.apache.kerby:kerb-identity:1.0.1 (1 constraints: 5f0cb602) org.apache.kerby:kerb-server:1.0.1 (1 constraints: d10b65f2) org.apache.kerby:kerb-simplekdc:1.0.1 (1 constraints: dc0d7e3e) org.apache.tomcat.embed:tomcat-embed-el:9.0.76 (1 constraints: d41558cf) +org.bouncycastle:bcpkix-jdk18on:1.77 (1 constraints: e3040431) +org.bouncycastle:bcprov-jdk18on:1.77 (2 constraints: c51a825c) +org.bouncycastle:bcutil-jdk18on:1.77 (1 constraints: 620d2d29) org.freemarker:freemarker:2.3.32 (1 constraints: f00e9371) org.glassfish.grizzly:grizzly-framework:2.4.4 (1 constraints: 670fe271) org.glassfish.grizzly:grizzly-http:2.4.4 (1 constraints: 2b127cf5) diff --git a/versions.props b/versions.props index 1a2527610ab..10b583c43e2 100644 --- a/versions.props +++ b/versions.props @@ -52,6 +52,7 @@ org.apache.tika:*=1.28.5 org.apache.tomcat:annotations-api=6.0.53 org.apache.zookeeper:*=3.9.1 org.bitbucket.b_c:jose4j=0.9.4 +org.bouncycastle:bcpkix-jdk18on=1.77 org.carrot2:carrot2-core=4.5.1 org.codehaus.woodstox:stax2-api=4.2.2 org.eclipse.jetty*:*=10.0.18 From 755d619d9aff90444cd817c0dcfee9f19429dce7 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Tue, 12 Dec 2023 09:50:07 -0600 Subject: [PATCH 04/21] remove duplicate section from solrconfig.xml (#2146) Co-authored-by: Andrey Bozhko --- .../configsets/_default/conf/solrconfig.xml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml index d68a091f163..a74f283ed68 100644 --- a/solr/server/solr/configsets/_default/conf/solrconfig.xml +++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml @@ -494,23 +494,6 @@ --> 200 - - - + +The two binary files were created by the following commands: + +```bash +echo "Hello" > hello.txt && \ + tar -cvf hello.tar.bin hello.txt && \ + rm hello.txt + +cp HelloWorld.java.txt HelloWorld.java && \ + javac HelloWorld.java && \ + mv HelloWorld.class HelloWorldJavaClass.class.bin && \ + rm HelloWorld.java +``` \ No newline at end of file diff --git a/solr/core/src/test-files/magic/hello.tar.bin b/solr/core/src/test-files/magic/hello.tar.bin new file mode 100644 index 0000000000000000000000000000000000000000..68ca23c362ac50e132f7fba22fe858d178b0f546 GIT binary patch literal 4096 zcmdOk&q&S5$=55XC}E%#FfcGMGci$M0Mh1WreNB@2*L*n835VF3Wg@8#%6}b#s+5Q z3I>M8W(I}~3I?=t5VE$R`3s2h){q6kQf6e5E}t;ArOQ3i-DL3;207T1XTvoF$0LX(X=rl#6^?y zbM+Dn3UX5Q3X1Z}Qu7k?l2aLg3O0P~;5y?waW0zHQ7$z@ARw{ABQ-H4wMd_KrzW2+#jUmI}EgnYpR9hUUO>*uc`TW+;=40LoX4siE$F0Sy)jxyjfu~Y~KYVwK1rwJ*G zbK#1eQE?iF0PS);!r_GSKd^W}%m2o}w!mopj|f59u{=PnztQYRy8s?Fc{Bt@Ltr!n I25ATY0C-`G*Z=?k literal 0 HcmV?d00001 diff --git a/solr/core/src/test-files/magic/plain.txt b/solr/core/src/test-files/magic/plain.txt new file mode 100644 index 00000000000..70c379b63ff --- /dev/null +++ b/solr/core/src/test-files/magic/plain.txt @@ -0,0 +1 @@ +Hello world \ No newline at end of file diff --git a/solr/core/src/test-files/magic/shell.sh.txt b/solr/core/src/test-files/magic/shell.sh.txt new file mode 100644 index 00000000000..9ea411e111d --- /dev/null +++ b/solr/core/src/test-files/magic/shell.sh.txt @@ -0,0 +1,2 @@ +#! /usr/bin/env bash +echo Hello \ No newline at end of file diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index 1da60d85ff9..f7c9431c296 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -592,14 +593,14 @@ public void testOverwrite(boolean v2) throws Exception { assertEquals( "Can't overwrite an existing configset unless the overwrite parameter is set", 400, - uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false)); + uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false, false)); unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper"); assertEquals( "Expecting version to remain equal", solrconfigZkVersion, getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml")); assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); assertTrue( "Expecting version bump", solrconfigZkVersion @@ -638,13 +639,14 @@ public void testOverwriteWithCleanup(boolean v2) throws Exception { zkClient.makePath(f, true); } assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); for (String f : extraFiles) { assertTrue( "Expecting file " + f + " to exist in ConfigSet but it's gone", zkClient.exists(f, true)); } - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false)); + assertEquals( + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false)); for (String f : extraFiles) { assertFalse( "Expecting file " + f + " to be deleted from ConfigSet but it wasn't", @@ -675,7 +677,8 @@ public void testOverwriteWithForbiddenFiles(boolean v2) throws Exception { .withConnTimeOut(45000, TimeUnit.MILLISECONDS) .build()) { String configPath = "/configs/" + configsetName + configsetSuffix; - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true)); + assertEquals( + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true, false)); for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) { String f = configPath + "/test." + fileEnding; assertFalse( @@ -710,7 +713,7 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"); // Was untrusted, overwrite with untrusted assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); assertTrue( "Expecting version bump", solrconfigZkVersion @@ -721,7 +724,8 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { // Was untrusted, overwrite with trusted but no cleanup assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false)); + 0, + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false)); assertTrue( "Expecting version bump", solrconfigZkVersion @@ -747,7 +751,7 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { // Was untrusted, overwrite with trusted with cleanup assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false)); + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false)); assertTrue( "Expecting version bump", solrconfigZkVersion @@ -761,7 +765,7 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { assertEquals( "Can't upload a trusted configset with an untrusted request", 400, - uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false)); + uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false)); assertEquals( "Expecting version to remain equal", solrconfigZkVersion, @@ -773,7 +777,7 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { assertEquals( "Can't upload a trusted configset with an untrusted request", 400, - uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false)); + uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false)); assertEquals( "Expecting version to remain equal", solrconfigZkVersion, @@ -783,7 +787,8 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { // Was trusted, overwrite with trusted no cleanup assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false)); + 0, + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false)); assertTrue( "Expecting version bump", solrconfigZkVersion @@ -794,7 +799,7 @@ public void testOverwriteWithTrust(boolean v2) throws Exception { // Was trusted, overwrite with trusted with cleanup assertEquals( - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false)); + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false)); assertTrue( "Expecting version bump", solrconfigZkVersion @@ -1457,6 +1462,13 @@ public void testUploadWithLibDirective() throws Exception { .get("id")); } + @Test + public void testUploadWithForbiddenContent() throws Exception { + // Uploads a config set containing a script, a class file and jar file, will return 400 error + long res = uploadConfigSet("forbidden", "suffix", "foo", true, false, true, false, true); + assertEquals(400, res); + } + private static String getSecurityJson() { return "{\n" + " 'authentication':{\n" @@ -1511,7 +1523,7 @@ private long uploadConfigSet( String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2) throws IOException { assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix)); - return uploadConfigSet(configSetName, suffix, username, false, false, v2, false); + return uploadConfigSet(configSetName, suffix, username, false, false, v2, false, false); } private long uploadConfigSet( @@ -1521,21 +1533,25 @@ private long uploadConfigSet( boolean overwrite, boolean cleanup, boolean v2, - boolean forbiddenTypes) + boolean forbiddenTypes, + boolean forbiddenContent) throws IOException { + File zipFile; + if (forbiddenTypes) { + log.info("Uploading configset with forbidden file endings"); + zipFile = + createTempZipFileWithForbiddenTypes( + "solr/configsets/upload/" + configSetName + "/solrconfig.xml"); + } else if (forbiddenContent) { + log.info("Uploading configset with forbidden file content"); + zipFile = createTempZipFileWithForbiddenContent("magic"); + } else { + zipFile = createTempZipFile("solr/configsets/upload/" + configSetName); + } + // Read zipped sample config - return uploadGivenConfigSet( - forbiddenTypes - ? createTempZipFileWithForbiddenTypes( - "solr/configsets/upload/" + configSetName + "/solrconfig.xml") - : createTempZipFile("solr/configsets/upload/" + configSetName), - configSetName, - suffix, - username, - overwrite, - cleanup, - v2); + return uploadGivenConfigSet(zipFile, configSetName, suffix, username, overwrite, cleanup, v2); } private long uploadBadConfigSet(String configSetName, String suffix, String username, boolean v2) @@ -1702,31 +1718,68 @@ private File createTempZipFileWithForbiddenTypes(String file) { } } - private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException { - OutputStream out = new FileOutputStream(zipfile); - ZipOutputStream zout = new ZipOutputStream(out); + /** Create a zip file (in the temp directory) containing files with forbidden content */ + private File createTempZipFileWithForbiddenContent(String resourcePath) { try { - for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) { - zout.putNextEntry(new ZipEntry("test." + fileType)); + final File zipFile = createTempFile("configset", "zip").toFile(); + final File directory = SolrTestCaseJ4.getFile(resourcePath); + if (log.isInfoEnabled()) { + log.info("Directory: {}", directory.getAbsolutePath()); + } + zipWithForbiddenContent(directory, zipFile); + if (log.isInfoEnabled()) { + log.info("Zipfile: {}", zipFile.getAbsolutePath()); + } + return zipFile; + } catch (IOException e) { + throw new RuntimeException(e); + } + } - InputStream in = new FileInputStream(file); - try { - byte[] buffer = new byte[1024]; - while (true) { - int readCount = in.read(buffer); - if (readCount < 0) { - break; + private static void zipWithForbiddenContent(File directory, File zipfile) throws IOException { + OutputStream out = Files.newOutputStream(zipfile.toPath()); + assertTrue(directory.isDirectory()); + try (ZipOutputStream zout = new ZipOutputStream(out)) { + // Copy in all files from the directory + for (File file : Objects.requireNonNull(directory.listFiles())) { + zout.putNextEntry(new ZipEntry(file.getName())); + zout.write(Files.readAllBytes(file.toPath())); + zout.closeEntry(); + } + } + } + + private static void zipWithForbiddenEndings(File fileOrDirectory, File zipfile) + throws IOException { + OutputStream out = new FileOutputStream(zipfile); + try (ZipOutputStream zout = new ZipOutputStream(out)) { + if (fileOrDirectory.isFile()) { + // Create entries with given file, one for each forbidden endding + for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) { + zout.putNextEntry(new ZipEntry("test." + fileType)); + + try (InputStream in = new FileInputStream(fileOrDirectory)) { + byte[] buffer = new byte[1024]; + while (true) { + int readCount = in.read(buffer); + if (readCount < 0) { + break; + } + zout.write(buffer, 0, readCount); } - zout.write(buffer, 0, readCount); } - } finally { - in.close(); - } - zout.closeEntry(); + zout.closeEntry(); + } + } + if (fileOrDirectory.isDirectory()) { + // Copy in all files from the directory + for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) { + zout.putNextEntry(new ZipEntry(file.getName())); + zout.write(Files.readAllBytes(file.toPath())); + zout.closeEntry(); + } } - } finally { - zout.close(); } } diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java new file mode 100644 index 00000000000..b8e9a35a3d9 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.util; + +import org.apache.solr.SolrTestCaseJ4; + +public class FileTypeMagicUtilTest extends SolrTestCaseJ4 { + public void testGuessMimeType() { + assertEquals( + "application/x-java-applet", + FileTypeMagicUtil.INSTANCE.guessMimeType( + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin"))); + assertEquals( + "application/zip", + FileTypeMagicUtil.INSTANCE.guessMimeType( + FileTypeMagicUtil.class.getResourceAsStream( + "/runtimecode/containerplugin.v.1.jar.bin"))); + assertEquals( + "application/x-tar", + FileTypeMagicUtil.INSTANCE.guessMimeType( + FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin"))); + assertEquals( + "text/x-shellscript", + FileTypeMagicUtil.INSTANCE.guessMimeType( + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt"))); + } + + public void testIsFileForbiddenInConfigset() { + assertTrue( + FileTypeMagicUtil.isFileForbiddenInConfigset( + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin"))); + assertTrue( + FileTypeMagicUtil.isFileForbiddenInConfigset( + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt"))); + assertFalse( + FileTypeMagicUtil.isFileForbiddenInConfigset( + FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt"))); + } +} diff --git a/solr/licenses/simplemagic-1.17.jar.sha1 b/solr/licenses/simplemagic-1.17.jar.sha1 new file mode 100644 index 00000000000..cf101094cc8 --- /dev/null +++ b/solr/licenses/simplemagic-1.17.jar.sha1 @@ -0,0 +1 @@ +b6e2d1e47d7172e57fa858a2e3940c09a590e61e diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt new file mode 100644 index 00000000000..9228230f933 --- /dev/null +++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt @@ -0,0 +1,15 @@ +ISC License (https://opensource.org/licenses/ISC) + +Copyright 2021, Gray Watson + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file diff --git a/solr/licenses/simplemagic-NOTICE.txt b/solr/licenses/simplemagic-NOTICE.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java index d571339880c..e40294a6683 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java @@ -348,6 +348,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) USE_FORBIDDEN_FILE_TYPES); return FileVisitResult.CONTINUE; } + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core String zkNode = createZkNodeName(zkPath, rootPath, file); try { // if the path exists (and presumably we're uploading data to it) just set its data @@ -437,6 +438,7 @@ public static void downloadFromZK(SolrZkClient zkClient, String zkPath, Path fil if (isFileForbiddenInConfigSets(zkPath)) { log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath); } else { + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core if (copyDataDown(zkClient, zkPath, file) == 0) { Files.createFile(file); } diff --git a/versions.lock b/versions.lock index 9c0f9ef53d4..6b6ea467e0d 100644 --- a/versions.lock +++ b/versions.lock @@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5) com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201) com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints: 5c0cf101) com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784) +com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830) com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c) com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36) com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b) diff --git a/versions.props b/versions.props index 10b583c43e2..cfa127e1094 100644 --- a/versions.props +++ b/versions.props @@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0 com.google.errorprone:*=2.23.0 com.google.guava:guava=32.1.3-jre com.google.re2j:re2j=1.7 +com.j256.simplemagic:simplemagic=1.17 com.jayway.jsonpath:json-path=2.8.0 com.lmax:disruptor=3.4.4 com.tdunning:t-digest=3.1 From 955074b8b3a9213c9b106d3b6d273e8562de453f Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Thu, 14 Dec 2023 12:36:05 -0500 Subject: [PATCH 11/21] SOLR-16880: Add OAS to release automation (#2125) Modifies our 'assembleRelease' gradle task and smoketester to handle a new release artifact: an OpenAPI spec ("OAS") covering Solr's v2 APIs. The spec (along with associated checksum and signature files) are made available under a separate 'openApi' directory, similar to our 'changes' files. --------- Co-authored-by: Houston Putman --- dev-tools/scripts/smokeTestRelease.py | 22 +++++++++++++++++ solr/CHANGES.txt | 3 +++ solr/api/build.gradle | 5 ++-- solr/distribution/build.gradle | 35 ++++++++++++++++++++++++--- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 554c664559d..3d57f6d98f9 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -220,6 +220,7 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): ents = getDirEntries(urlString) artifact = None changesURL = None + openApiURL = None mavenURL = None dockerURL = None artifactURL = None @@ -243,6 +244,10 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): if text not in ('changes/', 'changes-%s/' % version): raise RuntimeError('solr: found %s vs expected changes-%s/' % (text, version)) changesURL = subURL + elif text.startswith('openApi'): + if text not in ('openApi/', 'openApi-%s/' % version): + raise RuntimeError('solr: found %s vs expected openApi-%s/' % (text, version)) + openApiURL = subURL elif artifact is None: artifact = text artifactURL = subURL @@ -296,6 +301,12 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): raise RuntimeError('solr is missing changes-%s' % version) testChanges(version, changesURL) + if openApiURL is None: + stopGpgAgent(gpgHomeDir, logfile) + raise RuntimeError('solr is missing OpenAPI specification' % version) + testOpenApi(version, openApiURL) + + for artifact, urlString in artifacts: # pylint: disable=redefined-argument-from-local print(' download %s...' % artifact) scriptutil.download(artifact, urlString, tmpDir, force_clean=FORCE_CLEAN) @@ -349,6 +360,17 @@ def testChanges(version, changesURLString): s = load(changesURL) checkChangesContent(s, version, changesURL, True) +def testOpenApi(version, openApiDirUrl): + print(' check OpenAPI specification...') + specFound = False + expectedSpecFileName = 'solr-openapi-' + version + '.json' + for text, subURL in getDirEntries(openApiDirUrl): + if text == expectedSpecFileName: + specFound = True + + if not specFound: + raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl) + def testChangesText(dir, version): "Checks all CHANGES.txt under this dir." diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2fd94304a24..9505b2c6f53 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -171,6 +171,9 @@ Other Changes * SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman) +* SOLR-16880: Solr now produces an OpenAPI Specification artifact on releases ("solr-openapi-x.y.z.json") that covers + Solr's v2 APIs. (Jason Gerlowski, Houston Putman) + ================== 9.4.0 ================== New Features --------------------- diff --git a/solr/api/build.gradle b/solr/api/build.gradle index ecd93125e86..bbcf66170e1 100644 --- a/solr/api/build.gradle +++ b/solr/api/build.gradle @@ -28,7 +28,7 @@ ext { jsClientDir = "${buildDir}/generated/js" pythonClientDir = "${buildDir}/generated/python" openApiSpecDir = "${buildDir}/generated/openapi" - openApiSpecFile = "${project.openApiSpecDir}/openapi.json" + openApiSpecFile = "${project.openApiSpecDir}/solr-openapi-${version}.json" } configurations { @@ -50,6 +50,7 @@ resolve { classpath = sourceSets.main.runtimeClasspath resourcePackages = ["org.apache.solr.client.api.util", "org.apache.solr.client.api.endpoint"] outputDir = file(project.openApiSpecDir) + outputFileName = "solr-openapi-${version}" prettyPrint = true } @@ -91,7 +92,7 @@ tasks.withType(org.openapitools.generator.gradle.plugin.tasks.GenerateTask) { artifacts { // Ensure the OAS is available to other modules who want to generate code (i.e. solrj) - openapiSpec resolve.outputDir, { + openapiSpec file(openApiSpecFile), { builtBy resolve } diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index d1ad6b62256..8ebc5872f53 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -54,11 +54,13 @@ apply from: buildscript.sourceFile.toPath().resolveSibling("source-release.gradl // Set up the HTML-rendered "changes" distribution artifact by linking to documentation's output. configurations { changesHtml + openApiSpecFile docker } dependencies { changesHtml project(path: ":solr:documentation", configuration: "changesHtml") + openApiSpecFile project(path: ":solr:api", configuration: "openapiSpec") docker project(path: ':solr:docker', configuration: 'packagingOfficial') } @@ -73,11 +75,13 @@ task computeChecksums(type: Checksum) { [ tasks.assembleSourceTgz, fullDistTarTask, - slimDistTarTask, + slimDistTarTask ].each { dep -> dependsOn dep files += dep.outputs.files } + dependsOn configurations.openApiSpecFile + files += configurations.openApiSpecFile outputDir = file("${buildDir}/checksums") } @@ -93,11 +97,19 @@ task signSourceTgz(type: Sign) { dependsOn tasks.assembleSourceTgz sign tasks.assembleSourceTgz.destination } +task signOpenApiSpec(type: Sign) { + dependsOn configurations.openApiSpecFile + // This is not an artifact, so we need to use "file" not "configuration" signing + doFirst { + sign configurations.openApiSpecFile.singleFile + } +} task signReleaseArchives(type: Sync) { from tasks.signFullBinaryTgz from tasks.signSlimBinaryTgz from tasks.signSourceTgz + from tasks.signOpenApiSpec into "${buildDir}/signatures" } @@ -125,6 +137,10 @@ task assembleRelease(type: Sync) { into "changes" }) + from(configurations.openApiSpecFile, { + into "openApi" + }) + from(configurations.docker, { include 'Dockerfile.official*' into "docker" @@ -139,11 +155,24 @@ task assembleRelease(type: Sync) { from fullDistTarTask from slimDistTarTask - from tasks.computeChecksums + from(tasks.computeChecksums, { + exclude { it.file.getName().contains("openapi") } + }) + from(tasks.computeChecksums, { + include { it.file.getName().contains("openapi") } + into "openApi" + }) // Conditionally, attach signatures of all the release archives. if (project.ext.withSignedArtifacts) { - from tasks.signReleaseArchives + from(tasks.signReleaseArchives, { + exclude { it.file.getName().contains("openapi") } + }) + + from(tasks.signReleaseArchives, { + include { it.file.getName().contains("openapi") } + into "openApi" + }) } into releaseDir From 8768be25387336184de7248abf39fe488826be67 Mon Sep 17 00:00:00 2001 From: mariemat Date: Fri, 15 Dec 2023 10:10:51 +0100 Subject: [PATCH 12/21] Update replica-placement-plugins.adoc (#2155) --- .../configuration-guide/pages/replica-placement-plugins.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/replica-placement-plugins.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/replica-placement-plugins.adoc index 0ce84efaf68..6cf06903014 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/replica-placement-plugins.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/replica-placement-plugins.adoc @@ -133,7 +133,7 @@ If `withCollection` or `withCollectionShards` are defined and applicable to the ** Iterate over the number of replicas to place (for the current replica type for the current shard): *** Based on the number of replicas per AZ collected previously, pick the non-empty set of nodes having the lowest number of replicas. Then pick the first node in that set. -That's the node the replica is placed one. +That's the node the replica is placed on. Remove the node from the set of available nodes for the given AZ and increase the number of replicas placed on that AZ. ** During this process, the number of cores on the nodes in general is tracked to take into account placement decisions so that not all shards decide to put their replicas on the same nodes (they might though if these are the less loaded nodes). From 0b2326d701f7a19d2a2e4091eb6685736234c83c Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Fri, 15 Dec 2023 15:32:59 -0500 Subject: [PATCH 13/21] SOLR-17066: Add 'defaultCollection' to SolrClients (#2066) This commit adds a setter ('withDefaultCollection(String)') to all SolrClient builders, and propagates the field down to each client implementation. All SolrClient implementations now have a getter to fetch this property ('getDefaultCollection'). This will help users replace the anti-pattern of baking the collection/ core name into their client's base URL to achieve a sort of "poor man's" default. --- solr/CHANGES.txt | 4 ++++ .../apache/solr/client/solrj/SolrClient.java | 10 +++++++++ .../solrj/impl/CloudLegacySolrClient.java | 7 ------- .../client/solrj/impl/CloudSolrClient.java | 6 ------ .../impl/ConcurrentUpdateHttp2SolrClient.java | 9 ++++++++ .../impl/ConcurrentUpdateSolrClient.java | 2 ++ .../client/solrj/impl/Http2SolrClient.java | 10 ++++++++- .../client/solrj/impl/HttpSolrClient.java | 2 ++ .../client/solrj/impl/LBHttp2SolrClient.java | 21 ++++++++++++------- .../client/solrj/impl/LBHttpSolrClient.java | 4 ++++ .../solr/client/solrj/impl/LBSolrClient.java | 1 + .../client/solrj/impl/SolrClientBuilder.java | 6 ++++++ .../impl/CloudHttp2SolrClientBuilderTest.java | 11 ++++++++++ .../impl/CloudSolrClientBuilderTest.java | 10 +++++++++ ...ConcurrentUpdateSolrClientBuilderTest.java | 11 ++++++++++ .../solrj/impl/HttpSolrClientBuilderTest.java | 9 ++++++++ .../impl/LBHttpSolrClientBuilderTest.java | 11 ++++++++++ 17 files changed, 112 insertions(+), 22 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9505b2c6f53..6f8e0477bf9 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -124,6 +124,10 @@ Improvements * SOLR-17063: Do not retain log param references in LogWatcher (Michael Gibney) +* SOLR-17066: SolrClient builders now allow users to specify a "default" collection or core + using the `withDefaultCollection` method. This is preferable to including the collection + in the base URL accepted by certain client implementations. (Jason Gerlowski) + Optimizations --------------------- * SOLR-17084: LBSolrClient (used by CloudSolrClient) now returns the count of core tracked as not live AKA zombies diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrClient.java index f79bba0a64b..a134e2f001a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/SolrClient.java @@ -52,6 +52,7 @@ public abstract class SolrClient implements Serializable, Closeable { private static final long serialVersionUID = 1L; private DocumentObjectBinder binder; + protected String defaultCollection; /** * Adds a collection of documents @@ -1215,4 +1216,13 @@ public DocumentObjectBinder getBinder() { public SolrRequest.SolrClientContext getContext() { return SolrRequest.SolrClientContext.CLIENT; } + + /** + * Gets the collection used by default for collection or core-based requests + * + *

If no value is specified at client-creation time, this method will return null. + */ + public String getDefaultCollection() { + return defaultCollection; + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java index c1bc6811411..3547c652293 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java @@ -167,7 +167,6 @@ public static class Builder extends SolrClientBuilder { protected boolean shardLeadersOnly = true; protected boolean directUpdatesToLeadersOnly = false; protected boolean parallelUpdates = true; - protected String defaultCollection; protected long retryExpiryTimeNano = TimeUnit.NANOSECONDS.convert(3, TimeUnit.SECONDS); // 3 seconds or 3 million nanos protected ClusterStateProvider stateProvider; @@ -343,12 +342,6 @@ public Builder withParallelUpdates(boolean parallelUpdates) { return this; } - /** Sets the default collection for request. */ - public Builder withDefaultCollection(String collection) { - this.defaultCollection = collection; - return this; - } - /** * Sets the Zk connection timeout * diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 58f98c6bbc8..2d08fac6a35 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -90,7 +90,6 @@ public abstract class CloudSolrClient extends SolrClient { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - protected volatile String defaultCollection; // no of times collection state to be reloaded if stale state error is received private static final int MAX_STALE_RETRIES = Integer.parseInt(System.getProperty("cloudSolrClientMaxStaleRetries", "5")); @@ -292,11 +291,6 @@ public RequestWriter getRequestWriter() { return getLbClient().getRequestWriter(); } - /** Gets the default collection for request */ - public String getDefaultCollection() { - return defaultCollection; - } - /** Gets whether direct updates are sent in parallel */ public boolean isParallelUpdates() { return parallelUpdates; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java index 69c54ba693a..617e853db52 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateHttp2SolrClient.java @@ -149,6 +149,7 @@ protected ConcurrentUpdateHttp2SolrClient(Builder builder) { this.runners = new ArrayDeque<>(); this.streamDeletes = builder.streamDeletes; this.basePath = builder.baseSolrUrl; + this.defaultCollection = builder.defaultCollection; this.pollQueueTimeMillis = builder.pollQueueTimeMillis; this.stallTimeMillis = Integer.getInteger("solr.cloud.client.stallTime", 15000); @@ -359,6 +360,7 @@ private void addRunner() { @Override public NamedList request(final SolrRequest request, String collection) throws SolrServerException, IOException { + if (collection == null) collection = defaultCollection; if (!(request instanceof UpdateRequest)) { request.setBasePath(basePath); return client.request(request, collection); @@ -697,6 +699,7 @@ public void shutdownNow() { public static class Builder { protected Http2SolrClient client; protected String baseSolrUrl; + protected String defaultCollection; protected int queueSize = 10; protected int threadCount; protected ExecutorService executorService; @@ -787,6 +790,12 @@ public Builder neverStreamDeletes() { return this; } + /** Sets a default collection for collection-based requests. */ + public Builder withDefaultCollection(String defaultCollection) { + this.defaultCollection = defaultCollection; + return this; + } + /** * @param pollQueueTime time for an open connection to wait for updates when the queue is empty. */ diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java index 298811c8f39..6bac5c0457b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java @@ -116,6 +116,7 @@ protected ConcurrentUpdateSolrClient(Builder builder) { this.soTimeout = builder.socketTimeoutMillis; this.pollQueueTimeMillis = builder.pollQueueTime; this.stallTimeMillis = Integer.getInteger("solr.cloud.client.stallTime", 15000); + this.defaultCollection = builder.defaultCollection; // make sure the stall time is larger than the polling time // to give a chance for the queue to change @@ -475,6 +476,7 @@ public void setCollection(String collection) { @Override public NamedList request(final SolrRequest request, String collection) throws SolrServerException, IOException { + if (collection == null) collection = defaultCollection; if (!(request instanceof UpdateRequest)) { return client.request(request, collection); } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java index 00e94715cfd..af5164f2611 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java @@ -191,6 +191,7 @@ protected Http2SolrClient(String serverBaseUrl, Builder builder) { this.parser = builder.responseParser; } updateDefaultMimeTypeForParser(); + this.defaultCollection = builder.defaultCollection; if (builder.requestTimeoutMillis != null) { this.requestTimeoutMillis = builder.requestTimeoutMillis; } else { @@ -554,7 +555,7 @@ public void onFailure(Response response, Throwable failure) { @Override public NamedList request(SolrRequest solrRequest, String collection) throws SolrServerException, IOException { - + if (collection == null) collection = defaultCollection; String url = getRequestPath(solrRequest, collection); Throwable abortCause = null; Request req = null; @@ -1070,6 +1071,7 @@ public static class Builder { private ExecutorService executor; protected RequestWriter requestWriter; protected ResponseParser responseParser; + protected String defaultCollection; private Set urlParamNames; private CookieStore cookieStore = getDefaultCookieStore(); private String proxyHost; @@ -1194,6 +1196,12 @@ public Builder withResponseParser(ResponseParser responseParser) { return this; } + /** Sets a default collection for collection-based requests. */ + public Builder withDefaultCollection(String defaultCollection) { + this.defaultCollection = defaultCollection; + return this; + } + public Builder withFollowRedirects(boolean followRedirects) { this.followRedirects = followRedirects; return this; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java index 67f3162f1cb..9d11dbe7768 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java @@ -188,6 +188,7 @@ protected HttpSolrClient(Builder builder) { this.soTimeout = builder.socketTimeoutMillis; this.useMultiPartPost = builder.useMultiPartPost; this.urlParamNames = builder.urlParamNames; + this.defaultCollection = builder.defaultCollection; } public Set getUrlParamNames() { @@ -242,6 +243,7 @@ public NamedList request(final SolrRequest request, final ResponsePar public NamedList request( final SolrRequest request, final ResponseParser processor, String collection) throws SolrServerException, IOException { + if (collection == null) collection = defaultCollection; HttpRequestBase method = createMethod(request, collection); setBasicAuthHeader(request, method); if (request.getHeaders() != null) { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java index 18b89440568..82650e525b1 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttp2SolrClient.java @@ -23,7 +23,6 @@ import java.net.SocketException; import java.net.SocketTimeoutException; import java.util.Arrays; -import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -86,9 +85,11 @@ public class LBHttp2SolrClient extends LBSolrClient { private final Http2SolrClient solrClient; - private LBHttp2SolrClient(Http2SolrClient solrClient, List baseSolrUrls) { - super(baseSolrUrls); - this.solrClient = solrClient; + private LBHttp2SolrClient(Builder builder) { + super(Arrays.asList(builder.baseSolrUrls)); + this.solrClient = builder.http2SolrClient; + this.aliveCheckIntervalMillis = builder.aliveCheckIntervalMillis; + this.defaultCollection = builder.defaultCollection; } @Override @@ -258,6 +259,7 @@ public static class Builder { private final String[] baseSolrUrls; private long aliveCheckIntervalMillis = TimeUnit.MILLISECONDS.convert(60, TimeUnit.SECONDS); // 1 minute between checks + protected String defaultCollection; public Builder(Http2SolrClient http2Client, String... baseSolrUrls) { this.http2SolrClient = http2Client; @@ -279,11 +281,14 @@ public LBHttp2SolrClient.Builder setAliveCheckInterval(int aliveCheckInterval, T return this; } + /** Sets a default collection for collection-based requests. */ + public LBHttp2SolrClient.Builder withDefaultCollection(String defaultCollection) { + this.defaultCollection = defaultCollection; + return this; + } + public LBHttp2SolrClient build() { - LBHttp2SolrClient solrClient = - new LBHttp2SolrClient(this.http2SolrClient, Arrays.asList(this.baseSolrUrls)); - solrClient.aliveCheckIntervalMillis = this.aliveCheckIntervalMillis; - return solrClient; + return new LBHttp2SolrClient(this); } } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java index 1ba16f521f2..923d651afb3 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java @@ -93,6 +93,10 @@ protected LBHttpSolrClient(Builder builder) { builder.httpClient == null ? constructClient(builder.baseSolrUrls.toArray(new String[0])) : builder.httpClient; + this.defaultCollection = builder.defaultCollection; + if (httpSolrClientBuilder != null && this.defaultCollection != null) { + httpSolrClientBuilder.defaultCollection = this.defaultCollection; + } this.connectionTimeoutMillis = builder.connectionTimeoutMillis; this.soTimeoutMillis = builder.socketTimeoutMillis; this.parser = builder.responseParser; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java index 2723465b739..6acf04aea57 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/LBSolrClient.java @@ -574,6 +574,7 @@ public NamedList request( final int maxTries = (numServersToTry == null ? serverList.length : numServersToTry.intValue()); int numServersTried = 0; Map justFailed = null; + if (collection == null) collection = defaultCollection; boolean timeAllowedExceeded = false; long timeAllowedNano = getTimeAllowedInNanos(request); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java index d0c97f8ed22..aee3bf55f23 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientBuilder.java @@ -41,6 +41,7 @@ public abstract class SolrClientBuilder> { protected int socketTimeoutMillis = 120000; // 120 seconds private boolean socketTimeoutMillisUpdate = false; protected boolean followRedirects = false; + protected String defaultCollection; protected Set urlParamNames; /** The solution for the unchecked cast warning. */ @@ -97,6 +98,11 @@ public B withFollowRedirects(boolean followRedirects) { return getThis(); } + public B withDefaultCollection(String defaultCollection) { + this.defaultCollection = defaultCollection; + return getThis(); + } + /** * Tells {@link Builder} that created clients should obey the following timeout when connecting to * Solr servers. diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientBuilderTest.java index 6cf26e440b6..b776755a2ed 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientBuilderTest.java @@ -161,4 +161,15 @@ public void testProvideExternalClient() throws IOException { // it's external, should be NOT closed when closing CloudSolrClient verify(http2Client, never()).close(); } + + @Test + public void testDefaultCollectionPassedFromBuilderToClient() throws IOException { + try (CloudHttp2SolrClient createdClient = + new CloudHttp2SolrClient.Builder( + Collections.singletonList(ANY_ZK_HOST), Optional.of(ANY_CHROOT)) + .withDefaultCollection("aCollection") + .build()) { + assertEquals("aCollection", createdClient.getDefaultCollection()); + } + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java index b96d5762c5d..348e8e1d65e 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientBuilderTest.java @@ -101,4 +101,14 @@ public void test0Timeouts() throws IOException { assertNotNull(createdClient); } } + + @Test + public void testDefaultCollectionPassedFromBuilderToClient() throws IOException { + try (CloudSolrClient createdClient = + new Builder(Collections.singletonList(ANY_ZK_HOST), Optional.of(ANY_CHROOT)) + .withDefaultCollection("aCollection") + .build()) { + assertEquals("aCollection", createdClient.getDefaultCollection()); + } + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java index 902aeb61842..c6cd4779f96 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClientBuilderTest.java @@ -23,6 +23,7 @@ import java.net.SocketTimeoutException; import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCase; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.Builder; import org.junit.Test; @@ -71,4 +72,14 @@ public void testSocketTimeoutOnCommit() throws IOException, SolrServerException // else test passses } } + + @Test + public void testDefaultCollectionPassedFromBuilderToClient() throws IOException { + try (SolrClient createdClient = + new ConcurrentUpdateSolrClient.Builder("someurl") + .withDefaultCollection("aCollection") + .build()) { + assertEquals("aCollection", createdClient.getDefaultCollection()); + } + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java index eea4fcfc3b8..6aba55d47bf 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientBuilderTest.java @@ -22,6 +22,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.solr.SolrTestCase; import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder; import org.apache.solr.common.params.ModifiableSolrParams; import org.junit.Test; @@ -83,4 +84,12 @@ public void testDefaultsToBinaryResponseParserWhenNoneProvided() throws IOExcept assertTrue(usedParser instanceof BinaryResponseParser); } } + + @Test + public void testDefaultCollectionPassedFromBuilderToClient() throws IOException { + try (final SolrClient createdClient = + new Builder(ANY_BASE_SOLR_URL).withDefaultCollection("aCollection").build()) { + assertEquals("aCollection", createdClient.getDefaultCollection()); + } + } } diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java index 25a70f5460c..6db869cf96b 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/LBHttpSolrClientBuilderTest.java @@ -77,4 +77,15 @@ public void testUsesTimeoutProvidedByHttpClient() throws IOException { } HttpClientUtil.close(httpClient); } + + @Test + public void testDefaultCollectionPassedFromBuilderToClient() throws IOException { + try (LBHttpSolrClient createdClient = + new LBHttpSolrClient.Builder() + .withBaseSolrUrl(ANY_BASE_SOLR_URL) + .withDefaultCollection("aCollection") + .build()) { + assertEquals("aCollection", createdClient.getDefaultCollection()); + } + } } From c89813a675663bb82f18236840b2a84cac05e1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Sun, 17 Dec 2023 23:19:14 +0100 Subject: [PATCH 14/21] SOLR-16949: Handle inputStream leaks --- .../solr/core/FileSystemConfigSetService.java | 5 +- .../apache/solr/util/FileTypeMagicUtil.java | 93 +++++++++++++------ .../solr/util/FileTypeMagicUtilTest.java | 58 ++++++------ 3 files changed, 96 insertions(+), 60 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java index 4b041252211..c4686195d64 100644 --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java @@ -214,12 +214,11 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) "Not including uploading file to config, as it is a forbidden type: {}", file.getFileName()); } else { - if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) { + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(file)) { Files.copy( file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING); } else { - String mimeType = - FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file)); + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(file); log.warn( "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}", file.getFileName(), diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java index cfb6c9fa0af..692cda83bd3 100644 --- a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java +++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java @@ -20,7 +20,6 @@ import com.j256.simplemagic.ContentInfo; import com.j256.simplemagic.ContentInfoUtil; import com.j256.simplemagic.ContentType; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.nio.file.FileVisitResult; @@ -58,30 +57,23 @@ public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack { public static void assertConfigSetFolderLegal(Path confPath) throws IOException { Files.walkFileTree( confPath, - new SimpleFileVisitor() { + new SimpleFileVisitor<>() { @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - // Read first 100 bytes of the file to determine the mime type - try (InputStream fileStream = Files.newInputStream(file)) { - byte[] bytes = new byte[100]; - fileStream.read(bytes); - if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - String.format( - Locale.ROOT, - "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s", - file, - FileTypeMagicUtil.INSTANCE.guessMimeType(bytes))); - } - return FileVisitResult.CONTINUE; + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (FileTypeMagicUtil.isFileForbiddenInConfigset(file)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + String.format( + Locale.ROOT, + "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s", + file, + FileTypeMagicUtil.INSTANCE.guessMimeType(file))); } + return FileVisitResult.CONTINUE; } @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { if (SKIP_FOLDERS.contains(dir.getFileName().toString())) return FileVisitResult.SKIP_SUBTREE; @@ -90,19 +82,29 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) }); } + /** + * Guess the mime type of file based on its magic number. + * + * @param file file to check + * @return string with content-type or "application/octet-stream" if unknown + */ + public String guessMimeType(Path file) { + try { + return guessTypeFallbackToOctetStream(util.findMatch(file.toFile())); + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + } + /** * Guess the mime type of file based on its magic number. * * @param stream input stream of the file * @return string with content-type or "application/octet-stream" if unknown */ - public String guessMimeType(InputStream stream) { + String guessMimeType(InputStream stream) { try { - ContentInfo info = util.findMatch(stream); - if (info == null) { - return ContentType.OTHER.getMimeType(); - } - return info.getContentType().getMimeType(); + return guessTypeFallbackToOctetStream(util.findMatch(stream)); } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } @@ -115,7 +117,7 @@ public String guessMimeType(InputStream stream) { * @return string with content-type or "application/octet-stream" if unknown */ public String guessMimeType(byte[] bytes) { - return guessMimeType(new ByteArrayInputStream(bytes)); + return guessTypeFallbackToOctetStream(util.findMatch(bytes)); } @Override @@ -126,6 +128,31 @@ public void error(String line, String details, Exception e) { e); } + /** + * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types + * are: + * + *
    + *
  • application/x-java-applet: java class file + *
  • application/zip: jar or zip archives + *
  • application/x-tar: tar archives + *
  • text/x-shellscript: shell or bash script + *
+ * + * @param file file to check + * @return true if file is among the forbidden mime-types + */ + public static boolean isFileForbiddenInConfigset(Path file) { + try (InputStream fileStream = Files.newInputStream(file)) { + return isFileForbiddenInConfigset(fileStream); + } catch (IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + String.format(Locale.ROOT, "Error reading file %s", file), + e); + } + } + /** * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types * are: @@ -140,7 +167,7 @@ public void error(String line, String details, Exception e) { * @param fileStream stream from the file content * @return true if file is among the forbidden mime-types */ - public static boolean isFileForbiddenInConfigset(InputStream fileStream) { + static boolean isFileForbiddenInConfigset(InputStream fileStream) { return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream)); } @@ -153,7 +180,7 @@ public static boolean isFileForbiddenInConfigset(InputStream fileStream) { public static boolean isFileForbiddenInConfigset(byte[] bytes) { if (bytes == null || bytes.length == 0) return false; // A ZK znode may be a folder with no content - return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes)); + return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)); } private static final Set forbiddenTypes = @@ -163,4 +190,12 @@ public static boolean isFileForbiddenInConfigset(byte[] bytes) { "solr.configset.upload.mimetypes.forbidden", "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript") .split(","))); + + private String guessTypeFallbackToOctetStream(ContentInfo contentInfo) { + if (contentInfo == null) { + return ContentType.OTHER.getMimeType(); + } else { + return contentInfo.getContentType().getMimeType(); + } + } } diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java index b8e9a35a3d9..5c02528a7f4 100644 --- a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java +++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java @@ -17,38 +17,40 @@ package org.apache.solr.util; +import java.io.IOException; +import java.io.InputStream; import org.apache.solr.SolrTestCaseJ4; public class FileTypeMagicUtilTest extends SolrTestCaseJ4 { - public void testGuessMimeType() { - assertEquals( - "application/x-java-applet", - FileTypeMagicUtil.INSTANCE.guessMimeType( - FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin"))); - assertEquals( - "application/zip", - FileTypeMagicUtil.INSTANCE.guessMimeType( - FileTypeMagicUtil.class.getResourceAsStream( - "/runtimecode/containerplugin.v.1.jar.bin"))); - assertEquals( - "application/x-tar", - FileTypeMagicUtil.INSTANCE.guessMimeType( - FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin"))); - assertEquals( - "text/x-shellscript", - FileTypeMagicUtil.INSTANCE.guessMimeType( - FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt"))); + public void testGuessMimeType() throws IOException { + assertResourceMimeType("application/x-java-applet", "/magic/HelloWorldJavaClass.class.bin"); + assertResourceMimeType("application/zip", "/runtimecode/containerplugin.v.1.jar.bin"); + assertResourceMimeType("application/x-tar", "/magic/hello.tar.bin"); + assertResourceMimeType("text/x-shellscript", "/magic/shell.sh.txt"); } - public void testIsFileForbiddenInConfigset() { - assertTrue( - FileTypeMagicUtil.isFileForbiddenInConfigset( - FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin"))); - assertTrue( - FileTypeMagicUtil.isFileForbiddenInConfigset( - FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt"))); - assertFalse( - FileTypeMagicUtil.isFileForbiddenInConfigset( - FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt"))); + public void testIsFileForbiddenInConfigset() throws IOException { + assertResourceForbiddenInConfigset("/magic/HelloWorldJavaClass.class.bin"); + assertResourceForbiddenInConfigset("/magic/shell.sh.txt"); + assertResourceAllowedInConfigset("/magic/plain.txt"); + } + + private void assertResourceMimeType(String mimeType, String resourcePath) throws IOException { + try (InputStream stream = + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")) { + assertEquals("application/x-java-applet", FileTypeMagicUtil.INSTANCE.guessMimeType(stream)); + } + } + + private void assertResourceForbiddenInConfigset(String resourcePath) throws IOException { + try (InputStream stream = FileTypeMagicUtil.class.getResourceAsStream(resourcePath)) { + assertTrue(FileTypeMagicUtil.isFileForbiddenInConfigset(stream)); + } + } + + private void assertResourceAllowedInConfigset(String resourcePath) throws IOException { + try (InputStream stream = FileTypeMagicUtil.class.getResourceAsStream(resourcePath)) { + assertFalse(FileTypeMagicUtil.isFileForbiddenInConfigset(stream)); + } } } From 602edbd7ce8b0d6830d251d4044e62fd3c19a06d Mon Sep 17 00:00:00 2001 From: James Dyer Date: Mon, 18 Dec 2023 09:26:48 -0600 Subject: [PATCH 15/21] Copy the file from the "solrj" module (#2158) --- .../solrj-streaming/src/test-files/log4j2.xml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 solr/solrj-streaming/src/test-files/log4j2.xml diff --git a/solr/solrj-streaming/src/test-files/log4j2.xml b/solr/solrj-streaming/src/test-files/log4j2.xml new file mode 100644 index 00000000000..96f69f1dc8b --- /dev/null +++ b/solr/solrj-streaming/src/test-files/log4j2.xml @@ -0,0 +1,42 @@ + + + + + + + + + %maxLen{%-4r %-5p (%t) [%notEmpty{n:%X{node_name}}%notEmpty{ c:%X{collection}}%notEmpty{ s:%X{shard}}%notEmpty{ r:%X{replica}}%notEmpty{ x:%X{core}}%notEmpty{ t:%X{trace_id}}] %c{1.} %m%notEmpty{ + =>%ex{short}}}{10240}%n + + + + + + + + + + + + + + + + + From 5e87f8ecef5f890f9bacc1377df66b15eec9bf4e Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Tue, 19 Dec 2023 02:57:45 -0500 Subject: [PATCH 16/21] Update outdated link in PULL_REQUEST_TEMPLATE.md (#2163) The previous link was a soft redirect to the github page, so replace it. --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 84d3023d91f..960021a380c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -35,7 +35,7 @@ Please describe the tests you've developed or run to confirm this patch implemen Please review the following and check all that apply: -- [ ] I have reviewed the guidelines for [How to Contribute](https://wiki.apache.org/solr/HowToContribute) and my code conforms to the standards described there to the best of my ability. +- [ ] I have reviewed the guidelines for [How to Contribute](https://github.com/apache/solr/blob/main/CONTRIBUTING.md) and my code conforms to the standards described there to the best of my ability. - [ ] I have created a Jira issue and added the issue ID to my pull request title. - [ ] I have given Solr maintainers [access](https://help.github.com/en/articles/allowing-changes-to-a-pull-request-branch-created-from-a-fork) to contribute to my PR branch. (optional but recommended) - [ ] I have developed this patch against the `main` branch. From 5876c530c6e3e86891a36432acd29c9e64cdc774 Mon Sep 17 00:00:00 2001 From: Alex D Date: Tue, 19 Dec 2023 12:48:36 -0800 Subject: [PATCH 17/21] SOLR-17060 CoreContainer#create may deadlock with concurrent requests for metrics (#2101) --- solr/CHANGES.txt | 2 + .../java/org/apache/solr/core/SolrCore.java | 17 ++++-- .../org/apache/solr/core/SolrCoreTest.java | 60 +++++++++++++++++++ .../handler/admin/MetricsHandlerTest.java | 4 ++ 4 files changed, 78 insertions(+), 5 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 6f8e0477bf9..a6b21ee9fb3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -156,6 +156,8 @@ Bug Fixes * SOLR-17057: JSON Query regression: If "query" is specified with a String (not JSON structure), "defType" should parse it. Since 9.4 defType was ignored. (David Smiley) +* SOLR-17060: CoreContainer#create may deadlock with concurrent requests for metrics (Alex Deparvu, David Smiley) + Dependency Upgrades --------------------- * SOLR-17012: Update Apache Hadoop to 3.3.6 and Apache Curator to 5.5.0 (Kevin Risden) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 5f22e95b3fc..be6f7526b3d 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -259,6 +259,7 @@ public class SolrCore implements SolrInfoBean, Closeable { private Counter newSearcherCounter; private Counter newSearcherMaxReachedCounter; private Counter newSearcherOtherErrorsCounter; + private volatile boolean newSearcherReady = false; private final String metricTag = SolrMetricProducer.getUniqueMetricTag(this, null); private final SolrMetricsContext solrMetricsContext; @@ -1338,14 +1339,14 @@ public void initializeMetrics(SolrMetricsContext parentContext, String scope) { "sizeInBytes", Category.INDEX.toString()); parentContext.gauge( - () -> isClosed() ? parentContext.nullNumber() : getSegmentCount(), + () -> isClosed() ? parentContext.nullString() : NumberUtils.readableSize(getIndexSize()), true, - "segments", + "size", Category.INDEX.toString()); parentContext.gauge( - () -> isClosed() ? parentContext.nullString() : NumberUtils.readableSize(getIndexSize()), + () -> isReady() ? getSegmentCount() : parentContext.nullNumber(), true, - "size", + "segments", Category.INDEX.toString()); final CloudDescriptor cd = getCoreDescriptor().getCloudDescriptor(); @@ -1899,6 +1900,11 @@ public boolean isClosed() { return refCount.get() <= 0; } + /** Returns true if the core is ready for use. It is not initializing or closing/closed. */ + public boolean isReady() { + return !isClosed() && newSearcherReady; + } + private Collection closeHooks = null; /** Add a close callback hook */ @@ -2107,6 +2113,7 @@ public RefCounted getSearcher() { * because it might be closed soon after this method returns; it really depends. */ public R withSearcher(IOFunction lambda) throws IOException { + assert isReady(); final RefCounted refCounted = getSearcher(); try { return lambda.apply(refCounted.get()); @@ -2704,7 +2711,6 @@ public RefCounted getSearcher( if (!success) { newSearcherOtherErrorsCounter.inc(); - ; synchronized (searcherLock) { onDeckSearchers--; @@ -2734,6 +2740,7 @@ public RefCounted getSearcher( // we want to do this after we decrement onDeckSearchers so another thread // doesn't increment first and throw a false warning. openSearcherLock.unlock(); + newSearcherReady = true; } } diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java index e401bcc695a..28cfbc70cae 100644 --- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java +++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java @@ -16,6 +16,8 @@ */ package org.apache.solr.core; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.MetricFilter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -24,6 +26,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.ExecutorUtil; @@ -32,6 +35,7 @@ import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.handler.component.QueryComponent; import org.apache.solr.handler.component.SpellCheckComponent; +import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -327,6 +331,62 @@ public void testReloadLeak() throws Exception { assertTrue(core.areAllSearcherReferencesEmpty()); } + /** + * Best effort attempt to recreate a deadlock between SolrCore initialization and Index metrics + * poll. + * + *

See https://issues.apache.org/jira/browse/SOLR-17060 + */ + @Test + public void testCoreInitDeadlockMetrics() throws Exception { + SolrMetricManager metricManager = h.getCoreContainer().getMetricManager(); + CoreContainer coreContainer = h.getCoreContainer(); + + String coreName = "tmpCore"; + AtomicBoolean created = new AtomicBoolean(false); + AtomicBoolean atLeastOnePoll = new AtomicBoolean(false); + + final ExecutorService executor = + ExecutorUtil.newMDCAwareFixedThreadPool( + 1, new SolrNamedThreadFactory("testCoreInitDeadlockMetrics")); + executor.submit( + () -> { + while (!created.get()) { + var metrics = + metricManager.getMetrics( + "solr.core." + coreName, + MetricFilter.startsWith(SolrInfoBean.Category.INDEX.toString())); + for (var m : metrics.values()) { + if (m instanceof Gauge) { + var v = ((Gauge) m).getValue(); + atLeastOnePoll.compareAndSet(false, v != null); + } + } + + try { + TimeUnit.MILLISECONDS.sleep(5); + } catch (InterruptedException e1) { + throw new RuntimeException(e1); + } + } + }); + + TimeUnit.MILLISECONDS.sleep(25); + try (var tmpCore = coreContainer.create(coreName, Map.of("configSet", "minimal"))) { + tmpCore.open(); + for (int i = 0; i < 10; i++) { + TimeUnit.MILLISECONDS.sleep(50); // to allow metrics to be checked at least once + if (atLeastOnePoll.get()) { + break; + } + } + } finally { + created.set(true); + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + assertTrue(atLeastOnePoll.get()); + } + private static class NewSearcherRunnable implements Runnable { private final SolrCore core; diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java index 1f4697ecc10..c2ccdb43baf 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java @@ -93,6 +93,10 @@ public void test() throws Exception { // response wasn't serialized, so we get here whatever MetricUtils produced instead of NamedList assertNotNull(((MapWriter) o)._get("count", null)); assertEquals(0L, ((MapWriter) nl.get("SEARCHER.new.errors"))._get("count", null)); + assertNotNull(nl.get("INDEX.segments")); // int gauge + assertTrue((int) ((MapWriter) nl.get("INDEX.segments"))._get("value", null) >= 0); + assertNotNull(nl.get("INDEX.sizeInBytes")); // long gauge + assertTrue((long) ((MapWriter) nl.get("INDEX.sizeInBytes"))._get("value", null) >= 0); nl = (NamedList) values.get("solr.node"); assertNotNull(nl.get("CONTAINER.cores.loaded")); // int gauge assertEquals(1, ((MapWriter) nl.get("CONTAINER.cores.loaded"))._get("value", null)); From 65920af20553d71b403a82fbf1e7b4bdf37cafd4 Mon Sep 17 00:00:00 2001 From: Zauberfisch Date: Wed, 20 Dec 2023 15:10:29 +0100 Subject: [PATCH 18/21] Update outdated link in docker-networking.adoc (#2164) --- .../modules/deployment-guide/pages/docker-networking.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/docker-networking.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/docker-networking.adoc index 463fde67a92..364f06b1042 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/docker-networking.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/docker-networking.adoc @@ -16,7 +16,7 @@ // specific language governing permissions and limitations // under the License. -_Note: this article dates from Jan 2016. While this approach would still work, in Jan 2019 this would typically done with Docker cluster and orchestration tools like Kubernetes. See for example https://lucidworks.com/2019/02/07/running-solr-on-kubernetes-part-1/[this blog post]._ +_Note: this article dates from Jan 2016. While this approach would still work, in Jan 2019 this would typically done with Docker cluster and orchestration tools like Kubernetes. See for example https://lucidworks.com/post/running-solr-on-kubernetes-part-1/[this blog post]._ In this example I'll create a cluster with 3 ZooKeeper nodes and 3 Solr nodes, distributed over 3 machines (trinity10, trinity20, trinity30). I'll use an overlay network, specify fixed IP addresses when creating containers, and I'll pass in explicit `/etc/hosts` entries to make sure they are available even when nodes are down. From e2bf1f434aad873fbb24c21d46ac00e888806d98 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 5 Dec 2023 14:35:25 -0500 Subject: [PATCH 19/21] SOLR-17098: Only use ZK ACLs for default ZK Host --- solr/CHANGES.txt | 3 + .../org/apache/solr/core/CoreContainer.java | 1 + .../pages/stream-decorator-reference.adoc | 1 + .../pages/stream-source-reference.adoc | 3 + .../solr/client/solrj/io/SolrClientCache.java | 45 +++++++++-- .../client/solrj/io/SolrClientCacheTest.java | 77 +++++++++++++++++++ .../impl/ZkClientClusterStateProvider.java | 11 ++- .../solr/common/cloud/SolrZkClient.java | 24 ++++-- .../solr/common/cloud/ZkStateReader.java | 15 +++- .../solrj/impl/CloudHttp2SolrClient.java | 10 ++- .../solrj/impl/CloudLegacySolrClient.java | 10 ++- .../solrj/impl/ClusterStateProvider.java | 6 +- solr/solrj/src/test-files/solrj/solr/solr.xml | 3 + .../solr/cloud/MiniSolrCloudCluster.java | 1 + 14 files changed, 190 insertions(+), 20 deletions(-) create mode 100644 solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a6b21ee9fb3..128d6e06839 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -158,6 +158,9 @@ Bug Fixes * SOLR-17060: CoreContainer#create may deadlock with concurrent requests for metrics (Alex Deparvu, David Smiley) +* SOLR-17098: ZK Credentials and ACLs are no longer sent to all ZK Servers when using Streaming Expressions. + They will only be used when sent to the default ZK Host. (Houston Putman, Jan Høydahl, David Smiley, Gus Heck, Qing Xu) + Dependency Upgrades --------------------- * SOLR-17012: Update Apache Hadoop to 3.3.6 and Apache Curator to 5.5.0 (Kevin Risden) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 6681e71c814..1514aafe2ee 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -829,6 +829,7 @@ private void loadInternal() { zkSys.initZooKeeper(this, cfg.getCloudConfig()); if (isZooKeeperAware()) { + solrClientCache.setDefaultZKHost(getZkController().getZkServerAddress()); // initialize ZkClient metrics zkSys.getZkMetricsProducer().initializeMetrics(solrMetricsContext, "zkClient"); pkiAuthenticationSecurityBuilder = diff --git a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc index 1c2d8afccdf..d5e25ba98fa 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc @@ -1187,6 +1187,7 @@ Worker collections can be empty collections that exist only to execute streaming * `StreamExpression`: Expression to send to the worker collection. * `workers`: Number of workers in the worker collection to send the expression to. * `zkHost`: (Optional) The ZooKeeper connect string where the worker collection resides. +Zookeeper Credentials and ACLs will only be included if the same ZkHost is used as the Solr instance that you are connecting to (the `chroot` can be different). * `sort`: The sort criteria for ordering tuples returned by the worker nodes. === parallel Syntax diff --git a/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc b/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc index 39352261a2f..a4213776f6b 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/stream-source-reference.adoc @@ -36,6 +36,7 @@ To read more about the `/export` handler requirements review the section xref:ex * `fl`: (Mandatory) The list of fields to return. * `sort`: (Mandatory) The sort criteria. * `zkHost`: Only needs to be defined if the collection being searched is found in a different zkHost than the local stream handler. +Zookeeper Credentials and ACLs will only be included if the same ZkHost is used as the Solr instance that you are connecting to (the `chroot` can be different). * `qt`: Specifies the query type, or request handler, to use. Set this to `/export` to work with large result sets. The default is `/select`. @@ -484,6 +485,7 @@ When used in parallel mode the partitionKeys parameter must be provided. * `fl`: (Mandatory) The list of fields to return. * `sort`: (Mandatory) The sort criteria. * `zkHost`: Only needs to be defined if the collection being searched is found in a different zkHost than the local stream handler. +Zookeeper Credentials and ACLs will only be included if the same ZkHost is used as the Solr instance that you are connecting to (the `chroot` can be different). * `partitionKeys`: Comma delimited list of keys to partition the search results by. To be used with the parallel function for parallelizing operations across worker nodes. See the xref:stream-decorator-reference.adoc#parallel[parallel] function for details. @@ -648,6 +650,7 @@ The checkpoints will be saved under this id. If not set, it defaults to the highest version in the index. Setting to 0 will process all records that match query in the index. * `zkHost`: (Optional) Only needs to be defined if the collection being searched is found in a different zkHost than the local stream handler. +Zookeeper Credentials and ACLs will only be included if the same ZkHost is used as the Solr instance that you are connecting to (the `chroot` can be different). === topic Syntax diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java index e56d1a55c13..915d9fbafc7 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/SolrClientCache.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.apache.http.client.HttpClient; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient; @@ -57,6 +58,7 @@ public class SolrClientCache implements Closeable { private final HttpClient apacheHttpClient; private final Http2SolrClient http2SolrClient; private final AtomicBoolean isClosed = new AtomicBoolean(false); + private final AtomicReference defaultZkHost = new AtomicReference<>(); public SolrClientCache() { this.apacheHttpClient = null; @@ -74,40 +76,71 @@ public SolrClientCache(Http2SolrClient http2SolrClient) { this.http2SolrClient = http2SolrClient; } + public void setDefaultZKHost(String zkHost) { + if (zkHost != null) { + zkHost = zkHost.split("/")[0]; + if (!zkHost.isEmpty()) { + defaultZkHost.set(zkHost); + } else { + defaultZkHost.set(null); + } + } + } + public synchronized CloudSolrClient getCloudSolrClient(String zkHost) { ensureOpen(); Objects.requireNonNull(zkHost, "ZooKeeper host cannot be null!"); if (solrClients.containsKey(zkHost)) { return (CloudSolrClient) solrClients.get(zkHost); } + // Can only use ZK ACLs if there is a default ZK Host, and the given ZK host contains that + // default. + // Basically the ZK ACLs are assumed to be only used for the default ZK host, + // thus we should only provide the ACLs to that Zookeeper instance. + String zkHostNoChroot = zkHost.split("/")[0]; + boolean canUseACLs = + Optional.ofNullable(defaultZkHost.get()).map(zkHostNoChroot::equals).orElse(false); final CloudSolrClient client; if (apacheHttpClient != null) { - client = newCloudLegacySolrClient(zkHost, apacheHttpClient); + client = newCloudLegacySolrClient(zkHost, apacheHttpClient, canUseACLs); } else { - client = newCloudHttp2SolrClient(zkHost, http2SolrClient); + client = newCloudHttp2SolrClient(zkHost, http2SolrClient, canUseACLs); } solrClients.put(zkHost, client); return client; } @Deprecated - private static CloudSolrClient newCloudLegacySolrClient(String zkHost, HttpClient httpClient) { + private static CloudSolrClient newCloudLegacySolrClient( + String zkHost, HttpClient httpClient, boolean canUseACLs) { final List hosts = List.of(zkHost); var builder = new CloudLegacySolrClient.Builder(hosts, Optional.empty()); + builder.canUseZkACLs(canUseACLs); adjustTimeouts(builder, httpClient); var client = builder.build(); - client.connect(); + try { + client.connect(); + } catch (Exception e) { + IOUtils.closeQuietly(client); + throw e; + } return client; } private static CloudHttp2SolrClient newCloudHttp2SolrClient( - String zkHost, Http2SolrClient http2SolrClient) { + String zkHost, Http2SolrClient http2SolrClient, boolean canUseACLs) { final List hosts = List.of(zkHost); var builder = new CloudHttp2SolrClient.Builder(hosts, Optional.empty()); + builder.canUseZkACLs(canUseACLs); // using internal builder to ensure the internal client gets closed builder = builder.withInternalClientBuilder(newHttp2SolrClientBuilder(null, http2SolrClient)); var client = builder.build(); - client.connect(); + try { + client.connect(); + } catch (Exception e) { + IOUtils.closeQuietly(client); + throw e; + } return client; } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java new file mode 100644 index 00000000000..1f7ee0cffbf --- /dev/null +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/SolrClientCacheTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.solrj.io; + +import java.util.Map; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.DigestZkACLProvider; +import org.apache.solr.common.cloud.DigestZkCredentialsProvider; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SolrClientCacheTest extends SolrCloudTestCase { + + private static final Map sysProps = + Map.of( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + VMParamsZkCredentialsInjector.class.getName(), + SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + DigestZkCredentialsProvider.class.getName(), + SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + DigestZkACLProvider.class.getName(), + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, "admin-user", + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, "pass", + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME, "read-user", + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME, "pass"); + + @BeforeClass + public static void before() throws Exception { + sysProps.forEach(System::setProperty); + configureCluster(1) + .formatZkServer(true) + .addConfig("config", getFile("solrj/solr/configsets/streaming/conf").toPath()) + .configure(); + } + + @AfterClass + public static void after() { + sysProps.keySet().forEach(System::clearProperty); + } + + @Test + public void testZkACLsNotUsedWithDifferentZkHost() { + try (SolrClientCache cache = new SolrClientCache()) { + // This ZK Host is fake, thus the ZK ACLs should not be used + cache.setDefaultZKHost("test:2181"); + expectThrows( + SolrException.class, () -> cache.getCloudSolrClient(zkClient().getZkServerAddress())); + } + } + + @Test + public void testZkACLsUsedWithDifferentChroot() { + try (SolrClientCache cache = new SolrClientCache()) { + // The same ZK Host is used, so the ZK ACLs should still be applied + cache.setDefaultZKHost(zkClient().getZkServerAddress() + "/random/chroot"); + cache.getCloudSolrClient(zkClient().getZkServerAddress()); + } + } +} diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java index 075c1e4d3de..36c5891da1e 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/ZkClientClusterStateProvider.java @@ -48,6 +48,7 @@ public class ZkClientClusterStateProvider volatile ZkStateReader zkStateReader; private boolean closeZkStateReader = true; private final String zkHost; + private final boolean canUseZkACLs; private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; @@ -65,14 +66,22 @@ public ZkClientClusterStateProvider(ZkStateReader zkStateReader) { this.zkStateReader = zkStateReader; this.closeZkStateReader = false; this.zkHost = null; + this.canUseZkACLs = true; } public ZkClientClusterStateProvider(Collection zkHosts, String chroot) { + this(zkHosts, chroot, true); + } + + public ZkClientClusterStateProvider( + Collection zkHosts, String chroot, boolean canUseZkACLs) { zkHost = buildZkHostString(zkHosts, chroot); + this.canUseZkACLs = canUseZkACLs; } public ZkClientClusterStateProvider(String zkHost) { this.zkHost = zkHost; + this.canUseZkACLs = true; } /** @@ -212,7 +221,7 @@ public ZkStateReader getZkStateReader() { if (zkStateReader == null) { ZkStateReader zk = null; try { - zk = new ZkStateReader(zkHost, zkClientTimeout, zkConnectTimeout); + zk = new ZkStateReader(zkHost, zkClientTimeout, zkConnectTimeout, canUseZkACLs); zk.createClusterStateWatchersAndUpdate(); log.info("Cluster at {} ready", zkHost); zkStateReader = zk; diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java index ddbd70d375f..4ae9d16f123 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SolrZkClient.java @@ -118,7 +118,8 @@ public SolrZkClient(Builder builder) { builder.zkACLProvider, builder.higherLevelIsClosed, builder.compressor, - builder.solrClassLoader); + builder.solrClassLoader, + builder.useDefaultCredsAndACLs); } private SolrZkClient( @@ -131,7 +132,8 @@ private SolrZkClient( ZkACLProvider zkACLProvider, IsClosed higherLevelIsClosed, Compressor compressor, - SolrClassLoader solrClassLoader) { + SolrClassLoader solrClassLoader, + boolean useDefaultCredsAndACLs) { if (zkServerAddress == null) { // only tests should create one without server address @@ -148,9 +150,14 @@ private SolrZkClient( this.solrClassLoader = solrClassLoader; if (!strat.hasZkCredentialsToAddAutomatically()) { - zkCredentialsInjector = createZkCredentialsInjector(); + zkCredentialsInjector = + useDefaultCredsAndACLs + ? createZkCredentialsInjector() + : new DefaultZkCredentialsInjector(); ZkCredentialsProvider zkCredentialsToAddAutomatically = - createZkCredentialsToAddAutomatically(); + useDefaultCredsAndACLs + ? createZkCredentialsToAddAutomatically() + : new DefaultZkCredentialsProvider(); strat.setZkCredentialsToAddAutomatically(zkCredentialsToAddAutomatically); } @@ -210,7 +217,8 @@ private SolrZkClient( } assert ObjectReleaseTracker.track(this); if (zkACLProvider == null) { - this.zkACLProvider = createZkACLProvider(); + this.zkACLProvider = + useDefaultCredsAndACLs ? createZkACLProvider() : new DefaultZkACLProvider(); } else { this.zkACLProvider = zkACLProvider; } @@ -1134,6 +1142,7 @@ public static class Builder { public ZkACLProvider zkACLProvider; public IsClosed higherLevelIsClosed; public SolrClassLoader solrClassLoader; + public boolean useDefaultCredsAndACLs = true; public Compressor compressor; @@ -1199,6 +1208,11 @@ public Builder withSolrClassLoader(SolrClassLoader solrClassLoader) { return this; } + public Builder withUseDefaultCredsAndACLs(boolean useDefaultCredsAndACLs) { + this.useDefaultCredsAndACLs = useDefaultCredsAndACLs; + return this; + } + public SolrZkClient build() { return new SolrZkClient(this); } diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java index a89b3f8dd60..715d9858195 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -405,11 +405,20 @@ public ZkStateReader(SolrZkClient zkClient, Runnable securityNodeListener) { } public ZkStateReader(String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout) { - this.zkClient = + this(zkServerAddress, zkClientTimeout, zkClientConnectTimeout, true); + } + + public ZkStateReader( + String zkServerAddress, + int zkClientTimeout, + int zkClientConnectTimeout, + boolean canUseZkACLs) { + SolrZkClient.Builder builder = new SolrZkClient.Builder() .withUrl(zkServerAddress) .withTimeout(zkClientTimeout, TimeUnit.MILLISECONDS) .withConnTimeOut(zkClientConnectTimeout, TimeUnit.MILLISECONDS) + .withUseDefaultCredsAndACLs(canUseZkACLs) .withReconnectListener( () -> { // on reconnect, reload cloud info @@ -425,8 +434,8 @@ public ZkStateReader(String zkServerAddress, int zkClientTimeout, int zkClientCo log.error("Interrupted", e); throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "Interrupted", e); } - }) - .build(); + }); + this.zkClient = builder.build(); this.closeClient = true; this.securityNodeWatcher = null; diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java index d32a8ecbec9..cc3861cc5b5 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudHttp2SolrClient.java @@ -138,6 +138,7 @@ public static class Builder { private int parallelCacheRefreshesLocks = 3; private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; + private boolean canUseZkACLs = true; /** * Provide a series of Solr URLs to be used when configuring {@link CloudHttp2SolrClient} @@ -189,6 +190,12 @@ public Builder(List zkHosts, Optional zkChroot) { if (zkChroot.isPresent()) this.zkChroot = zkChroot.get(); } + /** Whether or not to use the default ZK ACLs when building a ZK Client. */ + public Builder canUseZkACLs(boolean canUseZkACLs) { + this.canUseZkACLs = canUseZkACLs; + return this; + } + /** * Tells {@link Builder} that created clients should be configured such that {@link * CloudSolrClient#isUpdatesToLeaders} returns true. @@ -406,7 +413,8 @@ public CloudHttp2SolrClient build() { throw new IllegalArgumentException( "Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); } else if (!zkHosts.isEmpty()) { - stateProvider = ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot); + stateProvider = + ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot, canUseZkACLs); if (stateProvider instanceof SolrZkClientTimeoutAware) { var timeoutAware = (SolrZkClientTimeoutAware) stateProvider; timeoutAware.setZkClientTimeout(zkClientTimeout); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java index 3547c652293..e1f3840cfbd 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudLegacySolrClient.java @@ -172,6 +172,7 @@ public static class Builder extends SolrClientBuilder { protected ClusterStateProvider stateProvider; private int zkConnectTimeout = SolrZkClientTimeout.DEFAULT_ZK_CONNECT_TIMEOUT; private int zkClientTimeout = SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT; + private boolean canUseZkACLs = true; /** Constructor for use by subclasses. This constructor was public prior to version 9.0 */ protected Builder() {} @@ -231,6 +232,12 @@ public Builder(List zkHosts, Optional zkChroot) { if (zkChroot.isPresent()) this.zkChroot = zkChroot.get(); } + /** Whether or not to use the default ZK ACLs when building a ZK Client. */ + public Builder canUseZkACLs(boolean canUseZkACLs) { + this.canUseZkACLs = canUseZkACLs; + return this; + } + /** Provides a {@link HttpClient} for the builder to use when creating clients. */ public Builder withLBHttpSolrClientBuilder(LBHttpSolrClient.Builder lbHttpSolrClientBuilder) { this.lbClientBuilder = lbHttpSolrClientBuilder; @@ -371,7 +378,8 @@ public CloudLegacySolrClient build() { throw new IllegalArgumentException( "Both zkHost(s) & solrUrl(s) have been specified. Only specify one."); } else if (!zkHosts.isEmpty()) { - this.stateProvider = ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot); + this.stateProvider = + ClusterStateProvider.newZkClusterStateProvider(zkHosts, zkChroot, canUseZkACLs); if (stateProvider instanceof SolrZkClientTimeoutAware) { var timeoutAware = (SolrZkClientTimeoutAware) stateProvider; timeoutAware.setZkClientTimeout(zkClientTimeout); diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java index 9673f08e48d..e6b7f2097a4 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java @@ -31,14 +31,14 @@ public interface ClusterStateProvider extends SolrCloseable { static ClusterStateProvider newZkClusterStateProvider( - Collection zkHosts, String zkChroot) { + Collection zkHosts, String zkChroot, boolean canUseZkACLs) { // instantiate via reflection so that we don't depend on ZK try { var constructor = Class.forName("org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider") .asSubclass(ClusterStateProvider.class) - .getConstructor(Collection.class, String.class); - return constructor.newInstance(zkHosts, zkChroot); + .getConstructor(Collection.class, String.class, Boolean.TYPE); + return constructor.newInstance(zkHosts, zkChroot, canUseZkACLs); } catch (InvocationTargetException e) { if (e.getCause() instanceof RuntimeException) { throw (RuntimeException) e.getCause(); diff --git a/solr/solrj/src/test-files/solrj/solr/solr.xml b/solr/solrj/src/test-files/solrj/solr/solr.xml index c4413d38ce3..81e2e31ac8a 100644 --- a/solr/solrj/src/test-files/solrj/solr/solr.xml +++ b/solr/solrj/src/test-files/solrj/solr/solr.xml @@ -41,6 +41,9 @@ 0 ${distribUpdateConnTimeout:45000} ${distribUpdateSoTimeout:340000} + ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} + ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} + ${zkCredentialsInjector:org.apache.solr.common.cloud.DefaultZkCredentialsInjector} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java index a8ac4b1ac60..42879306f61 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java @@ -129,6 +129,7 @@ public class MiniSolrCloudCluster { + " ${leaderVoteWait:10000}\n" + " ${distribUpdateConnTimeout:45000}\n" + " ${distribUpdateSoTimeout:340000}\n" + + " ${zkCredentialsInjector:org.apache.solr.common.cloud.DefaultZkCredentialsInjector} \n" + " ${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider} \n" + " ${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider} \n" + " ${pkiHandlerPrivateKeyPath:" From 17f8ee445e5e86f9d93852232b832317784e685f Mon Sep 17 00:00:00 2001 From: Justin Sweeney Date: Fri, 22 Dec 2023 05:29:38 -0700 Subject: [PATCH 20/21] SOLR-17022: Support for glob patterns for fields in Export handler, Stream handler and with SelectStream streaming expression (#1996) * Adding support for Glob patterns in Export handler and Select stream handler using the same logic to match glob patterns to fields as is used in select requests --- .../solr/handler/export/ExportWriter.java | 77 +++++++++++-------- .../apache/solr/search/SolrReturnFields.java | 5 +- .../solr/handler/export/TestExportWriter.java | 37 +++++++++ .../pages/exporting-result-sets.adoc | 5 +- .../pages/stream-decorator-reference.adoc | 2 +- .../client/solrj/io/stream/SelectStream.java | 30 +++++++- .../StreamExpressionToExpessionTest.java | 3 +- .../solr/common/util/GlobPatternUtil.java | 37 +++++++++ .../solr/common/util/TestGlobPatternUtil.java | 33 ++++++++ 9 files changed, 187 insertions(+), 42 deletions(-) create mode 100644 solr/solrj/src/java/org/apache/solr/common/util/GlobPatternUtil.java create mode 100644 solr/solrj/src/test/org/apache/solr/common/util/TestGlobPatternUtil.java diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java index 51ba5551b69..e5e40e6d07e 100644 --- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java +++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java @@ -27,6 +27,7 @@ import java.io.PrintWriter; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeSet; @@ -76,6 +77,7 @@ import org.apache.solr.schema.StrField; import org.apache.solr.search.DocValuesIteratorCache; import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.SolrReturnFields; import org.apache.solr.search.SortSpec; import org.apache.solr.search.SyntaxError; import org.slf4j.Logger; @@ -121,7 +123,7 @@ public boolean write( private int priorityQueueSize; StreamExpression streamExpression; StreamContext streamContext; - FieldWriter[] fieldWriters; + List fieldWriters; int totalHits = 0; FixedBitSet[] sets = null; PushWriter writer; @@ -293,7 +295,7 @@ private void _write(OutputStream os) throws IOException { } try { - fieldWriters = getFieldWriters(fields, req.getSearcher()); + fieldWriters = getFieldWriters(fields, req); } catch (Exception e) { writeException(e, writer, true); return; @@ -473,7 +475,7 @@ void fillOutDocs(MergeIterator mergeIterator, ExportBuffers.Buffer buffer) throw } void writeDoc( - SortDoc sortDoc, List leaves, EntryWriter ew, FieldWriter[] writers) + SortDoc sortDoc, List leaves, EntryWriter ew, List writers) throws IOException { int ord = sortDoc.ord; LeafReaderContext context = leaves.get(ord); @@ -485,82 +487,89 @@ void writeDoc( } } - public FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) + public List getFieldWriters(String[] fields, SolrQueryRequest req) throws IOException { - IndexSchema schema = searcher.getSchema(); - FieldWriter[] writers = new FieldWriter[fields.length]; - DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(searcher, false); - for (int i = 0; i < fields.length; i++) { - String field = fields[i]; - SchemaField schemaField = null; + DocValuesIteratorCache dvIterCache = new DocValuesIteratorCache(req.getSearcher(), false); - try { - schemaField = schema.getField(field); - } catch (Exception e) { - throw new IOException(e); - } + SolrReturnFields solrReturnFields = new SolrReturnFields(fields, req); + List writers = new ArrayList<>(); + for (String field : req.getSearcher().getFieldNames()) { + if (!solrReturnFields.wantsField(field)) { + continue; + } + SchemaField schemaField = req.getSchema().getField(field); if (!schemaField.hasDocValues()) { throw new IOException(schemaField + " must have DocValues to use this feature."); } boolean multiValued = schemaField.multiValued(); FieldType fieldType = schemaField.getType(); - - if (fieldType instanceof SortableTextField && schemaField.useDocValuesAsStored() == false) { - throw new IOException( - schemaField + " Must have useDocValuesAsStored='true' to be used with export writer"); + FieldWriter writer; + + if (fieldType instanceof SortableTextField && !schemaField.useDocValuesAsStored()) { + if (solrReturnFields.getRequestedFieldNames() != null + && solrReturnFields.getRequestedFieldNames().contains(field)) { + // Explicitly requested field cannot be used due to not having useDocValuesAsStored=true, + // throw exception + throw new IOException( + schemaField + " Must have useDocValuesAsStored='true' to be used with export writer"); + } else { + // Glob pattern matched field cannot be used due to not having useDocValuesAsStored=true + continue; + } } DocValuesIteratorCache.FieldDocValuesSupplier docValuesCache = dvIterCache.getSupplier(field); if (docValuesCache == null) { - writers[i] = EMPTY_FIELD_WRITER; + writer = EMPTY_FIELD_WRITER; } else if (fieldType instanceof IntValueFieldType) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); } else { - writers[i] = new IntFieldWriter(field, docValuesCache); + writer = new IntFieldWriter(field, docValuesCache); } } else if (fieldType instanceof LongValueFieldType) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); } else { - writers[i] = new LongFieldWriter(field, docValuesCache); + writer = new LongFieldWriter(field, docValuesCache); } } else if (fieldType instanceof FloatValueFieldType) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); } else { - writers[i] = new FloatFieldWriter(field, docValuesCache); + writer = new FloatFieldWriter(field, docValuesCache); } } else if (fieldType instanceof DoubleValueFieldType) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); } else { - writers[i] = new DoubleFieldWriter(field, docValuesCache); + writer = new DoubleFieldWriter(field, docValuesCache); } } else if (fieldType instanceof StrField || fieldType instanceof SortableTextField) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, false, docValuesCache); } else { - writers[i] = new StringFieldWriter(field, fieldType, docValuesCache); + writer = new StringFieldWriter(field, fieldType, docValuesCache); } } else if (fieldType instanceof DateValueFieldType) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, false, docValuesCache); } else { - writers[i] = new DateFieldWriter(field, docValuesCache); + writer = new DateFieldWriter(field, docValuesCache); } } else if (fieldType instanceof BoolField) { if (multiValued) { - writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); + writer = new MultiFieldWriter(field, fieldType, schemaField, true, docValuesCache); } else { - writers[i] = new BoolFieldWriter(field, fieldType, docValuesCache); + writer = new BoolFieldWriter(field, fieldType, docValuesCache); } } else { throw new IOException( "Export fields must be one of the following types: int,float,long,double,string,date,boolean,SortableText"); } + writers.add(writer); } return writers; } diff --git a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java index 7d0583ce63a..af35245af15 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java +++ b/solr/core/src/java/org/apache/solr/search/SolrReturnFields.java @@ -28,7 +28,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Supplier; -import org.apache.commons.io.FilenameUtils; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.QueryValueSource; @@ -37,6 +36,7 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.GlobPatternUtil; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.response.transform.DocTransformers; @@ -577,8 +577,7 @@ public boolean wantsField(String name) { return true; } for (String s : globs) { - // TODO something better? - if (FilenameUtils.wildcardMatch(name, s)) { + if (GlobPatternUtil.matches(s, name)) { okFieldNames.add(name); // Don't calculate it again return true; } diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java index 8337609faf9..e37f26efc94 100644 --- a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java +++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java @@ -1298,6 +1298,43 @@ public void testExpr() throws Exception { .contains("Must have useDocValuesAsStored='true'")); } + @Test + public void testGlobFields() throws Exception { + assertU(delQ("*:*")); + assertU(commit()); + createLargeIndex(); + SolrQueryRequest req = + req("q", "*:*", "qt", "/export", "fl", "id,*_udvas,*_i_p", "sort", "id asc"); + assertJQ( + req, + "response/numFound==100000", + "response/docs/[0]/id=='0'", + "response/docs/[1]/id=='1'", + "response/docs/[0]/sortabledv_udvas=='0'", + "response/docs/[1]/sortabledv_udvas=='1'", + "response/docs/[0]/small_i_p==0", + "response/docs/[1]/small_i_p==1"); + + assertU(delQ("*:*")); + assertU(commit()); + createLargeIndex(); + req = req("q", "*:*", "qt", "/export", "fl", "*", "sort", "id asc"); + assertJQ( + req, + "response/numFound==100000", + "response/docs/[0]/id=='0'", + "response/docs/[1]/id=='1'", + "response/docs/[0]/sortabledv_udvas=='0'", + "response/docs/[1]/sortabledv_udvas=='1'", + "response/docs/[0]/small_i_p==0", + "response/docs/[1]/small_i_p==1"); + + String jq = JQ(req); + assertFalse( + "Fields without docvalues and useDocValuesAsStored should not be returned", + jq.contains("\"sortabledv\"")); + } + @SuppressWarnings("rawtypes") private void validateSort(int numDocs) throws Exception { // 10 fields diff --git a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc index 28c395daa46..bbd31c7b358 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/exporting-result-sets.adoc @@ -70,7 +70,10 @@ It can get worse otherwise. The `fl` property defines the fields that will be exported with the result set. Any of the field types that can be sorted (i.e., int, long, float, double, string, date, boolean) can be used in the field list. The fields can be single or multi-valued. -However, returning scores and wildcards are not supported at this time. + +Wildcard patterns can be used for the field list (e.g. `fl=*_i`) and will be expanded to the list of fields that match the pattern and are able to be exported, see <>. + +Returning scores is not supported at this time. === Specifying the Local Streaming Expression diff --git a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc index d5e25ba98fa..28a570ffae4 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/stream-decorator-reference.adoc @@ -1376,7 +1376,7 @@ One can provide a list of operations and evaluators to perform on any fields, su === select Parameters * `StreamExpression` -* `fieldName`: name of field to include in the output tuple (can include multiple of these), such as `outputTuple[fieldName] = inputTuple[fieldName]` +* `fieldName`: name of field to include in the output tuple (can include multiple of these), such as `outputTuple[fieldName] = inputTuple[fieldName]`. The `fieldName` can be a wildcard pattern, e.g. `a_*` to select all fields that start with `a_`. * `fieldName as aliasFieldName`: aliased field name to include in the output tuple (can include multiple of these), such as `outputTuple[aliasFieldName] = incomingTuple[fieldName]` * `replace(fieldName, value, withValue=replacementValue)`: if `incomingTuple[fieldName] == value` then `outgoingTuple[fieldName]` will be set to `replacementValue`. `value` can be the string "null" to replace a null value with some other value. diff --git a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java index 80219e797bb..647a1c59d4c 100644 --- a/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java +++ b/solr/solrj-streaming/src/java/org/apache/solr/client/solrj/io/stream/SelectStream.java @@ -38,6 +38,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionParser; import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionValue; import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; +import org.apache.solr.common.util.GlobPatternUtil; /** * Selects fields from the incoming stream and applies optional field renaming. Does not reorder the @@ -52,14 +53,21 @@ public class SelectStream extends TupleStream implements Expressible { private TupleStream stream; private StreamContext streamContext; private Map selectedFields; + private List selectedFieldGlobPatterns; private Map selectedEvaluators; private List operations; public SelectStream(TupleStream stream, List selectedFields) throws IOException { this.stream = stream; this.selectedFields = new HashMap<>(); + this.selectedFieldGlobPatterns = new ArrayList<>(); for (String selectedField : selectedFields) { - this.selectedFields.put(selectedField, selectedField); + if (selectedField.contains("*")) { + // selected field is a glob pattern + this.selectedFieldGlobPatterns.add(selectedField); + } else { + this.selectedFields.put(selectedField, selectedField); + } } operations = new ArrayList<>(); selectedEvaluators = new LinkedHashMap<>(); @@ -68,6 +76,7 @@ public SelectStream(TupleStream stream, List selectedFields) throws IOEx public SelectStream(TupleStream stream, Map selectedFields) throws IOException { this.stream = stream; this.selectedFields = selectedFields; + selectedFieldGlobPatterns = new ArrayList<>(); operations = new ArrayList<>(); selectedEvaluators = new LinkedHashMap<>(); } @@ -123,6 +132,7 @@ public SelectStream(StreamExpression expression, StreamFactory factory) throws I stream = factory.constructStream(streamExpressions.get(0)); selectedFields = new HashMap<>(); + selectedFieldGlobPatterns = new ArrayList<>(); selectedEvaluators = new LinkedHashMap<>(); for (StreamExpressionParameter parameter : selectAsFieldsExpressions) { StreamExpressionValue selectField = (StreamExpressionValue) parameter; @@ -175,7 +185,11 @@ public SelectStream(StreamExpression expression, StreamFactory factory) throws I selectedFields.put(asValue, asName); } } else { - selectedFields.put(value, value); + if (value.contains("*")) { + selectedFieldGlobPatterns.add(value); + } else { + selectedFields.put(value, value); + } } } @@ -217,6 +231,11 @@ private StreamExpression toExpression(StreamFactory factory, boolean includeStre } } + // selected glob patterns + for (String selectFieldGlobPattern : selectedFieldGlobPatterns) { + expression.addParameter(selectFieldGlobPattern); + } + // selected evaluators for (Map.Entry selectedEvaluator : selectedEvaluators.entrySet()) { expression.addParameter( @@ -308,6 +327,13 @@ public Tuple read() throws IOException { workingForEvaluators.put(fieldName, original.get(fieldName)); if (selectedFields.containsKey(fieldName)) { workingToReturn.put(selectedFields.get(fieldName), original.get(fieldName)); + } else { + for (String globPattern : selectedFieldGlobPatterns) { + if (GlobPatternUtil.matches(globPattern, fieldName)) { + workingToReturn.put(fieldName, original.get(fieldName)); + break; + } + } } } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java index 4069b671b32..2c941f142d7 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionToExpessionTest.java @@ -105,7 +105,7 @@ public void testSelectStream() throws Exception { try (SelectStream stream = new SelectStream( StreamExpressionParser.parse( - "select(\"a_s as fieldA\", search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"))"), + "select(\"a_s as fieldA\", a_*, search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\"))"), factory)) { expressionString = stream.toExpression(factory).toString(); assertTrue(expressionString.contains("select(search(collection1,")); @@ -113,6 +113,7 @@ public void testSelectStream() throws Exception { assertTrue(expressionString.contains("fl=\"id,a_s,a_i,a_f\"")); assertTrue(expressionString.contains("sort=\"a_f asc, a_i asc\"")); assertTrue(expressionString.contains("a_s as fieldA")); + assertTrue(expressionString.contains("a_*")); } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/GlobPatternUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/GlobPatternUtil.java new file mode 100644 index 00000000000..8b26ab5a355 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/common/util/GlobPatternUtil.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common.util; + +import java.nio.file.FileSystems; +import java.nio.file.Paths; + +/** Provides methods for matching glob patterns against input strings. */ +public class GlobPatternUtil { + + /** + * Matches an input string against a provided glob patterns. This uses Java NIO FileSystems + * PathMatcher to match glob patterns in the same way to how glob patterns are matches for file + * paths, rather than implementing our own glob pattern matching. + * + * @param pattern the glob pattern to match against + * @param input the input string to match against a glob pattern + * @return true if the input string matches the glob pattern, false otherwise + */ + public static boolean matches(String pattern, String input) { + return FileSystems.getDefault().getPathMatcher("glob:" + pattern).matches(Paths.get(input)); + } +} diff --git a/solr/solrj/src/test/org/apache/solr/common/util/TestGlobPatternUtil.java b/solr/solrj/src/test/org/apache/solr/common/util/TestGlobPatternUtil.java new file mode 100644 index 00000000000..a5bdcad92fa --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/common/util/TestGlobPatternUtil.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common.util; + +import org.apache.solr.SolrTestCase; + +public class TestGlobPatternUtil extends SolrTestCase { + + public void testMatches() { + assertTrue(GlobPatternUtil.matches("*_str", "user_str")); + assertFalse(GlobPatternUtil.matches("*_str", "str_user")); + assertTrue(GlobPatternUtil.matches("str_*", "str_user")); + assertFalse(GlobPatternUtil.matches("str_*", "user_str")); + assertTrue(GlobPatternUtil.matches("str?", "str1")); + assertFalse(GlobPatternUtil.matches("str?", "str_user")); + assertTrue(GlobPatternUtil.matches("user_*_str", "user_type_str")); + assertFalse(GlobPatternUtil.matches("user_*_str", "user_str")); + } +} From a04ec07eadf72a5957d1f159f77f36a2c7b990ee Mon Sep 17 00:00:00 2001 From: Vincent P Date: Sat, 23 Dec 2023 07:15:05 +0100 Subject: [PATCH 21/21] SOLR-17094: Close ObjectCache when closing CoreContainer (#2166) Co-authored-by: Vincent Primault --- solr/core/src/java/org/apache/solr/core/CoreContainer.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 1514aafe2ee..ce5e05b2657 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -1299,7 +1299,11 @@ public void shutdown() { } } - objectCache.clear(); + try { + objectCache.close(); + } catch (IOException e) { + log.warn("Exception while closing ObjectCache.", e); + } // It's still possible that one of the pending dynamic load operation is waiting, so wake it // up if so. Since all the pending operations queues have been drained, there should be