Bump to Lucene99

CHANGELOG.md

@@ -153,6 +153,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.wiremock:wiremock-standalone` from 3.1.0 to 3.3.1 ([#11555](https://github.com/opensearch-project/OpenSearch/pull/11555))
 - Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.25.0 ([#11556](https://github.com/opensearch-project/OpenSearch/pull/11556))
 - Bump `actions/stale` from 8 to 9 ([#11557](https://github.com/opensearch-project/OpenSearch/pull/11557))
+- Bump `Lucene` from 9.8.0 to 9.9.0 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.8.0
+lucene            = 9.9.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.1+12

libs/core/licenses/lucene-core-9.9.0.jar.sha1

@@ -0,0 +1 @@
+2f0876192ee21f2b6f85b9435c99b9f0e04f3fcc

libs/core/src/main/java/org/opensearch/Version.java

@@ -99,7 +99,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_8_0);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_8_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

modules/lang-expression/licenses/lucene-expressions-9.9.0.jar.sha1

@@ -0,0 +1 @@
+a36fc5b5087b704852a5f228c38cb3a47fea8c36

modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldTypeTests.java

@@ -39,6 +39,7 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
@@ -122,35 +123,40 @@ public void testRangeQuery() throws IOException {
     public void testRoundsUpperBoundCorrectly() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
         Query scaledFloatQ = ft.rangeQuery(null, 0.1, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.1, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.095, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.095, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 9]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 9]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.105, true, false, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 0.105, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 10]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 10]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(null, 79.99, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9223372036854775808 TO 7999]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9223372036854775808 TO 7999]", getQueryString(scaledFloatQ));
     }
 
     public void testRoundsLowerBoundCorrectly() {
         ScaledFloatFieldMapper.ScaledFloatFieldType ft = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float", 100);
         Query scaledFloatQ = ft.rangeQuery(-0.1, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.1, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.095, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.095, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-9 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-9 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.105, null, false, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
         scaledFloatQ = ft.rangeQuery(-0.105, null, true, true, MOCK_QSC);
-        assertEquals("scaled_float:[-10 TO 9223372036854775807]", scaledFloatQ.toString());
+        assertEquals("scaled_float:[-10 TO 9223372036854775807]", getQueryString(scaledFloatQ));
+    }
+
+    private String getQueryString(Query query) {
+        assertTrue(query instanceof IndexOrDocValuesQuery);
+        return ((IndexOrDocValuesQuery) query).getIndexQuery().toString();
     }
 
     public void testValueForSearch() {

modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpPipeliningHandlerTests.java

@@ -79,7 +79,7 @@ public class Netty4HttpPipeliningHandlerTests extends OpenSearchTestCase {
     @After
     public void tearDown() throws Exception {
         waitingRequests.keySet().forEach(this::finishRequest);
-        shutdownExecutorService();
+        shutdownExecutorServices();
         super.tearDown();
     }
 
@@ -88,7 +88,7 @@ private CountDownLatch finishRequest(String url) {
         return finishingRequests.get(url);
     }
 
-    private void shutdownExecutorService() throws InterruptedException {
+    private void shutdownExecutorServices() throws InterruptedException {
         if (!handlerService.isShutdown()) {
             handlerService.shutdown();
             handlerService.awaitTermination(10, TimeUnit.SECONDS);

plugins/analysis-icu/licenses/lucene-analysis-icu-9.9.0.jar.sha1

@@ -0,0 +1 @@
+fa8ddcddd8ad086d123dbcf1b2804683a14f252f

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.9.0.jar.sha1

@@ -0,0 +1 @@
+70903a94800e3f09898f279987a591e1443e7f11

plugins/analysis-kuromoji/src/main/java/org/opensearch/index/analysis/KuromojiTokenizerFactory.java

@@ -38,7 +38,7 @@
 import org.apache.lucene.analysis.ja.JapaneseTokenizer;
 import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
 import org.apache.lucene.analysis.ja.dict.UserDictionary;
-import org.apache.lucene.analysis.ja.util.CSVUtil;
+import org.apache.lucene.analysis.util.CSVUtil;
 import org.opensearch.OpenSearchException;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.env.Environment;

plugins/analysis-nori/licenses/lucene-analysis-nori-9.9.0.jar.sha1

@@ -0,0 +1 @@
+521d3e833ffb371dd4712b8241239cae031507e5

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.9.0.jar.sha1

@@ -0,0 +1 @@
+40d1fb2e36efc071d1b96958f23bfe3d5e3d5758

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.9.0.jar.sha1

@@ -0,0 +1 @@
+4a4f7ba40b82f108a2c3d150624cb46f8281d212

plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.9.0.jar.sha1

@@ -0,0 +1 @@
+067dd0df852c5df001d77505dd048a4e90a4b365

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.9.0.jar.sha1

@@ -0,0 +1 @@
+ac5225e8a5fa4e1d9784b46f64942b75b2716b26

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/CorrelationCodecVersion.java

@@ -9,7 +9,7 @@
 package org.opensearch.plugin.correlation.core.index.codec;
 
 import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.correlation950.CorrelationCodec;
 import org.opensearch.plugin.correlation.core.index.codec.correlation950.PerFieldCorrelationVectorsFormat;
@@ -24,15 +24,15 @@
  * @opensearch.internal
  */
 public enum CorrelationCodecVersion {
-    V_9_5_0(
+    V_9_9_0(
         "CorrelationCodec",
-        new Lucene95Codec(),
+        new Lucene99Codec(),
         new PerFieldCorrelationVectorsFormat(Optional.empty()),
         (userCodec, mapperService) -> new CorrelationCodec(userCodec, new PerFieldCorrelationVectorsFormat(Optional.of(mapperService))),
         CorrelationCodec::new
     );
 
-    private static final CorrelationCodecVersion CURRENT = V_9_5_0;
+    private static final CorrelationCodecVersion CURRENT = V_9_9_0;
     private final String codecName;
     private final Codec defaultCodecDelegate;
     private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat;

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodec.java

@@ -19,7 +19,7 @@
  * @opensearch.internal
  */
 public class CorrelationCodec extends FilterCodec {
-    private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_5_0;
+    private static final CorrelationCodecVersion VERSION = CorrelationCodecVersion.V_9_9_0;
     private final PerFieldCorrelationVectorsFormat perFieldCorrelationVectorsFormat;
 
     /**

plugins/events-correlation-engine/src/main/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/PerFieldCorrelationVectorsFormat.java

@@ -8,7 +8,7 @@
 
 package org.opensearch.plugin.correlation.core.index.codec.correlation950;
 
-import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat;
 
@@ -26,10 +26,10 @@ public class PerFieldCorrelationVectorsFormat extends BasePerFieldCorrelationVec
     public PerFieldCorrelationVectorsFormat(final Optional<MapperService> mapperService) {
         super(
             mapperService,
-            Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN,
-            Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
-            Lucene95HnswVectorsFormat::new,
-            Lucene95HnswVectorsFormat::new
+            Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN,
+            Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH,
+            Lucene99HnswVectorsFormat::new,
+            Lucene99HnswVectorsFormat::new
         );
     }
 }

plugins/events-correlation-engine/src/test/java/org/opensearch/plugin/correlation/core/index/codec/correlation950/CorrelationCodecTests.java

@@ -32,7 +32,7 @@
 
 import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_EF_CONSTRUCTION;
 import static org.opensearch.plugin.correlation.core.index.codec.BasePerFieldCorrelationVectorsFormat.METHOD_PARAMETER_M;
-import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_5_0;
+import static org.opensearch.plugin.correlation.core.index.codec.CorrelationCodecVersion.V_9_9_0;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -56,7 +56,7 @@ public void testCorrelationVectorIndex() throws Exception {
         Function<MapperService, PerFieldCorrelationVectorsFormat> perFieldCorrelationVectorsProvider =
             mapperService -> new PerFieldCorrelationVectorsFormat(Optional.of(mapperService));
         Function<PerFieldCorrelationVectorsFormat, Codec> correlationCodecProvider = (correlationVectorsFormat -> new CorrelationCodec(
-            V_9_5_0.getDefaultCodecDelegate(),
+            V_9_9_0.getDefaultCodecDelegate(),
             correlationVectorsFormat
         ));
         testCorrelationVectorIndex(correlationCodecProvider, perFieldCorrelationVectorsProvider);

plugins/transport-nio/src/test/java/org/opensearch/http/nio/NioHttpPipeliningHandlerTests.java

@@ -80,15 +80,15 @@ public class NioHttpPipeliningHandlerTests extends OpenSearchTestCase {
     @After
     public void cleanup() throws Exception {
         waitingRequests.keySet().forEach(this::finishRequest);
-        shutdownExecutorService();
+        shutdownExecutorServices();
     }
 
     private CountDownLatch finishRequest(String url) {
         waitingRequests.get(url).countDown();
         return finishingRequests.get(url);
     }
 
-    private void shutdownExecutorService() throws InterruptedException {
+    private void shutdownExecutorServices() throws InterruptedException {
         if (!handlerService.isShutdown()) {
             handlerService.shutdown();
             handlerService.awaitTermination(10, TimeUnit.SECONDS);

server/licenses/lucene-analysis-common-9.9.0.jar.sha1

@@ -0,0 +1 @@
+445448c09a0c8ba40f81df7d86ac97303ce0b19e

server/licenses/lucene-backward-codecs-9.9.0.jar.sha1

@@ -0,0 +1 @@
+fd7a9d02972955bb85896ded80e9bf5ffa173b2b

server/licenses/lucene-core-9.9.0.jar.sha1

@@ -0,0 +1 @@
+2f0876192ee21f2b6f85b9435c99b9f0e04f3fcc

server/licenses/lucene-grouping-9.9.0.jar.sha1

@@ -0,0 +1 @@
+becfa739de2d3e29c129ed0347890a2dcab24b9b

server/licenses/lucene-highlighter-9.9.0.jar.sha1

@@ -0,0 +1 @@
+fc7392800a3baff95efa592b1dca0df047e862e5

server/licenses/lucene-join-9.9.0.jar.sha1

@@ -0,0 +1 @@
+58e2ff4e9dbb7bad804f08a1f482f08b72676421

server/licenses/lucene-memory-9.9.0.jar.sha1

@@ -0,0 +1 @@
+266c6dc593217f606e06903d15d20aaa502d3ad2

server/licenses/lucene-misc-9.9.0.jar.sha1

@@ -0,0 +1 @@
+873e4beb02f028a41c981a86e04c6d86a4e4cd68

server/licenses/lucene-queries-9.9.0.jar.sha1

@@ -0,0 +1 @@
+298606babedbd3a67b7539fd1f84ccca7140c4de

server/licenses/lucene-queryparser-9.9.0.jar.sha1

@@ -0,0 +1 @@
+1968cd1b3ccaff22d30ffa5e5e54ec51b4a4e66e

server/licenses/lucene-sandbox-9.9.0.jar.sha1

@@ -0,0 +1 @@
+25472d0b679ba12ce34dfe925b7541c8b7d74025

server/licenses/lucene-spatial-extras-9.9.0.jar.sha1

@@ -0,0 +1 @@
+37b89d9250edf4f2b1940395c053c66869b4449d

server/licenses/lucene-spatial3d-9.9.0.jar.sha1

@@ -0,0 +1 @@
+e51e7d3ca30c1789b7779b54afce4ab140830718

server/licenses/lucene-suggest-9.9.0.jar.sha1

@@ -0,0 +1 @@
+400c160b7fef8a4d142aa7cfc18f98469a5aec24

server/src/internalClusterTest/java/org/opensearch/index/store/CorruptedFileIT.java

@@ -710,6 +710,7 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro
 
         final NodeStats primaryNode = dataNodeStats.get(0);
         final NodeStats replicaNode = dataNodeStats.get(1);
+
         assertAcked(
             prepareCreate("test").setSettings(
                 Settings.builder()
@@ -795,6 +796,17 @@ public void testPrimaryCorruptionDuringReplicationDoesNotFailReplicaShard() thro
 
         // Assert the cluster returns to green status because the replica will be promoted to primary
         ensureGreen();
+
+        // After Lucene 9.9 check index will flag corruption with old (not the latest) commit points.
+        // For this test our previous corrupt commit "segments_2" will remain on the primary.
+        // This asserts this is the case, and then resets check index status.
+        assertEquals("Check index has a single failure", 1, checkIndexFailures.size());
+        assertTrue(
+            checkIndexFailures.get(0)
+                .getMessage()
+                .contains("could not read old (not latest) commit point segments file \"segments_2\" in directory")
+        );
+        resetCheckIndexStatus();
     }
 
     private int numShards(String... index) {

server/src/internalClusterTest/java/org/opensearch/validate/SimpleValidateQueryIT.java

@@ -270,7 +270,10 @@ public void testExplainDateRangeInQueryString() {
 
         long twoMonthsAgo = now.minus(2, ChronoUnit.MONTHS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000;
         long rangeEnd = (now.plus(1, ChronoUnit.DAYS).truncatedTo(ChronoUnit.DAYS).toEpochSecond() * 1000) - 1;
-        assertThat(response.getQueryExplanation().get(0).getExplanation(), equalTo("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]"));
+        assertThat(
+            response.getQueryExplanation().get(0).getExplanation(),
+            containsString("past:[" + twoMonthsAgo + " TO " + rangeEnd + "]")
+        );
         assertThat(response.isValid(), equalTo(true));
     }
 

server/src/main/java/org/apache/lucene/queries/BlendedTermQuery.java

@@ -100,11 +100,10 @@ public Query rewrite(IndexSearcher searcher) throws IOException {
             return rewritten;
         }
         IndexReader reader = searcher.getIndexReader();
-        IndexReaderContext context = reader.getContext();
         TermStates[] ctx = new TermStates[terms.length];
         int[] docFreqs = new int[ctx.length];
         for (int i = 0; i < terms.length; i++) {
-            ctx[i] = TermStates.build(context, terms[i], true);
+            ctx[i] = TermStates.build(searcher, terms[i], true);
             docFreqs[i] = ctx[i].docFreq();
         }