diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java index 0a9a09968a8..4d52a4c777f 100644 --- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java +++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java @@ -751,13 +751,6 @@ public final void doSyncOrReplicateRecovery(SolrCore core) throws Exception { } } - // if replay was skipped (possibly to due pulling a full index from the leader), - // then we still need to update version bucket seeds after recovery - if (successfulRecovery && replayFuture == null) { - log.info("Updating version bucket highest from index after successful recovery."); - core.seedVersionBuckets(); - } - if (log.isInfoEnabled()) { log.info( "Finished recovery process, successful=[{}] msTimeTaken={}", diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 094ccb285fc..5f22e95b3fc 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -1199,10 +1199,6 @@ private SolrCore( .initializeMetrics(solrMetricsContext, "directoryFactory"); } - // seed version buckets with max from index during core initialization ... requires a - // searcher! - seedVersionBuckets(); - bufferUpdatesIfConstructing(coreDescriptor); this.ruleExpiryLock = new ReentrantLock(); @@ -1238,22 +1234,6 @@ private SolrCore( assert ObjectReleaseTracker.track(this); } - public void seedVersionBuckets() { - UpdateHandler uh = getUpdateHandler(); - if (uh != null && uh.getUpdateLog() != null) { - RefCounted newestSearcher = getRealtimeSearcher(); - if (newestSearcher != null) { - try { - uh.getUpdateLog().seedBucketsWithHighestVersion(newestSearcher.get()); - } finally { - newestSearcher.decref(); - } - } else { - log.warn("No searcher available! Cannot seed version buckets with max from index."); - } - } - } - /** Set UpdateLog to buffer updates if the slice is in construction. */ private void bufferUpdatesIfConstructing(CoreDescriptor coreDescriptor) { diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java index fdb272709ba..6d674464ffc 100644 --- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java @@ -61,7 +61,6 @@ import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -104,7 +103,6 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.security.AuthorizationContext; import org.apache.solr.update.SolrIndexWriter; -import org.apache.solr.update.VersionInfo; import org.apache.solr.util.NumberUtils; import org.apache.solr.util.PropertiesInputStream; import org.apache.solr.util.RefCounted; @@ -730,18 +728,6 @@ public CoreReplicationAPI.IndexVersionResponse getIndexVersionResponse() throws return rsp; } - /** - * Retrieves the maximum version number from an index commit. NOTE: The commit MUST be - * reserved before calling this method - */ - private long getMaxVersion(IndexCommit commit) throws IOException { - try (DirectoryReader reader = DirectoryReader.open(commit)) { - IndexSearcher searcher = new IndexSearcher(reader); - VersionInfo vinfo = core.getUpdateHandler().getUpdateLog().getVersionInfo(); - return Math.abs(vinfo.getMaxVersionFromIndex(searcher)); - } - } - /** * For configuration files, checksum of the file is included because, unlike index files, they may * have same content but different timestamps. diff --git a/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java b/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java index abd530d85e6..cdddede5a79 100644 --- a/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java +++ b/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java @@ -23,12 +23,11 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.solr.common.SolrException; -/** - * @lucene.internal - */ /** * This implementation uses lock and condition and will throw exception if it can't obtain the lock * within lockTimeoutMs. + * + * @lucene.internal */ public class TimedVersionBucket extends VersionBucket { diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 255f0489af4..c9301356a0c 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -82,7 +82,6 @@ import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.util.LongSet; import org.apache.solr.util.OrderedExecutor; -import org.apache.solr.util.RTimer; import org.apache.solr.util.RefCounted; import org.apache.solr.util.TestInjection; import org.apache.solr.util.TimeOut; @@ -222,7 +221,6 @@ public String toString() { // This should only be used to initialize VersionInfo... the actual number of buckets may be // rounded up to a power of two. protected int numVersionBuckets; - protected Long maxVersionFromIndex = null; protected boolean existOldBufferLog = false; // keep track of deletes only... this is not updated on an add @@ -1889,12 +1887,6 @@ public void run() { // change the state while updates are still blocked to prevent races state = State.ACTIVE; if (finishing) { - - // after replay, update the max from the index - log.info("Re-computing max version from index after log re-play."); - maxVersionFromIndex = null; - getMaxVersionFromIndex(); - versionInfo.unblockUpdates(); } @@ -2327,71 +2319,6 @@ public void clearLog(SolrCore core, PluginInfo ulogPluginInfo) { } } - public Long getCurrentMaxVersion() { - return maxVersionFromIndex; - } - - // this method is primarily used for unit testing and is not part of the public API for this class - Long getMaxVersionFromIndex() { - RefCounted newestSearcher = - (uhandler != null && uhandler.core != null) ? uhandler.core.getRealtimeSearcher() : null; - if (newestSearcher == null) - throw new IllegalStateException("No searcher available to lookup max version from index!"); - - try { - seedBucketsWithHighestVersion(newestSearcher.get()); - return getCurrentMaxVersion(); - } finally { - newestSearcher.decref(); - } - } - - /** Used to seed all version buckets with the max value of the version field in the index. */ - protected Long seedBucketsWithHighestVersion( - SolrIndexSearcher newSearcher, VersionInfo versions) { - Long highestVersion = null; - final RTimer timer = new RTimer(); - - try (RecentUpdates recentUpdates = getRecentUpdates()) { - long maxVersionFromRecent = recentUpdates.getMaxRecentVersion(); - long maxVersionFromIndex = versions.getMaxVersionFromIndex(newSearcher); - - long maxVersion = Math.max(maxVersionFromIndex, maxVersionFromRecent); - if (maxVersion == 0L) { - maxVersion = versions.getNewClock(); - log.info( - "Could not find max version in index or recent updates, using new clock {}", - maxVersion); - } - - // seed all version buckets with the highest value from recent and index - versions.seedBucketsWithHighestVersion(maxVersion); - - highestVersion = maxVersion; - } catch (IOException ioExc) { - log.warn("Failed to determine the max value of the version field due to: ", ioExc); - } - - if (debug) { - log.debug( - "Took {}ms to seed version buckets with highest version {}", - timer.getTime(), - highestVersion); - } - - return highestVersion; - } - - public void seedBucketsWithHighestVersion(SolrIndexSearcher newSearcher) { - log.debug("Looking up max value of version field to seed version buckets"); - versionInfo.blockUpdates(); - try { - maxVersionFromIndex = seedBucketsWithHighestVersion(newSearcher, versionInfo); - } finally { - versionInfo.unblockUpdates(); - } - } - @SuppressForbidden(reason = "extends linkedhashmap") private static class OldDeletesLinkedHashMap extends LinkedHashMap { private final int numDeletesToKeepInternal; diff --git a/solr/core/src/java/org/apache/solr/update/VersionBucket.java b/solr/core/src/java/org/apache/solr/update/VersionBucket.java index 948da57b8c2..7724b3a2cb4 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionBucket.java +++ b/solr/core/src/java/org/apache/solr/update/VersionBucket.java @@ -23,21 +23,13 @@ // TODO: store the highest possible in the index on a commit (but how to not block adds?) // TODO: could also store highest possible in the transaction log after a commit. // Or on a new index, just scan "version" for the max? -/** - * @lucene.internal - */ /** * The default implementation which uses the intrinsic object monitor. It uses less memory but * ignores the lockTimeoutMs. + * + * @lucene.internal */ public class VersionBucket { - public long highest; - - public void updateHighest(long val) { - if (highest != 0) { - highest = Math.max(highest, Math.abs(val)); - } - } @FunctionalInterface public interface CheckedFunction { diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index 139873fdc96..d805ccab702 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -19,41 +19,32 @@ import static org.apache.solr.common.params.CommonParams.VERSION_FIELD; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.Map; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.Terms; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SuppressForbidden; -import org.apache.solr.index.SlowCompositeReaderWrapper; -import org.apache.solr.legacy.LegacyNumericUtils; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class VersionInfo { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS = "bucketVersionLockTimeoutMs"; private final UpdateLog ulog; - private final VersionBucket[] buckets; - private SchemaField versionField; + private final int numBuckets; + private volatile VersionBucket[] buckets; + private final Object bucketsSync = new Object(); + private final SchemaField versionField; final ReadWriteLock lock = new ReentrantReadWriteLock(true); - private int versionBucketLockTimeoutMs; + private final int versionBucketLockTimeoutMs; /** * Gets and returns the {@link org.apache.solr.common.params.CommonParams#VERSION_FIELD} from the @@ -103,14 +94,7 @@ public VersionInfo(UpdateLog ulog, int nBuckets) { .get("versionBucketLockTimeoutMs") .intVal( Integer.parseInt(System.getProperty(SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS, "0"))); - buckets = new VersionBucket[BitUtil.nextHighestPowerOfTwo(nBuckets)]; - for (int i = 0; i < buckets.length; i++) { - if (versionBucketLockTimeoutMs > 0) { - buckets[i] = new TimedVersionBucket(); - } else { - buckets[i] = new VersionBucket(); - } - } + numBuckets = BitUtil.nextHighestPowerOfTwo(nBuckets); } public int getVersionBucketLockTimeoutMs() { @@ -205,11 +189,25 @@ public VersionBucket bucket(int hash) { // Make sure high bits are moved down, since only the low bits will matter. // int h = hash + (hash >>> 8) + (hash >>> 16) + (hash >>> 24); // Assume good hash codes for now. - - int slot = hash & (buckets.length - 1); + int slot = hash & (numBuckets - 1); + if (buckets == null) { + synchronized (bucketsSync) { + if (buckets == null) { + buckets = createVersionBuckets(); + } + } + } return buckets[slot]; } + private VersionBucket[] createVersionBuckets() { + VersionBucket[] buckets = new VersionBucket[numBuckets]; + for (int i = 0; i < buckets.length; i++) { + buckets[i] = versionBucketLockTimeoutMs > 0 ? new TimedVersionBucket() : new VersionBucket(); + } + return buckets; + } + public Long lookupVersion(BytesRef idBytes) { return ulog.lookupVersion(idBytes); } @@ -246,85 +244,4 @@ public Long getVersionFromIndex(BytesRef idBytes) { } } } - - /** Returns the highest version from the index, or 0L if no versions can be found in the index. */ - @SuppressWarnings({"unchecked"}) - public Long getMaxVersionFromIndex(IndexSearcher searcher) throws IOException { - - final String versionFieldName = versionField.getName(); - - log.debug( - "Refreshing highest value of {} for {} version buckets from index", - versionFieldName, - buckets.length); - // if indexed, then we have terms to get the max from - if (versionField.indexed()) { - if (versionField.getType().isPointField()) { - return getMaxVersionFromIndexedPoints(searcher); - } else { - return getMaxVersionFromIndexedTerms(searcher); - } - } - // else: not indexed, use docvalues via value source ... - - long maxVersionInIndex = 0L; - ValueSource vs = versionField.getType().getValueSource(versionField, null); - Map funcContext = ValueSource.newContext(searcher); - vs.createWeight(funcContext, searcher); - // TODO: multi-thread this - for (LeafReaderContext ctx : searcher.getTopReaderContext().leaves()) { - int maxDoc = ctx.reader().maxDoc(); - FunctionValues fv = vs.getValues(funcContext, ctx); - for (int doc = 0; doc < maxDoc; doc++) { - long v = fv.longVal(doc); - maxVersionInIndex = Math.max(v, maxVersionInIndex); - } - } - return maxVersionInIndex; - } - - public void seedBucketsWithHighestVersion(long highestVersion) { - for (int i = 0; i < buckets.length; i++) { - // should not happen, but in case other threads are calling updateHighest on the version - // bucket - synchronized (buckets[i]) { - if (buckets[i].highest < highestVersion) buckets[i].highest = highestVersion; - } - } - } - - private long getMaxVersionFromIndexedTerms(IndexSearcher searcher) throws IOException { - assert !versionField.getType().isPointField(); - - final String versionFieldName = versionField.getName(); - final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(searcher.getIndexReader()); - final Terms versionTerms = leafReader.terms(versionFieldName); - final Long max = (versionTerms != null) ? LegacyNumericUtils.getMaxLong(versionTerms) : null; - if (null != max) { - log.debug("Found MAX value {} from Terms for {} in index", max, versionFieldName); - return max.longValue(); - } - return 0L; - } - - private long getMaxVersionFromIndexedPoints(IndexSearcher searcher) throws IOException { - assert versionField.getType().isPointField(); - - final String versionFieldName = versionField.getName(); - final byte[] maxBytes = - PointValues.getMaxPackedValue(searcher.getIndexReader(), versionFieldName); - if (null == maxBytes) { - return 0L; - } - final Object maxObj = versionField.getType().toObject(versionField, new BytesRef(maxBytes)); - if (null == maxObj || !(maxObj instanceof Number)) { - // HACK: aparently nothing asserts that the FieldType is numeric (let alone a Long???) - log.error("Unable to convert MAX byte[] from Points for {} in index", versionFieldName); - return 0L; - } - - final long max = ((Number) maxObj).longValue(); - log.debug("Found MAX value {} from Points for {} in index", max, versionFieldName); - return max; - } } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index b8e77f64295..3bbe0a8b38c 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -390,8 +390,6 @@ private boolean doVersionAdd( if (versionsStored) { - long bucketVersion = bucket.highest; - if (leaderLogic) { if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) { @@ -449,7 +447,6 @@ private boolean doVersionAdd( long version = vinfo.getNewClock(); cmd.setVersion(version); cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version); - bucket.updateHighest(version); } else { // The leader forwarded us this update. cmd.setVersion(versionOnUpdate); @@ -498,7 +495,7 @@ private boolean doVersionAdd( assert cmd.isInPlaceUpdate() == false; } } else { - if (lastVersion != null && Math.abs(lastVersion) > prev) { + if (Math.abs(lastVersion) > prev) { // this means we got a newer full doc update and in that case it makes no sense to // apply the older inplace update. Drop this update log.info( @@ -507,29 +504,19 @@ private boolean doVersionAdd( lastVersion); return true; } else { - // We're good, we should apply this update. First, update the bucket's highest. - if (bucketVersion != 0 && bucketVersion < versionOnUpdate) { - bucket.updateHighest(versionOnUpdate); - } + // We're good, we should apply this update. } } } else { // if we aren't the leader, then we need to check that updates were not re-ordered - if (bucketVersion != 0 && bucketVersion < versionOnUpdate) { - // we're OK... this update has a version higher than anything we've seen - // in this bucket so far, so we know that no reordering has yet occurred. - bucket.updateHighest(versionOnUpdate); - } else { - // there have been updates higher than the current update. we need to check - // the specific version for this id. - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { - // This update is a repeat, or was reordered. We need to drop this update. - if (log.isDebugEnabled()) { - log.debug("Dropping add update due to version {}", idBytes.utf8ToString()); - } - return true; + // we need to check the specific version for this id. + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { + // This update is a repeat, or was reordered. We need to drop this update. + if (log.isDebugEnabled()) { + log.debug("Dropping add update due to version {}", idBytes.utf8ToString()); } + return true; } } if (!isSubShardLeader @@ -1107,7 +1094,6 @@ private boolean doVersionDelete( try { BytesRef idBytes = cmd.getIndexedId(); if (versionsStored) { - long bucketVersion = bucket.highest; if (leaderLogic) { @@ -1156,7 +1142,6 @@ private boolean doVersionDelete( long version = vinfo.getNewClock(); cmd.setVersion(-version); - bucket.updateHighest(version); } else { cmd.setVersion(-versionOnUpdate); @@ -1168,21 +1153,14 @@ private boolean doVersionDelete( } // if we aren't the leader, then we need to check that updates were not re-ordered - if (bucketVersion != 0 && bucketVersion < versionOnUpdate) { - // we're OK... this update has a version higher than anything we've seen - // in this bucket so far, so we know that no reordering has yet occurred. - bucket.updateHighest(versionOnUpdate); - } else { - // there have been updates higher than the current update. we need to check - // the specific version for this id. - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { - // This update is a repeat, or was reordered. We need to drop this update. - if (log.isDebugEnabled()) { - log.debug("Dropping delete update due to version {}", idBytes.utf8ToString()); - } - return true; + // we need to check the specific version for this id. + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { + // This update is a repeat, or was reordered. We need to drop this update. + if (log.isDebugEnabled()) { + log.debug("Dropping delete update due to version {}", idBytes.utf8ToString()); } + return true; } if (!isSubShardLeader diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml deleted file mode 100644 index 375311a9b69..00000000000 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-dv.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - id - - - - - - - - - diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml b/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml deleted file mode 100644 index 4523d507ece..00000000000 --- a/solr/core/src/test-files/solr/collection1/conf/schema-version-indexed.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - id - - - - - - - - - diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java index c1092da34c7..f6e74fa2c52 100644 --- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java @@ -52,7 +52,6 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.update.UpdateLog; import org.apache.solr.util.RTimer; import org.apache.solr.util.TestInjection; import org.apache.solr.util.TimeOut; @@ -245,14 +244,9 @@ protected void testRf2() throws Exception { CoreContainer coreContainer = replicaJetty.getCoreContainer(); ZkCoreNodeProps replicaCoreNodeProps = new ZkCoreNodeProps(notLeader); String coreName = replicaCoreNodeProps.getCoreName(); - Long maxVersionBefore = null; try (SolrCore core = coreContainer.getCore(coreName)) { assertNotNull("Core '" + coreName + "' not found for replica: " + notLeader.getName(), core); - UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); - maxVersionBefore = ulog.getCurrentMaxVersion(); } - assertNotNull("max version bucket seed not set for core " + coreName, maxVersionBefore); - log.info("Looked up max version bucket seed {} for core {}", maxVersionBefore, coreName); // now up the stakes and do more docs int numDocs = TEST_NIGHTLY ? 1000 : 105; @@ -286,19 +280,6 @@ protected void testRf2() throws Exception { notLeaders = ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive); - try (SolrCore core = coreContainer.getCore(coreName)) { - assertNotNull("Core '" + coreName + "' not found for replica: " + notLeader.getName(), core); - Long currentMaxVersion = core.getUpdateHandler().getUpdateLog().getCurrentMaxVersion(); - log.info( - "After recovery, looked up NEW max version bucket seed {} for core {}, was: {}", - currentMaxVersion, - coreName, - maxVersionBefore); - assertTrue( - "max version bucket seed not updated after recovery!", - currentMaxVersion > maxVersionBefore); - } - // verify all docs received assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, numDocs + 3); diff --git a/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java b/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java deleted file mode 100644 index a80dd3f20cd..00000000000 --- a/solr/core/src/test/org/apache/solr/update/VersionInfoTest.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.update; - -import org.apache.lucene.util.BytesRef; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.util.Hash; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.schema.SchemaField; -import org.junit.Test; - -public class VersionInfoTest extends SolrTestCaseJ4 { - - @Test - public void testMaxIndexedVersionFromIndex() throws Exception { - initCore("solrconfig-tlog.xml", "schema-version-indexed.xml"); - try (SolrQueryRequest r = req()) { - SchemaField v = - r.getCore().getUpdateHandler().getUpdateLog().getVersionInfo().getVersionField(); - assertNotNull(v); - assertTrue(v.indexed()); - assertFalse(v.hasDocValues()); - - testMaxVersionLogic(r); - } finally { - deleteCore(); - } - } - - @Test - public void testMaxDocValuesVersionFromIndex() throws Exception { - initCore("solrconfig-tlog.xml", "schema-version-dv.xml"); - try (SolrQueryRequest r = req()) { - SchemaField v = - r.getCore().getUpdateHandler().getUpdateLog().getVersionInfo().getVersionField(); - assertNotNull(v); - assertFalse(v.indexed()); - assertTrue(v.hasDocValues()); - - testMaxVersionLogic(r); - } finally { - deleteCore(); - } - } - - protected void testMaxVersionLogic(SolrQueryRequest req) throws Exception { - UpdateHandler uhandler = req.getCore().getUpdateHandler(); - UpdateLog ulog = uhandler.getUpdateLog(); - ulog.init(uhandler, req.getCore()); - - clearIndex(); - assertU(commit()); - - // index the first doc - String docId = Integer.toString(1); - BytesRef idBytes = new BytesRef(docId); - assertU(adoc("id", docId)); - assertU(commit()); - - // max from the ulog should not be 0 or null - Long maxVersionFromUlog = ulog.getMaxVersionFromIndex(); - assertNotNull(maxVersionFromUlog); - assertTrue(maxVersionFromUlog != 0L); - - VersionInfo vInfo = ulog.getVersionInfo(); - try (SolrQueryRequest newReq = req()) { - // max version direct from the index should not be null, and should match what ulog reports - // (since doc is committed) - Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); - assertNotNull(vInfoMax); - assertEquals(maxVersionFromUlog, vInfoMax); - } - - // max version from ulog (and index) should be exactly the same as our single committed doc - Long version = vInfo.getVersionFromIndex(idBytes); - assertNotNull("version info should not be null for test doc: " + docId, version); - assertEquals(maxVersionFromUlog, version); - - int bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); - VersionBucket bucket = vInfo.bucket(bucketHash); - assertEquals(bucket.highest, version.longValue()); - - // send 2nd doc ... BUT DO NOT COMMIT - docId = Integer.toString(2); - idBytes = new BytesRef(docId); - assertU(adoc("id", docId)); - - try (SolrQueryRequest newReq = req()) { - // max version direct from the index should not be null, and should still match what ulog - // previously reported (since new doc is un-committed) - Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); - assertNotNull(vInfoMax); - assertEquals(maxVersionFromUlog, vInfoMax); - } - - maxVersionFromUlog = ulog.getMaxVersionFromIndex(); - assertNotNull(maxVersionFromUlog); - assertTrue( - "max version in ulog should have increased since our last committed doc: " - + version - + " ?< " - + maxVersionFromUlog, - version < maxVersionFromUlog); - - version = vInfo.getVersionFromIndex(idBytes); - assertNull("version info should be null for uncommitted test doc: " + docId, version); - - Long versionFromTLog = ulog.lookupVersion(idBytes); - assertNotNull( - "version from tlog should be non-null for uncommitted test doc: " + docId, versionFromTLog); - - // now commit that 2nd doc - assertU(commit()); - try (SolrQueryRequest newReq = req()) { - // max version direct from the index should match the new doc we just committed - Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); - assertEquals(versionFromTLog, vInfoMax); - } - assertEquals( - "committing doc should not have changed version from ulog", - versionFromTLog, - ulog.lookupVersion(idBytes)); - Long versionFromIndex = version = vInfo.getVersionFromIndex(idBytes); - assertNotNull( - "version from index should be non-null for committed test doc: " + docId, versionFromIndex); - assertEquals( - "version from tlog and version from index should be the same", - versionFromTLog, - versionFromIndex); - - bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); - bucket = vInfo.bucket(bucketHash); - assertEquals(bucket.highest, version.longValue()); - - // reload the core, which should reset the max - CoreContainer coreContainer = req.getCore().getCoreContainer(); - coreContainer.reload(req.getCore().getName()); - maxVersionFromUlog = ulog.getMaxVersionFromIndex(); - assertEquals( - "after reload, max version from ulog should be equal to version of last doc added", - maxVersionFromUlog, - versionFromIndex); - - // one more doc after reload - docId = Integer.toString(3); - idBytes = new BytesRef(docId); - assertU(adoc("id", docId)); - assertU(commit()); - - maxVersionFromUlog = ulog.getMaxVersionFromIndex(); - assertNotNull(maxVersionFromUlog); - assertTrue(maxVersionFromUlog != 0L); - - vInfo = ulog.getVersionInfo(); - try (SolrQueryRequest newReq = req()) { - // max version direct from the index should not be null, and should match what ulog reports - // (since doc is committed) - Long vInfoMax = vInfo.getMaxVersionFromIndex(newReq.getSearcher()); - assertNotNull(vInfoMax); - assertEquals(maxVersionFromUlog, vInfoMax); - } - version = vInfo.getVersionFromIndex(idBytes); - assertNotNull("version info should not be null for test doc: " + docId, version); - assertEquals(maxVersionFromUlog, version); - - bucketHash = Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); - bucket = vInfo.bucket(bucketHash); - assertEquals(bucket.highest, version.longValue()); - } -}