From 12b0915b8b6f422412ab2a1c838d4c846efea380 Mon Sep 17 00:00:00 2001 From: Oleksandr Porunov Date: Thu, 17 Oct 2024 02:57:10 +0100 Subject: [PATCH] Expose BerkeleyJE configs and setup default lock timeout to 30 seconds [tp-tests] Related to #1623 and #4425 Signed-off-by: Oleksandr Porunov --- docs/changelog.md | 17 ++++++++ docs/storage-backend/bdb.md | 42 +++++++++++++++++++ .../berkeleyje/BerkeleyJEStoreManager.java | 29 +++++++++++++ .../configuration/ConfigOption.java | 3 ++ .../util/system/ConfigurationUtil.java | 34 +++++++++++++++ .../diskstorage/es/ElasticSearchSetup.java | 38 +++-------------- 6 files changed, 131 insertions(+), 32 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9c5512e34e..775a0b1326 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -98,6 +98,23 @@ For more information on features and bug fixes in 1.1.0, see the GitHub mileston * [JanusGraph zip](https://github.com/JanusGraph/janusgraph/releases/download/v1.1.0/janusgraph-1.1.0.zip) * [JanusGraph zip with embedded Cassandra and ElasticSearch](https://github.com/JanusGraph/janusgraph/releases/download/v1.1.0/janusgraph-full-1.1.0.zip) +#### Upgrade Instructions + +##### BerkeleyJE lock timeout set to 30 seconds instead of default 500 ms + +Many users reported that in multi-thread environment the default lock timeout of `500 ms` is not enough and often lead +to lock timeout errors. Thus, the new configuration is set to `30000 ms`. +It's possible to switch back to previous settings by setting configuration option +`storage.berkeleyje.ext.je.lock.timeout` to `500 ms`. + +##### BerkeleyJE ability to overwrite arbitrary settings applied at `EnvironmentConfig` creation + +The new namespace `storage.berkeleyje.ext` now allows to set custom configurations which were not directly exposed by +JanusGraph. +The full list of possible setting is available inside the Java class `com.sleepycat.je.EnvironmentConfig`. +All configurations values should be specified as `String` and be formated the same as specified in the official sleepycat +[documentation](https://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/EnvironmentConfig.html). + ### Version 1.0.1 (Release Date: ???) /// tab | Maven diff --git a/docs/storage-backend/bdb.md b/docs/storage-backend/bdb.md index 1db34d1a67..333a98c22f 100644 --- a/docs/storage-backend/bdb.md +++ b/docs/storage-backend/bdb.md @@ -85,3 +85,45 @@ In order to not run out of memory, it is advised to disable transactions transactions enabled requires BerkeleyDB to acquire read locks on the data it is reading. When iterating over the entire graph, these read locks can easily require more memory than is available. + +## Additional BerkeleyDB JE configuration options + +It's possible to set additional BerkeleyDB JE configuration which are not +directly exposed by JanusGraph by leveraging `storage.berkeleyje.ext` +namespace. + +JanusGraph iterates over all properties prefixed with +`storage.berkeleyje.ext.`. It strips the prefix from each property key. +The remainder of the stripped key will be interpreted as a parameter +ke for `com.sleepycat.je.EnvironmentConfig`. The value associated with the +key is not modified. +This allows embedding arbitrary settings in JanusGraph’s properties. Here’s an +example configuration fragment that customizes three BerkeleyDB settings +using the `storage.berkeleyje.ext.` config mechanism: + +```properties +storage.backend=berkeleyje +storage.berkeleyje.ext.je.lock.timeout=5000 ms +storage.berkeleyje.ext.je.lock.deadlockDetect=false +storage.berkeleyje.ext.je.txn.timeout=5000 ms +storage.berkeleyje.ext.je.log.fileMax=100000000 +``` + +## Deadlock troubleshooting + +In concurrent environment deadlocks are possible when using BerkeleyDB JE storage +backend. +It may be complicated to deal with deadlocks in use-cases when multiple threads are +modifying same vertices (including edges creation between affected vertices). +More insights on this topic can be found in the GitHub issue +[#1623](https://github.com/JanusGraph/janusgraph/issues/1623). + +Some users suggest the following configuration to deal with deadlocks: +```properties +storage.berkeleyje.isolation-level=READ_UNCOMMITTED +storage.berkeleyje.lock-mode=LockMode.READ_UNCOMMITTED +storage.berkeleyje.ext.je.lock.timeout=0 +storage.lock.wait-time=5000 +ids.authority.wait-time=2000 +tx.max-commit-time=30000 +``` diff --git a/janusgraph-berkeleyje/src/main/java/org/janusgraph/diskstorage/berkeleyje/BerkeleyJEStoreManager.java b/janusgraph-berkeleyje/src/main/java/org/janusgraph/diskstorage/berkeleyje/BerkeleyJEStoreManager.java index 55d405adff..2578524179 100644 --- a/janusgraph-berkeleyje/src/main/java/org/janusgraph/diskstorage/berkeleyje/BerkeleyJEStoreManager.java +++ b/janusgraph-berkeleyje/src/main/java/org/janusgraph/diskstorage/berkeleyje/BerkeleyJEStoreManager.java @@ -48,6 +48,7 @@ import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; import org.janusgraph.graphdb.configuration.PreInitializeConfigOptions; import org.janusgraph.graphdb.transaction.TransactionConfiguration; +import org.janusgraph.util.system.ConfigurationUtil; import org.janusgraph.util.system.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,6 +94,23 @@ public class BerkeleyJEStoreManager extends LocalStoreManager implements Ordered ConfigOption.Type.MASKABLE, String.class, IsolationLevel.REPEATABLE_READ.toString(), disallowEmpty(String.class)); + public static final ConfigNamespace BERKELEY_EXTRAS_NS = + new ConfigNamespace(BERKELEY_NS, "ext", "Overrides for arbitrary settings applied at `EnvironmentConfig` creation.\n" + + "The full list of possible setting is available inside the Java class `com.sleepycat.je.EnvironmentConfig`. " + + "All configurations values should be specified as `String` and be formated the same as specified in the following " + + "[documentation](https://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/EnvironmentConfig.html)."); + + // This setting isn't used directly in Java, but this setting will be picked up indirectly during parsing of the + // subset configuration of `BERKELEY_EXTRAS_NS` namespace + public static final ConfigOption EXT_LOCK_TIMEOUT = + new ConfigOption<>(BERKELEY_EXTRAS_NS, EnvironmentConfig.LOCK_TIMEOUT, + "Lock timeout configuration. " + + "`0` disabled lock timeout completely. To set lock timeout via this configuration it's required to use " + + "String formated time representation. For example: `500 ms`, `5 min`, etc. See information about value " + + "constraints in the official " + + "[sleepycat documentation](https://docs.oracle.com/cd/E17277_02/html/java/com/sleepycat/je/EnvironmentConfig.html#LOCK_TIMEOUT)", + ConfigOption.Type.MASKABLE, "30000 ms"); + private final ConcurrentMap stores; protected volatile Environment environment; @@ -143,6 +161,9 @@ private synchronized void initialize() throws BackendException { envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false"); } + Map extraSettings = getSettingsFromJanusGraphConf(storageConfig); + extraSettings.forEach(envConfig::setConfigParam); + // Open the environment environment = new Environment(directory, envConfig); @@ -153,7 +174,15 @@ private synchronized void initialize() throws BackendException { } catch (DatabaseException e) { throw new PermanentBackendException("Error during BerkeleyJE initialization: ", e); } + } + static Map getSettingsFromJanusGraphConf(Configuration config) { + final Map settings = ConfigurationUtil.getSettingsFromJanusGraphConf(config, BERKELEY_EXTRAS_NS); + if(log.isDebugEnabled()){ + settings.forEach((key, val) -> log.debug("[BERKELEY ext.* cfg] Set {}: {}", key, val)); + log.debug("Loaded {} settings from the {} JanusGraph config namespace", settings.size(), BERKELEY_EXTRAS_NS); + } + return settings; } private synchronized void reInitialize(DatabaseException exception) throws BackendException { diff --git a/janusgraph-core/src/main/java/org/janusgraph/diskstorage/configuration/ConfigOption.java b/janusgraph-core/src/main/java/org/janusgraph/diskstorage/configuration/ConfigOption.java index 407a83824f..e2ebb7ae8d 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/diskstorage/configuration/ConfigOption.java +++ b/janusgraph-core/src/main/java/org/janusgraph/diskstorage/configuration/ConfigOption.java @@ -275,5 +275,8 @@ public static Predicate positiveLong() { return num -> num!=null && num>0; } + public static Predicate nonnegativeLong() { + return num -> num!=null && num>=0; + } } diff --git a/janusgraph-core/src/main/java/org/janusgraph/util/system/ConfigurationUtil.java b/janusgraph-core/src/main/java/org/janusgraph/util/system/ConfigurationUtil.java index f79b7d249a..05ecb0ef42 100644 --- a/janusgraph-core/src/main/java/org/janusgraph/util/system/ConfigurationUtil.java +++ b/janusgraph-core/src/main/java/org/janusgraph/util/system/ConfigurationUtil.java @@ -14,6 +14,7 @@ package org.janusgraph.util.system; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import org.apache.commons.configuration2.BaseConfiguration; import org.apache.commons.configuration2.Configuration; @@ -24,11 +25,14 @@ import org.apache.commons.configuration2.builder.fluent.PropertiesBuilderParameters; import org.apache.commons.configuration2.convert.DefaultListDelimiterHandler; import org.apache.commons.configuration2.ex.ConfigurationException; +import org.janusgraph.diskstorage.configuration.ConfigNamespace; import java.io.File; +import java.lang.reflect.Array; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -169,4 +173,34 @@ private static PropertiesConfiguration loadPropertiesConfig(PropertiesBuilderPar } return builder.configure(newParams).getConfiguration(); } + + public static Map getSettingsFromJanusGraphConf(org.janusgraph.diskstorage.configuration.Configuration config, ConfigNamespace namespace) { + + final Map settings = new HashMap<>(); + + final Map configSub = config.getSubset(namespace); + for (Map.Entry entry : configSub.entrySet()) { + String key = entry.getKey(); + Object val = entry.getValue(); + if (null == val) continue; + if (List.class.isAssignableFrom(val.getClass())) { + // Pretty print lists using comma-separated values and no surrounding square braces for ES + List l = (List) val; + settings.put(key, Joiner.on(",").join(l)); + } else if (val.getClass().isArray()) { + // As with Lists, but now for arrays + // The Object copy[] business lets us avoid repetitive primitive array type checking and casting + Object[] copy = new Object[Array.getLength(val)]; + for (int i= 0; i < copy.length; i++) { + copy[i] = Array.get(val, i); + } + settings.put(key, Joiner.on(",").join(copy)); + } else { + // Copy anything else unmodified + settings.put(key, val.toString()); + } + } + + return settings; + } } diff --git a/janusgraph-es/src/main/java/org/janusgraph/diskstorage/es/ElasticSearchSetup.java b/janusgraph-es/src/main/java/org/janusgraph/diskstorage/es/ElasticSearchSetup.java index 9ea643b536..fe865495d8 100644 --- a/janusgraph-es/src/main/java/org/janusgraph/diskstorage/es/ElasticSearchSetup.java +++ b/janusgraph-es/src/main/java/org/janusgraph/diskstorage/es/ElasticSearchSetup.java @@ -14,17 +14,15 @@ package org.janusgraph.diskstorage.es; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import org.janusgraph.diskstorage.configuration.Configuration; import org.janusgraph.diskstorage.es.rest.RestClientSetup; +import org.janusgraph.util.system.ConfigurationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.reflect.Array; import java.util.HashMap; -import java.util.List; import java.util.Map; /** @@ -53,36 +51,12 @@ public Connection connect(Configuration config) throws IOException { }; static Map getSettingsFromJanusGraphConf(Configuration config) { - - final Map settings = new HashMap<>(); - - int keysLoaded = 0; - final Map configSub = config.getSubset(ElasticSearchIndex.ES_CREATE_EXTRAS_NS); - for (Map.Entry entry : configSub.entrySet()) { - String key = entry.getKey(); - Object val = entry.getValue(); - if (null == val) continue; - if (List.class.isAssignableFrom(val.getClass())) { - // Pretty print lists using comma-separated values and no surrounding square braces for ES - List l = (List) val; - settings.put(key, Joiner.on(",").join(l)); - } else if (val.getClass().isArray()) { - // As with Lists, but now for arrays - // The Object copy[] business lets us avoid repetitive primitive array type checking and casting - Object[] copy = new Object[Array.getLength(val)]; - for (int i= 0; i < copy.length; i++) { - copy[i] = Array.get(val, i); - } - settings.put(key, Joiner.on(",").join(copy)); - } else { - // Copy anything else unmodified - settings.put(key, val.toString()); - } - log.debug("[ES ext.* cfg] Set {}: {}", key, val); - keysLoaded++; + final Map settings = ConfigurationUtil.getSettingsFromJanusGraphConf(config, ElasticSearchIndex.ES_CREATE_EXTRAS_NS); + if(log.isDebugEnabled()){ + settings.forEach((key, val) -> log.debug("[ES ext.* cfg] Set {}: {}", key, val)); + log.debug("Loaded {} settings from the {} JanusGraph config namespace", settings.size(), ElasticSearchIndex.ES_CREATE_EXTRAS_NS); } - log.debug("Loaded {} settings from the {} JanusGraph config namespace", keysLoaded, ElasticSearchIndex.ES_CREATE_EXTRAS_NS); - return settings; + return new HashMap<>(settings); } private static final Logger log = LoggerFactory.getLogger(ElasticSearchSetup.class);