From fd7a4238fbffd136dfb821907247075ae4fe35fe Mon Sep 17 00:00:00 2001 From: Himanshu Date: Wed, 20 Jul 2022 09:36:12 -0700 Subject: [PATCH 1/7] GORA-698 Implement Geode Store --- gora-geode/pom.xml | 82 ++++++++++++ .../apache/gora/geode/query/GeodeQuery.java | 38 ++++++ .../apache/gora/geode/query/GeodeResult.java | 91 +++++++++++++ .../apache/gora/geode/store/GeodeStore.java | 120 ++++++++++++++++++ pom.xml | 8 ++ 5 files changed, 339 insertions(+) create mode 100644 gora-geode/pom.xml create mode 100644 gora-geode/src/main/java/org/apache/gora/geode/query/GeodeQuery.java create mode 100644 gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java create mode 100644 gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java diff --git a/gora-geode/pom.xml b/gora-geode/pom.xml new file mode 100644 index 00000000..9a12dde8 --- /dev/null +++ b/gora-geode/pom.xml @@ -0,0 +1,82 @@ + + + + gora + org.apache.gora + 1.0-SNAPSHOT + ../ + + 4.0.0 + + gora-geode + + + 8 + 8 + + + + + + org.apache.gora + gora-core + + + + org.apache.gora + gora-core + test-jar + test + + + + org.apache.avro + avro + + + + + org.apache.hadoop + hadoop-client + + + + + org.apache.geode + geode-core + + + + + org.slf4j + slf4j-log4j12 + + + + log4j + log4j + + + javax.jms + jms + + + + + + + junit + junit + test + + + + org.apache.hadoop + hadoop-minicluster + test + + + + \ No newline at end of file diff --git a/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeQuery.java b/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeQuery.java new file mode 100644 index 00000000..f6684d5d --- /dev/null +++ b/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeQuery.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gora.geode.query; + +import org.apache.gora.persistency.impl.PersistentBase; +import org.apache.gora.query.impl.QueryBase; +import org.apache.gora.store.DataStore; + +/** + * {@link GeodeQuery} is the primary class + * responsible for representing a cache manipulation query. + */ +public class GeodeQuery extends QueryBase { + + public GeodeQuery() { + super(null); + } + + public GeodeQuery(DataStore dataStore) { + super(dataStore); + } + +} diff --git a/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java b/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java new file mode 100644 index 00000000..2f77f7db --- /dev/null +++ b/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gora.geode.query; + +import org.apache.gora.geode.store.GeodeStore; +import org.apache.gora.persistency.impl.PersistentBase; +import org.apache.gora.query.Query; +import org.apache.gora.query.impl.ResultBase; +import org.apache.gora.store.DataStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Iterator; +import java.util.NavigableSet; + +/** + * {@link GeodeResult} is the primary class + * responsible for representing result set of a cache manipulation query + * {@link org.apache.gora.geode.query.GeodeQuery} + */ +public class GeodeResult extends ResultBase { + + private static final Logger LOG = LoggerFactory.getLogger(GeodeResult.class); + private NavigableSet cacheKeySet; + private Iterator iterator; + private int current; + + public GeodeResult(DataStore dataStore, Query query) { + super(dataStore, query); + } + + public GeodeResult(DataStore dataStore, Query query, NavigableSet cacheKeySet) { + super(dataStore, query); + this.cacheKeySet = cacheKeySet; + this.iterator = cacheKeySet.iterator(); + this.current = 0; + } + + public GeodeStore getDataStore() { + return (GeodeStore) super.getDataStore(); + } + + @Override + public float getProgress() throws IOException { + if (cacheKeySet.size() == 0) { + return 1; + } + float progress = ((float) current / (float) cacheKeySet.size()); + return progress; + } + + @Override + public void close() throws IOException { + + } + + @Override + protected boolean nextInner() throws IOException { + if (!iterator.hasNext()) { + return false; + } + key = iterator.next(); + LOG.info("Results set pointer is now moved to key {}.", key); + persistent = dataStore.get(key); + this.current++; + return true; + } + + @Override + public int size() { + int totalSize = cacheKeySet.size(); + int intLimit = (int) this.limit; + return intLimit > 0 && totalSize > intLimit ? intLimit : totalSize; + } +} diff --git a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java new file mode 100644 index 00000000..e6afe9a5 --- /dev/null +++ b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java @@ -0,0 +1,120 @@ +package org.apache.gora.geode.store; + +import org.apache.geode.cache.Cache; +import org.apache.geode.cache.CacheFactory; +import org.apache.geode.cache.Region; +import org.apache.geode.cache.RegionFactory; +import org.apache.geode.cache.client.ClientCache; +import org.apache.geode.cache.client.ClientCacheFactory; +import org.apache.gora.persistency.impl.PersistentBase; +import org.apache.gora.query.PartitionQuery; +import org.apache.gora.query.Query; +import org.apache.gora.query.Result; +import org.apache.gora.store.impl.DataStoreBase; +import org.apache.gora.util.GoraException; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +import static org.apache.geode.cache.RegionShortcut.REPLICATE; + +public class GeodeStore extends DataStoreBase { + + ClientCache cache; + Region region; + + @Override + public void initialize(Class keyClass, Class persistentClass, Properties properties) throws GoraException { + super.initialize(keyClass, persistentClass, properties); + String geodeHostName = "127.0.0.1";//properties.get(""); + int portNumber = 10334;// properties.get(""); + cache = new ClientCacheFactory().addPoolLocator(geodeHostName, portNumber).create(); + } + + @Override + public String getSchemaName() { + return null; + } + + @Override + public void createSchema() throws GoraException { + try { + Properties properties = cache.getDistributedSystem().getProperties(); + CacheFactory factory = new CacheFactory(properties); + Cache cache = factory.create(); + RegionFactory rf = cache.createRegionFactory(REPLICATE); + Region tempRegion = rf.create(persistentClass.getSimpleName()); + region = tempRegion; + } catch (Exception e) { + throw new GoraException(e); + } + } + + @Override + public void deleteSchema() { + region.destroyRegion(); + } + + @Override + public boolean schemaExists() { + return false; + } + + @Override + public boolean exists(K key) { + for (K existingKey : region.getInterestList()) { + if (existingKey.equals(key)) { + return true; + } + } + return false; + } + + @Override + public T get(K key, String[] fields) { + return region.get(key); + + } + + @Override + public void put(K key, T obj) { + region.put(key, obj); + } + + @Override + public boolean delete(K key) { + region.destroy(key); + return true; + } + + @Override + public long deleteByQuery(Query query) { + return 0; + } + + @Override + public Result execute(Query query) { + return null; + } + + @Override + public Query newQuery() { + return null; + } + + @Override + public List> getPartitions(Query query) throws IOException { + return null; + } + + @Override + public void flush() { + + } + + @Override + public void close() { + + } +} diff --git a/pom.xml b/pom.xml index 063ce98e..a3b3267e 100755 --- a/pom.xml +++ b/pom.xml @@ -820,6 +820,7 @@ gora-tutorial gora-benchmark sources-dist + gora-geode @@ -1656,6 +1657,13 @@ 2.0.10 + + + org.apache.geode + geode-core + 1.15.0 + + com.amazonaws From a2a0e2184d7c7d0832cf2e4883ef4cee5e8cc2a4 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Sat, 3 Sep 2022 12:25:39 -0700 Subject: [PATCH 2/7] GORA-698 Add Gora Geode Store Implementation --- gora-geode/pom.xml | 2 +- .../apache/gora/geode/query/GeodeResult.java | 8 +- .../apache/gora/geode/store/GeodeStore.java | 138 +++++++++++++----- .../geode/store/GeodeStoreParameters.java | 60 ++++++++ 4 files changed, 166 insertions(+), 42 deletions(-) create mode 100644 gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStoreParameters.java diff --git a/gora-geode/pom.xml b/gora-geode/pom.xml index 9a12dde8..02e974ef 100644 --- a/gora-geode/pom.xml +++ b/gora-geode/pom.xml @@ -6,7 +6,7 @@ gora org.apache.gora 1.0-SNAPSHOT - ../ + ../pom.xml 4.0.0 diff --git a/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java b/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java index 2f77f7db..9583270b 100644 --- a/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java +++ b/gora-geode/src/main/java/org/apache/gora/geode/query/GeodeResult.java @@ -61,13 +61,7 @@ public float getProgress() throws IOException { if (cacheKeySet.size() == 0) { return 1; } - float progress = ((float) current / (float) cacheKeySet.size()); - return progress; - } - - @Override - public void close() throws IOException { - + return ((float) current / (float) cacheKeySet.size()); } @Override diff --git a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java index e6afe9a5..e0d84dba 100644 --- a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java +++ b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java @@ -1,68 +1,98 @@ package org.apache.gora.geode.store; -import org.apache.geode.cache.Cache; -import org.apache.geode.cache.CacheFactory; -import org.apache.geode.cache.Region; -import org.apache.geode.cache.RegionFactory; +import org.apache.geode.cache.*; import org.apache.geode.cache.client.ClientCache; import org.apache.geode.cache.client.ClientCacheFactory; +import org.apache.gora.geode.query.GeodeQuery; +import org.apache.gora.geode.query.GeodeResult; import org.apache.gora.persistency.impl.PersistentBase; import org.apache.gora.query.PartitionQuery; import org.apache.gora.query.Query; import org.apache.gora.query.Result; +import org.apache.gora.query.impl.PartitionQueryImpl; import org.apache.gora.store.impl.DataStoreBase; import org.apache.gora.util.GoraException; import java.io.IOException; -import java.util.List; -import java.util.Properties; +import java.util.*; +import java.util.concurrent.ConcurrentSkipListSet; import static org.apache.geode.cache.RegionShortcut.REPLICATE; +import static org.apache.gora.geode.store.GeodeStoreParameters.*; + public class GeodeStore extends DataStoreBase { - ClientCache cache; - Region region; + private ClientCache clientCache; + private Region region; + private Properties geodeProperties; + + + private CacheFactory cacheFactory; @Override public void initialize(Class keyClass, Class persistentClass, Properties properties) throws GoraException { super.initialize(keyClass, persistentClass, properties); - String geodeHostName = "127.0.0.1";//properties.get(""); - int portNumber = 10334;// properties.get(""); - cache = new ClientCacheFactory().addPoolLocator(geodeHostName, portNumber).create(); + String geodeHostName = (String) properties.get(GEODE_SERVER_HOST); + int portNumber = Integer.parseInt((String) properties.get(GEODE_SERVER_PORT)); + clientCache = new ClientCacheFactory().addPoolLocator(geodeHostName, portNumber).create(); + String userName = properties.getProperty(GEODE_USERNAME); + String password = properties.getProperty(GEODE_PASSWORD); + geodeProperties = properties; + + Properties clientProperties = clientCache.getDistributedSystem().getProperties(); + if (userName != null) { + clientProperties.setProperty("security-username", userName); + clientProperties.setProperty("security-password", password); + } + cacheFactory = new CacheFactory(clientProperties); } @Override + /* + Schema Name can be assigned via Property file using @PREFERRED_SCHEMA_NAME, or else PersistentClass name is used as the default schema name. + */ public String getSchemaName() { - return null; + String preferredSchemaName = properties.getProperty(PREFERRED_SCHEMA_NAME); + if (preferredSchemaName == null) { + return persistentClass.getSimpleName(); + } + return preferredSchemaName; } @Override public void createSchema() throws GoraException { try { - Properties properties = cache.getDistributedSystem().getProperties(); - CacheFactory factory = new CacheFactory(properties); - Cache cache = factory.create(); - RegionFactory rf = cache.createRegionFactory(REPLICATE); - Region tempRegion = rf.create(persistentClass.getSimpleName()); - region = tempRegion; + Cache cache = cacheFactory.create(); + String regionShortCut = geodeProperties.getProperty(GEODE_REGION_SHORTCUT); + RegionFactory regionFactory; + if (regionShortCut != null) { + regionFactory = cache.createRegionFactory(RegionShortcut.valueOf(regionShortCut)); + } else { + regionFactory = cache.createRegionFactory(REPLICATE); + } + region = regionFactory.create(getSchemaName()); } catch (Exception e) { throw new GoraException(e); } } @Override - public void deleteSchema() { + public void deleteSchema() { region.destroyRegion(); } @Override - public boolean schemaExists() { - return false; + public boolean schemaExists() { + Properties properties = clientCache.getDistributedSystem().getProperties(); + CacheFactory factory = new CacheFactory(properties); + Cache cache = factory.create(); + Region rf = cache.getRegion(getSchemaName()); + return rf != null; } @Override - public boolean exists(K key) { + public boolean exists(K key) { for (K existingKey : region.getInterestList()) { if (existingKey.equals(key)) { return true; @@ -72,45 +102,85 @@ public boolean exists(K key) { } @Override - public T get(K key, String[] fields) { + public T get(K key, String[] fields) { return region.get(key); - } @Override - public void put(K key, T obj) { + public void put(K key, T obj) { region.put(key, obj); } @Override - public boolean delete(K key) { + public boolean delete(K key) { region.destroy(key); return true; } @Override - public long deleteByQuery(Query query) { - return 0; + public long deleteByQuery(Query query) throws GoraException { + try { + long deletedRows = 0; + Result result = query.execute(); + while (result.next()) { + if (delete(result.getKey())) { + deletedRows++; + } + } + LOG.info("Geode datastore deleted {} rows from Persistent datastore.", deletedRows); + return deletedRows; + } catch (Exception e) { + throw new GoraException(e); + } } @Override - public Result execute(Query query) { - return null; + public Result execute(Query query) { + + K startKey = query.getStartKey(); + K endKey = query.getEndKey(); + NavigableSet cacheEntrySubList = new ConcurrentSkipListSet<>(); + if (startKey != null && endKey != null) { + boolean isInTheRegion = false; + for (K key : region.keySet()) { + if (key == startKey) { + isInTheRegion = true; + } + if (isInTheRegion) { + cacheEntrySubList.add(key); + } + if (key == endKey) { + break; + } + } + } else { + // Empty + cacheEntrySubList = Collections.emptyNavigableSet(); + } + return new GeodeResult<>(this, query, cacheEntrySubList); } + @Override public Query newQuery() { - return null; + GeodeQuery query = new GeodeQuery<>(this); + query.setFields(getFieldsToQuery(null)); + return query; } @Override - public List> getPartitions(Query query) throws IOException { - return null; + public List> getPartitions(Query query)throws IOException { + List> partitions = new ArrayList<>(); + PartitionQueryImpl partitionQuery = new PartitionQueryImpl<>( + query); + partitionQuery.setConf(this.getConf()); + partitions.add(partitionQuery); + return partitions; } @Override public void flush() { - + LOG.info("Geode datastore flushed successfully."); } @Override diff --git a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStoreParameters.java b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStoreParameters.java new file mode 100644 index 00000000..138bc299 --- /dev/null +++ b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStoreParameters.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gora.geode.store; + +/** + * Configuration Properties. + */ +public class GeodeStoreParameters { + + /** + * Property pointing to geode server contact points. + * string (multiple values with comma separated) + */ + public static final String GEODE_SERVER_HOST = "gora.geode.server.hostname"; + + /** + * Property pointing to the port to use to connect to the geode hosts. + * integer + */ + public static final String GEODE_SERVER_PORT = "gora.geode.server.port"; + + /** + * Property pointing to the gora schemaName. + * integer + */ + public static final String PREFERRED_SCHEMA_NAME = "gora.geode.preferred.schemaName"; + + /** + * Property pointing to the geode region shortcut. + * integer + */ + public static final String GEODE_REGION_SHORTCUT = "gora.geode.region.shortcut"; + + /** + * Property pointing to the username to connect to the server. + * string + */ + public static final String GEODE_USERNAME = "gora.geode.username"; + + /** + * Property pointing to the password to connect to the server. + * string + */ + public static final String GEODE_PASSWORD = "gora.geode.password"; + +} From 13fc58561aab36c5debd33f90449288f4307b3d2 Mon Sep 17 00:00:00 2001 From: Himanshu Date: Sat, 3 Sep 2022 12:41:05 -0700 Subject: [PATCH 3/7] GORA-698 Code format --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a3b3267e..5c6fd3e1 100755 --- a/pom.xml +++ b/pom.xml @@ -817,10 +817,10 @@ gora-jet gora-rethinkdb gora-elasticsearch + gora-geode gora-tutorial gora-benchmark sources-dist - gora-geode From 3acc4ed3a3aed78ec3efd8f4313a4972d37d8453 Mon Sep 17 00:00:00 2001 From: himanshu Date: Mon, 5 Sep 2022 13:54:06 -0700 Subject: [PATCH 4/7] 3 amendments performed as suggested. --- .../java/org/apache/gora/geode/store/GeodeStore.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java index e0d84dba..3b7bdfea 100644 --- a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java +++ b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java @@ -26,8 +26,6 @@ public class GeodeStore extends DataStoreBase private ClientCache clientCache; private Region region; private Properties geodeProperties; - - private CacheFactory cacheFactory; @Override @@ -44,7 +42,7 @@ public void initialize(Class keyClass, Class persistentClass, Properties p if (userName != null) { clientProperties.setProperty("security-username", userName); clientProperties.setProperty("security-password", password); - } + } else throw new GoraException(); cacheFactory = new CacheFactory(clientProperties); } @@ -65,7 +63,7 @@ public void createSchema() throws GoraException { try { Cache cache = cacheFactory.create(); String regionShortCut = geodeProperties.getProperty(GEODE_REGION_SHORTCUT); - RegionFactory regionFactory; + RegionFactory regionFactory; if (regionShortCut != null) { regionFactory = cache.createRegionFactory(RegionShortcut.valueOf(regionShortCut)); } else { @@ -136,7 +134,6 @@ public long deleteByQuery(Query query) throws GoraException { @Override public Result execute(Query query) { - K startKey = query.getStartKey(); K endKey = query.getEndKey(); NavigableSet cacheEntrySubList = new ConcurrentSkipListSet<>(); @@ -169,7 +166,7 @@ public Query newQuery() { } @Override - public List> getPartitions(Query query)throws IOException { + public List> getPartitions(Query query) throws IOException { List> partitions = new ArrayList<>(); PartitionQueryImpl partitionQuery = new PartitionQueryImpl<>( query); From 53c5628db7a9e116dfa1e02a975a7886b9cade15 Mon Sep 17 00:00:00 2001 From: himanshu Date: Sat, 17 Sep 2022 11:44:45 -0700 Subject: [PATCH 5/7] Test cases for Apache Geode --- gora-geode/src/test/conf/gora.properties | 18 ++ .../geode/GeodeStartupLogWaitStrategy.java | 36 ++++ .../gora/geode/GoraGeodeTestDriver.java | 88 ++++++++++ .../src/test/java/store/TestGeodeStore.java | 161 ++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 gora-geode/src/test/conf/gora.properties create mode 100644 gora-geode/src/test/java/org/apache/gora/geode/GeodeStartupLogWaitStrategy.java create mode 100644 gora-geode/src/test/java/org/apache/gora/geode/GoraGeodeTestDriver.java create mode 100644 gora-geode/src/test/java/store/TestGeodeStore.java diff --git a/gora-geode/src/test/conf/gora.properties b/gora-geode/src/test/conf/gora.properties new file mode 100644 index 00000000..3e8e111a --- /dev/null +++ b/gora-geode/src/test/conf/gora.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +gora.datastore.default=org.apache.gora.geode.store.GeodeStore +gora.datastore.geode.server=localhost +gora.datastore.geode.port="gora.geode.server.port" \ No newline at end of file diff --git a/gora-geode/src/test/java/org/apache/gora/geode/GeodeStartupLogWaitStrategy.java b/gora-geode/src/test/java/org/apache/gora/geode/GeodeStartupLogWaitStrategy.java new file mode 100644 index 00000000..67110ba1 --- /dev/null +++ b/gora-geode/src/test/java/org/apache/gora/geode/GeodeStartupLogWaitStrategy.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gora.geode; + +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; + + +/** + * Log based Geode server startup wait strategy to sync server + * startup to test suit startup. + */ +public class GeodeStartupLogWaitStrategy extends LogMessageWaitStrategy { + + private static final String regEx = ".*Apache Geode has started. Time to relax..*"; + + public GeodeStartupLogWaitStrategy() { + withRegEx(regEx); + } + +} \ No newline at end of file diff --git a/gora-geode/src/test/java/org/apache/gora/geode/GoraGeodeTestDriver.java b/gora-geode/src/test/java/org/apache/gora/geode/GoraGeodeTestDriver.java new file mode 100644 index 00000000..c11e5c51 --- /dev/null +++ b/gora-geode/src/test/java/org/apache/gora/geode/GoraGeodeTestDriver.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gora.geode; + +import org.apache.gora.GoraTestDriver; + +import org.apache.gora.geode.store.GeodeStore; +import org.apache.gora.geode.store.GeodeStoreParameters; +import org.apache.gora.persistency.Persistent; +import org.apache.gora.store.DataStore; +import org.apache.gora.store.DataStoreFactory; +import org.apache.gora.util.GoraException; +import org.testcontainers.containers.GenericContainer; + +import java.util.Properties; + +import static org.apache.gora.geode.store.GeodeStoreParameters.GEODE_SERVER_PORT; + +/** + * Helper class for third party tests using gora-geode backend. + * @see GoraTestDriver for test specifics. + * This driver is the base for all test cases that require an Geode server. + * In this case we use docker container. A docker container is run before tests + * and it is stopped after tests. + * + */ +public class GoraGeodeTestDriver extends GoraTestDriver { + + private final GenericContainer GeodeContainer; + private Properties properties = DataStoreFactory.createProps(); + + /** + * Default constructor + */ + public GoraGeodeTestDriver(GenericContainer GeodeContainer) { + super(GeodeStore.class); + this.GeodeContainer = GeodeContainer; + } + + @Override + public void setUpClass() { + log.info("Setting up Geode Test Driver"); + properties.put(GeodeStoreParameters.GEODE_SERVER_HOST, GeodeContainer.getContainerIpAddress()); + properties.put(GEODE_SERVER_PORT, GeodeContainer.getMappedPort(10334).toString()); + } + + @Override + public void tearDownClass() { + log.info("Teardown Geode test driver"); + } + + /** + * Instantiate a new {@link DataStore}. Uses 'null' schema. + * + * @param keyClass The key class. + * @param persistentClass The value class. + * @return A new store instance. + * @throws GoraException + */ + @Override + public DataStore createDataStore(Class keyClass, Class persistentClass) + throws GoraException { + + final DataStore dataStore = DataStoreFactory + .createDataStore((Class>) dataStoreClass, keyClass, persistentClass, conf, + properties); + dataStores.add(dataStore); + log.info("Datastore for {} was added.", persistentClass); + return dataStore; + } + +} diff --git a/gora-geode/src/test/java/store/TestGeodeStore.java b/gora-geode/src/test/java/store/TestGeodeStore.java new file mode 100644 index 00000000..421a7666 --- /dev/null +++ b/gora-geode/src/test/java/store/TestGeodeStore.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package store; + + + +import org.apache.avro.util.Utf8; +import org.apache.gora.examples.WebPageDataCreator; +import org.apache.gora.examples.generated.WebPage; +import org.apache.gora.geode.GeodeStartupLogWaitStrategy; +import org.apache.gora.geode.GoraGeodeTestDriver; +import org.apache.gora.geode.query.GeodeResult; +import org.apache.gora.query.Query; +import org.apache.gora.store.DataStoreTestBase; +import org.apache.gora.util.GoraException; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.testcontainers.containers.GenericContainer; + + + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.time.Duration; + +import static org.junit.Assert.*; + +/** + * Tests extending {@link DataStoreTestBase} + * which run the base JUnit test suite for Gora. + */ +public class TestGeodeStore extends DataStoreTestBase { + + private static final String DOCKER_CONTAINER_NAME = "apachegeode/" + + "" + + "geode:1.15.0"; + + /** + * JUnit integration testing with Docker and Testcontainers + */ + @ClassRule + public static GenericContainer Geode_CONTAINER = new GenericContainer(DOCKER_CONTAINER_NAME) + .withExposedPorts(10334) + .waitingFor(new GeodeStartupLogWaitStrategy()) + .withStartupTimeout(Duration.ofSeconds(300)); + + + // void withCommand("tail","-f","/dev/null").start(); + + // geode = new GenericContainer<>("apachegeode/geode:1.12.0"); + + + static { + try { + setTestDriver(new GoraGeodeTestDriver(Geode_CONTAINER)); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @Test + public void testPutAndGet() throws GoraException { + WebPage page = webPageStore.newPersistent(); + + // Write webpage data + page.setUrl(new Utf8("http://example.com")); + byte[] contentBytes = "example content in example.com".getBytes(Charset.defaultCharset()); + ByteBuffer buff = ByteBuffer.wrap(contentBytes); + page.setContent(buff); + webPageStore.put("com.example/http", page); + webPageStore.flush(); + + WebPage storedPage = webPageStore.get("com.example/http"); + + assertNotNull(storedPage); + assertEquals(page.getUrl(), storedPage.getUrl()); + } + + @Test + public void testCreateAndDeleteSchema() throws IOException { + WebPage page = webPageStore.newPersistent(); + + // Write webpage data + page.setUrl(new Utf8("http://example.com")); + webPageStore.put("com.example/http", page); + webPageStore.flush(); + + assertEquals("WebPage isn't created.", page.getUrl(), webPageStore.get("com.example/http").getUrl()); + + webPageStore.deleteSchema(); + + assertNull(webPageStore.get("com.example/http")); + } + + @Test + public void testGetSchemaName() throws IOException { + assertEquals("WebPage", webPageStore.getSchemaName()); + assertEquals("Employee", employeeStore.getSchemaName()); + } + + @Test + public void testExecute() throws IOException { + WebPageDataCreator.createWebPageData(webPageStore); + + final Query query = webPageStore.newQuery(); + + int limit = 5; + query.setLimit(limit); + GeodeResult result = (GeodeResult) webPageStore.execute(query); + assertEquals(limit, result.size()); + + limit = 10; + query.setLimit(limit); + result = (GeodeResult) webPageStore.execute(query); + assertEquals(limit, result.size()); + + } + + /** + * By design, you cannot update a Geode document blindly, you can only attempt to update a specific revision of a document. FIXME + */ + @Test + @Ignore + public void testUpdate() throws Exception { + //By design, you cannot update a Geode document blindly, you can only attempt to update a specific revision of a document. FIXME + } + + @Ignore("GeodeStore doesn't support 3 types union field yet") + @Override + public void testGet3UnionField() throws Exception { + // GeodeStore doesn't support 3 types union field yet + } + + @Ignore("Skip until GORA-66 is fixed: need better semantic for end/start keys") + @Override + public void testDeleteByQueryFields() throws IOException { + // Skip until GORA-66 is fixed: need better semantic for end/start keys + } + +} From 8e4fbee5b192227e9b3af5c39b2c59cd0ecd1e2d Mon Sep 17 00:00:00 2001 From: himanshu Date: Sat, 17 Sep 2022 13:26:24 -0700 Subject: [PATCH 6/7] Test Cases for Apache Geode --- gora-geode/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gora-geode/pom.xml b/gora-geode/pom.xml index 02e974ef..420e90ca 100644 --- a/gora-geode/pom.xml +++ b/gora-geode/pom.xml @@ -77,6 +77,11 @@ hadoop-minicluster test + + org.testcontainers + testcontainers + test + \ No newline at end of file From ea275745ee40fb6e351f6e4d972997411d6cfde7 Mon Sep 17 00:00:00 2001 From: himanshu Date: Sun, 18 Sep 2022 14:10:01 -0700 Subject: [PATCH 7/7] Replaced wildcard import statements with only required ones --- .../apache/gora/geode/store/GeodeStore.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java index 3b7bdfea..97136c1e 100644 --- a/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java +++ b/gora-geode/src/main/java/org/apache/gora/geode/store/GeodeStore.java @@ -1,6 +1,10 @@ package org.apache.gora.geode.store; -import org.apache.geode.cache.*; +import org.apache.geode.cache.Region; +import org.apache.geode.cache.CacheFactory; +import org.apache.geode.cache.RegionFactory; +import org.apache.geode.cache.RegionShortcut; +import org.apache.geode.cache.Cache; import org.apache.geode.cache.client.ClientCache; import org.apache.geode.cache.client.ClientCacheFactory; import org.apache.gora.geode.query.GeodeQuery; @@ -14,11 +18,20 @@ import org.apache.gora.util.GoraException; import java.io.IOException; -import java.util.*; +import java.util.NavigableSet; +import java.util.Properties; +import java.util.List; +import java.util.ArrayList; +import java.util.Collections; import java.util.concurrent.ConcurrentSkipListSet; import static org.apache.geode.cache.RegionShortcut.REPLICATE; -import static org.apache.gora.geode.store.GeodeStoreParameters.*; +import static org.apache.gora.geode.store.GeodeStoreParameters.GEODE_USERNAME; +import static org.apache.gora.geode.store.GeodeStoreParameters.GEODE_SERVER_PORT; +import static org.apache.gora.geode.store.GeodeStoreParameters.GEODE_SERVER_HOST; +import static org.apache.gora.geode.store.GeodeStoreParameters.GEODE_PASSWORD; +import static org.apache.gora.geode.store.GeodeStoreParameters.GEODE_REGION_SHORTCUT; +import static org.apache.gora.geode.store.GeodeStoreParameters.PREFERRED_SCHEMA_NAME; public class GeodeStore extends DataStoreBase {