#1536 Added working Valkey sample with sorts but no filters.

example/apache-ignite/src/main/resources/logback.xml

@@ -8,6 +8,8 @@
         </encoder>
     </appender>
 
+
+
     <!-- uncomment this whole section if you want socket level logging -->
     <!--appender name="server-inout" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <file>target/vuu-server-inout.log</file>

example/basket/src/main/scala/org/finos/vuu/core/module/basket/service/BasketTradingService.scala

@@ -101,7 +101,7 @@ class BasketTradingService(val table: DataTable, val tableContainer: TableContai
           val constituentTable = tableContainer.getTable(BasketTradingConstituentTable)
           val constituents = constituentTable.primaryKeys.map(key => constituentTable.pullRow(key)).filter(_.get(BTC.InstanceId) == key)
           constituents.foreach(row => {
-            val unitsAsInt = data.asInstanceOf[Int]
+            val unitsAsInt = data.asInstanceOf[String].toInt
             val weighting = row.get(BTC.Weighting)
             val quantity = (weighting.asInstanceOf[Double] * unitsAsInt).toLong
             constituentTable.processUpdate(row.key(), RowWithData(row.key(), Map(BTC.InstanceIdRic -> row.key(), BTC.Quantity -> quantity)), clock.now())

example/main/pom.xml

@@ -25,6 +25,12 @@
             <version>0.9.65-SNAPSHOT</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.finos.vuu</groupId>
+            <artifactId>valkey-redis</artifactId>
+            <version>0.9.65-SNAPSHOT</version>
+        </dependency>
+
         <dependency>
             <groupId>org.finos.vuu</groupId>
             <artifactId>virtualized-table</artifactId>

example/valkey-redis/README.md

@@ -4,14 +4,15 @@ For more information on Valkey go to [Valkey.io](https://valkey.io)
 
 ## Description
 
-This sample is designed to show the Vuu server having a virtualized data implementation backed by Valkey (Redis.)
+This sample is designed to show the Vuu server having a virtualized data table implementation backed by Valkey (Redis, but better)
 
 ## How to Run the Example
 
 Prerequisites (on mac, may differ on Linux or Windows): 
 
-* Have Homebrew installed 
+* Have Homebrew installed (https://brew.sh/)
 * Install Valkey (https://formulae.brew.sh/formula/valkey#default)
+* You should have built the UI already in npm as documented in the README.md in the root folder 
 
 Hardware: This has been coded on an m2 Macbook pro with 16Gb RAM. On smaller machines you might want to decrease the total count of records
 to fix your core/mem profile. On a machine with more RAM you should be able to go higher than the default 25m rows. 
@@ -22,35 +23,69 @@ to fix your core/mem profile. On a machine with more RAM you should be able to g
 /opt/homebrew/opt/valkey/bin/valkey-server /opt/homebrew/etc/valkey.conf
 ```
 
-2. Run the CreateValkeyDataFile static main.
+2. Run the CreateValkeyDataFile static main. This creates a large text file of data in the ./vuu/target folder to import into Valkey 
 ```
 org.finos.vuu.example.valkey.populate.CreateValKeyDataFile
 ```
-3. Import the data into the locally running Valkey (this follows the value mass insert process https://valkey.io/topics/mass-insertion/) THis can take a minute or two. 
+3. Import the data into the locally running Valkey (this follows the value mass insert process https://valkey.io/topics/mass-insertion/) This can take a minute or two. 
 ```
 cat /path/to/checkout/of/vuu/target/valkey-sample-data.txt | valkey-cli --pipe
 ```
-4. Open Valkey command line, check we can load 100_000 records by key.  
+4. Run the ValkeyVuuMain from within the IDE 
+```
+org.finos.vuu.example.valkey.ValkeyVuuMain
+```
+
+## Implementing Filters in Valkey 
+
+| Sample filter                        | Example Implementation in Valkey                                                                                                                                                                                                           | Docco Link |
+|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------| 
+| quantity > 10000                     | ZRANGE quantity.idx 10000 +inf BYSCORE                                                                                                                                                                                                     | https://valkey.io/topics/indexing/ |
+| currency = "GBP"                     | ZRANGE currency.idx "[GBP:" + BYLEX LIMIT 0 +inf                                                                                                                                                                                           | https://valkey.io/topics/indexing/ |
+| currency starts "GB"                 | ZRANGE myindex "[GB" "[GB\xff" BYLEX //need to check this (this does work)                                                                                                                                                                 | https://valkey.io/topics/indexing/ |
+| currency = "GBP" and ric starts "VOD" | There seems to be 2 ways to implemented this, 1 brute force, evaluate 1, then evaluate 2, then look for overlap (not nice), or two create a multifield index. <br/>Another approach might be to use composite indices. Need to check this. | https://valkey.io/topics/indexing/ |
+
+(more to follow)
+
+## Implementing Sorts
+
+Its felt that implementing indices and restricting the fields that people can sort on to one per time is probably the only effective way to enable sorting on datasets > 10m rows. 
+We could drop back to brute force, but that would require potentially a lot of cpu power.
+
+A third option would be a compound index, as discussed in the Valkey indexing page. But we'd likely want to restrict that only supported combinations otherwise the in mem size could balloon. 
+
+## Checking Valkey - Useful Commands 
+
+* Open Valkey command line interface, check we can load 100_000 records by key.  
 ```
 #Load the keys between 100_000 and 101_000
 ZRANGE order.id.pk 100000 101000 BYSCORE
 ```
-5. Load the object from key: 
+* Load the object from key: 
 ```
 HGETALL order:1
 ```
-6. Load the row ids via secondary index:
+* Load the row ids via secondary index:
 ```
-ZRANGE order.currency.idx [: + BYLEX LIMIT 10000 11000
+ZRANGE order.currency.idx [: + BYLEX LIMIT 2210000 2211000
 ```
-7. Load the rows in an auto complete manner using secondary index:
+* Load the rows in an auto complete manner using secondary index:
 ```
-ZRANGE order.currency.idx [GBP: + BYLEX LIMIT 10000 11000
+ZRANGE order.currency.idx [GBP: + BYLEX LIMIT 2210000 2211000
 ```
-8. Getting the full length of the index: 
+* Getting the full length of the index: 
 ```
 ZCOUNT order.currency.idx -inf +inf
 ```
+* Getting the length of a filtered index (max 1_000_000) (so we can give some feedback on filter)
+```
+ZRANGE order.currency.idx [USD: + BYLEX LIMIT 0 1000000
+```
+* Using an index in reverse (cannot use in ByLex fashion)
+```
+ZREVRANGE order.currency.idx 0 1000
+```
+
 
 FYI, the data file is stored here by default (FYI)
 

example/valkey-redis/src/main/resources/application.conf

@@ -0,0 +1,3 @@
+vuu.webroot="vuu-ui/deployed_apps/app-vuu-example"
+vuu.certPath="example/main/src/main/resources/certs/cert.pem"
+vuu.keyPath="example/main/src/main/resources/certs/key.pem"

example/valkey-redis/src/main/resources/certs/cert.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC3jCCAcYCCQDluS1n11toPDANBgkqhkiG9w0BAQsFADAxMQswCQYDVQQGEwJH
+QjEOMAwGA1UECgwFVmVudXUxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMTAzMTgw
+OTUzMzVaFw0zMTAzMTYwOTUzMzVaMDExCzAJBgNVBAYTAkdCMQ4wDAYDVQQKDAVW
+ZW51dTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAoXS+P4cOprmMIByOOD4+O69Qfnjmrh2Raq5DBHmEDObqF6mA2E+o
+SSRvPTdTsufK0tIKQM7iH8ot5P0+ydaVmaHHML7Bys3WYQNfCBm+MZRl79O7yiHX
+tZYtIiJoagiQ4IX19Q00tkaNPqsj5ANBfKVtzokRzXzw+BjZtZpBXWUeETJE4wGB
+5fSQCHMdA8pnblucqsMsmDhGYloH5sBPLIyu9wV/h6AunxhZ37cKlV3U6R8np1ov
+T2fW3fHj6LtrUceMiZKWWgM9NrcJyeSZrqtwWe4lS3VeGZbGG1EZ+oc0m4cbv9FI
+3pz47cK+NTRQq43JQERmA3Yb9G2xRNFTxQIDAQABMA0GCSqGSIb3DQEBCwUAA4IB
+AQB5mMt5eJuwnttcIfv7CKpb9icdYQUsQwN76n33EBetwoBj4r0F2gvxVCUo+9p9
+QTWET7K/nnCxB5ToP031baKZBOU8vIwhAU4WT1Y04ydYLu7vWBCVS7Cp1cygeJ5Z
+3WufvWo8EEqRhAKYFKIj0bj9XP6yOOBSNlmQk5VRRMKLp9aqmT1rV6+IgS6RJQim
+heZVnEUuKkHo6HZygv3Pl6DMXzD+pawcfYh7HYf+70qzmKvfBZkDE8x74KvdQ21V
+7bYRiL+yglzpkYPcAdpUAoHEUwh4DuCcaDUdGgGCBzjh7QJNvel5HccpXSGbrezY
+wYfUHIkVCMlDIRYJJArHJChO
+-----END CERTIFICATE-----

example/valkey-redis/src/main/resources/certs/key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQChdL4/hw6muYwg
+HI44Pj47r1B+eOauHZFqrkMEeYQM5uoXqYDYT6hJJG89N1Oy58rS0gpAzuIfyi3k
+/T7J1pWZoccwvsHKzdZhA18IGb4xlGXv07vKIde1li0iImhqCJDghfX1DTS2Ro0+
+qyPkA0F8pW3OiRHNfPD4GNm1mkFdZR4RMkTjAYHl9JAIcx0DymduW5yqwyyYOEZi
+WgfmwE8sjK73BX+HoC6fGFnftwqVXdTpHyenWi9PZ9bd8ePou2tRx4yJkpZaAz02
+twnJ5Jmuq3BZ7iVLdV4ZlsYbURn6hzSbhxu/0UjenPjtwr41NFCrjclARGYDdhv0
+bbFE0VPFAgMBAAECggEBAIa5QZ8E9YbNwU1yRoJL/eYTzhkajXKwnYzlSGrNcxDJ
+BoEkIa1eGtCdpGzyY9cfb7EnLwMRHsLkFBPF9cS6WC7QPcRBCLhPthlRlNTPllMM
+v1q11Tc/sh2F6LdY+sOvziRr4+bNVDTc/jNS3yn+RYDbEZuL6N0M6o47Pj85Ll3d
+FNHQf/4JHm8H2CpKzumnBeSQoo+gVgd/cY/WL52SuMYwjOWmEsSR1TTn/mp2128u
+0f4CcZ+OlqqI1u3T05V1wCHVkqZiuK4Uz7kjo4TPtNb7QKd8G2MhRxAhu8zoQflY
+T6TEwjw0vCicJAmDuNO6b8MsPpfyfpjH5Ne985bGPwECgYEA1g1SHbnKNSzfa+o2
+2ZuTBNgb0gueRpMH4SkvfrMTmPXbArWiRt0YYJp5D6/1ZTT/cnBtC4ob9bkk2Gi7
+QSsVYkFvn9kwHoF/dNPGG7NALf8uld6izMYuWsaZXGJFv7wV57gqsD13bLyrOypp
+c6xRkMgvlU68g/HYa1jwj8gG0iUCgYEAwRjB21ACWjHqDI+yDD2nj1K0kLHDFI2J
+yLQk8vZWzKknftFs/m/+OSZxh2MpBxV1MhdhIol0T7Iu4FqqxX0RobQMhDGgzIka
+pG8/Gqu3Xz+y0iLaKG3GOI/24zoWhvrhkkJvjWNA42ank9k9UtnpoieQkKtQ5PJO
+Kov9dPquOSECgYEAiZQ4QmjnAfScidUZehZXkZaBkyjqizv3pY3+O/ZH01m6cexi
+qB91iqnew+l51ZoR0fFW9ULIo9BjUUFHkiRp8yRR5bEID6N2U5XGUImO58GLpMJb
+azi6FJhvfASiBhGTFsO1lF99ggNmH3osGzAXoPI6O0JHQptQWfbpb9DGKY0CgYBU
+sA39qNaPRkr3HWKiWbwOcOB+gWgksHrYiqGV6ZJiCmInDaS25tlhqUmrbkMFN1UC
+Fhzk79ISTfyfK0Swfszn1baTl8yFSDBmI+BSSzOOU7qSzPRPPO3tyr4mJEjTBF9A
+LWENapV/5nririlmF+qO0rllqa097EN6B5Gu527X4QKBgGuqnOeiy8UVpTRcN6OL
+h5RRS10utpQcpU1nJHx1Us23As0LJXV+CZuFGNWVnR0GLFABKNBZMV7ZqPHxwmr0
+WBhvxP1xOwCUsla1Cg/dEG+J1q7dPyVfXFK0813Rb5S9omnrQolTczbEh2LnF3RE
+JJmkkwWfm/jOTEl8F3iJQLQN
+-----END PRIVATE KEY-----

example/valkey-redis/src/main/resources/logback.xml

@@ -68,11 +68,10 @@
     <!--logger name="org.finos.vuu.core.groupby.WrappedUpdateHandlingKeyObserver" level="DEBUG">
     </logger-->
 
-    <!--logger name="org.finos.vuu.core.groupby.GroupBySessionTableImpl" level="DEBUG">
-    </logger-->
-
+    <logger name="org.finos.vuu.example.valkey.provider" level="DEBUG">
+    </logger>
 
-    <root level="ERROR">
+    <root level="INFO">
         <appender-ref ref="STDOUT"/>
     </root>
 

example/valkey-redis/src/main/scala/org/finos/vuu/example/valkey/ValkeyVuuMain.scala

@@ -0,0 +1,91 @@
+package org.finos.vuu.example.valkey
+
+import com.typesafe.config.ConfigFactory
+import com.typesafe.scalalogging.StrictLogging
+import org.finos.toolbox.jmx.{JmxInfra, MetricsProvider, MetricsProviderImpl}
+import org.finos.toolbox.lifecycle.LifecycleContainer
+import org.finos.toolbox.time.{Clock, DefaultClock}
+import org.finos.vuu.core._
+import org.finos.vuu.core.module.TableDefContainer
+import org.finos.vuu.core.module.metrics.MetricsModule
+import org.finos.vuu.core.module.typeahead.TypeAheadModule
+import org.finos.vuu.example.valkey.factory.ValkeyConnectionPool
+import org.finos.vuu.example.valkey.module.ValkeyModule
+import org.finos.vuu.net.auth.AlwaysHappyAuthenticator
+import org.finos.vuu.net.http.VuuHttp2ServerOptions
+import org.finos.vuu.net.{AlwaysHappyLoginValidator, Authenticator, LoggedInTokenValidator}
+import org.finos.vuu.plugin.virtualized.VirtualizedTablePlugin
+import org.finos.vuu.state.MemoryBackedVuiStateStore
+
+/*
+//to allow self signed certs
+chrome://flags/#allow-insecure-localhost
+ */
+
+object ValkeyVuuMain extends App with StrictLogging {
+
+  JmxInfra.enableJmx()
+
+  implicit val metrics: MetricsProvider = new MetricsProviderImpl
+  implicit val clock: Clock = new DefaultClock
+  implicit val lifecycle: LifecycleContainer = new LifecycleContainer
+  implicit val tableDefContainer: TableDefContainer = new TableDefContainer(Map())
+
+  logger.info("[VUU] Starting...")
+  val runAsIgniteServer = false
+
+  val store = new MemoryBackedVuiStateStore()
+
+  //store.add(VuiState(VuiHeader("chris", "latest", "chris.latest", clock.now()), VuiJsonState("{ uiState : ['chris','foo'] }")))
+
+  lifecycle.autoShutdownHook()
+
+  val authenticator: Authenticator = new AlwaysHappyAuthenticator
+  val loginTokenValidator: LoggedInTokenValidator = new LoggedInTokenValidator
+
+  val defaultConfig = ConfigFactory.load()
+
+  //look in application.conf for default values
+  val webRoot = defaultConfig.getString("vuu.webroot")
+  val certPath = defaultConfig.getString("vuu.certPath")
+  val keyPath = defaultConfig.getString("vuu.keyPath")
+
+  logger.debug(s"[Ignite] Starting Valkey Pool")
+
+  val valkeyPool = new ValkeyConnectionPool("127.0.0.1", 6379)
+
+  val config = VuuServerConfig(
+    VuuHttp2ServerOptions()
+      //only specify webroot if we want to load the source locally, we'll load it from the jar
+      //otherwise
+      .withWebRoot(webRoot)
+      .withSsl(certPath, keyPath)
+      //don't leave me on in prod pls....
+      .withDirectoryListings(true)
+      .withBindAddress("0.0.0.0")
+      .withPort(8443),
+    VuuWebSocketOptions()
+      .withUri("websocket")
+      .withWsPort(8090)
+      .withWss(certPath, keyPath)
+      .withBindAddress("0.0.0.0"),
+    VuuSecurityOptions()
+      .withAuthenticator(authenticator)
+      .withLoginValidator(new AlwaysHappyLoginValidator),
+    VuuThreadingOptions()
+      .withViewPortThreads(4)
+      .withTreeThreads(4)
+  ).withModule(TypeAheadModule())
+    .withModule(MetricsModule())
+    .withModule(ValkeyModule(valkeyPool))
+    .withPlugin(VirtualizedTablePlugin)
+
+  val vuuServer = new VuuServer(config)
+
+  lifecycle.start()
+
+  logger.info("[VUU] Ready.")
+
+  vuuServer.join()
+
+}

example/valkey-redis/src/main/scala/org/finos/vuu/example/valkey/factory/ValkeyConnectionPool.scala

@@ -0,0 +1,18 @@
+package org.finos.vuu.example.valkey.factory
+
+import io.valkey.{Jedis, JedisPool}
+
+class ValkeyConnectionPool(val ipaddress: String, val port: Int) {
+
+  val config = new io.valkey.JedisPoolConfig()
+  // It is recommended that you set maxTotal = maxIdle = 2*minIdle for best performance
+  config.setMaxTotal(32)
+  config.setMaxIdle(32)
+  config.setMinIdle(16)
+
+  final val jedisPool: JedisPool = new io.valkey.JedisPool(config, ipaddress, port, false)
+
+  def getConnection(): Jedis = {
+    jedisPool.getResource()
+  }
+}

example/valkey-redis/src/main/scala/org/finos/vuu/example/valkey/module/ValkeyModule.scala

@@ -2,10 +2,11 @@ package org.finos.vuu.example.valkey.module
 
 import org.finos.toolbox.lifecycle.LifecycleContainer
 import org.finos.toolbox.time.Clock
-import org.finos.vuu.api.ViewPortDef
+import org.finos.vuu.api.{Index, Indices, ViewPortDef}
 import org.finos.vuu.core.module.ModuleFactory.stringToString
 import org.finos.vuu.core.module.{ModuleFactory, TableDefContainer, ViewServerModule}
 import org.finos.vuu.core.table.{Column, Columns}
+import org.finos.vuu.example.valkey.factory.ValkeyConnectionPool
 import org.finos.vuu.example.valkey.provider.ValkeyVirtualizedProvider
 import org.finos.vuu.net.rpc.RpcHandler
 import org.finos.vuu.plugin.virtualized.api.VirtualizedSessionTableDef
@@ -16,22 +17,30 @@ object ValkeyModule {
 
   private final val NAME = "VALKEY"
 
-  def apply()(implicit clock: Clock, lifecycle: LifecycleContainer, tableDefContainer: TableDefContainer): ViewServerModule = {
+  def apply(pool: ValkeyConnectionPool)(implicit clock: Clock, lifecycle: LifecycleContainer, tableDefContainer: TableDefContainer): ViewServerModule = {
     ModuleFactory.withNamespace(NAME)
       .addSessionTable(
         VirtualizedSessionTableDef(
           name = "bigOrders3",
-          keyField = "orderId",
-          columns
+          keyField = "id",
+          columns,
+          Indices(
+            //id is not required, its implicitly indexed as the primary key
+            Index("ric"),
+            Index("strategy"),
+            Index("parentId"),
+            Index("orderTimeMs"),
+            Index("currency"),
+            Index("quantity"),
+          )
         ),
-        (_, _) => new ValkeyVirtualizedProvider(),
+        (_, _) => new ValkeyVirtualizedProvider(pool),
         (table, _, _, _) => ViewPortDef(
           columns = table.getTableDef.columns,
           service = new NoOpValkeyService()
         )
       ).asModule()
   }
 
-  val columns: Array[Column] = Columns.fromNames("orderId".int(), "ric".string(), "quantity".int(), "price".double(), "side".string(), "strategy".string(), "parentId".int())
-
+  val columns: Array[Column] = Columns.fromNames("id".int(), "ric".string(), "quantity".int(), "price".double(), "side".string(), "strategy".string(), "parentId".int(), "orderTimeMs".long(), "currency".string())
 }