Postgres tests

Makefile

@@ -208,4 +208,12 @@ test-onchain:
 	${GOBIN} test -v -count 1 -tags="${BUILD_TAGS}" github.com/waku-org/go-waku/waku/v2/protocol/rln
 
 test-onchain-with-race:
-	${GOBIN} test -race -v -count 1 -tags="${BUILD_TAGS}" github.com/waku-org/go-waku/waku/v2/protocol/rln
+	${GOBIN} test -race -v -count 1 -tags="${BUILD_TAGS}" github.com/waku-org/go-waku/waku/v2/protocol/rln
+
+test-postgres: PG_BUILD_TAGS = ${BUILD_TAGS} include_postgres_tests
+test-postgres:
+	${GOBIN} test -p 1 -v -count 1 -tags="${PG_BUILD_TAGS}" github.com/waku-org/go-waku/waku/persistence
+
+test-postgres-with-race:
+	${GOBIN} test -race -p 1 -v -count 1 -tags="${PG_BUILD_TAGS}" github.com/waku-org/go-waku/waku/persistence
+

ci/Jenkinsfile.tests

@@ -30,6 +30,8 @@ pipeline {
 
   environment {
     TARGET  = 'tests'
+    DB_CONT  = "go-waku-test-db-${env.EXECUTOR_NUMBER.toInteger() + 1}"
+    DB_PORT  = "${5432 + env.EXECUTOR_NUMBER.toInteger()}"
     REPO    = "${env.WORKSPACE}/src/github.com/waku-org/go-waku"
     GOCACHE = "${env.WORKSPACE_TMP}/go-build"
     GOPATH  = "${env.WORKSPACE}/go"
@@ -58,6 +60,28 @@ pipeline {
       } } }
     }
 
+    stage('PostgresSQL') {
+      environment {
+        TEST_DB_PORT = "${env.DB_PORT}"
+      }
+      steps { script { dir(env.REPO) {
+        db = docker.image('postgres:9.6-alpine').withRun([
+          "--name=${DB_CONT}",
+          "--env=POSTGRES_HOST_AUTH_METHOD=trust",
+          "--publish=${DB_PORT}:${DB_PORT}",
+        ].join(' '), "-p ${DB_PORT}") { c ->
+          if (params.RACE) {
+            nix.develop('make test-postgres-with-race', pure: false)
+          }else {
+            nix.develop('make test-postgres', pure: false)
+          }
+        }
+      } } }
+      post { cleanup { /* Leftover DB containers. */
+        sh "docker rm ${DB_CONT} || true"
+      } }
+    }
+
     stage('Ganache') {
       steps { script {
         ganache = docker.image(

cmd/waku/node.go

@@ -18,7 +18,6 @@ import (
 	"github.com/pbnjay/memory"
 	"github.com/prometheus/client_golang/prometheus"
 
-	"github.com/waku-org/go-waku/waku/persistence/sqlite"
 	dbutils "github.com/waku-org/go-waku/waku/persistence/utils"
 	wakupeerstore "github.com/waku-org/go-waku/waku/v2/peerstore"
 	"github.com/waku-org/go-waku/waku/v2/rendezvous"
@@ -203,7 +202,7 @@ func Execute(options NodeOptions) {
 
 	if options.Store.Enable && options.PersistPeers {
 		// Create persistent peerstore
-		queries, err := sqlite.NewQueries("peerstore", db)
+		queries, err := dbutils.NewQueries("peerstore", db)
 		failOnErr(err, "Peerstore")
 
 		datastore := dssql.NewDatastore(db, queries)

waku/persistence/driver_type.go

@@ -0,0 +1,22 @@
+package persistence
+
+import (
+	"database/sql"
+	"reflect"
+)
+
+const (
+	UndefinedDriver = iota
+	PostgresDriver
+	SQLiteDriver
+)
+
+func GetDriverType(db *sql.DB) int {
+	switch reflect.TypeOf(db.Driver()).String() {
+	case "*sqlite3.SQLiteDriver":
+		return SQLiteDriver
+	case "*stdlib.Driver":
+		return PostgresDriver
+	}
+	return UndefinedDriver
+}

waku/persistence/postgres/mock.go

@@ -0,0 +1,50 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"os"
+)
+
+var dbUrlTemplate = "postgres://postgres@localhost:%s/%s?sslmode=disable"
+
+func ResetDefaultTestPostgresDB(dropDBUrl string) error {
+	db, err := sql.Open("pgx", dropDBUrl)
+	if err != nil {
+		return err
+	}
+
+	deletePrevConnectionsSql := `
+	SELECT pid, pg_terminate_backend(pid)
+	FROM pg_stat_activity
+	WHERE datname in ('template1', 'postgres') AND pid <> pg_backend_pid();`
+	_, err = db.Exec(deletePrevConnectionsSql)
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec("DROP DATABASE IF EXISTS postgres;")
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec("CREATE DATABASE postgres;")
+	return err
+}
+
+func NewMockPgDB() *sql.DB {
+	mockPgDBPort := os.Getenv("TEST_DB_PORT")
+
+	//
+	dropDBUrl := fmt.Sprintf(dbUrlTemplate, mockPgDBPort, "template1")
+	if err := ResetDefaultTestPostgresDB(dropDBUrl); err != nil {
+		log.Fatalf("an error '%s' while reseting the db", err)
+	}
+	mockDBUrl := fmt.Sprintf(dbUrlTemplate, mockPgDBPort, "postgres")
+	db, err := sql.Open("pgx", mockDBUrl)
+	if err != nil {
+		log.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
+	}
+	return db
+}

waku/persistence/postgres/postgres.go

@@ -71,5 +71,5 @@ func NewQueries(tbl string, db *sql.DB) (*persistence.Queries, error) {
 	if err != nil {
 		return nil, err
 	}
-	return persistence.CreateQueries(tbl, db), nil
+	return persistence.CreateQueries(tbl), nil
 }

waku/persistence/postgres/postgres_test.go

@@ -0,0 +1,52 @@
+//go:build include_postgres_tests
+// +build include_postgres_tests
+
+package postgres
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/waku-org/go-waku/waku/persistence"
+)
+
+func TestQueries(t *testing.T) {
+	db := persistence.NewMockPgDB()
+
+	queries, err := NewQueries("test_queries", db)
+	require.NoError(t, err)
+
+	query := queries.Delete()
+	require.NotEmpty(t, query)
+
+	query = queries.Exists()
+	require.NotEmpty(t, query)
+
+	query = queries.Get()
+	require.NotEmpty(t, query)
+
+	query = queries.Put()
+	require.NotEmpty(t, query)
+
+	query = queries.Query()
+	require.NotEmpty(t, query)
+
+	query = queries.Prefix()
+	require.NotEmpty(t, query)
+
+	query = queries.Limit()
+	require.NotEmpty(t, query)
+
+	query = queries.Offset()
+	require.NotEmpty(t, query)
+
+	query = queries.GetSize()
+	require.NotEmpty(t, query)
+}
+
+func TestCreateTable(t *testing.T) {
+	db := persistence.NewMockPgDB()
+
+	err := CreateTable(db, "test_create_table")
+	require.NoError(t, err)
+}

waku/persistence/sql_queries.go

@@ -1,7 +1,6 @@
 package persistence
 
 import (
-	"database/sql"
 	"fmt"
 )
 
@@ -20,7 +19,7 @@ type Queries struct {
 
 // CreateQueries Function creates a set of queries for an SQL table.
 // Note: Do not use this function to create queries for a table, rather use <rdb>.NewQueries to create table as well as queries.
-func CreateQueries(tbl string, db *sql.DB) *Queries {
+func CreateQueries(tbl string) *Queries {
 	return &Queries{
 		deleteQuery:  fmt.Sprintf("DELETE FROM %s WHERE key = $1", tbl),
 		existsQuery:  fmt.Sprintf("SELECT exists(SELECT 1 FROM %s WHERE key=$1)", tbl),

waku/persistence/sqlite/mock.go

@@ -0,0 +1,17 @@
+package sqlite
+
+import (
+	"database/sql"
+	"log"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+func NewMockSqliteDB() *sql.DB {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		log.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
+	}
+
+	return db
+}

waku/persistence/sqlite/sqlite.go

@@ -91,5 +91,5 @@ func NewQueries(tbl string, db *sql.DB) (*persistence.Queries, error) {
 	if err != nil {
 		return nil, err
 	}
-	return persistence.CreateQueries(tbl, db), nil
+	return persistence.CreateQueries(tbl), nil
 }

waku/persistence/sqlite/sqlite_test.go

@@ -1,24 +1,13 @@
 package sqlite
 
 import (
-	"database/sql"
-	"log"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
-func NewMock() *sql.DB {
-	db, err := sql.Open("sqlite3", ":memory:")
-	if err != nil {
-		log.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
-	}
-
-	return db
-}
-
 func TestQueries(t *testing.T) {
-	db := NewMock()
+	db := NewMockSqliteDB()
 	queries, err := NewQueries("test_queries", db)
 	require.NoError(t, err)
 
@@ -51,7 +40,7 @@ func TestQueries(t *testing.T) {
 }
 
 func TestCreateTable(t *testing.T) {
-	db := NewMock()
+	db := NewMockSqliteDB()
 
 	err := CreateTable(db, "test_create_table")
 	require.NoError(t, err)

waku/persistence/store.go

@@ -224,8 +224,8 @@ func (d *DBStore) cleanOlderRecords(ctx context.Context) error {
 	// Limit number of records to a max N
 	if d.maxMessages > 0 {
 		start := time.Now()
-		sqlStmt := `DELETE FROM message WHERE id IN (SELECT id FROM message ORDER BY receiverTimestamp DESC LIMIT -1 OFFSET $1)`
-		_, err := d.db.Exec(sqlStmt, d.maxMessages)
+
+		_, err := d.db.Exec(d.getDeleteOldRowsQuery(), d.maxMessages)
 		if err != nil {
 			d.metrics.RecordError(retPolicyFailure)
 			return err
@@ -239,6 +239,17 @@ func (d *DBStore) cleanOlderRecords(ctx context.Context) error {
 	return nil
 }
 
+func (d *DBStore) getDeleteOldRowsQuery() string {
+	sqlStmt := `DELETE FROM message WHERE id IN (SELECT id FROM message ORDER BY receiverTimestamp DESC %s OFFSET $1)`
+	switch GetDriverType(d.db) {
+	case SQLiteDriver:
+		sqlStmt = fmt.Sprintf(sqlStmt, "LIMIT -1")
+	case PostgresDriver:
+		sqlStmt = fmt.Sprintf(sqlStmt, "")
+	}
+	return sqlStmt
+}
+
 func (d *DBStore) checkForOlderRecords(ctx context.Context, t time.Duration) {
 	defer d.wg.Done()
 

waku/persistence/utils/db.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/waku-org/go-waku/waku/persistence"
 	"github.com/waku-org/go-waku/waku/persistence/postgres"
 	"github.com/waku-org/go-waku/waku/persistence/sqlite"
 	"go.uber.org/zap"
@@ -65,3 +66,13 @@ func ExtractDBAndMigration(databaseURL string, dbSettings DBSettings, logger *za
 	return db, migrationFn, nil
 
 }
+
+func NewQueries(tbl string, db *sql.DB) (*persistence.Queries, error) {
+	switch persistence.GetDriverType(db) {
+	case persistence.SQLiteDriver:
+		return sqlite.NewQueries(tbl, db)
+	case persistence.PostgresDriver:
+		return postgres.NewQueries(tbl, db)
+	}
+	return nil, errors.New("unsupported database engine")
+}