diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0906dea..5a5af08 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,20 +4,14 @@ on: [push] jobs: build: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v11 with: - java-version: 1.8 - - name: Download gradle - run: ./gradlew --version - - name: Run tests - run: ./gradlew test - - name: Run compile - run: ./gradlew compile - - name: Run build - run: ./gradlew build \ No newline at end of file + java-version: openjdk@1.14 + - name: Test + run: sbt +test diff --git a/.github/workflows/publish-manually.yml b/.github/workflows/publish-manually.yml deleted file mode 100644 index 671d213..0000000 --- a/.github/workflows/publish-manually.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Publish-manually - -on: [workflow_dispatch] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - shell: bash - env: - GRADLE_PROPERTIES: ${{ secrets.GRADLE_PROPERTIES }} - SIGNING_GPG_KEY: ${{ secrets.SIGNING_GPG_KEY }} - run: | - base64 -d <<< "$SIGNING_GPG_KEY" > /tmp/secring.gpg - echo "$GRADLE_PROPERTIES" > gradle.properties - - name: Get the tag - id: get_tag - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - name: Build shadowJar for Github - run: ./gradlew -Prelease -Pversion=${{ steps.get_tag.outputs.VERSION }} shadowJar - - name: Release to Github - uses: softprops/action-gh-release@v1 - with: - files: build/libs/secret-provider-${{ steps.get_tag.outputs.VERSION }}-all.jar - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Upload archive - run: ./gradlew -Prelease -Pversion=${{ steps.get_tag.outputs.VERSION }} signArchives uploadArchives - - name: Wait for nexus to settle - run: sleep 30 - - name: Release archive - run: ./gradlew -Prelease -Pversion=${{ steps.get_tag.outputs.VERSION }} closeAndReleaseRepository - diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c29bbaa..6a6ceaf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -11,32 +11,25 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v11 with: - java-version: 1.8 - - shell: bash - env: - GRADLE_PROPERTIES: ${{ secrets.GRADLE_PROPERTIES }} - SIGNING_GPG_KEY: ${{ secrets.SIGNING_GPG_KEY }} - run: | - base64 -d <<< "$SIGNING_GPG_KEY" > /tmp/secring.gpg - echo "$GRADLE_PROPERTIES" > gradle.properties + java-version: openjdk@1.14 - name: Get the tag id: get_tag run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - name: Build shadowJar for Github - run: ./gradlew -Prelease -Pversion=${{ steps.get_tag.outputs.VERSION }} shadowJar + - name: Assembly + run: sbt +assembly + env: + LENSES_TAG_NAME: ${{ steps.get_tag.outputs.VERSION }} - name: Release to Github uses: softprops/action-gh-release@v1 with: - files: build/libs/secret-provider-${{ steps.get_tag.outputs.VERSION }}-all.jar + files: | + target/scala-2.12/secret-provider_2.12-${{ steps.get_tag.outputs.VERSION }}-all.jar + target/scala-2.13/secret-provider_2.13-${{ steps.get_tag.outputs.VERSION }}-all.jar env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build and upload jar to nexus - run: ./gradlew -Prelease -Pversion=${{ steps.get_tag.outputs.VERSION }} signArchives uploadArchives - - name: Wait for nexus to settle - run: sleep 30 - - name: Release to nexus - run: ./gradlew -Prelease -Pversion=${{ steps.get_tag.outputs.VERSION }} closeAndReleaseRepository + LENSES_TAG_NAME: ${{ steps.get_tag.outputs.VERSION }} diff --git a/.gitignore b/.gitignore index 59392e0..aab635d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ /build/ /.classpath /.project +target/ +/.bsp/ +.DS_Store diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 0000000..5a87016 --- /dev/null +++ b/.sbtopts @@ -0,0 +1,4 @@ +-J-Xmx4G +-J-Xms1024M +-J-Xss2M +-J-XX:MaxMetaspaceSize=2G \ No newline at end of file diff --git a/README.md b/README.md index 083f04f..16d29e1 100644 --- a/README.md +++ b/README.md @@ -6,32 +6,10 @@ Secret provider for Kafka to provide indirect look up of configuration values. - -## Installing - -Maven -```xml - - io.lenses - secret-provider - 2.1.3 - -``` - -SBT -```bash -libraryDependencies += "io.lenses" % "secret-provider" % "2.1.3" -``` - -Gradle -```bash -compile 'io.lenses:secret-provider:2.1.3' -``` - ## Description External secret providers allow for indirect references to be placed in an -applications configuration, so for example, that secrets are not exposed in the +applications' configuration, so for example, that secrets are not exposed in the Worker API endpoints of Kafka Connect. For [Documentation](https://docs.lenses.io/4.0/integrations/connectors/secret-providers/). @@ -46,28 +24,16 @@ and submit a pull request. Thanks! ### Building -***Requires gradle 6.0 to build.*** - -To build - -```bash -gradle compile -``` +***Requires SBT to build.*** -To test +To build the (scala 2.12 and 2.13) assemblies for use with Kafka Connect (also runs tests): ```bash -gradle test +sbt +assembly ``` -To create a fat jar +To run tests: ```bash -gradle shadowJar -``` - -You can also use the gradle wrapper - -``` -./gradlew shadowJar +sbt +test ``` diff --git a/build.gradle b/build.gradle deleted file mode 100644 index c5da531..0000000 --- a/build.gradle +++ /dev/null @@ -1,223 +0,0 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -plugins { - id "com.github.maiflai.scalatest" version "0.26" - id "com.github.johnrengelman.shadow" version "5.2.0" - id 'io.codearte.nexus-staging' version '0.21.2' - id "net.researchgate.release" version "2.8.1" -} - -allprojects { - group = 'io.lenses' - version = project.version - description = "connect-secret-provider" - apply plugin: 'java' - apply plugin: 'scala' - apply plugin: 'com.github.johnrengelman.shadow' - apply plugin: 'maven' - apply plugin: 'maven-publish' - apply plugin: 'signing' - apply plugin: 'com.github.maiflai.scalatest' - apply plugin: 'io.codearte.nexus-staging' - - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - - - ext { - scalaMajorVersion = "2.12" - scala = "2.12.10" - scalaCheck = "1.14.3" - scalaLoggingVersion = "3.9.2" - kafkaVersion = "2.5.0" - vaultVersion = "5.1.0" - azureVersion = "1.22.0" - azureKeyVaultVersion = "4.1.1" - azureIdentityVersion = "1.0.5" - awsSecretsVersion = "1.11.762" - - //test - scalaTest = "3.1.1" - mockitoVersion = "1.13.0" - pegDownVersion = "1.1.0" - slf4jVersion = "1.7.26" - commonsIOVersion = "1.3.2" - jettyVersion = "9.4.19.v20190610" - testContainersVersion = "1.12.3" - flexmarkVersion = "0.35.10" - - gitCommitHash = ("git rev-parse HEAD").execute().text.trim() - gitTag = ("git describe --abbrev=0 --tags").execute().text.trim() - gitRepo = ("git remote get-url origin").execute().text.trim() - } - - repositories { - mavenLocal() - mavenCentral() - } - - jar { - manifest { - attributes( - "Version": project.version, - "Kafka-Version": "${kafkaVersion}", - "Created-By" : "Lenses", - "Created-At" : new Date().format("YYYYMMDDHHmm"), - "Git-Repo": "${gitRepo}", - "Git-Commit-Hash": "${gitCommitHash}", - "Git-Tag": "${gitTag}", - "Docs" : "https://docs.lenses.io/connectors/" - ) - } - } - - shadowJar { - archiveFileName = "${project.name}-${project.version}-all.jar" - zip64 true - mergeServiceFiles() - - manifest { - attributes( - "Version": project.version, - "Kafka-Version": "${kafkaVersion}", - "Created-By" : "Lenses", - "Created-At" : new Date().format("YYYYMMDDHHmm"), - "Git-Repo": "${gitRepo}", - "Git-Commit-Hash": "${gitCommitHash}", - "Git-Tag": "${gitTag}", - "Docs" : "https://docs.lenses.io/connectors/" - ) - } - - dependencies { - exclude(dependency("org.apache.avro:.*")) - exclude(dependency("org.apache.kafka:.*")) - exclude(dependency("io.confluent:.*")) - exclude(dependency("org.apache.kafka:.*")) - exclude(dependency("org.apache.zookeeper:.*")) - } - } - - dependencies { - compile "org.scala-lang:scala-library:${scala}" - compile "org.scala-lang:scala-compiler:${scala}" - compile "com.typesafe.scala-logging:scala-logging_${scalaMajorVersion}:${scalaLoggingVersion}" - compile "org.apache.kafka:connect-api:${kafkaVersion}" - compile "com.bettercloud:vault-java-driver:${vaultVersion}" - compile "com.azure:azure-security-keyvault-secrets:${azureKeyVaultVersion}" - compile "com.azure:azure-identity:${azureIdentityVersion}" - compile "com.amazonaws:aws-java-sdk-secretsmanager:${awsSecretsVersion}" - - testImplementation "org.mockito:mockito-scala_${scalaMajorVersion}:${mockitoVersion}" - testImplementation "org.scalacheck:scalacheck_${scalaMajorVersion}:${scalaCheck}" - testImplementation "org.scalatest:scalatest_${scalaMajorVersion}:${scalaTest}" - testImplementation "org.eclipse.jetty:jetty-server:${jettyVersion}" - testImplementation "org.apache.commons:commons-io:${commonsIOVersion}" - testImplementation "org.slf4j:slf4j-api:${slf4jVersion}" - testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}" - testImplementation "org.pegdown:pegdown:${pegDownVersion}" - testImplementation "com.vladsch.flexmark:flexmark-all:${flexmarkVersion}" - } - - test { - minHeapSize '256m' - maxHeapSize '2048m' - } - - task testJar(type: Jar, dependsOn: testClasses) { - archiveFileName = "test-${project.archivesBaseName}" - from sourceSets.test.output - } - - task sourcesJar(type: Jar) { - classifier = 'sources' - from sourceSets.main.allSource - } - - task javadocJar(type: Jar) { - classifier = 'javadoc' - from javadoc - } - - task scaladocJar(type: Jar) { - classifier = 'scaladoc' - from '../LICENSE' - from scaladoc - } - - task compile(dependsOn: 'compileScala') - - task fatJar(dependsOn: [test, shadowJar]) - - artifacts { - archives javadocJar - archives scaladocJar - archives sourcesJar - } - - nexusStaging { - username ossrhUsername - password ossrhPassword - } - - signing { - required { gradle.taskGraph.hasTask("uploadArchives") } - sign configurations.archives - } - - nexusStaging { - username ossrhUsername - password ossrhPassword - } - -// OSSRH publication - if (project.hasProperty('release')) { - uploadArchives { - repositories { - mavenDeployer { - // POM signature - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - // Target repository - repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - pom.project { - name project.name - description project.description - packaging 'jar' - url 'https://github.com/lensesio/connect-secret-provider' - - scm { - connection 'scm:git:https://github.com/lensesio/connect-secret-provider.git' - developerConnection 'scm:git:git@github.com:lensesio/connect-secret-provider.git' - url 'https://github.com/lensesio/connect-secret-provider.git' - } - - licenses { - license { - name 'Apache License 2.0' - url 'https://www.apache.org/licenses/LICENSE-2.0.html' - distribution 'repo' - } - } - - developers { - developer { - id = 'andrewstevenson' - name = 'Andrew Stevenson' - email = 'andrew@lenses.io' - } - } - } - } - } - } - } -} - -project.tasks.compileScala.scalaCompileOptions.additionalParameters = ["-target:jvm-1.8"] -project.tasks.compileTestScala.scalaCompileOptions.additionalParameters = ["-target:jvm-1.8"] diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..49b7e0b --- /dev/null +++ b/build.sbt @@ -0,0 +1,16 @@ +import Settings.modulesSettings + +name := "secret-provider" +javacOptions ++= Seq("--release", "11") +javaOptions ++= Seq("-Xms512M", "-Xmx2048M") +lazy val root = (project in file(".")) + .settings(modulesSettings) + +addCommandAlias( + "validateAll", + ";scalafmtCheck;scalafmtSbtCheck;test:scalafmtCheck;" +) +addCommandAlias( + "formatAll", + ";scalafmt;scalafmtSbt;test:scalafmt;" +) diff --git a/gradle.properties b/gradle.properties deleted file mode 100644 index ce85008..0000000 --- a/gradle.properties +++ /dev/null @@ -1,9 +0,0 @@ -# -# /* -# * Copyright 2017-2020 Lenses.io Ltd -# */ -# - -version=0.0.2 -ossrhUsername=you -ossrhPassword=me diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 490fda8..0000000 Binary files a/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index a4b4429..0000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,5 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew deleted file mode 100755 index 2fe81a7..0000000 --- a/gradlew +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env sh - -# -# Copyright 2015 the original author or authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=`expr $i + 1` - done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat deleted file mode 100644 index 9109989..0000000 --- a/gradlew.bat +++ /dev/null @@ -1,103 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/project/.sbtopts b/project/.sbtopts new file mode 100644 index 0000000..5a87016 --- /dev/null +++ b/project/.sbtopts @@ -0,0 +1,4 @@ +-J-Xmx4G +-J-Xms1024M +-J-Xss2M +-J-XX:MaxMetaspaceSize=2G \ No newline at end of file diff --git a/project/Dependencies.scala b/project/Dependencies.scala new file mode 100644 index 0000000..1d63148 --- /dev/null +++ b/project/Dependencies.scala @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017-2020 Lenses.io Ltd + */ + +import sbt._ + +trait Dependencies { + + object Versions { + + val scalaLoggingVersion = "3.9.5" + val kafkaVersion = "3.4.0" + val vaultVersion = "5.1.0" + val azureKeyVaultVersion = "4.5.2" + val azureIdentityVersion = "1.8.0" + val awsSecretsVersion = "1.12.411" + + //test + val scalaTestVersion = "3.2.15" + val mockitoVersion = "3.2.15.0" + val byteBuddyVersion = "1.14.0" + val slf4jVersion = "2.0.5" + val commonsIOVersion = "1.3.2" + val jettyVersion = "11.0.13" + val testContainersVersion = "1.12.3" + val flexmarkVersion = "0.64.0" + + val scalaCollectionCompatVersion = "2.8.1" + val jakartaServletVersion = "6.0.0" + + } + + object Dependencies { + + import Versions._ + + val `scala-logging` = + "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingVersion + val `kafka-connect-api` = "org.apache.kafka" % "connect-api" % kafkaVersion + val `vault-java-driver` = + "com.bettercloud" % "vault-java-driver" % vaultVersion + val `azure-key-vault` = + "com.azure" % "azure-security-keyvault-secrets" % azureKeyVaultVersion + val `azure-identity` = "com.azure" % "azure-identity" % azureIdentityVersion + val `aws-secrets-manager` = + "com.amazonaws" % "aws-java-sdk-secretsmanager" % awsSecretsVersion + + val `mockito` = "org.scalatestplus" %% "mockito-4-6" % mockitoVersion + val `scalatest` = "org.scalatest" %% "scalatest" % scalaTestVersion + val `jetty` = "org.eclipse.jetty" % "jetty-server" % jettyVersion + val `commons-io` = "org.apache.commons" % "commons-io" % commonsIOVersion + val `flexmark` = "com.vladsch.flexmark" % "flexmark-all" % flexmarkVersion + val `slf4j-api` = "org.slf4j" % "slf4j-api" % slf4jVersion + val `slf4j-simple` = "org.slf4j" % "slf4j-simple" % slf4jVersion + + val `byteBuddy` = "net.bytebuddy" % "byte-buddy" % byteBuddyVersion + val `scalaCollectionCompat` = + "org.scala-lang.modules" %% "scala-collection-compat" % scalaCollectionCompatVersion + val `jakartaServlet` = + "jakarta.servlet" % "jakarta.servlet-api" % jakartaServletVersion + + } + + import Dependencies._ + val secretProviderDeps = Seq( + `scala-logging`, + `kafka-connect-api` % Provided, + `vault-java-driver`, + `azure-key-vault`, + `azure-identity` exclude ("javax.activation", "activation"), + `aws-secrets-manager`, + `scalaCollectionCompat`, + `jakartaServlet` % Test, + `mockito` % Test, + `byteBuddy` % Test, + `scalatest` % Test, + `jetty` % Test, + `commons-io` % Test, + `flexmark` % Test, + `slf4j-api` % Test, + `slf4j-simple` % Test + ) + +} diff --git a/project/Settings.scala b/project/Settings.scala new file mode 100644 index 0000000..6a7edd4 --- /dev/null +++ b/project/Settings.scala @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2017-2020 Lenses.io Ltd + */ + +import sbt.Keys._ +import sbt._ +import sbtassembly.AssemblyKeys._ +import sbtassembly.MergeStrategy + +object Settings extends Dependencies { + + val scala212 = "2.12.14" + val scala213 = "2.13.10" + val scala3 = "3.2.2" + + val nextVersion = "2.1.7" + val artifactVersion = { + sys.env.get("LENSES_TAG_NAME") match { + case Some(tag) => tag + case _ => s"$nextVersion-SNAPSHOT" + } + } + + object ScalacFlags { + val FatalWarnings212 = "-Xfatal-warnings" + val FatalWarnings213 = "-Werror" + val WarnUnusedImports212 = "-Ywarn-unused-import" + val WarnUnusedImports213 = "-Ywarn-unused:imports" + } + + val modulesSettings: Seq[Setting[_]] = Seq( + organization := "io.lenses", + version := artifactVersion, + scalaOrganization := "org.scala-lang", + resolvers ++= Seq( + Resolver.mavenLocal, + Resolver.mavenCentral + ), + libraryDependencies ++= secretProviderDeps, + crossScalaVersions := List( /*scala3, */ scala213 /*scala212*/ ), + Compile / scalacOptions ++= Seq( + "-release:11", + "-encoding", + "utf8", + "-deprecation", + "-unchecked", + "-feature", + "11" + ), + Compile / scalacOptions ++= { + CrossVersion.partialVersion(scalaVersion.value) match { + case Some((2, n)) if n <= 12 => + Seq( + "-Ypartial-unification", + ScalacFlags.WarnUnusedImports212 + ) + case _ => + Seq( + ScalacFlags.WarnUnusedImports213 + ) + } + }, + Compile / console / scalacOptions --= Seq( + ScalacFlags.FatalWarnings212, + ScalacFlags.FatalWarnings213, + ScalacFlags.WarnUnusedImports212, + ScalacFlags.WarnUnusedImports213 + ), + assembly / assemblyJarName := s"secret-provider_${SemanticVersioning(scalaVersion.value).majorMinor}-${artifactVersion}-all.jar", + assembly / assemblyExcludedJars := { + val cp = (assembly / fullClasspath).value + val excludes = Set( + "org.apache.avro", + "org.apache.kafka", + "io.confluent", + "org.apache.zookeeper" + ) + cp filter { f => excludes.exists(f.data.getName.contains(_)) } + }, + assembly / assemblyMergeStrategy := { + case "module-info.class" => MergeStrategy.discard + case x if x.contains("io.netty.versions.properties") => + MergeStrategy.concat + case x => + val oldStrategy = (assembly / assemblyMergeStrategy).value + oldStrategy(x) + }, + Test / fork := true + ) + +} diff --git a/project/Versioning.scala b/project/Versioning.scala new file mode 100644 index 0000000..bcfc176 --- /dev/null +++ b/project/Versioning.scala @@ -0,0 +1,14 @@ +import java.util.regex.{Matcher, Pattern} + +case class SemanticVersioning(version: String) { + + private val versionPattern: Pattern = + Pattern.compile("([1-9]\\d*)\\.(\\d+)\\.(\\d+)(?:-([a-zA-Z0-9]+))?") + private val matcher: Matcher = versionPattern.matcher(version) + + def majorMinor = { + require(matcher.matches()) + s"${matcher.group(1)}.${matcher.group(2)}" + } + +} diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..875272d --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.8.2 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 0000000..a2af00e --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,3 @@ +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") + +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") diff --git a/src/main/scala/io/lenses/connect/secrets/async/AsyncFunctionLoop.scala b/src/main/scala/io/lenses/connect/secrets/async/AsyncFunctionLoop.scala index 2de7ca8..e7575f5 100644 --- a/src/main/scala/io/lenses/connect/secrets/async/AsyncFunctionLoop.scala +++ b/src/main/scala/io/lenses/connect/secrets/async/AsyncFunctionLoop.scala @@ -6,17 +6,14 @@ package io.lenses.connect.secrets.async -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLong - import com.typesafe.scalalogging.StrictLogging +import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} +import java.util.concurrent.{Executors, TimeUnit} import scala.concurrent.duration.Duration class AsyncFunctionLoop(interval: Duration, description: String)(thunk: => Unit) - extends AutoCloseable + extends AutoCloseable with StrictLogging { private val running = new AtomicBoolean(false) @@ -28,7 +25,9 @@ class AsyncFunctionLoop(interval: Duration, description: String)(thunk: => Unit) if (!running.compareAndSet(false, true)) { throw new IllegalStateException(s"$description already running.") } - logger.info(s"Starting $description loop with an interval of ${interval.toMillis}ms.") + logger.info( + s"Starting $description loop with an interval of ${interval.toMillis}ms." + ) executorService.submit(new Runnable { override def run(): Unit = { while (running.get()) { @@ -36,8 +35,7 @@ class AsyncFunctionLoop(interval: Duration, description: String)(thunk: => Unit) Thread.sleep(interval.toMillis) thunk success.incrementAndGet() - } - catch { + } catch { case _: InterruptedException => case t: Throwable => logger.warn("Failed to renew the Kerberos ticket", t) diff --git a/src/main/scala/io/lenses/connect/secrets/config/AWSProviderConfig.scala b/src/main/scala/io/lenses/connect/secrets/config/AWSProviderConfig.scala index 4af13e4..544368e 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/AWSProviderConfig.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/AWSProviderConfig.scala @@ -1,64 +1,64 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import java.util - -import io.lenses.connect.secrets.connect.{AuthMode, _} -import org.apache.kafka.common.config.ConfigDef.{Importance, Type} -import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} - -object AWSProviderConfig { - - val AWS_REGION: String = "aws.region" - val AWS_ACCESS_KEY: String = "aws.access.key" - val AWS_SECRET_KEY: String = "aws.secret.key" - val AUTH_METHOD: String = "aws.auth.method" - - val config: ConfigDef = new ConfigDef() - .define( - AWS_REGION, - Type.STRING, - Importance.HIGH, - "AWS region the Secrets manager is in" - ) - .define( - AWS_ACCESS_KEY, - Type.STRING, - null, - Importance.HIGH, - "AWS access key" - ) - .define( - AWS_SECRET_KEY, - Type.PASSWORD, - null, - Importance.HIGH, - "AWS password key" - ) - .define( - AUTH_METHOD, - Type.STRING, - AuthMode.CREDENTIALS.toString, - Importance.HIGH, - """ - | AWS authenticate method, 'credentials' to use the provided credentials - | or 'default' for the standard AWS provider chain. - | Default is 'credentials' - |""".stripMargin - ) - .define( - FILE_DIR, - Type.STRING, - "", - Importance.MEDIUM, - FILE_DIR_DESC - ) -} - -case class AWSProviderConfig(props: util.Map[String, _]) - extends AbstractConfig(AWSProviderConfig.config, props) +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import io.lenses.connect.secrets.connect.{AuthMode, _} +import org.apache.kafka.common.config.ConfigDef.{Importance, Type} +import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} + +import java.util + +object AWSProviderConfig { + + val AWS_REGION: String = "aws.region" + val AWS_ACCESS_KEY: String = "aws.access.key" + val AWS_SECRET_KEY: String = "aws.secret.key" + val AUTH_METHOD: String = "aws.auth.method" + + val config: ConfigDef = new ConfigDef() + .define( + AWS_REGION, + Type.STRING, + Importance.HIGH, + "AWS region the Secrets manager is in" + ) + .define( + AWS_ACCESS_KEY, + Type.STRING, + null, + Importance.HIGH, + "AWS access key" + ) + .define( + AWS_SECRET_KEY, + Type.PASSWORD, + null, + Importance.HIGH, + "AWS password key" + ) + .define( + AUTH_METHOD, + Type.STRING, + AuthMode.CREDENTIALS.toString, + Importance.HIGH, + """ + | AWS authenticate method, 'credentials' to use the provided credentials + | or 'default' for the standard AWS provider chain. + | Default is 'credentials' + |""".stripMargin + ) + .define( + FILE_DIR, + Type.STRING, + "", + Importance.MEDIUM, + FILE_DIR_DESC + ) +} + +case class AWSProviderConfig(props: util.Map[String, _]) + extends AbstractConfig(AWSProviderConfig.config, props) diff --git a/src/main/scala/io/lenses/connect/secrets/config/AWSProviderSettings.scala b/src/main/scala/io/lenses/connect/secrets/config/AWSProviderSettings.scala index a5465c6..d11b096 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/AWSProviderSettings.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/AWSProviderSettings.scala @@ -1,54 +1,54 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import io.lenses.connect.secrets.connect.AuthMode.AuthMode -import io.lenses.connect.secrets.connect._ -import org.apache.kafka.common.config.types.Password -import org.apache.kafka.connect.errors.ConnectException - -case class AWSProviderSettings( - region: String, - accessKey: String, - secretKey: Password, - authMode: AuthMode, - fileDir: String -) - -import AbstractConfigExtensions._ -object AWSProviderSettings { - def apply(configs: AWSProviderConfig): AWSProviderSettings = { - val region = configs.getStringOrThrowOnNull(AWSProviderConfig.AWS_REGION) - val accessKey = - configs.getStringOrThrowOnNull(AWSProviderConfig.AWS_ACCESS_KEY) - val secretKey = - configs.getPasswordOrThrowOnNull(AWSProviderConfig.AWS_SECRET_KEY) - - val authMode = - getAuthenticationMethod(configs.getString(AWSProviderConfig.AUTH_METHOD)) - - if (authMode == AuthMode.CREDENTIALS) { - if (accessKey.isEmpty) - throw new ConnectException( - s"${AWSProviderConfig.AWS_ACCESS_KEY} not set" - ) - if (secretKey.value().isEmpty) - throw new ConnectException( - s"${AWSProviderConfig.AWS_SECRET_KEY} not set" - ) - } - val fileDir = configs.getString(FILE_DIR) - - new AWSProviderSettings( - region = region, - accessKey = accessKey, - secretKey = secretKey, - authMode = authMode, - fileDir = fileDir - ) - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import io.lenses.connect.secrets.connect.AuthMode.AuthMode +import io.lenses.connect.secrets.connect._ +import org.apache.kafka.common.config.types.Password +import org.apache.kafka.connect.errors.ConnectException + +case class AWSProviderSettings( + region: String, + accessKey: String, + secretKey: Password, + authMode: AuthMode, + fileDir: String +) + +import io.lenses.connect.secrets.config.AbstractConfigExtensions._ +object AWSProviderSettings { + def apply(configs: AWSProviderConfig): AWSProviderSettings = { + val region = configs.getStringOrThrowOnNull(AWSProviderConfig.AWS_REGION) + val accessKey = + configs.getStringOrThrowOnNull(AWSProviderConfig.AWS_ACCESS_KEY) + val secretKey = + configs.getPasswordOrThrowOnNull(AWSProviderConfig.AWS_SECRET_KEY) + + val authMode = + getAuthenticationMethod(configs.getString(AWSProviderConfig.AUTH_METHOD)) + + if (authMode == AuthMode.CREDENTIALS) { + if (accessKey.isEmpty) + throw new ConnectException( + s"${AWSProviderConfig.AWS_ACCESS_KEY} not set" + ) + if (secretKey.value().isEmpty) + throw new ConnectException( + s"${AWSProviderConfig.AWS_SECRET_KEY} not set" + ) + } + val fileDir = configs.getString(FILE_DIR) + + new AWSProviderSettings( + region = region, + accessKey = accessKey, + secretKey = secretKey, + authMode = authMode, + fileDir = fileDir + ) + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/config/AbstractConfigProvider.scala b/src/main/scala/io/lenses/connect/secrets/config/AbstractConfigExtensions.scala similarity index 96% rename from src/main/scala/io/lenses/connect/secrets/config/AbstractConfigProvider.scala rename to src/main/scala/io/lenses/connect/secrets/config/AbstractConfigExtensions.scala index f4b3e19..5cebe46 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/AbstractConfigProvider.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/AbstractConfigExtensions.scala @@ -1,29 +1,29 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import org.apache.kafka.common.config.types.Password -import org.apache.kafka.common.config.AbstractConfig -import org.apache.kafka.connect.errors.ConnectException - -object AbstractConfigExtensions { - implicit class AbstractConfigExtension(val config: AbstractConfig) - extends AnyVal { - def getStringOrThrowOnNull(field: String): String = - Option(config.getString(field)).getOrElse(raiseException(field)) - - def getPasswordOrThrowOnNull(field: String): Password = { - val password = config.getPassword(field) - if (password == null) raiseException(field) - password - } - - private def raiseException(fieldName: String) = throw new ConnectException( - s"$fieldName not set" - ) - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import org.apache.kafka.common.config.AbstractConfig +import org.apache.kafka.common.config.types.Password +import org.apache.kafka.connect.errors.ConnectException + +object AbstractConfigExtensions { + implicit class AbstractConfigExtension(val config: AbstractConfig) + extends AnyVal { + def getStringOrThrowOnNull(field: String): String = + Option(config.getString(field)).getOrElse(raiseException(field)) + + def getPasswordOrThrowOnNull(field: String): Password = { + val password = config.getPassword(field) + if (password == null) raiseException(field) + password + } + + private def raiseException(fieldName: String) = throw new ConnectException( + s"$fieldName not set" + ) + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/config/Aes256DecodingProviderConfig.scala b/src/main/scala/io/lenses/connect/secrets/config/Aes256ProviderConfig.scala similarity index 77% rename from src/main/scala/io/lenses/connect/secrets/config/Aes256DecodingProviderConfig.scala rename to src/main/scala/io/lenses/connect/secrets/config/Aes256ProviderConfig.scala index 717367e..8c164e1 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/Aes256DecodingProviderConfig.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/Aes256ProviderConfig.scala @@ -1,13 +1,14 @@ package io.lenses.connect.secrets.config import io.lenses.connect.secrets.connect.{FILE_DIR, FILE_DIR_DESC} -import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} import org.apache.kafka.common.config.ConfigDef.{Importance, Type} +import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} + import java.util object Aes256ProviderConfig { val SECRET_KEY = "aes256.key" - + val config = new ConfigDef() .define( SECRET_KEY, @@ -26,7 +27,7 @@ object Aes256ProviderConfig { } case class Aes256ProviderConfig(props: util.Map[String, _]) - extends AbstractConfig(Aes256ProviderConfig.config, props) { - def aes256Key: String = getString(Aes256ProviderConfig.SECRET_KEY) - def writeDirectory: String = getString(FILE_DIR) - } + extends AbstractConfig(Aes256ProviderConfig.config, props) { + def aes256Key: String = getString(Aes256ProviderConfig.SECRET_KEY) + def writeDirectory: String = getString(FILE_DIR) +} diff --git a/src/main/scala/io/lenses/connect/secrets/config/AzureProviderConfig.scala b/src/main/scala/io/lenses/connect/secrets/config/AzureProviderConfig.scala index d05fcb4..a8b5c5e 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/AzureProviderConfig.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/AzureProviderConfig.scala @@ -1,65 +1,65 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import java.util - -import io.lenses.connect.secrets.connect._ -import org.apache.kafka.common.config.ConfigDef.{Importance, Type} -import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} - -object AzureProviderConfig { - - val AZURE_CLIENT_ID = "azure.client.id" - val AZURE_TENANT_ID = "azure.tenant.id" - val AZURE_SECRET_ID = "azure.secret.id" - val AUTH_METHOD = "azure.auth.method" - - val config: ConfigDef = new ConfigDef() - .define( - AZURE_CLIENT_ID, - Type.STRING, - null, - Importance.HIGH, - "Azure client id for the service principal" - ) - .define( - AZURE_TENANT_ID, - Type.STRING, - null, - Importance.HIGH, - "Azure tenant id for the service principal" - ) - .define( - AZURE_SECRET_ID, - Type.PASSWORD, - null, - Importance.HIGH, - "Azure secret id for the service principal" - ) - .define( - AUTH_METHOD, - Type.STRING, - AuthMode.CREDENTIALS.toString, - Importance.MEDIUM, - """ - |Azure authenticate method, 'credentials' to use the provided credentials or - |'default' for the standard Azure provider chain - |Default is 'credentials' - |""".stripMargin - ) - .define( - FILE_DIR, - Type.STRING, - "", - Importance.MEDIUM, - FILE_DIR_DESC - ) -} - -case class AzureProviderConfig(props: util.Map[String, _]) - extends AbstractConfig(AzureProviderConfig.config, props) +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import io.lenses.connect.secrets.connect._ +import org.apache.kafka.common.config.ConfigDef.{Importance, Type} +import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} + +import java.util + +object AzureProviderConfig { + + val AZURE_CLIENT_ID = "azure.client.id" + val AZURE_TENANT_ID = "azure.tenant.id" + val AZURE_SECRET_ID = "azure.secret.id" + val AUTH_METHOD = "azure.auth.method" + + val config: ConfigDef = new ConfigDef() + .define( + AZURE_CLIENT_ID, + Type.STRING, + null, + Importance.HIGH, + "Azure client id for the service principal" + ) + .define( + AZURE_TENANT_ID, + Type.STRING, + null, + Importance.HIGH, + "Azure tenant id for the service principal" + ) + .define( + AZURE_SECRET_ID, + Type.PASSWORD, + null, + Importance.HIGH, + "Azure secret id for the service principal" + ) + .define( + AUTH_METHOD, + Type.STRING, + AuthMode.CREDENTIALS.toString, + Importance.MEDIUM, + """ + |Azure authenticate method, 'credentials' to use the provided credentials or + |'default' for the standard Azure provider chain + |Default is 'credentials' + |""".stripMargin + ) + .define( + FILE_DIR, + Type.STRING, + "", + Importance.MEDIUM, + FILE_DIR_DESC + ) +} + +case class AzureProviderConfig(props: util.Map[String, _]) + extends AbstractConfig(AzureProviderConfig.config, props) diff --git a/src/main/scala/io/lenses/connect/secrets/config/AzureProviderSettings.scala b/src/main/scala/io/lenses/connect/secrets/config/AzureProviderSettings.scala index cab4fec..31d58ce 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/AzureProviderSettings.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/AzureProviderSettings.scala @@ -1,63 +1,63 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import com.typesafe.scalalogging.StrictLogging -import io.lenses.connect.secrets.connect.AuthMode.AuthMode -import io.lenses.connect.secrets.connect._ -import org.apache.kafka.common.config.types.Password -import org.apache.kafka.connect.errors.ConnectException - -case class AzureProviderSettings( - clientId: String, - tenantId: String, - secretId: Password, - authMode: AuthMode, - fileDir: String -) - -import AbstractConfigExtensions._ -object AzureProviderSettings extends StrictLogging { - def apply(config: AzureProviderConfig): AzureProviderSettings = { - - val authMode = getAuthenticationMethod( - config.getString(AzureProviderConfig.AUTH_METHOD) - ) - - if (authMode == AuthMode.CREDENTIALS) { - val clientId = - config.getStringOrThrowOnNull(AzureProviderConfig.AZURE_CLIENT_ID) - val tenantId = - config.getStringOrThrowOnNull(AzureProviderConfig.AZURE_TENANT_ID) - val secretId = - config.getPasswordOrThrowOnNull(AzureProviderConfig.AZURE_SECRET_ID) - - if (clientId.isEmpty) - throw new ConnectException( - s"${AzureProviderConfig.AZURE_CLIENT_ID} not set" - ) - if (tenantId.isEmpty) - throw new ConnectException( - s"${AzureProviderConfig.AZURE_TENANT_ID} not set" - ) - if (secretId.value().isEmpty) - throw new ConnectException( - s"${AzureProviderConfig.AZURE_SECRET_ID} not set" - ) - } - - val fileDir = config.getString(FILE_DIR) - - AzureProviderSettings( - clientId = config.getString(AzureProviderConfig.AZURE_CLIENT_ID), - tenantId = config.getString(AzureProviderConfig.AZURE_TENANT_ID), - secretId = config.getPassword(AzureProviderConfig.AZURE_SECRET_ID), - authMode = authMode, - fileDir = fileDir - ) - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import com.typesafe.scalalogging.StrictLogging +import io.lenses.connect.secrets.connect.AuthMode.AuthMode +import io.lenses.connect.secrets.connect._ +import org.apache.kafka.common.config.types.Password +import org.apache.kafka.connect.errors.ConnectException + +case class AzureProviderSettings( + clientId: String, + tenantId: String, + secretId: Password, + authMode: AuthMode, + fileDir: String +) + +import io.lenses.connect.secrets.config.AbstractConfigExtensions._ +object AzureProviderSettings extends StrictLogging { + def apply(config: AzureProviderConfig): AzureProviderSettings = { + + val authMode = getAuthenticationMethod( + config.getString(AzureProviderConfig.AUTH_METHOD) + ) + + if (authMode == AuthMode.CREDENTIALS) { + val clientId = + config.getStringOrThrowOnNull(AzureProviderConfig.AZURE_CLIENT_ID) + val tenantId = + config.getStringOrThrowOnNull(AzureProviderConfig.AZURE_TENANT_ID) + val secretId = + config.getPasswordOrThrowOnNull(AzureProviderConfig.AZURE_SECRET_ID) + + if (clientId.isEmpty) + throw new ConnectException( + s"${AzureProviderConfig.AZURE_CLIENT_ID} not set" + ) + if (tenantId.isEmpty) + throw new ConnectException( + s"${AzureProviderConfig.AZURE_TENANT_ID} not set" + ) + if (secretId.value().isEmpty) + throw new ConnectException( + s"${AzureProviderConfig.AZURE_SECRET_ID} not set" + ) + } + + val fileDir = config.getString(FILE_DIR) + + AzureProviderSettings( + clientId = config.getString(AzureProviderConfig.AZURE_CLIENT_ID), + tenantId = config.getString(AzureProviderConfig.AZURE_TENANT_ID), + secretId = config.getPassword(AzureProviderConfig.AZURE_SECRET_ID), + authMode = authMode, + fileDir = fileDir + ) + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/config/ENVProviderConfig.scala b/src/main/scala/io/lenses/connect/secrets/config/ENVProviderConfig.scala index fc8641e..e21f531 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/ENVProviderConfig.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/ENVProviderConfig.scala @@ -1,26 +1,26 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import java.util - -import io.lenses.connect.secrets.connect.{FILE_DIR, FILE_DIR_DESC} -import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} -import org.apache.kafka.common.config.ConfigDef.{Importance, Type} - -object ENVProviderConfig { - val config = new ConfigDef().define( - FILE_DIR, - Type.STRING, - "", - Importance.MEDIUM, - FILE_DIR_DESC - ) -} - -case class ENVProviderConfig(props: util.Map[String, _]) - extends AbstractConfig(ENVProviderConfig.config, props) +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import io.lenses.connect.secrets.connect.{FILE_DIR, FILE_DIR_DESC} +import org.apache.kafka.common.config.ConfigDef.{Importance, Type} +import org.apache.kafka.common.config.{AbstractConfig, ConfigDef} + +import java.util + +object ENVProviderConfig { + val config = new ConfigDef().define( + FILE_DIR, + Type.STRING, + "", + Importance.MEDIUM, + FILE_DIR_DESC + ) +} + +case class ENVProviderConfig(props: util.Map[String, _]) + extends AbstractConfig(ENVProviderConfig.config, props) diff --git a/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala b/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala index 3df6828..05eb808 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/VaultProviderConfig.scala @@ -1,362 +1,363 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import java.util - -import io.lenses.connect.secrets.connect.{FILE_DIR, FILE_DIR_DESC} -import org.apache.kafka.common.config.ConfigDef.{Importance, Type} -import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, SslConfigs} - -object VaultAuthMethod extends Enumeration { - type VaultAuthMethod = Value - val KUBERNETES, AWSIAM, GCP, USERPASS, LDAP, JWT, CERT, APPROLE, TOKEN, - GITHUB = Value - - def withNameOpt(s: String): Option[Value] = values.find(_.toString == s) -} - -object VaultProviderConfig { - val VAULT_ADDR: String = "vault.addr" - val VAULT_TOKEN: String = "vault.token" - val VAULT_NAMESPACE: String = "vault.namespace" - val VAULT_CLIENT_PEM: String = "vault.client.pem" - val VAULT_PEM: String = "vault.pem" - val VAULT_ENGINE_VERSION = "vault.engine.version" - val AUTH_METHOD: String = "vault.auth.method" - - val VAULT_TRUSTSTORE_LOC: String = - s"vault.${SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG}" - val VAULT_KEYSTORE_LOC: String = - s"vault.${SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG}" - val VAULT_KEYSTORE_PASS: String = - s"vault.${SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG}" - - val KUBERNETES_ROLE: String = "kubernetes.role" - val KUBERNETES_TOKEN_PATH: String = "kubernetes.token.path" - val KUBERNETES_DEFAULT_TOKEN_PATH: String = - "/var/run/secrets/kubernetes.io/serviceaccount/token" - - val APP_ROLE: String = "app.role.id" - val APP_ROLE_SECRET_ID: String = "app.role.secret.id" - - val AWS_ROLE: String = "aws.role" - val AWS_REQUEST_URL: String = "aws.request.url" - val AWS_REQUEST_HEADERS: String = "aws.request.headers" - val AWS_REQUEST_BODY: String = "aws.request.body" - val AWS_MOUNT: String = "aws.mount" - - val GCP_ROLE: String = "gcp.role" - val GCP_JWT: String = "gcp.jwt" - - val LDAP_USERNAME: String = "ldap.username" - val LDAP_PASSWORD: String = "ldap.password" - val LDAP_MOUNT: String = "ldap.mount" - - val USERNAME: String = "username" - val PASSWORD: String = "password" - val UP_MOUNT: String = "mount" - - val JWT_ROLE: String = "jwt.role" - val JWT_PROVIDER: String = "jwt.provider" - val JWT: String = "jwt" - - val CERT_MOUNT: String = "cert.mount" - - val GITHUB_TOKEN: String = "github.token" - val GITHUB_MOUNT: String = "github.mount" - - val TOKEN_RENEWAL: String = "token.renewal.ms" - val TOKEN_RENEWAL_DEFAULT: Int = 600000 - - val config: ConfigDef = new ConfigDef() - .define( - VAULT_ADDR, - ConfigDef.Type.STRING, - "http://localhost:8200", - Importance.HIGH, - "Address of the Vault server" - ) - .define( - VAULT_TOKEN, - ConfigDef.Type.PASSWORD, - null, - Importance.HIGH, - s"Vault app role token. $AUTH_METHOD must be 'token'" - ) - .define( - VAULT_NAMESPACE, - Type.STRING, - "", - Importance.MEDIUM, - "Sets a global namespace to the Vault server instance. Required Vault Enterprize Pro" - ) - .define( - VAULT_KEYSTORE_LOC, - ConfigDef.Type.STRING, - "", - ConfigDef.Importance.HIGH, - SslConfigs.SSL_KEYSTORE_LOCATION_DOC - ) - .define( - VAULT_KEYSTORE_PASS, - ConfigDef.Type.PASSWORD, - "", - ConfigDef.Importance.HIGH, - SslConfigs.SSL_KEYSTORE_PASSWORD_DOC - ) - .define( - VAULT_TRUSTSTORE_LOC, - ConfigDef.Type.STRING, - "", - ConfigDef.Importance.HIGH, - SslConfigs.SSL_TRUSTSTORE_LOCATION_DOC - ) - .define( - VAULT_PEM, - Type.STRING, - "", - Importance.HIGH, - "File containing the Vault server certificate string contents" - ) - .define( - VAULT_CLIENT_PEM, - Type.STRING, - "", - Importance.HIGH, - "File containing the Client certificate string contents" - ) - .define( - VAULT_ENGINE_VERSION, - Type.INT, - 2, - Importance.HIGH, - "KV Secrets Engine version of the Vault server instance. Defaults to 2" - ) - // auth mode - .define( - AUTH_METHOD, - Type.STRING, - "token", - Importance.HIGH, - """ - |The authentication mode for Vault. - |Available values are approle, userpass, kubernetes, cert, token, ldap, gcp, awsiam, jwt, github - | - |""".stripMargin - ) - // app role auth mode - .define( - APP_ROLE, - Type.STRING, - null, - Importance.HIGH, - s"Vault App role id. $AUTH_METHOD must be 'approle'" - ) - .define( - APP_ROLE_SECRET_ID, - Type.PASSWORD, - null, - Importance.HIGH, - s"Vault App role name secret id. $AUTH_METHOD must be 'approle'" - ) - // userpass - .define( - USERNAME, - Type.STRING, - null, - Importance.HIGH, - s"Username to connect to Vault with. $AUTH_METHOD must be 'userpass'" - ) - .define( - PASSWORD, - Type.PASSWORD, - null, - Importance.HIGH, - s"Password for the username. $AUTH_METHOD must be 'uerspass'" - ) - .define( - UP_MOUNT, - Type.STRING, - "userpass", - Importance.HIGH, - s"The mount name of the userpass authentication back end. Defaults to 'userpass'. $AUTH_METHOD must be 'userpass'" - ) - // kubernetes - .define( - KUBERNETES_ROLE, - Type.STRING, - null, - Importance.HIGH, - s"The kubernetes role used for authentication. $AUTH_METHOD must be 'kubernetes'" - ) - .define( - KUBERNETES_TOKEN_PATH, - Type.STRING, - KUBERNETES_DEFAULT_TOKEN_PATH, - Importance.HIGH, - s""" - | s"Path to the service account token. $AUTH_METHOD must be 'kubernetes'. - | Defaults to $KUBERNETES_DEFAULT_TOKEN_PATH - - | $AUTH_METHOD must be ' - - |""".stripMargin - ) - // awsiam - .define( - AWS_ROLE, - Type.STRING, - null, - Importance.HIGH, - s""" - |Name of the role against which the login is being attempted. If role is not specified, t - in endpoint - |looks for a role bearing the name of the AMI ID of the EC2 instance that is trying to - ing the ec2 - |auth method, or the "friendly name" (i.e., role name or username) of the IAM pr - henticated. - |If a matching role is not found, login fails. $AUTH_METHOD - must be 'awsiam' - |""".stripMargin - ) - .define( - AWS_REQUEST_URL, - Type.STRING, - null, - Importance.HIGH, - s""" - |PKCS7 signature of the identity document with all \n characters removed.Base64-encoded Hsed in the signed request. - |Most likely just aHR0cHM6Ly9zdHMuYW1hem9uYXdzLmNvbS8= (base64-encoding of https://sts.amom/) as most requests will - |probably use POST with an empty URI. $AUTH_METHOD must be 'awsiam' - |""".stripMargin - ) - .define( - AWS_REQUEST_HEADERS, - Type.PASSWORD, - null, - Importance.HIGH, - s"Request headers. $AUTH_METHOD must be 'awsiam'" - ) - .define( - AWS_REQUEST_BODY, - Type.PASSWORD, - null, - Importance.HIGH, - s""" - - ded body of the signed request. - |Most likely QWN0aW9uPUdldENhbGxlcklkZ - nNpb249MjAxMS0wNi0xNQ== which is - |the base64 encoding of Action=GetCallerIdentity&Versi - 5. $AUTH_METHOD must be 'awsiam' - |""".stripMargin - ) - .define( - AWS_MOUNT, - Type.STRING, - "aws", - Importance.HIGH, - s"AWS auth mount. $AUTH_METHOD must be 'awsiam'. Default 'aws'" - ) - //ldap - .define( - LDAP_USERNAME, - Type.STRING, - null, - Importance.HIGH, - s"LDAP username to connect to Vault with. $AUTH_METHOD must be 'ldap'" - ) - .define( - LDAP_PASSWORD, - Type.PASSWORD, - null, - Importance.HIGH, - s"LDAP Password for the username. $AUTH_METHOD must be 'ldap'" - ) - .define( - LDAP_MOUNT, - Type.STRING, - "ldap", - Importance.HIGH, - s"The mount name of the ldap authentication back end. Defaults to 'ldap'. $AUTH_METHOD must be 'ldap'" - ) - //jwt - .define( - JWT_ROLE, - Type.STRING, - null, - Importance.HIGH, - s"Role the JWT token belongs to. $AUTH_METHOD must be 'jwt'" - ) - .define( - JWT_PROVIDER, - Type.STRING, - null, - Importance.HIGH, - s"Provider of JWT token. $AUTH_METHOD must be 'jwt'" - ) - .define( - JWT, - Type.PASSWORD, - null, - Importance.HIGH, - s"JWT token. $AUTH_METHOD must be 'jwt'" - ) - //gcp - .define( - GCP_ROLE, - Type.STRING, - null, - Importance.HIGH, - s"The gcp role used for authentication. $AUTH_METHOD must be 'gcp'" - ) - .define( - GCP_JWT, - Type.PASSWORD, - null, - Importance.HIGH, - s"JWT token. $AUTH_METHOD must be 'gcp'" - ) - // cert mount - .define( - CERT_MOUNT, - Type.STRING, - "cert", - Importance.HIGH, - s"The mount name of the cert authentication back end. Defaults to 'cert'. $AUTH_METHOD must be 'cert'" - ) - .define( - GITHUB_TOKEN, - Type.PASSWORD, - null, - Importance.HIGH, - s"The github app-id used for authentication. $AUTH_METHOD must be 'github'" - ) - .define( - GITHUB_MOUNT, - Type.STRING, - "github", - Importance.HIGH, - s"The mount name of the github authentication back end. Defaults to 'cert'. $AUTH_METHOD must be 'github'" - ) - .define( - FILE_DIR, - Type.STRING, - "", - Importance.MEDIUM, - FILE_DIR_DESC - ).define( - TOKEN_RENEWAL, - Type.INT, - TOKEN_RENEWAL_DEFAULT, - Importance.MEDIUM, - "The time in milliseconds to renew the Vault token" - ) -} -case class VaultProviderConfig(props: util.Map[String, _]) - extends AbstractConfig(VaultProviderConfig.config, props) +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import io.lenses.connect.secrets.connect.{FILE_DIR, FILE_DIR_DESC} +import org.apache.kafka.common.config.ConfigDef.{Importance, Type} +import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, SslConfigs} + +import java.util + +object VaultAuthMethod extends Enumeration { + type VaultAuthMethod = Value + val KUBERNETES, AWSIAM, GCP, USERPASS, LDAP, JWT, CERT, APPROLE, TOKEN, + GITHUB = Value + + def withNameOpt(s: String): Option[Value] = values.find(_.toString == s) +} + +object VaultProviderConfig { + val VAULT_ADDR: String = "vault.addr" + val VAULT_TOKEN: String = "vault.token" + val VAULT_NAMESPACE: String = "vault.namespace" + val VAULT_CLIENT_PEM: String = "vault.client.pem" + val VAULT_PEM: String = "vault.pem" + val VAULT_ENGINE_VERSION = "vault.engine.version" + val AUTH_METHOD: String = "vault.auth.method" + + val VAULT_TRUSTSTORE_LOC: String = + s"vault.${SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG}" + val VAULT_KEYSTORE_LOC: String = + s"vault.${SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG}" + val VAULT_KEYSTORE_PASS: String = + s"vault.${SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG}" + + val KUBERNETES_ROLE: String = "kubernetes.role" + val KUBERNETES_TOKEN_PATH: String = "kubernetes.token.path" + val KUBERNETES_DEFAULT_TOKEN_PATH: String = + "/var/run/secrets/kubernetes.io/serviceaccount/token" + + val APP_ROLE: String = "app.role.id" + val APP_ROLE_SECRET_ID: String = "app.role.secret.id" + + val AWS_ROLE: String = "aws.role" + val AWS_REQUEST_URL: String = "aws.request.url" + val AWS_REQUEST_HEADERS: String = "aws.request.headers" + val AWS_REQUEST_BODY: String = "aws.request.body" + val AWS_MOUNT: String = "aws.mount" + + val GCP_ROLE: String = "gcp.role" + val GCP_JWT: String = "gcp.jwt" + + val LDAP_USERNAME: String = "ldap.username" + val LDAP_PASSWORD: String = "ldap.password" + val LDAP_MOUNT: String = "ldap.mount" + + val USERNAME: String = "username" + val PASSWORD: String = "password" + val UP_MOUNT: String = "mount" + + val JWT_ROLE: String = "jwt.role" + val JWT_PROVIDER: String = "jwt.provider" + val JWT: String = "jwt" + + val CERT_MOUNT: String = "cert.mount" + + val GITHUB_TOKEN: String = "github.token" + val GITHUB_MOUNT: String = "github.mount" + + val TOKEN_RENEWAL: String = "token.renewal.ms" + val TOKEN_RENEWAL_DEFAULT: Int = 600000 + + val config: ConfigDef = new ConfigDef() + .define( + VAULT_ADDR, + ConfigDef.Type.STRING, + "http://localhost:8200", + Importance.HIGH, + "Address of the Vault server" + ) + .define( + VAULT_TOKEN, + ConfigDef.Type.PASSWORD, + null, + Importance.HIGH, + s"Vault app role token. $AUTH_METHOD must be 'token'" + ) + .define( + VAULT_NAMESPACE, + Type.STRING, + "", + Importance.MEDIUM, + "Sets a global namespace to the Vault server instance. Required Vault Enterprize Pro" + ) + .define( + VAULT_KEYSTORE_LOC, + ConfigDef.Type.STRING, + "", + ConfigDef.Importance.HIGH, + SslConfigs.SSL_KEYSTORE_LOCATION_DOC + ) + .define( + VAULT_KEYSTORE_PASS, + ConfigDef.Type.PASSWORD, + "", + ConfigDef.Importance.HIGH, + SslConfigs.SSL_KEYSTORE_PASSWORD_DOC + ) + .define( + VAULT_TRUSTSTORE_LOC, + ConfigDef.Type.STRING, + "", + ConfigDef.Importance.HIGH, + SslConfigs.SSL_TRUSTSTORE_LOCATION_DOC + ) + .define( + VAULT_PEM, + Type.STRING, + "", + Importance.HIGH, + "File containing the Vault server certificate string contents" + ) + .define( + VAULT_CLIENT_PEM, + Type.STRING, + "", + Importance.HIGH, + "File containing the Client certificate string contents" + ) + .define( + VAULT_ENGINE_VERSION, + Type.INT, + 2, + Importance.HIGH, + "KV Secrets Engine version of the Vault server instance. Defaults to 2" + ) + // auth mode + .define( + AUTH_METHOD, + Type.STRING, + "token", + Importance.HIGH, + """ + |The authentication mode for Vault. + |Available values are approle, userpass, kubernetes, cert, token, ldap, gcp, awsiam, jwt, github + | + |""".stripMargin + ) + // app role auth mode + .define( + APP_ROLE, + Type.STRING, + null, + Importance.HIGH, + s"Vault App role id. $AUTH_METHOD must be 'approle'" + ) + .define( + APP_ROLE_SECRET_ID, + Type.PASSWORD, + null, + Importance.HIGH, + s"Vault App role name secret id. $AUTH_METHOD must be 'approle'" + ) + // userpass + .define( + USERNAME, + Type.STRING, + null, + Importance.HIGH, + s"Username to connect to Vault with. $AUTH_METHOD must be 'userpass'" + ) + .define( + PASSWORD, + Type.PASSWORD, + null, + Importance.HIGH, + s"Password for the username. $AUTH_METHOD must be 'uerspass'" + ) + .define( + UP_MOUNT, + Type.STRING, + "userpass", + Importance.HIGH, + s"The mount name of the userpass authentication back end. Defaults to 'userpass'. $AUTH_METHOD must be 'userpass'" + ) + // kubernetes + .define( + KUBERNETES_ROLE, + Type.STRING, + null, + Importance.HIGH, + s"The kubernetes role used for authentication. $AUTH_METHOD must be 'kubernetes'" + ) + .define( + KUBERNETES_TOKEN_PATH, + Type.STRING, + KUBERNETES_DEFAULT_TOKEN_PATH, + Importance.HIGH, + s""" + | s"Path to the service account token. $AUTH_METHOD must be 'kubernetes'. + | Defaults to $KUBERNETES_DEFAULT_TOKEN_PATH + + | $AUTH_METHOD must be ' + + |""".stripMargin + ) + // awsiam + .define( + AWS_ROLE, + Type.STRING, + null, + Importance.HIGH, + s""" + |Name of the role against which the login is being attempted. If role is not specified, t + in endpoint + |looks for a role bearing the name of the AMI ID of the EC2 instance that is trying to + ing the ec2 + |auth method, or the "friendly name" (i.e., role name or username) of the IAM pr + henticated. + |If a matching role is not found, login fails. $AUTH_METHOD + must be 'awsiam' + |""".stripMargin + ) + .define( + AWS_REQUEST_URL, + Type.STRING, + null, + Importance.HIGH, + s""" + |PKCS7 signature of the identity document with all \n characters removed.Base64-encoded Hsed in the signed request. + |Most likely just aHR0cHM6Ly9zdHMuYW1hem9uYXdzLmNvbS8= (base64-encoding of https://sts.amom/) as most requests will + |probably use POST with an empty URI. $AUTH_METHOD must be 'awsiam' + |""".stripMargin + ) + .define( + AWS_REQUEST_HEADERS, + Type.PASSWORD, + null, + Importance.HIGH, + s"Request headers. $AUTH_METHOD must be 'awsiam'" + ) + .define( + AWS_REQUEST_BODY, + Type.PASSWORD, + null, + Importance.HIGH, + s""" + + ded body of the signed request. + |Most likely QWN0aW9uPUdldENhbGxlcklkZ + nNpb249MjAxMS0wNi0xNQ== which is + |the base64 encoding of Action=GetCallerIdentity&Versi + 5. $AUTH_METHOD must be 'awsiam' + |""".stripMargin + ) + .define( + AWS_MOUNT, + Type.STRING, + "aws", + Importance.HIGH, + s"AWS auth mount. $AUTH_METHOD must be 'awsiam'. Default 'aws'" + ) + //ldap + .define( + LDAP_USERNAME, + Type.STRING, + null, + Importance.HIGH, + s"LDAP username to connect to Vault with. $AUTH_METHOD must be 'ldap'" + ) + .define( + LDAP_PASSWORD, + Type.PASSWORD, + null, + Importance.HIGH, + s"LDAP Password for the username. $AUTH_METHOD must be 'ldap'" + ) + .define( + LDAP_MOUNT, + Type.STRING, + "ldap", + Importance.HIGH, + s"The mount name of the ldap authentication back end. Defaults to 'ldap'. $AUTH_METHOD must be 'ldap'" + ) + //jwt + .define( + JWT_ROLE, + Type.STRING, + null, + Importance.HIGH, + s"Role the JWT token belongs to. $AUTH_METHOD must be 'jwt'" + ) + .define( + JWT_PROVIDER, + Type.STRING, + null, + Importance.HIGH, + s"Provider of JWT token. $AUTH_METHOD must be 'jwt'" + ) + .define( + JWT, + Type.PASSWORD, + null, + Importance.HIGH, + s"JWT token. $AUTH_METHOD must be 'jwt'" + ) + //gcp + .define( + GCP_ROLE, + Type.STRING, + null, + Importance.HIGH, + s"The gcp role used for authentication. $AUTH_METHOD must be 'gcp'" + ) + .define( + GCP_JWT, + Type.PASSWORD, + null, + Importance.HIGH, + s"JWT token. $AUTH_METHOD must be 'gcp'" + ) + // cert mount + .define( + CERT_MOUNT, + Type.STRING, + "cert", + Importance.HIGH, + s"The mount name of the cert authentication back end. Defaults to 'cert'. $AUTH_METHOD must be 'cert'" + ) + .define( + GITHUB_TOKEN, + Type.PASSWORD, + null, + Importance.HIGH, + s"The github app-id used for authentication. $AUTH_METHOD must be 'github'" + ) + .define( + GITHUB_MOUNT, + Type.STRING, + "github", + Importance.HIGH, + s"The mount name of the github authentication back end. Defaults to 'cert'. $AUTH_METHOD must be 'github'" + ) + .define( + FILE_DIR, + Type.STRING, + "", + Importance.MEDIUM, + FILE_DIR_DESC + ) + .define( + TOKEN_RENEWAL, + Type.INT, + TOKEN_RENEWAL_DEFAULT, + Importance.MEDIUM, + "The time in milliseconds to renew the Vault token" + ) +} +case class VaultProviderConfig(props: util.Map[String, _]) + extends AbstractConfig(VaultProviderConfig.config, props) diff --git a/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala b/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala index cdf4fec..e7b61e9 100644 --- a/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala +++ b/src/main/scala/io/lenses/connect/secrets/config/VaultProviderSettings.scala @@ -1,210 +1,215 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.config - -import com.typesafe.scalalogging.StrictLogging -import io.lenses.connect.secrets.config.VaultAuthMethod.VaultAuthMethod -import io.lenses.connect.secrets.connect._ -import AbstractConfigExtensions._ -import scala.concurrent.duration._ -import io.lenses.connect.secrets.config.VaultProviderConfig.TOKEN_RENEWAL -import org.apache.kafka.common.config.types.Password -import org.apache.kafka.connect.errors.ConnectException - -import scala.concurrent.duration.FiniteDuration -import scala.io.Source -import scala.util.{Failure, Success, Try} - -case class AwsIam( - role: String, - url: String, - headers: Password, - body: Password, - mount: String -) -case class Gcp(role: String, jwt: Password) -case class Jwt(role: String, provider: String, jwt: Password) -case class UserPass(username: String, password: Password, mount: String) -case class Ldap(username: String, password: Password, mount: String) -case class AppRole(role: String, secretId: Password) -case class K8s(role: String, jwt: Password) -case class Cert(mount: String) -case class Github(token: Password, mount: String) - -case class VaultSettings( - addr: String, - namespace: String, - token: Password, - authMode: VaultAuthMethod, - keystoreLoc: String, - keystorePass: Password, - truststoreLoc: String, - pem: String, - clientPem: String, - engineVersion: Int = 2, - appRole: Option[AppRole], - awsIam: Option[AwsIam], - gcp: Option[Gcp], - jwt: Option[Jwt], - userPass: Option[UserPass], - ldap: Option[Ldap], - k8s: Option[K8s], - cert: Option[Cert], - github: Option[Github], - fileDir: String, - tokenRenewal: FiniteDuration -) - -object VaultSettings extends StrictLogging { - def apply(config: VaultProviderConfig): VaultSettings = { - val addr = config.getString(VaultProviderConfig.VAULT_ADDR) - val token = config.getPassword(VaultProviderConfig.VAULT_TOKEN) - val namespace = config.getString(VaultProviderConfig.VAULT_NAMESPACE) - val keystoreLoc = config.getString(VaultProviderConfig.VAULT_KEYSTORE_LOC) - val keystorePass = - config.getPassword(VaultProviderConfig.VAULT_KEYSTORE_PASS) - val truststoreLoc = - config.getString(VaultProviderConfig.VAULT_TRUSTSTORE_LOC) - val pem = config.getString(VaultProviderConfig.VAULT_PEM) - val clientPem = config.getString(VaultProviderConfig.VAULT_CLIENT_PEM) - val engineVersion = config.getInt(VaultProviderConfig.VAULT_ENGINE_VERSION) - - val authMode = VaultAuthMethod.withNameOpt( - config.getString(VaultProviderConfig.AUTH_METHOD).toUpperCase - ) match { - case Some(auth) => auth - case None => - throw new ConnectException( - s"Unsupported ${VaultProviderConfig.AUTH_METHOD}" - ) - } - - val awsIam = - if (authMode.equals(VaultAuthMethod.AWSIAM)) Some(getAWS(config)) - else None - val gcp = - if (authMode.equals(VaultAuthMethod.GCP)) Some(getGCP(config)) else None - val appRole = - if (authMode.equals(VaultAuthMethod.APPROLE)) Some(getAppRole(config)) - else None - val jwt = - if (authMode.equals(VaultAuthMethod.JWT)) Some(getJWT(config)) else None - val k8s = - if (authMode.equals(VaultAuthMethod.KUBERNETES)) Some(getK8s(config)) - else None - val userpass = - if (authMode.equals(VaultAuthMethod.USERPASS)) Some(getUserPass(config)) - else None - val ldap = - if (authMode.equals(VaultAuthMethod.LDAP)) Some(getLDAP(config)) else None - val cert = - if (authMode.equals(VaultAuthMethod.CERT)) Some(getCert(config)) else None - val github = - if (authMode.equals(VaultAuthMethod.GITHUB)) Some(getGitHub(config)) - else None - - val fileDir = config.getString(FILE_DIR) - - val tokenRenewal = config.getInt(TOKEN_RENEWAL).toInt.milliseconds - VaultSettings( - addr = addr, - namespace = namespace, - token = token, - authMode = authMode, - keystoreLoc = keystoreLoc, - keystorePass = keystorePass, - truststoreLoc = truststoreLoc, - pem = pem, - clientPem = clientPem, - engineVersion = engineVersion, - appRole = appRole, - awsIam = awsIam, - gcp = gcp, - jwt = jwt, - userPass = userpass, - ldap = ldap, - k8s = k8s, - cert = cert, - github = github, - fileDir = fileDir, - tokenRenewal = tokenRenewal - ) - } - - def getCert(config: VaultProviderConfig): Cert = - Cert(config.getString(VaultProviderConfig.CERT_MOUNT)) - - def getGitHub(config: VaultProviderConfig): Github = { - val token = config.getPasswordOrThrowOnNull(VaultProviderConfig.GITHUB_TOKEN) - val mount = config.getStringOrThrowOnNull(VaultProviderConfig.GITHUB_MOUNT) - Github(token = token, mount = mount) - } - - def getAWS(config: VaultProviderConfig): AwsIam = { - val role = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_ROLE) - val url = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_URL) - val headers = config.getPasswordOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_HEADERS) - val body = config.getPasswordOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_BODY) - val mount = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_MOUNT) - AwsIam( - role = role, - url = url, - headers = headers, - body = body, - mount = mount - ) - } - - def getAppRole(config: VaultProviderConfig): AppRole = { - val role = config.getStringOrThrowOnNull(VaultProviderConfig.APP_ROLE) - val secretId = config.getPasswordOrThrowOnNull(VaultProviderConfig.APP_ROLE_SECRET_ID) - AppRole(role = role, secretId = secretId) - } - - def getK8s(config: VaultProviderConfig): K8s = { - val role = config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_ROLE) - val path = config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_TOKEN_PATH) - val file = Try(Source.fromFile(path)) match { - case Success(file) => file - case Failure(exception) => - throw new ConnectException( - s"Failed to load kubernetes token file [$path]", - exception - ) - } - val jwt = new Password(file.getLines.mkString) - file.close() - K8s(role = role, jwt = jwt) - } - - def getUserPass(config: VaultProviderConfig): UserPass = { - val user = config.getStringOrThrowOnNull(VaultProviderConfig.USERNAME) - val pass = config.getPasswordOrThrowOnNull(VaultProviderConfig.PASSWORD) - val mount = config.getStringOrThrowOnNull(VaultProviderConfig.UP_MOUNT) - UserPass(username = user, password = pass, mount = mount) - } - - def getLDAP(config: VaultProviderConfig): Ldap = { - val user = config.getStringOrThrowOnNull(VaultProviderConfig.LDAP_USERNAME) - val pass = config.getPasswordOrThrowOnNull(VaultProviderConfig.LDAP_PASSWORD) - val mount = config.getStringOrThrowOnNull(VaultProviderConfig.LDAP_MOUNT) - Ldap(username = user, password = pass, mount = mount) - } - - def getGCP(config: VaultProviderConfig): Gcp = { - val role = config.getStringOrThrowOnNull(VaultProviderConfig.GCP_ROLE) - val jwt = config.getPasswordOrThrowOnNull(VaultProviderConfig.GCP_JWT) - Gcp(role = role, jwt = jwt) - } - - def getJWT(config: VaultProviderConfig): Jwt = { - val role = config.getStringOrThrowOnNull(VaultProviderConfig.JWT_ROLE) - val provider = config.getStringOrThrowOnNull(VaultProviderConfig.JWT_PROVIDER) - val jwt = config.getPasswordOrThrowOnNull(VaultProviderConfig.JWT) - Jwt(role = role, provider = provider, jwt = jwt) - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.config + +import com.typesafe.scalalogging.StrictLogging +import io.lenses.connect.secrets.config.AbstractConfigExtensions._ +import io.lenses.connect.secrets.config.VaultAuthMethod.VaultAuthMethod +import io.lenses.connect.secrets.config.VaultProviderConfig.TOKEN_RENEWAL +import io.lenses.connect.secrets.connect._ +import org.apache.kafka.common.config.types.Password +import org.apache.kafka.connect.errors.ConnectException + +import scala.concurrent.duration.{FiniteDuration, _} +import scala.io.Source +import scala.util.{Failure, Success, Using} + +case class AwsIam( + role: String, + url: String, + headers: Password, + body: Password, + mount: String +) +case class Gcp(role: String, jwt: Password) +case class Jwt(role: String, provider: String, jwt: Password) +case class UserPass(username: String, password: Password, mount: String) +case class Ldap(username: String, password: Password, mount: String) +case class AppRole(role: String, secretId: Password) +case class K8s(role: String, jwt: Password) +case class Cert(mount: String) +case class Github(token: Password, mount: String) + +case class VaultSettings( + addr: String, + namespace: String, + token: Password, + authMode: VaultAuthMethod, + keystoreLoc: String, + keystorePass: Password, + truststoreLoc: String, + pem: String, + clientPem: String, + engineVersion: Int = 2, + appRole: Option[AppRole], + awsIam: Option[AwsIam], + gcp: Option[Gcp], + jwt: Option[Jwt], + userPass: Option[UserPass], + ldap: Option[Ldap], + k8s: Option[K8s], + cert: Option[Cert], + github: Option[Github], + fileDir: String, + tokenRenewal: FiniteDuration +) + +object VaultSettings extends StrictLogging { + def apply(config: VaultProviderConfig): VaultSettings = { + val addr = config.getString(VaultProviderConfig.VAULT_ADDR) + val token = config.getPassword(VaultProviderConfig.VAULT_TOKEN) + val namespace = config.getString(VaultProviderConfig.VAULT_NAMESPACE) + val keystoreLoc = config.getString(VaultProviderConfig.VAULT_KEYSTORE_LOC) + val keystorePass = + config.getPassword(VaultProviderConfig.VAULT_KEYSTORE_PASS) + val truststoreLoc = + config.getString(VaultProviderConfig.VAULT_TRUSTSTORE_LOC) + val pem = config.getString(VaultProviderConfig.VAULT_PEM) + val clientPem = config.getString(VaultProviderConfig.VAULT_CLIENT_PEM) + val engineVersion = config.getInt(VaultProviderConfig.VAULT_ENGINE_VERSION) + + val authMode = VaultAuthMethod.withNameOpt( + config.getString(VaultProviderConfig.AUTH_METHOD).toUpperCase + ) match { + case Some(auth) => auth + case None => + throw new ConnectException( + s"Unsupported ${VaultProviderConfig.AUTH_METHOD}" + ) + } + + val awsIam = + if (authMode.equals(VaultAuthMethod.AWSIAM)) Some(getAWS(config)) + else None + val gcp = + if (authMode.equals(VaultAuthMethod.GCP)) Some(getGCP(config)) else None + val appRole = + if (authMode.equals(VaultAuthMethod.APPROLE)) Some(getAppRole(config)) + else None + val jwt = + if (authMode.equals(VaultAuthMethod.JWT)) Some(getJWT(config)) else None + val k8s = + if (authMode.equals(VaultAuthMethod.KUBERNETES)) Some(getK8s(config)) + else None + val userpass = + if (authMode.equals(VaultAuthMethod.USERPASS)) Some(getUserPass(config)) + else None + val ldap = + if (authMode.equals(VaultAuthMethod.LDAP)) Some(getLDAP(config)) else None + val cert = + if (authMode.equals(VaultAuthMethod.CERT)) Some(getCert(config)) else None + val github = + if (authMode.equals(VaultAuthMethod.GITHUB)) Some(getGitHub(config)) + else None + + val fileDir = config.getString(FILE_DIR) + + val tokenRenewal = config.getInt(TOKEN_RENEWAL).toInt.milliseconds + VaultSettings( + addr = addr, + namespace = namespace, + token = token, + authMode = authMode, + keystoreLoc = keystoreLoc, + keystorePass = keystorePass, + truststoreLoc = truststoreLoc, + pem = pem, + clientPem = clientPem, + engineVersion = engineVersion, + appRole = appRole, + awsIam = awsIam, + gcp = gcp, + jwt = jwt, + userPass = userpass, + ldap = ldap, + k8s = k8s, + cert = cert, + github = github, + fileDir = fileDir, + tokenRenewal = tokenRenewal + ) + } + + def getCert(config: VaultProviderConfig): Cert = + Cert(config.getString(VaultProviderConfig.CERT_MOUNT)) + + def getGitHub(config: VaultProviderConfig): Github = { + val token = + config.getPasswordOrThrowOnNull(VaultProviderConfig.GITHUB_TOKEN) + val mount = config.getStringOrThrowOnNull(VaultProviderConfig.GITHUB_MOUNT) + Github(token = token, mount = mount) + } + + def getAWS(config: VaultProviderConfig): AwsIam = { + val role = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_ROLE) + val url = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_URL) + val headers = + config.getPasswordOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_HEADERS) + val body = + config.getPasswordOrThrowOnNull(VaultProviderConfig.AWS_REQUEST_BODY) + val mount = config.getStringOrThrowOnNull(VaultProviderConfig.AWS_MOUNT) + AwsIam( + role = role, + url = url, + headers = headers, + body = body, + mount = mount + ) + } + + def getAppRole(config: VaultProviderConfig): AppRole = { + val role = config.getStringOrThrowOnNull(VaultProviderConfig.APP_ROLE) + val secretId = + config.getPasswordOrThrowOnNull(VaultProviderConfig.APP_ROLE_SECRET_ID) + AppRole(role = role, secretId = secretId) + } + + def getK8s(config: VaultProviderConfig): K8s = { + val role = + config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_ROLE) + val path = + config.getStringOrThrowOnNull(VaultProviderConfig.KUBERNETES_TOKEN_PATH) + Using(Source.fromFile(path))(_.getLines().mkString) match { + case Failure(exception) => + throw new ConnectException( + s"Failed to load kubernetes token file [$path]", + exception + ) + case Success(fileContents) => + K8s(role = role, jwt = new Password(fileContents)) + } + } + + def getUserPass(config: VaultProviderConfig): UserPass = { + val user = config.getStringOrThrowOnNull(VaultProviderConfig.USERNAME) + val pass = config.getPasswordOrThrowOnNull(VaultProviderConfig.PASSWORD) + val mount = config.getStringOrThrowOnNull(VaultProviderConfig.UP_MOUNT) + UserPass(username = user, password = pass, mount = mount) + } + + def getLDAP(config: VaultProviderConfig): Ldap = { + val user = config.getStringOrThrowOnNull(VaultProviderConfig.LDAP_USERNAME) + val pass = + config.getPasswordOrThrowOnNull(VaultProviderConfig.LDAP_PASSWORD) + val mount = config.getStringOrThrowOnNull(VaultProviderConfig.LDAP_MOUNT) + Ldap(username = user, password = pass, mount = mount) + } + + def getGCP(config: VaultProviderConfig): Gcp = { + val role = config.getStringOrThrowOnNull(VaultProviderConfig.GCP_ROLE) + val jwt = config.getPasswordOrThrowOnNull(VaultProviderConfig.GCP_JWT) + Gcp(role = role, jwt = jwt) + } + + def getJWT(config: VaultProviderConfig): Jwt = { + val role = config.getStringOrThrowOnNull(VaultProviderConfig.JWT_ROLE) + val provider = + config.getStringOrThrowOnNull(VaultProviderConfig.JWT_PROVIDER) + val jwt = config.getPasswordOrThrowOnNull(VaultProviderConfig.JWT) + Jwt(role = role, provider = provider, jwt = jwt) + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/io/FileWriter.scala b/src/main/scala/io/lenses/connect/secrets/io/FileWriter.scala index 894a168..c4a3326 100644 --- a/src/main/scala/io/lenses/connect/secrets/io/FileWriter.scala +++ b/src/main/scala/io/lenses/connect/secrets/io/FileWriter.scala @@ -5,18 +5,14 @@ */ package io.lenses.connect.secrets.io -import java.io.BufferedOutputStream -import java.io.FileOutputStream -import java.nio.file.Path -import java.nio.file.Paths - import com.typesafe.scalalogging.StrictLogging import io.lenses.connect.secrets.utils.WithRetry +import java.io.{BufferedOutputStream, FileOutputStream} +import java.nio.file.attribute.PosixFilePermissions +import java.nio.file.{Files, Path, Paths} import scala.concurrent.duration._ import scala.util.Try -import java.nio.file.attribute.PosixFilePermissions -import java.nio.file.Files trait FileWriter { def write(fileName: String, content: Array[Byte], key: String): Path @@ -28,11 +24,14 @@ class FileWriterOnce(rootPath: Path) with StrictLogging { private val folderPermissions = PosixFilePermissions.fromString("rwx------") - private val filePermissions = PosixFilePermissions.fromString("rw-------") - private val folderAttributes = PosixFilePermissions.asFileAttribute(folderPermissions) - private val fileAttributes = PosixFilePermissions.asFileAttribute(filePermissions) - - if (!rootPath.toFile.exists) Files.createDirectories(rootPath, folderAttributes) + private val filePermissions = PosixFilePermissions.fromString("rw-------") + private val folderAttributes = + PosixFilePermissions.asFileAttribute(folderPermissions) + private val fileAttributes = + PosixFilePermissions.asFileAttribute(filePermissions) + + if (!rootPath.toFile.exists) + Files.createDirectories(rootPath, folderAttributes) def write(fileName: String, content: Array[Byte], key: String): Path = { val fullPath = Paths.get(rootPath.toString, fileName) diff --git a/src/main/scala/io/lenses/connect/secrets/package.scala b/src/main/scala/io/lenses/connect/secrets/package.scala index 8779641..2334ecb 100644 --- a/src/main/scala/io/lenses/connect/secrets/package.scala +++ b/src/main/scala/io/lenses/connect/secrets/package.scala @@ -1,176 +1,172 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets - -import java.io.File -import java.io.FileOutputStream -import java.time.OffsetDateTime -import java.util.Base64 - -import com.typesafe.scalalogging.StrictLogging -import org.apache.kafka.common.config.ConfigData -import org.apache.kafka.connect.errors.ConnectException - -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.util.Failure -import scala.util.Success -import scala.util.Try - -package object connect extends StrictLogging { - - val FILE_ENCODING: String = "file-encoding" - val FILE_DIR: String = "file.dir" - val FILE_DIR_DESC: String = - """ - | Location to write any files for any secrets that need to - | be written to disk. For example java keystores. - | Files will be written under this directory following the - | pattern /file.dir/[path|keyvault]/key - |""".stripMargin - - object AuthMode extends Enumeration { - type AuthMode = Value - val DEFAULT, CREDENTIALS = Value - def withNameOpt(s: String): Option[Value] = values.find(_.toString == s) - } - - object Encoding extends Enumeration { - type Encoding = Value - val BASE64, BASE64_FILE, UTF8, UTF8_FILE = Value - def withNameOpt(s: String): Option[Value] = values.find(_.toString == s) - } - - // get the authmethod - def getAuthenticationMethod(method: String): AuthMode.Value = { - AuthMode.withNameOpt(method.toUpperCase) match { - case Some(auth) => auth - case None => - throw new ConnectException( - s"Unsupported authentication method" - ) - } - } - - // base64 decode secret - def decode(key: String, value: String): String = { - Try(Base64.getDecoder.decode(value)) match { - case Success(decoded) => decoded.map(_.toChar).mkString - case Failure(exception) => - throw new ConnectException( - s"Failed to decode value for key [$key]", - exception - ) - } - } - - def decodeToBytes(key: String, value: String): Array[Byte] = { - Try(Base64.getDecoder.decode(value)) match { - case Success(decoded) => decoded - case Failure(exception) => - throw new ConnectException( - s"Failed to decode value for key [$key]", - exception - ) - } - } - - // decode a key bases on the prefix encoding - def decodeKey( - encoding: Option[Encoding.Value], - key: String, - value: String, - writeFileFn: Array[Byte] => String - ): String = { - encoding.fold(value) { - case Encoding.BASE64 => decode(key, value) - case Encoding.BASE64_FILE => - val decoded = decodeToBytes(key, value) - writeFileFn(decoded) - case Encoding.UTF8 => value - case Encoding.UTF8_FILE => writeFileFn(value.getBytes()) - } - } - - // write secrets to file - private def writer(file: File, payload: Array[Byte], key: String): Unit = { - Try(file.createNewFile()) match { - case Success(_) => - Try(new FileOutputStream(file)) match { - case Success(fos) => - fos.write(payload) - logger.info( - s"Payload written to [${file.getAbsolutePath}] for key [$key]" - ) - - case Failure(exception) => - throw new ConnectException( - s"Failed to write payload to file [${file.getAbsolutePath}] for key [$key]", - exception - ) - } - - case Failure(exception) => - throw new ConnectException( - s"Failed to create file [${file.getAbsolutePath}] for key [$key]", - exception - ) - } - } - - // write secrets to a file - def fileWriter( - fileName: String, - payload: Array[Byte], - key: String, - overwrite: Boolean = false - ): Unit = { - val file = new File(fileName) - file.getParentFile.mkdirs() - - if (file.exists()) { - logger.warn(s"File [$fileName] already exists") - if (overwrite) { - writer(file, payload, key) - } - } else { - writer(file, payload, key) - } - } - - //calculate the min expiry for secrets and return the configData and expiry - def getSecretsAndExpiry( - secrets: Map[String, (String, Option[OffsetDateTime])] - ): (Option[OffsetDateTime], ConfigData) = { - var expiryList = mutable.ListBuffer.empty[OffsetDateTime] - - val data = secrets - .map({ - case (key, (value, expiry)) => - expiry.foreach(e => expiryList.append(e)) - (key, value) - }) - .asJava - - if (expiryList.isEmpty) { - (None, new ConfigData(data)) - } else { - val minExpiry = expiryList.min - val ttl = minExpiry.toInstant.toEpochMilli - OffsetDateTime.now.toInstant - .toEpochMilli - (Some(minExpiry), new ConfigData(data, ttl)) - } - } - - def getFileName( - rootDir: String, - path: String, - key: String, - separator: String - ): String = - s"${rootDir.stripSuffix(separator)}$separator$path$separator${key.toLowerCase}" -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets + +import com.typesafe.scalalogging.StrictLogging +import org.apache.kafka.common.config.ConfigData +import org.apache.kafka.connect.errors.ConnectException + +import java.io.{File, FileOutputStream} +import java.time.OffsetDateTime +import java.util.Base64 +import scala.collection.mutable +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try} + +package object connect extends StrictLogging { + + val FILE_ENCODING: String = "file-encoding" + val FILE_DIR: String = "file.dir" + val FILE_DIR_DESC: String = + """ + | Location to write any files for any secrets that need to + | be written to disk. For example java keystores. + | Files will be written under this directory following the + | pattern /file.dir/[path|keyvault]/key + |""".stripMargin + + object AuthMode extends Enumeration { + type AuthMode = Value + val DEFAULT, CREDENTIALS = Value + def withNameOpt(s: String): Option[Value] = values.find(_.toString == s) + } + + object Encoding extends Enumeration { + type Encoding = Value + val BASE64, BASE64_FILE, UTF8, UTF8_FILE = Value + def withNameOpt(s: String): Option[Value] = values.find(_.toString == s) + } + + // get the authmethod + def getAuthenticationMethod(method: String): AuthMode.Value = { + AuthMode.withNameOpt(method.toUpperCase) match { + case Some(auth) => auth + case None => + throw new ConnectException( + s"Unsupported authentication method" + ) + } + } + + // base64 decode secret + def decode(key: String, value: String): String = { + Try(Base64.getDecoder.decode(value)) match { + case Success(decoded) => decoded.map(_.toChar).mkString + case Failure(exception) => + throw new ConnectException( + s"Failed to decode value for key [$key]", + exception + ) + } + } + + def decodeToBytes(key: String, value: String): Array[Byte] = { + Try(Base64.getDecoder.decode(value)) match { + case Success(decoded) => decoded + case Failure(exception) => + throw new ConnectException( + s"Failed to decode value for key [$key]", + exception + ) + } + } + + // decode a key bases on the prefix encoding + def decodeKey( + encoding: Option[Encoding.Value], + key: String, + value: String, + writeFileFn: Array[Byte] => String + ): String = { + encoding.fold(value) { + case Encoding.BASE64 => decode(key, value) + case Encoding.BASE64_FILE => + val decoded = decodeToBytes(key, value) + writeFileFn(decoded) + case Encoding.UTF8 => value + case Encoding.UTF8_FILE => writeFileFn(value.getBytes()) + } + } + + // write secrets to file + private def writer(file: File, payload: Array[Byte], key: String): Unit = { + Try(file.createNewFile()) match { + case Success(_) => + Try(new FileOutputStream(file)) match { + case Success(fos) => + fos.write(payload) + logger.info( + s"Payload written to [${file.getAbsolutePath}] for key [$key]" + ) + + case Failure(exception) => + throw new ConnectException( + s"Failed to write payload to file [${file.getAbsolutePath}] for key [$key]", + exception + ) + } + + case Failure(exception) => + throw new ConnectException( + s"Failed to create file [${file.getAbsolutePath}] for key [$key]", + exception + ) + } + } + + // write secrets to a file + def fileWriter( + fileName: String, + payload: Array[Byte], + key: String, + overwrite: Boolean = false + ): Unit = { + val file = new File(fileName) + file.getParentFile.mkdirs() + + if (file.exists()) { + logger.warn(s"File [$fileName] already exists") + if (overwrite) { + writer(file, payload, key) + } + } else { + writer(file, payload, key) + } + } + + //calculate the min expiry for secrets and return the configData and expiry + def getSecretsAndExpiry( + secrets: Map[String, (String, Option[OffsetDateTime])] + ): (Option[OffsetDateTime], ConfigData) = { + val expiryList = mutable.ListBuffer.empty[OffsetDateTime] + + val data = secrets + .map({ + case (key, (value, expiry)) => + expiry.foreach(e => expiryList.append(e)) + (key, value) + }) + .asJava + + if (expiryList.isEmpty) { + (None, new ConfigData(data)) + } else { + val minExpiry = expiryList.min + val ttl = minExpiry.toInstant.toEpochMilli - OffsetDateTime.now.toInstant + .toEpochMilli + (Some(minExpiry), new ConfigData(data, ttl)) + } + } + + def getFileName( + rootDir: String, + path: String, + key: String, + separator: String + ): String = + s"${rootDir.stripSuffix(separator)}$separator$path$separator${key.toLowerCase}" +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/AWSHelper.scala b/src/main/scala/io/lenses/connect/secrets/providers/AWSHelper.scala index 26742ad..cb75a56 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/AWSHelper.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/AWSHelper.scala @@ -1,137 +1,151 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.nio.file.FileSystems -import java.nio.file.Paths -import java.time.OffsetDateTime -import java.util.Calendar - -import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials, DefaultAWSCredentialsProviderChain} -import com.amazonaws.services.secretsmanager.model.{DescribeSecretRequest, GetSecretValueRequest} -import com.amazonaws.services.secretsmanager.{AWSSecretsManager, AWSSecretsManagerClientBuilder} -import com.fasterxml.jackson.databind.ObjectMapper -import com.typesafe.scalalogging.StrictLogging -import io.lenses.connect.secrets.config.AWSProviderSettings -import io.lenses.connect.secrets.connect.{decodeKey, getFileName, AuthMode} -import io.lenses.connect.secrets.io.FileWriter -import io.lenses.connect.secrets.io.FileWriterOnce -import io.lenses.connect.secrets.utils.EncodingAndId -import org.apache.kafka.connect.errors.ConnectException - -import scala.collection.JavaConverters._ -import scala.util.{Failure, Success, Try} - -trait AWSHelper extends StrictLogging { - private val separator: String = FileSystems.getDefault.getSeparator - - // initialize the AWS client based on the auth mode - def createClient(settings: AWSProviderSettings): AWSSecretsManager = { - - logger.info( - s"Initializing client with mode [${settings.authMode}]" - ) - - val credentialProvider = settings.authMode match { - case AuthMode.CREDENTIALS => - new AWSStaticCredentialsProvider(new BasicAWSCredentials(settings.accessKey, settings.secretKey.value())) - case _ => - new DefaultAWSCredentialsProviderChain() - } - - AWSSecretsManagerClientBuilder - .standard() - .withCredentials(credentialProvider) - .withRegion(settings.region) - .build() - - } - - // determine the ttl for the secret - private def getTTL( - client: AWSSecretsManager, - secretId: String - ): Option[OffsetDateTime] = { - - // describe to get the ttl - val descRequest: DescribeSecretRequest = - new DescribeSecretRequest().withSecretId(secretId) - - Try(client.describeSecret(descRequest)) match { - case Success(d) => - if (d.getRotationEnabled) { - val lastRotation = d.getLastRotatedDate - val nextRotationInDays = - d.getRotationRules.getAutomaticallyAfterDays - val cal = Calendar.getInstance() - //set to last rotation date - cal.setTime(lastRotation) - //increment - cal.add(Calendar.DAY_OF_MONTH, nextRotationInDays.toInt) - Some( - OffsetDateTime.ofInstant(cal.toInstant, cal.getTimeZone.toZoneId)) - - } else None - - case Failure(exception) => - throw new ConnectException( - s"Failed to describe secret [$secretId]", - exception - ) - } - } - - // get the key value and ttl in the specified secret - def getSecretValue( - client: AWSSecretsManager, - rootDir: String, - secretId: String, - key: String - ): (String, Option[OffsetDateTime]) = { - - // get the secret - Try( - client.getSecretValue(new GetSecretValueRequest().withSecretId(secretId)) - ) match { - case Success(secret) => - val value = - new ObjectMapper() - .readValue( - secret.getSecretString, - classOf[java.util.HashMap[String, String]] - ) - .asScala - .getOrElse( - key, - throw new ConnectException( - s"Failed to look up key [$key] in secret [${secret.getName}]. key not found" - ) - ) - - val fileWriter:FileWriter = new FileWriterOnce(Paths.get(rootDir, secretId)) - // decode the value - val encodingAndId = EncodingAndId.from(key) - ( - decodeKey( - key = key, - value = value, - encoding = encodingAndId.encoding, - writeFileFn = content=>{ - fileWriter.write(key.toLowerCase, content, key).toString - } - ), - getTTL(client, secretId) - ) - - case Failure(exception) => - throw new ConnectException( - s"Failed to look up key [$key] in secret [$secretId}]", - exception - ) - } - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.amazonaws.auth.{ + AWSStaticCredentialsProvider, + BasicAWSCredentials, + DefaultAWSCredentialsProviderChain +} +import com.amazonaws.services.secretsmanager.model.{ + DescribeSecretRequest, + GetSecretValueRequest +} +import com.amazonaws.services.secretsmanager.{ + AWSSecretsManager, + AWSSecretsManagerClientBuilder +} +import com.fasterxml.jackson.databind.ObjectMapper +import com.typesafe.scalalogging.StrictLogging +import io.lenses.connect.secrets.config.AWSProviderSettings +import io.lenses.connect.secrets.connect.{AuthMode, decodeKey} +import io.lenses.connect.secrets.io.{FileWriter, FileWriterOnce} +import io.lenses.connect.secrets.utils.EncodingAndId +import org.apache.kafka.connect.errors.ConnectException + +import java.nio.file.Paths +import java.time.OffsetDateTime +import java.util.Calendar +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try} + +trait AWSHelper extends StrictLogging { + + // initialize the AWS client based on the auth mode + def createClient(settings: AWSProviderSettings): AWSSecretsManager = { + + logger.info( + s"Initializing client with mode [${settings.authMode}]" + ) + + val credentialProvider = settings.authMode match { + case AuthMode.CREDENTIALS => + new AWSStaticCredentialsProvider( + new BasicAWSCredentials( + settings.accessKey, + settings.secretKey.value() + ) + ) + case _ => + new DefaultAWSCredentialsProviderChain() + } + + AWSSecretsManagerClientBuilder + .standard() + .withCredentials(credentialProvider) + .withRegion(settings.region) + .build() + + } + + // determine the ttl for the secret + private def getTTL( + client: AWSSecretsManager, + secretId: String + ): Option[OffsetDateTime] = { + + // describe to get the ttl + val descRequest: DescribeSecretRequest = + new DescribeSecretRequest().withSecretId(secretId) + + Try(client.describeSecret(descRequest)) match { + case Success(d) => + if (d.getRotationEnabled) { + val lastRotation = d.getLastRotatedDate + val nextRotationInDays = + d.getRotationRules.getAutomaticallyAfterDays + val cal = Calendar.getInstance() + //set to last rotation date + cal.setTime(lastRotation) + //increment + cal.add(Calendar.DAY_OF_MONTH, nextRotationInDays.toInt) + Some( + OffsetDateTime.ofInstant(cal.toInstant, cal.getTimeZone.toZoneId) + ) + + } else None + + case Failure(exception) => + throw new ConnectException( + s"Failed to describe secret [$secretId]", + exception + ) + } + } + + // get the key value and ttl in the specified secret + def getSecretValue( + client: AWSSecretsManager, + rootDir: String, + secretId: String, + key: String + ): (String, Option[OffsetDateTime]) = { + + // get the secret + Try( + client.getSecretValue(new GetSecretValueRequest().withSecretId(secretId)) + ) match { + case Success(secret) => + val value = + new ObjectMapper() + .readValue( + secret.getSecretString, + classOf[java.util.HashMap[String, String]] + ) + .asScala + .getOrElse( + key, + throw new ConnectException( + s"Failed to look up key [$key] in secret [${secret.getName}]. key not found" + ) + ) + + val fileWriter: FileWriter = new FileWriterOnce( + Paths.get(rootDir, secretId) + ) + // decode the value + val encodingAndId = EncodingAndId.from(key) + ( + decodeKey( + key = key, + value = value, + encoding = encodingAndId.encoding, + writeFileFn = content => { + fileWriter.write(key.toLowerCase, content, key).toString + } + ), + getTTL(client, secretId) + ) + + case Failure(exception) => + throw new ConnectException( + s"Failed to look up key [$key] in secret [$secretId] due to [${exception.getMessage}]", + exception + ) + } + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/AWSSecretProvider.scala b/src/main/scala/io/lenses/connect/secrets/providers/AWSSecretProvider.scala index f1fa126..0cc4a1a 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/AWSSecretProvider.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/AWSSecretProvider.scala @@ -1,64 +1,66 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.time.OffsetDateTime -import java.util - -import io.lenses.connect.secrets.config.{AWSProviderConfig, AWSProviderSettings} -import io.lenses.connect.secrets.connect.getSecretsAndExpiry -import com.amazonaws.services.secretsmanager.AWSSecretsManager -import org.apache.kafka.common.config.ConfigData -import org.apache.kafka.common.config.provider.ConfigProvider -import org.apache.kafka.connect.errors.ConnectException - -import scala.collection.JavaConverters._ - -class AWSSecretProvider() extends ConfigProvider with AWSHelper { - - var client: Option[AWSSecretsManager] = None - var rootDir: String = "" - - override def get(path: String): ConfigData = - new ConfigData(Map.empty[String, String].asJava) - - // path is expected to be the name of the AWS secret - // keys are expect to be the keys in the payload - override def get(path: String, keys: util.Set[String]): ConfigData = { - - client match { - case Some(awsClient) => - //aws client caches so we don't need to check here - val (expiry, data) = getSecretsAndExpiry( - getSecrets(awsClient, path, keys.asScala.toSet)) - expiry.foreach(exp => - logger.info(s"Min expiry for TTL set to [${exp.toString}]")) - data - - case None => throw new ConnectException("AWS client is not set.") - } - } - - override def close(): Unit = client.foreach(_.shutdown()) - - override def configure(configs: util.Map[String, _]): Unit = { - val settings = AWSProviderSettings(AWSProviderConfig(props = configs)) - rootDir = settings.fileDir - client = Some(createClient(settings)) - } - - def getSecrets( - awsClient: AWSSecretsManager, - path: String, - keys: Set[String]): Map[String, (String, Option[OffsetDateTime])] = { - keys.map { key => - logger.info(s"Looking up value at [$path] for key [$key]") - val (value, expiry) = getSecretValue(awsClient, rootDir, path, key) - (key, (value, expiry)) - }.toMap - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.amazonaws.services.secretsmanager.AWSSecretsManager +import io.lenses.connect.secrets.config.{AWSProviderConfig, AWSProviderSettings} +import io.lenses.connect.secrets.connect.getSecretsAndExpiry +import org.apache.kafka.common.config.ConfigData +import org.apache.kafka.common.config.provider.ConfigProvider +import org.apache.kafka.connect.errors.ConnectException + +import java.time.OffsetDateTime +import java.util +import scala.jdk.CollectionConverters._ + +class AWSSecretProvider() extends ConfigProvider with AWSHelper { + + var client: Option[AWSSecretsManager] = None + var rootDir: String = "" + + override def get(path: String): ConfigData = + new ConfigData(Map.empty[String, String].asJava) + + // path is expected to be the name of the AWS secret + // keys are expect to be the keys in the payload + override def get(path: String, keys: util.Set[String]): ConfigData = { + + client match { + case Some(awsClient) => + //aws client caches so we don't need to check here + val (expiry, data) = getSecretsAndExpiry( + getSecrets(awsClient, path, keys.asScala.toSet) + ) + expiry.foreach(exp => + logger.info(s"Min expiry for TTL set to [${exp.toString}]") + ) + data + + case None => throw new ConnectException("AWS client is not set.") + } + } + + override def close(): Unit = client.foreach(_.shutdown()) + + override def configure(configs: util.Map[String, _]): Unit = { + val settings = AWSProviderSettings(AWSProviderConfig(props = configs)) + rootDir = settings.fileDir + client = Some(createClient(settings)) + } + + def getSecrets( + awsClient: AWSSecretsManager, + path: String, + keys: Set[String] + ): Map[String, (String, Option[OffsetDateTime])] = { + keys.map { key => + logger.info(s"Looking up value at [$path] for key [$key]") + val (value, expiry) = getSecretValue(awsClient, rootDir, path, key) + (key, (value, expiry)) + }.toMap + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelper.scala b/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelper.scala index b1b948a..dfd64be 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelper.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelper.scala @@ -2,13 +2,9 @@ package io.lenses.connect.secrets.providers import java.security.SecureRandom import java.util.Base64 - import javax.crypto.Cipher -import javax.crypto.spec.IvParameterSpec -import javax.crypto.spec.SecretKeySpec - -import scala.util.Failure -import scala.util.Try +import javax.crypto.spec.{IvParameterSpec, SecretKeySpec} +import scala.util.{Failure, Try} private[providers] object Aes256DecodingHelper { @@ -33,10 +29,10 @@ private[providers] object Aes256DecodingHelper { } } -private[providers] class Aes256DecodingHelper private( - key: String, - ivSeparator: String - ) { +private[providers] class Aes256DecodingHelper private ( + key: String, + ivSeparator: String +) { import Aes256DecodingHelper.CHARSET import B64._ @@ -45,15 +41,18 @@ private[providers] class Aes256DecodingHelper private( def decrypt(s: String): Try[String] = for { - (iv, encoded) <- InitializationVector.extractInitialisationVector(s, ivSeparator) + (iv, encoded) <- InitializationVector.extractInitialisationVector( + s, + ivSeparator + ) decoded <- base64Decode(encoded) decrypted <- decryptBytes(iv, decoded) } yield new String(decrypted, CHARSET) private def decryptBytes( - iv: InitializationVector, - bytes: Array[Byte] - ): Try[Array[Byte]] = + iv: InitializationVector, + bytes: Array[Byte] + ): Try[Array[Byte]] = for { cipher <- getCipher(Cipher.DECRYPT_MODE, iv) encrypted <- Try(cipher.doFinal(bytes)) @@ -68,7 +67,7 @@ private[providers] class Aes256DecodingHelper private( } } -private case class InitializationVector private(bytes: Array[Byte]) +private case class InitializationVector private (bytes: Array[Byte]) private object InitializationVector { @@ -85,9 +84,9 @@ private object InitializationVector { } def extractInitialisationVector( - s: String, - ivSeparator: String - ): Try[(InitializationVector, String)] = + s: String, + ivSeparator: String + ): Try[(InitializationVector, String)] = s.indexOf(ivSeparator) match { case -1 => Failure( diff --git a/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingProvider.scala b/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingProvider.scala index faa33e7..43edbac 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingProvider.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/Aes256DecodingProvider.scala @@ -1,27 +1,23 @@ package io.lenses.connect.secrets.providers -import java.nio.file.Paths -import java.util - import io.lenses.connect.secrets.config.Aes256ProviderConfig import io.lenses.connect.secrets.connect.decodeKey -import io.lenses.connect.secrets.connect.Encoding -import io.lenses.connect.secrets.io.FileWriter -import io.lenses.connect.secrets.io.FileWriterOnce +import io.lenses.connect.secrets.io.{FileWriter, FileWriterOnce} import io.lenses.connect.secrets.utils.EncodingAndId -import org.apache.kafka.common.config.ConfigData import org.apache.kafka.common.config.provider.ConfigProvider -import org.apache.kafka.common.config.ConfigException +import org.apache.kafka.common.config.{ConfigData, ConfigException} import org.apache.kafka.connect.errors.ConnectException -import scala.collection.JavaConverters._ +import java.nio.file.Paths +import java.util +import scala.jdk.CollectionConverters._ class Aes256DecodingProvider extends ConfigProvider { var decoder: Option[Aes256DecodingHelper] = None - + private var fileWriter: FileWriter = _ - + override def configure(configs: util.Map[String, _]): Unit = { val aes256Cfg = Aes256ProviderConfig(configs) val aes256Key = aes256Cfg.aes256Key @@ -33,22 +29,33 @@ class Aes256DecodingProvider extends ConfigProvider { fileWriter = new FileWriterOnce(Paths.get(writeDir, "secrets")) } - override def get(path: String): ConfigData = new ConfigData(Map.empty[String, String].asJava) + override def get(path: String): ConfigData = + new ConfigData(Map.empty[String, String].asJava) override def get(path: String, keys: util.Set[String]): ConfigData = { val encodingAndId = EncodingAndId.from(path) decoder match { case Some(d) => def decrypt(key: String): String = { - val decrypted = d.decrypt(key).fold(e => throw new ConnectException("Failed to decrypt the secret.", e), identity) + val decrypted = d + .decrypt(key) + .fold( + e => + throw new ConnectException("Failed to decrypt the secret.", e), + identity + ) decodeKey( key = key, value = decrypted, encoding = encodingAndId.encoding, writeFileFn = { content => encodingAndId.id match { - case Some(value) => fileWriter.write(value, content, key).toString - case None => throw new ConnectException(s"Invalid argument received for key:$key. Expecting a file identifier.") + case Some(value) => + fileWriter.write(value, content, key).toString + case None => + throw new ConnectException( + s"Invalid argument received for key:$key. Expecting a file identifier." + ) } } ) diff --git a/src/main/scala/io/lenses/connect/secrets/providers/AzureHelper.scala b/src/main/scala/io/lenses/connect/secrets/providers/AzureHelper.scala index dc95a5c..0acd6bd 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/AzureHelper.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/AzureHelper.scala @@ -1,117 +1,111 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.nio.file.FileSystems -import java.time.OffsetDateTime - -import com.azure.core.credential.TokenCredential -import com.azure.identity.{ - ClientSecretCredentialBuilder, - DefaultAzureCredentialBuilder -} -import com.azure.security.keyvault.secrets.SecretClient -import com.typesafe.scalalogging.StrictLogging -import io.lenses.connect.secrets.config.AzureProviderSettings -import io.lenses.connect.secrets.connect.{ - AuthMode, - Encoding, - FILE_ENCODING, - decode, - decodeToBytes, - fileWriter, - getFileName -} -import org.apache.kafka.connect.errors.ConnectException - -import scala.util.{Failure, Success, Try} - -trait AzureHelper extends StrictLogging { - - private val separator: String = FileSystems.getDefault.getSeparator - - // look up secret in Azure - def getSecretValue( - rootDir: String, - path: String, - client: SecretClient, - key: String - ): (String, Option[OffsetDateTime]) = { - - Try(client.getSecret(key)) match { - case Success(secret) => - val value = secret.getValue - val props = secret.getProperties - - // check if the file-encoding - val encoding = - Encoding.withName( - Option(props.getTags) - .map { _.getOrDefault(FILE_ENCODING, Encoding.UTF8.toString) } - .getOrElse(Encoding.UTF8.toString) - .toUpperCase) - - val content = encoding match { - case Encoding.UTF8 => - value - - case Encoding.UTF8_FILE => - val fileName = getFileName(rootDir, path, key.toLowerCase, separator) - fileWriter( - fileName, - value.getBytes, - key.toLowerCase - ) - fileName - - case Encoding.BASE64 => - decode(key, value) - - // write to file and set the file name as the value - case Encoding.BASE64_FILE | Encoding.UTF8_FILE => - val fileName = getFileName(rootDir, path, key.toLowerCase, separator) - val decoded = decodeToBytes(key, value) - fileWriter( - fileName, - decoded, - key.toLowerCase - ) - fileName - } - - val expiry = Option(props.getExpiresOn) - (content, expiry) - - case Failure(e) => - throw new ConnectException( - s"Failed to look up secret [$key] at [${client.getVaultUrl}]", - e - ) - } - } - - // setup azure credentials - def createCredentials(settings: AzureProviderSettings): TokenCredential = { - - logger.info( - s"Initializing client with mode [${settings.authMode.toString}]" - ) - - settings.authMode match { - case AuthMode.CREDENTIALS => - new ClientSecretCredentialBuilder() - .clientId(settings.clientId) - .clientSecret(settings.secretId.value()) - .tenantId(settings.tenantId) - .build() - - case _ => - new DefaultAzureCredentialBuilder().build() - - } - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.azure.core.credential.TokenCredential +import com.azure.identity.{ + ClientSecretCredentialBuilder, + DefaultAzureCredentialBuilder +} +import com.azure.security.keyvault.secrets.SecretClient +import com.typesafe.scalalogging.StrictLogging +import io.lenses.connect.secrets.config.AzureProviderSettings +import io.lenses.connect.secrets.connect._ +import org.apache.kafka.connect.errors.ConnectException + +import java.nio.file.FileSystems +import java.time.OffsetDateTime +import scala.util.{Failure, Success, Try} + +trait AzureHelper extends StrictLogging { + + private val separator: String = FileSystems.getDefault.getSeparator + + // look up secret in Azure + def getSecretValue( + rootDir: String, + path: String, + client: SecretClient, + key: String + ): (String, Option[OffsetDateTime]) = { + + Try(client.getSecret(key)) match { + case Success(secret) => + val value = secret.getValue + val props = secret.getProperties + + // check if the file-encoding + val encoding = + Encoding.withName( + Option(props.getTags) + .map { _.getOrDefault(FILE_ENCODING, Encoding.UTF8.toString) } + .getOrElse(Encoding.UTF8.toString) + .toUpperCase + ) + + val content = encoding match { + case Encoding.UTF8 => + value + + case Encoding.UTF8_FILE => + val fileName = + getFileName(rootDir, path, key.toLowerCase, separator) + fileWriter( + fileName, + value.getBytes, + key.toLowerCase + ) + fileName + + case Encoding.BASE64 => + decode(key, value) + + // write to file and set the file name as the value + case Encoding.BASE64_FILE | Encoding.UTF8_FILE => + val fileName = + getFileName(rootDir, path, key.toLowerCase, separator) + val decoded = decodeToBytes(key, value) + fileWriter( + fileName, + decoded, + key.toLowerCase + ) + fileName + } + + val expiry = Option(props.getExpiresOn) + (content, expiry) + + case Failure(e) => + throw new ConnectException( + s"Failed to look up secret [$key] at [${client.getVaultUrl}]", + e + ) + } + } + + // setup azure credentials + def createCredentials(settings: AzureProviderSettings): TokenCredential = { + + logger.info( + s"Initializing client with mode [${settings.authMode.toString}]" + ) + + settings.authMode match { + case AuthMode.CREDENTIALS => + new ClientSecretCredentialBuilder() + .clientId(settings.clientId) + .clientSecret(settings.secretId.value()) + .tenantId(settings.tenantId) + .build() + + case _ => + new DefaultAzureCredentialBuilder().build() + + } + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/AzureSecretProvider.scala b/src/main/scala/io/lenses/connect/secrets/providers/AzureSecretProvider.scala index 3b2bcb5..41efa0a 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/AzureSecretProvider.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/AzureSecretProvider.scala @@ -1,107 +1,119 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.time.OffsetDateTime -import java.util - -import com.azure.core.credential.TokenCredential -import com.azure.security.keyvault.secrets.{SecretClient, SecretClientBuilder} -import io.lenses.connect.secrets.config.{ - AzureProviderConfig, - AzureProviderSettings -} -import io.lenses.connect.secrets.connect.getSecretsAndExpiry -import org.apache.kafka.common.config.ConfigData -import org.apache.kafka.common.config.provider.ConfigProvider - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -class AzureSecretProvider() extends ConfigProvider with AzureHelper { - - private var rootDir: String = _ - private var credentials: Option[TokenCredential] = None - val clientMap: mutable.Map[String, SecretClient] = mutable.Map.empty - val cache = mutable.Map.empty[String, (Option[OffsetDateTime], ConfigData)] - - // configure the vault client - override def configure(configs: util.Map[String, _]): Unit = { - val settings = AzureProviderSettings(AzureProviderConfig(configs)) - rootDir = settings.fileDir - credentials = Some(createCredentials(settings)) - } - - // lookup secrets at a path - // returns and empty map since Azure is flat - // and we need to know the secret to lookup - override def get(path: String): ConfigData = - new ConfigData(Map.empty[String, String].asJava) - - // get secret keys at a path - // paths is expected to be the url of the azure keyvault without the protocol (https://) - // since the connect work will not parse it correctly do to the : - // including the azure environment. - override def get(path: String, keys: util.Set[String]): ConfigData = { - - val keyVaultUrl = - if (path.startsWith("https://")) path else s"https://$path" - - // don't need to cache but allows for testing - // this way we don't require a keyvault set in the - // worker properties and we take the path as the target keyvault - val client = clientMap.getOrElse( - keyVaultUrl, - new SecretClientBuilder() - .vaultUrl(keyVaultUrl) - .credential(credentials.get) - .buildClient - ) - - clientMap += (keyVaultUrl -> client) - - val (expiry, data) = cache.get(keyVaultUrl) match { - case Some((expiresAt, data)) => - // we have all the keys and are before the expiry - val now = OffsetDateTime.now() - - if (keys.asScala.subsetOf(data.data().asScala.keySet) && (expiresAt - .getOrElse(now.plusSeconds(1)) - .isAfter(now))) { - logger.info("Fetching secrets from cache") - (expiresAt, - new ConfigData( - data.data().asScala.filterKeys(k => keys.contains(k)).asJava, - data.ttl())) - } else { - // missing some or expired so reload - getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet)) - } - - case None => - getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet)) - } - - expiry.foreach(exp => - logger.info(s"Min expiry for TTL set to [${exp.toString}]")) - cache += (keyVaultUrl -> (expiry, data)) - data - } - - override def close(): Unit = {} - - private def getSecrets( - client: SecretClient, - keys: Set[String]): Map[String, (String, Option[OffsetDateTime])] = { - val path = client.getVaultUrl.stripPrefix("https://") - keys.map { key => - logger.info(s"Looking up value at [$path] for key [$key]") - val (value, expiry) = getSecretValue(rootDir, path, client, key) - (key, (value, expiry)) - }.toMap - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.azure.core.credential.TokenCredential +import com.azure.security.keyvault.secrets.{SecretClient, SecretClientBuilder} +import io.lenses.connect.secrets.config.{ + AzureProviderConfig, + AzureProviderSettings +} +import io.lenses.connect.secrets.connect.getSecretsAndExpiry +import org.apache.kafka.common.config.ConfigData +import org.apache.kafka.common.config.provider.ConfigProvider + +import java.time.OffsetDateTime +import java.util +import scala.collection.mutable +import scala.jdk.CollectionConverters._ + +class AzureSecretProvider() extends ConfigProvider with AzureHelper { + + private var rootDir: String = _ + private var credentials: Option[TokenCredential] = None + val clientMap: mutable.Map[String, SecretClient] = mutable.Map.empty + val cache = mutable.Map.empty[String, (Option[OffsetDateTime], ConfigData)] + + // configure the vault client + override def configure(configs: util.Map[String, _]): Unit = { + val settings = AzureProviderSettings(AzureProviderConfig(configs)) + rootDir = settings.fileDir + credentials = Some(createCredentials(settings)) + } + + // lookup secrets at a path + // returns and empty map since Azure is flat + // and we need to know the secret to lookup + override def get(path: String): ConfigData = + new ConfigData(Map.empty[String, String].asJava) + + // get secret keys at a path + // paths is expected to be the url of the azure keyvault without the protocol (https://) + // since the connect work will not parse it correctly do to the : + // including the azure environment. + override def get(path: String, keys: util.Set[String]): ConfigData = { + + val keyVaultUrl = + if (path.startsWith("https://")) path else s"https://$path" + + // don't need to cache but allows for testing + // this way we don't require a keyvault set in the + // worker properties and we take the path as the target keyvault + val client = clientMap.getOrElse( + keyVaultUrl, + new SecretClientBuilder() + .vaultUrl(keyVaultUrl) + .credential(credentials.get) + .buildClient + ) + + clientMap += (keyVaultUrl -> client) + + val (expiry, data) = cache.get(keyVaultUrl) match { + case Some((expiresAt, data)) => + // we have all the keys and are before the expiry + val now = OffsetDateTime.now() + + if (keys.asScala.subsetOf(data.data().asScala.keySet) && (expiresAt + .getOrElse(now.plusSeconds(1)) + .isAfter(now))) { + logger.info("Fetching secrets from cache") + ( + expiresAt, + new ConfigData( + data + .data() + .asScala + .view + .filter { + case (k, _) => keys.contains(k) + } + .toMap + .asJava, + data.ttl() + ) + ) + } else { + // missing some or expired so reload + getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet)) + } + + case None => + getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet)) + } + + expiry.foreach(exp => + logger.info(s"Min expiry for TTL set to [${exp.toString}]") + ) + cache += (keyVaultUrl -> (expiry, data)) + data + } + + override def close(): Unit = {} + + private def getSecrets( + client: SecretClient, + keys: Set[String] + ): Map[String, (String, Option[OffsetDateTime])] = { + val path = client.getVaultUrl.stripPrefix("https://") + keys.map { key => + logger.info(s"Looking up value at [$path] for key [$key]") + val (value, expiry) = getSecretValue(rootDir, path, client, key) + (key, (value, expiry)) + }.toMap + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/ENVSecretProvider.scala b/src/main/scala/io/lenses/connect/secrets/providers/ENVSecretProvider.scala index b9de025..4fa8353 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/ENVSecretProvider.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/ENVSecretProvider.scala @@ -1,77 +1,84 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.nio.file.FileSystems -import java.util - -import io.lenses.connect.secrets.config.ENVProviderConfig -import io.lenses.connect.secrets.connect.{FILE_DIR, decode, decodeToBytes, fileWriter} -import org.apache.kafka.common.config.ConfigData -import org.apache.kafka.common.config.provider.ConfigProvider -import org.apache.kafka.connect.errors.ConnectException - -import scala.collection.JavaConverters._ - -class ENVSecretProvider extends ConfigProvider { - - var vars = Map.empty[String, String] - var fileDir: String = "" - private val separator: String = FileSystems.getDefault.getSeparator - private val BASE64_FILE = "(ENV-mounted-base64:)(.*$)".r - private val UTF8_FILE = "(ENV-mounted:)(.*$)".r - private val BASE64 = "(ENV-base64:)(.*$)".r - - override def get(path: String): ConfigData = - new ConfigData(Map.empty[String, String].asJava) - - override def get(path: String, keys: util.Set[String]): ConfigData = { - val data = - keys.asScala - .map { key => - { - val envVarVal = - vars.getOrElse(key, - throw new ConnectException( - s"Failed to lookup environment variable [$key]")) - - // match the value to see if its coming from contains - // the value metadata pattern - envVarVal match { - case BASE64_FILE(m, v) => - //decode and write to file - val fileName = s"${fileDir}$separator${key.toLowerCase}" - fileWriter(fileName, decodeToBytes(key, v), key) - (key, fileName) - - case UTF8_FILE(m, v) => - val fileName = s"${fileDir}$separator${key.toLowerCase}" - fileWriter(fileName, v.getBytes(), key) - (key, fileName) - - case BASE64(m, v) => - (key, decode(key, v)) - - case _ => - (key, envVarVal) - } - } - } - .toMap - .asJava - - new ConfigData(data) - } - - override def configure(configs: util.Map[String, _]): Unit = { - vars = System.getenv().asScala.toMap - val config = ENVProviderConfig(configs) - fileDir = config.getString(FILE_DIR).stripSuffix(separator) - } - - override def close(): Unit = {} -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import io.lenses.connect.secrets.config.ENVProviderConfig +import io.lenses.connect.secrets.connect.{ + FILE_DIR, + decode, + decodeToBytes, + fileWriter +} +import org.apache.kafka.common.config.ConfigData +import org.apache.kafka.common.config.provider.ConfigProvider +import org.apache.kafka.connect.errors.ConnectException + +import java.nio.file.FileSystems +import java.util +import scala.jdk.CollectionConverters._ + +class ENVSecretProvider extends ConfigProvider { + + var vars = Map.empty[String, String] + var fileDir: String = "" + private val separator: String = FileSystems.getDefault.getSeparator + private val BASE64_FILE = "(ENV-mounted-base64:)(.*$)".r + private val UTF8_FILE = "(ENV-mounted:)(.*$)".r + private val BASE64 = "(ENV-base64:)(.*$)".r + + override def get(path: String): ConfigData = + new ConfigData(Map.empty[String, String].asJava) + + override def get(path: String, keys: util.Set[String]): ConfigData = { + val data = + keys.asScala + .map { key => + { + val envVarVal = + vars.getOrElse( + key, + throw new ConnectException( + s"Failed to lookup environment variable [$key]" + ) + ) + + // match the value to see if its coming from contains + // the value metadata pattern + envVarVal match { + case BASE64_FILE(_, v) => + //decode and write to file + val fileName = s"$fileDir$separator${key.toLowerCase}" + fileWriter(fileName, decodeToBytes(key, v), key) + (key, fileName) + + case UTF8_FILE(_, v) => + val fileName = s"$fileDir$separator${key.toLowerCase}" + fileWriter(fileName, v.getBytes(), key) + (key, fileName) + + case BASE64(_, v) => + (key, decode(key, v)) + + case _ => + (key, envVarVal) + } + } + } + .toMap + .asJava + + new ConfigData(data) + } + + override def configure(configs: util.Map[String, _]): Unit = { + vars = System.getenv().asScala.toMap + val config = ENVProviderConfig(configs) + fileDir = config.getString(FILE_DIR).stripSuffix(separator) + } + + override def close(): Unit = {} +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala b/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala index 10411f9..b9fef96 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/VaultHelper.scala @@ -1,165 +1,165 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.io.File - -import com.bettercloud.vault.{SslConfig, Vault, VaultConfig} -import com.typesafe.scalalogging.StrictLogging -import io.lenses.connect.secrets.config.{VaultAuthMethod, VaultSettings} -import org.apache.kafka.connect.errors.ConnectException - -trait VaultHelper extends StrictLogging { - - // initialize the vault client - def createClient(settings: VaultSettings): Vault = { - val config = - new VaultConfig().address(settings.addr) - - // set ssl if configured - config.sslConfig(configureSSL(settings)) - - if (settings.namespace.nonEmpty) { - logger.info(s"Setting namespace to ${settings.namespace}") - config.nameSpace(settings.namespace) - } - - logger.info(s"Setting engine version to ${settings.engineVersion}") - config.engineVersion(settings.engineVersion) - - val vault = new Vault(config.build()) - - logger.info( - s"Initializing client with mode [${settings.authMode.toString}]" - ) - - val token = settings.authMode match { - case VaultAuthMethod.USERPASS => - settings.userPass - .map( - up => - vault - .auth() - .loginByUserPass(up.username, up.password.value(), up.mount) - .getAuthClientToken) - - case VaultAuthMethod.APPROLE => - settings.appRole - .map( - ar => - vault - .auth() - .loginByAppRole(ar.role, ar.secretId.value()) - .getAuthClientToken) - - case VaultAuthMethod.CERT => - settings.cert - .map(c => vault.auth().loginByCert(c.mount).getAuthClientToken) - - case VaultAuthMethod.AWSIAM => - settings.awsIam - .map( - aws => - vault - .auth() - .loginByAwsIam( - aws.role, - aws.url, - aws.body.value(), - aws.headers.value(), - aws.mount - ) - .getAuthClientToken) - - case VaultAuthMethod.KUBERNETES => - settings.k8s - .map( - k8s => - vault - .auth() - .loginByKubernetes(k8s.role, k8s.jwt.value()) - .getAuthClientToken) - case VaultAuthMethod.GCP => - settings.gcp - .map( - gcp => - vault - .auth() - .loginByGCP(gcp.role, gcp.jwt.value()) - .getAuthClientToken) - - case VaultAuthMethod.LDAP => - settings.ldap - .map( - l => - vault - .auth() - .loginByLDAP(l.username, l.password.value(), l.mount) - .getAuthClientToken) - - case VaultAuthMethod.JWT => - settings.jwt - .map( - j => - vault - .auth() - .loginByJwt(j.provider, j.role, j.jwt.value()) - .getAuthClientToken) - - case VaultAuthMethod.TOKEN => - Some(settings.token.value()) - - case VaultAuthMethod.GITHUB => - - settings.github - .map( - gh => - vault - .auth() - .loginByGithub(gh.token.value(), gh.mount) - .getAuthClientToken) - - case _ => - throw new ConnectException( - s"Unsupported auth method [${settings.authMode.toString}]") - } - - config.token(token.get) - config.build() - new Vault(config) - } - - // set up tls - private def configureSSL(settings: VaultSettings): SslConfig = { - val ssl = new SslConfig() - - if (settings.keystoreLoc != "") { - logger.info(s"Configuring keystore at [${settings.keystoreLoc}]") - ssl.keyStoreFile( - new File(settings.keystoreLoc), - settings.keystorePass.value() - ) - } - - if (settings.truststoreLoc != "") { - logger.info(s"Configuring keystore at [${settings.truststoreLoc}]") - ssl.trustStoreFile(new File(settings.truststoreLoc)) - } - - if (settings.clientPem != "") { - logger.info(s"Configuring client PEM. Ignored if JKS set.") - ssl.clientKeyPemFile(new File(settings.clientPem)) - } - - if (settings.pem != "") { - logger.info(s"Configuring Vault Server PEM. Ignored if JKS set.") - ssl.pemFile(new File(settings.pem)) - } - - ssl.build() - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.bettercloud.vault.{SslConfig, Vault, VaultConfig} +import com.typesafe.scalalogging.StrictLogging +import io.lenses.connect.secrets.config.{VaultAuthMethod, VaultSettings} +import org.apache.kafka.connect.errors.ConnectException + +import java.io.File + +trait VaultHelper extends StrictLogging { + + // initialize the vault client + def createClient(settings: VaultSettings): Vault = { + val config = + new VaultConfig().address(settings.addr) + + // set ssl if configured + config.sslConfig(configureSSL(settings)) + + if (settings.namespace.nonEmpty) { + logger.info(s"Setting namespace to ${settings.namespace}") + config.nameSpace(settings.namespace) + } + + logger.info(s"Setting engine version to ${settings.engineVersion}") + config.engineVersion(settings.engineVersion) + + val vault = new Vault(config.build()) + + logger.info( + s"Initializing client with mode [${settings.authMode.toString}]" + ) + + val token = settings.authMode match { + case VaultAuthMethod.USERPASS => + settings.userPass + .map(up => + vault + .auth() + .loginByUserPass(up.username, up.password.value(), up.mount) + .getAuthClientToken + ) + + case VaultAuthMethod.APPROLE => + settings.appRole + .map(ar => + vault + .auth() + .loginByAppRole(ar.role, ar.secretId.value()) + .getAuthClientToken + ) + + case VaultAuthMethod.CERT => + settings.cert + .map(c => vault.auth().loginByCert(c.mount).getAuthClientToken) + + case VaultAuthMethod.AWSIAM => + settings.awsIam + .map(aws => + vault + .auth() + .loginByAwsIam( + aws.role, + aws.url, + aws.body.value(), + aws.headers.value(), + aws.mount + ) + .getAuthClientToken + ) + + case VaultAuthMethod.KUBERNETES => + settings.k8s + .map(k8s => + vault + .auth() + .loginByKubernetes(k8s.role, k8s.jwt.value()) + .getAuthClientToken + ) + case VaultAuthMethod.GCP => + settings.gcp + .map(gcp => + vault + .auth() + .loginByGCP(gcp.role, gcp.jwt.value()) + .getAuthClientToken + ) + + case VaultAuthMethod.LDAP => + settings.ldap + .map(l => + vault + .auth() + .loginByLDAP(l.username, l.password.value(), l.mount) + .getAuthClientToken + ) + + case VaultAuthMethod.JWT => + settings.jwt + .map(j => + vault + .auth() + .loginByJwt(j.provider, j.role, j.jwt.value()) + .getAuthClientToken + ) + + case VaultAuthMethod.TOKEN => + Some(settings.token.value()) + + case VaultAuthMethod.GITHUB => + settings.github + .map(gh => + vault + .auth() + .loginByGithub(gh.token.value(), gh.mount) + .getAuthClientToken + ) + + case _ => + throw new ConnectException( + s"Unsupported auth method [${settings.authMode.toString}]" + ) + } + + config.token(token.get) + config.build() + new Vault(config) + } + + // set up tls + private def configureSSL(settings: VaultSettings): SslConfig = { + val ssl = new SslConfig() + + if (settings.keystoreLoc != "") { + logger.info(s"Configuring keystore at [${settings.keystoreLoc}]") + ssl.keyStoreFile( + new File(settings.keystoreLoc), + settings.keystorePass.value() + ) + } + + if (settings.truststoreLoc != "") { + logger.info(s"Configuring keystore at [${settings.truststoreLoc}]") + ssl.trustStoreFile(new File(settings.truststoreLoc)) + } + + if (settings.clientPem != "") { + logger.info(s"Configuring client PEM. Ignored if JKS set.") + ssl.clientKeyPemFile(new File(settings.clientPem)) + } + + if (settings.pem != "") { + logger.info(s"Configuring Vault Server PEM. Ignored if JKS set.") + ssl.pemFile(new File(settings.pem)) + } + + ssl.build() + } +} diff --git a/src/main/scala/io/lenses/connect/secrets/providers/VaultSecretProvider.scala b/src/main/scala/io/lenses/connect/secrets/providers/VaultSecretProvider.scala index 3d4f9a0..44f2827 100644 --- a/src/main/scala/io/lenses/connect/secrets/providers/VaultSecretProvider.scala +++ b/src/main/scala/io/lenses/connect/secrets/providers/VaultSecretProvider.scala @@ -6,14 +6,8 @@ package io.lenses.connect.secrets.providers -import java.nio.file.FileSystems -import java.nio.file.Paths -import java.time.OffsetDateTime -import java.util - -import _root_.io.lenses.connect.secrets.config.VaultProviderConfig -import _root_.io.lenses.connect.secrets.config.VaultSettings -import _root_.io.lenses.connect.secrets.connect._ +import io.lenses.connect.secrets.config.{VaultProviderConfig, VaultSettings} +import io.lenses.connect.secrets.connect._ import com.bettercloud.vault.Vault import io.lenses.connect.secrets.async.AsyncFunctionLoop import io.lenses.connect.secrets.io.FileWriterOnce @@ -22,26 +16,30 @@ import org.apache.kafka.common.config.ConfigData import org.apache.kafka.common.config.provider.ConfigProvider import org.apache.kafka.connect.errors.ConnectException -import scala.collection.JavaConverters._ +import java.nio.file.Paths +import java.time.OffsetDateTime +import java.util import scala.collection.mutable -import scala.util.Failure -import scala.util.Success -import scala.util.Try +import scala.jdk.CollectionConverters._ +import scala.util.{Failure, Success, Try} class VaultSecretProvider() extends ConfigProvider with VaultHelper { - private val separator: String = FileSystems.getDefault.getSeparator private var settings: VaultSettings = _ private var vaultClient: Option[Vault] = None private var tokenRenewal: Option[AsyncFunctionLoop] = None - private val cache = mutable.Map.empty[String, (Option[OffsetDateTime], ConfigData)] + private val cache = + mutable.Map.empty[String, (Option[OffsetDateTime], ConfigData)] def getClient: Option[Vault] = vaultClient // configure the vault client override def configure(configs: util.Map[String, _]): Unit = { settings = VaultSettings(VaultProviderConfig(configs)) vaultClient = Some(createClient(settings)) - val renewalLoop = new AsyncFunctionLoop(settings.tokenRenewal, "Vault Token Renewal")(renewToken()) + val renewalLoop = + new AsyncFunctionLoop(settings.tokenRenewal, "Vault Token Renewal")( + renewToken() + ) tokenRenewal = Some(renewalLoop) renewalLoop.start() } @@ -50,9 +48,7 @@ class VaultSecretProvider() extends ConfigProvider with VaultHelper { def tokenRenewalFailure: Long = tokenRenewal.map(_.failureRate).getOrElse(-1) private def renewToken(): Unit = { - vaultClient.foreach { client => - client.auth().renewSelf() - } + vaultClient.foreach { client => client.auth().renewSelf() } } // lookup secrets at a path @@ -75,7 +71,8 @@ class VaultSecretProvider() extends ConfigProvider with VaultHelper { } expiry.foreach(exp => - logger.info(s"Min expiry for TTL set to [${exp.toString}]")) + logger.info(s"Min expiry for TTL set to [${exp.toString}]") + ) cache += (path -> (expiry, data)) data } @@ -89,25 +86,40 @@ class VaultSecretProvider() extends ConfigProvider with VaultHelper { val now = OffsetDateTime.now() if (keys.asScala.subsetOf(data.data().asScala.keySet) && expiresAt - .getOrElse(now.plusSeconds(1)) - .isAfter(now)) { + .getOrElse(now.plusSeconds(1)) + .isAfter(now)) { logger.info("Fetching secrets from cache") - (expiresAt, + ( + expiresAt, new ConfigData( - data.data().asScala.filterKeys(k => keys.contains(k)).asJava, - data.ttl())) + data + .data() + .asScala + .view + .filter { + case (k, _) => keys.contains(k) + } + .toMap + .asJava, + data.ttl() + ) + ) } else { // missing some or expired so reload - getSecretsAndExpiry( - getSecrets(path).filterKeys(k => keys.contains(k))) + getSecretsAndExpiry(getSecrets(path).view.filter { + case (k, _) => keys.contains(k) + }.toMap) } case None => - getSecretsAndExpiry(getSecrets(path).filterKeys(k => keys.contains(k))) + getSecretsAndExpiry(getSecrets(path).view.filter { + case (k, _) => keys.contains(k) + }.toMap) } expiry.foreach(exp => - logger.info(s"Min expiry for TTL set to [${exp.toString}]")) + logger.info(s"Min expiry for TTL set to [${exp.toString}]") + ) cache += (path -> (expiry, data)) data } @@ -117,7 +129,9 @@ class VaultSecretProvider() extends ConfigProvider with VaultHelper { } // get the secrets and ttl under a path - def getSecrets(path: String): Map[String, (String, Option[OffsetDateTime])] = { + def getSecrets( + path: String + ): Map[String, (String, Option[OffsetDateTime])] = { val now = OffsetDateTime.now() logger.info(s"Looking up value at [$path]") @@ -126,16 +140,13 @@ class VaultSecretProvider() extends ConfigProvider with VaultHelper { case Success(response) => if (response.getRestResponse.getStatus != 200) { throw new ConnectException( - s"No secrets found at path [$path]. Vault response: ${ - new String( - response.getRestResponse.getBody) - }" + s"No secrets found at path [$path]. Vault response: ${new String(response.getRestResponse.getBody)}" ) } val ttl = Option(vaultClient.get.logical().read(path).getLeaseDuration) match { case Some(duration) => Some(now.plusSeconds(duration)) - case None => None + case None => None } if (response.getData.isEmpty) { @@ -148,9 +159,14 @@ class VaultSecretProvider() extends ConfigProvider with VaultHelper { case (k, v) => val encodingAndId = EncodingAndId.from(k) val decoded = - decodeKey(encoding = encodingAndId.encoding, key = k, value = v, writeFileFn = { content => - fileWriter.write(k.toLowerCase, content, k).toString - }) + decodeKey( + encoding = encodingAndId.encoding, + key = k, + value = v, + writeFileFn = { content => + fileWriter.write(k.toLowerCase, content, k).toString + } + ) (k, (decoded, ttl)) }.toMap diff --git a/src/main/scala/io/lenses/connect/secrets/utils/WithRetry.scala b/src/main/scala/io/lenses/connect/secrets/utils/WithRetry.scala index 2bf0340..1ac541b 100644 --- a/src/main/scala/io/lenses/connect/secrets/utils/WithRetry.scala +++ b/src/main/scala/io/lenses/connect/secrets/utils/WithRetry.scala @@ -7,20 +7,22 @@ package io.lenses.connect.secrets.utils import scala.annotation.tailrec import scala.concurrent.duration.FiniteDuration +import scala.util.{Failure, Success, Try} trait WithRetry { - protected final def withRetry[T](retry: Int = 5, interval:Option[FiniteDuration])(thunk: => T): T = - try { + + @tailrec + protected final def withRetry[T]( + retry: Int = 5, + interval: Option[FiniteDuration] + )(thunk: => T): T = + Try { thunk - } catch { - case t: Throwable => + } match { + case Failure(t) => if (retry == 0) throw t - interval match { - case Some(value) => - Thread.sleep(value.toMillis) - withRetry(retry - 1, interval)(thunk) - case None => - withRetry(retry-1,interval)(thunk) - } + interval.foreach(sleepValue => Thread.sleep(sleepValue.toMillis)) + withRetry(retry - 1, interval)(thunk) + case Success(value) => value } } diff --git a/src/test/java/io/lenses/connect/secrets/vault/MockVault.java b/src/test/java/io/lenses/connect/secrets/vault/MockVault.java index e587c09..2968960 100644 --- a/src/test/java/io/lenses/connect/secrets/vault/MockVault.java +++ b/src/test/java/io/lenses/connect/secrets/vault/MockVault.java @@ -10,9 +10,9 @@ import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; import java.io.IOException; import java.util.Map; import java.util.Optional; diff --git a/src/test/java/io/lenses/connect/secrets/vault/VaultTestUtils.java b/src/test/java/io/lenses/connect/secrets/vault/VaultTestUtils.java index d41884d..b788875 100644 --- a/src/test/java/io/lenses/connect/secrets/vault/VaultTestUtils.java +++ b/src/test/java/io/lenses/connect/secrets/vault/VaultTestUtils.java @@ -12,7 +12,7 @@ import org.eclipse.jetty.server.*; import org.eclipse.jetty.util.ssl.SslContextFactory; -import javax.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequest; import java.io.IOException; import java.util.Collections; import java.util.Map; @@ -78,9 +78,6 @@ public static Optional readRequestBody(HttpServletRequest request) { } public static Map readRequestHeaders(HttpServletRequest request) { -// ArrayList x = new ArrayList<>(); -// x.add(request.getHeaderNames()).stream().collect(toMap(identity(), request::getHeader)); -// return Collections.list(request.getHeaderNames()).stream().collect(toMap(identity(), request::getHeader)); } diff --git a/src/test/scala/io/lenses/connect/secrets/TmpDirUtil.scala b/src/test/scala/io/lenses/connect/secrets/TmpDirUtil.scala new file mode 100644 index 0000000..68d45a3 --- /dev/null +++ b/src/test/scala/io/lenses/connect/secrets/TmpDirUtil.scala @@ -0,0 +1,11 @@ +package io.lenses.connect.secrets + +import java.nio.file.FileSystems + +object TmpDirUtil { + + val separator: String = FileSystems.getDefault.getSeparator + + def getTempDir: String = + System.getProperty("java.io.tmpdir").stripSuffix(separator) +} diff --git a/src/test/scala/io/lenses/connect/secrets/async/AsyncFunctionLoopTest.scala b/src/test/scala/io/lenses/connect/secrets/async/AsyncFunctionLoopTest.scala index 095fe27..d0b0c22 100644 --- a/src/test/scala/io/lenses/connect/secrets/async/AsyncFunctionLoopTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/async/AsyncFunctionLoopTest.scala @@ -6,12 +6,10 @@ package io.lenses.connect.secrets.async -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import java.util.concurrent.{CountDownLatch, TimeUnit} import scala.concurrent.duration.DurationInt class AsyncFunctionLoopTest extends AnyFunSuite with Matchers { diff --git a/src/test/scala/io/lenses/connect/secrets/io/FileWriterOnceTest.scala b/src/test/scala/io/lenses/connect/secrets/io/FileWriterOnceTest.scala index 9ba13f6..e86c306 100644 --- a/src/test/scala/io/lenses/connect/secrets/io/FileWriterOnceTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/io/FileWriterOnceTest.scala @@ -1,15 +1,14 @@ package io.lenses.connect.secrets.io -import java.io.File -import java.nio.file.Paths -import java.util.UUID - +import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import org.scalatest.BeforeAndAfterAll +import java.io.File +import java.nio.file.Paths +import java.util.UUID import scala.io.Source -import scala.util.Try +import scala.util.{Success, Try, Using} class FileWriterOnceTest extends AnyFunSuite @@ -34,10 +33,7 @@ class FileWriterOnceTest val file = Paths.get(folder.toPath.toString, "thisone").toFile file.exists() shouldBe true - val source = Source.fromFile(file) - val size = source.size - source.close() - size shouldBe content.length + Using(Source.fromFile(file))(_.size) shouldBe Success(content.length) } test("does not write the file twice") { @@ -47,16 +43,11 @@ class FileWriterOnceTest val file = Paths.get(folder.toPath.toString, fileName).toFile file.exists() shouldBe true - var source = Source.fromFile(file) - var size = source.size - source.close() - size shouldBe content1.length + Using(Source.fromFile(file))(_.size) shouldBe Success(content1.length) - val content2 = Array(1, 2, 3,4,5,6,7,8).map(_.toByte) + val content2 = Array(1, 2, 3, 4, 5, 6, 7, 8).map(_.toByte) writer.write(fileName, content2, "key1") - source = Source.fromFile(file) - size = source.size - source.close() - size shouldBe content1.length + Using(Source.fromFile(file))(_.size) shouldBe Success(content1.length) + } } diff --git a/src/test/scala/io/lenses/connect/secrets/providers/AWSSecretProviderTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/AWSSecretProviderTest.scala index c4b17ce..a568daf 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/AWSSecretProviderTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/AWSSecretProviderTest.scala @@ -1,339 +1,336 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.nio.file.FileSystems -import java.util.Base64 -import java.util.Date - -import com.amazonaws.services.secretsmanager.AWSSecretsManager -import com.amazonaws.services.secretsmanager.model._ -import com.bettercloud.vault.json.JsonObject -import io.lenses.connect.secrets.config.AWSProviderConfig -import io.lenses.connect.secrets.config.AWSProviderSettings -import io.lenses.connect.secrets.connect._ -import io.lenses.connect.secrets.connect.AuthMode -import io.lenses.connect.secrets.connect.Encoding -import io.lenses.connect.secrets.utils.EncodingAndId -import org.apache.kafka.common.config.ConfigTransformer -import org.apache.kafka.common.config.provider.ConfigProvider -import org.apache.kafka.connect.errors.ConnectException -import org.mockito.ArgumentMatchers.any -import org.mockito.MockitoSugar -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import scala.collection.JavaConverters._ -import scala.io.Source - -class AWSSecretProviderTest - extends AnyWordSpec - with Matchers - with MockitoSugar { - - val separator: String = FileSystems.getDefault.getSeparator - val tmp: String = - System.getProperty("java.io.tmpdir") + separator + "provider-tests-aws" - - "should authenticate with credentials" in { - val props = Map( - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", - AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", - AWSProviderConfig.AWS_REGION -> "someregion" - ).asJava - - val provider = new AWSSecretProvider() - provider.configure(props) - provider.close() - } - - "should authenticate with credentials and lookup a secret" in { - val props = Map( - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", - AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", - AWSProviderConfig.AWS_REGION -> "someregion" - ).asJava - - val secretKey = "my-secret-key" - val secretName = "my-secret-name" - val secretValue = "secret-value" - - val provider = new AWSSecretProvider() - provider.configure(props) - - val mockClient = mock[AWSSecretsManager] - val secretValRequest = - new GetSecretValueRequest().withSecretId(secretName) - val secretValResponse = new GetSecretValueResult() - val secretJson = new JsonObject().add(secretKey, secretValue) - secretValResponse.setName(secretName) - secretValResponse.setSecretString(secretJson.toString()) - - val now = new Date() - val describeSecretResponse = new DescribeSecretResult() - describeSecretResponse.setLastRotatedDate(now) - describeSecretResponse.setRotationEnabled(true) - describeSecretResponse.setLastRotatedDate(now) - - val rotationRulesType = new RotationRulesType() - rotationRulesType.setAutomaticallyAfterDays(1.toLong) - describeSecretResponse.setRotationRules(rotationRulesType) - - when(mockClient.describeSecret(any[DescribeSecretRequest])) - .thenReturn(describeSecretResponse) - when(mockClient.getSecretValue(secretValRequest)) - .thenReturn(secretValResponse) - - provider.client = Some(mockClient) - val data = provider.get(secretName, Set(secretKey).asJava) - data.data().get(secretKey) shouldBe secretValue - provider.close() - } - - "should authenticate with credentials and lookup a base64 secret" in { - val props = Map( - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", - AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", - AWSProviderConfig.AWS_REGION -> "someregion" - ).asJava - - val secretKey = Encoding.BASE64.toString - val secretName = "my-secret-name" - val secretValue = "base64-secret-value" - - val provider = new AWSSecretProvider() - provider.configure(props) - - val mockClient = mock[AWSSecretsManager] - val secretValRequest = - new GetSecretValueRequest().withSecretId(secretName) - val secretValResponse = new GetSecretValueResult() - secretValResponse.setName(secretName) - val secretJson = new JsonObject().add( - secretKey, - Base64.getEncoder.encodeToString(secretValue.getBytes) - ) - secretValResponse.setSecretString(secretJson.toString) - - val now = new Date() - val describeSecretResponse = new DescribeSecretResult() - describeSecretResponse.setLastRotatedDate(now) - describeSecretResponse.setRotationEnabled(true) - describeSecretResponse.setLastRotatedDate(now) - - val rotationRulesType = new RotationRulesType() - rotationRulesType.setAutomaticallyAfterDays(1.toLong) - describeSecretResponse.setRotationRules(rotationRulesType) - - when(mockClient.describeSecret(any[DescribeSecretRequest])) - .thenReturn(describeSecretResponse) - when(mockClient.getSecretValue(secretValRequest)) - .thenReturn(secretValResponse) - - provider.client = Some(mockClient) - val data = provider.get(secretName, Set(secretKey).asJava) - data.data().get(secretKey) shouldBe secretValue - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should authenticate with credentials and lookup a base64 secret and write to file" in { - val props = Map( - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", - AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", - AWSProviderConfig.AWS_REGION -> "someregion", - FILE_DIR -> tmp - ).asJava - - val secretKey = Encoding.BASE64_FILE.toString - val secretName = "my-secret-name" - val secretValue = "base64-secret-value" - - val provider = new AWSSecretProvider() - provider.configure(props) - - val mockClient = mock[AWSSecretsManager] - val secretValRequest = - new GetSecretValueRequest().withSecretId(secretName) - val secretValResponse = new GetSecretValueResult() - secretValResponse.setName(secretName) - val secretJson = new JsonObject().add( - secretKey, - Base64.getEncoder.encodeToString("base64-secret-value".getBytes) - ) - secretValResponse.setSecretString(secretJson.toString) - - val now = new Date() - val describeSecretResponse = new DescribeSecretResult() - describeSecretResponse.setLastRotatedDate(now) - describeSecretResponse.setRotationEnabled(true) - describeSecretResponse.setLastRotatedDate(now) - - val rotationRulesType = new RotationRulesType() - rotationRulesType.setAutomaticallyAfterDays(1.toLong) - describeSecretResponse.setRotationRules(rotationRulesType) - - when(mockClient.describeSecret(any[DescribeSecretRequest])) - .thenReturn(describeSecretResponse) - when(mockClient.getSecretValue(secretValRequest)) - .thenReturn(secretValResponse) - - provider.client = Some(mockClient) - val data = provider.get(secretName, Set(secretKey).asJava) - val outputFile = data.data().get(secretKey) - outputFile shouldBe s"$tmp$separator$secretName$separator${secretKey.toLowerCase}" - - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe secretValue - result.close() - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should authenticate with credentials and lookup a utf8 secret and write to file" in { - val props = Map( - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", - AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", - AWSProviderConfig.AWS_REGION -> "someregion", - FILE_DIR -> tmp - ).asJava - - val secretKey = - s"${Encoding.UTF8_FILE}${EncodingAndId.Separator}my-secret-key" - val secretName = "my-secret-name" - val secretValue = "utf8-secret-value" - - val provider = new AWSSecretProvider() - provider.configure(props) - - val mockClient = mock[AWSSecretsManager] - val secretValRequest = - new GetSecretValueRequest().withSecretId(secretName) - val secretValResponse = new GetSecretValueResult() - secretValResponse.setName(secretName) - val secretJson = new JsonObject().add( - secretKey, - secretValue - ) - secretValResponse.setSecretString(secretJson.toString) - - val now = new Date() - val describeSecretResponse = new DescribeSecretResult() - describeSecretResponse.setLastRotatedDate(now) - describeSecretResponse.setRotationEnabled(true) - describeSecretResponse.setLastRotatedDate(now) - - val rotationRulesType = new RotationRulesType() - rotationRulesType.setAutomaticallyAfterDays(1.toLong) - describeSecretResponse.setRotationRules(rotationRulesType) - - when(mockClient.describeSecret(any[DescribeSecretRequest])) - .thenReturn(describeSecretResponse) - when(mockClient.getSecretValue(secretValRequest)) - .thenReturn(secretValResponse) - - provider.client = Some(mockClient) - val data = provider.get(secretName, Set(secretKey).asJava) - val outputFile = data.data().get(secretKey) - outputFile shouldBe s"$tmp$separator$secretName$separator${secretKey.toLowerCase}" - - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe secretValue - result.close() - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should throw an exception if access key not set and not default auth mode" in { - - intercept[ConnectException] { - AWSProviderSettings( - AWSProviderConfig( - Map( - AWSProviderConfig.AWS_REGION -> "someregion", - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_SECRET_KEY -> "secretId" - ).asJava - ) - ) - } - } - - "should throw an exception if secret key not set and not default auth mode" in { - - intercept[ConnectException] { - AWSProviderSettings( - AWSProviderConfig( - Map( - AWSProviderConfig.AWS_REGION -> "someregion", - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "someclientid" - ).asJava - ) - ) - } - } - - "should check transformer" in { - - val secretKey = s"my-secret-key" - val secretName = "my-secret-name" - val secretValue = "utf8-secret-value" - - val mockClient = mock[AWSSecretsManager] - val secretValRequest = - new GetSecretValueRequest().withSecretId(secretName) - val secretValResponse = new GetSecretValueResult() - val secretJson = new JsonObject().add(secretKey, secretValue) - secretValResponse.setName(secretName) - secretValResponse.setSecretString(secretJson.toString()) - - val now = new Date() - val describeSecretResponse = new DescribeSecretResult() - describeSecretResponse.setLastRotatedDate(now) - describeSecretResponse.setRotationEnabled(true) - describeSecretResponse.setLastRotatedDate(now) - - val rotationRulesType = new RotationRulesType() - rotationRulesType.setAutomaticallyAfterDays(1.toLong) - describeSecretResponse.setRotationRules(rotationRulesType) - - when(mockClient.describeSecret(any[DescribeSecretRequest])) - .thenReturn(describeSecretResponse) - when(mockClient.getSecretValue(secretValRequest)) - .thenReturn(secretValResponse) - - val props = Map( - AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", - AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", - AWSProviderConfig.AWS_REGION -> "someregion" - ).asJava - - val provider = new AWSSecretProvider() - provider.configure(props) - provider.client = Some(mockClient) - - // check the workerconfigprovider - val map = new java.util.HashMap[String, ConfigProvider]() - map.put("aws", provider) - val transformer = new ConfigTransformer(map) - val props2 = - Map("mykey" -> "${aws:my-secret-name:my-secret-key}").asJava - val data = transformer.transform(props2) - data.data().get("mykey") shouldBe secretValue - provider.close() - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.amazonaws.services.secretsmanager.AWSSecretsManager +import com.amazonaws.services.secretsmanager.model._ +import com.bettercloud.vault.json.JsonObject +import io.lenses.connect.secrets.TmpDirUtil.getTempDir +import io.lenses.connect.secrets.config.{AWSProviderConfig, AWSProviderSettings} +import io.lenses.connect.secrets.connect.{AuthMode, Encoding, _} +import io.lenses.connect.secrets.utils.EncodingAndId +import org.apache.kafka.common.config.ConfigTransformer +import org.apache.kafka.common.config.provider.ConfigProvider +import org.apache.kafka.connect.errors.ConnectException +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.when +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import org.scalatestplus.mockito.MockitoSugar + +import java.nio.file.FileSystems +import java.util.{Base64, Date} +import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.{Success, Using} + +class AWSSecretProviderTest + extends AnyWordSpec + with Matchers + with MockitoSugar { + + val separator: String = FileSystems.getDefault.getSeparator + val tmp: String = s"$getTempDir${separator}provider-tests-aws" + + "should authenticate with credentials" in { + val props = Map( + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", + AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", + AWSProviderConfig.AWS_REGION -> "someregion" + ).asJava + + val provider = new AWSSecretProvider() + provider.configure(props) + provider.close() + } + + "should authenticate with credentials and lookup a secret" in { + val props = Map( + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", + AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", + AWSProviderConfig.AWS_REGION -> "someregion" + ).asJava + + val secretKey = "my-secret-key" + val secretName = "my-secret-name" + val secretValue = "secret-value" + + val provider = new AWSSecretProvider() + provider.configure(props) + + val mockClient = mock[AWSSecretsManager] + val secretValRequest = + new GetSecretValueRequest().withSecretId(secretName) + val secretValResponse = new GetSecretValueResult() + val secretJson = new JsonObject().add(secretKey, secretValue) + secretValResponse.setName(secretName) + secretValResponse.setSecretString(secretJson.toString()) + + val now = new Date() + val describeSecretResponse = new DescribeSecretResult() + describeSecretResponse.setLastRotatedDate(now) + describeSecretResponse.setRotationEnabled(true) + describeSecretResponse.setLastRotatedDate(now) + + val rotationRulesType = new RotationRulesType() + rotationRulesType.setAutomaticallyAfterDays(1.toLong) + describeSecretResponse.setRotationRules(rotationRulesType) + + when(mockClient.describeSecret(any[DescribeSecretRequest])) + .thenReturn(describeSecretResponse) + when(mockClient.getSecretValue(secretValRequest)) + .thenReturn(secretValResponse) + + provider.client = Some(mockClient) + val data = provider.get(secretName, Set(secretKey).asJava) + data.data().get(secretKey) shouldBe secretValue + provider.close() + } + + "should authenticate with credentials and lookup a base64 secret" in { + val props = Map( + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", + AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", + AWSProviderConfig.AWS_REGION -> "someregion" + ).asJava + + val secretKey = Encoding.BASE64.toString + val secretName = "my-secret-name" + val secretValue = "base64-secret-value" + + val provider = new AWSSecretProvider() + provider.configure(props) + + val mockClient = mock[AWSSecretsManager] + val secretValRequest = + new GetSecretValueRequest().withSecretId(secretName) + val secretValResponse = new GetSecretValueResult() + secretValResponse.setName(secretName) + val secretJson = new JsonObject().add( + secretKey, + Base64.getEncoder.encodeToString(secretValue.getBytes) + ) + secretValResponse.setSecretString(secretJson.toString) + + val now = new Date() + val describeSecretResponse = new DescribeSecretResult() + describeSecretResponse.setLastRotatedDate(now) + describeSecretResponse.setRotationEnabled(true) + describeSecretResponse.setLastRotatedDate(now) + + val rotationRulesType = new RotationRulesType() + rotationRulesType.setAutomaticallyAfterDays(1.toLong) + describeSecretResponse.setRotationRules(rotationRulesType) + + when(mockClient.describeSecret(any[DescribeSecretRequest])) + .thenReturn(describeSecretResponse) + when(mockClient.getSecretValue(secretValRequest)) + .thenReturn(secretValResponse) + + provider.client = Some(mockClient) + val data = provider.get(secretName, Set(secretKey).asJava) + data.data().get(secretKey) shouldBe secretValue + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should authenticate with credentials and lookup a base64 secret and write to file" in { + val props = Map( + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", + AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", + AWSProviderConfig.AWS_REGION -> "someregion", + FILE_DIR -> tmp + ).asJava + + val secretKey = Encoding.BASE64_FILE.toString + val secretName = "my-secret-name" + val secretValue = "base64-secret-value" + + val provider = new AWSSecretProvider() + provider.configure(props) + + val mockClient = mock[AWSSecretsManager] + val secretValRequest = + new GetSecretValueRequest().withSecretId(secretName) + val secretValResponse = new GetSecretValueResult() + secretValResponse.setName(secretName) + val secretJson = new JsonObject().add( + secretKey, + Base64.getEncoder.encodeToString("base64-secret-value".getBytes) + ) + secretValResponse.setSecretString(secretJson.toString) + + val now = new Date() + val describeSecretResponse = new DescribeSecretResult() + describeSecretResponse.setLastRotatedDate(now) + describeSecretResponse.setRotationEnabled(true) + describeSecretResponse.setLastRotatedDate(now) + + val rotationRulesType = new RotationRulesType() + rotationRulesType.setAutomaticallyAfterDays(1.toLong) + describeSecretResponse.setRotationRules(rotationRulesType) + + when(mockClient.describeSecret(any[DescribeSecretRequest])) + .thenReturn(describeSecretResponse) + when(mockClient.getSecretValue(secretValRequest)) + .thenReturn(secretValResponse) + + provider.client = Some(mockClient) + val data = provider.get(secretName, Set(secretKey).asJava) + val outputFile = data.data().get(secretKey) + outputFile shouldBe s"$tmp$separator$secretName$separator${secretKey.toLowerCase}" + + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + secretValue + ) + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should authenticate with credentials and lookup a utf8 secret and write to file" in { + val props = Map( + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", + AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", + AWSProviderConfig.AWS_REGION -> "someregion", + FILE_DIR -> tmp + ).asJava + + val secretKey = + s"${Encoding.UTF8_FILE}${EncodingAndId.Separator}my-secret-key" + val secretName = "my-secret-name" + val secretValue = "utf8-secret-value" + + val provider = new AWSSecretProvider() + provider.configure(props) + + val mockClient = mock[AWSSecretsManager] + val secretValRequest = + new GetSecretValueRequest().withSecretId(secretName) + val secretValResponse = new GetSecretValueResult() + secretValResponse.setName(secretName) + val secretJson = new JsonObject().add( + secretKey, + secretValue + ) + secretValResponse.setSecretString(secretJson.toString) + + val now = new Date() + val describeSecretResponse = new DescribeSecretResult() + describeSecretResponse.setLastRotatedDate(now) + describeSecretResponse.setRotationEnabled(true) + describeSecretResponse.setLastRotatedDate(now) + + val rotationRulesType = new RotationRulesType() + rotationRulesType.setAutomaticallyAfterDays(1.toLong) + describeSecretResponse.setRotationRules(rotationRulesType) + + when(mockClient.describeSecret(any[DescribeSecretRequest])) + .thenReturn(describeSecretResponse) + when(mockClient.getSecretValue(secretValRequest)) + .thenReturn(secretValResponse) + + provider.client = Some(mockClient) + val data = provider.get(secretName, Set(secretKey).asJava) + val outputFile = data.data().get(secretKey) + outputFile shouldBe s"$tmp$separator$secretName$separator${secretKey.toLowerCase}" + + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + secretValue + ) + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should throw an exception if access key not set and not default auth mode" in { + + intercept[ConnectException] { + AWSProviderSettings( + AWSProviderConfig( + Map( + AWSProviderConfig.AWS_REGION -> "someregion", + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_SECRET_KEY -> "secretId" + ).asJava + ) + ) + } + } + + "should throw an exception if secret key not set and not default auth mode" in { + + intercept[ConnectException] { + AWSProviderSettings( + AWSProviderConfig( + Map( + AWSProviderConfig.AWS_REGION -> "someregion", + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "someclientid" + ).asJava + ) + ) + } + } + + "should check transformer" in { + + val secretKey = s"my-secret-key" + val secretName = "my-secret-name" + val secretValue = "utf8-secret-value" + + val mockClient = mock[AWSSecretsManager] + val secretValRequest = + new GetSecretValueRequest().withSecretId(secretName) + val secretValResponse = new GetSecretValueResult() + val secretJson = new JsonObject().add(secretKey, secretValue) + secretValResponse.setName(secretName) + secretValResponse.setSecretString(secretJson.toString()) + + val now = new Date() + val describeSecretResponse = new DescribeSecretResult() + describeSecretResponse.setLastRotatedDate(now) + describeSecretResponse.setRotationEnabled(true) + describeSecretResponse.setLastRotatedDate(now) + + val rotationRulesType = new RotationRulesType() + rotationRulesType.setAutomaticallyAfterDays(1.toLong) + describeSecretResponse.setRotationRules(rotationRulesType) + + when(mockClient.describeSecret(any[DescribeSecretRequest])) + .thenReturn(describeSecretResponse) + when(mockClient.getSecretValue(secretValRequest)) + .thenReturn(secretValResponse) + + val props = Map( + AWSProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AWSProviderConfig.AWS_ACCESS_KEY -> "somekey", + AWSProviderConfig.AWS_SECRET_KEY -> "secretkey", + AWSProviderConfig.AWS_REGION -> "someregion" + ).asJava + + val provider = new AWSSecretProvider() + provider.configure(props) + provider.client = Some(mockClient) + + // check the workerconfigprovider + val map = new java.util.HashMap[String, ConfigProvider]() + map.put("aws", provider) + val transformer = new ConfigTransformer(map) + val props2 = + Map("mykey" -> "${aws:my-secret-name:my-secret-key}").asJava + val data = transformer.transform(props2) + data.data().get("mykey") shouldBe secretValue + provider.close() + } +} diff --git a/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelperTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelperTest.scala index ad8404e..4e313de 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelperTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingHelperTest.scala @@ -1,25 +1,25 @@ package io.lenses.connect.secrets.providers +import io.lenses.connect.secrets.providers.Aes256DecodingHelper.INITIALISATION_VECTOR_SEPARATOR import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec import org.scalatest.prop.TableDrivenPropertyChecks +import org.scalatest.wordspec.AnyWordSpec + +import java.util.UUID.randomUUID import scala.util.Random.nextString -import scala.util.Try -import Aes256DecodingHelper.INITIALISATION_VECTOR_SEPARATOR import scala.util.Success -import java.util.UUID.randomUUID class Aes256DecodingHelperTest extends AnyWordSpec with Matchers with TableDrivenPropertyChecks { - - import AesDecodingTestHelper.encrypt + + import AesDecodingTestHelper.encrypt "AES-256 decorer" should { "not be created for invalid key length" in { val secretKey = randomUUID.toString.take(16) - Aes256DecodingHelper.init(secretKey) shouldBe 'left + Aes256DecodingHelper.init(secretKey) shouldBe Symbol("left") } "not be able to decrypt message for uncrecognized key" in new TestContext { @@ -54,7 +54,7 @@ class Aes256DecodingHelperTest } trait TestContext { - val key = generateKey() + val key = generateKey() def generateKey(): String = randomUUID.toString.take(32) diff --git a/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingProviderTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingProviderTest.scala index 282a768..7f8e187 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingProviderTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/Aes256DecodingProviderTest.scala @@ -1,32 +1,23 @@ package io.lenses.connect.secrets.providers +import io.lenses.connect.secrets.config.Aes256ProviderConfig +import io.lenses.connect.secrets.connect.{Encoding, FILE_DIR} +import io.lenses.connect.secrets.utils.EncodingAndId import org.apache.kafka.common.config.provider.ConfigProvider +import org.apache.kafka.common.config.{ConfigException, ConfigTransformer} +import org.apache.kafka.connect.errors.ConnectException import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.wordspec.AnyWordSpec -import java.util.Base64 -import java.util.UUID.randomUUID - -import io.lenses.connect.secrets.config.Aes256ProviderConfig - -import scala.collection.JavaConverters._ -import org.apache.kafka.connect.errors.ConnectException -import org.apache.kafka.common.config.ConfigException -import org.apache.kafka.common.config.ConfigTransformer -import io.lenses.connect.secrets.connect.FILE_DIR -import java.io.File -import java.nio.file.Files - -import org.scalatest.compatible.Assertion -import scala.io.Source import java.io.FileInputStream +import java.nio.file.Files import java.util - -import io.lenses.connect.secrets.connect.Encoding -import io.lenses.connect.secrets.utils.EncodingAndId - -import scala.util.Random +import java.util.Base64 +import java.util.UUID.randomUUID +import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.{Random, Success, Using} class Aes256DecodingProviderTest extends AnyWordSpec @@ -35,42 +26,60 @@ class Aes256DecodingProviderTest import AesDecodingTestHelper.encrypt "aes256 provider" should { - "decrypt aes 256 utf-8 encoded value" in new TestContext with ConfiguredProvider { + "decrypt aes 256 utf-8 encoded value" in new TestContext + with ConfiguredProvider { val encrypted = encrypt(value, key) forAll(Table("encoding", "", "utf-8")) { encoding => - val decrypted = provider.get(encoding, Set(encrypted).asJava).data().asScala + val decrypted = + provider.get(encoding, Set(encrypted).asJava).data().asScala decrypted.get(encrypted) shouldBe Some(value) } } - "decrypt aes 256 base64 encoded value" in new TestContext with ConfiguredProvider { - val encrypted = encrypt(Base64.getEncoder.encodeToString(value.getBytes()), key) + "decrypt aes 256 base64 encoded value" in new TestContext + with ConfiguredProvider { + val encrypted = + encrypt(Base64.getEncoder.encodeToString(value.getBytes()), key) - val decrypted = provider.get("base64", Set(encrypted).asJava).data().asScala + val decrypted = + provider.get("base64", Set(encrypted).asJava).data().asScala decrypted.get(encrypted) shouldBe Some(value) } - "decrypt aes 256 encoded value stored in file with utf-8 encoding" in new TestContext with ConfiguredProvider { + "decrypt aes 256 encoded value stored in file with utf-8 encoding" in new TestContext + with ConfiguredProvider { val encrypted = encrypt(value, key) - val providerData = provider.get(s"utf8_file${EncodingAndId.Separator}id1", Set(encrypted).asJava).data().asScala + val providerData = provider + .get(s"utf8_file${EncodingAndId.Separator}id1", Set(encrypted).asJava) + .data() + .asScala val decryptedPath = providerData(encrypted) decryptedPath should startWith(s"$tmpDir/secrets/") decryptedPath.toLowerCase.contains(encrypted.toLowerCase) shouldBe false - Source.fromFile(decryptedPath).getLines.mkString shouldBe value + Using(Source.fromFile(decryptedPath))(_.getLines().mkString) shouldBe Success( + value + ) } - "decrypt aes 256 encoded value stored in file with base64 encoding" in new TestContext with ConfiguredProvider { + "decrypt aes 256 encoded value stored in file with base64 encoding" in new TestContext + with ConfiguredProvider { val bytesAmount = 100 val bytesInput = Array.fill[Byte](bytesAmount)(0) Random.nextBytes(bytesInput) val encrypted = encrypt(Base64.getEncoder.encodeToString(bytesInput), key) - val providerData = provider.get(s"${Encoding.BASE64_FILE}${EncodingAndId.Separator}fileId1", Set(encrypted).asJava).data().asScala + val providerData = provider + .get( + s"${Encoding.BASE64_FILE}${EncodingAndId.Separator}fileId1", + Set(encrypted).asJava + ) + .data() + .asScala val decryptedPath = providerData(encrypted) decryptedPath should startWith(s"$tmpDir/secrets/") @@ -81,7 +90,6 @@ class Aes256DecodingProviderTest bytesConsumed.toList shouldBe bytesInput.toList } - "transform value referencing to the provider" in new TestContext { val value = "hi!" val encrypted = encrypt(value, key) diff --git a/src/test/scala/io/lenses/connect/secrets/providers/AesDecodingTestHelper.scala b/src/test/scala/io/lenses/connect/secrets/providers/AesDecodingTestHelper.scala index 0f2bcbb..6a957d0 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/AesDecodingTestHelper.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/AesDecodingTestHelper.scala @@ -1,41 +1,49 @@ package io.lenses.connect.secrets.providers -import javax.crypto.Cipher -import javax.crypto.spec.IvParameterSpec -import javax.crypto.spec.SecretKeySpec +import io.lenses.connect.secrets.providers.Aes256DecodingHelper.INITIALISATION_VECTOR_SEPARATOR + import java.util.Base64 +import javax.crypto.Cipher +import javax.crypto.spec.{IvParameterSpec, SecretKeySpec} import scala.util.Try -import Aes256DecodingHelper.INITIALISATION_VECTOR_SEPARATOR object AesDecodingTestHelper { private val AES = "AES" - - def encrypt(s: String, key: String): String = { - val iv = InitializationVector() - encryptBytes(s.getBytes("UTF-8"), iv, key).map(encrypted => - base64Encode(iv.bytes) + INITIALISATION_VECTOR_SEPARATOR + base64Encode(encrypted) - ).get - } - private def encryptBytes( - bytes: Array[Byte], - iv: InitializationVector, - key: String - ): Try[Array[Byte]] = - for { - cipher <- getCipher(Cipher.ENCRYPT_MODE, iv, key) - encrypted <- Try(cipher.doFinal(bytes)) - } yield encrypted + def encrypt(s: String, key: String): String = { + val iv = InitializationVector() + encryptBytes(s.getBytes("UTF-8"), iv, key) + .map(encrypted => + base64Encode(iv.bytes) + INITIALISATION_VECTOR_SEPARATOR + base64Encode( + encrypted + ) + ) + .get + } + + private def encryptBytes( + bytes: Array[Byte], + iv: InitializationVector, + key: String + ): Try[Array[Byte]] = + for { + cipher <- getCipher(Cipher.ENCRYPT_MODE, iv, key) + encrypted <- Try(cipher.doFinal(bytes)) + } yield encrypted - private def getCipher(mode: Int, iv: InitializationVector, key: String): Try[Cipher] = - Try { - val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") - val ivSpec = new IvParameterSpec(iv.bytes) - val secret = new SecretKeySpec(key.getBytes("UTF-8"), AES) - cipher.init(mode, secret, ivSpec) - cipher - } + private def getCipher( + mode: Int, + iv: InitializationVector, + key: String + ): Try[Cipher] = + Try { + val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding") + val ivSpec = new IvParameterSpec(iv.bytes) + val secret = new SecretKeySpec(key.getBytes("UTF-8"), AES) + cipher.init(mode, secret, ivSpec) + cipher + } - private def base64Encode(bytes: Array[Byte]) = - Base64.getEncoder().encodeToString(bytes) + private def base64Encode(bytes: Array[Byte]) = + Base64.getEncoder().encodeToString(bytes) } diff --git a/src/test/scala/io/lenses/connect/secrets/providers/AzureSecretProviderTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/AzureSecretProviderTest.scala index 7331e7d..3ae7696 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/AzureSecretProviderTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/AzureSecretProviderTest.scala @@ -1,419 +1,438 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.nio.file.FileSystems -import java.time.OffsetDateTime -import java.util.Base64 - -import com.azure.security.keyvault.secrets.SecretClient -import com.azure.security.keyvault.secrets.models.{KeyVaultSecret, SecretProperties} -import io.lenses.connect.secrets.config.{AzureProviderConfig, AzureProviderSettings} -import io.lenses.connect.secrets.connect -import io.lenses.connect.secrets.connect.AuthMode -import org.apache.kafka.common.config.{ConfigData, ConfigDef, ConfigTransformer} -import org.apache.kafka.common.config.provider.ConfigProvider -import org.apache.kafka.connect.errors.ConnectException -import org.mockito.MockitoSugar -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import scala.collection.JavaConverters._ -import scala.io.Source - -class AzureSecretProviderTest - extends AnyWordSpec - with Matchers - with MockitoSugar { - - val separator: String = FileSystems.getDefault.getSeparator - val tmp: String = - System.getProperty("java.io.tmpdir") + separator + "provider-tests-azure" - - "should get secrets at a path with service principal credentials" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid" - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - - val secretKey = "my-key" - val secretValue = "secret-value" - val secretPath = "my-path.vault.azure.net" - - val client = mock[SecretClient] - val secret = mock[KeyVaultSecret] - val secretProperties = mock[SecretProperties] - val offset = OffsetDateTime.now() - - // string secret - when(secretProperties.getExpiresOn).thenReturn(offset) - when(secret.getValue).thenReturn(secretValue) - when(secret.getProperties).thenReturn(secretProperties) - when(client.getSecret(secretKey)).thenReturn(secret) - - // poke in the mocked client - provider.clientMap += (s"https://$secretPath" -> client) - val data = provider.get(secretPath, Set(secretKey).asJava) - data.data().containsKey(secretKey) - data.data().get(secretKey) shouldBe secretValue - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should get base64 secrets at a path with service principal credentials" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid" - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - - val secretKey = "base64-key" - val secretValue = "base64-secret-value" - val secretPath = "my-path.vault.azure.net" - - val client = mock[SecretClient] - - //base64 secret - val secretb64 = mock[KeyVaultSecret] - val secretPropertiesb64 = mock[SecretProperties] - val ttl = OffsetDateTime.now() - - when(secretPropertiesb64.getExpiresOn).thenReturn(ttl) - when(secretPropertiesb64.getTags) - .thenReturn( - Map(connect.FILE_ENCODING -> connect.Encoding.BASE64.toString).asJava - ) - when(secretb64.getValue).thenReturn( - Base64.getEncoder.encodeToString(secretValue.getBytes) - ) - when(secretb64.getProperties).thenReturn(secretPropertiesb64) - when(client.getSecret(secretKey)).thenReturn(secretb64) - - // poke in the mocked client - provider.clientMap += (s"https://$secretPath" -> client) - val data = provider.get(secretPath, Set(secretKey).asJava) - data.data().get(secretKey) shouldBe secretValue - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should get base64 secrets and write to file" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", - connect.FILE_DIR -> tmp - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - val secretKey = "base64-key" - val secretValue = "base64-secret-value" - val secretPath = "my-path.vault.azure.net" - - val client = mock[SecretClient] - - //base64 secret - val secretb64 = mock[KeyVaultSecret] - val secretPropertiesb64 = mock[SecretProperties] - val ttl = OffsetDateTime.now() - - when(secretPropertiesb64.getExpiresOn).thenReturn(ttl) - when(secretPropertiesb64.getTags) - .thenReturn( - Map(connect.FILE_ENCODING -> connect.Encoding.BASE64_FILE.toString).asJava - ) - when(secretb64.getValue).thenReturn( - Base64.getEncoder.encodeToString(secretValue.getBytes) - ) - when(secretb64.getProperties).thenReturn(secretPropertiesb64) - when(client.getSecret(secretKey)).thenReturn(secretb64) - when(client.getVaultUrl).thenReturn(s"https://$secretPath") - - // poke in the mocked client - provider.clientMap += (s"https://$secretPath" -> client) - val data = provider.get(secretPath, Set(secretKey).asJava) - val outputFile = data.data().get(secretKey) - - outputFile shouldBe s"$tmp$separator$secretPath$separator$secretKey" - - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe secretValue - result.close() - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should get utf secrets and write to file" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", - connect.FILE_DIR -> tmp - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - - val secretKey = "utf8-key" - val secretValue = "utf8-secret-value" - val secretPath = "my-path.vault.azure.net" - - val client = mock[SecretClient] - val secret = mock[KeyVaultSecret] - val secretProperties = mock[SecretProperties] - val ttl = OffsetDateTime.now() - - when(secretProperties.getExpiresOn).thenReturn(ttl) - when(secretProperties.getTags) - .thenReturn( - Map(connect.FILE_ENCODING -> connect.Encoding.UTF8_FILE.toString).asJava - ) - when(secret.getValue).thenReturn(secretValue) - when(secret.getProperties).thenReturn(secretProperties) - when(client.getSecret(secretKey)).thenReturn(secret) - when(client.getVaultUrl).thenReturn(s"https://$secretPath") - - // poke in the mocked client - provider.clientMap += (s"https://$secretPath" -> client) - val data = provider.get(secretPath, Set(secretKey).asJava) - val outputFile = data.data().get(secretKey) - - outputFile shouldBe s"$tmp$separator$secretPath$separator$secretKey" - - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe secretValue - result.close() - - provider.get("").data().isEmpty shouldBe true - provider.close() - } - - "should use cache" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", - connect.FILE_DIR -> tmp - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - - val secretKey = "utf8-key" - val secretValue = "utf8-secret-value" - val secretPath = "my-path.vault.azure.net" - - val client = mock[SecretClient] - - // poke in the mocked client - provider.clientMap += (s"https://$secretPath" -> client) - val now = OffsetDateTime.now().plusMinutes(10) - val cachedData = new ConfigData(Map(secretKey -> secretPath).asJava) - val cached = (Some(now), cachedData) - - // add to cache - provider.cache += (s"https://$secretPath" -> cached) - val data = provider.get(secretPath, Set(secretKey).asJava) - data.data().containsKey(secretKey) - } - - "should use not cache because of expiry" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", - connect.FILE_DIR -> tmp - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - - val secretKey = "utf8-key" - val secretValue = "utf8-secret-value" - val secretPath = "my-path.vault.azure.net" - val vaultUrl = s"https://$secretPath" - - val client = mock[SecretClient] - val secret = mock[KeyVaultSecret] - val secretProperties = mock[SecretProperties] - val ttl = OffsetDateTime.now().plusHours(1) - - when(secretProperties.getExpiresOn).thenReturn(ttl) - when(secretProperties.getTags) - .thenReturn( - Map(connect.FILE_ENCODING -> connect.Encoding.UTF8_FILE.toString).asJava - ) - when(secret.getValue).thenReturn(secretValue) - when(secret.getProperties).thenReturn(secretProperties) - when(client.getSecret(secretKey)).thenReturn(secret) - when(client.getVaultUrl).thenReturn(vaultUrl) - - // poke in the mocked client - provider.clientMap += (vaultUrl -> client) - //put expiry of cache 1 second behind - val now = OffsetDateTime.now().minusSeconds(1) - val cachedData = new ConfigData(Map(secretKey -> secretPath).asJava) - val cached = (Some(now), cachedData) - - // add to cache - provider.cache += (vaultUrl -> cached) - val data = provider.get(secretPath, Set(secretKey).asJava) - data.data().containsKey(secretKey) - // ttl should be in future now in cache - provider.cache(vaultUrl)._1.get shouldBe ttl - } - - "should use not cache because of different keys" in { - val props = Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", - connect.FILE_DIR -> tmp - ).asJava - - val provider = new AzureSecretProvider - provider.configure(props) - - val secretKey = "utf8-key" - val secretValue = "utf8-secret-value" - val secretPath = "my-path.vault.azure.net" - val vaultUrl = s"https://$secretPath" - - val client = mock[SecretClient] - val secret = mock[KeyVaultSecret] - val secretProperties = mock[SecretProperties] - val ttl = OffsetDateTime.now().plusHours(1) - - when(secretProperties.getExpiresOn).thenReturn(ttl) - when(secretProperties.getTags) - .thenReturn( - Map(connect.FILE_ENCODING -> connect.Encoding.UTF8_FILE.toString).asJava - ) - when(secret.getValue).thenReturn(secretValue) - when(secret.getProperties).thenReturn(secretProperties) - when(client.getSecret(secretKey)).thenReturn(secret) - when(client.getVaultUrl).thenReturn(vaultUrl) - - // poke in the mocked client - provider.clientMap += (vaultUrl -> client) - //put expiry of cache 1 second behind - val now = OffsetDateTime.now() - val cachedData = new ConfigData(Map("old-key" -> secretPath).asJava) - val cached = (Some(now), cachedData) - - // add to cache - provider.cache += (vaultUrl -> cached) - val data = provider.get(secretPath, Set(secretKey).asJava) - data.data().containsKey(secretKey) - } - - "should throw an exception if client id not set and not default auth mode" in { - - intercept[ConnectException] { - AzureProviderSettings( - AzureProviderConfig( - Map(AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString).asJava - ) - ) - } - } - - "should throw an exception if tenant id not set and not default auth mode" in { - - intercept[ConnectException] { - AzureProviderSettings( - AzureProviderConfig( - Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid" - ).asJava - ) - ) - } - } - - "should throw an exception if secret id not set and not default auth mode" in { - - intercept[ConnectException] { - AzureProviderSettings( - AzureProviderConfig( - Map( - AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid" - ).asJava - ) - ) - } - } - - "should not throw an exception if service principals not set and default auth mode" in { - - val settings = AzureProviderSettings( - AzureProviderConfig( - Map(AzureProviderConfig.AUTH_METHOD -> AuthMode.DEFAULT.toString).asJava - ) - ) - - settings.authMode shouldBe AuthMode.DEFAULT - } - - "check transformer" in { - val props1 = Map( - AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", - AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", - AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid" - ).asJava - - val secretKey = "key-1" - val secretValue = "utf8-secret-value" - val secretPath = "my-path.vault.azure.net" - - val client = mock[SecretClient] - val secret = mock[KeyVaultSecret] - val secretProperties = mock[SecretProperties] - val offset = OffsetDateTime.now() - - // string secret - when(secretProperties.getExpiresOn).thenReturn(offset) - when(secret.getValue).thenReturn(secretValue) - when(secret.getProperties).thenReturn(secretProperties) - when(client.getSecret(secretKey)).thenReturn(secret) - - val provider = new AzureSecretProvider() - provider.configure(props1) - - provider.clientMap += (s"https://$secretPath" -> client) - - // check the workerconfigprovider - val map = new java.util.HashMap[String, ConfigProvider]() - map.put("azure", provider) - val transformer = new ConfigTransformer(map) - val props2 = - Map("mykey" -> "${azure:my-path.vault.azure.net:key-1}").asJava - val data = transformer.transform(props2) - data.data().get("mykey") shouldBe secretValue - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import com.azure.security.keyvault.secrets.SecretClient +import com.azure.security.keyvault.secrets.models.{ + KeyVaultSecret, + SecretProperties +} +import io.lenses.connect.secrets.TmpDirUtil.getTempDir +import io.lenses.connect.secrets.config.{ + AzureProviderConfig, + AzureProviderSettings +} +import io.lenses.connect.secrets.connect +import io.lenses.connect.secrets.connect.AuthMode +import org.apache.kafka.common.config.provider.ConfigProvider +import org.apache.kafka.common.config.{ConfigData, ConfigTransformer} +import org.apache.kafka.connect.errors.ConnectException +import org.mockito.Mockito.when +import org.scalatestplus.mockito.MockitoSugar +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.nio.file.FileSystems +import java.time.OffsetDateTime +import java.util.Base64 +import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.{Success, Using} + +class AzureSecretProviderTest + extends AnyWordSpec + with Matchers + with MockitoSugar { + + val separator: String = FileSystems.getDefault.getSeparator + val tmp: String = + s"$getTempDir${separator}provider-tests-azure" + + "should get secrets at a path with service principal credentials" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid" + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + + val secretKey = "my-key" + val secretValue = "secret-value" + val secretPath = "my-path.vault.azure.net" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + val secret = mock[KeyVaultSecret] + val secretProperties = mock[SecretProperties] + val offset = OffsetDateTime.now() + + // string secret + when(secretProperties.getExpiresOn).thenReturn(offset) + when(secret.getValue).thenReturn(secretValue) + when(secret.getProperties).thenReturn(secretProperties) + when(client.getSecret(secretKey)).thenReturn(secret) + + // poke in the mocked client + provider.clientMap += (s"https://$secretPath" -> client) + val data = provider.get(secretPath, Set(secretKey).asJava) + data.data().containsKey(secretKey) + data.data().get(secretKey) shouldBe secretValue + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should get base64 secrets at a path with service principal credentials" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid" + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + + val secretKey = "base64-key" + val secretValue = "base64-secret-value" + val secretPath = "my-path.vault.azure.net" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + //base64 secret + val secretb64 = mock[KeyVaultSecret] + val secretPropertiesb64 = mock[SecretProperties] + val ttl = OffsetDateTime.now() + + when(secretPropertiesb64.getExpiresOn).thenReturn(ttl) + when(secretPropertiesb64.getTags) + .thenReturn( + Map(connect.FILE_ENCODING -> connect.Encoding.BASE64.toString).asJava + ) + when(secretb64.getValue).thenReturn( + Base64.getEncoder.encodeToString(secretValue.getBytes) + ) + when(secretb64.getProperties).thenReturn(secretPropertiesb64) + when(client.getSecret(secretKey)).thenReturn(secretb64) + + // poke in the mocked client + provider.clientMap += (s"https://$secretPath" -> client) + val data = provider.get(secretPath, Set(secretKey).asJava) + data.data().get(secretKey) shouldBe secretValue + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should get base64 secrets and write to file" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", + connect.FILE_DIR -> tmp + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + val secretKey = "base64-key" + val secretValue = "base64-secret-value" + val secretPath = "my-path.vault.azure.net" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + //base64 secret + val secretb64 = mock[KeyVaultSecret] + val secretPropertiesb64 = mock[SecretProperties] + val ttl = OffsetDateTime.now() + + when(secretPropertiesb64.getExpiresOn).thenReturn(ttl) + when(secretPropertiesb64.getTags) + .thenReturn( + Map(connect.FILE_ENCODING -> connect.Encoding.BASE64_FILE.toString).asJava + ) + when(secretb64.getValue).thenReturn( + Base64.getEncoder.encodeToString(secretValue.getBytes) + ) + when(secretb64.getProperties).thenReturn(secretPropertiesb64) + when(client.getSecret(secretKey)).thenReturn(secretb64) + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + // poke in the mocked client + provider.clientMap += (s"https://$secretPath" -> client) + val data = provider.get(secretPath, Set(secretKey).asJava) + val outputFile = data.data().get(secretKey) + + outputFile shouldBe s"$tmp$separator$secretPath$separator$secretKey" + + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + secretValue + ) + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should get utf secrets and write to file" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", + connect.FILE_DIR -> tmp + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + + val secretKey = "utf8-key" + val secretValue = "utf8-secret-value" + val secretPath = "my-path.vault.azure.net" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + val secret = mock[KeyVaultSecret] + val secretProperties = mock[SecretProperties] + val ttl = OffsetDateTime.now() + + when(secretProperties.getExpiresOn).thenReturn(ttl) + when(secretProperties.getTags) + .thenReturn( + Map(connect.FILE_ENCODING -> connect.Encoding.UTF8_FILE.toString).asJava + ) + when(secret.getValue).thenReturn(secretValue) + when(secret.getProperties).thenReturn(secretProperties) + when(client.getSecret(secretKey)).thenReturn(secret) + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + // poke in the mocked client + provider.clientMap += (s"https://$secretPath" -> client) + val data = provider.get(secretPath, Set(secretKey).asJava) + val outputFile = data.data().get(secretKey) + + outputFile shouldBe s"$tmp$separator$secretPath$separator$secretKey" + + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + secretValue + ) + + provider.get("").data().isEmpty shouldBe true + provider.close() + } + + "should use cache" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", + connect.FILE_DIR -> tmp + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + + val secretKey = "utf8-key" + val secretPath = "my-path.vault.azure.net" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + // poke in the mocked client + provider.clientMap += (s"https://$secretPath" -> client) + val now = OffsetDateTime.now().plusMinutes(10) + val cachedData = new ConfigData(Map(secretKey -> secretPath).asJava) + val cached = (Some(now), cachedData) + + // add to cache + provider.cache += (s"https://$secretPath" -> cached) + val data = provider.get(secretPath, Set(secretKey).asJava) + data.data().containsKey(secretKey) + } + + "should use not cache because of expiry" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", + connect.FILE_DIR -> tmp + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + + val secretKey = "utf8-key" + val secretValue = "utf8-secret-value" + val secretPath = "my-path.vault.azure.net" + val vaultUrl = s"https://$secretPath" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + val secret = mock[KeyVaultSecret] + val secretProperties = mock[SecretProperties] + val ttl = OffsetDateTime.now().plusHours(1) + + when(secretProperties.getExpiresOn).thenReturn(ttl) + when(secretProperties.getTags) + .thenReturn( + Map(connect.FILE_ENCODING -> connect.Encoding.UTF8_FILE.toString).asJava + ) + when(secret.getValue).thenReturn(secretValue) + when(secret.getProperties).thenReturn(secretProperties) + when(client.getSecret(secretKey)).thenReturn(secret) + when(client.getVaultUrl).thenReturn(vaultUrl) + + // poke in the mocked client + provider.clientMap += (vaultUrl -> client) + //put expiry of cache 1 second behind + val now = OffsetDateTime.now().minusSeconds(1) + val cachedData = new ConfigData(Map(secretKey -> secretPath).asJava) + val cached = (Some(now), cachedData) + + // add to cache + provider.cache += (vaultUrl -> cached) + val data = provider.get(secretPath, Set(secretKey).asJava) + data.data().containsKey(secretKey) + // ttl should be in future now in cache + provider.cache(vaultUrl)._1.get shouldBe ttl + } + + "should use not cache because of different keys" in { + val props = Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid", + connect.FILE_DIR -> tmp + ).asJava + + val provider = new AzureSecretProvider + provider.configure(props) + + val secretKey = "utf8-key" + val secretValue = "utf8-secret-value" + val secretPath = "my-path.vault.azure.net" + val vaultUrl = s"https://$secretPath" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + val secret = mock[KeyVaultSecret] + val secretProperties = mock[SecretProperties] + val ttl = OffsetDateTime.now().plusHours(1) + + when(secretProperties.getExpiresOn).thenReturn(ttl) + when(secretProperties.getTags) + .thenReturn( + Map(connect.FILE_ENCODING -> connect.Encoding.UTF8_FILE.toString).asJava + ) + when(secret.getValue).thenReturn(secretValue) + when(secret.getProperties).thenReturn(secretProperties) + when(client.getSecret(secretKey)).thenReturn(secret) + when(client.getVaultUrl).thenReturn(vaultUrl) + + // poke in the mocked client + provider.clientMap += (vaultUrl -> client) + //put expiry of cache 1 second behind + val now = OffsetDateTime.now() + val cachedData = new ConfigData(Map("old-key" -> secretPath).asJava) + val cached = (Some(now), cachedData) + + // add to cache + provider.cache += (vaultUrl -> cached) + val data = provider.get(secretPath, Set(secretKey).asJava) + data.data().containsKey(secretKey) + } + + "should throw an exception if client id not set and not default auth mode" in { + + intercept[ConnectException] { + AzureProviderSettings( + AzureProviderConfig( + Map(AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString).asJava + ) + ) + } + } + + "should throw an exception if tenant id not set and not default auth mode" in { + + intercept[ConnectException] { + AzureProviderSettings( + AzureProviderConfig( + Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid" + ).asJava + ) + ) + } + } + + "should throw an exception if secret id not set and not default auth mode" in { + + intercept[ConnectException] { + AzureProviderSettings( + AzureProviderConfig( + Map( + AzureProviderConfig.AUTH_METHOD -> AuthMode.CREDENTIALS.toString, + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid" + ).asJava + ) + ) + } + } + + "should not throw an exception if service principals not set and default auth mode" in { + + val settings = AzureProviderSettings( + AzureProviderConfig( + Map(AzureProviderConfig.AUTH_METHOD -> AuthMode.DEFAULT.toString).asJava + ) + ) + + settings.authMode shouldBe AuthMode.DEFAULT + } + + "check transformer" in { + val props1 = Map( + AzureProviderConfig.AZURE_CLIENT_ID -> "someclientid", + AzureProviderConfig.AZURE_TENANT_ID -> "sometenantid", + AzureProviderConfig.AZURE_SECRET_ID -> "somesecretid" + ).asJava + + val secretKey = "key-1" + val secretValue = "utf8-secret-value" + val secretPath = "my-path.vault.azure.net" + + val client = mock[SecretClient] + when(client.getVaultUrl).thenReturn(s"https://$secretPath") + + val secret = mock[KeyVaultSecret] + val secretProperties = mock[SecretProperties] + val offset = OffsetDateTime.now() + + // string secret + when(secretProperties.getExpiresOn).thenReturn(offset) + when(secret.getValue).thenReturn(secretValue) + when(secret.getProperties).thenReturn(secretProperties) + when(client.getSecret(secretKey)).thenReturn(secret) + + val provider = new AzureSecretProvider() + provider.configure(props1) + + provider.clientMap += (s"https://$secretPath" -> client) + + // check the workerconfigprovider + val map = new java.util.HashMap[String, ConfigProvider]() + map.put("azure", provider) + val transformer = new ConfigTransformer(map) + val props2 = + Map("mykey" -> "${azure:my-path.vault.azure.net:key-1}").asJava + val data = transformer.transform(props2) + data.data().get("mykey") shouldBe secretValue + } +} diff --git a/src/test/scala/io/lenses/connect/secrets/providers/DecodeTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/DecodeTest.scala index 50e4168..95794b7 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/DecodeTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/DecodeTest.scala @@ -1,109 +1,109 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.io.File -import java.nio.file.FileSystems -import java.time.OffsetDateTime -import java.util.Base64 - -import io.lenses.connect.secrets.connect -import io.lenses.connect.secrets.connect.Encoding -import org.apache.commons.io.FileUtils -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -class DecodeTest extends AnyWordSpec with Matchers { - - val separator: String = FileSystems.getDefault.getSeparator - val tmp: String = - System.getProperty("java.io.tmpdir") + separator + "decoder-tests" - - def cleanUp(fileName: String): AnyVal = { - val tmpFile = new File(fileName) - if (tmpFile.exists) tmpFile.delete() - } - - "should decode UTF" in { - connect.decodeKey(None, "my-key", "secret", { _ => - fail("No files here") - }) shouldBe "secret" - connect.decodeKey(Some(Encoding.UTF8), "my-key", "secret", { _ => - fail("No files here") - }) shouldBe "secret" - } - - "should decode BASE64" in { - val value = Base64.getEncoder.encodeToString("secret".getBytes) - connect.decodeKey(Some(Encoding.BASE64), s"my-key", value, { _ => - fail("No files here") - }) shouldBe "secret" - } - - "should decode BASE64 and write to a file" in { - val fileName = s"${tmp}my-file-base64" - - val value = Base64.getEncoder.encodeToString("secret".getBytes) - var written = false - connect.decodeKey( - Some(Encoding.BASE64_FILE), - s"my-key", - value, { _ => - written = true - fileName - } - ) shouldBe fileName - written shouldBe true - } - - "should decode and write a jks" in { - val fileName = s"${tmp}my-file-base64-jks" - val jksFile: String = - getClass.getClassLoader.getResource("keystore.jks").getPath - val fileContent = FileUtils.readFileToByteArray(new File(jksFile)) - val jksEncoded = Base64.getEncoder.encodeToString(fileContent) - - var written = false - connect.decodeKey( - Some(Encoding.BASE64_FILE), - s"keystore.jks", - jksEncoded, { _ => - written = true - fileName - } - ) shouldBe fileName - - written shouldBe true - } - - "should decode UTF8 and write to a file" in { - val fileName = s"${tmp}my-file-utf8" - var written = false - - connect.decodeKey( - Some(Encoding.UTF8_FILE), - s"my-key", - "secret", { _ => - written = true - fileName - } - ) shouldBe fileName - written shouldBe true - } - - "min list test" in { - val now = OffsetDateTime.now() - val secrets = Map( - "ke3" -> ("value", Some(OffsetDateTime.now().plusHours(3))), - "key1" -> ("value", Some(now)), - "key2" -> ("value", Some(OffsetDateTime.now().plusHours(1))) - ) - - val (expiry, _) = connect.getSecretsAndExpiry(secrets) - expiry shouldBe Some(now) - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import io.lenses.connect.secrets.connect +import io.lenses.connect.secrets.connect.Encoding +import org.apache.commons.io.FileUtils +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.io.File +import java.nio.file.FileSystems +import java.time.OffsetDateTime +import java.util.Base64 + +class DecodeTest extends AnyWordSpec with Matchers { + + val separator: String = FileSystems.getDefault.getSeparator + val tmp: String = + System.getProperty("java.io.tmpdir") + separator + "decoder-tests" + + def cleanUp(fileName: String): AnyVal = { + val tmpFile = new File(fileName) + if (tmpFile.exists) tmpFile.delete() + } + + "should decode UTF" in { + connect.decodeKey(None, "my-key", "secret", { _ => + fail("No files here") + }) shouldBe "secret" + connect.decodeKey(Some(Encoding.UTF8), "my-key", "secret", { _ => + fail("No files here") + }) shouldBe "secret" + } + + "should decode BASE64" in { + val value = Base64.getEncoder.encodeToString("secret".getBytes) + connect.decodeKey(Some(Encoding.BASE64), s"my-key", value, { _ => + fail("No files here") + }) shouldBe "secret" + } + + "should decode BASE64 and write to a file" in { + val fileName = s"${tmp}my-file-base64" + + val value = Base64.getEncoder.encodeToString("secret".getBytes) + var written = false + connect.decodeKey( + Some(Encoding.BASE64_FILE), + s"my-key", + value, { _ => + written = true + fileName + } + ) shouldBe fileName + written shouldBe true + } + + "should decode and write a jks" in { + val fileName = s"${tmp}my-file-base64-jks" + val jksFile: String = + getClass.getClassLoader.getResource("keystore.jks").getPath + val fileContent = FileUtils.readFileToByteArray(new File(jksFile)) + val jksEncoded = Base64.getEncoder.encodeToString(fileContent) + + var written = false + connect.decodeKey( + Some(Encoding.BASE64_FILE), + s"keystore.jks", + jksEncoded, { _ => + written = true + fileName + } + ) shouldBe fileName + + written shouldBe true + } + + "should decode UTF8 and write to a file" in { + val fileName = s"${tmp}my-file-utf8" + var written = false + + connect.decodeKey( + Some(Encoding.UTF8_FILE), + s"my-key", + "secret", { _ => + written = true + fileName + } + ) shouldBe fileName + written shouldBe true + } + + "min list test" in { + val now = OffsetDateTime.now() + val secrets = Map( + "ke3" -> ("value", Some(OffsetDateTime.now().plusHours(3))), + "key1" -> ("value", Some(now)), + "key2" -> ("value", Some(OffsetDateTime.now().plusHours(1))) + ) + + val (expiry, _) = connect.getSecretsAndExpiry(secrets) + expiry shouldBe Some(now) + } +} diff --git a/src/test/scala/io/lenses/connect/secrets/providers/ENVSecretProviderTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/ENVSecretProviderTest.scala index 34d2168..9a7dbab 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/ENVSecretProviderTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/ENVSecretProviderTest.scala @@ -1,83 +1,84 @@ -/* - * - * * Copyright 2017-2020 Lenses.io Ltd - * - */ - -package io.lenses.connect.secrets.providers - -import java.nio.file.FileSystems -import java.util.Base64 - -import org.apache.kafka.common.config.ConfigTransformer -import org.apache.kafka.common.config.provider.ConfigProvider -import org.scalatest.matchers.should.Matchers -import org.scalatest.wordspec.AnyWordSpec - -import scala.collection.JavaConverters._ -import scala.io.Source - -class ENVSecretProviderTest extends AnyWordSpec with Matchers { - - val separator: String = FileSystems.getDefault.getSeparator - val tmp: String = - System.getProperty("java.io.tmpdir").stripSuffix(separator) + separator + "provider-tests-env" - - "should filter and match" in { - val provider = new ENVSecretProvider() - provider.vars = Map( - "RANDOM" -> "somevalue", - "CONNECT_CASSANDRA_PASSWORD" -> "secret", - "BASE64" -> s"ENV-base64:${Base64.getEncoder.encodeToString("my-base64-secret".getBytes)}", - "BASE64_FILE" -> s"ENV-mounted-base64:${Base64.getEncoder.encodeToString("my-base64-secret".getBytes)}", - "UTF8_FILE" -> s"ENV-mounted:my-secret" - ) - provider.fileDir = tmp - - val data = provider.get("", Set("CONNECT_CASSANDRA_PASSWORD").asJava) - data.data().get("CONNECT_CASSANDRA_PASSWORD") shouldBe "secret" - data.data().containsKey("RANDOM") shouldBe false - - val data2 = provider.get("", Set("CONNECT_CASSANDRA_PASSWORD", "RANDOM").asJava) - data2.data().get("CONNECT_CASSANDRA_PASSWORD") shouldBe "secret" - data2.data().containsKey("RANDOM") shouldBe true - - val data3 = provider.get("", Set("BASE64").asJava) - data3.data().get("BASE64") shouldBe "my-base64-secret" - - val data4 = provider.get("", Set("BASE64_FILE").asJava) - val outputFile = data4.data().get("BASE64_FILE") - outputFile shouldBe s"$tmp${separator}base64_file" - - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe "my-base64-secret" - result.close() - - val data5 = provider.get("", Set("UTF8_FILE").asJava) - val outputFile5 = data5.data().get("UTF8_FILE") - outputFile5 shouldBe s"$tmp${separator}utf8_file" - - val result2 = Source.fromFile(outputFile5) - result2.getLines().mkString shouldBe "my-secret" - result2.close() - - } - - "check transformer" in { - - val props = Map.empty[String, String].asJava - - val provider = new ENVSecretProvider() - provider.vars = Map("CONNECT_PASSWORD" -> "secret") - - // check the workerconfigprovider - val map = new java.util.HashMap[String, ConfigProvider]() - map.put("env", provider) - val transformer = new ConfigTransformer(map) - val props2 = - Map("mykey" -> "${env::CONNECT_PASSWORD}").asJava - val data = transformer.transform(props2) - data.data().containsKey("value") - data.data().get("mykey") shouldBe "secret" - } -} +/* + * + * * Copyright 2017-2020 Lenses.io Ltd + * + */ + +package io.lenses.connect.secrets.providers + +import org.apache.kafka.common.config.ConfigTransformer +import org.apache.kafka.common.config.provider.ConfigProvider +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec + +import java.nio.file.FileSystems +import java.util.Base64 +import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.{Success, Using} + +class ENVSecretProviderTest extends AnyWordSpec with Matchers { + + val separator: String = FileSystems.getDefault.getSeparator + val tmp: String = + System + .getProperty("java.io.tmpdir") + .stripSuffix(separator) + separator + "provider-tests-env" + + "should filter and match" in { + val provider = new ENVSecretProvider() + provider.vars = Map( + "RANDOM" -> "somevalue", + "CONNECT_CASSANDRA_PASSWORD" -> "secret", + "BASE64" -> s"ENV-base64:${Base64.getEncoder.encodeToString("my-base64-secret".getBytes)}", + "BASE64_FILE" -> s"ENV-mounted-base64:${Base64.getEncoder.encodeToString("my-base64-secret".getBytes)}", + "UTF8_FILE" -> s"ENV-mounted:my-secret" + ) + provider.fileDir = tmp + + val data = provider.get("", Set("CONNECT_CASSANDRA_PASSWORD").asJava) + data.data().get("CONNECT_CASSANDRA_PASSWORD") shouldBe "secret" + data.data().containsKey("RANDOM") shouldBe false + + val data2 = + provider.get("", Set("CONNECT_CASSANDRA_PASSWORD", "RANDOM").asJava) + data2.data().get("CONNECT_CASSANDRA_PASSWORD") shouldBe "secret" + data2.data().containsKey("RANDOM") shouldBe true + + val data3 = provider.get("", Set("BASE64").asJava) + data3.data().get("BASE64") shouldBe "my-base64-secret" + + val data4 = provider.get("", Set("BASE64_FILE").asJava) + val outputFile = data4.data().get("BASE64_FILE") + outputFile shouldBe s"$tmp${separator}base64_file" + + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + "my-base64-secret" + ) + + val data5 = provider.get("", Set("UTF8_FILE").asJava) + val outputFile5 = data5.data().get("UTF8_FILE") + outputFile5 shouldBe s"$tmp${separator}utf8_file" + + Using(Source.fromFile(outputFile5))(_.getLines().mkString) shouldBe Success( + "my-secret" + ) + + } + + "check transformer" in { + + val provider = new ENVSecretProvider() + provider.vars = Map("CONNECT_PASSWORD" -> "secret") + + // check the workerconfigprovider + val map = new java.util.HashMap[String, ConfigProvider]() + map.put("env", provider) + val transformer = new ConfigTransformer(map) + val props2 = + Map("mykey" -> "${env::CONNECT_PASSWORD}").asJava + val data = transformer.transform(props2) + data.data().containsKey("value") + data.data().get("mykey") shouldBe "secret" + } +} diff --git a/src/test/scala/io/lenses/connect/secrets/providers/VaultSecretProviderTest.scala b/src/test/scala/io/lenses/connect/secrets/providers/VaultSecretProviderTest.scala index bda5472..a1d82c7 100644 --- a/src/test/scala/io/lenses/connect/secrets/providers/VaultSecretProviderTest.scala +++ b/src/test/scala/io/lenses/connect/secrets/providers/VaultSecretProviderTest.scala @@ -6,36 +6,34 @@ package io.lenses.connect.secrets.providers -import java.io.File -import java.nio.file.FileSystems -import java.util.Base64 - -import com.bettercloud.vault.json.JsonArray -import com.bettercloud.vault.json.JsonObject -import io.lenses.connect.secrets.config.VaultAuthMethod -import io.lenses.connect.secrets.config.VaultProviderConfig -import io.lenses.connect.secrets.config.VaultSettings +import com.bettercloud.vault.json.{JsonArray, JsonObject} +import io.lenses.connect.secrets.TmpDirUtil.{getTempDir, separator} +import io.lenses.connect.secrets.config.{ + VaultAuthMethod, + VaultProviderConfig, + VaultSettings +} import io.lenses.connect.secrets.connect -import io.lenses.connect.secrets.vault.MockVault -import io.lenses.connect.secrets.vault.VaultTestUtils -import org.apache.kafka.common.config.ConfigData -import org.apache.kafka.common.config.ConfigTransformer +import io.lenses.connect.secrets.vault.{MockVault, VaultTestUtils} import org.apache.kafka.common.config.provider.ConfigProvider +import org.apache.kafka.common.config.{ConfigData, ConfigTransformer} import org.eclipse.jetty.server.Server import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec -import scala.collection.JavaConverters._ +import java.io.File +import java.util.Base64 import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.{Success, Using} class VaultSecretProviderTest - extends AnyWordSpec + extends AnyWordSpec with Matchers with BeforeAndAfterAll { - val separator: String = FileSystems.getDefault.getSeparator - val tmp: String = - System.getProperty("java.io.tmpdir") + separator + "provider-tests-vault" + + val tmp: String = s"$getTempDir${separator}provider-tests-vault" val data: JsonObject = new JsonObject().add( "data", @@ -57,7 +55,9 @@ class VaultSecretProviderTest .add("lease_duration", 0) .add("policies", new JsonArray()) - val root: JsonObject = new JsonObject().add("data", data).add("auth", auth) + val root: JsonObject = new JsonObject() + .add("data", data) + .add("auth", auth) .add("renewable", true) val mockVault = new MockVault(200, root.toString) val server: Server = VaultTestUtils.initHttpsMockVault(mockVault) @@ -333,9 +333,9 @@ class VaultSecretProviderTest .get(secretKey) outputFile shouldBe s"$tmp$separator$secretPath$separator$secretKey" - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe secretValue - result.close() + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + secretValue + ) provider.close() } @@ -363,9 +363,9 @@ class VaultSecretProviderTest .get(secretKey) outputFile shouldBe s"$tmp$separator$secretPath$separator$secretKey" - val result = Source.fromFile(outputFile) - result.getLines().mkString shouldBe secretValue - result.close() + Using(Source.fromFile(outputFile))(_.getLines().mkString) shouldBe Success( + secretValue + ) provider.close() }