Skip to content

Commit

Permalink
fix: update config for new new doc store format (#186)
Browse files Browse the repository at this point in the history
  • Loading branch information
aaron-steinfeld authored Sep 19, 2023
1 parent d6f6213 commit 23dab85
Show file tree
Hide file tree
Showing 18 changed files with 127 additions and 49 deletions.
2 changes: 1 addition & 1 deletion attribute-projection-registry/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ io.grpc:grpc-protobuf:1.57.2=compileClasspath,runtimeClasspath
io.grpc:grpc-stub:1.57.2=compileClasspath,runtimeClasspath
javax.annotation:javax.annotation-api:1.3.2=runtimeClasspath
org.checkerframework:checker-qual:3.33.0=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
empty=annotationProcessor
2 changes: 1 addition & 1 deletion attribute-service-api/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,6 @@ io.grpc:grpc-protobuf:1.57.2=compileClasspath,runtimeClasspath
io.grpc:grpc-stub:1.57.2=compileClasspath,runtimeClasspath
javax.annotation:javax.annotation-api:1.3.2=compileClasspath,runtimeClasspath
org.checkerframework:checker-qual:3.33.0=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
empty=annotationProcessor
2 changes: 1 addition & 1 deletion attribute-service-client/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ io.perfmark:perfmark-api:0.26.0=runtimeClasspath
javax.annotation:javax.annotation-api:1.3.2=runtimeClasspath
org.checkerframework:checker-qual:3.33.0=compileClasspath,runtimeClasspath
org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-client-utils:0.12.5=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5=runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
Expand Down
3 changes: 2 additions & 1 deletion attribute-service-factory/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,13 @@ org.eclipse.jetty:jetty-servlet:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-util-ajax:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-util:9.4.52.v20230823=runtimeClasspath
org.hdrhistogram:HdrHistogram:2.1.12=runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.documentstore:document-store:0.7.37=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-client-utils:0.12.5=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5=runtimeClasspath
org.hypertrace.core.grpcutils:grpc-server-utils:0.12.2=runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
org.hypertrace.core.serviceframework:docstore-metrics:0.1.60=runtimeClasspath
org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.60=compileClasspath,runtimeClasspath
org.hypertrace.core.serviceframework:platform-metrics:0.1.60=runtimeClasspath
org.hypertrace.core.serviceframework:platform-service-framework:0.1.60=compileClasspath,runtimeClasspath
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ public class AttributeServiceFactory implements GrpcPlatformServiceFactory {
@Override
public List<GrpcPlatformService> buildServices(GrpcServiceContainerEnvironment environment) {
return List.of(
new GrpcPlatformService(new AttributeServiceImpl(environment.getConfig(SERVICE_NAME))));
new GrpcPlatformService(
new AttributeServiceImpl(
environment.getConfig(SERVICE_NAME), environment.getLifecycle())));
}
}
2 changes: 2 additions & 0 deletions attribute-service-impl/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ plugins {

dependencies {
api(projects.attributeServiceApi)
api(commonLibs.hypertrace.framework.spi)
implementation(projects.attributeServiceTenantApi)
implementation(commonLibs.hypertrace.documentstore)
implementation(commonLibs.hypertrace.grpcutils.context)
Expand All @@ -14,6 +15,7 @@ dependencies {
implementation(commonLibs.slf4j2.api)
implementation(commonLibs.protobuf.javautil)
implementation(commonLibs.guava)
implementation(commonLibs.hypertrace.framework.documentstore.metrics)

testImplementation(commonLibs.mockito.core)
testImplementation(commonLibs.junit.jupiter)
Expand Down
35 changes: 34 additions & 1 deletion attribute-service-impl/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,26 +22,59 @@ com.google.protobuf:protobuf-java-util:3.24.1=compileClasspath,runtimeClasspath
com.google.protobuf:protobuf-java:3.24.1=compileClasspath,runtimeClasspath
com.typesafe:config:1.4.2=compileClasspath,runtimeClasspath
commons-logging:commons-logging:1.2=runtimeClasspath
io.dropwizard.metrics:metrics-core:4.2.16=runtimeClasspath
io.dropwizard.metrics:metrics-jvm:4.2.16=runtimeClasspath
io.github.mweirauch:micrometer-jvm-extras:0.2.2=runtimeClasspath
io.grpc:grpc-api:1.57.2=compileClasspath,runtimeClasspath
io.grpc:grpc-bom:1.57.2=compileClasspath,runtimeClasspath
io.grpc:grpc-context:1.57.2=runtimeClasspath
io.grpc:grpc-core:1.57.2=runtimeClasspath
io.grpc:grpc-protobuf-lite:1.57.2=compileClasspath,runtimeClasspath
io.grpc:grpc-protobuf:1.57.2=compileClasspath,runtimeClasspath
io.grpc:grpc-stub:1.57.2=compileClasspath,runtimeClasspath
io.micrometer:micrometer-commons:1.10.2=runtimeClasspath
io.micrometer:micrometer-core:1.10.2=runtimeClasspath
io.micrometer:micrometer-observation:1.10.2=runtimeClasspath
io.micrometer:micrometer-registry-prometheus:1.10.2=runtimeClasspath
io.perfmark:perfmark-api:0.26.0=runtimeClasspath
io.prometheus:simpleclient:0.16.0=runtimeClasspath
io.prometheus:simpleclient_common:0.16.0=runtimeClasspath
io.prometheus:simpleclient_dropwizard:0.12.0=runtimeClasspath
io.prometheus:simpleclient_pushgateway:0.12.0=runtimeClasspath
io.prometheus:simpleclient_servlet:0.12.0=runtimeClasspath
io.prometheus:simpleclient_servlet_common:0.12.0=runtimeClasspath
io.prometheus:simpleclient_tracer_common:0.16.0=runtimeClasspath
io.prometheus:simpleclient_tracer_otel:0.16.0=runtimeClasspath
io.prometheus:simpleclient_tracer_otel_agent:0.16.0=runtimeClasspath
javax.annotation:javax.annotation-api:1.3.2=runtimeClasspath
javax.servlet:javax.servlet-api:3.1.0=runtimeClasspath
javax.xml.bind:jaxb-api:2.3.0=runtimeClasspath
net.jodah:failsafe:2.4.4=runtimeClasspath
org.apache.commons:commons-collections4:4.4=runtimeClasspath
org.apache.commons:commons-dbcp2:2.9.0=runtimeClasspath
org.apache.commons:commons-lang3:3.12.0=runtimeClasspath
org.apache.commons:commons-pool2:2.10.0=runtimeClasspath
org.apache.logging.log4j:log4j-api:2.19.0=runtimeClasspath
org.apache.logging.log4j:log4j-core:2.19.0=runtimeClasspath
org.apache.logging.log4j:log4j-slf4j-impl:2.19.0=runtimeClasspath
org.checkerframework:checker-qual:3.33.0=compileClasspath,runtimeClasspath
org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.eclipse.jetty:jetty-http:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-io:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-security:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-server:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-servlet:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-util-ajax:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-util:9.4.52.v20230823=runtimeClasspath
org.hdrhistogram:HdrHistogram:2.1.12=runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.documentstore:document-store:0.7.37=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5=compileClasspath,runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
org.hypertrace.core.serviceframework:docstore-metrics:0.1.60=compileClasspath,runtimeClasspath
org.hypertrace.core.serviceframework:platform-metrics:0.1.60=runtimeClasspath
org.hypertrace.core.serviceframework:service-framework-spi:0.1.60=compileClasspath,runtimeClasspath
org.latencyutils:LatencyUtils:2.0.3=runtimeClasspath
org.mongodb:bson-record-codec:4.9.0=runtimeClasspath
org.mongodb:bson:4.9.0=runtimeClasspath
org.mongodb:mongodb-driver-core:4.9.0=runtimeClasspath
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,11 @@
import org.hypertrace.core.documentstore.JSONDocument;
import org.hypertrace.core.documentstore.Key;
import org.hypertrace.core.documentstore.Query;
import org.hypertrace.core.documentstore.model.config.DatastoreConfig;
import org.hypertrace.core.documentstore.model.config.TypesafeConfigDatastoreConfigExtractor;
import org.hypertrace.core.grpcutils.context.RequestContext;
import org.hypertrace.core.serviceframework.docstore.metrics.DocStoreMetricsRegistry;
import org.hypertrace.core.serviceframework.spi.PlatformServiceLifecycle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -84,8 +88,8 @@ private static String sourceMetadataPathFor(final AttributeSource source) {
* Initiates with a configuration. The configuration should be production configuration, but for
* V0 The attributes type data would be stored in the configuration
*/
public AttributeServiceImpl(Config config) {
Datastore store = initDataStore(config);
public AttributeServiceImpl(Config config, PlatformServiceLifecycle platformServiceLifecycle) {
Datastore store = initDataStore(config, platformServiceLifecycle);
this.collection = store.getCollection(ATTRIBUTE_METADATA_COLLECTION);
this.validator = new AttributeMetadataValidator(config);
this.converter = new AttributeMetadataConverterImpl();
Expand All @@ -99,11 +103,29 @@ public AttributeServiceImpl(Config config) {
this.updater = new AttributeUpdaterImpl(collection);
}

private Datastore initDataStore(Config config) {
Config docStoreConfig = config.getConfig(DOC_STORE_CONFIG_KEY);
String dataStoreType = docStoreConfig.getString(DATA_STORE_TYPE);
Config dataStoreConfig = docStoreConfig.getConfig(dataStoreType);
return DatastoreProvider.getDatastore(dataStoreType, dataStoreConfig);
private Datastore initDataStore(
Config config, PlatformServiceLifecycle platformServiceLifecycle) {
final Config docStoreConfig = config.getConfig(DOC_STORE_CONFIG_KEY);
final String dataStoreType = docStoreConfig.getString(DATA_STORE_TYPE);
final DatastoreConfig datastoreConfig =
TypesafeConfigDatastoreConfigExtractor.from(docStoreConfig, DATA_STORE_TYPE)
.hostKey(dataStoreType + ".host")
.portKey(dataStoreType + ".port")
.keysForEndpoints(dataStoreType + ".endpoints", "host", "port")
.authDatabaseKey(dataStoreType + ".authDb")
.replicaSetKey(dataStoreType + ".replicaSet")
.databaseKey(dataStoreType + ".database")
.usernameKey(dataStoreType + ".user")
.passwordKey(dataStoreType + ".password")
.applicationNameKey("appName")
.extract();

final Datastore datastore = DatastoreProvider.getDatastore(datastoreConfig);
new DocStoreMetricsRegistry(datastore)
.withPlatformLifecycle(platformServiceLifecycle)
.monitor();

return datastore;
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion attribute-service-tenant-api/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
# This file is expected to be part of source control.
com.fasterxml.jackson:jackson-bom:2.15.2=compileClasspath,runtimeClasspath
io.grpc:grpc-bom:1.57.2=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
empty=annotationProcessor
5 changes: 4 additions & 1 deletion attribute-service/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ plugins {
alias(commonLibs.plugins.hypertrace.jacoco)
}

val mongoPort = "37017"

tasks.register<DockerCreateNetwork>("createIntegrationTestNetwork") {
networkName.set("attr-svc-int-test")
}
Expand All @@ -33,7 +35,7 @@ tasks.register<DockerCreateContainer>("createMongoContainer") {
targetImageId(tasks.getByName<DockerPullImage>("pullMongoImage").image)
containerName.set("mongo-local")
hostConfig.network.set(tasks.getByName<DockerCreateNetwork>("createIntegrationTestNetwork").networkId)
hostConfig.portBindings.set(listOf("27017:27017"))
hostConfig.portBindings.set(listOf("$mongoPort:27017"))
hostConfig.autoRemove.set(true)
}

Expand All @@ -50,6 +52,7 @@ tasks.register<DockerStopContainer>("stopMongoContainer") {
tasks.integrationTest {
useJUnitPlatform()
dependsOn("startMongoContainer")
environment("MONGO_PORT", mongoPort)
finalizedBy("stopMongoContainer")
}

Expand Down
3 changes: 2 additions & 1 deletion attribute-service/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,13 @@ org.eclipse.jetty:jetty-servlet:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-util-ajax:9.4.52.v20230823=runtimeClasspath
org.eclipse.jetty:jetty-util:9.4.52.v20230823=runtimeClasspath
org.hdrhistogram:HdrHistogram:2.1.12=runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.documentstore:document-store:0.7.37=runtimeClasspath
org.hypertrace.core.grpcutils:grpc-client-utils:0.12.5=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5=runtimeClasspath
org.hypertrace.core.grpcutils:grpc-server-utils:0.12.2=runtimeClasspath
org.hypertrace.core.kafkastreams.framework:kafka-bom:0.3.2=compileClasspath,runtimeClasspath
org.hypertrace.core.serviceframework:docstore-metrics:0.1.60=runtimeClasspath
org.hypertrace.core.serviceframework:platform-grpc-service-framework:0.1.60=compileClasspath,runtimeClasspath
org.hypertrace.core.serviceframework:platform-metrics:0.1.60=runtimeClasspath
org.hypertrace.core.serviceframework:platform-service-framework:0.1.60=compileClasspath,runtimeClasspath
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +0,0 @@
main.class = org.hypertrace.core.attribute.service.AttributeServiceEntry
service.name = attribute-service
service.admin.port = 9013
document.store {
dataStoreType = mongo
mongo {
host = localhost
host = ${?MONGO_HOST} # provides a way to override the mongo_host via an environment variable
port = 27017
}
}
attributes.type.server.port = 9012
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,19 @@ service.name = attribute-service
service.admin.port = 9013
document.store {
dataStoreType = mongo
appName = attribute-service-local
mongo {
host = localhost
host = ${?MONGO_HOST} # provides a way to override the mongo_host via an environment variable
port = 27017
port = ${?MONGO_PORT}
database = default_db
user = ${?MONGO_SERVICE_USERNAME}
password = ${?MONGO_SERVICE_USER_PASSWORD}
endpoints = [
{
host = localhost
host = ${?MONGO_HOST} # provides a way to override the mongo_host via an environment variable
port = 27017
port = ${?MONGO_PORT}
}
]
}
}
attributes.type.server.port = 9012
Expand Down
2 changes: 1 addition & 1 deletion caching-attribute-service-client/gradle.lockfile
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ io.reactivex.rxjava3:rxjava:3.1.7=compileClasspath,runtimeClasspath
javax.annotation:javax.annotation-api:1.3.2=runtimeClasspath
org.checkerframework:checker-qual:3.33.0=compileClasspath,runtimeClasspath
org.codehaus.mojo:animal-sniffer-annotations:1.23=runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.3=compileClasspath,runtimeClasspath
org.hypertrace.bom:hypertrace-bom:0.2.4=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-client-rx-utils:0.12.5=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-client-utils:0.12.5=compileClasspath,runtimeClasspath
org.hypertrace.core.grpcutils:grpc-context-utils:0.12.5=compileClasspath,runtimeClasspath
Expand Down
7 changes: 6 additions & 1 deletion helm/templates/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,18 @@ data:
document.store {
{{- $dst := .Values.configMap.dataStoreType }}
dataStoreType = {{ $dst }}
appName = {{ .Values.service.name }}
{{ $dst }} {
{{- range $key, $value := (index .Values "configMap" (printf "%s" $dst)) }}
{{- if $value }}
{{- if eq $key "password" }}
{{- if hasPrefix "${?" (printf "%s" $value) }}
{{ $key }} = {{ $value }}
{{- else }}
{{- if eq $key "endpoints" }}
endpoints = {{- toJson $value }}
{{- else }}
{{ $key }} = {{ $value | quote }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
Expand Down
16 changes: 12 additions & 4 deletions helm/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,22 @@ spec:
containerPort: {{ .Values.containerHealthProbePort }}
protocol: TCP
env:
- name: SERVICE_NAME
value: "{{ .Chart.Name }}"
- name: BOOTSTRAP_CONFIG_URI
value: "file:///app/resources/configs"
- name: LOG4J_CONFIGURATION_FILE
value: "/var/{{ .Chart.Name }}/log/log4j2.properties"
- name: JAVA_OPTS
value: {{ .Values.javaOpts | quote }}
{{- with .Values.configMap.mongo.credentials }}
- name: MONGO_SERVICE_USERNAME
valueFrom:
secretKeyRef:
name: {{ .secretName }}
key: {{ .secretUsernameKey }}
- name: MONGO_SERVICE_USER_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .secretName }}
key: {{ .secretPasswordKey }}
{{- end }}
volumeMounts:
- name: service-config
mountPath: /app/resources/configs/{{ .Chart.Name }}/application.conf
Expand Down
25 changes: 15 additions & 10 deletions helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ image:
pullPolicy: IfNotPresent
tagOverride: ""

imagePullSecrets: {}
imagePullSecrets: { }

containerPort: 9012
containerHealthProbePort: 9013
Expand All @@ -21,15 +21,15 @@ service:
type: ClusterIP
port: 9012

nodeLabels: {}
nodeLabels: { }

tolerations: []
tolerations: [ ]

affinity: {}
affinity: { }

topologySpreadConstraints: []
topologySpreadConstraints: [ ]

securityContext: {}
securityContext: { }

javaOpts: "-XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0 -XX:MaxDirectMemorySize=128M -XX:+ExitOnOutOfMemoryError"

Expand All @@ -56,7 +56,7 @@ deploymentLabels:
podLabels:
app: attribute-service

podAnnotations: {}
podAnnotations: { }

deploymentSelectorMatchLabels:
app: attribute-service
Expand All @@ -68,8 +68,13 @@ configMap:
name: attribute-service-config
dataStoreType: "mongo"
mongo:
host: mongo
url: ""
endpoints:
- host: mongo
port: 27017
# credentials:
# secretName: <k8s secret name>
# secretUsernameKey: <secret key for username>
# secretPasswordKey: <secret key for password>
postgres:
host: postgres
port: 5432
Expand All @@ -88,7 +93,7 @@ logConfig:
config-bootstrapper:
job:
prefix: attribute
configurationCommands: {}
configurationCommands: { }

hpa:
enabled: false
Expand Down
2 changes: 1 addition & 1 deletion settings-gradle.lockfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This is a Gradle generated file for dependency locking.
# Manual edits can break the build and are not advised.
# This file is expected to be part of source control.
org.hypertrace.bom:hypertrace-version-catalog:0.2.3=incomingCatalogForCommonLibs0
org.hypertrace.bom:hypertrace-version-catalog:0.2.4=incomingCatalogForCommonLibs0
empty=

0 comments on commit 23dab85

Please sign in to comment.