Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue 1712 add status information #1747

Merged
merged 34 commits into from
Oct 4, 2023
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
fb5d33b
Update to store status information in the cache.
aindriu-aiven Sep 5, 2023
02c3f5d
Updated caching mechanism to reduce load on recreating all caches fro…
aindriu-aiven Sep 8, 2023
813e75d
Updated caching mechanism to reduce load on recreating all caches fro…
aindriu-aiven Sep 8, 2023
fae81be
Update to add new api endpoints for synchronizing the caches.
aindriu-aiven Sep 8, 2023
315667a
merge from main
aindriu-aiven Sep 8, 2023
506283a
Update minor consistency changes
aindriu-aiven Sep 8, 2023
38b94a9
format typescript
aindriu-aiven Sep 8, 2023
a1f1ec8
Merge branch 'main' of github.com:Aiven-Open/klaw into issue-1712-add…
aindriu-aiven Sep 11, 2023
6aa1f80
Updates from PR
aindriu-aiven Sep 11, 2023
6fb58bd
Addition of secure https communication
aindriu-aiven Sep 11, 2023
77d863c
Addition of secure https communication
aindriu-aiven Sep 11, 2023
05ced44
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Sep 12, 2023
140175c
Address PR Comments
aindriu-aiven Sep 12, 2023
fd57af8
Merge branch 'issue-1712-add-status-information' of github.com:Aiven-…
aindriu-aiven Sep 12, 2023
e03e2de
Updates to add app2app authentication and address a few comments from…
aindriu-aiven Sep 21, 2023
ef3f755
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Sep 21, 2023
f69daaf
Updates to coral to use envStatus
aindriu-aiven Sep 21, 2023
7e95843
fix typo
aindriu-aiven Sep 21, 2023
8306d37
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Sep 22, 2023
8c642b9
Updte formatting to resolve failing tests.
aindriu-aiven Sep 22, 2023
c54669f
Merge branch 'issue-1712-add-status-information' of github.com:Aiven-…
aindriu-aiven Sep 22, 2023
4d259bf
Updates to only exposes HA endpoints when HA is enabled
aindriu-aiven Sep 22, 2023
e3e980d
Add the APIs to coral
aindriu-aiven Sep 22, 2023
4c7091a
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Sep 22, 2023
ea8a2b3
change logging to debug
aindriu-aiven Sep 22, 2023
1e4bb84
Merge branch 'issue-1712-add-status-information' of github.com:Aiven-…
aindriu-aiven Sep 22, 2023
ef06588
fix formatting issue
aindriu-aiven Sep 22, 2023
4bbd37a
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Sep 26, 2023
b084f90
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Sep 26, 2023
65637a7
Update to make changes from PR feedback also updates to include the t…
aindriu-aiven Sep 26, 2023
1455587
Merge branch 'issue-1712-add-status-information' of github.com:Aiven-…
aindriu-aiven Sep 26, 2023
1a9a5c2
Update to alter the envStatusTimeString
aindriu-aiven Sep 26, 2023
2fb204a
Update to correct strings in coral tests
aindriu-aiven Sep 27, 2023
694e87f
Merge branch 'main' into issue-1712-add-status-information
aindriu-aiven Oct 4, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ describe("environment-test-helper.ts", () => {
defaultPartitions: undefined,
defaultReplicationFactor: undefined,
envStatus: "ONLINE",
envStatusTime: "2023-09-08T12:34:10.615919098",
id: "1",
maxPartitions: undefined,
maxReplicationFactor: undefined,
Expand Down
1 change: 1 addition & 0 deletions coral/src/domain/environment/environment-test-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const defaultEnvironmentDTO: KlawApiModel<"EnvModelResponse"> = {
tenantName: "default",
clusterName: "DEV",
envStatus: "ONLINE",
envStatusTime: "2023-09-08T12:34:10.615919098",
otherParams:
"default.partitions=2,max.partitions=2,default.replication.factor=1,max.replication.factor=1,topic.prefix=,topic.suffix=",
showDeleteEnv: false,
Expand Down
20 changes: 19 additions & 1 deletion coral/src/services/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ const API_PATHS = {
| "getKafkaEnv"
| "getKafkaConnectEnv"
| "getSchemaRegEnv"
| "addEnvToCache"
| "removeEnvFromCache"
>]: keyof ApiPaths;
};

Expand All @@ -219,6 +221,14 @@ type GetTopicRequest = (params: { topicReqId: string }) => keyof ApiPaths;
type GetKafkaEnv = (params: { envId: string }) => keyof ApiPaths;
type GetConnectEnv = (params: { envId: string }) => keyof ApiPaths;
type GetSchemaRegEnv = (params: { envId: string }) => keyof ApiPaths;
type AddEnvToCache = (params: {
tenantId: string;
id: string;
}) => keyof ApiPaths;
type RemoveEnvFromCache = (params: {
tenantId: string;
id: string;
}) => keyof ApiPaths;

const DYNAMIC_API_PATHS = {
getSchemaOfTopicFromSource: ({
Expand All @@ -238,6 +248,10 @@ const DYNAMIC_API_PATHS = {
`/environments/kafkaconnect/${envId}` as keyof ApiPaths,
getSchemaRegEnv: ({ envId }: Parameters<GetSchemaRegEnv>[0]) =>
`/environments/schemaRegistry/${envId}` as keyof ApiPaths,
addEnvToCache: ({ tenantId, id }: Parameters<AddEnvToCache>[0]) =>
`/environment/tenant/${tenantId}/id/${id}` as keyof ApiPaths,
removeEnvFromCache: ({ tenantId, id }: Parameters<RemoveEnvFromCache>[0]) =>
`/environment/tenant/${tenantId}/id/${id}` as keyof ApiPaths,
} satisfies {
[key in keyof Pick<
ApiOperations,
Expand All @@ -247,13 +261,17 @@ const DYNAMIC_API_PATHS = {
| "getKafkaEnv"
| "getKafkaConnectEnv"
| "getSchemaRegEnv"
| "addEnvToCache"
| "removeEnvFromCache"
>]:
| GetSchemaOfTopicFromSource
| GetSwitchTeams
| GetTopicRequest
| GetKafkaEnv
| GetConnectEnv
| GetSchemaRegEnv;
| GetSchemaRegEnv
| AddEnvToCache
| RemoveEnvFromCache;
};

type Params = URLSearchParams;
Expand Down
99 changes: 81 additions & 18 deletions coral/types/api.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,10 @@ export type paths = {
"/chPwd": {
post: operations["changePwd"];
};
"/cache/environment/tenant/{tenantId}/id/{id}": {
post: operations["addEnvToCache"];
delete: operations["removeEnvFromCache"];
};
"/addTenantId": {
post: operations["addTenantId"];
};
Expand Down Expand Up @@ -872,6 +876,40 @@ export type components = {
envId: string;
includeOnlyFailedTasks: boolean;
};
Env: {
id?: string;
/** Format: int32 */
tenantId?: number;
name?: string;
stretchCode?: string;
/** Format: int32 */
clusterId?: number;
type?: string;
otherParams?: string;
envExists?: string;
/** @enum {string} */
envStatus?: "OFFLINE" | "ONLINE" | "NOT_KNOWN";
/** Format: date-time */
envStatusTime?: string;
associatedEnv?: components["schemas"]["EnvTag"];
params?: components["schemas"]["EnvParams"];
};
EnvParams: {
defaultPartitions?: string;
maxPartitions?: string;
partitionsList?: (string)[];
defaultRepFactor?: string;
maxRepFactor?: string;
replicationFactorList?: (string)[];
topicPrefix?: (string)[];
topicSuffix?: (string)[];
topicRegex?: (string)[];
applyRegex?: boolean;
};
EnvTag: {
id?: string;
name?: string;
};
KwTenantModel: {
tenantName: string;
tenantDesc: string;
Expand Down Expand Up @@ -899,22 +937,6 @@ export type components = {
tenantId?: number;
params?: components["schemas"]["EnvParams"];
};
EnvParams: {
defaultPartitions?: string;
maxPartitions?: string;
partitionsList?: (string)[];
defaultRepFactor?: string;
maxRepFactor?: string;
replicationFactorList?: (string)[];
topicPrefix?: (string)[];
topicSuffix?: (string)[];
topicRegex?: (string)[];
applyRegex?: boolean;
};
EnvTag: {
id?: string;
name?: string;
};
KwClustersModel: {
/** Format: int32 */
clusterId?: number;
Expand Down Expand Up @@ -1114,6 +1136,8 @@ export type components = {
result: string;
/** @enum {string} */
envStatus: "OFFLINE" | "ONLINE" | "NOT_KNOWN";
/** Format: date-time */
envStatusTime: string;
};
TopicsCountPerEnv: {
status?: string;
Expand Down Expand Up @@ -1207,8 +1231,8 @@ export type components = {
hasSchema: boolean;
/** Format: int32 */
clusterId: number;
topicOwner?: boolean;
highestEnv?: boolean;
topicOwner?: boolean;
};
TopicBaseConfig: {
topicName: string;
Expand Down Expand Up @@ -1372,6 +1396,8 @@ export type components = {
clusterName: string;
/** @enum {string} */
envStatus: "OFFLINE" | "ONLINE" | "NOT_KNOWN";
/** Format: date-time */
envStatusTime: string;
otherParams: string;
showDeleteEnv: boolean;
totalNoPages: string;
Expand Down Expand Up @@ -2773,6 +2799,43 @@ export type operations = {
};
};
};
addEnvToCache: {
parameters: {
path: {
tenantId: number;
id: number;
};
};
requestBody: {
content: {
"application/json": components["schemas"]["Env"];
};
};
responses: {
/** @description OK */
200: {
content: {
"application/json": components["schemas"]["ApiResponse"];
};
};
};
};
removeEnvFromCache: {
parameters: {
path: {
tenantId: number;
id: number;
};
};
responses: {
/** @description OK */
200: {
content: {
"application/json": components["schemas"]["ApiResponse"];
};
};
};
};
addTenantId: {
requestBody: {
content: {
Expand Down Expand Up @@ -3668,7 +3731,7 @@ export type operations = {
getEnvParams: {
parameters: {
query: {
envSelected: string;
envSelected: number;
};
};
responses: {
Expand Down
31 changes: 31 additions & 0 deletions core/src/main/java/io/aiven/klaw/config/CacheConfig.java
Original file line number Diff line number Diff line change
@@ -1,19 +1,30 @@
package io.aiven.klaw.config;

import com.github.benmanes.caffeine.cache.Caffeine;
import io.aiven.klaw.dao.Env;
import io.aiven.klaw.service.CacheService;
import io.aiven.klaw.service.HighAvailabilityUtilsService;
import java.time.Duration;
import java.util.Arrays;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.concurrent.ConcurrentMapCache;
import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;

@Configuration
@EnableCaching
public class CacheConfig {

@Autowired private HighAvailabilityUtilsService highAvailabilityUtilsService;

@Value("${klaw.uiapi.servers:server1,server2}")
private String uiApiServers;

@Primary
public CacheManager cacheManager() {
SimpleCacheManager cacheManager = new SimpleCacheManager();
Expand All @@ -36,4 +47,24 @@ public CacheManager cacheManager() {
cacheManager.setCaches(Arrays.asList(userCache, teamsCache));
return cacheManager;
}

@Bean
public CacheService<Env> kafkaEnvListPerTenant() {
return new CacheService<Env>("environment", uiApiServers, highAvailabilityUtilsService);
aindriu-aiven marked this conversation as resolved.
Show resolved Hide resolved
}

@Bean
public CacheService<Env> schemaRegEnvListPerTenant() {
return new CacheService<Env>("environment", uiApiServers, highAvailabilityUtilsService);
aindriu-aiven marked this conversation as resolved.
Show resolved Hide resolved
}

@Bean
public CacheService<Env> kafkaConnectEnvListPerTenant() {
return new CacheService<Env>("environment", uiApiServers, highAvailabilityUtilsService);
}

@Bean
public CacheService<Env> allEnvListPerTenant() {
return new CacheService<Env>("environment", uiApiServers, highAvailabilityUtilsService);
}
}
Loading