From 953b79ebeaf067747cd3dc3d2a5dc2996f1417b4 Mon Sep 17 00:00:00 2001 From: Tommy Hughes Date: Tue, 17 Dec 2024 09:20:17 -0600 Subject: [PATCH 1/2] single deployment refactor Signed-off-by: Tommy Hughes --- .../api/v1alpha1/featurestore_types.go | 28 +- .../api/v1alpha1/zz_generated.deepcopy.go | 25 +- .../crd/bases/feast.dev_featurestores.yaml | 52 +-- ...v1alpha1_featurestore_pvc_persistence.yaml | 5 +- infra/feast-operator/dist/install.yaml | 52 +-- .../internal/controller/authz/authz.go | 27 +- .../featurestore_controller_db_store_test.go | 144 +++---- .../featurestore_controller_ephemeral_test.go | 171 +++----- ...restore_controller_kubernetes_auth_test.go | 195 +++------ .../featurestore_controller_loglevel_test.go | 61 +-- ...eaturestore_controller_objectstore_test.go | 158 ++----- .../featurestore_controller_oidc_auth_test.go | 162 +++---- .../featurestore_controller_pvc_test.go | 262 +++++------ .../featurestore_controller_test.go | 408 ++++++------------ .../featurestore_controller_tls_test.go | 172 +++----- .../internal/controller/services/client.go | 6 +- .../controller/services/repo_config.go | 181 +++++--- .../controller/services/repo_config_test.go | 243 +++-------- .../internal/controller/services/services.go | 365 +++++++++------- .../controller/services/services_types.go | 20 +- .../internal/controller/services/tls.go | 39 +- .../internal/controller/services/tls_test.go | 51 +-- .../internal/controller/services/util.go | 144 +++++-- 23 files changed, 1191 insertions(+), 1780 deletions(-) diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 2eb9ec8554..f73c7fc6a4 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -65,7 +65,7 @@ type FeatureStoreSpec struct { AuthzConfig *AuthzConfig `json:"authz,omitempty"` } -// FeatureStoreServices defines the desired feast service deployments. ephemeral registry is deployed by default. +// FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. type FeatureStoreServices struct { OfflineStore *OfflineStore `json:"offlineStore,omitempty"` OnlineStore *OnlineStore `json:"onlineStore,omitempty"` @@ -74,9 +74,9 @@ type FeatureStoreServices struct { // OfflineStore configures the deployed offline store service type OfflineStore struct { - StoreServiceConfigs `json:",inline"` - Persistence *OfflineStorePersistence `json:"persistence,omitempty"` - TLS *TlsConfigs `json:"tls,omitempty"` + ServiceConfigs `json:",inline"` + Persistence *OfflineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the offline store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -127,9 +127,9 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ // OnlineStore configures the deployed online store service type OnlineStore struct { - StoreServiceConfigs `json:",inline"` - Persistence *OnlineStorePersistence `json:"persistence,omitempty"` - TLS *TlsConfigs `json:"tls,omitempty"` + ServiceConfigs `json:",inline"` + Persistence *OnlineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the online store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -146,7 +146,7 @@ type OnlineStorePersistence struct { // OnlineStoreFilePersistence configures the file-based persistence for the offline store service // +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? self.path.startsWith('/') : true",message="Ephemeral stores must have absolute paths." // +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !self.path.startsWith('/') : true",message="PVC path must be a file name only, with no slashes." -// +kubebuilder:validation:XValidation:rule="has(self.path) && !self.path.startsWith('s3://') && !self.path.startsWith('gs://')",message="Online store does not support S3 or GS buckets." +// +kubebuilder:validation:XValidation:rule="has(self.path) ? !(self.path.startsWith('s3://') || self.path.startsWith('gs://')) : true",message="Online store does not support S3 or GS buckets." type OnlineStoreFilePersistence struct { Path string `json:"path,omitempty"` PvcConfig *PvcConfig `json:"pvc,omitempty"` @@ -235,11 +235,11 @@ type PvcConfig struct { Create *PvcCreate `json:"create,omitempty"` // MountPath within the container at which the volume should be mounted. // Must start by "/" and cannot contain ':'. - MountPath string `json:"mountPath,omitempty"` + MountPath string `json:"mountPath"` } // PvcCreate defines the immutable settings to create a new PVC mounted at the given path. -// The PVC name is the same as the associated deployment name. +// The PVC name is the same as the associated deployment & feast service name. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PvcCreate is immutable" type PvcCreate struct { // AccessModes k8s persistent volume access modes. Defaults to ["ReadWriteOnce"]. @@ -292,14 +292,6 @@ type DefaultConfigs struct { Image *string `json:"image,omitempty"` } -// StoreServiceConfigs k8s deployment settings -type StoreServiceConfigs struct { - // Replicas determines the number of pods for the feast service. - // When Replicas > 1, persistence is recommended. - Replicas *int32 `json:"replicas,omitempty"` - ServiceConfigs `json:",inline"` -} - // OptionalConfigs k8s container settings that are optional type OptionalConfigs struct { Env *[]corev1.EnvVar `json:"env,omitempty"` diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index 3241dff775..f1e0503088 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -273,7 +273,7 @@ func (in *LocalRegistryConfig) DeepCopy() *LocalRegistryConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { *out = *in - in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OfflineStorePersistence) @@ -376,7 +376,7 @@ func (in *OidcAuthz) DeepCopy() *OidcAuthz { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OnlineStore) DeepCopyInto(out *OnlineStore) { *out = *in - in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OnlineStorePersistence) @@ -721,27 +721,6 @@ func (in *ServiceHostnames) DeepCopy() *ServiceHostnames { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoreServiceConfigs) DeepCopyInto(out *StoreServiceConfigs) { - *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } - in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoreServiceConfigs. -func (in *StoreServiceConfigs) DeepCopy() *StoreServiceConfigs { - if in == nil { - return nil - } - out := new(StoreServiceConfigs) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index fd8861cef1..2cab2d8c5d 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -101,8 +101,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline store @@ -318,6 +318,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -375,12 +377,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -692,6 +688,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -711,8 +709,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -760,12 +758,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1082,6 +1074,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1369,8 +1363,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline @@ -1588,6 +1582,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1647,12 +1643,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1967,6 +1957,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1987,8 +1979,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -2038,12 +2030,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2368,6 +2354,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml index b7c7412c0f..15aa46c456 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml @@ -5,6 +5,7 @@ metadata: spec: feastProject: my_project services: + # demonstrates using a pre-existing PVC onlineStore: persistence: file: @@ -13,6 +14,7 @@ spec: ref: name: online-pvc mountPath: /data/online + # demonstrates specifying a storageClassName and storage size offlineStore: persistence: file: @@ -24,6 +26,7 @@ spec: requests: storage: 5Gi mountPath: /data/offline + # demonstrates letting the Operator create a PVC w/ defaults set registry: local: persistence: @@ -39,7 +42,7 @@ metadata: name: online-pvc spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index 435be789e5..cd63b3df8d 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -109,8 +109,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline store @@ -326,6 +326,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -383,12 +385,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -700,6 +696,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -719,8 +717,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -768,12 +766,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1090,6 +1082,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1377,8 +1371,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline @@ -1596,6 +1590,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1655,12 +1651,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1975,6 +1965,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1995,8 +1987,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -2046,12 +2038,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2376,6 +2362,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between diff --git a/infra/feast-operator/internal/controller/authz/authz.go b/infra/feast-operator/internal/controller/authz/authz.go index efcae23a4b..8596d99389 100644 --- a/infra/feast-operator/internal/controller/authz/authz.go +++ b/infra/feast-operator/internal/controller/authz/authz.go @@ -134,28 +134,11 @@ func (authz *FeastAuthorization) initFeastRoleBinding() *rbacv1.RoleBinding { func (authz *FeastAuthorization) setFeastRoleBinding(roleBinding *rbacv1.RoleBinding) error { roleBinding.Labels = authz.getLabels() - roleBinding.Subjects = []rbacv1.Subject{} - if authz.Handler.FeatureStore.Status.Applied.Services.OfflineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OfflineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if authz.Handler.FeatureStore.Status.Applied.Services.OnlineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OnlineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if services.IsLocalRegistry(authz.Handler.FeatureStore) { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.RegistryFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } + roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: services.GetFeastName(authz.Handler.FeatureStore), + Namespace: authz.Handler.FeatureStore.Namespace, + }) roleBinding.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, Kind: "Role", diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 0ee269bda1..0bde0dfd7b 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -127,7 +127,6 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Context("When deploying a resource with all db storage services", func() { const resourceName = "cr-name" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -206,7 +205,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ DBPersistence: &feastdevv1alpha1.OfflineStoreDBStorePersistence{ Type: string(offlineType), @@ -430,16 +429,17 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -529,7 +529,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -550,20 +550,20 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -579,29 +579,29 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineDBPersistenceSnowflakeConfigType, + DBParameters: unmarshallYamlString(snowflakeYamlString), + }, Registry: services.RegistryConfig{ Path: copyMap["path"].(string), RegistryType: services.RegistryDBPersistenceSQLConfigType, DBParameters: dbParametersMap, }, + OnlineStore: services.OnlineStoreConfig{ + Type: onlineType, + DBParameters: unmarshallYamlString(cassandraYamlString), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -610,38 +610,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineDBPersistenceSnowflakeConfigType, - DBParameters: unmarshallYamlString(snowflakeYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(1)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -650,25 +628,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Type: onlineType, - DBParameters: unmarshallYamlString(cassandraYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(repoConfigOnline).To(Equal(testConfig)) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(1)) // check client config cm := &corev1.ConfigMap{} @@ -682,6 +644,15 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -716,17 +687,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { feast.Handler.FeatureStore = resource // check online config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -736,9 +706,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType - onlineConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType + testConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index a762faa5a2..70ac81a056 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -48,7 +48,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { const resourceName = "services-ephemeral" const offlineType = "duckdb" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -66,7 +65,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ @@ -199,16 +198,17 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -244,7 +244,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -265,20 +265,21 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -291,28 +292,27 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, Registry: services.RegistryConfig{ RegistryType: services.RegistryFileConfigType, Path: registryPath, }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStorePath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -321,37 +321,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -360,25 +338,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStorePath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -396,12 +356,19 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, + OfflineStore: services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + }, OnlineStore: services.OnlineStoreConfig{ Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: regRemote, + Registry: services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfigClient).To(Equal(clientConfig)) @@ -424,17 +391,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) feast.Handler.FeatureStore = resource - // check registry config - deploy = &appsv1.Deployment{} + // check registry err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -443,21 +409,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStorePath testConfig.Registry.Path = newRegistryPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -466,20 +427,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -489,8 +444,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStorePath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStorePath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 57dd3a290d..57a73eb0eb 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -48,7 +48,6 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Context("When deploying a resource with all ephemeral services and Kubernetes authorization", func() { const resourceName = "kubernetes-authorization" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -63,7 +62,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{ Roles: roles, }} @@ -125,7 +124,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -134,7 +133,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -188,47 +187,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // check configured Roles for _, roleName := range roles { @@ -277,23 +248,21 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Kind: "Role", Name: feastRole.Name, } - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) - expectedSubject := rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: sa.Name, - Namespace: sa.Namespace, - } - Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) - Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) + expectedSubject := rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: sa.Name, + Namespace: sa.Namespace, } + Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) + Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) By("Updating the user roled and reconciling") resourceNew := resource.DeepCopy() @@ -394,7 +363,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -415,19 +384,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { }, } - // check registry deployment + // check registry deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -436,34 +405,22 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, - }, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + } + testConfig.Registry.RegistryType = services.RegistryFileConfigType + testConfig.AuthzConfig = services.AuthzConfig{ + Type: services.KubernetesAuthType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -472,37 +429,14 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -511,26 +445,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -544,6 +459,15 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -553,10 +477,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, + Registry: regRemote, AuthzConfig: services.AuthzConfig{ Type: services.KubernetesAuthType, }, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go index 70f33486fc..5139e14dd3 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go @@ -154,43 +154,26 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { Expect(cond.Message).To(Equal(feastdevv1alpha1.OnlineStoreReadyMessage)) Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("ERROR")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("INFO")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("DEBUG")) }) @@ -229,32 +212,22 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index f4a21a28f1..aff36f338e 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -46,7 +46,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-object-store" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -68,7 +67,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OnlineStore = nil resource.Spec.Services.OfflineStore = nil @@ -176,39 +175,23 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // update S3 additional args and reconcile resourceNew := resource.DeepCopy() @@ -234,16 +217,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.S3AdditionalKwargs).To(Equal(&newS3AdditionalKwargs)) // check registry deployment - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.VolumeMounts).To(HaveLen(1)) }) It("should properly encode a feature_store.yaml config", func() { @@ -290,21 +271,28 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -313,38 +301,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: registryPath, - S3AdditionalKwargs: &s3AdditionalKwargs, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryFileConfigType, + Path: registryPath, + S3AdditionalKwargs: &s3AdditionalKwargs, } - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -358,17 +321,12 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // remove S3 additional keywords and reconcile resourceNew := resource.DeepCopy() @@ -386,16 +344,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { feast.Handler.FeatureStore = resource // check registry config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -405,27 +361,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) testConfig.Registry.S3AdditionalKwargs = nil - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index eb320c5bb3..08c92a88a9 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -49,7 +49,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { const resourceName = "oidc-authorization" const oidcSecretName = "oidc-secret" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -74,7 +73,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{OidcAuthz: &feastdevv1alpha1.OidcAuthz{ SecretRef: corev1.LocalObjectReference{ Name: oidcSecretName, @@ -147,7 +146,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -156,7 +155,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -206,47 +205,22 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) // check Feast Role feastRole := &rbacv1.Role{} @@ -268,16 +242,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(err).To(HaveOccurred()) Expect(errors.IsNotFound(err)).To(BeTrue()) - // check ServiceAccounts - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) - } + // check ServiceAccount + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) By("Clearing the OIDC authorization and reconciling") resourceNew := resource.DeepCopy() @@ -328,7 +300,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -349,19 +321,19 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -374,28 +346,27 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + }, Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, + RegistryType: services.RegistryFileConfigType, + Path: services.EphemeralPath + "/" + services.DefaultRegistryPath, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: services.EphemeralPath + "/" + services.DefaultOnlineStorePath, + Type: services.OnlineSqliteConfigType, }, AuthzConfig: expectedServerOidcAuthorizConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -404,35 +375,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -441,23 +391,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) // check client config @@ -472,6 +405,11 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index e64e5cd624..887d9070ef 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -50,7 +50,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-pvc" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -79,7 +78,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ @@ -260,96 +259,87 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(offlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + name := feast.GetFeastServiceName(services.RegistryFeastType) + regVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(regVol.Name).To(Equal(name)) + Expect(regVol.PersistentVolumeClaim.ClaimName).To(Equal(name)) + + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.VolumeMounts).To(HaveLen(3)) + offlineVolMount := services.GetOfflineVolumeMount(feast.Handler.FeatureStore, offlineContainer.VolumeMounts) + Expect(offlineVolMount.MountPath).To(Equal(offlineStoreMountPath)) + offlinePvcName := feast.GetFeastServiceName(services.OfflineFeastType) + Expect(offlineVolMount.Name).To(Equal(offlinePvcName)) // check offline pvc pvc := &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: offlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) Expect(pvc.Spec.StorageClassName).To(Equal(&storageClassName)) Expect(pvc.Spec.AccessModes).To(Equal(accessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOfflineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(onlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check online + onlinePvcName := feast.GetFeastServiceName(services.OnlineFeastType) + onlineVol := services.GetOnlineVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(onlineVol.Name).To(Equal(onlinePvcName)) + Expect(onlineVol.PersistentVolumeClaim.ClaimName).To(Equal(onlinePvcName)) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.VolumeMounts).To(HaveLen(3)) + onlineVolMount := services.GetOnlineVolumeMount(feast.Handler.FeatureStore, onlineContainer.VolumeMounts) + Expect(onlineVolMount.MountPath).To(Equal(onlineStoreMountPath)) + Expect(onlineVolMount.Name).To(Equal(onlinePvcName)) // check online pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(onlinePvcName)) Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOnlineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(registryMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check registry + registryPvcName := feast.GetFeastServiceName(services.RegistryFeastType) + registryVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(registryVol.Name).To(Equal(registryPvcName)) + Expect(registryVol.PersistentVolumeClaim.ClaimName).To(Equal(registryPvcName)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.VolumeMounts).To(HaveLen(3)) + registryVolMount := services.GetRegistryVolumeMount(feast.Handler.FeatureStore, registryContainer.VolumeMounts) + Expect(registryVolMount.MountPath).To(Equal(registryMountPath)) + Expect(registryVolMount.Name).To(Equal(registryPvcName)) // check registry pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: registryPvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(registryPvcName)) Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultRegistryStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) @@ -375,19 +365,18 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check online deployment deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) // check online pvc is deleted log.FromContext(feast.Handler.Context).Info("Checking deletion of", "PersistentVolumeClaim", deploy.Name) pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) @@ -421,7 +410,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -442,21 +431,22 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -473,25 +463,24 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { RegistryType: services.RegistryFileConfigType, Path: registryMountedPath, }, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStoreMountedPath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -500,37 +489,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -539,25 +507,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStoreMountedPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -571,6 +521,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -610,14 +569,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check registry config deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -626,21 +585,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath testConfig.Registry.Path = newRegistryMountedPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -649,20 +603,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -672,8 +620,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStoreMountedPath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index debd63300b..71b5d400f8 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -172,15 +172,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) svc := &corev1.Service{} @@ -220,11 +221,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -232,7 +233,7 @@ var _ = Describe("FeatureStore Controller", func() { env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -241,17 +242,8 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfig).To(Equal(testConfig)) + testConfig := feast.GetDefaultRepoConfig() + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -265,17 +257,12 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-test-resource-registry.default.svc.cluster.local:80", - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-test-resource-registry.default.svc.cluster.local:80", } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // change feast project and reconcile resourceNew := resource.DeepCopy() @@ -291,8 +278,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -302,7 +289,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -310,7 +297,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should error on reconcile", func() { @@ -339,11 +326,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -378,17 +365,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.RegistryFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) Expect(cond).ToNot(BeNil()) @@ -404,7 +390,6 @@ var _ = Describe("FeatureStore Controller", func() { Context("When reconciling a resource with all services enabled", func() { const resourceName = "services" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -420,7 +405,7 @@ var _ = Describe("FeatureStore Controller", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } @@ -474,7 +459,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -483,7 +468,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -534,16 +519,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -579,12 +564,12 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) saList := corev1.ServiceAccountList{} err = k8sClient.List(ctx, &saList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(saList.Items).To(HaveLen(3)) + Expect(saList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -607,19 +592,20 @@ var _ = Describe("FeatureStore Controller", func() { // check registry config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -628,33 +614,20 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -663,38 +636,16 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-services-registry.default.svc.cluster.local:80", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -703,25 +654,7 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: "feast-services-offline.default.svc.cluster.local", - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -735,6 +668,15 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: "feast-services-offline.default.svc.cluster.local", + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-services-registry.default.svc.cluster.local:80", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -763,8 +705,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -774,7 +716,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -782,7 +724,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should properly set container env variables", func() { @@ -808,7 +750,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -830,129 +772,27 @@ var _ = Describe("FeatureStore Controller", func() { } fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) // check online config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - - // change feast project and reconcile - resourceNew := resource.DeepCopy() - resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} - err = k8sClient.Update(ctx, resourceNew) - Expect(err).NotTo(HaveOccurred()) - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) - }) - - It("Should scale online/offline store service", func() { - By("Reconciling the created resource") - controllerReconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - resource := &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - req, err := labels.NewRequirement(services.NameLabelKey, selection.Equals, []string{resource.Name}) - Expect(err).NotTo(HaveOccurred()) - labelSelector := labels.NewSelector().Add(*req) - listOpts := &client.ListOptions{Namespace: resource.Namespace, LabelSelector: labelSelector} - deployList := appsv1.DeploymentList{} - err = k8sClient.List(ctx, &deployList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) - - svcList := corev1.ServiceList{} - err = k8sClient.List(ctx, &svcList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(svcList.Items).To(HaveLen(3)) - - cmList := corev1.ConfigMapList{} - err = k8sClient.List(ctx, &cmList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) - - feast := services.FeastServices{ - Handler: handler.FeastHandler{ - Client: controllerReconciler.Client, - Context: ctx, - Scheme: controllerReconciler.Scheme, - FeatureStore: resource, - }, - } - - fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) - Expect(err).NotTo(HaveOccurred()) - - // check online config - deploy_online := &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy_online) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy_online.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_online.Name)) - Expect(deploy_online.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy_online.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy_online.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy_online.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - - // check offline config - deploy_offline := &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy_offline) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy_offline.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_offline.Name)) - Expect(deploy_offline.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy_offline.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy_offline.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) // change feast project and reconcile - // scale online replicas to 2 resourceNew := resource.DeepCopy() - new_replicas := int32(2) - resourceNew.Spec.Services.OnlineStore.Replicas = &new_replicas - resourceNew.Spec.Services.OfflineStore.Replicas = &new_replicas - + resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} err = k8sClient.Update(ctx, resourceNew) Expect(err).NotTo(HaveOccurred()) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -962,21 +802,16 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy_online) - Expect(err).NotTo(HaveOccurred()) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy_offline) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy_online.Spec.Replicas).To(Equal(&new_replicas)) - Expect(deploy_offline.Spec.Replicas).To(Equal(&new_replicas)) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) It("Should delete k8s objects owned by the FeatureStore CR", func() { @@ -1002,7 +837,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -1021,7 +856,7 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(2)) + Expect(deployList.Items).To(HaveLen(1)) err = k8sClient.List(ctx, &svcList, listOpts) Expect(err).NotTo(HaveOccurred()) @@ -1096,6 +931,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).To(HaveOccurred()) err = k8sClient.Get(ctx, nsName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).NotTo(BeEmpty()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType)).To(BeNil()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType)).To(BeNil()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeFalse()) @@ -1135,7 +971,6 @@ var _ = Describe("FeatureStore Controller", func() { Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType)).To(BeTrue()) - Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).To(Equal(resource.Namespace)) Expect(resource.Status.ServiceHostnames.Registry).ToNot(BeEmpty()) Expect(resource.Status.ServiceHostnames.Registry).To(Equal(referencedRegistry.Status.ServiceHostnames.Registry)) feast := services.FeastServices{ @@ -1147,6 +982,16 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment + deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + // check client config cm := &corev1.ConfigMap{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -1235,12 +1080,13 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -1275,7 +1121,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) @@ -1296,10 +1142,9 @@ var _ = Describe("FeatureStore Controller", func() { cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.OfflineStoreFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) Expect(cond).ToNot(BeNil()) @@ -1362,7 +1207,7 @@ var _ = Describe("FeatureStore Controller", func() { }) }) -func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, replicas int32, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { +func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -1373,17 +1218,14 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co Services: &feastdevv1alpha1.FeatureStoreServices{ OfflineStore: &feastdevv1alpha1.OfflineStore{}, OnlineStore: &feastdevv1alpha1.OnlineStore{ - StoreServiceConfigs: feastdevv1alpha1.StoreServiceConfigs{ - Replicas: &replicas, - ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ - DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ - Image: &image, - }, - OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ - Env: envVars, - ImagePullPolicy: &pullPolicy, - Resources: &corev1.ResourceRequirements{}, - }, + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ + Image: &image, + }, + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + Env: envVars, + ImagePullPolicy: &pullPolicy, + Resources: &corev1.ResourceRequirements{}, }, }, }, @@ -1394,7 +1236,7 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar { for _, e := range envs { - if e.Name == services.FeatureStoreYamlEnvVar { + if e.Name == services.TmpFeatureStoreYamlEnvVar { return &e } } diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index c191dae332..0b7fe84d22 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -27,7 +27,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -172,16 +171,17 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -225,7 +225,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -237,20 +237,21 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmList.Items).To(HaveLen(1)) - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -259,32 +260,19 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -293,37 +281,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), - Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -332,26 +298,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpsPort, - Scheme: services.HttpsScheme, - Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) + Expect(repoConfigOnline).To(Equal(&testConfig)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) // check client config @@ -366,6 +313,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpsPort, + Scheme: services.HttpsScheme, + Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), + Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -422,25 +381,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { // check registry deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(2)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -454,21 +406,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Path: remoteRegHost, Cert: services.GetTlsPath(services.RegistryFeastType) + "remote.crt", } - offlineConfig.Registry = regRemote - Expect(repoConfigOffline).To(Equal(offlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -478,8 +424,8 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.Registry = regRemote - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOnline).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/client.go b/infra/feast-operator/internal/controller/services/client.go index d4b78e2611..89e22f7be6 100644 --- a/infra/feast-operator/internal/controller/services/client.go +++ b/infra/feast-operator/internal/controller/services/client.go @@ -32,7 +32,7 @@ func (feast *FeastServices) deployClient() error { func (feast *FeastServices) createClientConfigMap() error { logger := log.FromContext(feast.Handler.Context) cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, cm, controllerutil.MutateFn(func() error { @@ -46,7 +46,7 @@ func (feast *FeastServices) createClientConfigMap() error { } func (feast *FeastServices) setClientConfigMap(cm *corev1.ConfigMap) error { - cm.Labels = feast.getLabels(ClientFeastType) + cm.Labels = feast.getFeastTypeLabels(ClientFeastType) clientYaml, err := feast.getClientFeatureStoreYaml(feast.extractConfigFromSecret) if err != nil { return err @@ -81,7 +81,7 @@ func (feast *FeastServices) setCaConfigMap(cm *corev1.ConfigMap) error { func (feast *FeastServices) initCaConfigMap() *corev1.ConfigMap { cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientCaFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientCaFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) return cm diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index 5433e99acf..675fbd047f 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -27,36 +27,74 @@ import ( ) // GetServiceFeatureStoreYamlBase64 returns a base64 encoded feature_store.yaml config for the feast service -func (feast *FeastServices) GetServiceFeatureStoreYamlBase64(feastType FeastServiceType) (string, error) { - fsYaml, err := feast.getServiceFeatureStoreYaml(feastType) +func (feast *FeastServices) GetServiceFeatureStoreYamlBase64() (string, error) { + fsYaml, err := feast.getServiceFeatureStoreYaml() if err != nil { return "", err } return base64.StdEncoding.EncodeToString(fsYaml), nil } -func (feast *FeastServices) getServiceFeatureStoreYaml(feastType FeastServiceType) ([]byte, error) { - repoConfig, err := feast.getServiceRepoConfig(feastType) +func (feast *FeastServices) getServiceFeatureStoreYaml() ([]byte, error) { + repoConfig, err := feast.getServiceRepoConfig() if err != nil { return nil, err } return yaml.Marshal(repoConfig) } -func (feast *FeastServices) getServiceRepoConfig(feastType FeastServiceType) (RepoConfig, error) { - return getServiceRepoConfig(feastType, feast.Handler.FeatureStore, feast.extractConfigFromSecret) +func (feast *FeastServices) getServiceRepoConfig() (RepoConfig, error) { + return getServiceRepoConfig(feast.Handler.FeatureStore, feast.extractConfigFromSecret) } func getServiceRepoConfig( - feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + repoConfig, err := getBaseServiceRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return repoConfig, err + } + appliedSpec := featureStore.Status.Applied + if appliedSpec.Services != nil { + services := appliedSpec.Services + if services.OfflineStore != nil { + err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if services.OnlineStore != nil { + err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if IsLocalRegistry(featureStore) { + err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + } + + return repoConfig, nil +} - repoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) +func getBaseServiceRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + + appliedSpec := featureStore.Status.Applied + repoConfig := defaultRepoConfig(featureStore) + clientRepoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) if err != nil { return repoConfig, err } + repoConfig.AuthzConfig = clientRepoConfig.AuthzConfig + if isRemoteRegistry(featureStore) { + repoConfig.Registry = clientRepoConfig.Registry + } if appliedSpec.AuthzConfig != nil && appliedSpec.AuthzConfig.OidcAuthz != nil { propertiesMap, authSecretErr := secretExtractionFunc("", appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") @@ -75,43 +113,10 @@ func getServiceRepoConfig( repoConfig.AuthzConfig.OidcParameters = oidcServerProperties } - if appliedSpec.Services != nil { - services := appliedSpec.Services - - switch feastType { - case OfflineFeastType: - // Offline server has an `offline_store` section and a remote `registry` - if services.OfflineStore != nil { - err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case OnlineFeastType: - // Online server has an `online_store` section, a remote `registry` and a remote `offline_store` - if services.OnlineStore != nil { - err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case RegistryFeastType: - // Registry server only has a `registry` section - if IsLocalRegistry(featureStore) { - err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - } - } - return repoConfig, nil } func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.Registry = RegistryConfig{} - repoConfig.Registry.Path = DefaultRegistryEphemeralPath registryPersistence := services.Registry.Local.Persistence if registryPersistence != nil { @@ -142,18 +147,10 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr repoConfig.Registry.DBParameters = parametersMap } } - - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OnlineStore = OnlineStoreConfig{} - - repoConfig.OnlineStore.Path = DefaultOnlineStoreEphemeralPath - repoConfig.OnlineStore.Type = OnlineSqliteConfigType onlineStorePersistence := services.OnlineStore.Persistence if onlineStorePersistence != nil { @@ -183,13 +180,11 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret repoConfig.OnlineStore.DBParameters = parametersMap } } - return nil } func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OfflineStore.Type = OfflineFilePersistenceDaskConfigType + repoConfig.OfflineStore = defaultOfflineStoreConfig offlineStorePersistence := services.OfflineStore.Persistence if offlineStorePersistence != nil { @@ -218,9 +213,6 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre repoConfig.OfflineStore.DBParameters = parametersMap } } - - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } @@ -237,10 +229,9 @@ func getClientRepoConfig( secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { status := featureStore.Status appliedServices := status.Applied.Services - clientRepoConfig := RepoConfig{ - Project: status.Applied.FeastProject, - Provider: LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + clientRepoConfig, err := getRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return clientRepoConfig, err } if len(status.ServiceHostnames.OfflineStore) > 0 { clientRepoConfig.OfflineStore = OfflineStoreConfig{ @@ -277,23 +268,27 @@ func getClientRepoConfig( } } - if status.Applied.AuthzConfig == nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ - Type: NoAuthAuthType, - } - } else { + return clientRepoConfig, nil +} + +func getRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + status := featureStore.Status + repoConfig := initRepoConfig(status.Applied.FeastProject) + if status.Applied.AuthzConfig != nil { if status.Applied.AuthzConfig.KubernetesAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: KubernetesAuthType, } } else if status.Applied.AuthzConfig.OidcAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: OidcAuthType, } propertiesMap, err := secretExtractionFunc("", status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") if err != nil { - return clientRepoConfig, err + return repoConfig, err } oidcClientProperties := map[string]interface{}{} @@ -301,13 +296,13 @@ func getClientRepoConfig( if val, exists := propertiesMap[string(oidcClientProperty)]; exists { oidcClientProperties[string(oidcClientProperty)] = val } else { - return clientRepoConfig, missingOidcSecretProperty(oidcClientProperty) + return repoConfig, missingOidcSecretProperty(oidcClientProperty) } } - clientRepoConfig.AuthzConfig.OidcParameters = oidcClientProperties + repoConfig.AuthzConfig.OidcParameters = oidcClientProperties } } - return clientRepoConfig, nil + return repoConfig, nil } func getActualPath(filePath string, pvcConfig *feastdevv1alpha1.PvcConfig) string { @@ -372,3 +367,49 @@ func mergeStructWithDBParametersMap(parametersMap *map[string]interface{}, s int return nil } + +func (feast *FeastServices) GetDefaultRepoConfig() RepoConfig { + return defaultRepoConfig(feast.Handler.FeatureStore) +} + +func defaultRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig { + repoConfig := initRepoConfig(featureStore.Status.Applied.FeastProject) + repoConfig.OnlineStore = defaultOnlineStoreConfig(featureStore) + repoConfig.Registry = defaultRegistryConfig(featureStore) + return repoConfig +} + +func (feast *FeastServices) GetInitRepoConfig() RepoConfig { + return initRepoConfig(feast.Handler.FeatureStore.Status.Applied.FeastProject) +} + +func initRepoConfig(feastProject string) RepoConfig { + return RepoConfig{ + Project: feastProject, + Provider: LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + AuthzConfig: defaultAuthzConfig, + } +} + +func defaultOnlineStoreConfig(featureStore *feastdevv1alpha1.FeatureStore) OnlineStoreConfig { + return OnlineStoreConfig{ + Type: OnlineSqliteConfigType, + Path: defaultOnlineStorePath(featureStore), + } +} + +func defaultRegistryConfig(featureStore *feastdevv1alpha1.FeatureStore) RegistryConfig { + return RegistryConfig{ + RegistryType: RegistryFileConfigType, + Path: defaultRegistryPath(featureStore), + } +} + +var defaultOfflineStoreConfig = OfflineStoreConfig{ + Type: OfflineFilePersistenceDaskConfigType, +} + +var defaultAuthzConfig = AuthzConfig{ + Type: NoAuthAuthType, +} diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 7f017f4d10..42525700a4 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -32,35 +32,26 @@ var projectName = "test-project" var _ = Describe("Repo Config", func() { Context("When creating the RepoConfig of a FeatureStore", func() { - It("should successfully create the repo configs", func() { By("Having the minimal created resource") featureStore := minimalFeatureStore() ApplyDefaultsToStatus(featureStore) - var repoConfig RepoConfig - repoConfig, err := getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig := RegistryConfig{ RegistryType: "file", - Path: DefaultRegistryEphemeralPath, + Path: EphemeralPath + "/" + DefaultRegistryPath, + } + expectedOnlineConfig := OnlineStoreConfig{ + Type: "sqlite", + Path: EphemeralPath + "/" + DefaultOnlineStorePath, } + + var repoConfig RepoConfig + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the local registry resource") @@ -77,29 +68,17 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: "file", Path: "file.db", } + + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the remote registry resource") @@ -108,33 +87,18 @@ var _ = Describe("Repo Config", func() { Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ FeastRef: &feastdevv1alpha1.FeatureStoreRef{ - Name: "registry", - Namespace: "remoteNS", + Name: "registry", }, }, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) By("Having the all the file services") featureStore = minimalFeatureStore() @@ -164,36 +128,24 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + expectedOfflineConfig := OfflineStoreConfig{ Type: "duckdb", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig := OnlineStoreConfig{ + expectedRegistryConfig = RegistryConfig{ + RegistryType: "file", + Path: "/data/registry.db", + } + expectedOnlineConfig = OnlineStoreConfig{ Type: "sqlite", Path: "/data/online.db", } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: "/data/registry.db", - } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having kubernetes authorization") @@ -202,56 +154,24 @@ var _ = Describe("Repo Config", func() { KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{}, } featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{ - Persistence: &feastdevv1alpha1.OfflineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{}, - }, - }, - OnlineStore: &feastdevv1alpha1.OnlineStore{ - Persistence: &feastdevv1alpha1.OnlineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OnlineStoreFilePersistence{}, - }, - }, + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, Registry: &feastdevv1alpha1.Registry{ - Local: &feastdevv1alpha1.LocalRegistryConfig{ - Persistence: &feastdevv1alpha1.RegistryPersistence{ - FilePersistence: &feastdevv1alpha1.RegistryFilePersistence{}, - }, - }, + Local: &feastdevv1alpha1.LocalRegistryConfig{}, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) + expectedOfflineConfig = OfflineStoreConfig{ Type: "dask", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) By("Having oidc authorization") featureStore.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{ @@ -269,46 +189,15 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - expectedOfflineConfig = OfflineStoreConfig{ - Type: "dask", - } Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) repoConfig, err = getClientRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) @@ -359,7 +248,7 @@ var _ = Describe("Repo Config", func() { featureStore.Spec.Services.OfflineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.OnlineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.Registry.Local.Persistence.FilePersistence = nil - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) newMap := CopyMap(parameterMap) port := parameterMap["port"].(int) @@ -369,29 +258,16 @@ var _ = Describe("Repo Config", func() { Port: port, DBParameters: newMap, } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - newMap = CopyMap(parameterMap) expectedOnlineConfig = OnlineStoreConfig{ Type: OnlineDBPersistenceSnowflakeConfigType, - DBParameters: newMap, + DBParameters: CopyMap(parameterMap), } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: RegistryDBPersistenceSnowflakeConfigType, DBParameters: parameterMap, } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) }) }) @@ -413,13 +289,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err := getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err := getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -440,13 +316,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientId): "client-id", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -455,17 +331,8 @@ var _ = Describe("Repo Config", func() { }) }) -func emptyOnlineStoreConfig() OnlineStoreConfig { - return OnlineStoreConfig{} -} - -func emptyOfflineStoreConfig() OfflineStoreConfig { - return OfflineStoreConfig{} -} - -func emptyRegistryConfig() RegistryConfig { - return RegistryConfig{} -} +var emptyOfflineStoreConfig = OfflineStoreConfig{} +var emptyRegistryConfig = RegistryConfig{} func minimalFeatureStore() *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 232e3e5874..32cf91d09b 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -106,7 +106,12 @@ func (feast *FeastServices) Deploy() error { return err } } - + if err := feast.createServiceAccount(); err != nil { + return err + } + if err := feast.createDeployment(); err != nil { + return err + } if err := feast.deployClient(); err != nil { return err } @@ -194,12 +199,6 @@ func (feast *FeastServices) deployFeastServiceByType(feastType FeastServiceType) if err := feast.createService(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } - if err := feast.createServiceAccount(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } - if err := feast.createDeployment(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } return feast.setFeastServiceCondition(nil, feastType) } @@ -207,12 +206,6 @@ func (feast *FeastServices) removeFeastServiceByType(feastType FeastServiceType) if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSvc(feastType)); err != nil { return err } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastDeploy(feastType)); err != nil { - return err - } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSA(feastType)); err != nil { - return err - } if err := feast.Handler.DeleteOwnedFeastObj(feast.initPVC(feastType)); err != nil { return err } @@ -233,11 +226,11 @@ func (feast *FeastServices) createService(feastType FeastServiceType) error { return nil } -func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) error { +func (feast *FeastServices) createServiceAccount() error { logger := log.FromContext(feast.Handler.Context) - sa := feast.initFeastSA(feastType) + sa := feast.initFeastSA() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, controllerutil.MutateFn(func() error { - return feast.setServiceAccount(sa, feastType) + return feast.setServiceAccount(sa) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -246,11 +239,11 @@ func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) err return nil } -func (feast *FeastServices) createDeployment(feastType FeastServiceType) error { +func (feast *FeastServices) createDeployment() error { logger := log.FromContext(feast.Handler.Context) - deploy := feast.initFeastDeploy(feastType) + deploy := feast.initFeastDeploy() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, deploy, controllerutil.MutateFn(func() error { - return feast.setDeployment(deploy, feastType) + return feast.setDeployment(deploy) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -280,78 +273,122 @@ func (feast *FeastServices) createPVC(pvcCreate *feastdevv1alpha1.PvcCreate, fea return nil } -func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType FeastServiceType) error { - fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64(feastType) - if err != nil { - return err - } - deploy.Labels = feast.getLabels(feastType) - sa := feast.initFeastSA(feastType) - tls := feast.getTlsConfigs(feastType) - serviceConfigs := feast.getServiceConfigs(feastType) - defaultServiceConfigs := serviceConfigs.DefaultConfigs - probeHandler := getProbeHandler(feastType, tls) - +func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { + deploy.Labels = feast.getLabels() deploy.Spec = appsv1.DeploymentSpec{ - Replicas: feast.getServiceReplicas(feastType), + Replicas: &DefaultReplicas, Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: deploy.GetLabels(), }, Spec: corev1.PodSpec{ - ServiceAccountName: sa.Name, - Containers: []corev1.Container{ - { - Name: string(feastType), - Image: *defaultServiceConfigs.Image, - Command: feast.getContainerCommand(feastType), - Ports: []corev1.ContainerPort{ - { - Name: string(feastType), - ContainerPort: getTargetPort(feastType, tls), - Protocol: corev1.ProtocolTCP, - }, - }, - Env: []corev1.EnvVar{ - { - Name: FeatureStoreYamlEnvVar, - Value: fsYamlB64, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 30, - PeriodSeconds: 30, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 20, - PeriodSeconds: 30, - }, - }, - }, + ServiceAccountName: feast.initFeastSA().Name, }, }, } + if err := feast.setPod(&deploy.Spec.Template.Spec); err != nil { + return err + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +} - // configs are applied here - podSpec := &deploy.Spec.Template.Spec - applyOptionalContainerConfigs(&podSpec.Containers[0], serviceConfigs.OptionalConfigs) - feast.mountTlsConfig(feastType, podSpec) - if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { - mountPvcConfig(podSpec, pvcConfig, deploy.Name) +func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { + if err := feast.setContainers(podSpec); err != nil { + return err } + feast.setRegistryClientInitContainer(podSpec) + feast.mountTlsConfigs(podSpec) + feast.mountPvcConfigs(podSpec) + feast.mountEmptyDirVolumes(podSpec) - switch feastType { - case OfflineFeastType: - feast.registryClientPodConfigs(podSpec) - case OnlineFeastType: - feast.registryClientPodConfigs(podSpec) - feast.offlineClientPodConfigs(podSpec) + return nil +} + +func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() + if err != nil { + return err + } + feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject + workingDir := getOfflineMountPath(feast.Handler.FeatureStore) + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "feast-init", + Image: DefaultImage, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + + feastProject + " ] || feast init " + feastProject + ";\necho $" + + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastProject + + "/feature_repo/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, + WorkingDir: workingDir, + }) + if feast.isLocalRegistry() { + feast.setContainer(&podSpec.Containers, RegistryFeastType, fsYamlB64) } + if feast.isOfflinStore() { + feast.setContainer(&podSpec.Containers, OfflineFeastType, fsYamlB64) + } + if feast.isOnlinStore() { + feast.setContainer(&podSpec.Containers, OnlineFeastType, fsYamlB64) + } + return nil +} - return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastType FeastServiceType, fsYamlB64 string) { + tls := feast.getTlsConfigs(feastType) + serviceConfigs := feast.getServiceConfigs(feastType) + defaultServiceConfigs := serviceConfigs.DefaultConfigs + probeHandler := getProbeHandler(feastType, tls) + container := &corev1.Container{ + Name: string(feastType), + Image: *defaultServiceConfigs.Image, + WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + "/feature_repo", + Command: feast.getContainerCommand(feastType), + Ports: []corev1.ContainerPort{ + { + Name: string(feastType), + ContainerPort: getTargetPort(feastType, tls), + Protocol: corev1.ProtocolTCP, + }, + }, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + /* + { + Name: mlpConfigVar, + Value: DefaultMlpConfigPath, + }, + */ + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 3, + FailureThreshold: 40, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 20, + FailureThreshold: 6, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 10, + }, + } + applyOptionalContainerConfigs(container, serviceConfigs.OptionalConfigs) + *containers = append(*containers, *container) } func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []string { @@ -380,38 +417,28 @@ func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []st return feastCommand } -func (feast *FeastServices) offlineClientPodConfigs(podSpec *corev1.PodSpec) { - feast.mountTlsConfig(OfflineFeastType, podSpec) -} - -func (feast *FeastServices) registryClientPodConfigs(podSpec *corev1.PodSpec) { - feast.setRegistryClientInitContainer(podSpec) - feast.mountRegistryClientTls(podSpec) -} - func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry - if len(hostname) > 0 { + // add grpc init container if remote registry reference (feastRef) is configured + if len(hostname) > 0 && feast.IsRemoteRefRegistry() { grpcurlFlag := "-plaintext" hostSplit := strings.Split(hostname, ":") if len(hostSplit) > 1 && hostSplit[1] == "443" { grpcurlFlag = "-insecure" } - podSpec.InitContainers = []corev1.Container{ - { - Name: "init-registry", - Image: "fullstorydev/grpcurl:v1.9.1-alpine", - Command: []string{ - "sh", "-c", - "until grpcurl " + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", - }, + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "init-registry", + Image: "fullstorydev/grpcurl:v1.9.1-alpine", + Command: []string{ + "sh", "-c", + "until grpcurl -H \"authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" " + + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", }, - } + }) } } - func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { - svc.Labels = feast.getLabels(feastType) + svc.Labels = feast.getFeastTypeLabels(feastType) if feast.isOpenShiftTls(feastType) { svc.Annotations = map[string]string{ "service.beta.openshift.io/serving-cert-secret-name": svc.Name + tlsNameSuffix, @@ -426,7 +453,7 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi scheme = HttpsScheme } svc.Spec = corev1.ServiceSpec{ - Selector: svc.GetLabels(), + Selector: feast.getLabels(), Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { @@ -441,8 +468,8 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi return controllerutil.SetControllerReference(feast.Handler.FeatureStore, svc, feast.Handler.Scheme) } -func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount, feastType FeastServiceType) error { - sa.Labels = feast.getLabels(feastType) +func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount) error { + sa.Labels = feast.getLabels() return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) } @@ -478,21 +505,6 @@ func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastd return feastdevv1alpha1.ServiceConfigs{} } -func (feast *FeastServices) getServiceReplicas(feastType FeastServiceType) *int32 { - appliedServices := feast.Handler.FeatureStore.Status.Applied.Services - switch feastType { - case OfflineFeastType: - if feast.isOfflinStore() { - return appliedServices.OfflineStore.Replicas - } - case OnlineFeastType: - if feast.isOnlinStore() { - return appliedServices.OnlineStore.Replicas - } - } - return &DefaultReplicas -} - func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *string { services := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { @@ -512,8 +524,13 @@ func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *stri return nil } -// GetObjectMeta returns the feast k8s object metadata -func (feast *FeastServices) GetObjectMeta(feastType FeastServiceType) metav1.ObjectMeta { +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMeta() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: GetFeastName(feast.Handler.FeatureStore), Namespace: feast.Handler.FeatureStore.Namespace} +} + +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMetaType(feastType FeastServiceType) metav1.ObjectMeta { return metav1.ObjectMeta{Name: feast.GetFeastServiceName(feastType), Namespace: feast.Handler.FeatureStore.Namespace} } @@ -530,10 +547,15 @@ func GetFeastName(featureStore *feastdevv1alpha1.FeatureStore) string { return handler.FeastPrefix + featureStore.Name } -func (feast *FeastServices) getLabels(feastType FeastServiceType) map[string]string { +func (feast *FeastServices) getFeastTypeLabels(feastType FeastServiceType) map[string]string { + labels := feast.getLabels() + labels[ServiceTypeLabelKey] = string(feastType) + return labels +} + +func (feast *FeastServices) getLabels() map[string]string { return map[string]string{ - NameLabelKey: feast.Handler.FeatureStore.Name, - ServiceTypeLabelKey: string(feastType), + NameLabelKey: feast.Handler.FeatureStore.Name, } } @@ -541,17 +563,17 @@ func (feast *FeastServices) setServiceHostnames() error { feast.Handler.FeatureStore.Status.ServiceHostnames = feastdevv1alpha1.ServiceHostnames{} domain := svcDomain + ":" if feast.isOfflinStore() { - objMeta := feast.GetObjectMeta(OfflineFeastType) + objMeta := feast.initFeastSvc(OfflineFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS) } if feast.isOnlinStore() { - objMeta := feast.GetObjectMeta(OnlineFeastType) + objMeta := feast.initFeastSvc(OnlineFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.OnlineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OnlineStore.TLS) } if feast.isLocalRegistry() { - objMeta := feast.GetObjectMeta(RegistryFeastType) + objMeta := feast.initFeastSvc(RegistryFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.Registry = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.Registry.Local.TLS) } else if feast.isRemoteRegistry() { @@ -598,10 +620,6 @@ func (feast *FeastServices) setRemoteRegistryURL() error { func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, error) { if feast.IsRemoteRefRegistry() { feastRemoteRef := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef - // default to FeatureStore namespace if not set - if len(feastRemoteRef.Namespace) == 0 { - feastRemoteRef.Namespace = feast.Handler.FeatureStore.Namespace - } nsName := types.NamespacedName{Name: feastRemoteRef.Name, Namespace: feastRemoteRef.Namespace} crNsName := client.ObjectKeyFromObject(feast.Handler.FeatureStore) if nsName == crNsName { @@ -635,19 +653,13 @@ func (feast *FeastServices) isRemoteRegistry() bool { } func (feast *FeastServices) IsRemoteRefRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.FeastRef != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef != nil } func (feast *FeastServices) isRemoteHostnameRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.Hostname != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.Hostname != nil } func (feast *FeastServices) isOfflinStore() bool { @@ -660,9 +672,9 @@ func (feast *FeastServices) isOnlinStore() bool { return appliedServices != nil && appliedServices.OnlineStore != nil } -func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1.Deployment { +func (feast *FeastServices) initFeastDeploy() *appsv1.Deployment { deploy := &appsv1.Deployment{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } deploy.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment")) return deploy @@ -670,15 +682,15 @@ func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1. func (feast *FeastServices) initFeastSvc(feastType FeastServiceType) *corev1.Service { svc := &corev1.Service{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } svc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) return svc } -func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.ServiceAccount { +func (feast *FeastServices) initFeastSA() *corev1.ServiceAccount { sa := &corev1.ServiceAccount{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) return sa @@ -686,7 +698,7 @@ func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.Serv func (feast *FeastServices) initPVC(feastType FeastServiceType) *corev1.PersistentVolumeClaim { pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } pvc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim")) return pvc @@ -704,28 +716,73 @@ func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs } } -func mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, deployName string) { +func (feast *FeastServices) mountPvcConfigs(podSpec *corev1.PodSpec) { + for _, feastType := range feastServerTypes { + if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { + feast.mountPvcConfig(podSpec, pvcConfig, feastType) + } + } +} + +func (feast *FeastServices) mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { if podSpec != nil && pvcConfig != nil { - container := &podSpec.Containers[0] - var pvcName string - if pvcConfig.Create != nil { - pvcName = deployName - } else { + volName := feast.initPVC(feastType).Name + pvcName := volName + if pvcConfig.Ref != nil { pvcName = pvcConfig.Ref.Name } - podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ - Name: pvcName, + Name: volName, VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcName, }, }, }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: pvcName, - MountPath: pvcConfig.MountPath, + if feastType == OfflineFeastType { + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } +} + +func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { + if shouldMountEmptyDir(feast.Handler.FeatureStore) { + mountEmptyDirVolume(podSpec) + } +} + +func mountEmptyDirVolume(podSpec *corev1.PodSpec) { + if podSpec != nil { + volName := strings.TrimPrefix(EphemeralPath, "/") + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, }) + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index b9e1a9d9d7..882839a429 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -25,13 +25,12 @@ import ( ) const ( - FeatureStoreYamlEnvVar = "FEATURE_STORE_YAML_BASE64" - FeatureStoreYamlCmKey = "feature_store.yaml" - DefaultRegistryEphemeralPath = "/tmp/registry.db" - DefaultRegistryPvcPath = "registry.db" - DefaultOnlineStoreEphemeralPath = "/tmp/online_store.db" - DefaultOnlineStorePvcPath = "online_store.db" - svcDomain = ".svc.cluster.local" + TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + FeatureStoreYamlCmKey = "feature_store.yaml" + EphemeralPath = "/feast-data" + DefaultRegistryPath = "registry.db" + DefaultOnlineStorePath = "online_store.db" + svcDomain = ".svc.cluster.local" HttpPort = 80 HttpsPort = 443 @@ -164,6 +163,13 @@ var ( OidcClientProperties = []OidcPropertyType{OidcClientSecret, OidcUsername, OidcPassword} ) +// feast server types, not the client types +var feastServerTypes = []FeastServiceType{ + RegistryFeastType, + OfflineFeastType, + OnlineFeastType, +} + // AuthzType defines the authorization type type AuthzType string diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go index a52cc707eb..6dcca7edea 100644 --- a/infra/feast-operator/internal/controller/services/tls.go +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -165,12 +165,19 @@ func (feast *FeastServices) mountRegistryClientTls(podSpec *corev1.PodSpec) { if feast.localRegistryTls() { feast.mountTlsConfig(RegistryFeastType, podSpec) } else if feast.remoteRegistryTls() { - mountTlsRemoteRegistryConfig(RegistryFeastType, podSpec, + mountTlsRemoteRegistryConfig(podSpec, feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.TLS) } } } +func (feast *FeastServices) mountTlsConfigs(podSpec *corev1.PodSpec) { + // how deal w/ client deployment tls mounts when the time comes? new function? + feast.mountRegistryClientTls(podSpec) + feast.mountTlsConfig(OfflineFeastType, podSpec) + feast.mountTlsConfig(OnlineFeastType, podSpec) +} + func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec *corev1.PodSpec) { tls := feast.getTlsConfigs(feastType) if tls.IsTLS() && podSpec != nil { @@ -183,18 +190,19 @@ func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec * }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + if i, container := getContainerByType(feastType, podSpec.Containers); container != nil { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(feastType), + ReadOnly: true, + }) + } } } -func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { +func mountTlsRemoteRegistryConfig(podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { if tls != nil { - volName := string(feastType) + tlsNameSuffix + volName := string(RegistryFeastType) + tlsNameSuffix podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: volName, VolumeSource: corev1.VolumeSource{ @@ -203,12 +211,13 @@ func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.Po }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(RegistryFeastType), + ReadOnly: true, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 17d23dcf72..522eb2265b 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -135,23 +135,15 @@ var _ = Describe("TLS Config", func() { Expect(openshiftTls).To(BeTrue()) // check k8s deployment objects - offlineDeploy := feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) + feastDeploy := feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) - onlineDeploy := feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) - Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[1].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[2].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(4)) // registry service w/ tls and in an openshift cluster feast.Handler.FeatureStore = minimalFeatureStore() @@ -258,22 +250,19 @@ var _ = Describe("TLS Config", func() { Expect(onlineSvc.Spec.Ports[0].Name).To(Equal(HttpScheme)) // check k8s deployment objects - offlineDeploy = feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) - Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - onlineDeploy = feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) + feastDeploy = feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).NotTo(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + + Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).To(ContainElements(ContainSubstring("--key"))) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) + Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 92ee2b5752..d0ca94ff86 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -32,20 +32,22 @@ func isRemoteRegistry(featureStore *feastdevv1alpha1.FeatureStore) bool { } func hasPvcConfig(featureStore *feastdevv1alpha1.FeatureStore, feastType FeastServiceType) (*feastdevv1alpha1.PvcConfig, bool) { + var pvcConfig *feastdevv1alpha1.PvcConfig services := featureStore.Status.Applied.Services - var pvcConfig *feastdevv1alpha1.PvcConfig = nil - switch feastType { - case OnlineFeastType: - if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig - } - case OfflineFeastType: - if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig - } - case RegistryFeastType: - if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { - pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + if services != nil { + switch feastType { + case OnlineFeastType: + if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig + } + case OfflineFeastType: + if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig + } + case RegistryFeastType: + if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { + pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + } } } return pvcConfig, pvcConfig != nil @@ -58,6 +60,20 @@ func shouldCreatePvc(featureStore *feastdevv1alpha1.FeatureStore, feastType Feas return nil, false } +func shouldMountEmptyDir(featureStore *feastdevv1alpha1.FeatureStore) bool { + _, ok := hasPvcConfig(featureStore, OfflineFeastType) + return !ok +} + +func getOfflineMountPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if featureStore.Status.Applied.Services != nil { + if pvcConfig, ok := hasPvcConfig(featureStore, OfflineFeastType); ok { + return pvcConfig.MountPath + } + } + return EphemeralPath +} + func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { cr.Status.FeastVersion = feastversion.FeastVersion applied := cr.Spec.DeepCopy() @@ -87,7 +103,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.Registry.Local.Persistence.FilePersistence.Path) == 0 { - services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(services.Registry.Local.Persistence.FilePersistence) + services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(cr) } if services.Registry.Local.Persistence.FilePersistence.PvcConfig != nil { @@ -97,7 +113,10 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } setServiceDefaultConfigs(&services.Registry.Local.ServiceConfigs.DefaultConfigs) + } else if services.Registry.Remote.FeastRef != nil && len(services.Registry.Remote.FeastRef.Namespace) == 0 { + services.Registry.Remote.FeastRef.Namespace = cr.Namespace } + if services.OfflineStore != nil { if services.OfflineStore.Persistence == nil { services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{} @@ -118,7 +137,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } } - setStoreServiceDefaultConfigs(&services.OfflineStore.StoreServiceConfigs) + setServiceDefaultConfigs(&services.OfflineStore.ServiceConfigs.DefaultConfigs) } if services.OnlineStore != nil { @@ -132,7 +151,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { - services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(services.OnlineStore.Persistence.FilePersistence) + services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) } if services.OnlineStore.Persistence.FilePersistence.PvcConfig != nil { @@ -141,7 +160,7 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } } - setStoreServiceDefaultConfigs(&services.OnlineStore.StoreServiceConfigs) + setServiceDefaultConfigs(&services.OnlineStore.ServiceConfigs.DefaultConfigs) } // overwrite status.applied with every reconcile applied.DeepCopyInto(&cr.Status.Applied) @@ -153,13 +172,6 @@ func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { } } -func setStoreServiceDefaultConfigs(storeServiceConfigs *feastdevv1alpha1.StoreServiceConfigs) { - if storeServiceConfigs.Replicas == nil { - storeServiceConfigs.Replicas = &DefaultReplicas - } - setServiceDefaultConfigs(&storeServiceConfigs.ServiceConfigs.DefaultConfigs) -} - func checkOfflineStoreFilePersistenceType(value string) error { if slices.Contains(feastdevv1alpha1.ValidOfflineStoreFilePersistenceTypes, value) { return nil @@ -194,18 +206,20 @@ func ensurePVCDefaults(pvc *feastdevv1alpha1.PvcConfig, feastType FeastServiceTy } } -func defaultOnlineStorePath(persistence *feastdevv1alpha1.OnlineStoreFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultOnlineStoreEphemeralPath +func defaultOnlineStorePath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, OnlineFeastType); ok { + return DefaultOnlineStorePath } - return DefaultOnlineStorePvcPath + // if online pvc not set, use offline's mount path. + return getOfflineMountPath(featureStore) + "/" + DefaultOnlineStorePath } -func defaultRegistryPath(persistence *feastdevv1alpha1.RegistryFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultRegistryEphemeralPath +func defaultRegistryPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, RegistryFeastType); ok { + return DefaultRegistryPath } - return DefaultRegistryPvcPath + // if registry pvc not set, use offline's mount path. + return getOfflineMountPath(featureStore) + "/" + DefaultRegistryPath } func checkOfflineStoreDBStorePersistenceType(value string) error { @@ -357,3 +371,69 @@ func envOverride(dst, src []corev1.EnvVar) []corev1.EnvVar { } return dst } + +func GetRegistryContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(RegistryFeastType, containers) + return container +} + +func GetOfflineContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(OfflineFeastType, containers) + return container +} + +func GetOnlineContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(OnlineFeastType, containers) + return container +} + +func getContainerByType(feastType FeastServiceType, containers []corev1.Container) (int, *corev1.Container) { + for i, c := range containers { + if c.Name == string(feastType) { + return i, &c + } + } + return -1, nil +} + +func GetRegistryVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(RegistryFeastType, featureStore, volumes) +} + +func GetOnlineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OnlineFeastType, featureStore, volumes) +} + +func GetOfflineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OfflineFeastType, featureStore, volumes) +} + +func getVolumeByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + for _, v := range volumes { + if v.Name == GetFeastServiceName(featureStore, feastType) { + return &v + } + } + return nil +} + +func GetRegistryVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(RegistryFeastType, featureStore, volumeMounts) +} + +func GetOnlineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OnlineFeastType, featureStore, volumeMounts) +} + +func GetOfflineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OfflineFeastType, featureStore, volumeMounts) +} + +func getVolumeMountByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + for _, vm := range volumeMounts { + if vm.Name == GetFeastServiceName(featureStore, feastType) { + return &vm + } + } + return nil +} From c6a4ed9ab945b9ab9c269203a341888df3e5cc1f Mon Sep 17 00:00:00 2001 From: Tommy Hughes Date: Fri, 20 Dec 2024 12:10:53 -0600 Subject: [PATCH 2/2] fix e2e tests Signed-off-by: Tommy Hughes --- infra/feast-operator/test/e2e/e2e_test.go | 26 +++++++++++++--------- infra/feast-operator/test/e2e/test_util.go | 6 +---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index fdf58d8f3b..bab57a3c00 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -27,9 +27,12 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/test/utils" ) -const feastControllerNamespace = "feast-operator-system" -const timeout = 2 * time.Minute -const controllerDeploymentName = "feast-operator-controller-manager" +const ( + feastControllerNamespace = "feast-operator-system" + timeout = 2 * time.Minute + controllerDeploymentName = "feast-operator-controller-manager" + feastPrefix = "feast-" +) var _ = Describe("controller", Ordered, func() { BeforeAll(func() { @@ -159,15 +162,18 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", featureStoreName, err)) - k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName), - fmt.Sprintf("feast-%s-offline", featureStoreName), + feastResourceName := feastPrefix + featureStoreName + k8sResourceNames := []string{feastResourceName} + feastK8sResourceNames := []string{ + feastResourceName + "-online", + feastResourceName + "-offline", } if !hasRemoteRegistry { - k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName)) + feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") } - for _, deploymentName := range k8ResourceNames { + for _, deploymentName := range k8sResourceNames { By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) Expect(err).To(BeNil(), fmt.Sprintf( @@ -178,7 +184,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st } By("Check if the feast client - kubernetes config map exists.") - configMapName := fmt.Sprintf("feast-%s-client", featureStoreName) + configMapName := feastResourceName + "-client" err = checkIfConfigMapExists(namespace, configMapName) Expect(err).To(BeNil(), fmt.Sprintf( "config map %s is not available but expected to be available. \nError: %v\n", @@ -186,7 +192,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st )) fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) - for _, serviceAccountName := range k8ResourceNames { + for _, serviceAccountName := range k8sResourceNames { By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) err = checkIfServiceAccountExists(namespace, serviceAccountName) Expect(err).To(BeNil(), fmt.Sprintf( @@ -196,7 +202,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) } - for _, serviceName := range k8ResourceNames { + for _, serviceName := range feastK8sResourceNames { By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) err = checkIfKubernetesServiceExists(namespace, serviceName) Expect(err).To(BeNil(), fmt.Sprintf( diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index d92f719fb9..017690a0ec 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -192,9 +192,5 @@ func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (boo hasValidFeastRef := registryConfig.Remote.FeastRef != nil && registryConfig.Remote.FeastRef.Name != "" - if hasHostname || hasValidFeastRef { - return true, nil - } - - return false, nil + return hasHostname || hasValidFeastRef, nil }