support spark crd use pyFiles (#3056) (#3058)

Co-authored-by: chengjoey <[email protected]>
erda-project · Nov 16, 2021 · 02583bd · 02583bd
1 parent 8a12314
commit 02583bd
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/apistructs/bigdata.go b/apistructs/bigdata.go
@@ -64,4 +64,9 @@ type SparkConf struct {
 	PythonVersion    *string         `json:"pythonVersion,omitempty"`
 	DriverResource   BigdataResource `json:"driverResource"`
 	ExecutorResource BigdataResource `json:"executorResource"`
+	Deps             Dependencies    `json:"deps"`
+}
+
+type Dependencies struct {
+	PyFiles []string `json:"pyFiles"`
 }
diff --git a/...pipeline/pipengine/actionexecutor/plugins/scheduler/executor/plugins/k8sspark/k8sspark.go b/...pipeline/pipengine/actionexecutor/plugins/scheduler/executor/plugins/k8sspark/k8sspark.go
@@ -388,6 +388,11 @@ func (k *K8sSpark) generateKubeSparkJob(job *apistructs.JobFromUser, conf *apist
 		},
 	}
 
+	// add deps pyFiles
+	if len(conf.Spec.SparkConf.Deps.PyFiles) > 0 {
+		sparkApp.Spec.Deps.PyFiles = conf.Spec.SparkConf.Deps.PyFiles
+	}
+
 	if sparkApp.Spec.Type == sparkv1beta2.PythonApplicationType {
 		sparkApp.Spec.PythonVersion = stringptr("3")
 		if conf.Spec.SparkConf.PythonVersion != nil {