script glue args

Dorianteffo · Apr 10, 2024 · e2336a8 · e2336a8
1 parent 63900d6
commit e2336a8
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/airflow/dags/vg-lakehouse-dag.py b/airflow/dags/vg-lakehouse-dag.py
@@ -15,8 +15,15 @@
 delta_path = "s3://vg-lakehouse/delta_jar/delta-core_2.12-2.1.0.jar,s3://vg-lakehouse/delta_jar/delta-storage-2.1.0.jar"
 glue_args = {
             "GlueVersion": "4.0", 
+            "Command" : {
+                'PythonVersion': '3'
+            },
             "WorkerType": "G.1X",
             "NumberOfWorkers": 2, 
+            "DefaultArguments":{
+                '--extra-jars':delta_path,
+                '--extra-py-files' : delta_path, 
+                },
         }
 
 # job_run_args = {