fix: grouping and column ordering (#153)

opentargets · Jan 27, 2023 · 645f872 · 645f872
1 parent 6a01be4
commit 645f872
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 1 deletion.
diff --git a/modules/GeneBurden.py b/modules/GeneBurden.py
@@ -100,11 +100,11 @@ def read_gene_burden_curation(curated_data: str) -> DataFrame:
             T.StructField('ancestry', T.StringType(), True),
             T.StructField('ancestryId', T.StringType(), True),
             T.StructField('cohortId', T.StringType(), True),
-            T.StructField('studyId', T.StringType(), True),
             T.StructField('studySampleSize', T.IntegerType(), True),
             T.StructField('studyCases', T.IntegerType(), True),
             T.StructField('studyCasesWithQualifyingVariants', T.IntegerType(), True),
             T.StructField('allelicRequirements', T.StringType(), True),
+            T.StructField('studyId', T.StringType(), True),
             T.StructField('statisticalMethod', T.StringType(), True),
             T.StructField('statisticalMethodOverview', T.StringType(), True),
             T.StructField('literature', T.StringType(), True),

diff --git a/modules/TargetSafety.py b/modules/TargetSafety.py
@@ -99,6 +99,18 @@ def process_aop(aopwiki: str) -> DataFrame:
         .withColumn('isHumanApplicable', F.when(F.col('isHumanApplicable') != F.lit(True), F.col("isHumanApplicable")))
         # data bug: some events have the substring "NA" at the start - removal and trim the string
         .withColumn('event', F.trim(F.regexp_replace(F.col('event'), '^NA', '')))
+        # data bug: effects.direction need to be in lowercase, this field is an enum
+        .withColumn(
+            'effect',
+            F.transform(
+                F.col('effects'),
+                lambda x: F.struct(
+                    F.lower(x.direction).alias('direction'),
+                    x.dosing.alias('dosing')).alias('effects')
+            )
+        )
+        # I need to convert the biosamples array into a struct so that data is parsed the same way as the rest of the sources
+        .withColumn('biosample', F.explode('biosamples'))
     )
 
 def process_adverse_events(adverse_events: str) -> DataFrame: