Enforce unique

mysociety · Nov 20, 2024 · 0761b35 · 0761b35
1 parent 52f3244
commit 0761b35
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/src/parl_motion_detector/process.py b/src/parl_motion_detector/process.py
@@ -100,4 +100,13 @@ def move_to_package(data_dir: Path = data_dir):
         df = pd.concat(dfs)
         # sort by first column
         df = df.sort_values(by=df.columns[0])
+
+        # remove duplicate rows
+        df = df.drop_duplicates()
+
+        # check there are no duplicated values in the first column
+
+        if df[df.columns[0]].duplicated().sum() != 0:
+            raise ValueError("Duplicated values in the first column")
+
         df.to_parquet(package_dir / file_ending)