Add testing action (#7)

- Add automated testing action
- Import chambers first.
- Add various debug catches..

.github/workflows/test.yml

@@ -0,0 +1,25 @@
+# Run linting and basic file format checks
+
+name: Test
+
+on:
+  pull_request:
+  workflow_call:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run tests and try and build project
+        uses: mysociety/run-in-devcontainer@v1
+        with:
+          dockerfile: Dockerfile
+          app: web
+          run: |
+            script/setup
+            script/populate --all
+            script/test
+     
+

docker-compose.yml

@@ -10,13 +10,13 @@ services:
       POSTGRES_PASSWORD: 'password'
       POSTGRES_DB: 'twfy-votes'
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
       interval: 5s
       timeout: 5s
       retries: 5
     working_dir: /app
   web:
-    image: mysociety/twfy-votes:${TAG:-latest}
+    image: mysociety/twfy-votes-django:${TAG:-latest}
     build: .
     command: /app/script/server --development
     volumes:
@@ -42,4 +42,4 @@ services:
 volumes:
   pgdata:
   build:
-  node_modules:
+  node_modules:

src/votes/populate/commons_votes.py

@@ -51,7 +51,7 @@ class Member(BaseModel):
     PartyAbbreviation: Optional[str]
     MemberFrom: str
     ListAs: Optional[str]
-    ProxyName: None
+    ProxyName: Optional[str]
 
 
 class Division(BaseModel):

src/votes/populate/divisions.py

@@ -62,7 +62,7 @@ class api_divisions_with_total_membership:
         LEFT JOIN org_membership_count on
             (CAST(api_divisions.division_date as DATE) between org_membership_count.start_date
             and org_membership_count.end_date
-            and api_divisions.chamber = org_membership_count.chamber_slug)
+            and CAST(api_divisions.chamber as STRING) = org_membership_count.chamber_slug)
         """
 
 

src/votes/populate/people.py

@@ -265,6 +265,10 @@ def import_popolo(quiet: bool = False):
             chamber_slug = ChamberSlug.from_parlparse(
                 post_org_or_self_org(membership), passthrough=True
             )
+            if chamber_slug in ["crown", "london-assembly"]:
+                chamber_id = 0
+            else:
+                chamber_id = chamber_slug_lookup[chamber_slug]
             post = membership.post()
             post_label = post.role if post else ""
             area_name = post.area.name if post else ""
@@ -278,7 +282,7 @@ def import_popolo(quiet: bool = False):
                 effective_party_slug=effective_party,
                 party_id=org_slug_lookup.get(membership.on_behalf_of_id or ""),
                 effective_party_id=org_slug_lookup.get(effective_party),
-                chamber_id=chamber_slug_lookup.get(chamber_slug),
+                chamber_id=chamber_id,
                 chamber_slug=chamber_slug,
                 area_name=area_name,
                 post_label=post_label,

src/votes/populate/policycalc.py

@@ -524,6 +524,9 @@ def generate_policy_distributions(
             .df()
         )
 
+        if len(df) == 0:
+            raise ValueError("No policy calc information returned")
+
         list_cols = ["num_comparators", "division_ids"]
         for col in df.columns:
             if col not in list_cols:
@@ -547,6 +550,18 @@ def run_policy_calculations(
 ):
     partial_update = update_since is not None
 
+    sources = [
+        policy_divisions_relevant,
+        policy_agreements_relevant,
+        policy_votes_relevant,
+        policy_collective_relevant,
+        pw_relevant_people,
+    ]
+
+    for s in sources:
+        if s.source.exists() is False:
+            raise ValueError(f"{s.source} not present")
+
     count = generate_policy_distributions(update_from_hash=partial_update, quiet=quiet)
 
     if not quiet:

src/votes/populate/register.py

@@ -22,8 +22,8 @@ def accepts_argument(func: Callable[[Any], Any], arg_name: str):
 
 class ImportOrder(IntEnum):
     DOWNLOAD = 1
+    CHAMBERS = 4
     PEOPLE = 5
-    CHAMBERS = 9
     LOOKUPS = 10
     API_VOTES = 14
     DECISIONS = 15