diff --git a/src/votes/models/base_model.py b/src/twfy_votes/helpers/base_model.py similarity index 96% rename from src/votes/models/base_model.py rename to src/twfy_votes/helpers/base_model.py index 6e4ddf9..ffc8979 100644 --- a/src/votes/models/base_model.py +++ b/src/twfy_votes/helpers/base_model.py @@ -6,8 +6,8 @@ from django.conf import settings from django.db import connection -from twfy_votes.helpers.duck import sync_to_postgres -from twfy_votes.helpers.typed_django.models import ModelType, TypedModel +from .duck import sync_to_postgres +from .typed_django.models import ModelType, TypedModel @contextmanager diff --git a/src/votes/management/commands/vr_validator.py b/src/votes/management/commands/vr_validator.py index fbf6a3e..122e36e 100644 --- a/src/votes/management/commands/vr_validator.py +++ b/src/votes/management/commands/vr_validator.py @@ -18,8 +18,14 @@ from typing_extensions import Self from ...consts import PolicyDirection, PolicyStrength, VotePosition -from ...models.decisions import Chamber, Policy, PolicyComparisonPeriod -from ...models.people import Membership, Organization, Person +from ...models import ( + Chamber, + Membership, + Organization, + Person, + Policy, + PolicyComparisonPeriod, +) from ...populate.policycalc import PolicyPivotTable, get_connected_duck diff --git a/src/votes/models/decisions.py b/src/votes/models.py similarity index 82% rename from src/votes/models/decisions.py rename to src/votes/models.py index 29ac1bb..17264c1 100644 --- a/src/votes/models/decisions.py +++ b/src/votes/models.py @@ -2,29 +2,33 @@ import datetime from dataclasses import dataclass -from typing import Optional, Protocol, Type, TypeVar +from typing import TYPE_CHECKING, Optional, Protocol, Type, TypeVar from django.db import models from django.urls import reverse import markdown +import numpy as np import pandas as pd from numpy import nan +from twfy_votes.helpers.base_model import DjangoVoteModel from twfy_votes.helpers.typed_django.models import ( DoNothingForeignKey, Dummy, DummyManyToMany, DummyOneToMany, ManyToMany, + PrimaryKey, TextField, field, related_name, ) -from ..consts import ( +from .consts import ( ChamberSlug, MotionType, + OrganisationType, PolicyDirection, PolicyGroupSlug, PolicyStatus, @@ -35,12 +39,19 @@ TagType, VotePosition, ) -from ..policy_generation.scoring import ( +from .policy_generation.scoring import ( ScoringFuncProtocol, SimplifiedScore, ) -from .base_model import DjangoVoteModel -from .people import Membership, Organization, Person + +if TYPE_CHECKING: + from .models import ( + Chamber, + PolicyComparisonPeriod, + RebellionRate, + Vote, + VoteDistribution, + ) @dataclass @@ -84,6 +95,170 @@ def is_valid_decision_model( return klass +@dataclass +class DistributionGroup: + party: Organization + chamber: Chamber + period: PolicyComparisonPeriod + + def key(self): + return f"{self.party.id}-{self.chamber.id}-{self.period.id}" + + +class Person(DjangoVoteModel): + id: PrimaryKey = None + name: str + memberships: DummyOneToMany["Membership"] = related_name("person") + votes: DummyOneToMany[Vote] = related_name("person") + vote_distributions: DummyOneToMany[VoteDistribution] = related_name("person") + rebellion_rates: DummyOneToMany[RebellionRate] = related_name("person") + + def str_id(self): + return f"uk.org.publicwhip/person/{self.id}" + + def votes_url(self, year: str = "all"): + return reverse("person_votes", kwargs={"person_id": self.id, "year": year}) + + def rebellion_rate_df(self): + items = self.rebellion_rates.filter( + period_type=RebellionPeriodType.YEAR + ).order_by("-period_number") + df = pd.DataFrame( + [ + { + "Year": UrlColumn( + reverse("person_votes", args=[self.id, r.period_number]), + str(r.period_number), + ), + "Party alignment": 1 - r.value, + "Total votes": r.total_votes, + } + for r in items + ] + ) + + return df + + def policy_distribution_groups(self): + groups: list[DistributionGroup] = [] + distributions = self.vote_distributions.all().prefetch_related( + "period", "chamber", "party" + ) + # iterate through this and create unique groups + + existing_keys = [] + + for distribution in distributions: + group = DistributionGroup( + party=distribution.party, + chamber=distribution.chamber, + period=distribution.period, + ) + if group.key() not in existing_keys: + groups.append(group) + existing_keys.append(group.key()) + + return groups + + @classmethod + def current(cls): + """ + Those with a membership that is current. + """ + return cls.objects.filter(memberships__end_date__gte=datetime.date.today()) + + def membership_in_chamber_on_date( + self, chamber_slug: ChamberSlug, date: datetime.date + ) -> Membership: + membership = self.memberships.filter( + chamber_slug=chamber_slug, start_date__lte=date, end_date__gte=date + ).first() + if membership: + return membership + else: + raise ValueError( + f"{self.name} was not a member of {chamber_slug} on {date}" + ) + + def votes_df(self, year: int | None = None) -> pd.DataFrame: + if year: + votes_query = self.votes.filter(division__date__year=year) + else: + votes_query = self.votes.all() + + data = [ + { + "Date": v.division.date, + "Division": UrlColumn( + url=v.division.url(), text=v.division.division_name + ), + "Vote": v.vote_desc(), + "Party alignment": ( + 1 + - ( + v.diff_from_party_average + if v.diff_from_party_average is not None + else np.nan + ) + ), + } + for v in votes_query + if v.division is not None + ] + + # sort by data decending + data = sorted(data, key=lambda x: x["Date"], reverse=True) + + return pd.DataFrame(data=data) + + +class Organization(DjangoVoteModel): + id: PrimaryKey = None + slug: str + name: str + classification: OrganisationType = OrganisationType.UNKNOWN + org_memberships: DummyOneToMany["Membership"] = related_name("organization") + party_memberships: DummyOneToMany["Membership"] = related_name("on_behalf_of") + + +class OrgMembershipCount(DjangoVoteModel): + chamber_slug: ChamberSlug + chamber_id: Dummy[int] = 0 + chamber: DoNothingForeignKey[Organization] = related_name("org_membership_counts") + start_date: datetime.date + end_date: datetime.date + count: int + + +class Membership(DjangoVoteModel): + """ + A timed connection between a person and a post. + """ + + id: PrimaryKey = None + person_id: Dummy[int] = 0 + person: DoNothingForeignKey[Person] = related_name("memberships") + start_date: datetime.date + end_date: datetime.date + party_slug: str + effective_party_slug: str + party_id: Dummy[Optional[int]] = None + party: DoNothingForeignKey[Organization] = field( + default=None, null=True, related_name="party_memberships" + ) + effective_party_id: Dummy[Optional[int]] = None + effective_party: DoNothingForeignKey[Organization] = field( + default=None, null=True, related_name="effective_party_memberships" + ) + chamber_id: Dummy[Optional[int]] = None + chamber: DoNothingForeignKey[Chamber] = field( + default=None, null=True, related_name="org_memberships" + ) + chamber_slug: str + post_label: str + area_name: str + + class Chamber(DjangoVoteModel): slug: ChamberSlug member_plural: str diff --git a/src/votes/models/__init__.py b/src/votes/models/__init__.py deleted file mode 100644 index 0f2e306..0000000 --- a/src/votes/models/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# reexport all models from .decisions -from . import decisions as decisions -from . import people as people diff --git a/src/votes/models/people.py b/src/votes/models/people.py deleted file mode 100644 index 36b90a2..0000000 --- a/src/votes/models/people.py +++ /dev/null @@ -1,199 +0,0 @@ -from __future__ import annotations - -import datetime -from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional - -from django.urls import reverse - -import numpy as np -import pandas as pd - -from twfy_votes.helpers.typed_django.models import ( - DoNothingForeignKey, - Dummy, - DummyOneToMany, - PrimaryKey, - field, - related_name, -) - -from ..consts import ChamberSlug, OrganisationType, RebellionPeriodType -from .base_model import DjangoVoteModel - -if TYPE_CHECKING: - from .decisions import ( - Chamber, - PolicyComparisonPeriod, - RebellionRate, - Vote, - VoteDistribution, - ) - - -@dataclass -class DistributionGroup: - party: Organization - chamber: Chamber - period: PolicyComparisonPeriod - - def key(self): - return f"{self.party.id}-{self.chamber.id}-{self.period.id}" - - -class Person(DjangoVoteModel): - id: PrimaryKey = None - name: str - memberships: DummyOneToMany["Membership"] = related_name("person") - votes: DummyOneToMany[Vote] = related_name("person") - vote_distributions: DummyOneToMany[VoteDistribution] = related_name("person") - rebellion_rates: DummyOneToMany[RebellionRate] = related_name("person") - - def str_id(self): - return f"uk.org.publicwhip/person/{self.id}" - - def votes_url(self): - return reverse("person_votes", kwargs={"person_id": self.id}) - - def rebellion_rate_df(self): - from .decisions import UrlColumn - - items = self.rebellion_rates.filter( - period_type=RebellionPeriodType.YEAR - ).order_by("-period_number") - df = pd.DataFrame( - [ - { - "Year": UrlColumn( - reverse("person_votes", args=[self.id, r.period_number]), - str(r.period_number), - ), - "Party alignment": 1 - r.value, - "Total votes": r.total_votes, - } - for r in items - ] - ) - - return df - - def policy_distribution_groups(self): - groups: list[DistributionGroup] = [] - distributions = self.vote_distributions.all().prefetch_related( - "period", "chamber", "party" - ) - # iterate through this and create unique groups - - existing_keys = [] - - for distribution in distributions: - group = DistributionGroup( - party=distribution.party, - chamber=distribution.chamber, - period=distribution.period, - ) - if group.key() not in existing_keys: - groups.append(group) - existing_keys.append(group.key()) - - return groups - - @classmethod - def current(cls): - """ - Those with a membership that is current. - """ - return cls.objects.filter(memberships__end_date__gte=datetime.date.today()) - - def membership_in_chamber_on_date( - self, chamber_slug: ChamberSlug, date: datetime.date - ) -> Membership: - membership = self.memberships.filter( - chamber_slug=chamber_slug, start_date__lte=date, end_date__gte=date - ).first() - if membership: - return membership - else: - raise ValueError( - f"{self.name} was not a member of {chamber_slug} on {date}" - ) - - def votes_df(self, year: int | None = None) -> pd.DataFrame: - from .decisions import UrlColumn - - if year: - votes_query = self.votes.filter(division__date__year=year) - else: - votes_query = self.votes.all() - - data = [ - { - "Date": v.division.date, - "Division": UrlColumn( - url=v.division.url(), text=v.division.division_name - ), - "Vote": v.vote_desc(), - "Party alignment": ( - 1 - - ( - v.diff_from_party_average - if v.diff_from_party_average is not None - else np.nan - ) - ), - } - for v in votes_query - if v.division is not None - ] - - # sort by data decending - data = sorted(data, key=lambda x: x["Date"], reverse=True) - - return pd.DataFrame(data=data) - - -class Organization(DjangoVoteModel): - id: PrimaryKey = None - slug: str - name: str - classification: OrganisationType = OrganisationType.UNKNOWN - org_memberships: DummyOneToMany["Membership"] = related_name("organization") - party_memberships: DummyOneToMany["Membership"] = related_name("on_behalf_of") - - -class OrgMembershipCount(DjangoVoteModel): - chamber_slug: ChamberSlug - chamber_id: Dummy[int] = 0 - chamber: DoNothingForeignKey[Organization] = related_name("org_membership_counts") - start_date: datetime.date - end_date: datetime.date - count: int - - -class Membership(DjangoVoteModel): - """ - A timed connection between a person and a post. - """ - - id: PrimaryKey = None - person_id: Dummy[int] = 0 - person: DoNothingForeignKey[Person] = related_name("memberships") - start_date: datetime.date - end_date: datetime.date - party_slug: str - effective_party_slug: str - party_id: Dummy[Optional[int]] = None - party: DoNothingForeignKey[Organization] = field( - default=None, null=True, related_name="party_memberships" - ) - effective_party_id: Dummy[Optional[int]] = None - effective_party: DoNothingForeignKey[Organization] = field( - default=None, null=True, related_name="effective_party_memberships" - ) - chamber_id: Dummy[Optional[int]] = None - chamber: DoNothingForeignKey[Chamber] = field( - default=None, null=True, related_name="org_memberships" - ) - chamber_slug: str - post_label: str - area_name: str diff --git a/src/votes/populate/agreements.py b/src/votes/populate/agreements.py index 3210bf8..a9df2aa 100644 --- a/src/votes/populate/agreements.py +++ b/src/votes/populate/agreements.py @@ -8,7 +8,7 @@ from twfy_votes.helpers.duck import DuckQuery from ..consts import ChamberSlug -from ..models.decisions import Agreement, Chamber +from ..models import Agreement, Chamber from .register import ImportOrder, import_register BASE_DIR = Path(settings.BASE_DIR) diff --git a/src/votes/populate/breakdowns.py b/src/votes/populate/breakdowns.py index 7494345..6e2b06f 100644 --- a/src/votes/populate/breakdowns.py +++ b/src/votes/populate/breakdowns.py @@ -8,13 +8,11 @@ from twfy_votes.helpers.duck import DuckQuery -from ..models.decisions import ( +from ..models import ( Division, DivisionBreakdown, DivisionPartyBreakdown, DivisionsIsGovBreakdown, -) -from ..models.people import ( Organization, ) from .register import ImportOrder, import_register diff --git a/src/votes/populate/chambers.py b/src/votes/populate/chambers.py index 68dd958..a71047f 100644 --- a/src/votes/populate/chambers.py +++ b/src/votes/populate/chambers.py @@ -1,7 +1,7 @@ import rich from ..consts import ChamberSlug -from ..models.decisions import Chamber +from ..models import Chamber from .register import ImportOrder, import_register member_name_lookup = { diff --git a/src/votes/populate/cluster_analysis.py b/src/votes/populate/cluster_analysis.py index 356aa08..ede70b7 100644 --- a/src/votes/populate/cluster_analysis.py +++ b/src/votes/populate/cluster_analysis.py @@ -10,7 +10,7 @@ from twfy_votes.helpers.duck import DuckQuery from ..consts import TagType -from ..models.decisions import Division, DivisionTag +from ..models import Division, DivisionTag from .register import ImportOrder, import_register BASE_DIR = Path(settings.BASE_DIR) diff --git a/src/votes/populate/comparison_periods.py b/src/votes/populate/comparison_periods.py index e5e0c26..23e8beb 100644 --- a/src/votes/populate/comparison_periods.py +++ b/src/votes/populate/comparison_periods.py @@ -5,7 +5,7 @@ import rich import tomllib -from ..models.decisions import Chamber, PolicyComparisonPeriod +from ..models import Chamber, PolicyComparisonPeriod from .register import ImportOrder, import_register diff --git a/src/votes/populate/divisions.py b/src/votes/populate/divisions.py index cc1d544..aa4ef93 100644 --- a/src/votes/populate/divisions.py +++ b/src/votes/populate/divisions.py @@ -9,7 +9,7 @@ from twfy_votes.helpers.duck import DuckQuery from ..consts import ChamberSlug -from ..models.decisions import Chamber, Division +from ..models import Chamber, Division from .register import ImportOrder, import_register duck = DuckQuery(postgres_database_settings=settings.DATABASES["default"]) diff --git a/src/votes/populate/government_parties.py b/src/votes/populate/government_parties.py index fc73625..3cf7628 100644 --- a/src/votes/populate/government_parties.py +++ b/src/votes/populate/government_parties.py @@ -5,7 +5,7 @@ import rich import tomllib -from ..models.decisions import Chamber, GovernmentParty +from ..models import Chamber, GovernmentParty from .register import ImportOrder, import_register diff --git a/src/votes/populate/motions.py b/src/votes/populate/motions.py index 02f8277..d77e9a2 100644 --- a/src/votes/populate/motions.py +++ b/src/votes/populate/motions.py @@ -7,7 +7,7 @@ import rich from ..consts import MotionType -from ..models.decisions import Motion +from ..models import Motion from .register import ImportOrder, import_register BASE_DIR = Path(settings.BASE_DIR) diff --git a/src/votes/populate/people.py b/src/votes/populate/people.py index e946bdd..2844770 100644 --- a/src/votes/populate/people.py +++ b/src/votes/populate/people.py @@ -11,8 +11,7 @@ from mysoc_validator.models.popolo import Person as PopoloPerson from ..consts import ChamberSlug, OrganisationType -from ..models.decisions import Chamber -from ..models.people import Membership, Organization, OrgMembershipCount, Person +from ..models import Chamber, Membership, Organization, OrgMembershipCount, Person from .register import ImportOrder, import_register diff --git a/src/votes/populate/policy.py b/src/votes/populate/policy.py index a672c1b..881d21a 100644 --- a/src/votes/populate/policy.py +++ b/src/votes/populate/policy.py @@ -8,12 +8,12 @@ import rich from ruamel.yaml import YAML +from twfy_votes.helpers.base_model import disable_constraints from twfy_votes.policy.models import ( PartialPolicy, ) -from ..models.base_model import disable_constraints -from ..models.decisions import ( +from ..models import ( Agreement, Chamber, Division, diff --git a/src/votes/populate/policy_groups.py b/src/votes/populate/policy_groups.py index 23fdcd3..a4c59a9 100644 --- a/src/votes/populate/policy_groups.py +++ b/src/votes/populate/policy_groups.py @@ -1,7 +1,7 @@ import rich from ..consts import PolicyGroupSlug -from ..models.decisions import PolicyGroup +from ..models import PolicyGroup from .register import ImportOrder, import_register description_lookup = { diff --git a/src/votes/populate/policycalc.py b/src/votes/populate/policycalc.py index c95352d..054d5f1 100644 --- a/src/votes/populate/policycalc.py +++ b/src/votes/populate/policycalc.py @@ -15,7 +15,7 @@ from twfy_votes.helpers.duck import DuckQuery from twfy_votes.helpers.duck.funcs import query_to_parquet from twfy_votes.helpers.duck.templates import EnforceIntJinjaQuery -from votes.models.decisions import Policy, VoteDistribution +from votes.models import Policy, VoteDistribution from votes.policy_generation.scoring import ScoreFloatPair from .register import ImportOrder, import_register diff --git a/src/votes/populate/rebellions.py b/src/votes/populate/rebellions.py index 7ce01fc..857d89e 100644 --- a/src/votes/populate/rebellions.py +++ b/src/votes/populate/rebellions.py @@ -8,7 +8,7 @@ from twfy_votes.helpers.duck import DuckQuery from ..consts import RebellionPeriodType -from ..models.decisions import RebellionRate +from ..models import RebellionRate from .register import ImportOrder, import_register BASE_DIR = Path(settings.BASE_DIR) diff --git a/src/votes/populate/votes.py b/src/votes/populate/votes.py index c95fc24..ba764ef 100644 --- a/src/votes/populate/votes.py +++ b/src/votes/populate/votes.py @@ -9,7 +9,7 @@ from twfy_votes.helpers.duck import DuckQuery from ..consts import VotePosition -from ..models.decisions import Division, Vote +from ..models import Division, Vote from .register import ImportOrder, import_register BASE_DIR = Path(settings.BASE_DIR) diff --git a/src/votes/templatetags/hub_filters.py b/src/votes/templatetags/hub_filters.py index d1eb6f8..62751ed 100644 --- a/src/votes/templatetags/hub_filters.py +++ b/src/votes/templatetags/hub_filters.py @@ -117,6 +117,8 @@ def format_percentage(value: float): # if value is na return "-" if pd.isna(value): return "-" + if isinstance(value, str): + return value return "{:.2%}".format(value) df = df.rename(columns=nice_headers) @@ -126,8 +128,8 @@ def format_percentage(value: float): raise ValueError(f"Column {p} not found in DataFrame") styled_df = df.style.hide(axis="index").format( - {x: format_percentage for x in percentage_columns}, - precision=2, # type: ignore + {x: format_percentage for x in percentage_columns}, # type: ignore + precision=2, ) return mark_safe(styled_df.to_html()) # type: ignore diff --git a/src/votes/views/api.py b/src/votes/views/api.py index 5842d27..70ae8cc 100644 --- a/src/votes/views/api.py +++ b/src/votes/views/api.py @@ -7,12 +7,13 @@ from pydantic import BaseModel from ..consts import PolicyStatus -from ..models.decisions import ( +from ..models import ( Agreement, Division, DivisionBreakdown, DivisionPartyBreakdown, DivisionsIsGovBreakdown, + Person, Policy, PolicyAgreementLink, PolicyDivisionLink, @@ -20,7 +21,6 @@ Vote, VoteDistribution, ) -from ..models.people import Person from .helper_models import PairedPolicy, PolicyDisplayGroup, PolicyReport from .twfy_bridge import PopoloPolicy diff --git a/src/votes/views/helper_models.py b/src/votes/views/helper_models.py index 65612d0..8af94b2 100644 --- a/src/votes/views/helper_models.py +++ b/src/votes/views/helper_models.py @@ -16,7 +16,7 @@ PolicyStrength, PowersAnalysis, ) -from ..models.decisions import ( +from ..models import ( Agreement, Chamber, Division, diff --git a/src/votes/views/twfy_bridge.py b/src/votes/views/twfy_bridge.py index 1e7998e..bddc022 100644 --- a/src/votes/views/twfy_bridge.py +++ b/src/votes/views/twfy_bridge.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict from ..consts import ChamberSlug, PolicyDirection, PolicyStrength, VotePosition -from ..models.decisions import ( +from ..models import ( Chamber, Division, DivisionBreakdown, diff --git a/src/votes/views/views.py b/src/votes/views/views.py index c8974da..6116378 100644 --- a/src/votes/views/views.py +++ b/src/votes/views/views.py @@ -17,17 +17,18 @@ from twfy_votes.helpers.routes import RouteApp from ..consts import ChamberSlug, PolicyStatus -from ..models.decisions import ( +from ..models import ( Agreement, Chamber, Division, + Organization, + Person, Policy, PolicyAgreementLink, PolicyComparisonPeriod, PolicyDivisionLink, Vote, ) -from ..models.people import Organization, Person from .helper_models import ( ChamberPolicyGroup, DivisionSearch,