Add generator functions

.github/workflows-templates/build_and_publish.yml → .github/workflows/build_and_publish.yml

@@ -13,6 +13,8 @@ on:
   push: 
     branches: ["main"]
   workflow_dispatch:
+  schedule: 
+    - cron: '0 9 * * *'
 
 jobs:
 
@@ -31,8 +33,11 @@ jobs:
           run: |
             export PATH="/root/.local/bin:$PATH"
             script/test
+            project process-current-year
             dataset build --all
-            dataset version auto --auto-ban major --all --publish
+            dataset version static --auto-ban major --all --publish
+            project remove-current-year-parquets
+
             
       - name: Push new data
         id: auto-commit-action

.gitignore

@@ -5,4 +5,7 @@ data/private/*
 _render/_parts
 _render/_papermills
 docs/_site
-UNKNOWN.egg-info
+UNKNOWN.egg-info
+data/interim/results/
+data/process
+data/packages/parliamentary_motions/*.parquet

data/packages/parliamentary_motions/agreements.resource.yaml

@@ -0,0 +1,74 @@
+title: Agreements
+description: dataset of agreements extracted from parliamentary debates
+custom:
+  row_count: 3234
+path: agreements.parquet
+name: agreements
+profile: data-resource
+scheme: file
+format: parquet
+hashing: md5
+encoding: utf-8
+schema:
+  fields:
+  - name: gid
+    type: string
+    description: unique identifier for the agreement - debate gid plus paragraph pid
+    constraints:
+      unique: true
+    example: uk.org.publicwhip/debate/2019-01-08b.277.4.2
+  - name: date
+    type: string
+    description: date of the debate
+    constraints:
+      unique: false
+    example: '2019-01-08'
+  - name: major_heading_id
+    type: string
+    description: ID of the major heading (if relevant)
+    constraints:
+      unique: false
+    example: uk.org.publicwhip/debate/2019-01-08b.211.0
+  - name: minor_heading_id
+    type: string
+    description: ID of the minor heading (if relevant)
+    constraints:
+      unique: false
+    example: ''
+  - name: speech_id
+    type: string
+    description: ID of the speech containing the agreement
+    constraints:
+      unique: false
+    example: uk.org.publicwhip/debate/2019-01-08b.277.4
+  - name: paragraph_pid
+    type: string
+    description: paragraph ID of the agreement
+    constraints:
+      unique: false
+    example: a100.1/12
+  - name: agreed_text
+    type: string
+    description: Text that contains the agreement
+    constraints:
+      unique: false
+    example: '(2) That a further day not later than 5 August be allotted for the consideration
+      of the following Estimate for financial year 2021-22: Foreign, Commonwealth
+      and Development Office, insofar as it relates to the spending of the Foreign,
+      Commonwealth and Development Office on Official Development Assistance and the
+      British Council.—(David Rutley.) Question agreed to.'
+  - name: motion_title
+    type: string
+    description: Title of the motion
+    constraints:
+      unique: false
+    example: ''
+  - name: chamber
+    type: string
+    description: Chamber in which the agreement was made
+    constraints:
+      unique: false
+      enum:
+      - house-of-commons
+    example: house-of-commons
+hash: b4e660d78499c28db479a8627a94f20a

data/packages/parliamentary_motions/datapackage.yaml

@@ -0,0 +1,46 @@
+name: parliamentary_motions
+title: Parliamentary motions
+description : |
+  Motions and agreements extracted from parliamentary debates
+version: 0.1.0
+licenses:
+
+  - name: CC-BY-4.0
+    path: https://creativecommons.org/licenses/by/4.0/
+    title: Creative Commons Attribution 4.0 International License
+
+contributors:
+  - title: mySociety
+    path: https://mysociety.org
+    role: author
+
+
+custom:
+  build: "parl_motion_detector.process:move_to_package"
+  tests:
+    - test_parliamentary_motions
+  dataset_order: 0
+  download_options:
+    gate: default
+    survey: default
+    header_text: default
+  formats:
+    csv: true
+    parquet: true
+    gpkg: false
+    geojson: false
+  is_geodata: false
+  composite:
+    xlsx:
+      include: all
+      exclude: none
+      render: true
+    sqlite:
+      include: all
+      exclude: none
+      render: true
+    json:
+      include: all
+      exclude: none
+      render: true
+

data/packages/parliamentary_motions/division-links.resource.yaml

@@ -0,0 +1,34 @@
+title: Division Links
+description: Lookup between GID for a division and the relevant motion
+custom:
+  row_count: 1094
+path: division-links.parquet
+name: division-links
+profile: data-resource
+scheme: file
+format: parquet
+hashing: md5
+encoding: utf-8
+schema:
+  fields:
+  - name: division_gid
+    type: string
+    description: GID of the division
+    constraints:
+      unique: true
+    example: uk.org.publicwhip/debate/2019-01-08b.243.1
+  - name: motion_gid
+    type: string
+    description: GID of the motion
+    constraints:
+      unique: true
+    example: uk.org.publicwhip/debate/2019-01-08b.212.3.1
+  - name: chamber
+    type: string
+    description: Chamber in which the motion was made
+    constraints:
+      unique: false
+      enum:
+      - house-of-commons
+    example: house-of-commons
+hash: 4f802b3551fbc111caa68a36d6d2eddc

data/packages/parliamentary_motions/motions.resource.yaml

@@ -0,0 +1,54 @@
+title: Motions
+description: Motions extracted from parliamentary debates
+custom:
+  row_count: 4328
+path: motions.parquet
+name: motions
+profile: data-resource
+scheme: file
+format: parquet
+hashing: md5
+encoding: utf-8
+schema:
+  fields:
+  - name: gid
+    type: string
+    description: unique identifier for the motion - debate gid plus paragraph pid
+    constraints:
+      unique: true
+    example: uk.org.publicwhip/debate/2019-01-08b.212.3.1
+  - name: speech_id
+    type: string
+    description: ID of the speech containing the motion (or the first entry)
+    constraints:
+      unique: false
+    example: uk.org.publicwhip/debate/2019-01-08b.212.3
+  - name: date
+    type: string
+    description: date of the debate
+    constraints:
+      unique: false
+    example: '2019-01-08'
+  - name: motion_title
+    type: string
+    description: Title of the motion
+    constraints:
+      unique: false
+    example: ''
+  - name: motion_text
+    type: string
+    description: Text of the motion
+    constraints:
+      unique: false
+    example: "After Clause 46 - Register of members: information to be included and\
+      \ powers to obtain it\nQuestion put, That amendment (a) to Lords amendment 23\
+      \ be made."
+  - name: chamber
+    type: string
+    description: Chamber in which the motion was made
+    constraints:
+      unique: false
+      enum:
+      - house-of-commons
+    example: house-of-commons
+hash: 933981b426e92431d3f5171e08d055d9

docs/index.md

@@ -3,6 +3,6 @@ title: "Download parl_motion_detector"
 layout: datasets/front
 ---
 
-# Dataset download site title
+# Parliamentary motion detection
 
-Here is some text underneath that.
+Extracted motions and agreements.