Initial commit

.github/workflows/nber.yml

@@ -0,0 +1,67 @@
+name: NBER
+
+on:
+  schedule:
+    - cron: "0 6 * * *"
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'Manual trigger'
+
+jobs:
+  printInputs:
+    runs-on: ubuntu-latest
+    steps:
+    - run: |
+        echo "Log level: ${{ github.event.inputs.logLevel }}"
+        echo "Tags: ${{ github.event.inputs.tags }}"
+  job_id:
+    name: NBER
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Run script
+      env:
+        # python argument
+        START: ${{ secrets.START }}
+        END: ${{ secrets.END }}
+        INTERVAL: ${{ secrets.INTERVAL }}
+
+        # proxy
+        PROXY_USER: ${{ secrets.PROXY_USER }}
+        PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
+        PROXY_HOST: ${{ secrets.PROXY_HOST }}
+        PROXY_PORT: ${{ secrets.PROXY_PORT }}
+        PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
+
+        # git
+        ACCESS_TOKEN: ${{ secrets.TOKEN }}
+        BRANCH: ${{ secrets.BRANCH }}
+        EMAIL: ${{ secrets.EMAIL }}
+        USERNAME: ${{ secrets.USERNAME }}
+      run: |
+        # set up git
+        git config --global user.email ${EMAIL}
+        git config --global user.name ${USERNAME}
+        git remote set-url origin https://${USERNAME}:${ACCESS_TOKEN}@github.com/${USERNAME}/nber.git
+        
+        # run script
+        python src/nber.py --start ${START} --end ${END} --interval ${INTERVAL}
+    - name: Commit changes
+      run: |
+        git add data/nber/*
+        git commit --allow-empty -m "Update nber data"
+        git pull --rebase origin ${BRANCH}
+        git push -u origin ${BRANCH}

.github/workflows/notebook.yml

@@ -0,0 +1,53 @@
+name: notebook
+
+on:
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'Manual trigger'
+
+jobs:
+  printInputs:
+    runs-on: ubuntu-latest
+    steps:
+    - run: |
+        echo "Log level: ${{ github.event.inputs.logLevel }}"
+        echo "Tags: ${{ github.event.inputs.tags }}"
+  job_id:
+    name: notebook
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        sudo apt install fonts-humor-sans
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        rm -rf ~/.cache/matplotlib
+    - name: Run script
+      env:
+        ACCESS_TOKEN: ${{ secrets.TOKEN }}
+        EMAIL: ${{ secrets.EMAIL }}
+        USERNAME: ${{ secrets.USERNAME }}
+      run: |
+        git config --global user.email ${EMAIL}
+        git config --global user.name ${USERNAME}
+        git remote set-url origin https://${USERNAME}:${ACCESS_TOKEN}@github.com/${USERNAME}/nber.git
+        jupyter nbconvert --to notebook --execute index.ipynb
+        mv index.nbconvert.ipynb index.ipynb
+    - name: Commit changes
+      env:
+        ACCESS_TOKEN: ${{ secrets.TOKEN }}
+      run: |
+        git add .
+        git commit -m "Updated notebook for use case example"
+        git pull --rebase origin main
+        git push -u origin main

.github/workflows/pdf_to_txt.yml

@@ -0,0 +1,61 @@
+name: PDF to text
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'Manual trigger'
+
+jobs:
+  printInputs:
+    runs-on: ubuntu-latest
+    steps:
+    - run: |
+        echo "Log level: ${{ github.event.inputs.logLevel }}"
+        echo "Tags: ${{ github.event.inputs.tags }}"
+  job_id:
+    name: PDF to text
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run script
+      env:
+        # bash argument
+        START: ${{ secrets.START }}
+        END: ${{ secrets.END }}
+
+        # proxy
+        PROXY_USER: ${{ secrets.PROXY_USER }}
+        PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
+        PROXY_HOST: ${{ secrets.PROXY_HOST }}
+        PROXY_PORT: ${{ secrets.PROXY_PORT }}
+        PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
+
+        # git
+        ACCESS_TOKEN: ${{ secrets.TOKEN }}
+        BRANCH: ${{ secrets.BRANCH }}
+        EMAIL: ${{ secrets.EMAIL }}
+        USERNAME: ${{ secrets.USERNAME }}
+      run: |
+        # set up git
+        git config --global user.email ${EMAIL}
+        git config --global user.name ${USERNAME}
+        git remote set-url origin https://${USERNAME}:${ACCESS_TOKEN}@github.com/${USERNAME}/nber.git
+
+        # download module for pdftotext
+        sudo apt-get install poppler-utils
+
+        # run bash script
+        bash pdf_to_txt.sh $START $END
+    - name: Commit changes
+      run: |
+        git add paper/*
+        git commit --allow-empty -m "Add paper(s)"
+        git pull --no-rebase origin ${BRANCH}
+        git push -u origin ${BRANCH}

.github/workflows/repec.yml

@@ -0,0 +1,67 @@
+name: RePEc
+
+on:
+  schedule:
+    - cron: "0 12 * * *"
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'Manual trigger'
+
+jobs:
+  printInputs:
+    runs-on: ubuntu-latest
+    steps:
+    - run: |
+        echo "Log level: ${{ github.event.inputs.logLevel }}"
+        echo "Tags: ${{ github.event.inputs.tags }}"
+  job_id:
+    name: RepEc
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Run script
+      env:
+        # python argument
+        START: ${{ secrets.START }}
+        END: ${{ secrets.END }}
+        INTERVAL: ${{ secrets.INTERVAL }}
+
+        # proxy
+        PROXY_USER: ${{ secrets.PROXY_USER }}
+        PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
+        PROXY_HOST: ${{ secrets.PROXY_HOST }}
+        PROXY_PORT: ${{ secrets.PROXY_PORT }}
+        PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
+
+        # git
+        ACCESS_TOKEN: ${{ secrets.TOKEN }}
+        BRANCH: ${{ secrets.BRANCH }}
+        EMAIL: ${{ secrets.EMAIL }}
+        USERNAME: ${{ secrets.USERNAME }}
+      run: |
+        # set up git
+        git config --global user.email ${EMAIL}
+        git config --global user.name ${USERNAME}
+        git remote set-url origin https://${USERNAME}:${ACCESS_TOKEN}@github.com/${USERNAME}/nber.git
+        
+        # run script
+        python src/repec.py --start ${START} --end ${END} --interval ${INTERVAL}
+    - name: Commit changes
+      run: |
+        git add data/repec/*
+        git commit --allow-empty -m "Update repec data"
+        git pull --rebase origin ${BRANCH}
+        git push -u origin ${BRANCH}

.github/workflows/wikipedia.yml

@@ -0,0 +1,48 @@
+name: Wikipedia
+
+on:
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+      tags:
+        description: 'Manual trigger'
+
+jobs:
+  printInputs:
+    runs-on: ubuntu-latest
+    steps:
+    - run: |
+        echo "Log level: ${{ github.event.inputs.logLevel }}"
+        echo "Tags: ${{ github.event.inputs.tags }}"
+  job_id:
+    name: Wikipedia
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.6
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Run script
+      run: |
+        python src/wikipedia.py
+    - name: Commit changes
+      env:
+        ACCESS_TOKEN: ${{ secrets.TOKEN }}
+        USER_EMAIL: ${{ secrets.USER_EMAIL }}
+        USER_NAME: ${{ secrets.USER_NAME }}
+      run: |
+        git config --global user.email ${USER_EMAIL}
+        git config --global user.name ${USER_NAME}
+        git remote set-url origin https://${USER_NAME}:${ACCESS_TOKEN}@github.com/${USER_NAME}/nber.git
+        git add -A
+        git diff-index --quiet HEAD || git commit -m "Added new NBER economist's profile from Wikipedia"
+        git pull --rebase origin main
+        git push -u origin main

.gitignore

@@ -0,0 +1,10 @@
+.DS_Store
+.ipynb_checkpoints/
+.pytest_cache/
+.vscode/
+.venv/
+__pycache__/
+pythonenv3.8/
+data/
+util/
+host.txt

LICENSE

@@ -0,0 +1,13 @@
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                    Version 2, December 2004
+
+ Copyright (C) 2004 Sam Hocevar <[email protected]>
+
+ Everyone is permitted to copy and distribute verbatim or modified
+ copies of this license document, and changing it is allowed as long
+ as the name is changed.
+
+            DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. You just DO WHAT THE FUCK YOU WANT TO.