Support Python 3.6 to 3.12 and release v1.5.0 (#227)

* Update dependencies and support Python 3.6 to 3.12

* Update circleci image name

* Update requirements.txt

* Update config.yml

* Support Python 3.6 to 3.12 and release v1.5.0

.circleci/config.yml

@@ -1,20 +1,14 @@
 # Python CircleCI 2.1 configuration file
 version: 2.1
-
-
 orbs:
   codecov: codecov/[email protected]
-
-
 jobs:
-  py37: &test-template
+  py39: &test-template
     docker:
-      - image: circleci/python:3.7
+      - image: cimg/python:3.9
     environment:
       SCRAPYDWEB_TESTMODE: True
-
     working_directory: ~/repo
-
     parameters:
       is-py27:
         type: boolean
@@ -39,13 +33,11 @@ jobs:
           name: Install telnet
           command: |
             sudo apt-get update && sudo apt-get install telnet
-
       - run:
           name: Setup env
           command: |
             mkdir ~/logs
             ls -l ~
-
       - checkout
       - when:
           condition: <<parameters.is-py27>>
@@ -54,15 +46,13 @@ jobs:
                 name: Create virtual env in PY2
                 command: |
                   virtualenv ./venv
-
       - unless:
           condition: <<parameters.is-py27>>
           steps:
             - run:
                 name: Create virtual env in PY3
                 command: |
                   python3 -m venv venv
-
       - when:
           condition: <<parameters.set-data-path>>
           steps:
@@ -112,7 +102,6 @@ jobs:
                   # mysql -h 127.0.0.1 -u root -prootpw -e "create database scrapydweb_timertasks"
                   # mysql -h 127.0.0.1 -u root -prootpw -e "create database scrapydweb_metadata"
                   # mysql -h 127.0.0.1 -u root -prootpw -e "create database scrapydweb_jobs"
-
       - run:
           name: Install dependencies
           command: |
@@ -125,7 +114,6 @@ jobs:
             pip install -r requirements.txt
             pip install -r requirements-tests.txt
             pip list
-
       - when:
           condition: <<parameters.use-git>>
           steps:
@@ -138,7 +126,6 @@ jobs:
                   pip install -U git+https://github.com/scrapy/scrapyd.git
                   pip install -U git+https://github.com/my8100/logparser.git
                   pip list
-
       - run:
           name: Launch Scrapyd
           command: |
@@ -150,7 +137,6 @@ jobs:
             sleep 5
             cat ~/scrapyd.log
             cd -
-
       - run:
           name: Run tests
           command: |
@@ -160,7 +146,6 @@ jobs:
             flake8 . --count --exclude=./venv* --select=E9,F63,F7,F82 --show-source --statistics
             coverage erase
             coverage run --source=scrapydweb -m pytest tests -s -vv -l --disable-warnings
-
       - run:
           name: Generate report
           command: |
@@ -178,23 +163,28 @@ jobs:
           path: coverage.xml
       - codecov/upload:
           file: coverage.xml
-
   py27:
     <<: *test-template
     docker:
-      - image: circleci/python:2.7
-
+      - image: cimg/python:2.7
   py36:
     <<: *test-template
     docker:
-      - image: circleci/python:3.6
-
-  py38-git-postgresql:
+      - image: cimg/python:3.6
+  py37:
     <<: *test-template
     docker:
-      - image: circleci/python:3.8
-      # CircleCI PostgreSQL images available at: https://hub.docker.com/r/circleci/postgres/
-      - image: circleci/postgres:9.6-alpine
+      - image: cimg/python:3.7
+  py38:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.8
+  py310-git-postgresql:
+    <<: *test-template
+    docker:
+      - image: cimg/python:3.10
+      # https://circleci.com/developer/images/image/cimg/postgres#image-tags
+      - image: cimg/postgres:9.6
         environment:
           POSTGRES_USER: circleci
           # psycopg2.OperationalError: FATAL: database "circleci" does not exist
@@ -204,16 +194,15 @@ jobs:
     environment:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'postgresql://circleci:fakepassword@localhost:5432'
-
-  py38-git-mysql:
+  py310-git-mysql:
     <<: *test-template
     docker:
-      - image: circleci/python:3.8
+      - image: cimg/python:3.10
       # https://circleci.com/docs/2.0/postgres-config/#example-mysql-project
       # Plugin caching_sha2_password could not be loaded
-      # - image: circleci/mysql:8.0.4
-      # https://circleci.com/docs/2.0/circleci-images/#mysql
-      - image: circleci/mysql:5.7.25
+      # - image: cimg/mysql:8.0.4
+      # https://circleci.com/developer/images/image/cimg/mysql#image-tags
+      - image: cimg/mysql:5.7
         environment:
           MYSQL_ROOT_PASSWORD: rootpw
           MYSQL_DATABASE: test_db
@@ -222,22 +211,19 @@ jobs:
     environment:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'mysql://root:[email protected]:3306'
-
-  py38-sqlite:
+  py310-sqlite:
     <<: *test-template
     docker:
-      - image: circleci/python:3.8
+      - image: cimg/python:3.10
     environment:
       SCRAPYDWEB_TESTMODE: True
       DATA_PATH: '/home/circleci/repo/scrapydweb_data'
       DATABASE_URL: 'sqlite:////home/circleci/repo/scrapydweb_database'
-
-  py38-postgresql:
+  py310-postgresql:
     <<: *test-template
     docker:
-      - image: circleci/python:3.8
-      # CircleCI PostgreSQL images available at: https://hub.docker.com/r/circleci/postgres/
-      - image: circleci/postgres:9.6-alpine
+      - image: cimg/python:3.10
+      - image: cimg/postgres:9.6
         environment:
           POSTGRES_USER: circleci
           # psycopg2.OperationalError: FATAL: database "circleci" does not exist
@@ -247,16 +233,14 @@ jobs:
     environment:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'postgresql://circleci:fakepassword@localhost:5432'
-
-  py38-mysql:
+  py310-mysql:
     <<: *test-template
     docker:
-      - image: circleci/python:3.8
+      - image: cimg/python:3.10
       # https://circleci.com/docs/2.0/postgres-config/#example-mysql-project
       # Plugin caching_sha2_password could not be loaded
-      # - image: circleci/mysql:8.0.4
-      # https://circleci.com/docs/2.0/circleci-images/#mysql
-      - image: circleci/mysql:5.7.25
+      # - image: cimg/mysql:8.0.4
+      - image: cimg/mysql:5.7
         environment:
           MYSQL_ROOT_PASSWORD: rootpw
           MYSQL_DATABASE: test_db
@@ -265,43 +249,35 @@ jobs:
     environment:
       SCRAPYDWEB_TESTMODE: True
       DATABASE_URL: 'mysql://root:[email protected]:3306'
-
-  py39:
+  py311:
     <<: *test-template
     docker:
-      - image: circleci/python:3.9
-
-  py310:
+      - image: cimg/python:3.11
+  py312:
     <<: *test-template
     docker:
-      - image: circleci/python:3.10
-
-
+      - image: cimg/python:3.12
 workflows:
   test:
     jobs:
       # - py27:
       #    is-py27: true
-
       - py36
-
       - py37
-
-      - py38-git-postgresql:
+      - py38
+      - py39
+      - py310-git-postgresql:
           use-git: true
           use-postgresql: true
-      - py38-git-mysql:
+      - py310-git-mysql:
           use-git: true
           use-mysql: true
-
-      - py38-sqlite:
+      - py310-sqlite:
           set-data-path: true
           use-sqlite: true
-      - py38-postgresql:
+      - py310-postgresql:
           use-postgresql: true
-      - py38-mysql:
+      - py310-mysql:
           use-mysql: true
-
-      - py39
-
-      - py310
+      - py311
+      - py312

.github/ISSUE_TEMPLATE/bug_report.md

@@ -28,8 +28,8 @@ If applicable, add screenshots to help explain your problem.
 
 **Environment (please complete the following information):**
  - Operating system: [e.g. Win 10, macOS 10.14, Ubuntu 18, centOS 7.6, Debian 9.6 or Fedora 29]
- - Python version: [e.g. 2.7 or 3.8]
- - ScrapydWeb version: [e.g. 1.4.1 or latest code on GitHub]
+ - Python version: [e.g. 3.6 or 3.12]
+ - ScrapydWeb version: [e.g. 1.5.0 or latest code on GitHub]
  - ScrapydWeb related settings [e.g. 'ENABLE_AUTH = True']
  - Scrapyd version: [e.g. 1.2.1 or latest code on GitHub]
  - Scrapyd amount [e.g. 1 or 5]

HISTORY.md

@@ -1,5 +1,11 @@
 Release History
 ===============
+1.5.0 (2024-02-11)
+------------------
+- Update dependencies like Flask==2.0.0 
+- Support Python 3.6 to 3.12
+
+
 1.4.1 (2023-08-31)
 ------------------
 - Fix dependencies and release v1.4.1

requirements.txt

@@ -1,22 +1,22 @@
 pip>=19.1.1
-APScheduler==3.5.3
-click==7.0
+APScheduler==3.6.0
+click==7.1.2
 colorama==0.4.0
-Flask==1.0.2
+Flask==2.0.0
 Flask-Compress==1.4.0
 Flask-SQLAlchemy==2.4.0
 idna==2.7
-itsdangerous==1.1.0
-Jinja2==2.10
+itsdangerous==2.0.0
+Jinja2==3.0.0
 logparser==0.8.2
-MarkupSafe==1.1.1
+MarkupSafe==2.0.0
 pexpect==4.7.0
 ptyprocess==0.6.0
 pytz==2018.9
 requests>=2.21.0
 setuptools>=40.6.3
-six==1.12.0
+six==1.16.0
 SQLAlchemy==1.3.24
 tzlocal==1.5.1
-w3lib==1.19.0
-Werkzeug==0.14.1
+w3lib==2.0.0
+Werkzeug==2.0.0

scrapydweb/__version__.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 __title__ = 'scrapydweb'
-__version__ = '1.4.1'
+__version__ = '1.5.0'
 __author__ = 'my8100'
 __author_email__ = '[email protected]'
 __url__ = 'https://github.com/my8100/scrapydweb'

scrapydweb/views/operations/scrapyd_deploy.py

@@ -12,7 +12,10 @@
 import tempfile
 
 from flask import current_app as app
-from six.moves.configparser import SafeConfigParser
+try:
+    from six.moves.configparser import SafeConfigParser
+except ImportError:
+    from six.moves.configparser import RawConfigParser as SafeConfigParser
 
 
 _SETUP_PY_TEMPLATE = """# Automatically created by: scrapydweb x scrapyd-client

setup.py

@@ -31,29 +31,29 @@
     packages=find_packages(exclude=("tests", )),
     include_package_data=True,
     zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, <=3.9",
+    python_requires=">=3.6",
     install_requires=[
-        "APScheduler==3.5.3",  # Aug 15, 2018
-        "click==7.0",  # Sep 26, 2018
+        "APScheduler==3.6.0",  # Mar 18, 2019
+        "click==7.1.2",  # Apr 28, 2020
         "colorama==0.4.0",  # Oct 10, 2018
-        "Flask==1.0.2",  # May 2, 2018
+        "Flask==2.0.0",  # May 12, 2021
         "Flask-Compress==1.4.0",  # Jan 5, 2017
         "Flask-SQLAlchemy==2.4.0",  # Apr 25, 2019
-        "idna==2.7",  # Jun 11, 2018    
-        "itsdangerous==1.1.0",  # Oct 27, 2018
-        "Jinja2==2.10",  # Nov 9, 2017
+        "idna==2.7",  # Jun 11, 2018
+        "itsdangerous==2.0.0",  # May 12, 2021
+        "Jinja2==3.0.0",  # May 12, 2021
         "logparser==0.8.2",
-        "MarkupSafe==1.1.1",  # Feb 24, 2019
+        "MarkupSafe==2.0.0",  # May 12, 2021
         "pexpect==4.7.0",  # Apr 7, 2019
         "ptyprocess==0.6.0",  # Jun 22, 2018
         "pytz==2018.9",  # Jan 7, 2019
         "requests>=2.21.0",  # Dec 10, 2018
         "setuptools>=40.6.3",  # Dec 11, 2018
-        "six==1.12.0",  # Dec 10, 2018
+        "six==1.16.0",  # May 5, 2021
         "SQLAlchemy==1.3.24",  # Mar 31, 2021
         "tzlocal==1.5.1",  # Dec 1, 2017
-        "w3lib==1.19.0",  # Jan 25, 2018
-        "Werkzeug==0.14.1",  # Jan 1, 2018
+        "w3lib==2.0.0",  # Aug 11, 2022
+        "Werkzeug==2.0.0",  # May 12, 2021
     ],
 
     entry_points={
@@ -64,10 +64,12 @@
 
     classifiers=[
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ]
 )