From 393ddc9b7885acd36b7475d0c8f7fe7cdf4134e7 Mon Sep 17 00:00:00 2001 From: wkpn Date: Thu, 27 Feb 2020 20:08:09 +0300 Subject: [PATCH] 2.0.3 --- CHANGELOG-ru.rst | 57 +++++++++++++++++++ CHANGELOG.rst | 18 ++++++ README-ru.rst | 132 +++++++++++++++++++++++++++++++++++++++++++ README.rst | 9 +++ aio2ch/__init__.py | 2 +- aio2ch/api.py | 39 ++++++++++++- aio2ch/exceptions.py | 8 +++ aio2ch/helpers.py | 32 +++++++++++ setup.py | 3 +- tests/conftest.py | 10 ++++ tests/test_api.py | 89 ++++++++++++++++++++++++----- 11 files changed, 380 insertions(+), 19 deletions(-) create mode 100644 CHANGELOG-ru.rst create mode 100644 README-ru.rst create mode 100644 aio2ch/helpers.py diff --git a/CHANGELOG-ru.rst b/CHANGELOG-ru.rst new file mode 100644 index 0000000..e019a70 --- /dev/null +++ b/CHANGELOG-ru.rst @@ -0,0 +1,57 @@ +Изменения +========= + +`2.0.3` +------- + +* ``get_thread_posts`` и ``get_thread_media`` теперь принимают тред по адресу + +.. code-block:: python + + >>> thread_media = await client.get_thread_media('https://2ch.hk/test/res/30972.html') + +* добавлен список досок и соответствующие проверки +* добавлены новые исключения +* больше тестов +* почищен код + +`2.0.2` +------- + +* добавлены докстринги +* у проекта теперь есть логотип (может измениться) +* ``api_client`` перенесен в отдельный модуль + +`2.0.1` +------- + +* уменьшение количества потребляемой памяти за счет использования кортежей вместо списков +* улучшено покрытие тестами +* другие api эндпоинты +* больше аннотаций типов +* чистка кода + +`2.0` +----- + +* Апи клиент теперь может быть использован как менеджер контекста +* f-строки везде +* ``aiohttp`` заменен на ``httpx`` +* аннотации типов +* ``download_thread_media`` теперь стримит файлы вместо полной загрузки + +`1.4.3.1` +--------- + +* небольшой рефакторинг импортов (теперь можно писать ``from aio2ch import Api``) +* по дефолту все методы не возвращают ``status``, надо юзать ``return_status=True`` для его получения + + +`1.4.3` +------- + +* добавлен параметр ``keywords`` к методу ``get_board_threads`` +* добавлен метод для скачивания медиа из треда ``download_thread_media`` + +.. _changelog: https://github.com/wkpn/aio2ch/CHANGELOG-ru.rst +.. _readme: https://github.com/wkpn/aio2ch/README-ru.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a225d2a..bbcd931 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,21 @@ Changelog ========= +`2.0.3` +------- + +* ``get_thread_posts`` and ``get_thread_media`` now accepts thread passed as url + +.. code-block:: python + + >>> thread_media = await client.get_thread_media('https://2ch.hk/test/res/30972.html') + +* added boards lists and according checks +* added new exceptions +* more tests +* more code cleanup +* added changelog_ and readme_ translations in Russian + `2.0.2` ------- @@ -38,3 +53,6 @@ Changelog * Added ``keywords`` parameter to ``get_board_threads`` method * Added ``download_thread_media`` method + +.. _changelog: https://github.com/wkpn/aio2ch/CHANGELOG-ru.rst +.. _readme: https://github.com/wkpn/aio2ch/README-ru.rst diff --git a/README-ru.rst b/README-ru.rst new file mode 100644 index 0000000..bdc05b3 --- /dev/null +++ b/README-ru.rst @@ -0,0 +1,132 @@ +|Logo| + +|License| +|Downloads| +|PyPi| +|Python| + +Полностью асинхронный read-only API wrapper для 2ch.hk (dvach, Двач) + +Требования +---------- + +- httpx_ +- aiofiles_ + +Установка через pip +------------------- +.. code-block:: bash + + $ pip3 install aio2ch + + +Сборка из исходников +-------------------- +.. code-block:: bash + + $ git clone https://github.com/wkpn/aio2ch + $ cd ./aio2ch + $ python3 setup.py install + +Использование +------------- + +Простой пример (тогда надо вызвать ``client.close()`` в конце использования) + +.. code-block:: python + + >>> from aio2ch import Api + >>> client = Api() + >>> ... + >>> await client.close() + +Или можно использовать как менеджер контекста + +.. code-block:: python + + >>> async with Api() as client: + ... boards = await client.get_boards() + +Получить все доски + +.. code-block:: python + + >>> boards = await client.get_boards() + + >>> boards + [, ... ] + +Также дополнительно можно получить ``status`` для каждого метода. Полезно, если нужны ретраи + +.. code-block:: python + + >>> status, boards = await client.get_boards(return_status=True) + + >>> status + 200 + + >>> boards + [, ... ] + +Получить все треды с доски + +.. code-block:: python + + >>> threads = await client.get_board_threads(board='b') + + >>> threads + [, ... ] + +Получить топ тредов с доски с заданной сортировкой (*views*, *score* or *posts_count*) + +.. code-block:: python + + >>> top_threads = await client.get_top_board_threads(board='b', method='views', num=3) + + >>> top_threads + [, , ] + +Получить все посты с треда (``thread`` инстанс ``Thread``) + +.. code-block:: python + + >>> thread_posts = await client.get_thread_posts(thread=thread) + + >>> thread_posts + [, ... ] + +Получить все посты с треда по его адресу + +.. code-block:: python + + >>> thread_posts = await client.get_thread_posts(thread='https://2ch.hk/test/res/30972.html') + + >>> thread_posts + [, ... ] + +Получить все медиа с треда (пикчи, webm-ки и прочее) + +.. code-block:: python + + >>> thread_media = await client.get_thread_media(thread=thread) + + >>> thread_media + [, ... ] + +Скачать все медиа с треда на диск в папку + +.. code-block:: python + + >>> await client.download_thread_media(files=thread_media, save_to='./downloads') + +.. |License| image:: https://img.shields.io/pypi/l/aio2ch.svg + :target: https://pypi.python.org/pypi/aio2ch +.. |Downloads| image:: https://pepy.tech/badge/aio2ch + :target: https://pepy.tech/project/aio2ch +.. |PyPi| image:: https://img.shields.io/pypi/v/aio2ch.svg + :target: https://pypi.python.org/pypi/aio2ch +.. |Python| image:: https://img.shields.io/pypi/pyversions/aio2ch.svg + :target: https://pypi.python.org/pypi/aio2ch +.. |Logo| image:: https://raw.githubusercontent.com/wkpn/aio2ch/master/docs/img/banner.jpg +.. _httpx: https://github.com/encode/httpx +.. _aiofiles: https://github.com/Tinche/aiofiles \ No newline at end of file diff --git a/README.rst b/README.rst index 41e0fa5..d87845f 100644 --- a/README.rst +++ b/README.rst @@ -95,6 +95,15 @@ Get all thread's posts (``thread`` is an instance of ``Thread``) >>> thread_posts [, ... ] +Get all thread's posts by url + +.. code-block:: python + + >>> thread_posts = await client.get_thread_posts(thread='https://2ch.hk/test/res/30972.html') + + >>> thread_posts + [, ... ] + Get all media in all thread's posts (images, webm and so on) .. code-block:: python diff --git a/aio2ch/__init__.py b/aio2ch/__init__.py index deda7e7..cdbc7cf 100644 --- a/aio2ch/__init__.py +++ b/aio2ch/__init__.py @@ -1,5 +1,5 @@ __author__ = 'wkpn' -__version__ = '2.0.2' +__version__ = '2.0.3' __license__ = 'MIT' __url__ = 'https://github.com/wkpn/aio2ch' diff --git a/aio2ch/api.py b/aio2ch/api.py index 4d60775..294607b 100644 --- a/aio2ch/api.py +++ b/aio2ch/api.py @@ -1,11 +1,30 @@ __all__ = 'Api' -from .exceptions import NoBoardProvidedException, WrongSortMethodException -from .objects import Board, File, Post, Thread +from .exceptions import ( + InvalidBoardIdException, + InvalidThreadUrlException, + NoBoardProvidedException, + WrongSortMethodException +) +from .objects import ( + Board, + File, + Post, + Thread +) +from .helpers import BOARDS_LIST, get_board_and_thread_from_url from .settings import SORTING_METHODS from .api_client import ApiClient -from typing import Any, Dict, Iterable, Optional, Tuple, Type, Union +from typing import ( + Any, + Dict, + Iterable, + Optional, + Tuple, + Type, + Union +) from types import TracebackType import aiofiles @@ -57,6 +76,9 @@ async def get_board_threads(self, if isinstance(board, Board): board = board.id + if board not in BOARDS_LIST: + raise InvalidBoardIdException(f'Board {board} doesn\'t exist') + status, threads = await self._get(url=f'{self._api_client.api_url}/{board}/threads.json') threads = threads['threads'] threads = tuple(Thread(thread, board) for thread in threads) @@ -90,6 +112,9 @@ async def get_top_board_threads(self, if isinstance(board, Board): board = board.id + if board not in BOARDS_LIST: + raise InvalidBoardIdException(f'Board {board} doesn\'t exist') + result = await self.get_board_threads(board, return_status=return_status) if return_status: @@ -126,11 +151,19 @@ async def get_thread_posts(self, if isinstance(thread, Thread): board = thread.board thread = thread.num + elif isinstance(thread, str): + board, thread = get_board_and_thread_from_url(thread) + + if not all((board, thread)): + raise InvalidThreadUrlException(f'Invalid thread url {thread}') elif isinstance(board, Board): board = board.id elif not board: raise NoBoardProvidedException('Board id is not provided') + if board not in BOARDS_LIST: + raise InvalidBoardIdException(f'Board {board} doesn\'t exist') + status, posts = await self._get(url=f'{self._api_client.api_url}/{board}/res/{thread}.json') posts = posts['threads'][0]['posts'] posts = tuple(Post(post) for post in posts) diff --git a/aio2ch/exceptions.py b/aio2ch/exceptions.py index 9e1623b..3ca4b92 100644 --- a/aio2ch/exceptions.py +++ b/aio2ch/exceptions.py @@ -4,3 +4,11 @@ class WrongSortMethodException(Exception): class NoBoardProvidedException(Exception): pass + + +class InvalidThreadUrlException(Exception): + pass + + +class InvalidBoardIdException(Exception): + pass diff --git a/aio2ch/helpers.py b/aio2ch/helpers.py new file mode 100644 index 0000000..1444205 --- /dev/null +++ b/aio2ch/helpers.py @@ -0,0 +1,32 @@ +from typing import Tuple, Union + +import re + + +BOARDS_LIST: Tuple = ( + 'b', 'vg', 'po', 'fag', 'news', '2d', 'v', 'hw', 'sex', 'a', 'au', 'biz', 'wm', 'soc', 'wrk', 'rf', 'pr', + 'brg', 'me', 'mobi', 'kpop', 'c', 'w', 'hi', 'mov', 'fiz', 'ftb', 'sp', 'cg', 'ma', 'hry', 'dr', 'tes', + 's', 'alco', 'ra', 'obr', 'em', 'di', 'gsg', 'ch', 'mlp', 'mus', 'fa', 'ga', 'fs', 'zog', 'psy', 'whn', + 'bi', 'by', 'fg', 'fet', 'tv', 'un', 'fd', 'mu', 'ukr', 'pa', 'qtr4', 'fl', 'mg', 're', 'gd', 'bo', 'sn', + 'spc', 't', 'd', 'ew', 'dom', 'wh', 'td', 'mc', 'mmo', 'fur', 'e', 'asmr', 'vape', 'sci', 'tr', 'p', 'gg', + 'cul', 'out', 'sw', 'pok', 'es', 'diy', 'trv', 'hc', 'media', 'jsf', 'wow', 'cute', 'kz', 'socionics', 'o', + 'ruvn', 'izd', 'wr', 'r', 'h', 'moba', 'cc', 'gabe', 'se', 'wwe', 'int', 'hh', 'mo', 'ne', 'pvc', 'ph', 'sf', + 'de', 'wp', 'bg', 'aa', 'ja', 'rm', 'to', 'vn', 'ho', 'web', 'br', 'gb', 'abu', 'old', 'guro', 'ussr', 'law', + 'm', 'ya', 'r34', '8', 'mlpr', 'ro', 'who', 'srv', 'electrach', 'ing', 'got', 'crypt', 'lap', 'smo', 'hg', + 'sad', 'fi', 'nvr', 'ind', 'ld', 'fem', 'vr', 'arg', 'char', 'hv', 'math', 'catalog', 'api', 'test' +) + + +def get_board_and_thread_from_url(thread_url: str) -> Union[Tuple[str, str], Tuple[None, None]]: + # https://2ch.hk/test/res/30972.html + + pattern = 'https://[2a-z]+.[a-z]+/[a-z]+/res/[0-9]+.html' + match = re.compile(pattern).match(thread_url) + + if match: + split = re.split('[/.]', thread_url) + board, thread = split[4], split[6] + + return board, thread + + return None, None diff --git a/setup.py b/setup.py index 8a30ad6..eaf72c0 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ -from setuptools import setup from aio2ch import __author__, __version__, __license__, __url__ +from setuptools import setup + with open('README.rst', 'r', encoding='utf-8') as f: long_description = f.read() diff --git a/tests/conftest.py b/tests/conftest.py index c505cad..287978d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,16 @@ def thread(): return Thread(test_thread_data, test_board) +@pytest.fixture +def thread_url(): + return 'https://2ch.hk/test/res/30972.html' + + +@pytest.fixture +def invalid_thread_url(): + return 'https://dvach.hk/blah/blah/blah.css' + + @pytest.fixture def number_of_threads(number=5): return number diff --git a/tests/test_api.py b/tests/test_api.py index 1153764..baf02bb 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1,6 +1,16 @@ -from aio2ch.exceptions import NoBoardProvidedException, WrongSortMethodException -from aio2ch.objects import Board, File, Post, Thread - +from aio2ch.exceptions import ( + InvalidBoardIdException, + InvalidThreadUrlException, + NoBoardProvidedException, + WrongSortMethodException +) +from aio2ch.helpers import get_board_and_thread_from_url +from aio2ch.objects import ( + Board, + File, + Post, + Thread +) import pytest @@ -34,6 +44,12 @@ async def test_get_boards_with_status(client): assert all(isinstance(board, Board) for board in boards) +@pytest.mark.asyncio +async def test_get_board_threads_invalid_board(client): + with pytest.raises(InvalidBoardIdException): + await client.get_board_threads(board='thisboarddoesntexist') + + @pytest.mark.asyncio async def test_get_board_threads_str(client): threads = await client.get_board_threads(board='test') @@ -43,18 +59,18 @@ async def test_get_board_threads_str(client): @pytest.mark.asyncio -async def test_get_board_threads_instance(client, board): - threads = await client.get_board_threads(board=board) +async def test_get_board_threads_str_with_status(client): + status, threads = await client.get_board_threads(board='test', return_status=True) + assert status >= 200 assert len(threads) > 0 assert all(isinstance(thread, Thread) for thread in threads) @pytest.mark.asyncio -async def test_get_board_threads_str_with_status(client): - status, threads = await client.get_board_threads(board='test', return_status=True) +async def test_get_board_threads_instance(client, board): + threads = await client.get_board_threads(board=board) - assert status >= 200 assert len(threads) > 0 assert all(isinstance(thread, Thread) for thread in threads) @@ -77,19 +93,19 @@ async def test_get_top_board_threads_str(client, number_of_threads): @pytest.mark.asyncio -async def test_get_top_board_threads_instance(client, board, number_of_threads): - threads = await client.get_top_board_threads(board=board, method='views', num=number_of_threads) +async def test_get_top_board_threads_str_with_status(client, number_of_threads): + status, threads = await client.get_top_board_threads(board='test', method='views', num=number_of_threads, + return_status=True) + assert status >= 200 assert len(threads) == number_of_threads assert all(isinstance(thread, Thread) for thread in threads) @pytest.mark.asyncio -async def test_get_top_board_threads_str_with_status(client, number_of_threads): - status, threads = await client.get_top_board_threads(board='test', method='views', num=number_of_threads, - return_status=True) +async def test_get_top_board_threads_instance(client, board, number_of_threads): + threads = await client.get_top_board_threads(board=board, method='views', num=number_of_threads) - assert status >= 200 assert len(threads) == number_of_threads assert all(isinstance(thread, Thread) for thread in threads) @@ -123,6 +139,13 @@ async def test_get_thread_posts(client, thread): assert all(isinstance(post, Post) for post in posts) +@pytest.mark.asyncio +async def test_get_thread_posts_by_url(client, thread_url): + posts = await client.get_thread_posts(thread_url) + + assert all(isinstance(post, Post) for post in posts) + + @pytest.mark.asyncio async def test_get_thread_posts_with_status(client, thread): status, posts = await client.get_thread_posts(thread, return_status=True) @@ -131,12 +154,26 @@ async def test_get_thread_posts_with_status(client, thread): assert all(isinstance(post, Post) for post in posts) +@pytest.mark.asyncio +async def test_get_thread_posts_with_status_by_url(client, thread_url): + status, posts = await client.get_thread_posts(thread_url, return_status=True) + + assert status >= 200 + assert all(isinstance(post, Post) for post in posts) + + @pytest.mark.asyncio async def test_get_thread_posts_no_board_provided(client, thread): with pytest.raises(NoBoardProvidedException): await client.get_thread_posts(thread.num) +@pytest.mark.asyncio +async def test_get_thread_posts_invalid_thread_url(client, invalid_thread_url): + with pytest.raises(InvalidThreadUrlException): + await client.get_thread_posts(invalid_thread_url) + + @pytest.mark.asyncio async def test_get_thread_media(client, thread): thread_media = await client.get_thread_media(thread) @@ -145,6 +182,14 @@ async def test_get_thread_media(client, thread): assert all(isinstance(file, File) for file in thread_media) +@pytest.mark.asyncio +async def test_get_thread_media_by_url(client, thread_url): + thread_media = await client.get_thread_media(thread_url) + + assert len(thread_media) > 0 + assert all(isinstance(file, File) for file in thread_media) + + @pytest.mark.asyncio async def test_get_thread_media_with_status(client, thread): status, thread_media = await client.get_thread_media(thread, return_status=True) @@ -152,3 +197,19 @@ async def test_get_thread_media_with_status(client, thread): assert status >= 200 assert len(thread_media) > 0 assert all(isinstance(file, File) for file in thread_media) + + +@pytest.mark.asyncio +async def test_get_thread_media_with_status_by_url(client, thread_url): + status, thread_media = await client.get_thread_media(thread_url, return_status=True) + + assert status >= 200 + assert len(thread_media) > 0 + assert all(isinstance(file, File) for file in thread_media) + + +def test_get_board_and_thread_from_url(thread_url): + board, thread = get_board_and_thread_from_url(thread_url) + + assert board == 'test' + assert thread == '30972'