diff --git a/Dockerfile b/Dockerfile index a4b9b4f..6c348a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,10 @@ COPY requirements.txt /cncnet-map-api COPY requirements-dev.txt /cncnet-map-api COPY start.sh /cncnet-map-api -RUN apt-get update && apt-get install -y liblzo2-dev +RUN apt-get update && apt-get install -y liblzo2-dev # Compression library used by westwood. +RUN apt-get install libmagic1 # File type checking. RUN pip install --upgrade pip +# The cflags are needed to build the lzo library on Apple silicon. RUN CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r ./requirements-dev.txt RUN chmod +x /cncnet-map-api/start.sh diff --git a/README.md b/README.md index 6d5940d..b9abe79 100644 --- a/README.md +++ b/README.md @@ -24,29 +24,34 @@ Just set up your environment file and run the full docker compose. ## Backend devs +You can use the docker files if you'd like, but Django + docker is known to have issue attaching +to debuggers and hitting breakpoints, so here are the native OS instructions. + 1. Download and install [pyenv](https://github.com/pyenv/pyenv) 2. Install [PostgreSQL](https://www.postgresql.org/) for your system. This is required for Django - On Mac you can do `brew install postgresql` if you have brew installed. -3. Checkout the repository -4. Switch to the repository directory -5. Setup Python +3. Install LibMagic for [Python Magic](https://github.com/ahupp/python-magic) + - On Mac you can do `brew install libmagic` if you have breq installed. +4. Checkout the repository +5. Switch to the repository directory +6. Setup Python - Install Python 3.12 `pyenv install 3.12` or whatever the latest python is. - Setup the virtual environments `pyenv virtualenv 3.12 cncnet-map-api` - Set the virtual enviornment for the directory `pyenv local cncnet-map-api` -6. Setup requirements `pip install -r requirements-dev.txt` +7. Setup requirements `pip install -r requirements-dev.txt` - On Apple Silicon you'll need to install lzo with `brew install lzo` then run `CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r requirements-dev.txt` to get `python-lzo` to install. You shouldn't need to include those flags again unless `python-lzo` updates. -7. Install the pre-commit hooks `pre-commit install` -8. Setup the environment variables +8. Install the pre-commit hooks `pre-commit install` +9. Setup the environment variables - Create a `.env` file at the root of the repo - Copy the contents of `example.env` to your `.env` file. - Fill out the required values in `.env` - If the app doesn't run due to a missing required variable, add said variable to `example.env` because the person who made the variable forgot to do so. -9. Run the `db` service in `docker-compose` -10. Load your `.env` file into your shell, (you can use `./load_env.sh`) then migrate the database `./manage.py migrate` -11. `./manage.py runserver` +10. Run the `db` service in `docker-compose` +11. Load your `.env` file into your shell, (you can use `./load_env.sh`) then migrate the database `./manage.py migrate` +12. `./manage.py runserver` I **strongly** recommend using PyCharm and the `.env` plugin for running the PyTests. diff --git a/kirovy/migrations/0004_cncmap_incomplete_upload.py b/kirovy/migrations/0004_cncmap_incomplete_upload.py new file mode 100644 index 0000000..d79f62c --- /dev/null +++ b/kirovy/migrations/0004_cncmap_incomplete_upload.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.5 on 2024-04-13 19:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("kirovy", "0003_map_categories"), + ] + + operations = [ + migrations.AddField( + model_name="cncmap", + name="incomplete_upload", + field=models.BooleanField( + default=False, + help_text="If true, then the map file has been uploaded, but the map info has not been set yet.", + ), + ), + ] diff --git a/kirovy/models/cnc_map.py b/kirovy/models/cnc_map.py index fe6e36e..9831dbe 100644 --- a/kirovy/models/cnc_map.py +++ b/kirovy/models/cnc_map.py @@ -107,6 +107,11 @@ class CncMap(cnc_user.CncNetUserOwnedModel): ) """:attr: Keep banned maps around so we can keep track of rule-breakers.""" + incomplete_upload = models.BooleanField( + default=False, + help_text="If true, then the map file has been uploaded, but the map info has not been set yet.", + ) + def next_version_number(self) -> int: """Generate the next version to use for a map file. diff --git a/kirovy/response.py b/kirovy/response.py index b438237..52a7ebe 100644 --- a/kirovy/response.py +++ b/kirovy/response.py @@ -5,7 +5,7 @@ class KirovyResponse(Response): def __init__( self, - data: t.Optional[t.Union[t.ListResponseData, t.ResponseData]] = None, + data: t.Optional[t.BaseResponseData] = None, status: t.Optional[int] = None, template_name: t.Optional[str] = None, headers: t.Optional[t.DictStrAny] = None, diff --git a/kirovy/services/cnc_gen_2_services.py b/kirovy/services/cnc_gen_2_services.py index 461fdf9..e50a5ac 100644 --- a/kirovy/services/cnc_gen_2_services.py +++ b/kirovy/services/cnc_gen_2_services.py @@ -1,6 +1,7 @@ import base64 import logging +import magic from PIL import Image import configparser import enum @@ -140,12 +141,10 @@ def is_text(cls, uploaded_file: File) -> bool: :return: True if readable as text. """ - try: - with uploaded_file.open("tr") as check_file: - check_file.read() - return True - except UnicodeDecodeError: - return False + magic_parser = magic.Magic(mime=True) + uploaded_file.seek(0) + mr_mime = magic_parser.from_buffer(uploaded_file.read()) + return mr_mime == "text/plain" def extract_preview(self) -> t.Optional[Image.Image]: """Extract the map preview if it exists. diff --git a/kirovy/typing/__init__.py b/kirovy/typing/__init__.py index 159d52c..834fb09 100644 --- a/kirovy/typing/__init__.py +++ b/kirovy/typing/__init__.py @@ -44,5 +44,9 @@ class ListResponseData(BaseResponseData): pagination_metadata: NotRequired[PaginationMetadata] -class ResponseData(TypedDict): +class ResponseData(BaseResponseData): result: DictStrAny + + +class ErrorResponseData(BaseResponseData): + additional: NotRequired[DictStrAny] diff --git a/kirovy/urls.py b/kirovy/urls.py index 2272945..c174ec3 100644 --- a/kirovy/urls.py +++ b/kirovy/urls.py @@ -14,13 +14,30 @@ 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import path +from django.urls import path, include from kirovy.views import test, cnc_map_views, permission_views +from kirovy import typing as t -urlpatterns = [ - path("admin/", admin.site.urls), - path("test/jwt", test.TestJwt.as_view()), - path("map-categories/", cnc_map_views.MapCategoryListCreateView.as_view()), - path("ui-permissions/", permission_views.ListPermissionForAuthUser.as_view()), + +def _get_url_patterns() -> t.List[path]: + """Return the root level url patterns. + + I added this because I wanted to have the root URLs at the top of the file, + but I didn't want to have other url files. + """ + return [ + path("admin/", admin.site.urls), + path("test/jwt", test.TestJwt.as_view()), + path("ui-permissions/", permission_views.ListPermissionForAuthUser.as_view()), + path("maps/", include(map_patterns)), + ] + + +map_patterns = [ + path("categories/", cnc_map_views.MapCategoryListCreateView.as_view()), + path("upload//", cnc_map_views.MapFileUploadView.as_view()), ] + + +urlpatterns = _get_url_patterns() diff --git a/kirovy/utils/file_utils.py b/kirovy/utils/file_utils.py index 062add4..afcb4ae 100644 --- a/kirovy/utils/file_utils.py +++ b/kirovy/utils/file_utils.py @@ -1,5 +1,7 @@ +import collections +import functools import hashlib -from functools import partial +from kirovy import typing as t from django.db.models.fields.files import FieldFile @@ -21,3 +23,108 @@ def _hash_file(hasher: "_HASH", file: FieldFile, block_size: int) -> str: file.seek(0) return hasher.hexdigest() + + +class ByteSized: + """A class to pretty format byte sizes, inspired by ``datetime.timedelta``'s functionality.""" + + _byte: int = 0 + _kilo: int = 0 + _mega: int = 0 + _giga: int = 0 + _tera: int = 0 + + def __new__( + cls, + byte: int = 0, + *, + kilo: int = 0, + mega: int = 0, + giga: int = 0, + tera: int = 0, + ) -> "ByteSized": + + if any([x < 0 for x in [tera, giga, mega, kilo, byte]]): + raise AttributeError("Does not support args < 0") + + self = object.__new__(cls) + b2k, br = divmod(byte, 1000) + self._byte = br + + kilo = kilo + b2k + k2m, kr = divmod(kilo, 1000) + self._kilo = kr + + mega = mega + k2m + m2g, mr = divmod(mega, 1000) + self._mega = mr + + giga = giga + m2g + g2t, gr = divmod(giga, 1000) + self._giga = gr + + self._tera = tera + g2t + + return self + + def __str__(self) -> str: + return ", ".join( + [f"{size}{desc}" for desc, size in self.__mapping.items() if size > 0] + ) + + @functools.cached_property + def __mapping(self) -> t.Dict[str, int]: + return collections.OrderedDict( + { + "TB": self.tera, + "GB": self.giga, + "MB": self.mega, + "KB": self.kilo, + "B": self.byte, + } + ) + + @property + def tera(self) -> int: + return self._tera + + @property + def giga(self) -> int: + return self._giga + + @property + def mega(self) -> int: + return self._mega + + @property + def kilo(self) -> int: + return self._kilo + + @property + def byte(self) -> int: + return self._byte + + @functools.cached_property + def total_bytes(self) -> int: + total = 0 + to_explode = [self._byte, self._kilo, self._mega, self._giga, self._tera] + for i, value in enumerate(to_explode): + exponent = 3 * i + magnitude = 10**exponent + total += value * magnitude + return total + + def __gt__(self, other: "ByteSized") -> bool: + return self.total_bytes > other.total_bytes + + def __lt__(self, other: "ByteSized") -> bool: + return self.total_bytes < other.total_bytes + + def __ge__(self, other: "ByteSized") -> bool: + return self.total_bytes >= other.total_bytes + + def __le__(self, other: "ByteSized") -> bool: + return self.total_bytes <= other.total_bytes + + def __eq__(self, other: "ByteSized") -> bool: + return self.total_bytes == other.total_bytes diff --git a/kirovy/views/cnc_map_views.py b/kirovy/views/cnc_map_views.py index 01a5396..9912cd3 100644 --- a/kirovy/views/cnc_map_views.py +++ b/kirovy/views/cnc_map_views.py @@ -1,15 +1,25 @@ +import logging + from django.conf import settings +from django.core.files.uploadedfile import UploadedFile from rest_framework import status +from rest_framework.parsers import FileUploadParser, MultiPartParser from rest_framework.response import Response from rest_framework.views import APIView -from kirovy import permissions +from kirovy import permissions, typing as t, exceptions from kirovy.models import MapCategory from kirovy.request import KirovyRequest +from kirovy.response import KirovyResponse from kirovy.serializers import cnc_map_serializers +from kirovy.services.cnc_gen_2_services import CncGen2MapParser +from kirovy.utils import file_utils from kirovy.views import base_views +_LOGGER = logging.getLogger(__name__) + + class MapCategoryListCreateView(base_views.KirovyListCreateView): permission_classes = [permissions.IsAdmin | permissions.ReadOnly] serializer_class = cnc_map_serializers.MapCategorySerializer @@ -28,3 +38,42 @@ class MapRetrieveUpdateView(base_views.KirovyRetrieveUpdateView): class MapDeleteView(base_views.KirovyDestroyView): ... + + +class MapFileUploadView(APIView): + parser_classes = [MultiPartParser] + permission_classes = [permissions.CanUpload] + + def post( + self, request: KirovyRequest, filename: str, format=None + ) -> KirovyResponse: + + uploaded_file: UploadedFile = request.data["file"] + max_size = file_utils.ByteSized(mega=25) + uploaded_size = file_utils.ByteSized(uploaded_file.size) + + if uploaded_size > max_size: + return KirovyResponse( + t.ErrorResponseData( + message="File too large", + additional={ + "max_bytes": str(max_size), + "your_bytes": str(uploaded_file), + }, + ), + status=status.HTTP_400_BAD_REQUEST, + ) + + try: + # TODO: Finish the map upload. + map_parser = CncGen2MapParser(uploaded_file) + except exceptions.InvalidMapFile as e: + return KirovyResponse( + t.ErrorResponseData(message="Invalid Map File"), + status=status.HTTP_400_BAD_REQUEST, + ) + + return KirovyResponse( + t.ResponseData(message="File uploaded successfully"), + status=status.HTTP_201_CREATED, + ) diff --git a/requirements-dev.txt b/requirements-dev.txt index b5a8177..c8baba5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,8 +1,7 @@ -r requirements.txt -pytest==7.* +pytest==8.* pytest-mock==3.* -black==22.* -pre-commit==2.* -pytest-django==4.5.2 -deptree==0.0.10 +black==24.* +pre-commit==3.* +pytest-django==4.* markdown>=3.4.4, <=4.0 diff --git a/requirements.txt b/requirements.txt index 8c5f1f4..3d9901a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -django>=4.2.*, <=4.3 # 4.2 is the Long-Term Service version. +django>=4.2.11, <=4.3 # 4.2 is the Long-Term Service version until 2026. psycopg2==2.* requests>=2.31.*, <3.0 djangorestframework>=3.14.*, <4.0 @@ -6,3 +6,4 @@ pyjwt[crypto]>=2.8.0 pillow==10.* python-lzo==1.15 ujson==5.* +python-magic>=0.4.27 diff --git a/tests/fixtures/common_fixtures.py b/tests/fixtures/common_fixtures.py index 982f863..a3c27fc 100644 --- a/tests/fixtures/common_fixtures.py +++ b/tests/fixtures/common_fixtures.py @@ -124,6 +124,16 @@ def post( """Wraps post to make it default to JSON.""" data = self.__convert_data(data, content_type) + + if content_type is None: + return super().post( + path, + data=data, + follow=follow, + secure=secure, + **extra, + ) + return super().post( path, data=data, diff --git a/tests/fixtures/file_fixtures.py b/tests/fixtures/file_fixtures.py index 0134008..6d70a5f 100644 --- a/tests/fixtures/file_fixtures.py +++ b/tests/fixtures/file_fixtures.py @@ -16,7 +16,7 @@ def test_data_path() -> pathlib.Path: def load_test_file(test_data_path): """Return a function to load a file from test_data.""" - def _inner(relative_path: t.Union[str, pathlib.Path]) -> File: + def _inner(relative_path: t.Union[str, pathlib.Path], read_mode: str = "r") -> File: """Load a file from test_data. :param relative_path: @@ -26,7 +26,7 @@ def _inner(relative_path: t.Union[str, pathlib.Path]) -> File: """ full_path = test_data_path / relative_path - return File(open(full_path, "r")) + return File(open(full_path, read_mode)) return _inner @@ -34,7 +34,7 @@ def _inner(relative_path: t.Union[str, pathlib.Path]) -> File: @pytest.fixture def file_binary(load_test_file) -> File: """Returns a random binary file.""" - file = load_test_file("binary_file.mp3") + file = load_test_file("binary_file.mp3", "rb") yield file file.close() diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py new file mode 100644 index 0000000..b9e7cdc --- /dev/null +++ b/tests/test_file_utils.py @@ -0,0 +1,67 @@ +import pytest + +from kirovy.utils import file_utils + + +def test_byte_sized_from_bytes(): + """Test the easiest case of converting bytes to other magnitudes.""" + total_byte = 1002456789290 + byte_sized = file_utils.ByteSized(byte=total_byte) + assert byte_sized.tera == 1 + assert byte_sized.giga == 2 + assert byte_sized.mega == 456 + assert byte_sized.kilo == 789 + assert byte_sized.byte == 290 + + assert byte_sized.total_bytes == total_byte + assert str(byte_sized) == "1TB, 2GB, 456MB, 789KB, 290B" + + +def test_byte_sized_mix_and_match(): + """Test that combinations of args add up correctly, including carrying over when > 1000.""" + total_byte = 3003457790290 + # The 2002 GB should roll over into 3TB 2GB + # The 1456 MB should roll over into 3GB 456MB + # 1789 KB should roll over into 457MB 789KB + # 1290 B should roll over into 790KB 290B + byte_sized = file_utils.ByteSized(1290, kilo=1789, mega=1456, giga=2002, tera=1) + assert byte_sized.tera == 3 + assert byte_sized.giga == 3 + assert byte_sized.mega == 457 + assert byte_sized.kilo == 790 + assert byte_sized.byte == 290 + + assert byte_sized.total_bytes == total_byte + assert str(byte_sized) == "3TB, 3GB, 457MB, 790KB, 290B" + + +def test_byte_sized_print_missing(): + """Test that magnitudes that are zero don't get included in the string.""" + # The 100MB should roll over into 3GB, and MB should be excluded from the string. + byte_sized = file_utils.ByteSized(23, kilo=117, mega=1000, giga=2, tera=0) + assert str(byte_sized) == "3GB, 117KB, 23B" + + assert str(file_utils.ByteSized(mega=25)) == "25MB" + + +@pytest.mark.parametrize("arg_to_negative", ["tera", "giga", "mega", "kilo", "byte"]) +def test_byte_sized_lt_zero(arg_to_negative: str): + """Test that we error out on args being negative.""" + args = {key: 1 for key in ["tera", "giga", "mega", "kilo", "byte"]} + args[arg_to_negative] = -1 + with pytest.raises(AttributeError): + file_utils.ByteSized(**args) + + +def test_byte_sized__operators(): + assert file_utils.ByteSized(1) < file_utils.ByteSized(2) + assert file_utils.ByteSized(2) > file_utils.ByteSized(1) + + assert file_utils.ByteSized(1) == file_utils.ByteSized(1) + assert file_utils.ByteSized(1) != file_utils.ByteSized(2) + + assert file_utils.ByteSized(1) >= file_utils.ByteSized(1) + assert file_utils.ByteSized(2) >= file_utils.ByteSized(1) + + assert file_utils.ByteSized(1) <= file_utils.ByteSized(1) + assert file_utils.ByteSized(1) <= file_utils.ByteSized(2) diff --git a/tests/test_views/test_map_category_api.py b/tests/test_views/test_map_category_api.py index 8d26fbc..a1aed3f 100644 --- a/tests/test_views/test_map_category_api.py +++ b/tests/test_views/test_map_category_api.py @@ -3,7 +3,7 @@ from kirovy.models import MapCategory from kirovy import typing as t -CATEGORY_URL = "/map-categories/" +CATEGORY_URL = "/maps/categories/" def test_map_category_create( diff --git a/tests/test_views/test_map_upload.py b/tests/test_views/test_map_upload.py new file mode 100644 index 0000000..d8d7409 --- /dev/null +++ b/tests/test_views/test_map_upload.py @@ -0,0 +1,19 @@ +import pathlib +from urllib import parse + +from rest_framework import status + +_UPLOAD_URL = "/maps/upload/" + + +def test_map_file_upload_happy_path(client_user, file_map_desert): + # TODO: Finish the tests. + file_name = parse.quote_plus(pathlib.Path(file_map_desert.name).name) + response = client_user.post( + f"{_UPLOAD_URL}{file_name}/", + {"file": file_map_desert}, + format="multipart", + content_type=None, + ) + + assert response.status_code == status.HTTP_200_OK