From d92272ffa0933b485a466e17aa7141ea56bef9f9 Mon Sep 17 00:00:00 2001 From: Jackson Kruger Date: Wed, 11 Oct 2023 13:32:58 -0500 Subject: [PATCH] Batch inserts [MOD-555] (#726) * Batch a bunch of inserts, but still more to do * Insert many for clickhouse (+ tests) * Batch the remaining ones except those requiring deduplication * Risky dedups * Bit o cleanup and formatting * cargo sqlx prepare * Add test around batch editing project categories * Add struct to satisfy clippy * Fix silly mistake that was caught by the tests! * Leave room for growth in dummy_data --- Cargo.lock | 12 + Cargo.toml | 2 + sqlx-data.json | 444 ++++++++---------- src/clickhouse/mod.rs | 6 +- src/database/models/collection_item.rs | 26 +- src/database/models/project_item.rs | 144 +++--- src/database/models/team_item.rs | 73 ++- src/database/models/thread_item.rs | 28 +- src/database/models/version_item.rs | 257 ++++++++--- src/queue/analytics.rs | 65 +-- src/queue/analytics/tests.rs | 128 ++++++ src/queue/payouts.rs | 31 +- src/routes/analytics.rs | 26 +- src/routes/v2/admin.rs | 64 ++- src/routes/v2/collections.rs | 32 +- src/routes/v2/projects.rs | 595 +++++++++++-------------- src/routes/v2/version_creation.rs | 4 +- src/routes/v2/versions.rs | 39 +- tests/common/database.rs | 5 + tests/common/dummy_data.rs | 10 + tests/common/environment.rs | 13 + tests/files/dummy_data.sql | 22 +- tests/project.rs | 71 ++- 23 files changed, 1188 insertions(+), 909 deletions(-) create mode 100644 src/queue/analytics/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 77159808..40237625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,17 @@ dependencies = [ "uuid 1.4.0", ] +[[package]] +name = "derive-new" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3418329ca0ad70234b9735dc4ceed10af4df60eff9c8e7b06cb5e520d92c3535" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -2260,6 +2271,7 @@ dependencies = [ "color-thief", "dashmap", "deadpool-redis", + "derive-new", "dotenvy", "env_logger", "flate2", diff --git a/Cargo.toml b/Cargo.toml index 885904b9..898a4ae4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,5 +106,7 @@ woothee = "0.13.0" lettre = "0.10.4" +derive-new = "0.5.9" + [dev-dependencies] actix-http = "3.4.0" diff --git a/sqlx-data.json b/sqlx-data.json index eabec5b7..9549939c 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -265,19 +265,6 @@ }, "query": "\n SELECT o.id, o.title, o.team_id, o.description, o.icon_url, o.color\n FROM organizations o\n WHERE o.id = ANY($1) OR o.title = ANY($2)\n GROUP BY o.id;\n " }, - "05baeb26d9856218e5c6f8856a96788b2a7ac3536ff9412a50552cef1d561a1e": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int4" - ] - } - }, - "query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, FALSE)\n " - }, "061a3e43df9464263aaf1555a27c1f4b6a0f381282f4fa75cc13b1d354857578": { "describe": { "columns": [ @@ -595,24 +582,6 @@ }, "query": "\n SELECT SUM(pv.amount) amount\n FROM payouts_values pv\n WHERE pv.user_id = $1\n " }, - "0c2addb0d7a87fa558821ff8e943bbb751fb2bdc22d1a5368f61cc7827586840": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Varchar", - "Varchar", - "Bool", - "Int4", - "Varchar" - ] - } - }, - "query": "\n INSERT INTO files (id, version_id, url, filename, is_primary, size, file_type)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n " - }, "0c6b7e51b0b9115d95b5dbb9bb88a3e266b78ae9375a90261503c2cccd5bdf1b": { "describe": { "columns": [], @@ -870,21 +839,6 @@ }, "query": "SELECT EXISTS(SELECT 1 FROM threads WHERE id=$1)" }, - "196c8ac2228e199f23eaf980f7ea15b37f76e66bb81da1115a754aad0be756e4": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Numeric", - "Timestamptz" - ] - } - }, - "query": "\n INSERT INTO payouts_values (user_id, mod_id, amount, created)\n VALUES ($1, $2, $3, $4)\n " - }, "19c7498a01f51b8220245a53490916191a153fa1fe14404d39ab2980e3386058": { "describe": { "columns": [], @@ -935,6 +889,34 @@ }, "query": "\n UPDATE collections\n SET icon_url = NULL, color = NULL\n WHERE (id = $1)\n " }, + "1b66b5d566aa6a969bacbb7897af829a569e13a619db295d2e6abcdb89fcac17": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "Int8Array" + ] + } + }, + "query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n SELECT * FROM UNNEST ($1::int8[], $2::int8[])\n ON CONFLICT DO NOTHING\n " + }, + "1c30a8a31b031f194f70dc2a3bac8e131513dd7e9d2c46432ca797f6422c6ecf": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "Int8Array", + "NumericArray", + "TimestamptzArray" + ] + } + }, + "query": "\n INSERT INTO payouts_values (user_id, mod_id, amount, created)\n SELECT * FROM UNNEST ($1::bigint[], $2::bigint[], $3::numeric[], $4::timestamptz[])\n " + }, "1cefe4924d3c1f491739858ce844a22903d2dbe26f255219299f1833a10ce3d7": { "describe": { "columns": [ @@ -1201,6 +1183,24 @@ }, "query": "\n UPDATE team_members\n SET permissions = $1\n WHERE (team_id = $2 AND user_id = $3)\n " }, + "24ae57ca296554a29b414caca866cfe7ab956ea28450d40a564498c3d27b937f": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "Int8Array", + "VarcharArray", + "VarcharArray", + "BoolArray", + "Int4Array", + "VarcharArray" + ] + } + }, + "query": "\n INSERT INTO files (id, version_id, url, filename, is_primary, size, file_type)\n SELECT * FROM UNNEST($1::bigint[], $2::bigint[], $3::varchar[], $4::varchar[], $5::bool[], $6::integer[], $7::varchar[])\n " + }, "25131559cb73a088000ab6379a769233440ade6c7511542da410065190d203fc": { "describe": { "columns": [ @@ -1233,18 +1233,6 @@ }, "query": "\n DELETE FROM threads_members\n WHERE thread_id = $1\n " }, - "299b8ea6e7a0048fa389cc4432715dc2a09e227d2f08e91167a43372a7ac6e35": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8" - ] - } - }, - "query": "\n DELETE FROM mods_categories\n WHERE joining_mod_id = $1 AND is_additional = FALSE\n " - }, "29e171bd746ac5dc1fabae4c9f81c3d1df4e69c860b7d0f6a907377664199217": { "describe": { "columns": [ @@ -1265,19 +1253,6 @@ }, "query": "\n SELECT id FROM reports\n WHERE closed = FALSE\n ORDER BY created ASC\n LIMIT $1;\n " }, - "29e657d26f0fb24a766f5b5eb6a94d01d1616884d8ca10e91536e974d5b585a6": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int4", - "Int8" - ] - } - }, - "query": "\n INSERT INTO loaders_versions (loader_id, version_id)\n VALUES ($1, $2)\n " - }, "29fcff0f1d36bd1a9e0c8c4005209308f0c5f383e4e52ed8c6b989994ead32df": { "describe": { "columns": [], @@ -1462,6 +1437,19 @@ }, "query": "\n SELECT id FROM users\n WHERE email = $1\n " }, + "3151420021b0c5a85f7c338e67be971915ff89073815e27fa6af5254db22dce8": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int4Array", + "Int8Array" + ] + } + }, + "query": "\n INSERT INTO loaders_versions (loader_id, version_id)\n SELECT * FROM UNNEST($1::integer[], $2::bigint[])\n " + }, "320d73cd900a6e00f0e74b7a8c34a7658d16034b01a35558cb42fa9c16185eb5": { "describe": { "columns": [ @@ -1768,31 +1756,6 @@ }, "query": "\n UPDATE mods\n SET title = $1\n WHERE (id = $2)\n " }, - "3f2f05653552ce8c1be95ce0a922ab41f52f40f8ff6c91c6621481102c8f35e3": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int4", - "Int8" - ] - } - }, - "query": "\n INSERT INTO game_versions_versions (game_version_id, joining_version_id)\n VALUES ($1, $2)\n " - }, - "3fcfed18cbfb37866e0fa57a4e95efb326864f8219941d1b696add39ed333ad1": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8" - ] - } - }, - "query": "\n DELETE FROM mods_categories\n WHERE joining_mod_id = $1 AND is_additional = TRUE\n " - }, "40f7c5bec98fe3503d6bd6db2eae5a4edb8d5d6efda9b9dc124f344ae5c60e08": { "describe": { "columns": [], @@ -2009,6 +1972,20 @@ }, "query": "\n UPDATE uploaded_images\n SET thread_message_id = $1\n WHERE id = $2\n " }, + "473db826b691ae1131990ef0927cfe5b63d48829dd41edb7def22248d5668ac7": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "Int4Array", + "VarcharArray" + ] + } + }, + "query": "\n INSERT INTO mods_donations (\n joining_mod_id, joining_platform_id, url\n )\n SELECT * FROM UNNEST($1::bigint[], $2::int[], $3::varchar[])\n " + }, "4778d2f5994fda2f978fa53e0840c1a9a2582ef0434a5ff7f21706f1dc4edcf4": { "describe": { "columns": [], @@ -2055,19 +2032,6 @@ }, "query": "\n SELECT d.dependency_id, COALESCE(vd.mod_id, 0) mod_id, d.mod_dependency_id\n FROM versions v\n INNER JOIN dependencies d ON d.dependent_id = v.id\n LEFT JOIN versions vd ON d.dependency_id = vd.id\n WHERE v.mod_id = $1\n " }, - "489913b3c32631fb329a3259cfe620d65053e2abf425a0d3f1bc01f1cdbdd73d": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - } - }, - "query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING\n " - }, "49813a96f007216072d69468aae705d73d5b85dcdd64a22060009b12d947ed5a": { "describe": { "columns": [], @@ -2649,6 +2613,23 @@ }, "query": "SELECT id FROM users WHERE gitlab_id = $1" }, + "5d65f89c020ae032f26d742c37afe47876911eb3a16a6852299b98f2a8251fb4": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "VarcharArray", + "BoolArray", + "VarcharArray", + "VarcharArray", + "Int8Array" + ] + } + }, + "query": "\n INSERT INTO mods_gallery (\n mod_id, image_url, featured, title, description, ordering\n )\n SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bool[], $4::varchar[], $5::varchar[], $6::bigint[])\n " + }, "5d7425cfa91e332bf7cc14aa5c300b997e941c49757606f6b906cb5e060d3179": { "describe": { "columns": [], @@ -2725,22 +2706,6 @@ }, "query": "\n UPDATE mods_gallery\n SET ordering = $2\n WHERE id = $1\n " }, - "5f94e9e767ec4be7f9136b991b4a29373dbe48feb2f61281e3212721095ed675": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Varchar", - "Int8", - "Int8", - "Varchar" - ] - } - }, - "query": "\n INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name)\n VALUES ($1, $2, $3, $4, $5)\n " - }, "60a251aea1efbc7d9357255e520f0ac13f3697fecb84b1e9edd5d9ea61fe0cb0": { "describe": { "columns": [ @@ -3187,19 +3152,6 @@ }, "query": "\n UPDATE mods\n SET loaders = (\n SELECT COALESCE(ARRAY_AGG(DISTINCT l.loader) filter (where l.loader is not null), array[]::varchar[])\n FROM versions v\n INNER JOIN loaders_versions lv ON lv.version_id = v.id\n INNER JOIN loaders l on lv.loader_id = l.id\n WHERE v.mod_id = mods.id AND v.status != ALL($2)\n )\n WHERE id = $1\n " }, - "6c7aeb0db4a4fb3387c37b8d7aca6fdafaa637fd883a44416b56270aeebb7a01": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int4", - "Int8" - ] - } - }, - "query": "\n INSERT INTO loaders_versions (loader_id, version_id)\n VALUES ($1, $2)\n " - }, "6c8b8a2f11c0b4e7a5973547fe1611a0fa4ef366d5c8a91d9fb9a1360ea04d46": { "describe": { "columns": [ @@ -3358,18 +3310,38 @@ }, "query": "SELECT EXISTS(SELECT 1 FROM uploaded_images WHERE id=$1)" }, - "70b510956a40583eef8c57dcced71c67f525eee455ae8b09e9b2403668068751": { + "7075dc0343dab7c4dd4469b4af095232dcdd056a15d928a6d93556daf6fd327c": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "Int8Array", + "Int8Array", + "VarcharArray", + "Int8Array", + "Int8Array", + "BoolArray", + "NumericArray", + "Int8Array" + ] + } + }, + "query": "\n INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering)\n SELECT * FROM UNNEST ($1::int8[], $2::int8[], $3::int8[], $4::varchar[], $5::int8[], $6::int8[], $7::bool[], $8::numeric[], $9::int8[])\n " + }, + "70c812c6a0d29465569169afde42c74a353a534aeedd5cdd81bceb2a7de6bc78": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ "Int8", - "Int8" + "Bool" ] } }, - "query": "\n INSERT INTO threads_members (\n thread_id, user_id\n )\n VALUES (\n $1, $2\n )\n " + "query": "\n DELETE FROM mods_categories\n WHERE joining_mod_id = $1 AND is_additional = $2\n " }, "71abd207410d123f9a50345ddcddee335fea0d0cc6f28762713ee01a36aee8a0": { "describe": { @@ -3743,19 +3715,6 @@ }, "query": "\n SELECT id FROM mods_gallery\n WHERE image_url = $1\n " }, - "7cb691738c28e0d1f28c84ba2dbcfa21a6dbd859bcf0f565f90cd7ce2ea5aa1c": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int4" - ] - } - }, - "query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, FALSE)\n " - }, "7e030d43f3412e7df63c970f873d0a73dd2deb9857aa6f201ec5eec628eb336c": { "describe": { "columns": [], @@ -3807,35 +3766,34 @@ }, "query": "\n DELETE FROM historical_payouts\n WHERE user_id = $1\n " }, - "85463fa221147ee8d409fc92ed681fa27df683e7c80b8dd8616ae94dc1205c24": { + "8475c7cb94786576012b16d53a017cb250f0de99b76746d8725798daa3345c5e": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ - "Int8", - "Int8" + "Int8Array", + "VarcharArray", + "Int8Array", + "Int8Array", + "VarcharArray" ] } }, - "query": "\n UPDATE versions\n SET author_id = $1\n WHERE (author_id = $2)\n " + "query": "\n INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name)\n SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bigint[], $4::bigint[], $5::varchar[])\n " }, - "85b40877c48fc4f23039c1b556007f92056a015f160fe1059b0d3b13615af0fb": { + "85463fa221147ee8d409fc92ed681fa27df683e7c80b8dd8616ae94dc1205c24": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ "Int8", - "Varchar", - "Bool", - "Varchar", - "Varchar", "Int8" ] } }, - "query": "\n INSERT INTO mods_gallery (\n mod_id, image_url, featured, title, description, ordering\n )\n VALUES (\n $1, $2, $3, $4, $5, $6\n )\n " + "query": "\n UPDATE versions\n SET author_id = $1\n WHERE (author_id = $2)\n " }, "85c6de008681d9fc9dc51b17330bed09204010813111e66a7ca84bc0e603f537": { "describe": { @@ -4820,19 +4778,21 @@ }, "query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2\n " }, - "b903ac4e686ef85ba28d698c668da07860e7f276b261d8f2cebb74e73b094970": { + "b86145932b1f919fc82414c303ade80f62d4c1bc155f948359b5f6578c680244": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ - "Int8" + "Int8Array", + "Int4Array", + "BoolArray" ] } }, - "query": "\n DELETE FROM hashes\n WHERE EXISTS(\n SELECT 1 FROM files WHERE\n (files.version_id = $1) AND\n (hashes.file_id = files.id)\n )\n " + "query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n SELECT * FROM UNNEST ($1::bigint[], $2::int[], $3::bool[])\n " }, - "b9399840dbbf807a03d69b7fcb3bd479ef20920ab1e3c91706a1c2c7089f48e7": { + "b903ac4e686ef85ba28d698c668da07860e7f276b261d8f2cebb74e73b094970": { "describe": { "columns": [], "nullable": [], @@ -4842,20 +4802,19 @@ ] } }, - "query": "\n INSERT INTO teams (id)\n VALUES ($1)\n " + "query": "\n DELETE FROM hashes\n WHERE EXISTS(\n SELECT 1 FROM files WHERE\n (files.version_id = $1) AND\n (hashes.file_id = files.id)\n )\n " }, - "b96ab39ab9624bfcdc8675107544307af9892504c4cbc40e4e7c40a1e4e83e14": { + "b9399840dbbf807a03d69b7fcb3bd479ef20920ab1e3c91706a1c2c7089f48e7": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ - "Int4", "Int8" ] } }, - "query": "\n INSERT INTO game_versions_versions (game_version_id, joining_version_id)\n VALUES ($1, $2)\n " + "query": "\n INSERT INTO teams (id)\n VALUES ($1)\n " }, "b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62": { "describe": { @@ -5157,20 +5116,6 @@ }, "query": "\n UPDATE users\n SET payout_wallet = NULL, payout_wallet_type = NULL, payout_address = NULL\n WHERE (id = $1)\n " }, - "c545a74e902c5c63bca1057b76e94b9547ee21fadbc61964f45837915d5f4608": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int4", - "Varchar" - ] - } - }, - "query": "\n INSERT INTO mods_donations (\n joining_mod_id, joining_platform_id, url\n )\n VALUES (\n $1, $2, $3\n )\n " - }, "c55d2132e3e6e92dd50457affab758623dca175dc27a2d3cd4aace9cfdecf789": { "describe": { "columns": [], @@ -5217,37 +5162,30 @@ }, "query": "\n UPDATE mods\n SET client_side = $1\n WHERE (id = $2)\n " }, - "c6060a389343c9f35aea5d931518b85ab7c71b6ba74eae7b4b51d881f1798c6e": { + "c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ - "Int8", - "Int8", - "Int8", - "Varchar", - "Int8", - "Int8", - "Bool", - "Numeric", - "Int8" + "Text" ] } }, - "query": "\n INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n " + "query": "\n UPDATE versions\n SET status = requested_status\n WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL\n " }, - "c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d": { + "c8c0bf5d298810a7a30caf03d7437af757303fa9aa0f500b83476e65cec7f1e9": { "describe": { "columns": [], "nullable": [], "parameters": { "Left": [ - "Text" + "Int8Array", + "Int8Array" ] } }, - "query": "\n UPDATE versions\n SET status = requested_status\n WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL\n " + "query": "\n INSERT INTO threads_members (\n thread_id, user_id\n )\n SELECT * FROM UNNEST ($1::int8[], $2::int8[])\n " }, "c8fde56e5d03eda085519b4407768de7ddf48cae18ce7138a97e8e8fba967e15": { "describe": { @@ -5376,20 +5314,6 @@ }, "query": "\n SELECT id, user_id, session, created, last_login, expires, refresh_expires, os, platform,\n city, country, ip, user_agent\n FROM sessions\n WHERE id = ANY($1) OR session = ANY($2)\n ORDER BY created DESC\n " }, - "cb57ae673f1a7e50cc319efddb9bdc82e2251596bcf85aea52e8def343e423b8": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Varchar", - "Bytea" - ] - } - }, - "query": "\n INSERT INTO hashes (file_id, algorithm, hash)\n VALUES ($1, $2, $3)\n " - }, "cb82bb6e22690fd5fee18bbc2975503371814ef1cbf95f32c195bfe7542b2b20": { "describe": { "columns": [], @@ -5408,19 +5332,6 @@ }, "query": "\n INSERT INTO team_members (\n id, team_id, user_id, role, permissions, organization_permissions, accepted\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n " }, - "ccce60dc60ca6c4ea1142ab6d0d81bdb1ee9ed97c992695324aec015e0e190bf": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - } - }, - "query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING\n " - }, "ccd913bb2f3006ffe881ce2fc4ef1e721d18fe2eed6ac62627046c955129610c": { "describe": { "columns": [ @@ -5453,19 +5364,6 @@ }, "query": "\n DELETE FROM hashes\n WHERE file_id = $1\n " }, - "cdf20036b29b61da40bf990c9ab04c509297a4d65bc9b136c9fb20f1e97e1149": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int4" - ] - } - }, - "query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, FALSE)\n " - }, "ce20a9c53249e255be7312819f505d935d3ab2ee3c21a6422e5b12155c159bd7": { "describe": { "columns": [ @@ -5660,6 +5558,20 @@ }, "query": "\n INSERT INTO notifications (\n id, user_id, body\n )\n VALUES (\n $1, $2, $3\n )\n " }, + "d2e826d4fa4e3e730cc84c97964c0c5fdd25cd49ddff8c593bd9b8a3b4d5ff1e": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "VarcharArray", + "ByteaArray" + ] + } + }, + "query": "\n INSERT INTO hashes (file_id, algorithm, hash)\n SELECT * FROM UNNEST($1::bigint[], $2::varchar[], $3::bytea[])\n " + }, "d331ca8f22da418cf654985c822ce4466824beaa00dea64cde90dc651a03024b": { "describe": { "columns": [], @@ -5791,19 +5703,6 @@ }, "query": "\n SELECT id, team_id, role AS member_role, permissions, organization_permissions,\n accepted, payouts_split, role,\n ordering, user_id\n FROM team_members\n WHERE (team_id = ANY($1) AND user_id = $2 AND accepted = TRUE)\n ORDER BY ordering\n " }, - "d59a0ca4725d40232eae8bf5735787e1b76282c390d2a8d07fb34e237a0b2132": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int4" - ] - } - }, - "query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, TRUE)\n " - }, "d6453e50041b5521fa9e919a9162e533bb9426f8c584d98474c6ad414db715c8": { "describe": { "columns": [ @@ -6505,6 +6404,19 @@ }, "query": "\n SELECT name FROM project_types pt\n INNER JOIN mods ON mods.project_type = pt.id\n WHERE mods.id = $1\n " }, + "efdaae627a24efdf522c913cfd3600d6331e30dffbba8c2d318e44e260ac5f59": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int8Array", + "Int8Array" + ] + } + }, + "query": "\n INSERT INTO collections_mods (collection_id, mod_id)\n SELECT * FROM UNNEST($1::bigint[], $2::bigint[])\n ON CONFLICT DO NOTHING\n " + }, "f1525930830e17b5ee8feb796d9950dd3741131965f050840fa75423b5a54f01": { "describe": { "columns": [], @@ -6723,6 +6635,19 @@ }, "query": "SELECT EXISTS(SELECT 1 FROM mods m INNER JOIN team_members tm ON tm.team_id = m.team_id AND user_id = $2 WHERE m.id = $1)" }, + "fa54ed32004b883daa44eeb413fc2e07b45883608afc6ac91ac6f74736a12256": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Int4Array", + "Int8Array" + ] + } + }, + "query": "\n INSERT INTO game_versions_versions (game_version_id, joining_version_id)\n SELECT * FROM UNNEST($1::integer[], $2::bigint[])\n " + }, "fb955ca41b95120f66c98c0b528b1db10c4be4a55e9641bb104d772e390c9bb7": { "describe": { "columns": [ @@ -6743,19 +6668,6 @@ }, "query": "SELECT EXISTS(SELECT 1 FROM notifications WHERE id=$1)" }, - "fcd15905507769ab7f9839d64d1be3ee3f61cd555aee57dace76f8e53e91d344": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Int8", - "Int4" - ] - } - }, - "query": "\n INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional)\n VALUES ($1, $2, TRUE)\n " - }, "fce67ce3d0c27c64af85fb7d36661513bc5ea2e96fcf12f3a51c97999b01b83c": { "describe": { "columns": [ diff --git a/src/clickhouse/mod.rs b/src/clickhouse/mod.rs index 09722462..c46c0089 100644 --- a/src/clickhouse/mod.rs +++ b/src/clickhouse/mod.rs @@ -6,8 +6,12 @@ mod fetch; pub use fetch::*; pub async fn init_client() -> clickhouse::error::Result { - let database = dotenvy::var("CLICKHOUSE_DATABASE").unwrap(); + init_client_with_database(&dotenvy::var("CLICKHOUSE_DATABASE").unwrap()).await +} +pub async fn init_client_with_database( + database: &str, +) -> clickhouse::error::Result { let client = { let mut http_connector = HttpConnector::new(); http_connector.enforce_http(false); // allow https URLs diff --git a/src/database/models/collection_item.rs b/src/database/models/collection_item.rs index 12ff7838..a24f5dcb 100644 --- a/src/database/models/collection_item.rs +++ b/src/database/models/collection_item.rs @@ -81,19 +81,19 @@ impl Collection { .execute(&mut *transaction) .await?; - for project_id in self.projects.iter() { - sqlx::query!( - " - INSERT INTO collections_mods (collection_id, mod_id) - VALUES ($1, $2) - ON CONFLICT DO NOTHING - ", - self.id as CollectionId, - *project_id as ProjectId, - ) - .execute(&mut *transaction) - .await?; - } + let (collection_ids, project_ids): (Vec<_>, Vec<_>) = + self.projects.iter().map(|p| (self.id.0, p.0)).unzip(); + sqlx::query!( + " + INSERT INTO collections_mods (collection_id, mod_id) + SELECT * FROM UNNEST($1::bigint[], $2::bigint[]) + ON CONFLICT DO NOTHING + ", + &collection_ids[..], + &project_ids[..], + ) + .execute(&mut *transaction) + .await?; Ok(()) } diff --git a/src/database/models/project_item.rs b/src/database/models/project_item.rs index f841f934..f214c9ad 100644 --- a/src/database/models/project_item.rs +++ b/src/database/models/project_item.rs @@ -5,6 +5,7 @@ use crate::database::redis::RedisPool; use crate::models::ids::base62_impl::{parse_base62, to_base62}; use crate::models::projects::{MonetizationStatus, ProjectStatus}; use chrono::{DateTime, Utc}; +use itertools::Itertools; use serde::{Deserialize, Serialize}; pub const PROJECTS_NAMESPACE: &str = "projects"; @@ -20,23 +21,25 @@ pub struct DonationUrl { } impl DonationUrl { - pub async fn insert_project( - &self, + pub async fn insert_many_projects( + donation_urls: Vec, project_id: ProjectId, transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result<(), sqlx::error::Error> { + let (project_ids, platform_ids, urls): (Vec<_>, Vec<_>, Vec<_>) = donation_urls + .into_iter() + .map(|url| (project_id.0, url.platform_id.0, url.url)) + .multiunzip(); sqlx::query!( " INSERT INTO mods_donations ( joining_mod_id, joining_platform_id, url ) - VALUES ( - $1, $2, $3 - ) + SELECT * FROM UNNEST($1::bigint[], $2::int[], $3::varchar[]) ", - project_id as ProjectId, - self.platform_id as DonationPlatformId, - self.url, + &project_ids[..], + &platform_ids[..], + &urls[..], ) .execute(&mut *transaction) .await?; @@ -56,26 +59,76 @@ pub struct GalleryItem { } impl GalleryItem { - pub async fn insert( - &self, + pub async fn insert_many( + items: Vec, project_id: ProjectId, transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result<(), sqlx::error::Error> { + let (project_ids, image_urls, featureds, titles, descriptions, orderings): ( + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + ) = items + .into_iter() + .map(|gi| { + ( + project_id.0, + gi.image_url, + gi.featured, + gi.title, + gi.description, + gi.ordering, + ) + }) + .multiunzip(); sqlx::query!( " INSERT INTO mods_gallery ( mod_id, image_url, featured, title, description, ordering ) - VALUES ( - $1, $2, $3, $4, $5, $6 - ) + SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bool[], $4::varchar[], $5::varchar[], $6::bigint[]) ", - project_id as ProjectId, - self.image_url, - self.featured, - self.title, - self.description, - self.ordering + &project_ids[..], + &image_urls[..], + &featureds[..], + &titles[..] as &[Option], + &descriptions[..] as &[Option], + &orderings[..] + ) + .execute(&mut *transaction) + .await?; + + Ok(()) + } +} + +#[derive(derive_new::new)] +pub struct ModCategory { + project_id: ProjectId, + category_id: CategoryId, + is_additional: bool, +} + +impl ModCategory { + pub async fn insert_many( + items: Vec, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result<(), DatabaseError> { + let (project_ids, category_ids, is_additionals): (Vec<_>, Vec<_>, Vec<_>) = items + .into_iter() + .map(|mc| (mc.project_id.0, mc.category_id.0, mc.is_additional)) + .multiunzip(); + sqlx::query!( + " + INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) + SELECT * FROM UNNEST ($1::bigint[], $2::int[], $3::bool[]) + ", + &project_ids[..], + &category_ids[..], + &is_additionals[..] ) .execute(&mut *transaction) .await?; @@ -160,46 +213,35 @@ impl ProjectBuilder { }; project_struct.insert(&mut *transaction).await?; + let ProjectBuilder { + donation_urls, + gallery_items, + categories, + additional_categories, + .. + } = self; + for mut version in self.initial_versions { version.project_id = self.project_id; version.insert(&mut *transaction).await?; } - for donation in self.donation_urls { - donation - .insert_project(self.project_id, &mut *transaction) - .await?; - } - - for gallery in self.gallery_items { - gallery.insert(self.project_id, &mut *transaction).await?; - } - - for category in self.categories { - sqlx::query!( - " - INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) - VALUES ($1, $2, FALSE) - ", - self.project_id as ProjectId, - category as CategoryId, - ) - .execute(&mut *transaction) + DonationUrl::insert_many_projects(donation_urls, self.project_id, &mut *transaction) .await?; - } - for category in self.additional_categories { - sqlx::query!( - " - INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) - VALUES ($1, $2, TRUE) - ", - self.project_id as ProjectId, - category as CategoryId, + GalleryItem::insert_many(gallery_items, self.project_id, &mut *transaction).await?; + + let project_id = self.project_id; + let mod_categories = categories + .into_iter() + .map(|c| ModCategory::new(project_id, c, false)) + .chain( + additional_categories + .into_iter() + .map(|c| ModCategory::new(project_id, c, true)), ) - .execute(&mut *transaction) - .await?; - } + .collect_vec(); + ModCategory::insert_many(mod_categories, &mut *transaction).await?; Project::update_game_versions(self.project_id, &mut *transaction).await?; Project::update_loaders(self.project_id, &mut *transaction).await?; diff --git a/src/database/models/team_item.rs b/src/database/models/team_item.rs index 31d60b20..a6e7c783 100644 --- a/src/database/models/team_item.rs +++ b/src/database/models/team_item.rs @@ -41,26 +41,61 @@ impl TeamBuilder { .execute(&mut *transaction) .await?; - for member in self.members { - let team_member_id = generate_team_member_id(&mut *transaction).await?; - sqlx::query!( - " - INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - ", - team_member_id as TeamMemberId, - team.id as TeamId, - member.user_id as UserId, - member.role, - member.permissions.bits() as i64, - member.organization_permissions.map(|p| p.bits() as i64), - member.accepted, - member.payouts_split, - member.ordering, - ) - .execute(&mut *transaction) - .await?; + let mut team_member_ids = Vec::new(); + for _ in self.members.iter() { + team_member_ids.push(generate_team_member_id(&mut *transaction).await?.0); } + let TeamBuilder { members } = self; + let ( + team_ids, + user_ids, + roles, + permissions, + organization_permissions, + accepteds, + payouts_splits, + orderings, + ): ( + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + ) = members + .into_iter() + .map(|m| { + ( + team.id.0, + m.user_id.0, + m.role, + m.permissions.bits() as i64, + m.organization_permissions.map(|p| p.bits() as i64), + m.accepted, + m.payouts_split, + m.ordering, + ) + }) + .multiunzip(); + sqlx::query!( + " + INSERT INTO team_members (id, team_id, user_id, role, permissions, organization_permissions, accepted, payouts_split, ordering) + SELECT * FROM UNNEST ($1::int8[], $2::int8[], $3::int8[], $4::varchar[], $5::int8[], $6::int8[], $7::bool[], $8::numeric[], $9::int8[]) + ", + &team_member_ids[..], + &team_ids[..], + &user_ids[..], + &roles[..], + &permissions[..], + &organization_permissions[..] as &[Option], + &accepteds[..], + &payouts_splits[..], + &orderings[..], + ) + .execute(&mut *transaction) + .await?; Ok(team_id) } diff --git a/src/database/models/thread_item.rs b/src/database/models/thread_item.rs index c81b2db4..b8582cee 100644 --- a/src/database/models/thread_item.rs +++ b/src/database/models/thread_item.rs @@ -90,22 +90,20 @@ impl ThreadBuilder { .execute(&mut *transaction) .await?; - for member in &self.members { - sqlx::query!( - " - INSERT INTO threads_members ( - thread_id, user_id - ) - VALUES ( - $1, $2 - ) - ", - thread_id as ThreadId, - *member as UserId, + let (thread_ids, members): (Vec<_>, Vec<_>) = + self.members.iter().map(|m| (thread_id.0, m.0)).unzip(); + sqlx::query!( + " + INSERT INTO threads_members ( + thread_id, user_id ) - .execute(&mut *transaction) - .await?; - } + SELECT * FROM UNNEST ($1::int8[], $2::int8[]) + ", + &thread_ids[..], + &members[..], + ) + .execute(&mut *transaction) + .await?; Ok(thread_id) } diff --git a/src/database/models/version_item.rs b/src/database/models/version_item.rs index f917b20d..451f22f4 100644 --- a/src/database/models/version_item.rs +++ b/src/database/models/version_item.rs @@ -39,12 +39,59 @@ pub struct DependencyBuilder { } impl DependencyBuilder { - pub async fn insert( - self, + pub async fn insert_many( + builders: Vec, version_id: VersionId, transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result<(), DatabaseError> { - let project_id = if let Some(project_id) = self.project_id { + let mut project_ids = Vec::new(); + for dependency in builders.iter() { + project_ids.push( + dependency + .try_get_project_id(transaction) + .await? + .map(|id| id.0), + ); + } + + let (version_ids, dependency_types, dependency_ids, filenames): ( + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + ) = builders + .into_iter() + .map(|d| { + ( + version_id.0, + d.dependency_type, + d.version_id.map(|v| v.0), + d.file_name, + ) + }) + .multiunzip(); + sqlx::query!( + " + INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name) + SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::bigint[], $4::bigint[], $5::varchar[]) + ", + &version_ids[..], + &dependency_types[..], + &dependency_ids[..] as &[Option], + &project_ids[..] as &[Option], + &filenames[..] as &[Option], + ) + .execute(&mut *transaction) + .await?; + + Ok(()) + } + + async fn try_get_project_id( + &self, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result, DatabaseError> { + Ok(if let Some(project_id) = self.project_id { Some(project_id) } else if let Some(version_id) = self.version_id { sqlx::query!( @@ -58,23 +105,7 @@ impl DependencyBuilder { .map(|x| ProjectId(x.mod_id)) } else { None - }; - - sqlx::query!( - " - INSERT INTO dependencies (dependent_id, dependency_type, dependency_id, mod_dependency_id, dependency_file_name) - VALUES ($1, $2, $3, $4, $5) - ", - version_id as VersionId, - self.dependency_type, - self.version_id.map(|x| x.0), - project_id.map(|x| x.0), - self.file_name, - ) - .execute(&mut *transaction) - .await?; - - Ok(()) + }) } } @@ -89,42 +120,70 @@ pub struct VersionFileBuilder { } impl VersionFileBuilder { - pub async fn insert( - self, + pub async fn insert_many( + version_files: Vec, version_id: VersionId, transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result { let file_id = generate_file_id(&mut *transaction).await?; + let (file_ids, version_ids, urls, filenames, primary, sizes, file_types): ( + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + Vec<_>, + ) = version_files + .iter() + .map(|f| { + ( + file_id.0, + version_id.0, + f.url.clone(), + f.filename.clone(), + f.primary, + f.size as i32, + f.file_type.map(|x| x.to_string()), + ) + }) + .multiunzip(); sqlx::query!( " INSERT INTO files (id, version_id, url, filename, is_primary, size, file_type) - VALUES ($1, $2, $3, $4, $5, $6, $7) + SELECT * FROM UNNEST($1::bigint[], $2::bigint[], $3::varchar[], $4::varchar[], $5::bool[], $6::integer[], $7::varchar[]) ", - file_id as FileId, - version_id as VersionId, - self.url, - self.filename, - self.primary, - self.size as i32, - self.file_type.map(|x| x.as_str()), + &file_ids[..], + &version_ids[..], + &urls[..], + &filenames[..], + &primary[..], + &sizes[..], + &file_types[..] as &[Option], ) .execute(&mut *transaction) .await?; - for hash in self.hashes { - sqlx::query!( - " - INSERT INTO hashes (file_id, algorithm, hash) - VALUES ($1, $2, $3) - ", - file_id as FileId, - hash.algorithm, - hash.hash, - ) - .execute(&mut *transaction) - .await?; - } + let (file_ids, algorithms, hashes): (Vec<_>, Vec<_>, Vec<_>) = version_files + .into_iter() + .flat_map(|f| { + f.hashes + .into_iter() + .map(|h| (file_id.0, h.algorithm, h.hash)) + }) + .multiunzip(); + sqlx::query!( + " + INSERT INTO hashes (file_id, algorithm, hash) + SELECT * FROM UNNEST($1::bigint[], $2::varchar[], $3::bytea[]) + ", + &file_ids[..], + &algorithms[..], + &hashes[..], + ) + .execute(&mut *transaction) + .await?; Ok(file_id) } @@ -170,41 +229,91 @@ impl VersionBuilder { .execute(&mut *transaction) .await?; - for file in self.files { - file.insert(self.version_id, transaction).await?; - } + let VersionBuilder { + dependencies, + loaders, + game_versions, + files, + version_id, + .. + } = self; + VersionFileBuilder::insert_many(files, self.version_id, transaction).await?; + + DependencyBuilder::insert_many(dependencies, self.version_id, transaction).await?; + + let loader_versions = loaders + .iter() + .map(|l| LoaderVersion::new(*l, version_id)) + .collect_vec(); + LoaderVersion::insert_many(loader_versions, &mut *transaction).await?; + + let game_version_versions = game_versions + .iter() + .map(|v| VersionVersion::new(*v, version_id)) + .collect_vec(); + VersionVersion::insert_many(game_version_versions, &mut *transaction).await?; - for dependency in self.dependencies { - dependency.insert(self.version_id, transaction).await?; - } + Ok(self.version_id) + } +} - for loader in self.loaders.clone() { - sqlx::query!( - " - INSERT INTO loaders_versions (loader_id, version_id) - VALUES ($1, $2) - ", - loader as LoaderId, - self.version_id as VersionId, - ) - .execute(&mut *transaction) - .await?; - } +#[derive(derive_new::new)] +pub struct LoaderVersion { + pub loader_id: LoaderId, + pub version_id: VersionId, +} - for game_version in self.game_versions.clone() { - sqlx::query!( - " - INSERT INTO game_versions_versions (game_version_id, joining_version_id) - VALUES ($1, $2) - ", - game_version as GameVersionId, - self.version_id as VersionId, - ) - .execute(&mut *transaction) - .await?; - } +impl LoaderVersion { + pub async fn insert_many( + items: Vec, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result<(), DatabaseError> { + let (loader_ids, version_ids): (Vec<_>, Vec<_>) = items + .iter() + .map(|l| (l.loader_id.0, l.version_id.0)) + .unzip(); + sqlx::query!( + " + INSERT INTO loaders_versions (loader_id, version_id) + SELECT * FROM UNNEST($1::integer[], $2::bigint[]) + ", + &loader_ids[..], + &version_ids[..], + ) + .execute(&mut *transaction) + .await?; - Ok(self.version_id) + Ok(()) + } +} + +#[derive(derive_new::new)] +pub struct VersionVersion { + pub game_version_id: GameVersionId, + pub joining_version_id: VersionId, +} + +impl VersionVersion { + pub async fn insert_many( + items: Vec, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + ) -> Result<(), DatabaseError> { + let (game_version_ids, version_ids): (Vec<_>, Vec<_>) = items + .into_iter() + .map(|i| (i.game_version_id.0, i.joining_version_id.0)) + .unzip(); + sqlx::query!( + " + INSERT INTO game_versions_versions (game_version_id, joining_version_id) + SELECT * FROM UNNEST($1::integer[], $2::bigint[]) + ", + &game_version_ids[..], + &version_ids[..], + ) + .execute(&mut *transaction) + .await?; + + Ok(()) } } diff --git a/src/queue/analytics.rs b/src/queue/analytics.rs index 6ff153a4..78a63ebc 100644 --- a/src/queue/analytics.rs +++ b/src/queue/analytics.rs @@ -1,6 +1,13 @@ use crate::models::analytics::{Download, PageView, Playtime}; use dashmap::DashSet; +#[cfg(test)] +mod tests; + +const VIEWS_TABLENAME: &str = "views"; +const DOWNLOADS_TABLENAME: &str = "downloads"; +const PLAYTIME_TABLENAME: &str = "playtime"; + pub struct AnalyticsQueue { views_queue: DashSet, downloads_queue: DashSet, @@ -17,54 +24,50 @@ impl AnalyticsQueue { } } - pub async fn add_view(&self, page_view: PageView) { + pub fn add_view(&self, page_view: PageView) { self.views_queue.insert(page_view); } - pub async fn add_download(&self, download: Download) { + pub fn add_download(&self, download: Download) { self.downloads_queue.insert(download); } - pub async fn add_playtime(&self, playtime: Playtime) { + pub fn add_playtime(&self, playtime: Playtime) { self.playtime_queue.insert(playtime); } pub async fn index(&self, client: clickhouse::Client) -> Result<(), clickhouse::error::Error> { - let views_queue = self.views_queue.clone(); - self.views_queue.clear(); - - let downloads_queue = self.downloads_queue.clone(); - self.downloads_queue.clear(); - - let playtime_queue = self.playtime_queue.clone(); - self.playtime_queue.clear(); - - if !views_queue.is_empty() || !downloads_queue.is_empty() || !playtime_queue.is_empty() { - let mut views = client.insert("views")?; - - for view in views_queue { - views.write(&view).await?; - } + Self::index_queue(&client, &self.views_queue, VIEWS_TABLENAME).await?; + Self::index_queue(&client, &self.downloads_queue, DOWNLOADS_TABLENAME).await?; + Self::index_queue(&client, &self.playtime_queue, PLAYTIME_TABLENAME).await?; - views.end().await?; - - let mut downloads = client.insert("downloads")?; - - for download in downloads_queue { - downloads.write(&download).await?; - } + Ok(()) + } - downloads.end().await?; + async fn index_queue( + client: &clickhouse::Client, + queue: &DashSet, + table_name: &str, + ) -> Result<(), clickhouse::error::Error> + where + T: serde::Serialize + Eq + std::hash::Hash + Clone + clickhouse::Row, + { + if queue.is_empty() { + return Ok(()); + } - let mut playtimes = client.insert("playtime")?; + let current_queue = queue.clone(); + queue.clear(); - for playtime in playtime_queue { - playtimes.write(&playtime).await?; - } + let mut inserter = client.inserter(table_name)?; - playtimes.end().await?; + for row in current_queue { + inserter.write(&row).await?; + inserter.commit().await?; } + inserter.end().await?; + Ok(()) } } diff --git a/src/queue/analytics/tests.rs b/src/queue/analytics/tests.rs new file mode 100644 index 00000000..52b3d8e1 --- /dev/null +++ b/src/queue/analytics/tests.rs @@ -0,0 +1,128 @@ +use futures::Future; +use uuid::Uuid; + +use super::*; +use crate::clickhouse::init_client_with_database; +use std::net::Ipv6Addr; + +#[tokio::test] +async fn test_indexing() { + with_test_clickhouse_db(|clickhouse_client| async move { + let analytics = AnalyticsQueue::new(); + + analytics.add_download(get_default_download()); + analytics.add_playtime(get_default_playtime()); + analytics.add_view(get_default_views()); + + analytics.index(clickhouse_client.clone()).await.unwrap(); + assert_table_counts(&clickhouse_client, 1, 1, 1).await; + + analytics.index(clickhouse_client.clone()).await.unwrap(); + assert_table_counts(&clickhouse_client, 1, 1, 1).await; + }) + .await; +} + +#[tokio::test] +async fn can_insert_many_downloads() { + with_test_clickhouse_db(|clickhouse_client| async move { + let analytics = AnalyticsQueue::new(); + let n_downloads = 100_000; + + for _ in 0..n_downloads { + analytics.add_download(get_default_download()); + } + + analytics.index(clickhouse_client.clone()).await.unwrap(); + assert_table_count(DOWNLOADS_TABLENAME, &clickhouse_client, n_downloads).await; + }) + .await; +} + +async fn assert_table_counts( + client: &clickhouse::Client, + downloads: u64, + playtimes: u64, + views: u64, +) { + assert_table_count(DOWNLOADS_TABLENAME, client, downloads).await; + assert_table_count(PLAYTIME_TABLENAME, client, playtimes).await; + assert_table_count(VIEWS_TABLENAME, client, views).await; +} + +async fn assert_table_count(table_name: &str, client: &clickhouse::Client, expected_count: u64) { + let count = client + .query(&format!("SELECT COUNT(*) from {table_name}")) + .fetch_one::() + .await + .unwrap(); + assert_eq!(expected_count, count); +} + +async fn with_test_clickhouse_db(f: impl FnOnce(clickhouse::Client) -> Fut) +where + Fut: Future, +{ + let db_name = format!("test_{}", uuid::Uuid::new_v4().as_simple()); + println!("Clickhouse test db: {}", db_name); + let clickhouse_client = init_client_with_database(&db_name) + .await + .expect("A real clickhouse instance should be running locally"); + + f(clickhouse_client.clone()).await; + + clickhouse_client + .query(&format!("DROP DATABASE IF EXISTS {db_name}")) + .execute() + .await + .unwrap(); +} + +fn get_default_download() -> Download { + Download { + id: Uuid::new_v4(), + recorded: Default::default(), + domain: Default::default(), + site_path: Default::default(), + user_id: Default::default(), + project_id: Default::default(), + version_id: Default::default(), + ip: get_default_ipv6(), + country: Default::default(), + user_agent: Default::default(), + headers: Default::default(), + } +} + +fn get_default_playtime() -> Playtime { + Playtime { + id: Uuid::new_v4(), + recorded: Default::default(), + seconds: Default::default(), + user_id: Default::default(), + project_id: Default::default(), + version_id: Default::default(), + loader: Default::default(), + game_version: Default::default(), + parent: Default::default(), + } +} + +fn get_default_views() -> PageView { + PageView { + id: Uuid::new_v4(), + recorded: Default::default(), + domain: Default::default(), + site_path: Default::default(), + user_id: Default::default(), + project_id: Default::default(), + ip: get_default_ipv6(), + country: Default::default(), + user_agent: Default::default(), + headers: Default::default(), + } +} + +fn get_default_ipv6() -> Ipv6Addr { + Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0) +} diff --git a/src/queue/payouts.rs b/src/queue/payouts.rs index 73924e9d..737fe31b 100644 --- a/src/queue/payouts.rs +++ b/src/queue/payouts.rs @@ -355,6 +355,8 @@ pub async fn process_payout( }; let mut clear_cache_users = Vec::new(); + let (mut insert_user_ids, mut insert_project_ids, mut insert_payouts, mut insert_starts) = + (Vec::new(), Vec::new(), Vec::new(), Vec::new()); for (id, project) in projects_map { if let Some(value) = &multipliers.values.get(&(id as u64)) { let project_multiplier: Decimal = @@ -367,18 +369,10 @@ pub async fn process_payout( let payout: Decimal = payout * project_multiplier * (split / sum_splits); if payout > Decimal::ZERO { - sqlx::query!( - " - INSERT INTO payouts_values (user_id, mod_id, amount, created) - VALUES ($1, $2, $3, $4) - ", - user_id, - id, - payout, - start - ) - .execute(&mut *transaction) - .await?; + insert_user_ids.push(user_id); + insert_project_ids.push(id); + insert_payouts.push(payout); + insert_starts.push(start); sqlx::query!( " @@ -399,6 +393,19 @@ pub async fn process_payout( } } + sqlx::query!( + " + INSERT INTO payouts_values (user_id, mod_id, amount, created) + SELECT * FROM UNNEST ($1::bigint[], $2::bigint[], $3::numeric[], $4::timestamptz[]) + ", + &insert_user_ids[..], + &insert_project_ids[..], + &insert_payouts[..], + &insert_starts[..] + ) + .execute(&mut *transaction) + .await?; + if !clear_cache_users.is_empty() { crate::database::models::User::clear_caches( &clear_cache_users diff --git a/src/routes/analytics.rs b/src/routes/analytics.rs index 5e06b4c5..932cc8b7 100644 --- a/src/routes/analytics.rs +++ b/src/routes/analytics.rs @@ -150,7 +150,7 @@ pub async fn page_view_ingest( view.user_id = user.id.0; } - analytics_queue.add_view(view).await; + analytics_queue.add_view(view); Ok(HttpResponse::NoContent().body("")) } @@ -202,19 +202,17 @@ pub async fn playtime_ingest( } if let Some(version) = versions.iter().find(|x| id == x.inner.id.into()) { - analytics_queue - .add_playtime(Playtime { - id: Default::default(), - recorded: Utc::now().timestamp_nanos() / 100_000, - seconds: playtime.seconds as u64, - user_id: user.id.0, - project_id: version.inner.project_id.0 as u64, - version_id: version.inner.id.0 as u64, - loader: playtime.loader, - game_version: playtime.game_version, - parent: playtime.parent.map(|x| x.0).unwrap_or(0), - }) - .await; + analytics_queue.add_playtime(Playtime { + id: Default::default(), + recorded: Utc::now().timestamp_nanos() / 100_000, + seconds: playtime.seconds as u64, + user_id: user.id.0, + project_id: version.inner.project_id.0 as u64, + version_id: version.inner.id.0 as u64, + loader: playtime.loader, + game_version: playtime.game_version, + parent: playtime.parent.map(|x| x.0).unwrap_or(0), + }); } } diff --git a/src/routes/v2/admin.rs b/src/routes/v2/admin.rs index be4db052..439e83ec 100644 --- a/src/routes/v2/admin.rs +++ b/src/routes/v2/admin.rs @@ -108,40 +108,36 @@ pub async fn count_download( let ip = crate::routes::analytics::convert_to_ip_v6(&download_body.ip) .unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped()); - analytics_queue - .add_download(Download { - id: Uuid::new_v4(), - recorded: Utc::now().timestamp_nanos() / 100_000, - domain: url.host_str().unwrap_or_default().to_string(), - site_path: url.path().to_string(), - user_id: user - .and_then(|(scopes, x)| { - if scopes.contains(Scopes::PERFORM_ANALYTICS) { - Some(x.id.0 as u64) - } else { - None - } - }) - .unwrap_or(0), - project_id: project_id as u64, - version_id: version_id as u64, - ip, - country: maxmind.query(ip).await.unwrap_or_default(), - user_agent: download_body - .headers - .get("user-agent") - .cloned() - .unwrap_or_default(), - headers: download_body - .headers - .clone() - .into_iter() - .filter(|x| { - !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase()) - }) - .collect(), - }) - .await; + analytics_queue.add_download(Download { + id: Uuid::new_v4(), + recorded: Utc::now().timestamp_nanos() / 100_000, + domain: url.host_str().unwrap_or_default().to_string(), + site_path: url.path().to_string(), + user_id: user + .and_then(|(scopes, x)| { + if scopes.contains(Scopes::PERFORM_ANALYTICS) { + Some(x.id.0 as u64) + } else { + None + } + }) + .unwrap_or(0), + project_id: project_id as u64, + version_id: version_id as u64, + ip, + country: maxmind.query(ip).await.unwrap_or_default(), + user_agent: download_body + .headers + .get("user-agent") + .cloned() + .unwrap_or_default(), + headers: download_body + .headers + .clone() + .into_iter() + .filter(|x| !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase())) + .collect(), + }); Ok(HttpResponse::NoContent().body("")) } diff --git a/src/routes/v2/collections.rs b/src/routes/v2/collections.rs index 01372b0e..89778754 100644 --- a/src/routes/v2/collections.rs +++ b/src/routes/v2/collections.rs @@ -15,6 +15,7 @@ use crate::{database, models}; use actix_web::web::Data; use actix_web::{delete, get, patch, post, web, HttpRequest, HttpResponse}; use chrono::Utc; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use sqlx::PgPool; use std::sync::Arc; @@ -301,6 +302,11 @@ pub async fn collection_edit( .execute(&mut *transaction) .await?; + let collection_item_ids = new_project_ids + .iter() + .map(|_| collection_item.id.0) + .collect_vec(); + let mut validated_project_ids = Vec::new(); for project_id in new_project_ids { let project = database::models::Project::get(project_id, &**pool, &redis) .await? @@ -309,20 +315,20 @@ pub async fn collection_edit( "The specified project {project_id} does not exist!" )) })?; - - // Insert- don't throw an error if it already exists - sqlx::query!( - " - INSERT INTO collections_mods (collection_id, mod_id) - VALUES ($1, $2) - ON CONFLICT DO NOTHING - ", - collection_item.id as database::models::ids::CollectionId, - project.inner.id as database::models::ids::ProjectId, - ) - .execute(&mut *transaction) - .await?; + validated_project_ids.push(project.inner.id.0); } + // Insert- don't throw an error if it already exists + sqlx::query!( + " + INSERT INTO collections_mods (collection_id, mod_id) + SELECT * FROM UNNEST ($1::int8[], $2::int8[]) + ON CONFLICT DO NOTHING + ", + &collection_item_ids[..], + &validated_project_ids[..], + ) + .execute(&mut *transaction) + .await?; } database::models::Collection::clear_cache(collection_item.id, &redis).await?; diff --git a/src/routes/v2/projects.rs b/src/routes/v2/projects.rs index 50967487..1a0e4c69 100644 --- a/src/routes/v2/projects.rs +++ b/src/routes/v2/projects.rs @@ -2,6 +2,7 @@ use crate::auth::{filter_authorized_projects, get_user_from_headers, is_authoriz use crate::database; use crate::database::models::image_item; use crate::database::models::notification_item::NotificationBuilder; +use crate::database::models::project_item::{GalleryItem, ModCategory}; use crate::database::models::thread_item::ThreadMessageBuilder; use crate::database::redis::RedisPool; use crate::file_hosting::FileHost; @@ -31,6 +32,9 @@ use sqlx::PgPool; use std::sync::Arc; use validator::Validate; +use database::models as db_models; +use db_models::ids as db_ids; + pub fn config(cfg: &mut web::ServiceConfig) { cfg.service(project_search); cfg.service(projects_get); @@ -97,11 +101,11 @@ pub async fn random_projects_get( .collect::>(), ) .fetch_many(&**pool) - .try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ids::ProjectId(m.id))) }) + .try_filter_map(|e| async { Ok(e.right().map(|m| db_ids::ProjectId(m.id))) }) .try_collect::>() .await?; - let projects_data = database::models::Project::get_many_ids(&project_ids, &**pool, &redis) + let projects_data = db_models::Project::get_many_ids(&project_ids, &**pool, &redis) .await? .into_iter() .map(Project::from) @@ -124,7 +128,7 @@ pub async fn projects_get( session_queue: web::Data, ) -> Result { let ids = serde_json::from_str::>(&ids.ids)?; - let projects_data = database::models::Project::get_many(&ids, &**pool, &redis).await?; + let projects_data = db_models::Project::get_many(&ids, &**pool, &redis).await?; let user_option = get_user_from_headers( &req, @@ -152,7 +156,7 @@ pub async fn project_get( ) -> Result { let string = info.into_inner().0; - let project_data = database::models::Project::get(&string, &**pool, &redis).await?; + let project_data = db_models::Project::get(&string, &**pool, &redis).await?; let user_option = get_user_from_headers( &req, &**pool, @@ -181,7 +185,7 @@ pub async fn project_get_check( ) -> Result { let slug = info.into_inner().0; - let project_data = database::models::Project::get(&slug, &**pool, &redis).await?; + let project_data = db_models::Project::get(&slug, &**pool, &redis).await?; if let Some(project) = project_data { Ok(HttpResponse::Ok().json(json! ({ @@ -208,7 +212,7 @@ pub async fn dependency_list( ) -> Result { let string = info.into_inner().0; - let result = database::models::Project::get(&string, &**pool, &redis).await?; + let result = db_models::Project::get(&string, &**pool, &redis).await?; let user_option = get_user_from_headers( &req, @@ -247,7 +251,7 @@ pub async fn dependency_list( let dep_version_ids = dependencies .iter() .filter_map(|x| x.0) - .collect::>(); + .collect::>(); let (projects_result, versions_result) = futures::future::try_join( database::Project::get_many_ids(&project_ids, &**pool, &redis), database::Version::get_many(&dep_version_ids, &**pool, &redis), @@ -399,13 +403,13 @@ pub async fn project_edit( .map_err(|err| ApiError::Validation(validation_errors_to_string(err, None)))?; let string = info.into_inner().0; - let result = database::models::Project::get(&string, &**pool, &redis).await?; + let result = db_models::Project::get(&string, &**pool, &redis).await?; if let Some(project_item) = result { let id = project_item.inner.id; let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -436,7 +440,7 @@ pub async fn project_edit( WHERE (id = $2) ", title.trim(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -457,7 +461,7 @@ pub async fn project_edit( WHERE (id = $2) ", description, - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -494,7 +498,7 @@ pub async fn project_edit( SET moderation_message = NULL, moderation_message_body = NULL, queued = NOW() WHERE (id = $1) ", - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -505,7 +509,7 @@ pub async fn project_edit( SET show_in_mod_inbox = FALSE WHERE id = $1 ", - project_item.thread_id as database::models::ids::ThreadId, + project_item.thread_id as db_ids::ThreadId, ) .execute(&mut *transaction) .await?; @@ -518,7 +522,7 @@ pub async fn project_edit( SET approved = NOW() WHERE id = $1 AND approved IS NULL ", - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -542,7 +546,7 @@ pub async fn project_edit( SET webhook_sent = TRUE WHERE id = $1 ", - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -580,12 +584,10 @@ pub async fn project_edit( FROM team_members tm WHERE tm.team_id = $1 AND tm.accepted ", - project_item.inner.team_id as database::models::ids::TeamId + project_item.inner.team_id as db_ids::TeamId ) .fetch_many(&mut *transaction) - .try_filter_map(|e| async { - Ok(e.right().map(|c| database::models::UserId(c.id))) - }) + .try_filter_map(|e| async { Ok(e.right().map(|c| db_models::UserId(c.id))) }) .try_collect::>() .await?; @@ -618,7 +620,7 @@ pub async fn project_edit( WHERE (id = $2) ", status.as_str(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -652,7 +654,7 @@ pub async fn project_edit( WHERE (id = $2) ", requested_status.map(|x| x.as_str()), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -665,7 +667,7 @@ pub async fn project_edit( DELETE FROM mods_categories WHERE joining_mod_id = $1 AND is_additional = FALSE ", - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -677,7 +679,7 @@ pub async fn project_edit( DELETE FROM mods_categories WHERE joining_mod_id = $1 AND is_additional = TRUE ", - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -685,67 +687,25 @@ pub async fn project_edit( } if let Some(categories) = &new_project.categories { - if !perms.contains(ProjectPermissions::EDIT_DETAILS) { - return Err(ApiError::CustomAuthentication( - "You do not have the permissions to edit the categories of this project!" - .to_string(), - )); - } - - for category in categories { - let category_id = - database::models::categories::Category::get_id(category, &mut *transaction) - .await? - .ok_or_else(|| { - ApiError::InvalidInput(format!( - "Category {} does not exist.", - category.clone() - )) - })?; - - sqlx::query!( - " - INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) - VALUES ($1, $2, FALSE) - ", - id as database::models::ids::ProjectId, - category_id as database::models::ids::CategoryId, - ) - .execute(&mut *transaction) - .await?; - } + edit_project_categories( + categories, + &perms, + id as db_ids::ProjectId, + false, + &mut transaction, + ) + .await?; } if let Some(categories) = &new_project.additional_categories { - if !perms.contains(ProjectPermissions::EDIT_DETAILS) { - return Err(ApiError::CustomAuthentication( - "You do not have the permissions to edit the additional categories of this project!" - .to_string(), - )); - } - - for category in categories { - let category_id = - database::models::categories::Category::get_id(category, &mut *transaction) - .await? - .ok_or_else(|| { - ApiError::InvalidInput(format!( - "Category {} does not exist.", - category.clone() - )) - })?; - - sqlx::query!( - " - INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) - VALUES ($1, $2, TRUE) - ", - id as database::models::ids::ProjectId, - category_id as database::models::ids::CategoryId, - ) - .execute(&mut *transaction) - .await?; - } + edit_project_categories( + categories, + &perms, + id as db_ids::ProjectId, + true, + &mut transaction, + ) + .await?; } if let Some(issues_url) = &new_project.issues_url { @@ -763,7 +723,7 @@ pub async fn project_edit( WHERE (id = $2) ", issues_url.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -784,7 +744,7 @@ pub async fn project_edit( WHERE (id = $2) ", source_url.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -805,7 +765,7 @@ pub async fn project_edit( WHERE (id = $2) ", wiki_url.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -826,7 +786,7 @@ pub async fn project_edit( WHERE (id = $2) ", license_url.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -847,7 +807,7 @@ pub async fn project_edit( WHERE (id = $2) ", discord_url.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -905,7 +865,7 @@ pub async fn project_edit( WHERE (id = $2) ", Some(slug), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -919,12 +879,10 @@ pub async fn project_edit( )); } - let side_type_id = database::models::categories::SideType::get_id( - new_side.as_str(), - &mut *transaction, - ) - .await? - .expect("No database entry found for side type"); + let side_type_id = + db_models::categories::SideType::get_id(new_side.as_str(), &mut *transaction) + .await? + .expect("No database entry found for side type"); sqlx::query!( " @@ -932,8 +890,8 @@ pub async fn project_edit( SET client_side = $1 WHERE (id = $2) ", - side_type_id as database::models::SideTypeId, - id as database::models::ids::ProjectId, + side_type_id as db_models::SideTypeId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -947,12 +905,10 @@ pub async fn project_edit( )); } - let side_type_id = database::models::categories::SideType::get_id( - new_side.as_str(), - &mut *transaction, - ) - .await? - .expect("No database entry found for side type"); + let side_type_id = + db_models::categories::SideType::get_id(new_side.as_str(), &mut *transaction) + .await? + .expect("No database entry found for side type"); sqlx::query!( " @@ -960,8 +916,8 @@ pub async fn project_edit( SET server_side = $1 WHERE (id = $2) ", - side_type_id as database::models::SideTypeId, - id as database::models::ids::ProjectId, + side_type_id as db_models::SideTypeId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -992,7 +948,7 @@ pub async fn project_edit( WHERE (id = $2) ", license, - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1010,13 +966,13 @@ pub async fn project_edit( DELETE FROM mods_donations WHERE joining_mod_id = $1 ", - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; for donation in donations { - let platform_id = database::models::categories::DonationPlatform::get_id( + let platform_id = db_models::categories::DonationPlatform::get_id( &donation.id, &mut *transaction, ) @@ -1033,8 +989,8 @@ pub async fn project_edit( INSERT INTO mods_donations (joining_mod_id, joining_platform_id, url) VALUES ($1, $2, $3) ", - id as database::models::ids::ProjectId, - platform_id as database::models::ids::DonationPlatformId, + id as db_ids::ProjectId, + platform_id as db_ids::DonationPlatformId, donation.url ) .execute(&mut *transaction) @@ -1059,7 +1015,7 @@ pub async fn project_edit( WHERE (id = $2) ", moderation_message.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1083,7 +1039,7 @@ pub async fn project_edit( WHERE (id = $2) ", moderation_message_body.as_deref(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1104,7 +1060,7 @@ pub async fn project_edit( WHERE (id = $2) ", body, - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1136,7 +1092,7 @@ pub async fn project_edit( WHERE (id = $2) ", monetization_status.as_str(), - id as database::models::ids::ProjectId, + id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1154,7 +1110,7 @@ pub async fn project_edit( }; img::delete_unused_images(context, checkable_strings, &mut transaction, &redis).await?; - database::models::Project::clear_cache( + db_models::Project::clear_cache( project_item.inner.id, project_item.inner.slug, None, @@ -1174,6 +1130,13 @@ pub async fn project_edit( } } +#[derive(derive_new::new)] +pub struct CategoryChanges<'a> { + pub categories: &'a Option>, + pub add_categories: &'a Option>, + pub remove_categories: &'a Option>, +} + #[derive(Deserialize, Validate)] pub struct BulkEditProject { #[validate(length(max = 3))] @@ -1260,14 +1223,12 @@ pub async fn projects_edit( .validate() .map_err(|err| ApiError::Validation(validation_errors_to_string(err, None)))?; - let project_ids: Vec = - serde_json::from_str::>(&ids.ids)? - .into_iter() - .map(|x| x.into()) - .collect(); + let project_ids: Vec = serde_json::from_str::>(&ids.ids)? + .into_iter() + .map(|x| x.into()) + .collect(); - let projects_data = - database::models::Project::get_many_ids(&project_ids, &**pool, &redis).await?; + let projects_data = db_models::Project::get_many_ids(&project_ids, &**pool, &redis).await?; if let Some(id) = project_ids .iter() @@ -1282,31 +1243,27 @@ pub async fn projects_edit( let team_ids = projects_data .iter() .map(|x| x.inner.team_id) - .collect::>(); + .collect::>(); let team_members = - database::models::TeamMember::get_from_team_full_many(&team_ids, &**pool, &redis).await?; + db_models::TeamMember::get_from_team_full_many(&team_ids, &**pool, &redis).await?; let organization_ids = projects_data .iter() .filter_map(|x| x.inner.organization_id) - .collect::>(); + .collect::>(); let organizations = - database::models::Organization::get_many_ids(&organization_ids, &**pool, &redis).await?; + db_models::Organization::get_many_ids(&organization_ids, &**pool, &redis).await?; let organization_team_ids = organizations .iter() .map(|x| x.team_id) - .collect::>(); - let organization_team_members = database::models::TeamMember::get_from_team_full_many( - &organization_team_ids, - &**pool, - &redis, - ) - .await?; + .collect::>(); + let organization_team_members = + db_models::TeamMember::get_from_team_full_many(&organization_team_ids, &**pool, &redis) + .await?; - let categories = database::models::categories::Category::list(&**pool, &redis).await?; - let donation_platforms = - database::models::categories::DonationPlatform::list(&**pool, &redis).await?; + let categories = db_models::categories::Category::list(&**pool, &redis).await?; + let donation_platforms = db_models::categories::DonationPlatform::list(&**pool, &redis).await?; let mut transaction = pool.begin().await?; @@ -1356,126 +1313,35 @@ pub async fn projects_edit( }; } - let mut set_categories = if let Some(categories) = bulk_edit_project.categories.clone() { - categories - } else { - project.categories.clone() - }; - - if let Some(delete_categories) = &bulk_edit_project.remove_categories { - for category in delete_categories { - if let Some(pos) = set_categories.iter().position(|x| x == category) { - set_categories.remove(pos); - } - } - } - - if let Some(add_categories) = &bulk_edit_project.add_categories { - for category in add_categories { - if set_categories.len() < 3 { - set_categories.push(category.clone()); - } else { - break; - } - } - } - - if set_categories != project.categories { - sqlx::query!( - " - DELETE FROM mods_categories - WHERE joining_mod_id = $1 AND is_additional = FALSE - ", - project.inner.id as database::models::ids::ProjectId, - ) - .execute(&mut *transaction) - .await?; - - for category in set_categories { - let category_id = categories - .iter() - .find(|x| x.category == category) - .ok_or_else(|| { - ApiError::InvalidInput(format!( - "Category {} does not exist.", - category.clone() - )) - })? - .id; - - sqlx::query!( - " - INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) - VALUES ($1, $2, FALSE) - ", - project.inner.id as database::models::ids::ProjectId, - category_id as database::models::ids::CategoryId, - ) - .execute(&mut *transaction) - .await?; - } - } - - let mut set_additional_categories = - if let Some(categories) = bulk_edit_project.additional_categories.clone() { - categories - } else { - project.additional_categories.clone() - }; - - if let Some(delete_categories) = &bulk_edit_project.remove_additional_categories { - for category in delete_categories { - if let Some(pos) = set_additional_categories.iter().position(|x| x == category) { - set_additional_categories.remove(pos); - } - } - } - - if let Some(add_categories) = &bulk_edit_project.add_additional_categories { - for category in add_categories { - if set_additional_categories.len() < 256 { - set_additional_categories.push(category.clone()); - } else { - break; - } - } - } - - if set_additional_categories != project.additional_categories { - sqlx::query!( - " - DELETE FROM mods_categories - WHERE joining_mod_id = $1 AND is_additional = TRUE - ", - project.inner.id as database::models::ids::ProjectId, - ) - .execute(&mut *transaction) - .await?; - - for category in set_additional_categories { - let category_id = categories - .iter() - .find(|x| x.category == category) - .ok_or_else(|| { - ApiError::InvalidInput(format!( - "Category {} does not exist.", - category.clone() - )) - })? - .id; + bulk_edit_project_categories( + &categories, + &project.categories, + project.inner.id as db_ids::ProjectId, + CategoryChanges::new( + &bulk_edit_project.categories, + &bulk_edit_project.add_categories, + &bulk_edit_project.remove_categories, + ), + 3, + false, + &mut transaction, + ) + .await?; - sqlx::query!( - " - INSERT INTO mods_categories (joining_mod_id, joining_category_id, is_additional) - VALUES ($1, $2, TRUE) - ", - project.inner.id as database::models::ids::ProjectId, - category_id as database::models::ids::CategoryId, - ) - .execute(&mut *transaction) - .await?; - } - } + bulk_edit_project_categories( + &categories, + &project.additional_categories, + project.inner.id as db_ids::ProjectId, + CategoryChanges::new( + &bulk_edit_project.additional_categories, + &bulk_edit_project.add_additional_categories, + &bulk_edit_project.remove_additional_categories, + ), + 256, + true, + &mut transaction, + ) + .await?; let project_donations: Vec = project .donation_urls @@ -1514,7 +1380,7 @@ pub async fn projects_edit( DELETE FROM mods_donations WHERE joining_mod_id = $1 ", - project.inner.id as database::models::ids::ProjectId, + project.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1536,8 +1402,8 @@ pub async fn projects_edit( INSERT INTO mods_donations (joining_mod_id, joining_platform_id, url) VALUES ($1, $2, $3) ", - project.inner.id as database::models::ids::ProjectId, - platform_id as database::models::ids::DonationPlatformId, + project.inner.id as db_ids::ProjectId, + platform_id as db_ids::DonationPlatformId, donation.url ) .execute(&mut *transaction) @@ -1553,7 +1419,7 @@ pub async fn projects_edit( WHERE (id = $2) ", issues_url.as_deref(), - project.inner.id as database::models::ids::ProjectId, + project.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1567,7 +1433,7 @@ pub async fn projects_edit( WHERE (id = $2) ", source_url.as_deref(), - project.inner.id as database::models::ids::ProjectId, + project.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1581,7 +1447,7 @@ pub async fn projects_edit( WHERE (id = $2) ", wiki_url.as_deref(), - project.inner.id as database::models::ids::ProjectId, + project.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -1595,14 +1461,13 @@ pub async fn projects_edit( WHERE (id = $2) ", discord_url.as_deref(), - project.inner.id as database::models::ids::ProjectId, + project.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; } - database::models::Project::clear_cache(project.inner.id, project.inner.slug, None, &redis) - .await?; + db_models::Project::clear_cache(project.inner.id, project.inner.slug, None, &redis).await?; } transaction.commit().await?; @@ -1610,6 +1475,96 @@ pub async fn projects_edit( Ok(HttpResponse::NoContent().body("")) } +pub async fn bulk_edit_project_categories( + all_db_categories: &[db_models::categories::Category], + project_categories: &Vec, + project_id: db_ids::ProjectId, + bulk_changes: CategoryChanges<'_>, + max_num_categories: usize, + is_additional: bool, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, +) -> Result<(), ApiError> { + let mut set_categories = if let Some(categories) = bulk_changes.categories.clone() { + categories + } else { + project_categories.clone() + }; + + if let Some(delete_categories) = &bulk_changes.remove_categories { + for category in delete_categories { + if let Some(pos) = set_categories.iter().position(|x| x == category) { + set_categories.remove(pos); + } + } + } + + if let Some(add_categories) = &bulk_changes.add_categories { + for category in add_categories { + if set_categories.len() < max_num_categories { + set_categories.push(category.clone()); + } else { + break; + } + } + } + + if &set_categories != project_categories { + sqlx::query!( + " + DELETE FROM mods_categories + WHERE joining_mod_id = $1 AND is_additional = $2 + ", + project_id as db_ids::ProjectId, + is_additional + ) + .execute(&mut *transaction) + .await?; + + let mut mod_categories = Vec::new(); + for category in set_categories { + let category_id = all_db_categories + .iter() + .find(|x| x.category == category) + .ok_or_else(|| { + ApiError::InvalidInput(format!("Category {} does not exist.", category.clone())) + })? + .id; + mod_categories.push(ModCategory::new(project_id, category_id, is_additional)); + } + ModCategory::insert_many(mod_categories, &mut *transaction).await?; + } + + Ok(()) +} + +pub async fn edit_project_categories( + categories: &Vec, + perms: &ProjectPermissions, + project_id: db_ids::ProjectId, + additional: bool, + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, +) -> Result<(), ApiError> { + if !perms.contains(ProjectPermissions::EDIT_DETAILS) { + let additional_str = if additional { "additional " } else { "" }; + return Err(ApiError::CustomAuthentication(format!( + "You do not have the permissions to edit the {additional_str}categories of this project!" + ))); + } + + let mut mod_categories = Vec::new(); + for category in categories { + let category_id = db_models::categories::Category::get_id(category, &mut *transaction) + .await? + .ok_or_else(|| { + ApiError::InvalidInput(format!("Category {} does not exist.", category.clone())) + })?; + mod_categories.push(ModCategory::new(project_id, category_id, additional)); + } + ModCategory::insert_many(mod_categories, &mut *transaction).await?; + + Ok(()) +} + #[derive(Deserialize)] pub struct SchedulingData { pub time: DateTime, @@ -1648,11 +1603,11 @@ pub async fn project_schedule( } let string = info.into_inner().0; - let result = database::models::Project::get(&string, &**pool, &redis).await?; + let result = db_models::Project::get(&string, &**pool, &redis).await?; if let Some(project_item) = result { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -1692,12 +1647,12 @@ pub async fn project_schedule( ", ProjectStatus::Scheduled.as_str(), scheduling_data.time, - project_item.inner.id as database::models::ids::ProjectId, + project_item.inner.id as db_ids::ProjectId, ) .execute(&**pool) .await?; - database::models::Project::clear_cache( + db_models::Project::clear_cache( project_item.inner.id, project_item.inner.slug, None, @@ -1741,7 +1696,7 @@ pub async fn project_icon_edit( .1; let string = info.into_inner().0; - let project_item = database::models::Project::get(&string, &**pool, &redis) + let project_item = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) @@ -1749,7 +1704,7 @@ pub async fn project_icon_edit( if !user.role.is_mod() { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -1810,12 +1765,12 @@ pub async fn project_icon_edit( ", format!("{}/{}", cdn_url, upload_data.file_name), color.map(|x| x as i32), - project_item.inner.id as database::models::ids::ProjectId, + project_item.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; - database::models::Project::clear_cache( + db_models::Project::clear_cache( project_item.inner.id, project_item.inner.slug, None, @@ -1854,7 +1809,7 @@ pub async fn delete_project_icon( .1; let string = info.into_inner().0; - let project_item = database::models::Project::get(&string, &**pool, &redis) + let project_item = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) @@ -1862,7 +1817,7 @@ pub async fn delete_project_icon( if !user.role.is_mod() { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -1906,18 +1861,13 @@ pub async fn delete_project_icon( SET icon_url = NULL, color = NULL WHERE (id = $1) ", - project_item.inner.id as database::models::ids::ProjectId, + project_item.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; - database::models::Project::clear_cache( - project_item.inner.id, - project_item.inner.slug, - None, - &redis, - ) - .await?; + db_models::Project::clear_cache(project_item.inner.id, project_item.inner.slug, None, &redis) + .await?; transaction.commit().await?; @@ -1963,7 +1913,7 @@ pub async fn add_gallery_item( .1; let string = info.into_inner().0; - let project_item = database::models::Project::get(&string, &**pool, &redis) + let project_item = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) @@ -1977,7 +1927,7 @@ pub async fn add_gallery_item( if !user.role.is_admin() { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -2040,25 +1990,24 @@ pub async fn add_gallery_item( SET featured = $2 WHERE mod_id = $1 ", - project_item.inner.id as database::models::ids::ProjectId, + project_item.inner.id as db_ids::ProjectId, false, ) .execute(&mut *transaction) .await?; } - database::models::project_item::GalleryItem { + let gallery_item = vec![db_models::project_item::GalleryItem { image_url: file_url, featured: item.featured, title: item.title, description: item.description, created: Utc::now(), ordering: item.ordering.unwrap_or(0), - } - .insert(project_item.inner.id, &mut transaction) - .await?; + }]; + GalleryItem::insert_many(gallery_item, project_item.inner.id, &mut transaction).await?; - database::models::Project::clear_cache( + db_models::Project::clear_cache( project_item.inner.id, project_item.inner.slug, None, @@ -2122,7 +2071,7 @@ pub async fn edit_gallery_item( item.validate() .map_err(|err| ApiError::Validation(validation_errors_to_string(err, None)))?; - let project_item = database::models::Project::get(&string, &**pool, &redis) + let project_item = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) @@ -2130,7 +2079,7 @@ pub async fn edit_gallery_item( if !user.role.is_mod() { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -2185,7 +2134,7 @@ pub async fn edit_gallery_item( SET featured = $2 WHERE mod_id = $1 ", - project_item.inner.id as database::models::ids::ProjectId, + project_item.inner.id as db_ids::ProjectId, false, ) .execute(&mut *transaction) @@ -2244,13 +2193,8 @@ pub async fn edit_gallery_item( .await?; } - database::models::Project::clear_cache( - project_item.inner.id, - project_item.inner.slug, - None, - &redis, - ) - .await?; + db_models::Project::clear_cache(project_item.inner.id, project_item.inner.slug, None, &redis) + .await?; transaction.commit().await?; @@ -2283,7 +2227,7 @@ pub async fn delete_gallery_item( .1; let string = info.into_inner().0; - let project_item = database::models::Project::get(&string, &**pool, &redis) + let project_item = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) @@ -2291,7 +2235,7 @@ pub async fn delete_gallery_item( if !user.role.is_mod() { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project_item.inner, user.id.into(), &**pool, @@ -2356,13 +2300,8 @@ pub async fn delete_gallery_item( .execute(&mut *transaction) .await?; - database::models::Project::clear_cache( - project_item.inner.id, - project_item.inner.slug, - None, - &redis, - ) - .await?; + db_models::Project::clear_cache(project_item.inner.id, project_item.inner.slug, None, &redis) + .await?; transaction.commit().await?; @@ -2389,7 +2328,7 @@ pub async fn project_delete( .1; let string = info.into_inner().0; - let project = database::models::Project::get(&string, &**pool, &redis) + let project = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) @@ -2397,7 +2336,7 @@ pub async fn project_delete( if !user.role.is_admin() { let (team_member, organization_team_member) = - database::models::TeamMember::get_for_project_permissions( + db_models::TeamMember::get_for_project_permissions( &project.inner, user.id.into(), &**pool, @@ -2429,8 +2368,7 @@ pub async fn project_delete( let context = ImageContext::Project { project_id: Some(project.inner.id.into()), }; - let uploaded_images = - database::models::Image::get_many_contexted(context, &mut transaction).await?; + let uploaded_images = db_models::Image::get_many_contexted(context, &mut transaction).await?; for image in uploaded_images { image_item::Image::remove(image.id, &mut transaction, &redis).await?; } @@ -2440,13 +2378,12 @@ pub async fn project_delete( DELETE FROM collections_mods WHERE mod_id = $1 ", - project.inner.id as database::models::ids::ProjectId, + project.inner.id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; - let result = - database::models::Project::remove(project.inner.id, &mut transaction, &redis).await?; + let result = db_models::Project::remove(project.inner.id, &mut transaction, &redis).await?; transaction.commit().await?; @@ -2478,14 +2415,14 @@ pub async fn project_follow( .1; let string = info.into_inner().0; - let result = database::models::Project::get(&string, &**pool, &redis) + let result = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) })?; - let user_id: database::models::ids::UserId = user.id.into(); - let project_id: database::models::ids::ProjectId = result.inner.id; + let user_id: db_ids::UserId = user.id.into(); + let project_id: db_ids::ProjectId = result.inner.id; if !is_authorized(&result.inner, &Some(user), &pool).await? { return Ok(HttpResponse::NotFound().body("")); @@ -2495,8 +2432,8 @@ pub async fn project_follow( " SELECT EXISTS(SELECT 1 FROM mod_follows mf WHERE mf.follower_id = $1 AND mf.mod_id = $2) ", - user_id as database::models::ids::UserId, - project_id as database::models::ids::ProjectId + user_id as db_ids::UserId, + project_id as db_ids::ProjectId ) .fetch_one(&**pool) .await? @@ -2512,7 +2449,7 @@ pub async fn project_follow( SET follows = follows + 1 WHERE id = $1 ", - project_id as database::models::ids::ProjectId, + project_id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -2522,8 +2459,8 @@ pub async fn project_follow( INSERT INTO mod_follows (follower_id, mod_id) VALUES ($1, $2) ", - user_id as database::models::ids::UserId, - project_id as database::models::ids::ProjectId + user_id as db_ids::UserId, + project_id as db_ids::ProjectId ) .execute(&mut *transaction) .await?; @@ -2557,21 +2494,21 @@ pub async fn project_unfollow( .1; let string = info.into_inner().0; - let result = database::models::Project::get(&string, &**pool, &redis) + let result = db_models::Project::get(&string, &**pool, &redis) .await? .ok_or_else(|| { ApiError::InvalidInput("The specified project does not exist!".to_string()) })?; - let user_id: database::models::ids::UserId = user.id.into(); + let user_id: db_ids::UserId = user.id.into(); let project_id = result.inner.id; let following = sqlx::query!( " SELECT EXISTS(SELECT 1 FROM mod_follows mf WHERE mf.follower_id = $1 AND mf.mod_id = $2) ", - user_id as database::models::ids::UserId, - project_id as database::models::ids::ProjectId + user_id as db_ids::UserId, + project_id as db_ids::ProjectId ) .fetch_one(&**pool) .await? @@ -2587,7 +2524,7 @@ pub async fn project_unfollow( SET follows = follows - 1 WHERE id = $1 ", - project_id as database::models::ids::ProjectId, + project_id as db_ids::ProjectId, ) .execute(&mut *transaction) .await?; @@ -2597,8 +2534,8 @@ pub async fn project_unfollow( DELETE FROM mod_follows WHERE follower_id = $1 AND mod_id = $2 ", - user_id as database::models::ids::UserId, - project_id as database::models::ids::ProjectId + user_id as db_ids::UserId, + project_id as db_ids::ProjectId ) .execute(&mut *transaction) .await?; diff --git a/src/routes/v2/version_creation.rs b/src/routes/v2/version_creation.rs index 80fc895d..9c7b8611 100644 --- a/src/routes/v2/version_creation.rs +++ b/src/routes/v2/version_creation.rs @@ -725,9 +725,7 @@ async fn upload_file_to_version_inner( "At least one file must be specified".to_string(), )); } else { - for file_builder in file_builders { - file_builder.insert(version_id, &mut *transaction).await?; - } + VersionFileBuilder::insert_many(file_builders, version_id, &mut *transaction).await?; } // Clear version cache diff --git a/src/routes/v2/versions.rs b/src/routes/v2/versions.rs index cfaa9da4..218706e9 100644 --- a/src/routes/v2/versions.rs +++ b/src/routes/v2/versions.rs @@ -3,6 +3,7 @@ use crate::auth::{ filter_authorized_versions, get_user_from_headers, is_authorized, is_authorized_version, }; use crate::database; +use crate::database::models::version_item::{DependencyBuilder, LoaderVersion, VersionVersion}; use crate::database::models::{image_item, Organization}; use crate::database::redis::RedisPool; use crate::models; @@ -450,11 +451,12 @@ pub async fn version_edit( }) .collect::>(); - for dependency in builders { - dependency - .insert(version_item.inner.id, &mut transaction) - .await?; - } + DependencyBuilder::insert_many( + builders, + version_item.inner.id, + &mut transaction, + ) + .await?; } } } @@ -469,6 +471,7 @@ pub async fn version_edit( .execute(&mut *transaction) .await?; + let mut version_versions = Vec::new(); for game_version in game_versions { let game_version_id = database::models::categories::GameVersion::get_id( &game_version.0, @@ -481,17 +484,9 @@ pub async fn version_edit( ) })?; - sqlx::query!( - " - INSERT INTO game_versions_versions (game_version_id, joining_version_id) - VALUES ($1, $2) - ", - game_version_id as database::models::ids::GameVersionId, - id as database::models::ids::VersionId, - ) - .execute(&mut *transaction) - .await?; + version_versions.push(VersionVersion::new(game_version_id, id)); } + VersionVersion::insert_many(version_versions, &mut transaction).await?; database::models::Project::update_game_versions( version_item.inner.project_id, @@ -510,6 +505,7 @@ pub async fn version_edit( .execute(&mut *transaction) .await?; + let mut loader_versions = Vec::new(); for loader in loaders { let loader_id = database::models::categories::Loader::get_id(&loader.0, &mut *transaction) @@ -519,18 +515,9 @@ pub async fn version_edit( "No database entry for loader provided.".to_string(), ) })?; - - sqlx::query!( - " - INSERT INTO loaders_versions (loader_id, version_id) - VALUES ($1, $2) - ", - loader_id as database::models::ids::LoaderId, - id as database::models::ids::VersionId, - ) - .execute(&mut *transaction) - .await?; + loader_versions.push(LoaderVersion::new(loader_id, id)); } + LoaderVersion::insert_many(loader_versions, &mut transaction).await?; database::models::Project::update_loaders( version_item.inner.project_id, diff --git a/tests/common/database.rs b/tests/common/database.rs index 63535125..483a44d9 100644 --- a/tests/common/database.rs +++ b/tests/common/database.rs @@ -29,6 +29,7 @@ pub const USER_USER_PAT: &str = "mrp_patuser"; pub const FRIEND_USER_PAT: &str = "mrp_patfriend"; pub const ENEMY_USER_PAT: &str = "mrp_patenemy"; +#[derive(Clone)] pub struct TemporaryDatabase { pub pool: PgPool, pub redis_pool: RedisPool, @@ -75,10 +76,14 @@ impl TemporaryDatabase { .await .expect("Connection to temporary database failed"); + println!("Running migrations on temporary database"); + // Performs migrations let migrations = sqlx::migrate!("./migrations"); migrations.run(&pool).await.expect("Migrations failed"); + println!("Migrations complete"); + // Gets new Redis pool let redis_pool = RedisPool::new(Some(temp_database_name.clone())); diff --git a/tests/common/dummy_data.rs b/tests/common/dummy_data.rs index d3cd9667..35cdc97d 100644 --- a/tests/common/dummy_data.rs +++ b/tests/common/dummy_data.rs @@ -13,6 +13,16 @@ use super::{ environment::TestEnvironment, }; +pub const DUMMY_CATEGORIES: &'static [&str] = &[ + "combat", + "decoration", + "economy", + "food", + "magic", + "mobs", + "optimization", +]; + pub struct DummyData { pub alpha_team_id: String, pub beta_team_id: String, diff --git a/tests/common/environment.rs b/tests/common/environment.rs index bcf5c686..cba05836 100644 --- a/tests/common/environment.rs +++ b/tests/common/environment.rs @@ -3,6 +3,19 @@ use super::{database::TemporaryDatabase, dummy_data}; use crate::common::setup; use actix_web::{dev::ServiceResponse, test, App}; +use futures::Future; + +pub async fn with_test_environment(f: impl FnOnce(TestEnvironment) -> Fut) +where + Fut: Future, +{ + let test_env = TestEnvironment::build_with_dummy().await; + let db = test_env.db.clone(); + + f(test_env).await; + + db.cleanup().await; +} // A complete test environment, with a test actix app and a database. // Must be called in an #[actix_rt::test] context. It also simulates a diff --git a/tests/files/dummy_data.sql b/tests/files/dummy_data.sql index 59391f48..0583f8af 100644 --- a/tests/files/dummy_data.sql +++ b/tests/files/dummy_data.sql @@ -27,10 +27,20 @@ INSERT INTO loaders (id, loader) VALUES (1, 'fabric'); INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) VALUES (1,1); INSERT INTO loaders_project_types (joining_loader_id, joining_project_type_id) VALUES (1,2); -INSERT INTO categories (id, category, project_type) VALUES (1, 'combat', 1); -INSERT INTO categories (id, category, project_type) VALUES (2, 'decoration', 1); -INSERT INTO categories (id, category, project_type) VALUES (3, 'economy', 1); +INSERT INTO categories (id, category, project_type) VALUES + (1, 'combat', 1), + (2, 'decoration', 1), + (3, 'economy', 1), + (4, 'food', 1), + (5, 'magic', 1), + (6, 'mobs', 1), + (7, 'optimization', 1); -INSERT INTO categories (id, category, project_type) VALUES (4, 'combat', 2); -INSERT INTO categories (id, category, project_type) VALUES (5, 'decoration', 2); -INSERT INTO categories (id, category, project_type) VALUES (6, 'economy', 2); \ No newline at end of file +INSERT INTO categories (id, category, project_type) VALUES + (101, 'combat', 2), + (102, 'decoration', 2), + (103, 'economy', 2), + (104, 'food', 2), + (105, 'magic', 2), + (106, 'mobs', 2), + (107, 'optimization', 2); \ No newline at end of file diff --git a/tests/project.rs b/tests/project.rs index 215bcb66..c28804f9 100644 --- a/tests/project.rs +++ b/tests/project.rs @@ -1,10 +1,14 @@ +use actix_http::StatusCode; +use actix_web::dev::ServiceResponse; use actix_web::test; +use common::environment::with_test_environment; use labrinth::database::models::project_item::{PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE}; use labrinth::models::ids::base62_impl::parse_base62; use serde_json::json; use crate::common::database::*; +use crate::common::dummy_data::DUMMY_CATEGORIES; use crate::common::{actix::AppendsMultipart, environment::TestEnvironment}; // importing common module. @@ -403,7 +407,7 @@ pub async fn test_patch_project() { "title": "New successful title", "description": "New successful description", "body": "New successful body", - "categories": ["combat"], + "categories": [DUMMY_CATEGORIES[0]], "license_id": "MIT", "issues_url": "https://github.com", "discord_url": "https://discord.gg", @@ -441,7 +445,7 @@ pub async fn test_patch_project() { assert_eq!(body["title"], json!("New successful title")); assert_eq!(body["description"], json!("New successful description")); assert_eq!(body["body"], json!("New successful body")); - assert_eq!(body["categories"], json!(["combat"])); + assert_eq!(body["categories"], json!([DUMMY_CATEGORIES[0]])); assert_eq!(body["license"]["id"], json!("MIT")); assert_eq!(body["issues_url"], json!("https://github.com")); assert_eq!(body["discord_url"], json!("https://discord.gg")); @@ -457,5 +461,68 @@ pub async fn test_patch_project() { test_env.cleanup().await; } +#[actix_rt::test] +pub async fn test_bulk_edit_categories() { + with_test_environment(|test_env| async move { + let alpha_project_id = &test_env.dummy.as_ref().unwrap().alpha_project_id; + let beta_project_id = &test_env.dummy.as_ref().unwrap().beta_project_id; + + let req = test::TestRequest::patch() + .uri(&format!( + "/v2/projects?ids={}", + urlencoding::encode(&format!("[\"{alpha_project_id}\",\"{beta_project_id}\"]")) + )) + .append_header(("Authorization", ADMIN_USER_PAT)) + .set_json(json!({ + "categories": [DUMMY_CATEGORIES[0], DUMMY_CATEGORIES[3]], + "add_categories": [DUMMY_CATEGORIES[1], DUMMY_CATEGORIES[2]], + "remove_categories": [DUMMY_CATEGORIES[3]], + "additional_categories": [DUMMY_CATEGORIES[4], DUMMY_CATEGORIES[6]], + "add_additional_categories": [DUMMY_CATEGORIES[5]], + "remove_additional_categories": [DUMMY_CATEGORIES[6]], + })) + .to_request(); + let resp = test_env.call(req).await; + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let alpha_body = get_project_body(&test_env, &alpha_project_id, ADMIN_USER_PAT).await; + assert_eq!(alpha_body["categories"], json!(DUMMY_CATEGORIES[0..=2])); + assert_eq!( + alpha_body["additional_categories"], + json!(DUMMY_CATEGORIES[4..=5]) + ); + + let beta_body = get_project_body(&test_env, &beta_project_id, ADMIN_USER_PAT).await; + assert_eq!(beta_body["categories"], alpha_body["categories"]); + assert_eq!( + beta_body["additional_categories"], + alpha_body["additional_categories"], + ); + }) + .await; +} + +async fn get_project( + test_env: &TestEnvironment, + project_slug: &str, + user_pat: &str, +) -> ServiceResponse { + let req = test::TestRequest::get() + .uri(&format!("/v2/project/{project_slug}")) + .append_header(("Authorization", user_pat)) + .to_request(); + test_env.call(req).await +} + +async fn get_project_body( + test_env: &TestEnvironment, + project_slug: &str, + user_pat: &str, +) -> serde_json::Value { + let resp = get_project(test_env, project_slug, user_pat).await; + assert_eq!(resp.status(), StatusCode::OK); + test::read_body_json(resp).await +} + // TODO: Missing routes on projects // TODO: using permissions/scopes, can we SEE projects existence that we are not allowed to? (ie 401 instead of 404)