From d14111d33e85ab0866a413c064bca6656c9146b0 Mon Sep 17 00:00:00 2001 From: Wyatt Verchere Date: Wed, 6 Dec 2023 17:21:35 -0800 Subject: [PATCH] extracts versions --- ...f619267c93d1561ac3ab7a4394c7230b0bce3.json | 28 +++ src/clickhouse/fetch.rs | 18 +- src/database/models/project_item.rs | 108 ++++++++++- src/models/v2/projects.rs | 167 +++++++++++++----- src/models/v3/projects.rs | 11 +- src/routes/maven.rs | 6 +- src/routes/updates.rs | 3 +- src/routes/v2/project_creation.rs | 11 +- src/routes/v2/projects.rs | 59 ++++--- src/routes/v2/users.rs | 17 +- src/routes/v2/version_creation.rs | 12 +- src/routes/v2/version_file.rs | 15 +- src/routes/v2/versions.rs | 14 +- src/routes/v3/project_creation.rs | 5 - src/routes/v3/projects.rs | 99 ++++++++++- src/routes/v3/version_file.rs | 42 +++-- src/routes/v3/versions.rs | 123 ++++++++++--- src/search/indexing/local_import.rs | 2 +- tests/common/api_common/models.rs | 1 - tests/common/api_v3/version.rs | 28 +++ tests/project.rs | 37 ++-- 21 files changed, 620 insertions(+), 186 deletions(-) create mode 100644 .sqlx/query-c8b154bf3e070475bac63bacb25f619267c93d1561ac3ab7a4394c7230b0bce3.json diff --git a/.sqlx/query-c8b154bf3e070475bac63bacb25f619267c93d1561ac3ab7a4394c7230b0bce3.json b/.sqlx/query-c8b154bf3e070475bac63bacb25f619267c93d1561ac3ab7a4394c7230b0bce3.json new file mode 100644 index 00000000..57def10c --- /dev/null +++ b/.sqlx/query-c8b154bf3e070475bac63bacb25f619267c93d1561ac3ab7a4394c7230b0bce3.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT m.id as mod_id, v.id as version_id\n FROM mods m\n INNER JOIN versions v ON m.id = v.mod_id\n WHERE m.id = ANY($1) \n ORDER BY m.id, v.id;\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "mod_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8Array" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "c8b154bf3e070475bac63bacb25f619267c93d1561ac3ab7a4394c7230b0bce3" +} diff --git a/src/clickhouse/fetch.rs b/src/clickhouse/fetch.rs index 2988ba72..b0245075 100644 --- a/src/clickhouse/fetch.rs +++ b/src/clickhouse/fetch.rs @@ -115,8 +115,9 @@ pub async fn fetch_countries_downloads( end_date: DateTime, client: Arc, ) -> Result, ApiError> { - let query = client.query( - " + let query = client + .query( + " SELECT country, project_id, @@ -126,8 +127,8 @@ pub async fn fetch_countries_downloads( GROUP BY country, project_id - " - ) + ", + ) .bind(start_date.timestamp()) .bind(end_date.timestamp()) .bind(projects.iter().map(|x| x.0).collect::>()); @@ -141,8 +142,9 @@ pub async fn fetch_countries_views( end_date: DateTime, client: Arc, ) -> Result, ApiError> { - let query = client.query( - " + let query = client + .query( + " SELECT country, project_id, @@ -152,8 +154,8 @@ pub async fn fetch_countries_views( GROUP BY country, project_id - " - ) + ", + ) .bind(start_date.timestamp()) .bind(end_date.timestamp()) .bind(projects.iter().map(|x| x.0).collect::>()); diff --git a/src/database/models/project_item.rs b/src/database/models/project_item.rs index b051d765..a66ff2ff 100644 --- a/src/database/models/project_item.rs +++ b/src/database/models/project_item.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; pub const PROJECTS_NAMESPACE: &str = "projects"; pub const PROJECTS_SLUGS_NAMESPACE: &str = "projects_slugs"; const PROJECTS_DEPENDENCIES_NAMESPACE: &str = "projects_dependencies"; +const PROJECT_VERSIONS_NAMESPACE: &str = "project_versions"; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct LinkUrl { @@ -309,6 +310,10 @@ impl Project { let project = Self::get_id(id, &mut **transaction, redis).await?; if let Some(project) = project { + let version_ids = Self::get_versions(id, &mut **transaction, redis) + .await? + .unwrap_or_default(); + Project::clear_cache(id, project.inner.slug, Some(true), redis).await?; sqlx::query!( @@ -371,7 +376,7 @@ impl Project { .execute(&mut **transaction) .await?; - for version in project.versions { + for version in version_ids { super::Version::remove_full(version, redis, transaction).await?; } @@ -723,7 +728,7 @@ impl Project { additional_categories: m.additional_categories.unwrap_or_default(), project_types: m.project_types.unwrap_or_default(), games: m.games.unwrap_or_default(), - versions: { + public_versions: { #[derive(Deserialize)] struct Version { pub id: VersionId, @@ -834,6 +839,102 @@ impl Project { Ok(dependencies) } + pub async fn get_versions<'a, E>( + id: ProjectId, + exec: E, + redis: &RedisPool, + ) -> Result>, DatabaseError> + where + E: sqlx::Executor<'a, Database = sqlx::Postgres>, + { + let versions = Self::get_versions_many(&[id], exec, redis) + .await? + .into_iter() + .next() + .map(|(_, versions)| versions); + + Ok(versions) + } + + pub async fn get_versions_many<'a, E>( + ids: &[ProjectId], + exec: E, + redis: &RedisPool, + ) -> Result)>, DatabaseError> + where + E: sqlx::Executor<'a, Database = sqlx::Postgres>, + { + let mut redis = redis.connect().await?; + + let mut found_versions = Vec::new(); + let mut remaining_ids = ids.to_vec(); + + if !remaining_ids.is_empty() { + let projects_versions = redis + .multi_get::( + PROJECT_VERSIONS_NAMESPACE, + remaining_ids.iter().map(|x| x.0), + ) + .await?; + for project_versions in projects_versions { + if let Some((project_id, version_ids)) = project_versions + .and_then(|x| serde_json::from_str::<(ProjectId, Vec)>(&x).ok()) + { + remaining_ids.retain(|x| project_id != *x); + found_versions.push((project_id, version_ids)); + continue; + } + } + } + + if !remaining_ids.is_empty() { + let project_ids_parsed: Vec = + remaining_ids.iter().map(|x| x.0).collect::>(); + + let versions: Vec<(ProjectId, Vec)> = sqlx::query!( + " + SELECT m.id as mod_id, v.id as version_id + FROM mods m + INNER JOIN versions v ON m.id = v.mod_id + WHERE m.id = ANY($1) + ORDER BY m.id, v.id; + ", + &project_ids_parsed + ) + .fetch_many(exec) + .try_filter_map(|e| async { + Ok(e.right() + .map(|x| (ProjectId(x.mod_id), VersionId(x.version_id)))) + }) + .try_collect::>() + .await? + .into_iter() + .group_by(|(project_id, _)| *project_id) + .into_iter() + .map(|(project_id, group)| { + ( + project_id, + group.map(|(_, version_id)| version_id).collect_vec(), + ) + }) + .collect(); + + for (project_id, versions) in &versions { + redis + .set_serialized_to_json( + PROJECT_VERSIONS_NAMESPACE, + project_id.0, + (project_id, versions), + None, + ) + .await?; + } + found_versions.extend(versions); + } + + Ok(found_versions) + } + pub async fn clear_cache( id: ProjectId, slug: Option, @@ -854,6 +955,7 @@ impl Project { None }, ), + (PROJECT_VERSIONS_NAMESPACE, Some(id.0.to_string())), ]) .await?; Ok(()) @@ -865,7 +967,7 @@ pub struct QueryProject { pub inner: Project, pub categories: Vec, pub additional_categories: Vec, - pub versions: Vec, + pub public_versions: Vec, pub project_types: Vec, pub games: Vec, pub urls: Vec, diff --git a/src/models/v2/projects.rs b/src/models/v2/projects.rs index 0f23b618..bc6d48c5 100644 --- a/src/models/v2/projects.rs +++ b/src/models/v2/projects.rs @@ -5,8 +5,8 @@ use std::collections::HashMap; use super::super::ids::OrganizationId; use super::super::teams::TeamId; use super::super::users::UserId; -use crate::database::models::legacy_loader_fields::MinecraftGameVersion; -use crate::database::models::{version_item, DatabaseError}; +use crate::database; +use crate::database::models::DatabaseError; use crate::database::redis::RedisPool; use crate::models::ids::{ProjectId, VersionId}; use crate::models::projects::{ @@ -14,9 +14,14 @@ use crate::models::projects::{ ProjectStatus, Version, VersionFile, VersionStatus, VersionType, }; use crate::models::threads::ThreadId; -use crate::routes::v2_reroute; +use crate::queue::session::AuthQueue; +use crate::routes::v3::versions::{VersionListFilters, VersionListFiltersWithProjects}; +use crate::routes::{v2_reroute, v3}; +use actix_web::web::Data; +use actix_web::{web, HttpRequest}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; +use sqlx::PgPool; use validator::Validate; /// A project returned from the API @@ -76,7 +81,11 @@ impl LegacyProject { // - Its conceivable that certain V3 projects that have many different ones may not have the same fields on all of them. // TODO: Should this return an error instead for v2 users? // It's safe to use a db version_item for this as the only info is side types, game versions, and loader fields (for loaders), which used to be public on project anyway. - pub fn from(data: Project, versions_item: Option) -> Self { + fn from_inner( + data: Project, + versions_option: Option, + visible_version_ids: Vec, + ) -> Self { let mut client_side = LegacySideType::Unknown; let mut server_side = LegacySideType::Unknown; let mut game_versions = Vec::new(); @@ -104,33 +113,47 @@ impl LegacyProject { let mut loaders = data.loaders; - if let Some(versions_item) = versions_item { + if let Some(versions_item) = versions_option { game_versions = versions_item - .version_fields + .fields .iter() - .find(|f| f.field_name == "game_versions") - .and_then(|f| MinecraftGameVersion::try_from_version_field(f).ok()) - .map(|v| v.into_iter().map(|v| v.version).collect()) + .find_map(|(name, value)| { + if *name == "game_versions" { + value.as_array().map(|v| { + v.iter() + .filter_map(|gv| gv.as_str().map(|gv| gv.to_string())) + .collect::>() + }) + } else { + None + } + }) .unwrap_or(Vec::new()); // Extract side types from remaining fields (singleplayer, client_only, etc) - let fields = versions_item - .version_fields - .iter() - .map(|f| (f.field_name.clone(), f.value.clone().serialize_internal())) - .collect::>(); - (client_side, server_side) = v2_reroute::convert_side_types_v2(&fields); + (client_side, server_side) = v2_reroute::convert_side_types_v2(&versions_item.fields); // - if loader is mrpack, this is a modpack // the loaders are whatever the corresponding loader fields are - if versions_item.loaders == vec!["mrpack".to_string()] { + if versions_item + .loaders + .contains(&Loader("mrpack".to_string())) + { project_type = "modpack".to_string(); if let Some(mrpack_loaders) = versions_item - .version_fields + .fields .iter() - .find(|f| f.field_name == "mrpack_loaders") + .find(|f| f.0 == "mrpack_loaders") { - loaders = mrpack_loaders.value.as_strings(); + loaders = mrpack_loaders + .1 + .as_array() + .map(|v| { + v.iter() + .filter_map(|l| l.as_str().map(|l| l.to_string())) + .collect::>() + }) + .unwrap_or(Vec::new()); } } } @@ -170,7 +193,7 @@ impl LegacyProject { categories: data.categories, additional_categories: data.additional_categories, loaders, - versions: data.versions, + versions: visible_version_ids.into_iter().map(|i| i.into()).collect(), icon_url: data.icon_url, issues_url, source_url, @@ -191,30 +214,88 @@ impl LegacyProject { } } - // Because from needs a version_item, this is a helper function to get many from one db query. - pub async fn from_many<'a, E>( - data: Vec, - exec: E, - redis: &RedisPool, - ) -> Result, DatabaseError> - where - E: sqlx::Executor<'a, Database = sqlx::Postgres>, - { - let version_ids: Vec<_> = data + pub async fn from( + req: HttpRequest, + client: Data, + redis: Data, + session_queue: Data, + project: Project, + ) -> Result { + // Call v3 project version get + let project_ids = serde_json::to_string(&[project.id])?; + let found_versions = match v3::versions::version_list_inner( + req, + web::Query(VersionListFiltersWithProjects { + ids: project_ids, + filters: VersionListFilters::default(), + }), + client.clone(), + redis.clone(), + session_queue.clone(), + ) + .await + { + Ok(versions) => versions, + Err(_) => vec![], + }; + + let version_ids = found_versions .iter() - .filter_map(|p| p.versions.first().map(|i| (*i).into())) - .collect(); - let example_versions = version_item::Version::get_many(&version_ids, exec, redis).await?; - let mut legacy_projects = Vec::new(); - for project in data { - let version_item = example_versions - .iter() - .find(|v| v.inner.project_id == project.id.into()) - .cloned(); - let project = LegacyProject::from(project, version_item); - legacy_projects.push(project); - } - Ok(legacy_projects) + .map(|v| v.id.into()) + .collect::>(); + let version_item = found_versions.into_iter().next().take(); + let project = LegacyProject::from_inner(project, version_item, version_ids); + Ok(project) + } + + pub async fn from_many( + req: HttpRequest, + client: Data, + redis: Data, + session_queue: Data, + projects: Vec, + ) -> Result, DatabaseError> { + let project_ids = + serde_json::to_string(&projects.iter().map(|p| p.id).collect::>())?; + // Call v3 project version get + let found_versions = v3::versions::version_list_inner( + req, + web::Query(VersionListFiltersWithProjects { + ids: project_ids, + filters: VersionListFilters::default(), + }), + client.clone(), + redis.clone(), + session_queue.clone(), + ) + .await + .unwrap_or_default(); + + let proj_version_hashmap = + found_versions + .into_iter() + .fold(HashMap::new(), |mut acc: HashMap>, version| { + acc.entry(version.project_id) + .or_default() + .push(version); + acc + }); + + Ok(projects + .into_iter() + .map(|project| { + let found_version = proj_version_hashmap + .get(&project.id) + .cloned() + .unwrap_or_default(); + let version_ids = found_version + .iter() + .map(|v| v.id.into()) + .collect::>(); + let version_item = found_version.into_iter().next().take(); + LegacyProject::from_inner(project, version_item, version_ids) + }) + .collect()) } } diff --git a/src/models/v3/projects.rs b/src/models/v3/projects.rs index 62103708..495a2076 100644 --- a/src/models/v3/projects.rs +++ b/src/models/v3/projects.rs @@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize}; use validator::Validate; /// The ID of a specific project, encoded as base62 for usage in the API -#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Debug)] #[serde(from = "Base62Id")] #[serde(into = "Base62Id")] pub struct ProjectId(pub u64); @@ -85,8 +85,6 @@ pub struct Project { /// A list of loaders this project supports pub loaders: Vec, - /// A list of ids for versions of the project. - pub versions: Vec, /// The URL of the icon of the project pub icon_url: Option, @@ -206,7 +204,6 @@ impl From for Project { categories: data.categories, additional_categories: data.additional_categories, loaders: m.loaders, - versions: data.versions.into_iter().map(|v| v.into()).collect(), icon_url: m.icon_url, link_urls: data .urls @@ -242,11 +239,6 @@ impl Project { .organization_id .and_then(|id| Some(OrganizationId(parse_base62(&id).ok()?))); let thread_id = ThreadId(parse_base62(&m.thread_id).ok()?); - let versions = m - .versions - .iter() - .filter_map(|id| Some(VersionId(parse_base62(id).ok()?))) - .collect(); let approved = DateTime::parse_from_rfc3339(&m.date_created).ok()?; let published = DateTime::parse_from_rfc3339(&m.date_published).ok()?.into(); @@ -367,7 +359,6 @@ impl Project { categories, additional_categories, loaders, - versions, icon_url, link_urls, gallery, diff --git a/src/routes/maven.rs b/src/routes/maven.rs index cb8fd45f..5138c2e8 100644 --- a/src/routes/maven.rs +++ b/src/routes/maven.rs @@ -166,7 +166,11 @@ async fn find_version( .ok() .map(|x| x as i64); - let all_versions = database::models::Version::get_many(&project.versions, pool, redis).await?; + // Maven routes check for authorized versions, so we can get all versions here rather than just public ones + let versions = database::models::Project::get_versions(project.inner.id, pool, redis) + .await? + .unwrap_or_default(); + let all_versions = database::models::Version::get_many(&versions, pool, redis).await?; let exact_matches = all_versions .iter() diff --git a/src/routes/updates.rs b/src/routes/updates.rs index f4d6d2f8..6148f0cb 100644 --- a/src/routes/updates.rs +++ b/src/routes/updates.rs @@ -60,7 +60,8 @@ pub async fn forge_updates( return Err(ApiError::InvalidInput(ERROR.to_string())); } - let versions = database::models::Version::get_many(&project.versions, &**pool, &redis).await?; + let versions = + database::models::Version::get_many(&project.public_versions, &**pool, &redis).await?; let loaders = match &*neo.neoforge { "only" => |x: &String| *x == "neoforge", diff --git a/src/routes/v2/project_creation.rs b/src/routes/v2/project_creation.rs index 7e1c5053..35d5b2ef 100644 --- a/src/routes/v2/project_creation.rs +++ b/src/routes/v2/project_creation.rs @@ -1,4 +1,3 @@ -use crate::database::models::version_item; use crate::database::redis::RedisPool; use crate::file_hosting::FileHost; use crate::models; @@ -235,23 +234,19 @@ pub async fn project_create( // Call V3 project creation let response = v3::project_creation::project_create( - req, + req.clone(), payload, client.clone(), redis.clone(), file_host, - session_queue, + session_queue.clone(), ) .await?; // Convert response to V2 format match v2_reroute::extract_ok_json::(response).await { Ok(project) => { - let version_item = match project.versions.first() { - Some(vid) => version_item::Version::get((*vid).into(), &**client, &redis).await?, - None => None, - }; - let project = LegacyProject::from(project, version_item); + let project = LegacyProject::from(req, client, redis, session_queue, project).await?; Ok(HttpResponse::Ok().json(project)) } Err(response) => Ok(response), diff --git a/src/routes/v2/projects.rs b/src/routes/v2/projects.rs index 80c43c8c..adb12de1 100644 --- a/src/routes/v2/projects.rs +++ b/src/routes/v2/projects.rs @@ -1,15 +1,16 @@ use crate::database::models::categories::LinkPlatform; -use crate::database::models::{project_item, version_item}; +use crate::database::models::project_item; use crate::database::redis::RedisPool; use crate::file_hosting::FileHost; use crate::models; use crate::models::projects::{ - Link, MonetizationStatus, Project, ProjectStatus, SearchRequest, Version, + Link, MonetizationStatus, Project, ProjectStatus, SearchRequest, }; use crate::models::v2::projects::{DonationLink, LegacyProject, LegacySideType}; use crate::models::v2::search::LegacySearchResults; use crate::queue::session::AuthQueue; use crate::routes::v3::projects::ProjectIds; +use crate::routes::v3::versions::{VersionListFilters, VersionListFiltersWithProjects}; use crate::routes::{v2_reroute, v3, ApiError}; use crate::search::{search_for_project, SearchConfig, SearchError}; use actix_web::{delete, get, patch, post, web, HttpRequest, HttpResponse}; @@ -138,6 +139,8 @@ pub struct RandomProjects { #[get("projects_random")] pub async fn random_projects_get( + req: HttpRequest, + session_queue: web::Data, web::Query(count): web::Query, pool: web::Data, redis: web::Data, @@ -151,8 +154,9 @@ pub async fn random_projects_get( .or_else(v2_reroute::flatten_404_error)?; // Convert response to V2 format match v2_reroute::extract_ok_json::>(response).await { - Ok(project) => { - let legacy_projects = LegacyProject::from_many(project, &**pool, &redis).await?; + Ok(projects) => { + let legacy_projects = + LegacyProject::from_many(req, pool, redis, session_queue, projects).await?; Ok(HttpResponse::Ok().json(legacy_projects)) } Err(response) => Ok(response), @@ -169,11 +173,11 @@ pub async fn projects_get( ) -> Result { // Call V3 project creation let response = v3::projects::projects_get( - req, + req.clone(), web::Query(ids), pool.clone(), redis.clone(), - session_queue, + session_queue.clone(), ) .await .or_else(v2_reroute::flatten_404_error) @@ -181,8 +185,9 @@ pub async fn projects_get( // Convert response to V2 format match v2_reroute::extract_ok_json::>(response).await { - Ok(project) => { - let legacy_projects = LegacyProject::from_many(project, &**pool, &redis).await?; + Ok(projects) => { + let legacy_projects = + LegacyProject::from_many(req, pool, redis, session_queue, projects).await?; Ok(HttpResponse::Ok().json(legacy_projects)) } Err(response) => Ok(response), @@ -199,19 +204,21 @@ pub async fn project_get( ) -> Result { // Convert V2 data to V3 data // Call V3 project creation - let response = v3::projects::project_get(req, info, pool.clone(), redis.clone(), session_queue) - .await - .or_else(v2_reroute::flatten_404_error) - .or_else(v2_reroute::flatten_404_error)?; + let response = v3::projects::project_get( + req.clone(), + info, + pool.clone(), + redis.clone(), + session_queue.clone(), + ) + .await + .or_else(v2_reroute::flatten_404_error) + .or_else(v2_reroute::flatten_404_error)?; // Convert response to V2 format match v2_reroute::extract_ok_json::(response).await { Ok(project) => { - let version_item = match project.versions.first() { - Some(vid) => version_item::Version::get((*vid).into(), &**pool, &redis).await?, - None => None, - }; - let project = LegacyProject::from(project, version_item); + let project = LegacyProject::from(req, pool, redis, session_queue, project).await?; Ok(HttpResponse::Ok().json(project)) } Err(response) => Ok(response), @@ -364,7 +371,6 @@ pub async fn project_edit( let v2_new_project = new_project.into_inner(); let client_side = v2_new_project.client_side; let server_side = v2_new_project.server_side; - let new_slug = v2_new_project.slug.clone(); // TODO: Some kind of handling here to ensure project type is fine. // We expect the version uploaded to be of loader type modpack, but there might not be a way to check here for that. @@ -473,12 +479,19 @@ pub async fn project_edit( // If client and server side were set, we will call // the version setting route for each version to set the side types for each of them. if response.status().is_success() && (client_side.is_some() || server_side.is_some()) { - let project_item = - project_item::Project::get(&new_slug.unwrap_or(project_id), &**pool, &redis).await?; - let version_ids = project_item.map(|x| x.versions).unwrap_or_default(); - let versions = version_item::Version::get_many(&version_ids, &**pool, &redis).await?; + let versions = v3::versions::version_list_inner( + req.clone(), + web::Query(VersionListFiltersWithProjects { + ids: serde_json::to_string(&[project_id])?, + filters: VersionListFilters::default(), + }), + pool.clone(), + redis.clone(), + session_queue.clone(), + ) + .await?; + for version in versions { - let version = Version::from(version); let mut fields = version.fields; let (current_client_side, current_server_side) = v2_reroute::convert_side_types_v2(&fields); diff --git a/src/routes/v2/users.rs b/src/routes/v2/users.rs index 37da2f9e..ecb00acb 100644 --- a/src/routes/v2/users.rs +++ b/src/routes/v2/users.rs @@ -78,14 +78,21 @@ pub async fn projects_list( redis: web::Data, session_queue: web::Data, ) -> Result { - let response = v3::users::projects_list(req, info, pool.clone(), redis.clone(), session_queue) - .await - .or_else(v2_reroute::flatten_404_error)?; + let response = v3::users::projects_list( + req.clone(), + info, + pool.clone(), + redis.clone(), + session_queue.clone(), + ) + .await + .or_else(v2_reroute::flatten_404_error)?; // Convert to V2 projects match v2_reroute::extract_ok_json::>(response).await { - Ok(project) => { - let legacy_projects = LegacyProject::from_many(project, &**pool, &redis).await?; + Ok(projects) => { + let legacy_projects = + LegacyProject::from_many(req, pool, redis, session_queue, projects).await?; Ok(HttpResponse::Ok().json(legacy_projects)) } Err(response) => Ok(response), diff --git a/src/routes/v2/version_creation.rs b/src/routes/v2/version_creation.rs index 997189dc..db0741c0 100644 --- a/src/routes/v2/version_creation.rs +++ b/src/routes/v2/version_creation.rs @@ -210,11 +210,13 @@ async fn get_example_version_fields( None => return Ok(None), }; - let vid = match project_item::Project::get_id(project_id.into(), &**pool, redis) - .await? - .and_then(|p| p.versions.first().cloned()) - { - Some(vid) => vid, + let example_versions = + match project_item::Project::get_versions(project_id.into(), &**pool, redis).await? { + Some(v) => v, + None => return Ok(None), + }; + let vid = match example_versions.first() { + Some(v) => *v, None => return Ok(None), }; diff --git a/src/routes/v2/version_file.rs b/src/routes/v2/version_file.rs index a50d25b4..8654c365 100644 --- a/src/routes/v2/version_file.rs +++ b/src/routes/v2/version_file.rs @@ -198,11 +198,11 @@ pub async fn get_projects_from_hashes( hashes: file_data.hashes, }; let response = v3::version_file::get_projects_from_hashes( - req, + req.clone(), pool.clone(), redis.clone(), web::Json(file_data), - session_queue, + session_queue.clone(), ) .await .or_else(v2_reroute::flatten_404_error)?; @@ -217,9 +217,14 @@ pub async fn get_projects_from_hashes( (hash.clone(), project_id) }) .collect::>(); - let legacy_projects = - LegacyProject::from_many(projects_hashes.into_values().collect(), &**pool, &redis) - .await?; + let legacy_projects = LegacyProject::from_many( + req, + pool, + redis, + session_queue, + projects_hashes.into_values().collect(), + ) + .await?; let legacy_projects_hashes = hash_to_project_id .into_iter() .filter_map(|(hash, project_id)| { diff --git a/src/routes/v2/versions.rs b/src/routes/v2/versions.rs index 3ff63e80..34b5d080 100644 --- a/src/routes/v2/versions.rs +++ b/src/routes/v2/versions.rs @@ -71,10 +71,16 @@ pub async fn version_list( offset: filters.offset, }; - let response = - v3::versions::version_list(req, info, web::Query(filters), pool, redis, session_queue) - .await - .or_else(v2_reroute::flatten_404_error)?; + let response = v3::versions::project_version_list( + req, + info, + web::Query(filters), + pool, + redis, + session_queue, + ) + .await + .or_else(v2_reroute::flatten_404_error)?; // Convert response to V2 format match v2_reroute::extract_ok_json::>(response).await { diff --git a/src/routes/v3/project_creation.rs b/src/routes/v3/project_creation.rs index 0c999ad9..a94c8a26 100644 --- a/src/routes/v3/project_creation.rs +++ b/src/routes/v3/project_creation.rs @@ -806,11 +806,6 @@ async fn project_create_inner( categories: project_create_data.categories, additional_categories: project_create_data.additional_categories, loaders: vec![], - versions: project_builder - .initial_versions - .iter() - .map(|v| v.version_id.into()) - .collect::>(), icon_url: project_builder.icon_url.clone(), link_urls: project_builder .link_urls diff --git a/src/routes/v3/projects.rs b/src/routes/v3/projects.rs index 3b35d163..5acf755a 100644 --- a/src/routes/v3/projects.rs +++ b/src/routes/v3/projects.rs @@ -34,10 +34,13 @@ use serde_json::json; use sqlx::PgPool; use validator::Validate; +use super::versions::projects_version_list; + pub fn config(cfg: &mut web::ServiceConfig) { cfg.route("search", web::get().to(project_search)); cfg.route("projects", web::get().to(projects_get)); cfg.route("projects", web::patch().to(projects_edit)); + cfg.route("projects/versions", web::patch().to(projects_version_list)); cfg.route("projects_random", web::get().to(random_projects_get)); cfg.service( @@ -60,7 +63,10 @@ pub fn config(cfg: &mut web::ServiceConfig) { "members", web::get().to(super::teams::team_members_get_project), ) - .route("version", web::get().to(super::versions::version_list)) + .route( + "version", + web::get().to(super::versions::project_version_list), + ) .route( "version/{slug}", web::get().to(super::versions::version_project_get), @@ -332,8 +338,12 @@ pub async fn project_edit( )); } + let versions = + database::models::Project::get_versions(project_item.inner.id, &**pool, &redis) + .await? + .unwrap_or_default(); if status == &ProjectStatus::Processing { - if project_item.versions.is_empty() { + if versions.is_empty() { return Err(ApiError::InvalidInput(String::from( "Project submitted for review with no initial versions", ))); @@ -2278,3 +2288,88 @@ pub async fn project_unfollow( )) } } + +// pub async fn project_get_versions( +// req: HttpRequest, +// info: web::Path<(String,)>, +// pool: web::Data, +// redis: web::Data, +// session_queue: web::Data, +// ) -> Result { +// project_get_versions_inner(req, info.into_inner().0, pool, redis, session_queue).await +// } +// pub async fn project_get_versions_inner( +// req: HttpRequest, +// info: String, +// pool: web::Data, +// redis: web::Data, +// session_queue: web::Data, +// ) -> Result { +// let user_option = get_user_from_headers( +// &req, +// &**pool, +// &redis, +// &session_queue, +// Some(&[Scopes::PROJECT_READ, Scopes::VERSION_READ]), +// ) +// .await.ok().map(|x| x.1); + +// let project = db_models::Project::get(&info, &**pool, &redis).await?.ok_or_else(|| {ApiError::NotFound})?; + +// if !is_authorized(&project.inner, &user_option, &pool).await? { +// return Err(ApiError::NotFound); +// } + +// let project_versions = db_models::Project::get_versions(project.inner.id, &**pool, &redis).await?.ok_or_else(|| {ApiError::NotFound})?; + +// let versions_data = filter_authorized_versions( +// database::models::Version::get_many(&project_versions, &**pool, &redis).await?, +// &user_option, +// &pool, +// ) +// .await?; + +// Ok(HttpResponse::Ok().json(versions_data)) +// } + +// pub async fn projects_get_versions( +// req: HttpRequest, +// web::Query(ids): web::Query, +// pool: web::Data, +// redis: web::Data, +// session_queue: web::Data, +// ) -> Result { +// let user_option = get_user_from_headers( +// &req, +// &**pool, +// &redis, +// &session_queue, +// Some(&[Scopes::PROJECT_READ, Scopes::VERSION_READ]), +// ) +// .await.ok().map(|x| x.1); + +// let ids = serde_json::from_str::>(&ids.ids)?; +// let projects_data = db_models::Project::get_many(&ids, &**pool, &redis).await?; +// let projects_data = filter_authorized_projects( +// projects_data, +// &user_option, +// &pool, +// ).await?; +// let allowed_project_ids = projects_data.iter().map(|x| x.id.into()).collect::>(); + +// let project_versions = db_models::Project::get_versions_many(&allowed_project_ids, &**pool, &redis).await?; +// let all_version_ids = project_versions.into_iter().map(|x| x.1).flatten().collect::>(); +// let versions_data = database::models::Version::get_many(&all_version_ids, &**pool, &redis).await?; + +// let versions_data = filter_authorized_versions( +// versions_data, +// &user_option, +// &pool, +// ) +// .await?.into_iter().fold(HashMap::new(), |mut acc, version | { +// acc.entry(version.project_id).or_insert_with(Vec::new).push(version); +// acc +// }); + +// Ok(HttpResponse::Ok().json(versions_data)) +// } diff --git a/src/routes/v3/version_file.rs b/src/routes/v3/version_file.rs index 2006d481..22bdb577 100644 --- a/src/routes/v3/version_file.rs +++ b/src/routes/v3/version_file.rs @@ -150,7 +150,11 @@ pub async fn get_update_from_hash( if let Some(project) = database::models::Project::get_id(file.project_id, &**pool, &redis).await? { - let versions = database::models::Version::get_many(&project.versions, &**pool, &redis) + let versions = + database::models::Project::get_versions(project.inner.id, &**pool, &redis) + .await? + .unwrap_or_default(); + let versions = database::models::Version::get_many(&versions, &**pool, &redis) .await? .into_iter() .filter(|x| { @@ -346,15 +350,15 @@ pub async fn update_files( &redis, ) .await?; - let all_versions = database::models::Version::get_many( - &projects - .iter() - .flat_map(|x| x.versions.clone()) - .collect::>(), - &**pool, - &redis, - ) - .await?; + let project_ids = projects.iter().map(|x| x.inner.id).collect::>(); + let all_version_ids = + database::models::Project::get_versions_many(&project_ids, &**pool, &redis) + .await? + .into_iter() + .flat_map(|x| x.1) + .collect::>(); + let all_versions = + database::models::Version::get_many(&all_version_ids, &**pool, &redis).await?; let mut response = HashMap::new(); @@ -467,15 +471,15 @@ pub async fn update_individual_files( &redis, ) .await?; - let all_versions = database::models::Version::get_many( - &projects - .iter() - .flat_map(|x| x.versions.clone()) - .collect::>(), - &**pool, - &redis, - ) - .await?; + let project_ids = projects.iter().map(|x| x.inner.id).collect::>(); + let all_version_ids = + database::models::Project::get_versions_many(&project_ids, &**pool, &redis) + .await? + .into_iter() + .flat_map(|x| x.1) + .collect::>(); + let all_versions = + database::models::Version::get_many(&all_version_ids, &**pool, &redis).await?; let mut response = HashMap::new(); diff --git a/src/routes/v3/versions.rs b/src/routes/v3/versions.rs index 0b55c865..2df9fd00 100644 --- a/src/routes/v3/versions.rs +++ b/src/routes/v3/versions.rs @@ -2,7 +2,8 @@ use std::collections::HashMap; use super::ApiError; use crate::auth::{ - filter_authorized_versions, get_user_from_headers, is_authorized, is_authorized_version, + filter_authorized_projects, filter_authorized_versions, get_user_from_headers, is_authorized, + is_authorized_version, }; use crate::database; use crate::database::models::loader_fields::{ @@ -13,10 +14,10 @@ use crate::database::models::{image_item, Organization}; use crate::database::redis::RedisPool; use crate::models; use crate::models::ids::base62_impl::parse_base62; -use crate::models::ids::VersionId; +use crate::models::ids::{ProjectId, VersionId}; use crate::models::images::ImageContext; use crate::models::pats::Scopes; -use crate::models::projects::{skip_nulls, Loader}; +use crate::models::projects::{skip_nulls, Loader, Version}; use crate::models::projects::{Dependency, FileType, VersionStatus, VersionType}; use crate::models::teams::ProjectPermissions; use crate::queue::session::AuthQueue; @@ -68,7 +69,6 @@ pub async fn version_project_get_helper( session_queue: web::Data, ) -> Result { let result = database::models::Project::get(&id.0, &**pool, &redis).await?; - let user_option = get_user_from_headers( &req, &**pool, @@ -81,21 +81,24 @@ pub async fn version_project_get_helper( .ok(); if let Some(project) = result { - if !is_authorized(&project.inner, &user_option, &pool).await? { - return Err(ApiError::NotFound); - } - let versions = - database::models::Version::get_many(&project.versions, &**pool, &redis).await?; + database::models::Project::get_versions(project.inner.id, &**pool, &redis).await?; + if let Some(versions) = versions { + if !is_authorized(&project.inner, &user_option, &pool).await? { + return Err(ApiError::NotFound); + } - let id_opt = parse_base62(&id.1).ok(); - let version = versions - .into_iter() - .find(|x| Some(x.inner.id.0 as u64) == id_opt || x.inner.version_number == id.1); + let versions = database::models::Version::get_many(&versions, &**pool, &redis).await?; - if let Some(version) = version { - if is_authorized_version(&version.inner, &user_option, &pool).await? { - return Ok(HttpResponse::Ok().json(models::projects::Version::from(version))); + let id_opt = parse_base62(&id.1).ok(); + let version = versions + .into_iter() + .find(|x| Some(x.inner.id.0 as u64) == id_opt || x.inner.version_number == id.1); + + if let Some(version) = version { + if is_authorized_version(&version.inner, &user_option, &pool).await? { + return Ok(HttpResponse::Ok().json(models::projects::Version::from(version))); + } } } } @@ -682,7 +685,7 @@ pub async fn version_edit_helper( } } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Default)] pub struct VersionListFilters { pub loaders: Option, pub featured: Option, @@ -698,7 +701,7 @@ pub struct VersionListFilters { pub loader_fields: Option, } -pub async fn version_list( +pub async fn project_version_list( req: HttpRequest, info: web::Path<(String,)>, web::Query(filters): web::Query, @@ -706,9 +709,60 @@ pub async fn version_list( redis: web::Data, session_queue: web::Data, ) -> Result { - let string = info.into_inner().0; + let project_id = info.into_inner().0; + let project_ids = serde_json::to_string(&[project_id])?; + let version_list = version_list_inner( + req, + web::Query(VersionListFiltersWithProjects { + ids: project_ids, + filters, + }), + pool, + redis, + session_queue, + ) + .await?; + Ok(HttpResponse::Ok().json(version_list)) +} - let result = database::models::Project::get(&string, &**pool, &redis).await?; +#[derive(Deserialize)] +pub struct VersionListFiltersWithProjects { + pub ids: String, + + #[serde(flatten)] + pub filters: VersionListFilters, +} + +pub async fn projects_version_list( + req: HttpRequest, + web::Query(filters): web::Query, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result { + let version_list = + version_list_inner(req, web::Query(filters), pool, redis, session_queue).await?; + let proj_version_hashmap = version_list + .into_iter() + .fold(HashMap::new(), |mut acc: HashMap<_, Vec<_>>, version| { + acc.entry(version.project_id) + .or_default() + .push(version); + acc + }); + Ok(HttpResponse::Ok().json(proj_version_hashmap)) +} + +pub async fn version_list_inner( + req: HttpRequest, + web::Query(filters): web::Query, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, +) -> Result, ApiError> { + let project_ids: Vec = serde_json::from_str(&filters.ids)?; + let filters = filters.filters; + let projects = database::models::Project::get_many(&project_ids, &**pool, &redis).await?; let user_option = get_user_from_headers( &req, @@ -721,11 +775,23 @@ pub async fn version_list( .map(|x| x.1) .ok(); - if let Some(project) = result { - if !is_authorized(&project.inner, &user_option, &pool).await? { - return Err(ApiError::NotFound); - } + let allowed_projects = filter_authorized_projects(projects.clone(), &user_option, &pool) + .await? + .into_iter() + .map(|x| x.id) + .collect::>(); + let projects = projects + .into_iter() + .filter_map(|x| { + if allowed_projects.contains(&x.inner.id.into()) { + Some(x.inner.id) + } else { + None + } + }) + .collect::>(); + if !projects.is_empty() { let loader_field_filters = filters.loader_fields.as_ref().map(|x| { serde_json::from_str::>>(x).unwrap_or_default() }); @@ -733,7 +799,12 @@ pub async fn version_list( .loaders .as_ref() .map(|x| serde_json::from_str::>(x).unwrap_or_default()); - let mut versions = database::models::Version::get_many(&project.versions, &**pool, &redis) + let version_ids = database::models::Project::get_versions_many(&projects, &**pool, &redis) + .await? + .into_iter() + .flat_map(|(_, v)| v) + .collect::>(); + let mut versions = database::models::Version::get_many(&version_ids, &**pool, &redis) .await? .into_iter() .skip(filters.offset.unwrap_or(0)) @@ -820,7 +891,7 @@ pub async fn version_list( let response = filter_authorized_versions(response, &user_option, &pool).await?; - Ok(HttpResponse::Ok().json(response)) + Ok(response) } else { Err(ApiError::NotFound) } diff --git a/src/search/indexing/local_import.rs b/src/search/indexing/local_import.rs index c5485eb5..4c4d478e 100644 --- a/src/search/indexing/local_import.rs +++ b/src/search/indexing/local_import.rs @@ -93,7 +93,7 @@ pub async fn index_local( let thread_id: crate::models::threads::ThreadId = m.thread_id.into(); let all_version_ids = m - .versions + .public_versions .iter() .map(|v| (*v).into()) .collect::>(); diff --git a/tests/common/api_common/models.rs b/tests/common/api_common/models.rs index cbf7ea96..7a48b88a 100644 --- a/tests/common/api_common/models.rs +++ b/tests/common/api_common/models.rs @@ -45,7 +45,6 @@ pub struct CommonProject { pub categories: Vec, pub additional_categories: Vec, pub loaders: Vec, - pub versions: Vec, pub icon_url: Option, pub gallery: Vec, pub color: Option, diff --git a/tests/common/api_v3/version.rs b/tests/common/api_v3/version.rs index 43226684..8e6bba58 100644 --- a/tests/common/api_v3/version.rs +++ b/tests/common/api_v3/version.rs @@ -88,6 +88,34 @@ impl ApiV3 { assert_eq!(resp.status(), 200); test::read_body_json(resp).await } + + #[allow(clippy::too_many_arguments)] + pub async fn get_project_versions_deserialized( + &self, + slug: &str, + game_versions: Option>, + loaders: Option>, + featured: Option, + version_type: Option, + limit: Option, + offset: Option, + pat: &str, + ) -> Vec { + let resp = self + .get_project_versions( + slug, + game_versions, + loaders, + featured, + version_type, + limit, + offset, + pat, + ) + .await; + assert_eq!(resp.status(), 200); + test::read_body_json(resp).await + } } #[async_trait(?Send)] diff --git a/tests/project.rs b/tests/project.rs index 32ba8616..bed7d5c4 100644 --- a/tests/project.rs +++ b/tests/project.rs @@ -32,22 +32,15 @@ async fn test_get_project() { let alpha_project_id = &test_env.dummy.as_ref().unwrap().project_alpha.project_id; let beta_project_id = &test_env.dummy.as_ref().unwrap().project_beta.project_id; let alpha_project_slug = &test_env.dummy.as_ref().unwrap().project_alpha.project_slug; - let alpha_version_id = &test_env.dummy.as_ref().unwrap().project_alpha.version_id; - // Perform request on dummy data - let req = test::TestRequest::get() - .uri(&format!("/v3/project/{alpha_project_id}")) - .append_header(("Authorization", USER_USER_PAT)) - .to_request(); - let resp = test_env.call(req).await; - let status = resp.status(); - let body: serde_json::Value = test::read_body_json(resp).await; + let api = &test_env.api; - assert_eq!(status, 200); - assert_eq!(body["id"], json!(alpha_project_id)); - assert_eq!(body["slug"], json!(alpha_project_slug)); - let versions = body["versions"].as_array().unwrap(); - assert_eq!(versions[0], json!(alpha_version_id)); + // Perform request on dummy data + let proj = api + .get_project_deserialized_common(alpha_project_slug, USER_USER_PAT) + .await; + assert_eq!(proj.id.to_string(), alpha_project_id.to_string()); + assert_eq!(proj.slug.unwrap(), alpha_project_slug.to_string()); // Confirm that the request was cached let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap(); @@ -189,8 +182,20 @@ async fn test_add_remove_project() { let project = api .get_project_deserialized_common("demo", USER_USER_PAT) .await; - assert!(project.versions.len() == 1); - let uploaded_version_id = project.versions[0]; + let versions = api + .get_project_versions_deserialized( + &project.id.to_string(), + None, + None, + None, + None, + None, + None, + USER_USER_PAT, + ) + .await; + assert!(versions.len() == 1); + let uploaded_version_id = versions[0].id; // Checks files to ensure they were uploaded and correctly identify the file let hash = sha1::Sha1::from(include_bytes!("../tests/files/basic-mod.jar"))