More staging fixes (#768)

* Fixes issues

* staging fixes

* passes tests

* fixes. fmt/clippy

* drops datapack/plugin extras

* fixed failing test

---------

Co-authored-by: Geometrically <[email protected]>

migrations/20231125080100_drops_mods_dp_plugins.sql

@@ -0,0 +1,31 @@
+-- For every loader that has a loaders_project_types entry that connects it to the project_types 'plugin',
+-- remove all non plugin project_types entries for that loader.
+-- This is to ensure that the plugin project_types is the only one that is used for the plugin loaders
+
+--plugin
+DELETE FROM loaders_project_types
+WHERE joining_loader_id IN (
+    SELECT DISTINCT l.id
+    FROM loaders l
+    LEFT JOIN loaders_project_types lpt ON lpt.joining_loader_id = l.id
+    LEFT JOIN project_types pt ON pt.id = lpt.joining_project_type_id
+    WHERE pt.name = 'plugin'
+)
+AND joining_project_type_id NOT IN (
+    SELECT id FROM project_types
+    WHERE name = 'plugin'
+);
+
+--datapack
+DELETE FROM loaders_project_types
+WHERE joining_loader_id IN (
+    SELECT DISTINCT l.id
+    FROM loaders l
+    LEFT JOIN loaders_project_types lpt ON lpt.joining_loader_id = l.id
+    LEFT JOIN project_types pt ON pt.id = lpt.joining_project_type_id
+    WHERE pt.name = 'datapack'
+)
+AND joining_project_type_id NOT IN (
+    SELECT id FROM project_types
+    WHERE name = 'datapack'
+);

src/database/models/loader_fields.rs

@@ -13,6 +13,7 @@ const GAMES_LIST_NAMESPACE: &str = "games";
 const LOADER_ID: &str = "loader_id";
 const LOADERS_LIST_NAMESPACE: &str = "loaders";
 const LOADER_FIELDS_NAMESPACE: &str = "loader_fields";
+const LOADER_FIELDS_NAMESPACE_ALL: &str = "loader_fields_all";
 const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums";
 const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = "loader_field_enum_values";
 
@@ -396,8 +397,57 @@ impl LoaderField {
             .collect();
         Ok(result)
     }
-}
 
+    // Gets all fields for a given loader(s)
+    // This is for tags, which need all fields for all loaders
+    // We want to return them even in testing situations where we dont have loaders or loader_fields_loaders set up
+    pub async fn get_fields_all<'a, E>(
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<Vec<LoaderField>, DatabaseError>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let mut redis = redis.connect().await?;
+
+        let cached_fields: Option<Vec<LoaderField>> = redis
+            .get(LOADER_FIELDS_NAMESPACE_ALL, "")
+            .await?
+            .and_then(|x| serde_json::from_str::<Vec<LoaderField>>(&x).ok());
+
+        if let Some(cached_fields) = cached_fields {
+            return Ok(cached_fields);
+        }
+
+        let result = sqlx::query!(
+            "
+            SELECT DISTINCT lf.id, lf.field, lf.field_type, lf.optional, lf.min_val, lf.max_val, lf.enum_type
+            FROM loader_fields lf
+            ",
+        )
+        .fetch_many(exec)
+        .try_filter_map(|e| async {
+            Ok(e.right().and_then(|r| {
+                Some(LoaderField {
+                    id: LoaderFieldId(r.id),
+                    field_type: LoaderFieldType::build(&r.field_type, r.enum_type)?,
+                    field: r.field,
+                    optional: r.optional,
+                    min_val: r.min_val,
+                    max_val: r.max_val,
+                })
+            }))
+        })
+        .try_collect::<Vec<LoaderField>>()
+        .await?;
+
+        redis
+            .set_serialized_to_json(LOADER_FIELDS_NAMESPACE_ALL, "", &result, None)
+            .await?;
+
+        Ok(result)
+    }
+}
 impl LoaderFieldEnum {
     pub async fn get<'a, E>(
         enum_name: &str, // Note: NOT loader field name

src/models/v2/mod.rs

@@ -1,2 +1,3 @@
 // Legacy models from V2, where its useful to keep the struct for rerouting/conversion
 pub mod projects;
+pub mod search;

src/models/v2/projects.rs

@@ -77,7 +77,11 @@ impl LegacyProject {
 
         // V2 versions only have one project type- v3 versions can rarely have multiple.
         // We'll just use the first one.
-        let mut project_type = data.project_types.first().cloned().unwrap_or_default();
+        let mut project_type = data
+            .project_types
+            .first()
+            .cloned()
+            .unwrap_or("unknown".to_string());
         let mut loaders = data.loaders;
 
         if let Some(versions_item) = versions_item {

src/models/v2/search.rs

@@ -0,0 +1,124 @@
+use serde::{Deserialize, Serialize};
+
+use crate::search::ResultSearchProject;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct LegacySearchResults {
+    pub hits: Vec<LegacyResultSearchProject>,
+    pub offset: usize,
+    pub limit: usize,
+    pub total_hits: usize,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct LegacyResultSearchProject {
+    pub project_id: String,
+    pub project_type: String,
+    pub slug: Option<String>,
+    pub author: String,
+    pub title: String,
+    pub description: String,
+    pub categories: Vec<String>,
+    pub display_categories: Vec<String>,
+    pub versions: Vec<String>,
+    pub downloads: i32,
+    pub follows: i32,
+    pub icon_url: String,
+    /// RFC 3339 formatted creation date of the project
+    pub date_created: String,
+    /// RFC 3339 formatted modification date of the project
+    pub date_modified: String,
+    pub latest_version: String,
+    pub license: String,
+    pub client_side: String,
+    pub server_side: String,
+    pub gallery: Vec<String>,
+    pub featured_gallery: Option<String>,
+    pub color: Option<u32>,
+}
+
+// TODO: In other PR, when these are merged, make sure the v2 search testing functions use these
+impl LegacyResultSearchProject {
+    pub fn from(result_search_project: ResultSearchProject) -> Self {
+        let mut categories = result_search_project.categories;
+        if categories.contains(&"mrpack".to_string()) {
+            if let Some(mrpack_loaders) = result_search_project.loader_fields.get("mrpack_loaders")
+            {
+                categories.extend(mrpack_loaders.clone());
+                categories.retain(|c| c != "mrpack");
+            }
+        }
+        let mut display_categories = result_search_project.display_categories;
+        if display_categories.contains(&"mrpack".to_string()) {
+            if let Some(mrpack_loaders) = result_search_project.loader_fields.get("mrpack_loaders")
+            {
+                display_categories.extend(mrpack_loaders.clone());
+                display_categories.retain(|c| c != "mrpack");
+            }
+        }
+
+        // Sort then remove duplicates
+        categories.sort();
+        categories.dedup();
+        display_categories.sort();
+        display_categories.dedup();
+
+        Self {
+            project_type: result_search_project
+                .project_types
+                .first()
+                .cloned()
+                .unwrap_or_default(),
+            client_side: result_search_project
+                .loader_fields
+                .get("client_side")
+                .cloned()
+                .unwrap_or_default()
+                .join(","),
+            server_side: result_search_project
+                .loader_fields
+                .get("server_side")
+                .cloned()
+                .unwrap_or_default()
+                .join(","),
+            versions: result_search_project
+                .loader_fields
+                .get("game_versions")
+                .cloned()
+                .unwrap_or_default(),
+            latest_version: result_search_project.version_id,
+            categories,
+
+            project_id: result_search_project.project_id,
+            slug: result_search_project.slug,
+            author: result_search_project.author,
+            title: result_search_project.title,
+            description: result_search_project.description,
+            display_categories,
+            downloads: result_search_project.downloads,
+            follows: result_search_project.follows,
+            icon_url: result_search_project.icon_url,
+            license: result_search_project.license,
+            date_created: result_search_project.date_created,
+            date_modified: result_search_project.date_modified,
+            gallery: result_search_project.gallery,
+            featured_gallery: result_search_project.featured_gallery,
+            color: result_search_project.color,
+        }
+    }
+}
+
+impl LegacySearchResults {
+    pub fn from(search_results: crate::search::SearchResults) -> Self {
+        Self {
+            hits: search_results
+                .hits
+                .into_iter()
+                .map(LegacyResultSearchProject::from)
+                .collect(),
+            offset: search_results.offset,
+            limit: search_results.limit,
+            total_hits: search_results.total_hits,
+        }
+    }
+}

src/routes/v2/projects.rs

@@ -6,6 +6,7 @@ use crate::models::projects::{
     DonationLink, MonetizationStatus, Project, ProjectStatus, SearchRequest, SideType,
 };
 use crate::models::v2::projects::LegacyProject;
+use crate::models::v2::search::LegacySearchResults;
 use crate::queue::session::AuthQueue;
 use crate::routes::v3::projects::ProjectIds;
 use crate::routes::{v2_reroute, v3, ApiError};
@@ -95,7 +96,7 @@ pub async fn project_search(
 
     let results = search_for_project(&info, &config).await?;
 
-    // TODO: convert to v2 format-we may need a new v2 struct for this for 'original' format
+    let results = LegacySearchResults::from(results);
 
     Ok(HttpResponse::Ok().json(results))
 }

src/routes/v3/tags.rs

@@ -7,7 +7,7 @@ use crate::database::models::loader_fields::{
 };
 use crate::database::redis::RedisPool;
 use actix_web::{web, HttpResponse};
-use itertools::Itertools;
+
 use serde_json::Value;
 use sqlx::PgPool;
 
@@ -121,20 +121,16 @@ pub async fn loader_fields_list(
     redis: web::Data<RedisPool>,
 ) -> Result<HttpResponse, ApiError> {
     let query = query.into_inner();
-    let all_loader_ids = Loader::list(&**pool, &redis)
+    let loader_field = LoaderField::get_fields_all(&**pool, &redis)
         .await?
         .into_iter()
-        .map(|x| x.id)
-        .collect_vec();
-    let loader_field =
-        LoaderField::get_field(&query.loader_field, &all_loader_ids, &**pool, &redis)
-            .await?
-            .ok_or_else(|| {
-                ApiError::InvalidInput(format!(
-                    "'{}' was not a valid loader field.",
-                    query.loader_field
-                ))
-            })?;
+        .find(|x| x.field == query.loader_field)
+        .ok_or_else(|| {
+            ApiError::InvalidInput(format!(
+                "'{}' was not a valid loader field.",
+                query.loader_field
+            ))
+        })?;
 
     let loader_field_enum_id = match loader_field.field_type {
         LoaderFieldType::Enum(enum_id) | LoaderFieldType::ArrayEnum(enum_id) => enum_id,

src/search/indexing/mod.rs

@@ -124,10 +124,16 @@ async fn create_and_add_to_index(
     let index = create_index(client, name, custom_rules).await?;
 
     let mut new_filterable_attributes = index.get_filterable_attributes().await?;
+    let mut new_displayed_attributes = index.get_displayed_attributes().await?;
+
     new_filterable_attributes.extend(additional_fields.iter().map(|s| s.to_string()));
+    new_displayed_attributes.extend(additional_fields.iter().map(|s| s.to_string()));
     index
         .set_filterable_attributes(new_filterable_attributes)
         .await?;
+    index
+        .set_displayed_attributes(new_displayed_attributes)
+        .await?;
 
     add_to_index(client, index, projects).await?;
     Ok(())

tests/common/api_common/generic.rs

@@ -2,12 +2,9 @@ use std::collections::HashMap;
 
 use actix_web::dev::ServiceResponse;
 use async_trait::async_trait;
-use labrinth::{
-    models::{
-        projects::{ProjectId, VersionType},
-        teams::{OrganizationPermissions, ProjectPermissions},
-    },
-    search::SearchResults,
+use labrinth::models::{
+    projects::{ProjectId, VersionType},
+    teams::{OrganizationPermissions, ProjectPermissions},
 };
 
 use crate::common::{api_v2::ApiV2, api_v3::ApiV3, dummy_data::TestFile};
@@ -76,7 +73,6 @@ delegate_api_variant!(
         [edit_project, ServiceResponse, id_or_slug: &str, patch: serde_json::Value, pat: &str],
         [edit_project_bulk, ServiceResponse, ids_or_slugs: &[&str], patch: serde_json::Value, pat: &str],
         [edit_project_icon, ServiceResponse, id_or_slug: &str, icon: Option<CommonImageData>, pat: &str],
-        [search_deserialized_common, SearchResults, query: Option<&str>, facets: Option<serde_json::Value>, pat: &str],
     }
 );