From 0b791b4bb8e946f1db40e80cb6a4df3d583327d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 20:40:12 -0500 Subject: [PATCH 1/9] Batch search indexing queries --- .../src/search/indexing/local_import.rs | 21 +++++++--- apps/labrinth/src/search/indexing/mod.rs | 42 ++++++++++++++----- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/apps/labrinth/src/search/indexing/local_import.rs b/apps/labrinth/src/search/indexing/local_import.rs index e1620d8ce1..7c0417880c 100644 --- a/apps/labrinth/src/search/indexing/local_import.rs +++ b/apps/labrinth/src/search/indexing/local_import.rs @@ -20,10 +20,12 @@ use crate::routes::v2_reroute; use crate::search::UploadSearchProject; use sqlx::postgres::PgPool; -pub async fn index_local( +pub async fn index_local_chunk( pool: &PgPool, -) -> Result, IndexingError> { - info!("Indexing local projects!"); + cursor: i64, + limit: i64, +) -> Result<(Vec, i64), IndexingError> { + info!(cursor = cursor, limit = limit, "Indexing local projects"); // todo: loaders, project type, game versions struct PartialProject { @@ -45,13 +47,16 @@ pub async fn index_local( SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color FROM mods m - WHERE m.status = ANY($1) - GROUP BY m.id; + WHERE m.status = ANY($1) AND m.id > $2 + GROUP BY m.id + LIMIT $3; ", &*crate::models::projects::ProjectStatus::iterator() .filter(|x| x.is_searchable()) .map(|x| x.to_string()) .collect::>(), + cursor, + limit, ) .fetch(pool) .map_ok(|m| { @@ -74,6 +79,10 @@ pub async fn index_local( let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); + let Some(largest) = project_ids.iter().max() else { + return Ok((vec![], i64::MAX)); + }; + struct PartialGallery { url: String, featured: bool, @@ -412,7 +421,7 @@ pub async fn index_local( } } - Ok(uploads) + Ok((uploads, *largest)) } struct PartialVersion { diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index 52c8c5d909..4243a3ab3e 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -8,7 +8,7 @@ use crate::search::{SearchConfig, UploadSearchProject}; use ariadne::ids::base62_impl::to_base62; use futures::StreamExt; use futures::stream::FuturesOrdered; -use local_import::index_local; +use local_import::index_local_chunk; use meilisearch_sdk::client::{Client, SwapIndexes}; use meilisearch_sdk::indexes::Index; use meilisearch_sdk::settings::{PaginationSetting, Settings}; @@ -34,8 +34,8 @@ pub enum IndexingError { // The chunk size for adding projects to the indexing database. If the request size // is too large (>10MiB) then the request fails with an error. This chunk size -// assumes a max average size of 4KiB per project to avoid this cap. -const MEILISEARCH_CHUNK_SIZE: usize = 10000000; +// assumes a max average size of 8KiB per project to avoid this cap. +const MEILISEARCH_CHUNK_SIZE: usize = 5000000; const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); pub async fn remove_documents( @@ -115,15 +115,35 @@ pub async fn index_projects( .map(|x| x.field) .collect::>(); - let uploads = index_local(&pool).await?; + let mut cursor = 0; + let mut idx = 0; + let mut total = 0; - add_projects_batch_client( - &indices, - uploads, - all_loader_fields.clone(), - config, - ) - .await?; + loop { + info!("Gathering index data chunk {idx}"); + idx += 1; + + let (uploads, next_cursor) = + index_local_chunk(&pool, cursor, 10000).await?; + total += uploads.len(); + + if total == 0 { + info!( + "No more projects to index, indexed {total} projects after {idx} chunks" + ); + break; + } + + cursor = next_cursor; + + add_projects_batch_client( + &indices, + uploads, + all_loader_fields.clone(), + config, + ) + .await?; + } // Swap the index swap_index(config, "projects").await?; From 7b96b696b706baf043b86017779c7ca386ac3df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 20:41:23 -0500 Subject: [PATCH 2/9] Query cache --- ...f4eeff66ab4165a9f4980032e114db4dc1286.json | 26 ------------------- ...1a44b780a2e3b8ecc3330ec26c7a70f82e29.json} | 8 +++--- ...d2402f52fea71e27b08e7926fcc2a9e62c0f3.json | 20 -------------- ...afedb074492b4ec7f2457c14113f5fd13aa02.json | 18 ------------- ...e5c93783c7641b019fdb698a1ec0be1393606.json | 17 ------------ 5 files changed, 5 insertions(+), 84 deletions(-) delete mode 100644 apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json rename apps/labrinth/.sqlx/{query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json => query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json} (88%) delete mode 100644 apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json delete mode 100644 apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json delete mode 100644 apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json diff --git a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json b/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json deleted file mode 100644 index 921f7f92d9..0000000000 --- a/apps/labrinth/.sqlx/query-1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id,\n status AS \"status: PayoutStatus\"\n FROM payouts\n ORDER BY id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "status: PayoutStatus", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false - ] - }, - "hash": "1adbd24d815107e13bc1440c7a8f4eeff66ab4165a9f4980032e114db4dc1286" -} diff --git a/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json b/apps/labrinth/.sqlx/query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json similarity index 88% rename from apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json rename to apps/labrinth/.sqlx/query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json index 6142e7dcaf..8af5783487 100644 --- a/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json +++ b/apps/labrinth/.sqlx/query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1)\n GROUP BY m.id;\n ", + "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1) AND m.id > $2\n GROUP BY m.id\n LIMIT $3;\n ", "describe": { "columns": [ { @@ -66,7 +66,9 @@ ], "parameters": { "Left": [ - "TextArray" + "TextArray", + "Int8", + "Int8" ] }, "nullable": [ @@ -84,5 +86,5 @@ true ] }, - "hash": "b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5" + "hash": "ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29" } diff --git a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json b/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json deleted file mode 100644 index 89bd8147dc..0000000000 --- a/apps/labrinth/.sqlx/query-b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status AS \"status: PayoutStatus\" FROM payouts WHERE id = 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "status: PayoutStatus", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "b92b5bb7d179c4fcdbc45600ccfd2402f52fea71e27b08e7926fcc2a9e62c0f3" -} diff --git a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json b/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json deleted file mode 100644 index 469c30168a..0000000000 --- a/apps/labrinth/.sqlx/query-cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, $3, $4, $5, 10.0, NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Varchar", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "cd5ccd618fb3cc41646a6de86f9afedb074492b4ec7f2457c14113f5fd13aa02" -} diff --git a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json b/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json deleted file mode 100644 index 52e020ebf2..0000000000 --- a/apps/labrinth/.sqlx/query-cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO payouts (id, method, platform_id, status, user_id, amount, created)\n VALUES ($1, $2, NULL, $3, $4, 10.00, NOW())\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Varchar", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "cec4240c7c848988b3dfd13e3f8e5c93783c7641b019fdb698a1ec0be1393606" -} From 0b337df27a8348e839852b368e60a9d43060e5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 22:07:27 -0500 Subject: [PATCH 3/9] why is ci doing that --- ...b3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json} | 4 ++-- apps/labrinth/src/search/indexing/local_import.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename apps/labrinth/.sqlx/{query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json => query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json} (94%) diff --git a/apps/labrinth/.sqlx/query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json b/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json similarity index 94% rename from apps/labrinth/.sqlx/query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json rename to apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json index 8af5783487..32dcb10585 100644 --- a/apps/labrinth/.sqlx/query-ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29.json +++ b/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1) AND m.id > $2\n GROUP BY m.id\n LIMIT $3;\n ", + "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1) AND m.id > $2\n GROUP BY m.id\n LIMIT $3\n ", "describe": { "columns": [ { @@ -86,5 +86,5 @@ true ] }, - "hash": "ab07bd689e83d7e1d26b0ccb13a01a44b780a2e3b8ecc3330ec26c7a70f82e29" + "hash": "2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa" } diff --git a/apps/labrinth/src/search/indexing/local_import.rs b/apps/labrinth/src/search/indexing/local_import.rs index 7c0417880c..001984b29d 100644 --- a/apps/labrinth/src/search/indexing/local_import.rs +++ b/apps/labrinth/src/search/indexing/local_import.rs @@ -49,7 +49,7 @@ pub async fn index_local_chunk( FROM mods m WHERE m.status = ANY($1) AND m.id > $2 GROUP BY m.id - LIMIT $3; + LIMIT $3 ", &*crate::models::projects::ProjectStatus::iterator() .filter(|x| x.is_searchable()) From a21bea96547acfa0921f494ae2646e1f7a1b887d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 23:07:04 -0500 Subject: [PATCH 4/9] Revert "Batch search indexing queries" This reverts commit 0b791b4bb8e946f1db40e80cb6a4df3d583327d7. --- .../src/search/indexing/local_import.rs | 21 +++------- apps/labrinth/src/search/indexing/mod.rs | 42 +++++-------------- 2 files changed, 17 insertions(+), 46 deletions(-) diff --git a/apps/labrinth/src/search/indexing/local_import.rs b/apps/labrinth/src/search/indexing/local_import.rs index 001984b29d..e1620d8ce1 100644 --- a/apps/labrinth/src/search/indexing/local_import.rs +++ b/apps/labrinth/src/search/indexing/local_import.rs @@ -20,12 +20,10 @@ use crate::routes::v2_reroute; use crate::search::UploadSearchProject; use sqlx::postgres::PgPool; -pub async fn index_local_chunk( +pub async fn index_local( pool: &PgPool, - cursor: i64, - limit: i64, -) -> Result<(Vec, i64), IndexingError> { - info!(cursor = cursor, limit = limit, "Indexing local projects"); +) -> Result, IndexingError> { + info!("Indexing local projects!"); // todo: loaders, project type, game versions struct PartialProject { @@ -47,16 +45,13 @@ pub async fn index_local_chunk( SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color FROM mods m - WHERE m.status = ANY($1) AND m.id > $2 - GROUP BY m.id - LIMIT $3 + WHERE m.status = ANY($1) + GROUP BY m.id; ", &*crate::models::projects::ProjectStatus::iterator() .filter(|x| x.is_searchable()) .map(|x| x.to_string()) .collect::>(), - cursor, - limit, ) .fetch(pool) .map_ok(|m| { @@ -79,10 +74,6 @@ pub async fn index_local_chunk( let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); - let Some(largest) = project_ids.iter().max() else { - return Ok((vec![], i64::MAX)); - }; - struct PartialGallery { url: String, featured: bool, @@ -421,7 +412,7 @@ pub async fn index_local_chunk( } } - Ok((uploads, *largest)) + Ok(uploads) } struct PartialVersion { diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index 4243a3ab3e..52c8c5d909 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -8,7 +8,7 @@ use crate::search::{SearchConfig, UploadSearchProject}; use ariadne::ids::base62_impl::to_base62; use futures::StreamExt; use futures::stream::FuturesOrdered; -use local_import::index_local_chunk; +use local_import::index_local; use meilisearch_sdk::client::{Client, SwapIndexes}; use meilisearch_sdk::indexes::Index; use meilisearch_sdk::settings::{PaginationSetting, Settings}; @@ -34,8 +34,8 @@ pub enum IndexingError { // The chunk size for adding projects to the indexing database. If the request size // is too large (>10MiB) then the request fails with an error. This chunk size -// assumes a max average size of 8KiB per project to avoid this cap. -const MEILISEARCH_CHUNK_SIZE: usize = 5000000; +// assumes a max average size of 4KiB per project to avoid this cap. +const MEILISEARCH_CHUNK_SIZE: usize = 10000000; const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); pub async fn remove_documents( @@ -115,35 +115,15 @@ pub async fn index_projects( .map(|x| x.field) .collect::>(); - let mut cursor = 0; - let mut idx = 0; - let mut total = 0; + let uploads = index_local(&pool).await?; - loop { - info!("Gathering index data chunk {idx}"); - idx += 1; - - let (uploads, next_cursor) = - index_local_chunk(&pool, cursor, 10000).await?; - total += uploads.len(); - - if total == 0 { - info!( - "No more projects to index, indexed {total} projects after {idx} chunks" - ); - break; - } - - cursor = next_cursor; - - add_projects_batch_client( - &indices, - uploads, - all_loader_fields.clone(), - config, - ) - .await?; - } + add_projects_batch_client( + &indices, + uploads, + all_loader_fields.clone(), + config, + ) + .await?; // Swap the index swap_index(config, "projects").await?; From 7f554929ed5fdcd9c6780fc2f8fcc421a236cca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 23:08:05 -0500 Subject: [PATCH 5/9] so is it actually my changes causing this or what --- ...ee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json} | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) rename apps/labrinth/.sqlx/{query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json => query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json} (88%) diff --git a/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json b/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json similarity index 88% rename from apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json rename to apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json index 32dcb10585..6142e7dcaf 100644 --- a/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json +++ b/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1) AND m.id > $2\n GROUP BY m.id\n LIMIT $3\n ", + "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1)\n GROUP BY m.id;\n ", "describe": { "columns": [ { @@ -66,9 +66,7 @@ ], "parameters": { "Left": [ - "TextArray", - "Int8", - "Int8" + "TextArray" ] }, "nullable": [ @@ -86,5 +84,5 @@ true ] }, - "hash": "2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa" + "hash": "b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5" } From c305ad3e22581a0abd0c7f4ce8c82c9912127337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 23:22:28 -0500 Subject: [PATCH 6/9] Reapply "Batch search indexing queries" This reverts commit a21bea96547acfa0921f494ae2646e1f7a1b887d. --- .../src/search/indexing/local_import.rs | 21 +++++++--- apps/labrinth/src/search/indexing/mod.rs | 42 ++++++++++++++----- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/apps/labrinth/src/search/indexing/local_import.rs b/apps/labrinth/src/search/indexing/local_import.rs index e1620d8ce1..001984b29d 100644 --- a/apps/labrinth/src/search/indexing/local_import.rs +++ b/apps/labrinth/src/search/indexing/local_import.rs @@ -20,10 +20,12 @@ use crate::routes::v2_reroute; use crate::search::UploadSearchProject; use sqlx::postgres::PgPool; -pub async fn index_local( +pub async fn index_local_chunk( pool: &PgPool, -) -> Result, IndexingError> { - info!("Indexing local projects!"); + cursor: i64, + limit: i64, +) -> Result<(Vec, i64), IndexingError> { + info!(cursor = cursor, limit = limit, "Indexing local projects"); // todo: loaders, project type, game versions struct PartialProject { @@ -45,13 +47,16 @@ pub async fn index_local( SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color FROM mods m - WHERE m.status = ANY($1) - GROUP BY m.id; + WHERE m.status = ANY($1) AND m.id > $2 + GROUP BY m.id + LIMIT $3 ", &*crate::models::projects::ProjectStatus::iterator() .filter(|x| x.is_searchable()) .map(|x| x.to_string()) .collect::>(), + cursor, + limit, ) .fetch(pool) .map_ok(|m| { @@ -74,6 +79,10 @@ pub async fn index_local( let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); + let Some(largest) = project_ids.iter().max() else { + return Ok((vec![], i64::MAX)); + }; + struct PartialGallery { url: String, featured: bool, @@ -412,7 +421,7 @@ pub async fn index_local( } } - Ok(uploads) + Ok((uploads, *largest)) } struct PartialVersion { diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index 52c8c5d909..4243a3ab3e 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -8,7 +8,7 @@ use crate::search::{SearchConfig, UploadSearchProject}; use ariadne::ids::base62_impl::to_base62; use futures::StreamExt; use futures::stream::FuturesOrdered; -use local_import::index_local; +use local_import::index_local_chunk; use meilisearch_sdk::client::{Client, SwapIndexes}; use meilisearch_sdk::indexes::Index; use meilisearch_sdk::settings::{PaginationSetting, Settings}; @@ -34,8 +34,8 @@ pub enum IndexingError { // The chunk size for adding projects to the indexing database. If the request size // is too large (>10MiB) then the request fails with an error. This chunk size -// assumes a max average size of 4KiB per project to avoid this cap. -const MEILISEARCH_CHUNK_SIZE: usize = 10000000; +// assumes a max average size of 8KiB per project to avoid this cap. +const MEILISEARCH_CHUNK_SIZE: usize = 5000000; const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); pub async fn remove_documents( @@ -115,15 +115,35 @@ pub async fn index_projects( .map(|x| x.field) .collect::>(); - let uploads = index_local(&pool).await?; + let mut cursor = 0; + let mut idx = 0; + let mut total = 0; - add_projects_batch_client( - &indices, - uploads, - all_loader_fields.clone(), - config, - ) - .await?; + loop { + info!("Gathering index data chunk {idx}"); + idx += 1; + + let (uploads, next_cursor) = + index_local_chunk(&pool, cursor, 10000).await?; + total += uploads.len(); + + if total == 0 { + info!( + "No more projects to index, indexed {total} projects after {idx} chunks" + ); + break; + } + + cursor = next_cursor; + + add_projects_batch_client( + &indices, + uploads, + all_loader_fields.clone(), + config, + ) + .await?; + } // Swap the index swap_index(config, "projects").await?; From 7f8bb9d4192c094d84b990bb2bf37fc202fa7550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Wed, 14 Jan 2026 23:23:31 -0500 Subject: [PATCH 7/9] how does this even happen --- ...2e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json} | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) rename apps/labrinth/.sqlx/{query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json => query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json} (88%) diff --git a/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json b/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json similarity index 88% rename from apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json rename to apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json index 6142e7dcaf..32dcb10585 100644 --- a/apps/labrinth/.sqlx/query-b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5.json +++ b/apps/labrinth/.sqlx/query-2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1)\n GROUP BY m.id;\n ", + "query": "\n SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,\n m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color\n FROM mods m\n WHERE m.status = ANY($1) AND m.id > $2\n GROUP BY m.id\n LIMIT $3\n ", "describe": { "columns": [ { @@ -66,7 +66,9 @@ ], "parameters": { "Left": [ - "TextArray" + "TextArray", + "Int8", + "Int8" ] }, "nullable": [ @@ -84,5 +86,5 @@ true ] }, - "hash": "b30d0365bd116fceee5de03fb9e3087a587633783894a5041889b856d47a4ed5" + "hash": "2fcd377a3c3db3a82e46ace4f8f63c4e5f10748010da75c58ec0c544525addaa" } From c59d4efa598931e2cf3e93936857e6940ff763cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Thu, 15 Jan 2026 00:01:39 -0500 Subject: [PATCH 8/9] ggs --- apps/labrinth/src/search/indexing/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index 4243a3ab3e..aede505f55 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -127,7 +127,7 @@ pub async fn index_projects( index_local_chunk(&pool, cursor, 10000).await?; total += uploads.len(); - if total == 0 { + if uploads.len() == 0 { info!( "No more projects to index, indexed {total} projects after {idx} chunks" ); From 7c9aaad5ac7b528a4f241e5bd46a3345f7a567ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois-X=2E=20T=2E?= Date: Thu, 15 Jan 2026 00:16:31 -0500 Subject: [PATCH 9/9] Lint --- apps/labrinth/src/search/indexing/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index aede505f55..f9f3b3d161 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -127,7 +127,7 @@ pub async fn index_projects( index_local_chunk(&pool, cursor, 10000).await?; total += uploads.len(); - if uploads.len() == 0 { + if uploads.is_empty() { info!( "No more projects to index, indexed {total} projects after {idx} chunks" );