From 00aa50ba66b1a15ce71f36b4d5afde840fe2fa84 Mon Sep 17 00:00:00 2001 From: jonvdveen <196496658+jonvdveen@users.noreply.github.com> Date: Thu, 2 Jul 2026 18:56:25 -0600 Subject: [PATCH] feat: paginate workflow queries so large libraries are fully scanned All workflow trigger queries were capped at .limit(10000) with no ORDER BY and no pagination, so on libraries larger than 10k assets an all_assets ("full library scan") trigger silently evaluated an arbitrary 10k subset. The IF/SWITCH condition queries had the same cap, and bulk actions sent every asset id in a single Immich API request. - resolveAssetTrigger: keyset pagination (ORDER BY id, batches of 5000) for all three trigger types - IF/SWITCH: evaluate conditions against the incoming asset set in chunks instead of one capped inArray query - actionExecutor: call Immich bulk endpoints in batches of 1000 ids Verified on a 26,063-asset library (Immich v3.0.1): all_assets now resolves 26,063 (previously 10,000), a person-filter IF matched 813 assets vs 142 under the cap, and a 25,250-asset remove_from_album branch completed via batched requests. Fixes #296. Co-Authored-By: Claude Fable 5 --- src/lib/workflow/actionExecutor.ts | 27 +++++-- src/lib/workflow/engine.ts | 118 +++++++++++++++++++---------- 2 files changed, 96 insertions(+), 49 deletions(-) diff --git a/src/lib/workflow/actionExecutor.ts b/src/lib/workflow/actionExecutor.ts index 1fd09b1..6597a64 100644 --- a/src/lib/workflow/actionExecutor.ts +++ b/src/lib/workflow/actionExecutor.ts @@ -28,6 +28,16 @@ async function getWorkflowApiKey(ownerId: string): Promise { return row?.value || null; } +// Immich bulk endpoints are called in batches so large asset sets don't +// produce oversized request bodies. +const API_BATCH_SIZE = 1000; + +async function immichFetchBatched(path: string, method: string, ids: string[], extraBody: Record, user: IUser): Promise { + for (let i = 0; i < ids.length; i += API_BATCH_SIZE) { + await immichFetch(path, method, { ...extraBody, ids: ids.slice(i, i + API_BATCH_SIZE) }, user); + } +} + async function immichFetch(path: string, method: string, body: any, user: IUser): Promise { const workflowApiKey = await getWorkflowApiKey(user.id); @@ -136,34 +146,37 @@ export async function executeAction( switch (subType) { case "create_album": { const albumName = await resolveTemplate(config.nameTemplate || "Auto Album", assetIds); - const album = await immichFetch("/albums", "POST", { albumName, assetIds }, user); + const album = await immichFetch("/albums", "POST", { albumName, assetIds: assetIds.slice(0, API_BATCH_SIZE) }, user); + if (assetIds.length > API_BATCH_SIZE) { + await immichFetchBatched(`/albums/${album.id}/assets`, "PUT", assetIds.slice(API_BATCH_SIZE), {}, user); + } return { action: "create_album", assetsProcessed: assetIds.length, albumId: album.id, albumName }; } case "add_to_album": { if (!config.albumId) throw new Error("Album ID is required for add_to_album"); - await immichFetch(`/albums/${config.albumId}/assets`, "PUT", { ids: assetIds }, user); + await immichFetchBatched(`/albums/${config.albumId}/assets`, "PUT", assetIds, {}, user); return { action: "add_to_album", assetsProcessed: assetIds.length, albumId: config.albumId }; } case "remove_from_album": { if (!config.albumId) throw new Error("Album ID is required for remove_from_album"); - await immichFetch(`/albums/${config.albumId}/assets`, "DELETE", { ids: assetIds }, user); + await immichFetchBatched(`/albums/${config.albumId}/assets`, "DELETE", assetIds, {}, user); return { action: "remove_from_album", assetsProcessed: assetIds.length, albumId: config.albumId }; } case "favorite": { - await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: true }, user); + await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: true }, user); return { action: "favorite", assetsProcessed: assetIds.length }; } case "unfavorite": { - await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: false }, user); + await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: false }, user); return { action: "unfavorite", assetsProcessed: assetIds.length }; } case "archive": { - await immichFetch("/assets", "PUT", { ids: assetIds, visibility: "archive" }, user); + await immichFetchBatched("/assets", "PUT", assetIds, { visibility: "archive" }, user); return { action: "archive", assetsProcessed: assetIds.length }; } @@ -173,7 +186,7 @@ export async function executeAction( // Create or get tag const tag = await immichFetch("/tags", "POST", { name: config.tagName }, user); // Tag assets - await immichFetch(`/tags/${tag.id}/assets`, "PUT", { ids: assetIds }, user); + await immichFetchBatched(`/tags/${tag.id}/assets`, "PUT", assetIds, {}, user); return { action: "tag", assetsProcessed: assetIds.length }; } catch (e: any) { return { action: "tag", assetsProcessed: 0, error: e.message }; diff --git a/src/lib/workflow/engine.ts b/src/lib/workflow/engine.ts index e49683f..4289c08 100644 --- a/src/lib/workflow/engine.ts +++ b/src/lib/workflow/engine.ts @@ -3,7 +3,7 @@ import { db } from "@/config/db"; import { workflows, workflowNodes, workflowEdges, workflowRuns, workflowProcessedAssets } from "@/db/schema/workflows.schema"; import { assets } from "@/schema/assets.schema"; import { exif } from "@/schema"; -import { eq, and, desc, gte, isNull, inArray, sql, ne } from "drizzle-orm"; +import { eq, and, desc, gt, gte, isNull, inArray, sql, ne, SQL } from "drizzle-orm"; import { buildConditions } from "./conditionBuilder"; import { executeAction } from "./actionExecutor"; import { IUser } from "@/types/user"; @@ -19,6 +19,53 @@ function logError(runId: string, ...args: any[]) { console.error(`${PREFIX} [run:${runId.slice(0, 8)}]`, ...args); } +const QUERY_BATCH_SIZE = 5000; + +// Keyset-paginate an asset query so large libraries aren't silently truncated. +// fetchBatch must ORDER BY assets.id and apply gt(assets.id, afterId) when given. +async function fetchAllBatches( + fetchBatch: (afterId: string | null, limit: number) => Promise, +): Promise { + const all: T[] = []; + let cursor: string | null = null; + for (;;) { + const batch = await fetchBatch(cursor, QUERY_BATCH_SIZE); + all.push(...batch); + if (batch.length < QUERY_BATCH_SIZE) break; + cursor = batch[batch.length - 1].id; + } + return all; +} + +// Evaluate condition clauses against an incoming asset set (chunked to keep +// IN-lists bounded), or against the whole library when scopeIds is null. +async function queryConditionMatches(whereClauses: SQL[], scopeIds: string[] | null): Promise { + if (scopeIds !== null) { + const matched: string[] = []; + for (let i = 0; i < scopeIds.length; i += QUERY_BATCH_SIZE) { + const chunk = scopeIds.slice(i, i + QUERY_BATCH_SIZE); + const rows = await db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .leftJoin(exif, eq(assets.id, exif.assetId)) + .where(and(...whereClauses, inArray(assets.id, chunk))); + matched.push(...rows.map((r) => r.id)); + } + return matched; + } + + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .leftJoin(exif, eq(assets.id, exif.assetId)) + .where(and(...whereClauses, ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); + return rows.map((r) => r.id); +} + async function resolveAssetTrigger( subType: string, workflowId: string, @@ -80,21 +127,27 @@ async function resolveAssetTrigger( let candidateIds: string[]; if (subType === "new_asset") { - const rows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .where(and(...baseConditions, gte(assets.createdAt, sinceDate))) - .limit(10000); + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .where(and(...baseConditions, gte(assets.createdAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); const totalCandidates = rows.length; // For new_asset: skip any asset we've ever processed candidateIds = rows.map((r) => r.id).filter((id) => !processedMap?.has(id)); log(runId, `Trigger [new_asset]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} already processed, ${candidateIds.length} remaining`); } else if (subType === "asset_updated") { - const rows = await db - .selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt }) - .from(assets) - .where(and(...baseConditions, gte(assets.updatedAt, sinceDate))) - .limit(10000); + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt }) + .from(assets) + .where(and(...baseConditions, gte(assets.updatedAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); const totalCandidates = rows.length; // For asset_updated: only skip if we processed it AFTER its current updatedAt candidateIds = rows @@ -107,11 +160,14 @@ async function resolveAssetTrigger( log(runId, `Trigger [asset_updated]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} skipped (not updated since processing), ${candidateIds.length} remaining`); } else { // all_assets — no dedup - const rows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .where(and(...baseConditions)) - .limit(10000); + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .where(and(...baseConditions, ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); log(runId, `Trigger [all_assets]: ${rows.length} assets found`); return rows.map((r) => r.id); } @@ -244,19 +300,8 @@ export async function executeWorkflow( const whereClauses = buildConditions(conditions, user.id); - // Scope to incoming assets if available - if (assetIds !== null && assetIds.length > 0) { - whereClauses.push(inArray(assets.id, assetIds)); - } - - const matchedRows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .leftJoin(exif, eq(assets.id, exif.assetId)) - .where(and(...whereClauses)) - .limit(10000); - - const matchedIds = matchedRows.map((r) => r.id); + // Scoped to incoming assets (chunked) when available + const matchedIds = await queryConditionMatches(whereClauses, assetIds); const matchedSet = new Set(matchedIds); let trueIds = matchedIds; @@ -298,19 +343,8 @@ export async function executeWorkflow( const whereClauses = buildConditions(c.conditions || [], user.id); - // Scope to remaining assets if available - if (remaining !== null && remaining.length > 0) { - whereClauses.push(inArray(assets.id, remaining)); - } - - const matchedRows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .leftJoin(exif, eq(assets.id, exif.assetId)) - .where(and(...whereClauses)) - .limit(10000); - - const matchedIds = matchedRows.map((r) => r.id); + // Scoped to remaining assets (chunked) when available + const matchedIds = await queryConditionMatches(whereClauses, remaining); const matchedSet = new Set(matchedIds); let caseIds: string[];