diff --git a/src/lib/workflow/actionExecutor.ts b/src/lib/workflow/actionExecutor.ts index 1fd09b1..6597a64 100644 --- a/src/lib/workflow/actionExecutor.ts +++ b/src/lib/workflow/actionExecutor.ts @@ -28,6 +28,16 @@ async function getWorkflowApiKey(ownerId: string): Promise { return row?.value || null; } +// Immich bulk endpoints are called in batches so large asset sets don't +// produce oversized request bodies. +const API_BATCH_SIZE = 1000; + +async function immichFetchBatched(path: string, method: string, ids: string[], extraBody: Record, user: IUser): Promise { + for (let i = 0; i < ids.length; i += API_BATCH_SIZE) { + await immichFetch(path, method, { ...extraBody, ids: ids.slice(i, i + API_BATCH_SIZE) }, user); + } +} + async function immichFetch(path: string, method: string, body: any, user: IUser): Promise { const workflowApiKey = await getWorkflowApiKey(user.id); @@ -136,34 +146,37 @@ export async function executeAction( switch (subType) { case "create_album": { const albumName = await resolveTemplate(config.nameTemplate || "Auto Album", assetIds); - const album = await immichFetch("/albums", "POST", { albumName, assetIds }, user); + const album = await immichFetch("/albums", "POST", { albumName, assetIds: assetIds.slice(0, API_BATCH_SIZE) }, user); + if (assetIds.length > API_BATCH_SIZE) { + await immichFetchBatched(`/albums/${album.id}/assets`, "PUT", assetIds.slice(API_BATCH_SIZE), {}, user); + } return { action: "create_album", assetsProcessed: assetIds.length, albumId: album.id, albumName }; } case "add_to_album": { if (!config.albumId) throw new Error("Album ID is required for add_to_album"); - await immichFetch(`/albums/${config.albumId}/assets`, "PUT", { ids: assetIds }, user); + await immichFetchBatched(`/albums/${config.albumId}/assets`, "PUT", assetIds, {}, user); return { action: "add_to_album", assetsProcessed: assetIds.length, albumId: config.albumId }; } case "remove_from_album": { if (!config.albumId) throw new Error("Album ID is required for remove_from_album"); - await immichFetch(`/albums/${config.albumId}/assets`, "DELETE", { ids: assetIds }, user); + await immichFetchBatched(`/albums/${config.albumId}/assets`, "DELETE", assetIds, {}, user); return { action: "remove_from_album", assetsProcessed: assetIds.length, albumId: config.albumId }; } case "favorite": { - await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: true }, user); + await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: true }, user); return { action: "favorite", assetsProcessed: assetIds.length }; } case "unfavorite": { - await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: false }, user); + await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: false }, user); return { action: "unfavorite", assetsProcessed: assetIds.length }; } case "archive": { - await immichFetch("/assets", "PUT", { ids: assetIds, visibility: "archive" }, user); + await immichFetchBatched("/assets", "PUT", assetIds, { visibility: "archive" }, user); return { action: "archive", assetsProcessed: assetIds.length }; } @@ -173,7 +186,7 @@ export async function executeAction( // Create or get tag const tag = await immichFetch("/tags", "POST", { name: config.tagName }, user); // Tag assets - await immichFetch(`/tags/${tag.id}/assets`, "PUT", { ids: assetIds }, user); + await immichFetchBatched(`/tags/${tag.id}/assets`, "PUT", assetIds, {}, user); return { action: "tag", assetsProcessed: assetIds.length }; } catch (e: any) { return { action: "tag", assetsProcessed: 0, error: e.message }; diff --git a/src/lib/workflow/engine.ts b/src/lib/workflow/engine.ts index e49683f..4289c08 100644 --- a/src/lib/workflow/engine.ts +++ b/src/lib/workflow/engine.ts @@ -3,7 +3,7 @@ import { db } from "@/config/db"; import { workflows, workflowNodes, workflowEdges, workflowRuns, workflowProcessedAssets } from "@/db/schema/workflows.schema"; import { assets } from "@/schema/assets.schema"; import { exif } from "@/schema"; -import { eq, and, desc, gte, isNull, inArray, sql, ne } from "drizzle-orm"; +import { eq, and, desc, gt, gte, isNull, inArray, sql, ne, SQL } from "drizzle-orm"; import { buildConditions } from "./conditionBuilder"; import { executeAction } from "./actionExecutor"; import { IUser } from "@/types/user"; @@ -19,6 +19,53 @@ function logError(runId: string, ...args: any[]) { console.error(`${PREFIX} [run:${runId.slice(0, 8)}]`, ...args); } +const QUERY_BATCH_SIZE = 5000; + +// Keyset-paginate an asset query so large libraries aren't silently truncated. +// fetchBatch must ORDER BY assets.id and apply gt(assets.id, afterId) when given. +async function fetchAllBatches( + fetchBatch: (afterId: string | null, limit: number) => Promise, +): Promise { + const all: T[] = []; + let cursor: string | null = null; + for (;;) { + const batch = await fetchBatch(cursor, QUERY_BATCH_SIZE); + all.push(...batch); + if (batch.length < QUERY_BATCH_SIZE) break; + cursor = batch[batch.length - 1].id; + } + return all; +} + +// Evaluate condition clauses against an incoming asset set (chunked to keep +// IN-lists bounded), or against the whole library when scopeIds is null. +async function queryConditionMatches(whereClauses: SQL[], scopeIds: string[] | null): Promise { + if (scopeIds !== null) { + const matched: string[] = []; + for (let i = 0; i < scopeIds.length; i += QUERY_BATCH_SIZE) { + const chunk = scopeIds.slice(i, i + QUERY_BATCH_SIZE); + const rows = await db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .leftJoin(exif, eq(assets.id, exif.assetId)) + .where(and(...whereClauses, inArray(assets.id, chunk))); + matched.push(...rows.map((r) => r.id)); + } + return matched; + } + + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .leftJoin(exif, eq(assets.id, exif.assetId)) + .where(and(...whereClauses, ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); + return rows.map((r) => r.id); +} + async function resolveAssetTrigger( subType: string, workflowId: string, @@ -80,21 +127,27 @@ async function resolveAssetTrigger( let candidateIds: string[]; if (subType === "new_asset") { - const rows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .where(and(...baseConditions, gte(assets.createdAt, sinceDate))) - .limit(10000); + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .where(and(...baseConditions, gte(assets.createdAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); const totalCandidates = rows.length; // For new_asset: skip any asset we've ever processed candidateIds = rows.map((r) => r.id).filter((id) => !processedMap?.has(id)); log(runId, `Trigger [new_asset]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} already processed, ${candidateIds.length} remaining`); } else if (subType === "asset_updated") { - const rows = await db - .selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt }) - .from(assets) - .where(and(...baseConditions, gte(assets.updatedAt, sinceDate))) - .limit(10000); + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt }) + .from(assets) + .where(and(...baseConditions, gte(assets.updatedAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); const totalCandidates = rows.length; // For asset_updated: only skip if we processed it AFTER its current updatedAt candidateIds = rows @@ -107,11 +160,14 @@ async function resolveAssetTrigger( log(runId, `Trigger [asset_updated]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} skipped (not updated since processing), ${candidateIds.length} remaining`); } else { // all_assets — no dedup - const rows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .where(and(...baseConditions)) - .limit(10000); + const rows = await fetchAllBatches((afterId, limit) => + db + .selectDistinctOn([assets.id], { id: assets.id }) + .from(assets) + .where(and(...baseConditions, ...(afterId ? [gt(assets.id, afterId)] : []))) + .orderBy(assets.id) + .limit(limit) + ); log(runId, `Trigger [all_assets]: ${rows.length} assets found`); return rows.map((r) => r.id); } @@ -244,19 +300,8 @@ export async function executeWorkflow( const whereClauses = buildConditions(conditions, user.id); - // Scope to incoming assets if available - if (assetIds !== null && assetIds.length > 0) { - whereClauses.push(inArray(assets.id, assetIds)); - } - - const matchedRows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .leftJoin(exif, eq(assets.id, exif.assetId)) - .where(and(...whereClauses)) - .limit(10000); - - const matchedIds = matchedRows.map((r) => r.id); + // Scoped to incoming assets (chunked) when available + const matchedIds = await queryConditionMatches(whereClauses, assetIds); const matchedSet = new Set(matchedIds); let trueIds = matchedIds; @@ -298,19 +343,8 @@ export async function executeWorkflow( const whereClauses = buildConditions(c.conditions || [], user.id); - // Scope to remaining assets if available - if (remaining !== null && remaining.length > 0) { - whereClauses.push(inArray(assets.id, remaining)); - } - - const matchedRows = await db - .selectDistinctOn([assets.id], { id: assets.id }) - .from(assets) - .leftJoin(exif, eq(assets.id, exif.assetId)) - .where(and(...whereClauses)) - .limit(10000); - - const matchedIds = matchedRows.map((r) => r.id); + // Scoped to remaining assets (chunked) when available + const matchedIds = await queryConditionMatches(whereClauses, remaining); const matchedSet = new Set(matchedIds); let caseIds: string[];