Merge pull request #297 from jonvdveen/feat/paginate-workflow-triggers

feat: paginate workflow asset queries — remove the silent 10,000-asset cap
This commit is contained in:
Varun Raj Manoharan authored and GitHub committed 2026-07-10 19:36:36 +05:30
commit 800751d493
2 files changed
+96 -49

No files matched your search

+20 -7
View File
@@ -28,6 +28,16 @@ async function getWorkflowApiKey(ownerId: string): Promise<string | null> {
return row?.value || null;
}
// Immich bulk endpoints are called in batches so large asset sets don't
// produce oversized request bodies.
const API_BATCH_SIZE = 1000;
async function immichFetchBatched(path: string, method: string, ids: string[], extraBody: Record<string, any>, user: IUser): Promise<void> {
for (let i = 0; i < ids.length; i += API_BATCH_SIZE) {
await immichFetch(path, method, { ...extraBody, ids: ids.slice(i, i + API_BATCH_SIZE) }, user);
}
}
async function immichFetch(path: string, method: string, body: any, user: IUser): Promise<any> {
const workflowApiKey = await getWorkflowApiKey(user.id);
@@ -136,34 +146,37 @@ export async function executeAction(
switch (subType) {
case "create_album": {
const albumName = await resolveTemplate(config.nameTemplate || "Auto Album", assetIds);
const album = await immichFetch("/albums", "POST", { albumName, assetIds }, user);
const album = await immichFetch("/albums", "POST", { albumName, assetIds: assetIds.slice(0, API_BATCH_SIZE) }, user);
if (assetIds.length > API_BATCH_SIZE) {
await immichFetchBatched(`/albums/${album.id}/assets`, "PUT", assetIds.slice(API_BATCH_SIZE), {}, user);
}
return { action: "create_album", assetsProcessed: assetIds.length, albumId: album.id, albumName };
}
case "add_to_album": {
if (!config.albumId) throw new Error("Album ID is required for add_to_album");
await immichFetch(`/albums/${config.albumId}/assets`, "PUT", { ids: assetIds }, user);
await immichFetchBatched(`/albums/${config.albumId}/assets`, "PUT", assetIds, {}, user);
return { action: "add_to_album", assetsProcessed: assetIds.length, albumId: config.albumId };
}
case "remove_from_album": {
if (!config.albumId) throw new Error("Album ID is required for remove_from_album");
await immichFetch(`/albums/${config.albumId}/assets`, "DELETE", { ids: assetIds }, user);
await immichFetchBatched(`/albums/${config.albumId}/assets`, "DELETE", assetIds, {}, user);
return { action: "remove_from_album", assetsProcessed: assetIds.length, albumId: config.albumId };
}
case "favorite": {
await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: true }, user);
await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: true }, user);
return { action: "favorite", assetsProcessed: assetIds.length };
}
case "unfavorite": {
await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: false }, user);
await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: false }, user);
return { action: "unfavorite", assetsProcessed: assetIds.length };
}
case "archive": {
await immichFetch("/assets", "PUT", { ids: assetIds, visibility: "archive" }, user);
await immichFetchBatched("/assets", "PUT", assetIds, { visibility: "archive" }, user);
return { action: "archive", assetsProcessed: assetIds.length };
}
@@ -173,7 +186,7 @@ export async function executeAction(
// Create or get tag
const tag = await immichFetch("/tags", "POST", { name: config.tagName }, user);
// Tag assets
await immichFetch(`/tags/${tag.id}/assets`, "PUT", { ids: assetIds }, user);
await immichFetchBatched(`/tags/${tag.id}/assets`, "PUT", assetIds, {}, user);
return { action: "tag", assetsProcessed: assetIds.length };
} catch (e: any) {
return { action: "tag", assetsProcessed: 0, error: e.message };
+76 -42
View File
@@ -3,7 +3,7 @@ import { db } from "@/config/db";
import { workflows, workflowNodes, workflowEdges, workflowRuns, workflowProcessedAssets } from "@/db/schema/workflows.schema";
import { assets } from "@/schema/assets.schema";
import { exif } from "@/schema";
import { eq, and, desc, gte, isNull, inArray, sql, ne } from "drizzle-orm";
import { eq, and, desc, gt, gte, isNull, inArray, sql, ne, SQL } from "drizzle-orm";
import { buildConditions } from "./conditionBuilder";
import { executeAction } from "./actionExecutor";
import { IUser } from "@/types/user";
@@ -19,6 +19,53 @@ function logError(runId: string, ...args: any[]) {
console.error(`${PREFIX} [run:${runId.slice(0, 8)}]`, ...args);
}
const QUERY_BATCH_SIZE = 5000;
// Keyset-paginate an asset query so large libraries aren't silently truncated.
// fetchBatch must ORDER BY assets.id and apply gt(assets.id, afterId) when given.
async function fetchAllBatches<T extends { id: string }>(
fetchBatch: (afterId: string | null, limit: number) => Promise<T[]>,
): Promise<T[]> {
const all: T[] = [];
let cursor: string | null = null;
for (;;) {
const batch = await fetchBatch(cursor, QUERY_BATCH_SIZE);
all.push(...batch);
if (batch.length < QUERY_BATCH_SIZE) break;
cursor = batch[batch.length - 1].id;
}
return all;
}
// Evaluate condition clauses against an incoming asset set (chunked to keep
// IN-lists bounded), or against the whole library when scopeIds is null.
async function queryConditionMatches(whereClauses: SQL[], scopeIds: string[] | null): Promise<string[]> {
if (scopeIds !== null) {
const matched: string[] = [];
for (let i = 0; i < scopeIds.length; i += QUERY_BATCH_SIZE) {
const chunk = scopeIds.slice(i, i + QUERY_BATCH_SIZE);
const rows = await db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.leftJoin(exif, eq(assets.id, exif.assetId))
.where(and(...whereClauses, inArray(assets.id, chunk)));
matched.push(...rows.map((r) => r.id));
}
return matched;
}
const rows = await fetchAllBatches((afterId, limit) =>
db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.leftJoin(exif, eq(assets.id, exif.assetId))
.where(and(...whereClauses, ...(afterId ? [gt(assets.id, afterId)] : [])))
.orderBy(assets.id)
.limit(limit)
);
return rows.map((r) => r.id);
}
async function resolveAssetTrigger(
subType: string,
workflowId: string,
@@ -80,21 +127,27 @@ async function resolveAssetTrigger(
let candidateIds: string[];
if (subType === "new_asset") {
const rows = await db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.where(and(...baseConditions, gte(assets.createdAt, sinceDate)))
.limit(10000);
const rows = await fetchAllBatches((afterId, limit) =>
db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.where(and(...baseConditions, gte(assets.createdAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : [])))
.orderBy(assets.id)
.limit(limit)
);
const totalCandidates = rows.length;
// For new_asset: skip any asset we've ever processed
candidateIds = rows.map((r) => r.id).filter((id) => !processedMap?.has(id));
log(runId, `Trigger [new_asset]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} already processed, ${candidateIds.length} remaining`);
} else if (subType === "asset_updated") {
const rows = await db
.selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt })
.from(assets)
.where(and(...baseConditions, gte(assets.updatedAt, sinceDate)))
.limit(10000);
const rows = await fetchAllBatches((afterId, limit) =>
db
.selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt })
.from(assets)
.where(and(...baseConditions, gte(assets.updatedAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : [])))
.orderBy(assets.id)
.limit(limit)
);
const totalCandidates = rows.length;
// For asset_updated: only skip if we processed it AFTER its current updatedAt
candidateIds = rows
@@ -107,11 +160,14 @@ async function resolveAssetTrigger(
log(runId, `Trigger [asset_updated]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} skipped (not updated since processing), ${candidateIds.length} remaining`);
} else {
// all_assets — no dedup
const rows = await db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.where(and(...baseConditions))
.limit(10000);
const rows = await fetchAllBatches((afterId, limit) =>
db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.where(and(...baseConditions, ...(afterId ? [gt(assets.id, afterId)] : [])))
.orderBy(assets.id)
.limit(limit)
);
log(runId, `Trigger [all_assets]: ${rows.length} assets found`);
return rows.map((r) => r.id);
}
@@ -244,19 +300,8 @@ export async function executeWorkflow(
const whereClauses = buildConditions(conditions, user.id);
// Scope to incoming assets if available
if (assetIds !== null && assetIds.length > 0) {
whereClauses.push(inArray(assets.id, assetIds));
}
const matchedRows = await db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.leftJoin(exif, eq(assets.id, exif.assetId))
.where(and(...whereClauses))
.limit(10000);
const matchedIds = matchedRows.map((r) => r.id);
// Scoped to incoming assets (chunked) when available
const matchedIds = await queryConditionMatches(whereClauses, assetIds);
const matchedSet = new Set(matchedIds);
let trueIds = matchedIds;
@@ -298,19 +343,8 @@ export async function executeWorkflow(
const whereClauses = buildConditions(c.conditions || [], user.id);
// Scope to remaining assets if available
if (remaining !== null && remaining.length > 0) {
whereClauses.push(inArray(assets.id, remaining));
}
const matchedRows = await db
.selectDistinctOn([assets.id], { id: assets.id })
.from(assets)
.leftJoin(exif, eq(assets.id, exif.assetId))
.where(and(...whereClauses))
.limit(10000);
const matchedIds = matchedRows.map((r) => r.id);
// Scoped to remaining assets (chunked) when available
const matchedIds = await queryConditionMatches(whereClauses, remaining);
const matchedSet = new Set(matchedIds);
let caseIds: string[];