Merge pull request #297 from jonvdveen/feat/paginate-workflow-triggers
feat: paginate workflow asset queries — remove the silent 10,000-asset cap
This commit is contained in:
2 files changed
+96
-49
No files matched your search
@@ -28,6 +28,16 @@ async function getWorkflowApiKey(ownerId: string): Promise<string | null> {
|
||||
return row?.value || null;
|
||||
}
|
||||
|
||||
// Immich bulk endpoints are called in batches so large asset sets don't
|
||||
// produce oversized request bodies.
|
||||
const API_BATCH_SIZE = 1000;
|
||||
|
||||
async function immichFetchBatched(path: string, method: string, ids: string[], extraBody: Record<string, any>, user: IUser): Promise<void> {
|
||||
for (let i = 0; i < ids.length; i += API_BATCH_SIZE) {
|
||||
await immichFetch(path, method, { ...extraBody, ids: ids.slice(i, i + API_BATCH_SIZE) }, user);
|
||||
}
|
||||
}
|
||||
|
||||
async function immichFetch(path: string, method: string, body: any, user: IUser): Promise<any> {
|
||||
const workflowApiKey = await getWorkflowApiKey(user.id);
|
||||
|
||||
@@ -136,34 +146,37 @@ export async function executeAction(
|
||||
switch (subType) {
|
||||
case "create_album": {
|
||||
const albumName = await resolveTemplate(config.nameTemplate || "Auto Album", assetIds);
|
||||
const album = await immichFetch("/albums", "POST", { albumName, assetIds }, user);
|
||||
const album = await immichFetch("/albums", "POST", { albumName, assetIds: assetIds.slice(0, API_BATCH_SIZE) }, user);
|
||||
if (assetIds.length > API_BATCH_SIZE) {
|
||||
await immichFetchBatched(`/albums/${album.id}/assets`, "PUT", assetIds.slice(API_BATCH_SIZE), {}, user);
|
||||
}
|
||||
return { action: "create_album", assetsProcessed: assetIds.length, albumId: album.id, albumName };
|
||||
}
|
||||
|
||||
case "add_to_album": {
|
||||
if (!config.albumId) throw new Error("Album ID is required for add_to_album");
|
||||
await immichFetch(`/albums/${config.albumId}/assets`, "PUT", { ids: assetIds }, user);
|
||||
await immichFetchBatched(`/albums/${config.albumId}/assets`, "PUT", assetIds, {}, user);
|
||||
return { action: "add_to_album", assetsProcessed: assetIds.length, albumId: config.albumId };
|
||||
}
|
||||
|
||||
case "remove_from_album": {
|
||||
if (!config.albumId) throw new Error("Album ID is required for remove_from_album");
|
||||
await immichFetch(`/albums/${config.albumId}/assets`, "DELETE", { ids: assetIds }, user);
|
||||
await immichFetchBatched(`/albums/${config.albumId}/assets`, "DELETE", assetIds, {}, user);
|
||||
return { action: "remove_from_album", assetsProcessed: assetIds.length, albumId: config.albumId };
|
||||
}
|
||||
|
||||
case "favorite": {
|
||||
await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: true }, user);
|
||||
await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: true }, user);
|
||||
return { action: "favorite", assetsProcessed: assetIds.length };
|
||||
}
|
||||
|
||||
case "unfavorite": {
|
||||
await immichFetch("/assets", "PUT", { ids: assetIds, isFavorite: false }, user);
|
||||
await immichFetchBatched("/assets", "PUT", assetIds, { isFavorite: false }, user);
|
||||
return { action: "unfavorite", assetsProcessed: assetIds.length };
|
||||
}
|
||||
|
||||
case "archive": {
|
||||
await immichFetch("/assets", "PUT", { ids: assetIds, visibility: "archive" }, user);
|
||||
await immichFetchBatched("/assets", "PUT", assetIds, { visibility: "archive" }, user);
|
||||
return { action: "archive", assetsProcessed: assetIds.length };
|
||||
}
|
||||
|
||||
@@ -173,7 +186,7 @@ export async function executeAction(
|
||||
// Create or get tag
|
||||
const tag = await immichFetch("/tags", "POST", { name: config.tagName }, user);
|
||||
// Tag assets
|
||||
await immichFetch(`/tags/${tag.id}/assets`, "PUT", { ids: assetIds }, user);
|
||||
await immichFetchBatched(`/tags/${tag.id}/assets`, "PUT", assetIds, {}, user);
|
||||
return { action: "tag", assetsProcessed: assetIds.length };
|
||||
} catch (e: any) {
|
||||
return { action: "tag", assetsProcessed: 0, error: e.message };
|
||||
|
||||
+76
-42
@@ -3,7 +3,7 @@ import { db } from "@/config/db";
|
||||
import { workflows, workflowNodes, workflowEdges, workflowRuns, workflowProcessedAssets } from "@/db/schema/workflows.schema";
|
||||
import { assets } from "@/schema/assets.schema";
|
||||
import { exif } from "@/schema";
|
||||
import { eq, and, desc, gte, isNull, inArray, sql, ne } from "drizzle-orm";
|
||||
import { eq, and, desc, gt, gte, isNull, inArray, sql, ne, SQL } from "drizzle-orm";
|
||||
import { buildConditions } from "./conditionBuilder";
|
||||
import { executeAction } from "./actionExecutor";
|
||||
import { IUser } from "@/types/user";
|
||||
@@ -19,6 +19,53 @@ function logError(runId: string, ...args: any[]) {
|
||||
console.error(`${PREFIX} [run:${runId.slice(0, 8)}]`, ...args);
|
||||
}
|
||||
|
||||
const QUERY_BATCH_SIZE = 5000;
|
||||
|
||||
// Keyset-paginate an asset query so large libraries aren't silently truncated.
|
||||
// fetchBatch must ORDER BY assets.id and apply gt(assets.id, afterId) when given.
|
||||
async function fetchAllBatches<T extends { id: string }>(
|
||||
fetchBatch: (afterId: string | null, limit: number) => Promise<T[]>,
|
||||
): Promise<T[]> {
|
||||
const all: T[] = [];
|
||||
let cursor: string | null = null;
|
||||
for (;;) {
|
||||
const batch = await fetchBatch(cursor, QUERY_BATCH_SIZE);
|
||||
all.push(...batch);
|
||||
if (batch.length < QUERY_BATCH_SIZE) break;
|
||||
cursor = batch[batch.length - 1].id;
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
// Evaluate condition clauses against an incoming asset set (chunked to keep
|
||||
// IN-lists bounded), or against the whole library when scopeIds is null.
|
||||
async function queryConditionMatches(whereClauses: SQL[], scopeIds: string[] | null): Promise<string[]> {
|
||||
if (scopeIds !== null) {
|
||||
const matched: string[] = [];
|
||||
for (let i = 0; i < scopeIds.length; i += QUERY_BATCH_SIZE) {
|
||||
const chunk = scopeIds.slice(i, i + QUERY_BATCH_SIZE);
|
||||
const rows = await db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.leftJoin(exif, eq(assets.id, exif.assetId))
|
||||
.where(and(...whereClauses, inArray(assets.id, chunk)));
|
||||
matched.push(...rows.map((r) => r.id));
|
||||
}
|
||||
return matched;
|
||||
}
|
||||
|
||||
const rows = await fetchAllBatches((afterId, limit) =>
|
||||
db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.leftJoin(exif, eq(assets.id, exif.assetId))
|
||||
.where(and(...whereClauses, ...(afterId ? [gt(assets.id, afterId)] : [])))
|
||||
.orderBy(assets.id)
|
||||
.limit(limit)
|
||||
);
|
||||
return rows.map((r) => r.id);
|
||||
}
|
||||
|
||||
async function resolveAssetTrigger(
|
||||
subType: string,
|
||||
workflowId: string,
|
||||
@@ -80,21 +127,27 @@ async function resolveAssetTrigger(
|
||||
let candidateIds: string[];
|
||||
|
||||
if (subType === "new_asset") {
|
||||
const rows = await db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.where(and(...baseConditions, gte(assets.createdAt, sinceDate)))
|
||||
.limit(10000);
|
||||
const rows = await fetchAllBatches((afterId, limit) =>
|
||||
db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.where(and(...baseConditions, gte(assets.createdAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : [])))
|
||||
.orderBy(assets.id)
|
||||
.limit(limit)
|
||||
);
|
||||
const totalCandidates = rows.length;
|
||||
// For new_asset: skip any asset we've ever processed
|
||||
candidateIds = rows.map((r) => r.id).filter((id) => !processedMap?.has(id));
|
||||
log(runId, `Trigger [new_asset]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} already processed, ${candidateIds.length} remaining`);
|
||||
} else if (subType === "asset_updated") {
|
||||
const rows = await db
|
||||
.selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt })
|
||||
.from(assets)
|
||||
.where(and(...baseConditions, gte(assets.updatedAt, sinceDate)))
|
||||
.limit(10000);
|
||||
const rows = await fetchAllBatches((afterId, limit) =>
|
||||
db
|
||||
.selectDistinctOn([assets.id], { id: assets.id, updatedAt: assets.updatedAt })
|
||||
.from(assets)
|
||||
.where(and(...baseConditions, gte(assets.updatedAt, sinceDate), ...(afterId ? [gt(assets.id, afterId)] : [])))
|
||||
.orderBy(assets.id)
|
||||
.limit(limit)
|
||||
);
|
||||
const totalCandidates = rows.length;
|
||||
// For asset_updated: only skip if we processed it AFTER its current updatedAt
|
||||
candidateIds = rows
|
||||
@@ -107,11 +160,14 @@ async function resolveAssetTrigger(
|
||||
log(runId, `Trigger [asset_updated]: ${totalCandidates} candidates, ${totalCandidates - candidateIds.length} skipped (not updated since processing), ${candidateIds.length} remaining`);
|
||||
} else {
|
||||
// all_assets — no dedup
|
||||
const rows = await db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.where(and(...baseConditions))
|
||||
.limit(10000);
|
||||
const rows = await fetchAllBatches((afterId, limit) =>
|
||||
db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.where(and(...baseConditions, ...(afterId ? [gt(assets.id, afterId)] : [])))
|
||||
.orderBy(assets.id)
|
||||
.limit(limit)
|
||||
);
|
||||
log(runId, `Trigger [all_assets]: ${rows.length} assets found`);
|
||||
return rows.map((r) => r.id);
|
||||
}
|
||||
@@ -244,19 +300,8 @@ export async function executeWorkflow(
|
||||
|
||||
const whereClauses = buildConditions(conditions, user.id);
|
||||
|
||||
// Scope to incoming assets if available
|
||||
if (assetIds !== null && assetIds.length > 0) {
|
||||
whereClauses.push(inArray(assets.id, assetIds));
|
||||
}
|
||||
|
||||
const matchedRows = await db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.leftJoin(exif, eq(assets.id, exif.assetId))
|
||||
.where(and(...whereClauses))
|
||||
.limit(10000);
|
||||
|
||||
const matchedIds = matchedRows.map((r) => r.id);
|
||||
// Scoped to incoming assets (chunked) when available
|
||||
const matchedIds = await queryConditionMatches(whereClauses, assetIds);
|
||||
const matchedSet = new Set(matchedIds);
|
||||
|
||||
let trueIds = matchedIds;
|
||||
@@ -298,19 +343,8 @@ export async function executeWorkflow(
|
||||
|
||||
const whereClauses = buildConditions(c.conditions || [], user.id);
|
||||
|
||||
// Scope to remaining assets if available
|
||||
if (remaining !== null && remaining.length > 0) {
|
||||
whereClauses.push(inArray(assets.id, remaining));
|
||||
}
|
||||
|
||||
const matchedRows = await db
|
||||
.selectDistinctOn([assets.id], { id: assets.id })
|
||||
.from(assets)
|
||||
.leftJoin(exif, eq(assets.id, exif.assetId))
|
||||
.where(and(...whereClauses))
|
||||
.limit(10000);
|
||||
|
||||
const matchedIds = matchedRows.map((r) => r.id);
|
||||
// Scoped to remaining assets (chunked) when available
|
||||
const matchedIds = await queryConditionMatches(whereClauses, remaining);
|
||||
const matchedSet = new Set(matchedIds);
|
||||
let caseIds: string[];
|
||||
|
||||
|
||||
Reference in new issue
Block a user