feat: add compression

This commit is contained in:
jfmcarreira committed 2026-09-24 01:16:30 +01:00
1 parent 505c6c7db7
commit ea20e8ddba
44 files changed
+4033 -30

No files matched your search

+1 -1
View File
@@ -25,7 +25,7 @@ ENV VERSION=$VERSION
ENV NODE_ENV=production
RUN apk add --no-cache curl
RUN apk add --no-cache curl ffmpeg
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nextjs -u 1001
+13 -6
View File
@@ -1,20 +1,27 @@
version: '3.8'
name: immich-power-tools
services:
immich_power_tools:
immich-power-tools:
container_name: immich_power_tools
image: ghcr.io/immich-power-tools/immich-power-tools:latest
build:
context: .
dockerfile: Dockerfile
ports:
- "3000:3000"
volumes:
- immich-power-tools-data:/app/data
- /data/docker/immich-power-tools:/app/data
env_file:
- .env
networks:
- default
- immich
healthcheck:
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
volumes:
immich-power-tools-data:
networks:
immich:
external: true
name: immich_default
+18 -9
View File
@@ -34,6 +34,11 @@ export interface AssetPhoto extends Photo {
isVideo: boolean;
duration?: string;
isSelected: boolean;
originalFileName?: string;
fileSizeInByte?: number;
exifImageWidth?: number;
exifImageHeight?: number;
compressionUnsupportedReason?: string;
}
interface AssetGridProps {
@@ -43,6 +48,8 @@ interface AssetGridProps {
onSelectionChange?: (ids: string[]) => void;
onDeleteAsset?: (id: string) => void;
onFavoriteAsset?: (id: string, isFavorite: boolean) => void;
showMetadataOverlay?: boolean;
allowAssetMutations?: boolean;
}
interface AssetGridRef {
@@ -51,7 +58,7 @@ interface AssetGridRef {
unselectAll: () => void;
}
const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal = true, selectable = false, onSelectionChange, onDeleteAsset, onFavoriteAsset }, ref) => {
const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal = true, selectable = false, onSelectionChange, onDeleteAsset, onFavoriteAsset, showMetadataOverlay = false, allowAssetMutations = true }, ref) => {
const [index, setIndex] = useState(-1);
const [lastSelectedIndex, setLastSelectedIndex] = useState(-1);
const [showInfoPanel, setShowInfoPanel] = useState(() => {
@@ -109,7 +116,7 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
useImperativeHandle(ref, () => ({
getSelectedIds: () => selectedIds,
selectAll: () => {
const allIds = assets.map((asset) => asset.id);
const allIds = assets.filter((asset) => !asset.compressionUnsupportedReason).map((asset) => asset.id);
updateContext({ selectedIds: allIds });
onSelectionChange?.(allIds);
},
@@ -123,6 +130,7 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
const handleSelect = (_idx: number, asset: AssetPhoto, event: React.MouseEvent) => {
event.stopPropagation();
if (asset.compressionUnsupportedReason) return;
const isPresent = selectedIds.includes(asset.id);
if (isPresent) {
const newSelectedIds = selectedIds.filter((id) => id !== asset.id);
@@ -132,7 +140,7 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
const clickedIndex = images.findIndex((image) => {
return image.id === asset.id;
});
if (event.shiftKey) {
if (event.shiftKey && lastSelectedIndex >= 0) {
const startIndex = Math.min(clickedIndex, lastSelectedIndex);
const endIndex = Math.max(clickedIndex, lastSelectedIndex);
const rangeSelectedIds = images.slice(startIndex, endIndex + 1).map((image) => image.id);
@@ -150,7 +158,7 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
};
const handleClick = (index: number, asset: AssetPhoto, event: React.MouseEvent) => {
if (selectable && (event.metaKey || event.ctrlKey || selectedIds.length > 0)) {
if (selectable && !asset.compressionUnsupportedReason && (event.metaKey || event.ctrlKey || selectedIds.length > 0)) {
handleSelect(index, asset, event);
} else {
setIndex(index);
@@ -214,9 +222,10 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
photo={context.photo}
width={context.width}
height={context.height}
selectable={selectable}
selectable={selectable && !context.photo.compressionUnsupportedReason}
onSelect={(event) => handleSelect(context.index, context.photo, event)}
selectionMode={selectable && selectedIds.length > 0}
showMetadataOverlay={showMetadataOverlay}
/>
);
};
@@ -231,7 +240,7 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
if (!currentAsset) return [];
const isFav = favoriteIds.has(currentAsset.id);
return [
<button
allowAssetMutations && <button
key="favorite"
type="button"
className="yarl__button"
@@ -242,7 +251,7 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
className={`h-6 w-6 ${isFav ? 'fill-red-500 text-red-500' : 'text-white'}`}
/>
</button>,
<button
allowAssetMutations && <button
key="info"
type="button"
className="yarl__button"
@@ -269,8 +278,8 @@ const AssetGrid = forwardRef<AssetGridRef, AssetGridProps>(({ assets, isInternal
>
<Trash2 className="h-6 w-6 text-white" />
</button>,
];
}, [currentAsset, showInfoPanel, onDeleteAsset, onFavoriteAsset, handleOpenInImmich, favoriteIds, handleToggleFavorite]);
].filter(Boolean) as React.ReactNode[];
}, [currentAsset, showInfoPanel, handleOpenInImmich, favoriteIds, handleToggleFavorite, allowAssetMutations]);
return (
<div>
+20 -4
View File
@@ -1,5 +1,5 @@
import { humanizeDuration } from '@/helpers/string.helper'
import { humanizeBytes, humanizeDuration } from '@/helpers/string.helper'
import { PlayIcon } from '@radix-ui/react-icons'
import { CircleCheck } from 'lucide-react'
import React, { useEffect } from 'react'
@@ -15,9 +15,10 @@ interface LazyGridImageProps {
onSelect?: (event: React.MouseEvent) => void;
/** When true, dim non-selected siblings so the selected set is visible in dense grids. */
selectionMode?: boolean;
showMetadataOverlay?: boolean;
}
export default function LazyGridImage({ imageProps, photo, width, height, selectable, onSelect, selectionMode = false }: LazyGridImageProps) {
export default function LazyGridImage({ imageProps, photo, width, height, selectable, onSelect, selectionMode = false, showMetadataOverlay = false }: LazyGridImageProps) {
const [isVisible, setIsVisible] = React.useState(false)
const imageRef = React.useRef<HTMLDivElement>(null)
@@ -61,13 +62,28 @@ export default function LazyGridImage({ imageProps, photo, width, height, select
className={`group ${photo.isSelected ? 'ring-4 ring-blue-500 ring-inset' : ''} ${dim}`.trim()}
>
<img {...imageProps} alt={imageProps.alt || ""} title="" style={{ ...imageProps.style, width, height, objectFit: 'cover' }} />
{photo.isVideo && <div className="absolute bottom-2 right-2 bg-black/50 p-1 rounded-full flex items-center gap-1">
{photo.isVideo && <div className={`absolute right-2 ${showMetadataOverlay ? 'bottom-12' : 'bottom-2'} bg-black/50 p-1 rounded-full flex items-center gap-1`}>
<PlayIcon className="w-3 h-3 text-white" />
{!!photo.duration && <span className="text-xs text-white">{humanizeDuration(photo.duration)}</span>}
</div>}
{showMetadataOverlay && photo.fileSizeInByte != null && (
<div className="absolute inset-x-0 bottom-0 bg-black/60 px-2 py-1 text-white pointer-events-none">
<div className="truncate text-[11px] font-medium">{photo.originalFileName}</div>
<div className="text-[10px] text-white/80">
{humanizeBytes(photo.fileSizeInByte)}
{photo.exifImageWidth && photo.exifImageHeight ? ` · ${photo.exifImageWidth}×${photo.exifImageHeight}` : ''}
</div>
</div>
)}
{photo.compressionUnsupportedReason && (
<div className="absolute top-2 right-2 max-w-[75%] rounded bg-destructive/90 px-2 py-1 text-[10px] text-white pointer-events-none">
{photo.compressionUnsupportedReason}
</div>
)}
{selectable && (
<div
className={`absolute top-2 left-2 z-10 cursor-pointer transition-opacity ${photo.isSelected ? 'opacity-100' : 'opacity-0 group-hover:opacity-100'}`}
className={`absolute top-2 left-2 z-10 cursor-pointer transition-opacity ${photo.isSelected ? 'opacity-100' : 'opacity-70 md:opacity-0 md:group-hover:opacity-100'}`}
aria-label={photo.isSelected ? "Unselect asset" : "Select asset"}
onClick={(e) => {
e.stopPropagation();
onSelect?.(e);
+2 -1
View File
@@ -1,4 +1,4 @@
import { Copy, GalleryHorizontal, GalleryVerticalEnd, Image as ImageIcon, MapPin, MapPinX, PackageSearch, Rewind, Search, Settings, Share2, User, Video, Workflow } from "lucide-react";
import { Copy, GalleryHorizontal, GalleryVerticalEnd, Image as ImageIcon, MapPin, MapPinX, PackageSearch, Rewind, Search, Settings, Share2, User, Video, Workflow, FileArchive } from "lucide-react";
interface SidebarNav {
title: string;
@@ -30,6 +30,7 @@ export const sidebarGroups: SidebarGroup[] = [
{ title: "Empty Videos", link: "/assets/empty-videos", icon: <Video className="h-4 w-4" /> },
{ title: "Bulk Duplicate Finder", link: "/assets/bulk-duplicate-finder", icon: <Copy className="h-4 w-4" /> },
{ title: "Orphan Finder", link: "/assets/orphan-finder", icon: <PackageSearch className="h-4 w-4" /> },
{ title: "Compression", link: "/assets/compression", icon: <FileArchive className="h-4 w-4" /> },
],
},
{
@@ -9,4 +9,5 @@ CREATE TABLE `__new_settings` (
INSERT INTO `__new_settings`("id", "key", "value", "owner_id", "updated_at") SELECT "key", "key", "value", NULL, "updated_at" FROM `settings`;--> statement-breakpoint
DROP TABLE `settings`;--> statement-breakpoint
ALTER TABLE `__new_settings` RENAME TO `settings`;--> statement-breakpoint
CREATE UNIQUE INDEX `settings_owner_key_unique` ON `settings` (`owner_id`,`key`);--> statement-breakpoint
PRAGMA foreign_keys=ON;
@@ -0,0 +1,36 @@
CREATE TABLE `compression_jobs` (
`id` text PRIMARY KEY NOT NULL,
`user_id` text NOT NULL,
`asset_id` text NOT NULL,
`original_filename` text NOT NULL,
`original_size` integer DEFAULT 0 NOT NULL,
`original_checksum` text,
`media_type` text NOT NULL,
`input_path` text NOT NULL,
`output_path` text NOT NULL,
`output_size` integer,
`output_checksum` text,
`status` text DEFAULT 'QUEUED' NOT NULL,
`attempt` integer DEFAULT 0 NOT NULL,
`progress` integer DEFAULT 0 NOT NULL,
`speed` text,
`ffmpeg_arguments` text NOT NULL,
`output_extension` text NOT NULL,
`ffmpeg_exit_code` integer,
`error_message` text,
`log_path` text NOT NULL,
`source_metadata` text DEFAULT '{}' NOT NULL,
`replacement_asset_id` text,
`replacement_stage` text,
`created_at` integer,
`started_at` integer,
`finished_at` integer,
`replaced_at` integer,
`heartbeat_at` integer
);
--> statement-breakpoint
CREATE INDEX `compression_jobs_user_status_created_idx` ON `compression_jobs` (`user_id`,`status`,`created_at`);
--> statement-breakpoint
CREATE INDEX `compression_jobs_asset_idx` ON `compression_jobs` (`asset_id`);
--> statement-breakpoint
CREATE UNIQUE INDEX `compression_jobs_one_active_asset_idx` ON `compression_jobs` (`user_id`,`asset_id`) WHERE "compression_jobs"."status" in ('QUEUED', 'DOWNLOADING', 'COMPRESSING', 'VALIDATING', 'READY', 'REPLACING', 'CANCELLING', 'RETRYING', 'NEEDS_ATTENTION');
+11 -2
View File
@@ -43,7 +43,16 @@
"autoincrement": false
}
},
"indexes": {},
"indexes": {
"settings_owner_key_unique": {
"name": "settings_owner_key_unique",
"columns": [
"owner_id",
"key"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
@@ -744,4 +753,4 @@
"internal": {
"indexes": {}
}
}
}
+993
View File
@@ -0,0 +1,993 @@
{
"version": "6",
"dialect": "sqlite",
"id": "67735475-ebe4-4a4d-8924-59c231c9e671",
"prevId": "c9948313-c4b4-4db4-8975-8c409c20379e",
"tables": {
"settings": {
"name": "settings",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"key": {
"name": "key",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"value": {
"name": "value",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"owner_id": {
"name": "owner_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"settings_owner_key_unique": {
"name": "settings_owner_key_unique",
"columns": [
"owner_id",
"key"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"api_keys": {
"name": "api_keys",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"purpose": {
"name": "purpose",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"key_name": {
"name": "key_name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"secret": {
"name": "secret",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"immich_key_id": {
"name": "immich_key_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"api_keys_user_id_purpose_unique": {
"name": "api_keys_user_id_purpose_unique",
"columns": [
"user_id",
"purpose"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"import_job_items": {
"name": "import_job_items",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"job_id": {
"name": "job_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"asset_id": {
"name": "asset_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'pending'"
},
"item_data": {
"name": "item_data",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{}'"
},
"immich_id": {
"name": "immich_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"import_job_items_job_id_import_jobs_id_fk": {
"name": "import_job_items_job_id_import_jobs_id_fk",
"tableFrom": "import_job_items",
"tableTo": "import_jobs",
"columnsFrom": [
"job_id"
],
"columnsTo": [
"id"
],
"onDelete": "no action",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"import_jobs": {
"name": "import_jobs",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"platform": {
"name": "platform",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'pending'"
},
"url": {
"name": "url",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"url_config": {
"name": "url_config",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{}'"
},
"import_data": {
"name": "import_data",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{}'"
},
"total_count": {
"name": "total_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"uploaded_count": {
"name": "uploaded_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"skipped_count": {
"name": "skipped_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"failed_count": {
"name": "failed_count",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"workflow_edges": {
"name": "workflow_edges",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"workflow_id": {
"name": "workflow_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_node_id": {
"name": "source_node_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"target_node_id": {
"name": "target_node_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_handle": {
"name": "source_handle",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"workflow_edges_workflow_id_workflows_id_fk": {
"name": "workflow_edges_workflow_id_workflows_id_fk",
"tableFrom": "workflow_edges",
"tableTo": "workflows",
"columnsFrom": [
"workflow_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"workflow_edges_source_node_id_workflow_nodes_id_fk": {
"name": "workflow_edges_source_node_id_workflow_nodes_id_fk",
"tableFrom": "workflow_edges",
"tableTo": "workflow_nodes",
"columnsFrom": [
"source_node_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"workflow_edges_target_node_id_workflow_nodes_id_fk": {
"name": "workflow_edges_target_node_id_workflow_nodes_id_fk",
"tableFrom": "workflow_edges",
"tableTo": "workflow_nodes",
"columnsFrom": [
"target_node_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"workflow_nodes": {
"name": "workflow_nodes",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"workflow_id": {
"name": "workflow_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"type": {
"name": "type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"sub_type": {
"name": "sub_type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"data": {
"name": "data",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{}'"
},
"position_x": {
"name": "position_x",
"type": "real",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"position_y": {
"name": "position_y",
"type": "real",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"workflow_nodes_workflow_id_workflows_id_fk": {
"name": "workflow_nodes_workflow_id_workflows_id_fk",
"tableFrom": "workflow_nodes",
"tableTo": "workflows",
"columnsFrom": [
"workflow_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"workflow_processed_assets": {
"name": "workflow_processed_assets",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"workflow_id": {
"name": "workflow_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"asset_id": {
"name": "asset_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"run_id": {
"name": "run_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"processed_at": {
"name": "processed_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"workflow_processed_assets_workflow_id_workflows_id_fk": {
"name": "workflow_processed_assets_workflow_id_workflows_id_fk",
"tableFrom": "workflow_processed_assets",
"tableTo": "workflows",
"columnsFrom": [
"workflow_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
},
"workflow_processed_assets_run_id_workflow_runs_id_fk": {
"name": "workflow_processed_assets_run_id_workflow_runs_id_fk",
"tableFrom": "workflow_processed_assets",
"tableTo": "workflow_runs",
"columnsFrom": [
"run_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"workflow_runs": {
"name": "workflow_runs",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"workflow_id": {
"name": "workflow_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"trigger": {
"name": "trigger",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'running'"
},
"result": {
"name": "result",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false,
"default": "'{}'"
},
"error": {
"name": "error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"started_at": {
"name": "started_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"completed_at": {
"name": "completed_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {
"workflow_runs_workflow_id_workflows_id_fk": {
"name": "workflow_runs_workflow_id_workflows_id_fk",
"tableFrom": "workflow_runs",
"tableTo": "workflows",
"columnsFrom": [
"workflow_id"
],
"columnsTo": [
"id"
],
"onDelete": "cascade",
"onUpdate": "no action"
}
},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"workflows": {
"name": "workflows",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"owner_id": {
"name": "owner_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"enabled": {
"name": "enabled",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": false
},
"cron_schedule": {
"name": "cron_schedule",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"webhook_token": {
"name": "webhook_token",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"viewport": {
"name": "viewport",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{\"x\":0,\"y\":0,\"zoom\":1}'"
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"workflows_webhook_token_unique": {
"name": "workflows_webhook_token_unique",
"columns": [
"webhook_token"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"compression_jobs": {
"name": "compression_jobs",
"columns": {
"id": {
"name": "id",
"type": "text",
"primaryKey": true,
"notNull": true,
"autoincrement": false
},
"user_id": {
"name": "user_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"asset_id": {
"name": "asset_id",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"original_filename": {
"name": "original_filename",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"original_size": {
"name": "original_size",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"original_checksum": {
"name": "original_checksum",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"media_type": {
"name": "media_type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"input_path": {
"name": "input_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"output_path": {
"name": "output_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"output_size": {
"name": "output_size",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"output_checksum": {
"name": "output_checksum",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'QUEUED'"
},
"attempt": {
"name": "attempt",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"progress": {
"name": "progress",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 0
},
"speed": {
"name": "speed",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"ffmpeg_arguments": {
"name": "ffmpeg_arguments",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"output_extension": {
"name": "output_extension",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"ffmpeg_exit_code": {
"name": "ffmpeg_exit_code",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"error_message": {
"name": "error_message",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"log_path": {
"name": "log_path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"source_metadata": {
"name": "source_metadata",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'{}'"
},
"replacement_asset_id": {
"name": "replacement_asset_id",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"replacement_stage": {
"name": "replacement_stage",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"started_at": {
"name": "started_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"finished_at": {
"name": "finished_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"replaced_at": {
"name": "replaced_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"heartbeat_at": {
"name": "heartbeat_at",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
}
},
"indexes": {
"compression_jobs_user_status_created_idx": {
"name": "compression_jobs_user_status_created_idx",
"columns": [
"user_id",
"status",
"created_at"
],
"isUnique": false
},
"compression_jobs_asset_idx": {
"name": "compression_jobs_asset_idx",
"columns": [
"asset_id"
],
"isUnique": false
},
"compression_jobs_one_active_asset_idx": {
"name": "compression_jobs_one_active_asset_idx",
"columns": [
"user_id",
"asset_id"
],
"isUnique": true,
"where": "\"compression_jobs\".\"status\" in ('QUEUED', 'DOWNLOADING', 'COMPRESSING', 'VALIDATING', 'READY', 'REPLACING', 'CANCELLING', 'RETRYING', 'NEEDS_ATTENTION')"
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}
+8 -1
View File
@@ -36,6 +36,13 @@
"when": 1775574030092,
"tag": "0004_equal_iron_monger",
"breakpoints": true
},
{
"idx": 5,
"version": "6",
"when": 1790200218320,
"tag": "0005_selective_compression",
"breakpoints": true
}
]
}
}
+39
View File
@@ -0,0 +1,39 @@
import { sql } from "drizzle-orm";
import { sqliteTable, text, integer, index, uniqueIndex } from "drizzle-orm/sqlite-core";
export const compressionJobs = sqliteTable("compression_jobs", {
id: text("id").primaryKey(),
userId: text("user_id").notNull(),
assetId: text("asset_id").notNull(),
originalFilename: text("original_filename").notNull(),
originalSize: integer("original_size").notNull().default(0),
originalChecksum: text("original_checksum"),
mediaType: text("media_type").notNull(),
inputPath: text("input_path").notNull(),
outputPath: text("output_path").notNull(),
outputSize: integer("output_size"),
outputChecksum: text("output_checksum"),
status: text("status").notNull().default("QUEUED"),
attempt: integer("attempt").notNull().default(0),
progress: integer("progress").notNull().default(0),
speed: text("speed"),
ffmpegArguments: text("ffmpeg_arguments").notNull(),
outputExtension: text("output_extension").notNull(),
ffmpegExitCode: integer("ffmpeg_exit_code"),
errorMessage: text("error_message"),
logPath: text("log_path").notNull(),
sourceMetadata: text("source_metadata").notNull().default("{}"),
replacementAssetId: text("replacement_asset_id"),
replacementStage: text("replacement_stage"),
createdAt: integer("created_at", { mode: "timestamp" }).$defaultFn(() => new Date()),
startedAt: integer("started_at", { mode: "timestamp" }),
finishedAt: integer("finished_at", { mode: "timestamp" }),
replacedAt: integer("replaced_at", { mode: "timestamp" }),
heartbeatAt: integer("heartbeat_at", { mode: "timestamp" }),
}, (table) => [
index("compression_jobs_user_status_created_idx").on(table.userId, table.status, table.createdAt),
index("compression_jobs_asset_idx").on(table.assetId),
uniqueIndex("compression_jobs_one_active_asset_idx")
.on(table.userId, table.assetId)
.where(sql`${table.status} in ('QUEUED', 'DOWNLOADING', 'COMPRESSING', 'VALIDATING', 'READY', 'REPLACING', 'CANCELLING', 'RETRYING', 'NEEDS_ATTENTION')`),
]);
+2
View File
@@ -2,3 +2,5 @@ export * from "./settings.schema";
export * from "./apiKeys.schema";
export * from "./importJobs.schema";
export * from "./workflows.schema";
export * from "./compressionJobs.schema";
+4 -2
View File
@@ -1,4 +1,4 @@
import { sqliteTable, text, integer } from "drizzle-orm/sqlite-core";
import { sqliteTable, text, integer, uniqueIndex } from "drizzle-orm/sqlite-core";
import { randomUUID } from "crypto";
export const settings = sqliteTable("settings", {
@@ -9,4 +9,6 @@ export const settings = sqliteTable("settings", {
updatedAt: integer("updated_at", { mode: "timestamp" }).$onUpdate(
() => new Date()
),
});
}, (table) => [
uniqueIndex("settings_owner_key_unique").on(table.ownerId, table.key),
]);
+3
View File
@@ -7,6 +7,9 @@ export async function register() {
const { resumePendingJobs } = await import("./workers/import/runner");
await resumePendingJobs();
const { resumeCompressionJobs } = await import("./workers/compression/runner");
await resumeCompressionJobs();
const { loadAllScheduledWorkflows } = await import("./lib/workflow/scheduler");
await loadAllScheduledWorkflows();
}
+88
View File
@@ -0,0 +1,88 @@
export interface CompressionSettings {
ffmpegArguments: string;
outputExtension: string;
}
export function tokenizeArguments(input: string): string[] {
const args: string[] = [];
let current = "";
let quote: "'" | '"' | null = null;
let escaping = false;
let tokenStarted = false;
for (const char of input) {
if (escaping) {
current += char;
escaping = false;
tokenStarted = true;
continue;
}
if (char === "\\" && quote !== "'") {
escaping = true;
tokenStarted = true;
continue;
}
if (quote) {
if (char === quote) {
quote = null;
} else {
current += char;
}
tokenStarted = true;
continue;
}
if (char === "'" || char === '"') {
quote = char;
tokenStarted = true;
continue;
}
if (/\s/.test(char)) {
if (tokenStarted) {
args.push(current);
current = "";
tokenStarted = false;
}
continue;
}
current += char;
tokenStarted = true;
}
if (escaping) throw new Error("FFmpeg arguments end with an incomplete escape sequence");
if (quote) throw new Error("FFmpeg arguments contain an unterminated quoted string");
if (tokenStarted) args.push(current);
return args;
}
export function validateCompressionSettings(settings: CompressionSettings): string | null {
const argumentsValue = settings.ffmpegArguments.trim();
const extension = settings.outputExtension.trim();
if (!argumentsValue) return "FFmpeg arguments cannot be empty";
if (!argumentsValue.includes("{input}")) return "FFmpeg arguments must contain {input}";
if (!argumentsValue.includes("{output}")) return "FFmpeg arguments must contain {output}";
if (!extension.startsWith(".") || extension.length < 2) return "Output extension must start with a dot";
if (!/^\.[A-Za-z0-9]+$/.test(extension)) return "Output extension may contain only letters and numbers after the dot";
try {
const parsed = tokenizeArguments(argumentsValue);
if (parsed.length === 0) return "FFmpeg arguments cannot be empty";
} catch (error: any) {
return error?.message ?? "Invalid FFmpeg arguments";
}
return null;
}
export function buildFfmpegArguments(template: string, inputPath: string, outputPath: string): string[] {
const tokens = tokenizeArguments(template);
return tokens.map((token) => token
.split("{input}").join(inputPath)
.split("{output}").join(outputPath));
}
+19
View File
@@ -0,0 +1,19 @@
import type { NextApiRequest, NextApiResponse } from "next";
import type { IUser } from "@/types/user";
import { getCurrentUser } from "@/handlers/serverUtils/user.utils";
export async function requireCompressionUser(
req: NextApiRequest,
res: NextApiResponse,
): Promise<IUser | null> {
const user = await getCurrentUser(req);
if (!user) {
res.status(401).json({ error: "Unauthorized" });
return null;
}
if (user.isUsingAPIKey || !user.accessToken) {
res.status(403).json({ error: "Compression requires an interactive Immich session" });
return null;
}
return user;
}
+21
View File
@@ -0,0 +1,21 @@
import path from "path";
export const FFMPEG_ARGUMENTS_SETTING = "compression_ffmpeg_arguments";
export const FFMPEG_OUTPUT_EXTENSION_SETTING = "compression_ffmpeg_output_extension";
// Do not silently choose an encoding configuration for the user. This is shown only as an example in the UI.
export const FFMPEG_ARGUMENTS_EXAMPLE = "-i {input} -map 0 -c:v libx265 -crf 24 -preset medium -c:a copy {output}";
export const DEFAULT_OUTPUT_EXTENSION = ".mp4";
const defaultDataDir = path.join(process.cwd(), "data");
const appDbPath = process.env.APP_DB_PATH || path.join(defaultDataDir, "app.db");
export const COMPRESSION_ROOT = process.env.COMPRESSION_DATA_PATH || path.join(path.dirname(appDbPath), "compression");
export const MAX_LOG_BYTES = 1024 * 1024;
export const CANCEL_GRACE_MS = 5000;
export const FFPROBE_TIMEOUT_MS = 5 * 60 * 1000;
export const MEDIA_DECODE_TIMEOUT_MS = 6 * 60 * 60 * 1000;
export const COMPRESSION_TIMEOUT_MS = 24 * 60 * 60 * 1000;
export const MAX_PROBE_OUTPUT_BYTES = 4 * 1024 * 1024;
export const STALE_JOB_MS = 5 * 60 * 1000;
export const AUTH_CACHE_TTL_MS = 10 * 60 * 1000;
+48
View File
@@ -0,0 +1,48 @@
import { eq, inArray } from "drizzle-orm";
import { db } from "@/config/db";
import { activities } from "@/schema/activities.schema";
import { albums } from "@/schema/albums.schema";
import { assetEdits } from "@/schema/assetEdits.schema";
import { assetFaces } from "@/schema/assetFaces.schema";
import { memoryAssets } from "@/schema/memoryAssets.schema";
export interface CompressionEligibilityAsset {
id: string;
visibility: string;
livePhotoVideoId?: string | null;
libraryId?: string | null;
isExternal?: boolean;
isOffline?: boolean;
stackId?: string | null;
duplicateId?: string | null;
}
export function baseCompressionUnsupportedReason(asset: CompressionEligibilityAsset): string | null {
if (!['timeline', 'archive'].includes(asset.visibility)) return "Locked and hidden assets are not supported";
if (asset.livePhotoVideoId) return "Live Photos are not supported";
if (asset.libraryId || asset.isExternal) return "External-library assets are not supported";
if (asset.isOffline) return "Offline assets are not supported";
if (asset.stackId) return "Stacked assets are not supported";
if (asset.duplicateId) return "Duplicate-group assets are not supported";
return null;
}
export async function getRelationshipUnsupportedReasons(assetIds: string[]): Promise<Map<string, string>> {
const reasons = new Map<string, string>();
if (assetIds.length === 0) return reasons;
const [edits, faces, memories, activityRows, covers] = await Promise.all([
db.select({ assetId: assetEdits.assetId }).from(assetEdits).where(inArray(assetEdits.assetId, assetIds)),
db.select({ assetId: assetFaces.assetId }).from(assetFaces).where(inArray(assetFaces.assetId, assetIds)),
db.select({ assetId: memoryAssets.assetId }).from(memoryAssets).where(inArray(memoryAssets.assetId, assetIds)),
db.select({ assetId: activities.assetId }).from(activities).where(inArray(activities.assetId, assetIds)),
db.select({ assetId: albums.albumThumbnailAssetId }).from(albums).where(inArray(albums.albumThumbnailAssetId, assetIds)),
]);
for (const row of edits) reasons.set(row.assetId, "Edited assets are not supported");
for (const row of faces) if (!reasons.has(row.assetId)) reasons.set(row.assetId, "Assets with detected faces are not supported");
for (const row of memories) if (!reasons.has(row.assetId)) reasons.set(row.assetId, "Assets used in memories are not supported");
for (const row of activityRows) if (row.assetId && !reasons.has(row.assetId)) reasons.set(row.assetId, "Assets with album comments or likes are not supported");
for (const row of covers) if (row.assetId && !reasons.has(row.assetId)) reasons.set(row.assetId, "Album cover assets are not supported");
return reasons;
}
+290
View File
@@ -0,0 +1,290 @@
import { spawn } from "child_process";
import { FFPROBE_TIMEOUT_MS, MAX_PROBE_OUTPUT_BYTES, MEDIA_DECODE_TIMEOUT_MS } from "./constants";
export interface ProbeStream {
codec_type?: string;
codec_name?: string;
width?: number;
height?: number;
avg_frame_rate?: string;
r_frame_rate?: string;
tags?: Record<string, string>;
side_data_list?: Array<Record<string, unknown>>;
duration?: string;
nb_frames?: string;
nb_read_frames?: string;
}
export interface ProbeResult {
streams: ProbeStream[];
format?: {
duration?: string;
size?: string;
format_name?: string;
tags?: Record<string, string>;
};
}
export async function ffprobe(filePath: string, signal?: AbortSignal): Promise<ProbeResult> {
return new Promise((resolve, reject) => {
const child = spawn("ffprobe", [
"-v", "error",
"-print_format", "json",
"-show_format",
"-show_streams",
"-count_frames",
filePath,
], { shell: false });
let stdout = "";
let stderr = "";
let settled = false;
const finish = (error?: Error, result?: ProbeResult) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
signal?.removeEventListener("abort", abort);
if (error) reject(error);
else resolve(result!);
};
const stop = () => {
if (child.exitCode === null && !child.killed) child.kill("SIGKILL");
};
const abort = () => {
stop();
finish(new Error("ffprobe was cancelled"));
};
const timeout = setTimeout(() => {
stop();
finish(new Error("ffprobe timed out"));
}, FFPROBE_TIMEOUT_MS);
timeout.unref();
signal?.addEventListener("abort", abort, { once: true });
if (signal?.aborted) return abort();
const append = (current: string, chunk: Buffer) => {
if (Buffer.byteLength(current) + chunk.length > MAX_PROBE_OUTPUT_BYTES) {
stop();
finish(new Error("ffprobe output exceeded the safety limit"));
return current;
}
return current + chunk.toString();
};
child.stdout.on("data", (chunk: Buffer) => { stdout = append(stdout, chunk); });
child.stderr.on("data", (chunk: Buffer) => { stderr = append(stderr, chunk); });
child.on("error", (error) => finish(error));
child.on("close", (code) => {
if (settled) return;
if (code !== 0) {
finish(new Error(`ffprobe failed with code ${code}: ${stderr.trim()}`));
return;
}
try {
finish(undefined, JSON.parse(stdout));
} catch {
finish(new Error("ffprobe returned invalid JSON"));
}
});
});
}
export function mediaDurationSeconds(probe: ProbeResult): number | null {
const raw = probe.format?.duration;
if (!raw) return null;
const value = Number(raw);
return Number.isFinite(value) ? value : null;
}
export async function verifyMediaDecodes(filePath: string, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const child = spawn("ffmpeg", [
"-v", "error",
"-xerror",
"-i", filePath,
"-map", "0:v?",
"-map", "0:a?",
"-f", "null",
"-",
], { shell: false });
let stderr = "";
let settled = false;
const finish = (error?: Error) => {
if (settled) return;
settled = true;
clearTimeout(timeout);
signal?.removeEventListener("abort", abort);
if (error) reject(error);
else resolve();
};
const stop = () => {
if (child.exitCode === null && !child.killed) child.kill("SIGKILL");
};
const abort = () => {
stop();
finish(new Error("Media validation was cancelled"));
};
const timeout = setTimeout(() => {
stop();
finish(new Error("Media decode validation timed out"));
}, MEDIA_DECODE_TIMEOUT_MS);
timeout.unref();
signal?.addEventListener("abort", abort, { once: true });
if (signal?.aborted) return abort();
child.stderr.on("data", (chunk: Buffer) => {
if (Buffer.byteLength(stderr) < MAX_PROBE_OUTPUT_BYTES) stderr += chunk.toString();
});
child.on("error", (error) => finish(error));
child.on("close", (code) => finish(code === 0 ? undefined : new Error(`Compressed output could not be fully decoded: ${stderr.trim() || `FFmpeg exited with code ${code}`}`)));
});
}
function frameCount(stream?: ProbeStream): number | null {
const value = Number(stream?.nb_read_frames ?? stream?.nb_frames);
return Number.isFinite(value) && value > 0 ? value : null;
}
function streamCount(probe: ProbeResult, type: string): number {
return probe.streams.filter((stream) => stream.codec_type === type).length;
}
function normalizedAspectRatio(stream?: ProbeStream): number | null {
if (!stream?.width || !stream?.height) return null;
return Math.min(stream.width, stream.height) / Math.max(stream.width, stream.height);
}
function validateDimensions(input: ProbeStream | undefined, output: ProbeStream | undefined, errors: string[]): void {
if (!input?.width || !input?.height) errors.push("Original has invalid dimensions");
if (!output?.width || !output?.height) {
errors.push("Output has invalid dimensions");
return;
}
if (!input?.width || !input?.height) return;
const inputRatio = normalizedAspectRatio(input);
const outputRatio = normalizedAspectRatio(output);
if (inputRatio != null && outputRatio != null && Math.abs(inputRatio - outputRatio) > 0.01) {
errors.push(`Output aspect ratio changed (${input.width}x${input.height} -> ${output.width}x${output.height})`);
}
const inputPixels = input.width * input.height;
const outputPixels = output.width * output.height;
if (outputPixels > inputPixels) {
errors.push(`Output resolution upscales the original (${input.width}x${input.height} -> ${output.width}x${output.height})`);
}
}
function validateOutputContainer(output: ProbeResult, mediaType: string, outputExtension: string, errors: string[]): void {
const extension = outputExtension.toLowerCase();
const primary = output.streams.find((stream) => stream.codec_type === "video");
if (mediaType === "IMAGE") {
const codecs: Record<string, string[]> = {
".jpg": ["mjpeg"],
".jpeg": ["mjpeg"],
".jfif": ["mjpeg"],
".png": ["png", "apng"],
".webp": ["webp"],
".gif": ["gif"],
".tif": ["tiff"],
".tiff": ["tiff"],
".bmp": ["bmp"],
".avif": ["av1"],
".heic": ["hevc", "av1"],
".heif": ["hevc", "av1"],
};
const allowed = codecs[extension];
if (allowed && (!primary?.codec_name || !allowed.includes(primary.codec_name))) {
errors.push(`Output codec ${primary?.codec_name ?? "unknown"} does not match ${extension}`);
}
return;
}
const formatName = output.format?.format_name;
if (!formatName) {
errors.push("Output container could not be verified");
return;
}
const formatTokens = new Set(formatName.toLowerCase().split(","));
const expectedFormats: Record<string, string[]> = {
".mp4": ["mp4"],
".m4v": ["mp4"],
".mov": ["mov"],
".mkv": ["matroska"],
".webm": ["webm"],
".avi": ["avi"],
".3gp": ["3gp"],
".3g2": ["3g2"],
".mts": ["mpegts"],
".m2ts": ["mpegts"],
};
const allowed = expectedFormats[extension];
if (allowed && !allowed.some((format) => formatTokens.has(format))) {
errors.push(`Output container ${formatName} does not match ${extension}`);
}
}
export function validateProbePair(
input: ProbeResult,
output: ProbeResult,
mediaType: string,
outputExtension: string,
): string[] {
const errors: string[] = [];
const inputVideo = input.streams.filter((stream) => stream.codec_type === "video");
const outputVideo = output.streams.filter((stream) => stream.codec_type === "video");
const inputPrimary = inputVideo[0];
const outputPrimary = outputVideo[0];
if (mediaType === "IMAGE") {
if (inputVideo.length !== 1 || outputVideo.length !== 1) {
errors.push(`Images must contain exactly one visual stream (${inputVideo.length} -> ${outputVideo.length})`);
}
for (const streamType of ["audio", "subtitle", "data", "attachment"]) {
const inputCount = streamCount(input, streamType);
const outputCount = streamCount(output, streamType);
if (inputCount !== 0 || outputCount !== 0) {
errors.push(`Image contains unsupported ${streamType} streams (${inputCount} -> ${outputCount})`);
}
}
const inputFrames = frameCount(inputPrimary) ?? 1;
const outputFrames = frameCount(outputPrimary) ?? 1;
if (inputFrames !== 1 || outputFrames !== 1) {
errors.push("Animated or multi-frame images are not supported by the compression replacement workflow");
}
} else if (mediaType === "VIDEO") {
if (inputVideo.length === 0 || outputVideo.length === 0 || inputVideo.length !== outputVideo.length) {
errors.push(`Output changed the number of video streams (${inputVideo.length} -> ${outputVideo.length})`);
}
const inputDuration = mediaDurationSeconds(input);
const outputDuration = mediaDurationSeconds(output);
if (inputDuration == null || outputDuration == null) {
errors.push("Video duration could not be verified");
} else {
const tolerance = Math.max(0.1, inputDuration * 0.005);
if (Math.abs(inputDuration - outputDuration) > tolerance) {
errors.push(`Output duration differs from the original (${inputDuration.toFixed(2)}s vs ${outputDuration.toFixed(2)}s)`);
}
}
for (const streamType of ["audio", "subtitle", "data", "attachment"]) {
const inputCount = streamCount(input, streamType);
const outputCount = streamCount(output, streamType);
if (outputCount !== inputCount) errors.push(`Output changed ${streamType} streams (${inputCount} -> ${outputCount})`);
}
const inputFrames = frameCount(inputPrimary);
const outputFrames = frameCount(outputPrimary);
if (inputFrames == null || outputFrames == null) {
errors.push("Video frame count could not be verified");
} else if (inputFrames !== outputFrames) {
errors.push(`Output frame count differs from the original (${inputFrames} -> ${outputFrames})`);
}
} else {
errors.push(`Unsupported media class ${mediaType}`);
}
validateDimensions(inputPrimary, outputPrimary, errors);
validateOutputContainer(output, mediaType, outputExtension, errors);
return errors;
}
+19
View File
@@ -0,0 +1,19 @@
import { createHash } from "crypto";
import { createReadStream } from "fs";
export async function sha1File(filePath: string): Promise<string> {
const hash = createHash("sha1");
for await (const chunk of createReadStream(filePath)) hash.update(chunk);
return hash.digest("base64");
}
export function checksumsEqual(left?: string | null, right?: string | null): boolean {
if (!left || !right) return false;
const normalize = (value: string) => {
const trimmed = value.trim();
return /^[a-f\d]{40}$/i.test(trimmed)
? Buffer.from(trimmed, "hex").toString("base64")
: trimmed;
};
return normalize(left) === normalize(right);
}
+117
View File
@@ -0,0 +1,117 @@
import { createHash } from "crypto";
import { ENV } from "@/config/environment";
export interface ImmichExifInfo {
fileSizeInByte?: number | null;
dateTimeOriginal?: string | null;
timeZone?: string | null;
description?: string | null;
rating?: number | null;
latitude?: number | null;
longitude?: number | null;
}
export interface ImmichAsset {
id: string;
ownerId: string;
type: "IMAGE" | "VIDEO";
originalFileName: string;
duration?: number | null;
fileCreatedAt: string;
fileModifiedAt: string;
updatedAt: string;
isFavorite: boolean;
isTrashed: boolean;
isOffline: boolean;
visibility: string;
livePhotoVideoId?: string | null;
libraryId?: string | null;
checksum: string;
isEdited: boolean;
duplicateId?: string | null;
stack?: { id: string; primaryAssetId: string; assetCount: number } | null;
tags?: Array<{ id: string }>;
people?: Array<{ id?: string; personGroupId?: string }>;
exifInfo?: ImmichExifInfo | null;
}
export interface ImmichAssetMetadata {
key: string;
value: Record<string, unknown>;
}
export class ImmichRequestError extends Error {
constructor(
message: string,
public readonly status: number,
) {
super(message);
}
}
export async function immichRequest(
route: string,
headers: Record<string, string>,
init: RequestInit = {},
): Promise<Response> {
const response = await fetch(`${ENV.IMMICH_URL}/api${route}`, {
...init,
headers: {
...headers,
...(init.body instanceof FormData ? {} : { "Content-Type": "application/json" }),
...(init.headers || {}),
},
});
if (!response.ok) {
const body = await response.text().catch(() => "");
throw new ImmichRequestError(
`Immich API ${init.method ?? "GET"} ${route} failed (${response.status}): ${body || response.statusText}`,
response.status,
);
}
return response;
}
export async function immichJson<T = any>(
route: string,
headers: Record<string, string>,
init: RequestInit = {},
): Promise<T> {
const response = await immichRequest(route, headers, init);
if (response.status === 204) return null as T;
return response.json() as Promise<T>;
}
export function getImmichAsset(assetId: string, headers: Record<string, string>): Promise<ImmichAsset> {
return immichJson<ImmichAsset>(`/assets/${assetId}`, headers);
}
export function getImmichAssetUnsupportedReason(asset: ImmichAsset, userId: string, mediaType?: string): string | null {
if (asset.ownerId !== userId) return "The asset is not owned by the current user";
if (mediaType && asset.type !== mediaType) return "The asset media type changed after it was queued";
if (asset.isTrashed) return "Trashed assets are not supported";
if (asset.isOffline) return "Offline assets are not supported";
if (!['timeline', 'archive'].includes(asset.visibility)) return "Locked and hidden assets are not supported";
if (asset.livePhotoVideoId) return "Live Photos are not supported";
if (asset.libraryId) return "External-library assets are not supported";
if (asset.isEdited) return "Edited assets are not supported";
if (asset.stack) return "Stacked assets are not supported";
if (asset.duplicateId) return "Duplicate-group assets are not supported";
if ((asset.people?.length ?? 0) > 0) return "Assets with assigned people are not supported";
return null;
}
export async function getImmichAssetMetadata(assetId: string, headers: Record<string, string>): Promise<ImmichAssetMetadata[]> {
return immichJson<ImmichAssetMetadata[]>(`/assets/${assetId}/metadata`, headers).catch((error) => {
if (error instanceof ImmichRequestError && error.status === 404) return [];
throw error;
});
}
export async function hashImmichOriginal(assetId: string, headers: Record<string, string>): Promise<string> {
const response = await immichRequest(`/assets/${assetId}/original`, headers);
if (!response.body) throw new Error("Immich returned an empty asset response");
const hash = createHash("sha1");
for await (const chunk of response.body as any) hash.update(chunk);
return hash.digest("base64");
}
+53
View File
@@ -0,0 +1,53 @@
export function guessContentType(fileName: string): string {
const extension = fileName.toLowerCase().split(".").pop();
switch (extension) {
case "jpg":
case "jpeg":
case "jfif":
return "image/jpeg";
case "png":
return "image/png";
case "heic":
return "image/heic";
case "heif":
return "image/heif";
case "webp":
return "image/webp";
case "avif":
return "image/avif";
case "gif":
return "image/gif";
case "tif":
case "tiff":
return "image/tiff";
case "bmp":
return "image/bmp";
case "mp4":
case "m4v":
return "video/mp4";
case "mov":
return "video/quicktime";
case "avi":
return "video/x-msvideo";
case "mkv":
return "video/x-matroska";
case "webm":
return "video/webm";
case "3gp":
return "video/3gpp";
case "3g2":
return "video/3gpp2";
case "mts":
case "m2ts":
return "video/MP2T";
default:
return "application/octet-stream";
}
}
export function isOutputExtensionCompatible(mediaType: string, extension: string): boolean {
const contentType = guessContentType(`output${extension}`);
if (mediaType === "IMAGE") return contentType.startsWith("image/");
if (mediaType === "VIDEO") return contentType.startsWith("video/");
return false;
}
+43
View File
@@ -0,0 +1,43 @@
import { and, eq } from "drizzle-orm";
import { appDb } from "@/db";
import { settings } from "@/db/schema";
import {
DEFAULT_OUTPUT_EXTENSION,
FFMPEG_ARGUMENTS_SETTING,
FFMPEG_OUTPUT_EXTENSION_SETTING,
} from "./constants";
import { CompressionSettings } from "./arguments";
async function getSetting(ownerId: string, key: string): Promise<string | null> {
const [row] = await appDb
.select({ value: settings.value })
.from(settings)
.where(and(eq(settings.ownerId, ownerId), eq(settings.key, key)));
return row?.value ?? null;
}
export async function getCompressionSettings(ownerId: string): Promise<CompressionSettings> {
const [ffmpegArguments, outputExtension] = await Promise.all([
getSetting(ownerId, FFMPEG_ARGUMENTS_SETTING),
getSetting(ownerId, FFMPEG_OUTPUT_EXTENSION_SETTING),
]);
return {
ffmpegArguments: ffmpegArguments ?? "",
outputExtension: outputExtension ?? DEFAULT_OUTPUT_EXTENSION,
};
}
export async function saveCompressionSettings(ownerId: string, value: CompressionSettings): Promise<void> {
await appDb.transaction(async (tx) => {
for (const [key, settingValue] of [
[FFMPEG_ARGUMENTS_SETTING, value.ffmpegArguments.trim()],
[FFMPEG_OUTPUT_EXTENSION_SETTING, value.outputExtension.trim().toLowerCase()],
] as const) {
await tx.insert(settings).values({ ownerId, key, value: settingValue }).onConflictDoUpdate({
target: [settings.ownerId, settings.key],
set: { value: settingValue },
});
}
});
}
+99
View File
@@ -0,0 +1,99 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { and, asc, desc, eq, gte, ilike, inArray, isNull, SQL } from "drizzle-orm";
import { db } from "@/config/db";
import { assets } from "@/schema/assets.schema";
import { exif } from "@/schema/exif.schema";
import { cleanUpAssets } from "@/helpers/asset.helper";
import { ASSET_VIDEO_PATH } from "@/config/routes";
import { requireCompressionUser } from "@/lib/compression/auth";
import { baseCompressionUnsupportedReason, getRelationshipUnsupportedReasons } from "@/lib/compression/eligibility";
import { getCompressionSettings } from "@/lib/compression/settings";
import { isOutputExtensionCompatible } from "@/lib/compression/mime";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "GET") {
res.setHeader("Allow", "GET");
return res.status(405).json({ error: "Method Not Allowed" });
}
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const type = String(req.query.type ?? "ALL").toUpperCase();
const filename = String(req.query.filename ?? "").trim();
const rawMinSizeMb = Number(req.query.minSizeMb ?? 0);
const minSizeMb = Number.isFinite(rawMinSizeMb) ? Math.max(0, rawMinSizeMb) : 0;
const sort = String(req.query.sort ?? "largest");
const rawLimit = Number(req.query.limit ?? 500);
const limit = Number.isFinite(rawLimit) ? Math.min(500, Math.max(1, Math.trunc(rawLimit))) : 500;
const conditions: SQL[] = [
eq(assets.ownerId, currentUser.id),
isNull(assets.deletedAt),
eq(assets.status, "active"),
inArray(assets.visibility, ["timeline", "archive"]),
];
if (type === "IMAGE" || type === "VIDEO") conditions.push(eq(assets.type, type));
if (filename) conditions.push(ilike(assets.originalFileName, `%${filename}%`));
if (minSizeMb > 0) conditions.push(gte(exif.fileSizeInByte, Math.round(minSizeMb * 1024 * 1024)));
const orderBy = sort === "smallest"
? asc(exif.fileSizeInByte)
: sort === "oldest"
? asc(assets.fileCreatedAt)
: sort === "newest"
? desc(assets.fileCreatedAt)
: desc(exif.fileSizeInByte);
const rows = await db
.select({
id: assets.id,
type: assets.type,
originalFileName: assets.originalFileName,
isFavorite: assets.isFavorite,
duration: assets.duration,
localDateTime: assets.localDateTime,
fileCreatedAt: assets.fileCreatedAt,
livePhotoVideoId: assets.livePhotoVideoId,
libraryId: assets.libraryId,
isExternal: assets.isExternal,
isOffline: assets.isOffline,
visibility: assets.visibility,
stackId: assets.stackId,
duplicateId: assets.duplicateId,
fileSizeInByte: exif.fileSizeInByte,
exifImageWidth: exif.exifImageWidth,
exifImageHeight: exif.exifImageHeight,
dateTimeOriginal: exif.dateTimeOriginal,
orientation: exif.orientation,
})
.from(assets)
.leftJoin(exif, eq(assets.id, exif.assetId))
.where(and(...conditions))
.orderBy(orderBy)
.limit(limit);
const [relationshipReasons, settings] = await Promise.all([
getRelationshipUnsupportedReasons(rows.map((row) => row.id)),
getCompressionSettings(currentUser.id),
]);
const mapped = rows.map((row) => ({
...row,
duration: row.duration == null ? null : Number(row.duration),
localDateTime: row.localDateTime,
dateTimeOriginal: row.dateTimeOriginal?.toISOString?.() ?? row.fileCreatedAt.toISOString(),
exifImageWidth: row.exifImageWidth || 1,
exifImageHeight: row.exifImageHeight || 1,
fileSizeInByte: row.fileSizeInByte ?? 0,
compressionUnsupportedReason: baseCompressionUnsupportedReason(row)
?? relationshipReasons.get(row.id)
?? (!isOutputExtensionCompatible(row.type, settings.outputExtension)
? `${settings.outputExtension} cannot be used for ${row.type.toLowerCase()} assets`
: undefined),
url: "",
previewUrl: "",
downloadUrl: ASSET_VIDEO_PATH(row.id),
}));
return res.status(200).json({ assets: cleanUpAssets(mapped as any) });
}
@@ -0,0 +1,24 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { eq } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { requireCompressionUser } from "@/lib/compression/auth";
import { cancelCompressionJob } from "@/workers/compression/runner";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "POST") return res.status(405).json({ error: "Method Not Allowed" });
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const id = String(req.query.id);
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, id));
if (!job || job.userId !== currentUser.id) return res.status(404).json({ error: "Job not found" });
if (!["QUEUED", "DOWNLOADING", "COMPRESSING"].includes(job.status)) {
return res.status(409).json({ error: `Cannot cancel a ${job.status} job` });
}
try {
await cancelCompressionJob(id, currentUser.id, job.attempt);
return res.status(200).json({ success: true });
} catch (error: any) {
return res.status(409).json({ error: error?.message ?? "Could not cancel job" });
}
}
@@ -0,0 +1,25 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { eq } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { requireCompressionUser } from "@/lib/compression/auth";
import { discardCompressionJob } from "@/workers/compression/runner";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "DELETE") {
res.setHeader("Allow", "DELETE");
return res.status(405).json({ error: "Method Not Allowed" });
}
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const id = String(req.query.id);
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, id));
if (!job || job.userId !== currentUser.id) return res.status(404).json({ error: "Job not found" });
try {
await discardCompressionJob(id, currentUser.id);
return res.status(204).end();
} catch (error: any) {
return res.status(409).json({ error: error?.message ?? "Could not discard job" });
}
}
@@ -0,0 +1,18 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { eq } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { requireCompressionUser } from "@/lib/compression/auth";
import { readCompressionLog } from "@/workers/compression/runner";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "GET") return res.status(405).json({ error: "Method Not Allowed" });
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const id = String(req.query.id);
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, id));
if (!job || job.userId !== currentUser.id) return res.status(404).json({ error: "Job not found" });
const log = await readCompressionLog(id);
res.setHeader("Content-Type", "text/plain; charset=utf-8");
return res.status(200).send(log);
}
@@ -0,0 +1,63 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { createReadStream } from "fs";
import { stat } from "fs/promises";
import { eq } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { requireCompressionUser } from "@/lib/compression/auth";
import { guessContentType } from "@/lib/compression/mime";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (!["GET", "HEAD"].includes(req.method ?? "")) {
res.setHeader("Allow", "GET, HEAD");
return res.status(405).json({ error: "Method Not Allowed" });
}
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const id = String(req.query.id);
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, id));
if (!job || job.userId !== currentUser.id) return res.status(404).json({ error: "Job not found" });
if (!["READY", "NO_SAVING"].includes(job.status)) return res.status(409).json({ error: "Preview is not ready" });
let fileStat;
try {
fileStat = await stat(job.outputPath);
} catch {
return res.status(404).json({ error: "Compressed output is no longer available" });
}
const mime = guessContentType(job.outputPath);
res.setHeader("Content-Type", mime);
res.setHeader("Accept-Ranges", "bytes");
const range = req.headers.range;
if (range) {
const match = range.match(/^bytes=(\d*)-(\d*)$/);
if (match) {
const hasStart = match[1] !== "";
const hasEnd = match[2] !== "";
const requestedStart = hasStart ? Number(match[1]) : null;
const requestedEnd = hasEnd ? Number(match[2]) : null;
const suffixLength = !hasStart && requestedEnd != null ? requestedEnd : null;
const start = suffixLength != null
? Math.max(0, fileStat.size - suffixLength)
: requestedStart ?? 0;
const end = suffixLength != null
? fileStat.size - 1
: Math.min(requestedEnd ?? fileStat.size - 1, fileStat.size - 1);
if ((hasStart || hasEnd) && start >= 0 && start <= end && start < fileStat.size) {
res.status(206);
res.setHeader("Content-Range", `bytes ${start}-${end}/${fileStat.size}`);
res.setHeader("Content-Length", String(end - start + 1));
if (req.method === "HEAD") return res.end();
createReadStream(job.outputPath, { start, end }).pipe(res);
return;
}
}
res.status(416).setHeader("Content-Range", `bytes */${fileStat.size}`);
return res.end();
}
res.setHeader("Content-Length", String(fileStat.size));
if (req.method === "HEAD") return res.end();
createReadStream(job.outputPath).pipe(res);
}
@@ -0,0 +1,23 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { eq } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { requireCompressionUser } from "@/lib/compression/auth";
import { getUserHeaders } from "@/helpers/user.helper";
import { replaceCompressionJob } from "@/workers/compression/replacement";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "POST") return res.status(405).json({ error: "Method Not Allowed" });
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const id = String(req.query.id);
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, id));
if (!job || job.userId !== currentUser.id) return res.status(404).json({ error: "Job not found" });
try {
await replaceCompressionJob(id, currentUser.id, getUserHeaders(currentUser) as Record<string, string>);
return res.status(200).json({ success: true });
} catch (error: any) {
return res.status(409).json({ error: error?.message ?? "Replacement failed" });
}
}
@@ -0,0 +1,31 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { eq } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { requireCompressionUser } from "@/lib/compression/auth";
import { getUserHeaders } from "@/helpers/user.helper";
import { registerCompressionAuth, retryCompressionJob } from "@/workers/compression/runner";
import { getCompressionSettings } from "@/lib/compression/settings";
import { validateCompressionSettings } from "@/lib/compression/arguments";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "POST") return res.status(405).json({ error: "Method Not Allowed" });
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const id = String(req.query.id);
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, id));
if (!job || job.userId !== currentUser.id) return res.status(404).json({ error: "Job not found" });
if (!["FAILED", "CANCELLED", "NO_SAVING"].includes(job.status)) {
return res.status(409).json({ error: `Cannot retry a ${job.status} job` });
}
const settings = await getCompressionSettings(currentUser.id);
const settingsError = validateCompressionSettings(settings);
if (settingsError) return res.status(400).json({ error: `${settingsError}. Configure compression in Settings first.` });
try {
await retryCompressionJob(id, currentUser.id, job.attempt, settings);
registerCompressionAuth(currentUser.id, getUserHeaders(currentUser) as Record<string, string>);
return res.status(200).json({ success: true });
} catch (error: any) {
return res.status(409).json({ error: error?.message ?? "Could not retry job" });
}
}
+151
View File
@@ -0,0 +1,151 @@
import type { NextApiRequest, NextApiResponse } from "next";
import path from "path";
import { randomUUID } from "crypto";
import { and, desc, eq, inArray, isNull, ne } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { db } from "@/config/db";
import { assets } from "@/schema/assets.schema";
import { exif } from "@/schema/exif.schema";
import { getUserHeaders } from "@/helpers/user.helper";
import { getCompressionSettings } from "@/lib/compression/settings";
import { validateCompressionSettings } from "@/lib/compression/arguments";
import { COMPRESSION_ROOT } from "@/lib/compression/constants";
import { registerCompressionAuth } from "@/workers/compression/runner";
import { requireCompressionUser } from "@/lib/compression/auth";
import { baseCompressionUnsupportedReason, getRelationshipUnsupportedReasons } from "@/lib/compression/eligibility";
import { isOutputExtensionCompatible } from "@/lib/compression/mime";
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
function publicJob(job: typeof compressionJobs.$inferSelect) {
return {
id: job.id,
assetId: job.assetId,
originalFilename: job.originalFilename,
originalSize: job.originalSize,
outputSize: job.outputSize,
status: job.status,
attempt: job.attempt,
progress: job.progress,
speed: job.speed,
errorMessage: job.errorMessage,
ffmpegExitCode: job.ffmpegExitCode,
replacementStage: job.replacementStage,
createdAt: job.createdAt,
finishedAt: job.finishedAt,
replacedAt: job.replacedAt,
};
}
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
const headers = getUserHeaders(currentUser) as Record<string, string>;
if (req.method === "GET") {
const [openJobs, completedJobs] = await Promise.all([
appDb.select().from(compressionJobs)
.where(and(eq(compressionJobs.userId, currentUser.id), ne(compressionJobs.status, "COMPLETED")))
.orderBy(desc(compressionJobs.createdAt), desc(compressionJobs.id)),
appDb.select().from(compressionJobs)
.where(and(eq(compressionJobs.userId, currentUser.id), eq(compressionJobs.status, "COMPLETED")))
.orderBy(desc(compressionJobs.createdAt), desc(compressionJobs.id))
.limit(10),
]);
return res.status(200).json({ jobs: [...openJobs, ...completedJobs].map(publicJob) });
}
if (req.method !== "POST") {
res.setHeader("Allow", "GET, POST");
return res.status(405).json({ error: "Method Not Allowed" });
}
const rawAssetIds: unknown[] = Array.isArray(req.body?.assetIds) ? req.body.assetIds : [];
const assetIds: string[] = [
...new Set(rawAssetIds.filter((id): id is string => typeof id === "string" && UUID.test(id))),
];
if (assetIds.length === 0) return res.status(400).json({ error: "assetIds is required" });
if (assetIds.length > 500) return res.status(400).json({ error: "A maximum of 500 assets can be queued at once" });
const settings = await getCompressionSettings(currentUser.id);
const settingsError = validateCompressionSettings(settings);
if (settingsError) return res.status(400).json({ error: `${settingsError}. Configure compression in Settings first.` });
const sourceAssets = await db.select({
id: assets.id,
type: assets.type,
originalFileName: assets.originalFileName,
livePhotoVideoId: assets.livePhotoVideoId,
libraryId: assets.libraryId,
isExternal: assets.isExternal,
isOffline: assets.isOffline,
stackId: assets.stackId,
duplicateId: assets.duplicateId,
visibility: assets.visibility,
originalSize: exif.fileSizeInByte,
})
.from(assets)
.leftJoin(exif, eq(assets.id, exif.assetId))
.where(and(
eq(assets.ownerId, currentUser.id),
inArray(assets.id, assetIds),
eq(assets.status, "active"),
isNull(assets.deletedAt),
inArray(assets.visibility, ["timeline", "archive"]),
));
const rejected: Array<{ assetId: string; reason: string }> = [];
const createdIds: string[] = [];
const createdAssetIds: string[] = [];
const relationshipReasons = await getRelationshipUnsupportedReasons(sourceAssets.map((asset) => asset.id));
for (const asset of sourceAssets) {
const unsupportedReason = baseCompressionUnsupportedReason(asset)
?? relationshipReasons.get(asset.id)
?? (!isOutputExtensionCompatible(asset.type, settings.outputExtension)
? `${settings.outputExtension} cannot be used for ${asset.type.toLowerCase()} assets`
: null);
if (unsupportedReason) {
rejected.push({ assetId: asset.id, reason: unsupportedReason });
continue;
}
const jobId = randomUUID();
const jobDir = path.join(COMPRESSION_ROOT, jobId);
const originalExt = path.extname(asset.originalFileName) || ".bin";
const inputPath = path.join(jobDir, `input${originalExt}`);
const outputPath = path.join(jobDir, `output${settings.outputExtension}`);
const logPath = path.join(jobDir, "ffmpeg.log");
const inserted = await appDb.insert(compressionJobs).values({
id: jobId,
userId: currentUser.id,
assetId: asset.id,
originalFilename: asset.originalFileName,
originalSize: asset.originalSize ?? 0,
mediaType: asset.type,
inputPath,
outputPath,
logPath,
status: "QUEUED",
ffmpegArguments: settings.ffmpegArguments,
outputExtension: settings.outputExtension,
sourceMetadata: "{}",
}).onConflictDoNothing().returning({ id: compressionJobs.id });
if (inserted.length === 0) {
rejected.push({ assetId: asset.id, reason: "Asset already has an active compression job" });
} else {
createdIds.push(jobId);
createdAssetIds.push(asset.id);
}
}
const foundIds = new Set(sourceAssets.map((asset) => asset.id));
for (const assetId of assetIds) {
if (!foundIds.has(assetId)) rejected.push({ assetId, reason: "Asset not found" });
}
registerCompressionAuth(currentUser.id, headers);
return res.status(createdIds.length > 0 ? 201 : 200).json({ jobIds: createdIds, createdAssetIds, rejected });
}
+15
View File
@@ -0,0 +1,15 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { requireCompressionUser } from "@/lib/compression/auth";
import { getUserHeaders } from "@/helpers/user.helper";
import { registerCompressionAuth } from "@/workers/compression/runner";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (req.method !== "POST") {
res.setHeader("Allow", "POST");
return res.status(405).json({ error: "Method Not Allowed" });
}
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
registerCompressionAuth(currentUser.id, getUserHeaders(currentUser) as Record<string, string>);
return res.status(202).json({ success: true });
}
+29
View File
@@ -0,0 +1,29 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { requireCompressionUser } from "@/lib/compression/auth";
import { getCompressionSettings, saveCompressionSettings } from "@/lib/compression/settings";
import { validateCompressionSettings } from "@/lib/compression/arguments";
import { FFMPEG_ARGUMENTS_EXAMPLE } from "@/lib/compression/constants";
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
const currentUser = await requireCompressionUser(req, res);
if (!currentUser) return;
if (req.method === "GET") {
const value = await getCompressionSettings(currentUser.id);
return res.status(200).json({ ...value, example: FFMPEG_ARGUMENTS_EXAMPLE });
}
if (req.method === "PUT") {
const value = {
ffmpegArguments: String(req.body?.ffmpegArguments ?? ""),
outputExtension: String(req.body?.outputExtension ?? ""),
};
const error = validateCompressionSettings(value);
if (error) return res.status(400).json({ error });
await saveCompressionSettings(currentUser.id, value);
return res.status(200).json({ success: true, ...value });
}
res.setHeader("Allow", "GET, PUT");
return res.status(405).json({ error: "Method Not Allowed" });
}
+349
View File
@@ -0,0 +1,349 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
import PageLayout from "@/components/layouts/PageLayout";
import Header from "@/components/shared/Header";
import AssetGrid from "@/components/shared/AssetGrid";
import FloatingBar from "@/components/shared/FloatingBar";
import Loader from "@/components/ui/loader";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Badge } from "@/components/ui/badge";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import PhotoSelectionContext, { IPhotoSelectionContext } from "@/contexts/PhotoSelectionContext";
import { IAsset } from "@/types/asset";
import API from "@/lib/api";
import { humanizeBytes } from "@/helpers/string.helper";
import toast from "react-hot-toast";
import { FileArchive, RefreshCw, Search, Settings, X, RotateCcw, Trash2, Play, FileText, ArrowRightLeft } from "lucide-react";
import Link from "next/link";
interface CompressionJob {
id: string;
assetId: string;
originalFilename: string;
originalSize: number;
outputSize: number | null;
status: string;
attempt: number;
progress: number;
speed: string | null;
errorMessage: string | null;
ffmpegExitCode: number | null;
createdAt: string | Date;
finishedAt: string | Date | null;
replacedAt: string | Date | null;
replacementStage: string | null;
}
type Filters = {
type: "ALL" | "IMAGE" | "VIDEO";
minSizeMb: string;
filename: string;
sort: "largest" | "smallest" | "newest" | "oldest";
};
const statusLabel: Record<string, string> = {
QUEUED: "Queued",
DOWNLOADING: "Downloading original",
COMPRESSING: "Compressing",
VALIDATING: "Validating",
READY: "Ready",
REPLACING: "Replacing in Immich",
CANCELLING: "Cancelling",
RETRYING: "Retrying",
NEEDS_ATTENTION: "Needs attention",
COMPLETED: "Completed",
FAILED: "Failed",
CANCELLED: "Cancelled",
NO_SAVING: "No space saving",
};
function savings(job: CompressionJob) {
if (!job.outputSize || !job.originalSize) return null;
const saved = job.originalSize - job.outputSize;
return { saved, percent: (saved / job.originalSize) * 100 };
}
export default function CompressionPage() {
const [assets, setAssets] = useState<IAsset[]>([]);
const [jobs, setJobs] = useState<CompressionJob[]>([]);
const [loadingAssets, setLoadingAssets] = useState(false);
const [queueing, setQueueing] = useState(false);
const [pendingActions, setPendingActions] = useState<Set<string>>(new Set());
const jobsRequest = useRef(0);
const [filters, setFilters] = useState<Filters>({ type: "ALL", minSizeMb: "500", filename: "", sort: "largest" });
const [appliedFilters, setAppliedFilters] = useState(filters);
const [contextState, setContextState] = useState<IPhotoSelectionContext>({
selectedIds: [],
assets: [],
config: {},
updateContext: (newConfig: Partial<IPhotoSelectionContext>) => {
setContextState((prev) => ({
...prev,
...newConfig,
config: newConfig.config ? { ...prev.config, ...newConfig.config } : prev.config,
}));
},
});
const fetchAssets = useCallback(async () => {
setLoadingAssets(true);
try {
const data = await API.get("/api/compression/assets", {
type: appliedFilters.type,
minSizeMb: appliedFilters.minSizeMb || "0",
filename: appliedFilters.filename,
sort: appliedFilters.sort,
limit: 500,
});
const fetched = (data.assets || []) as IAsset[];
setAssets(fetched);
contextState.updateContext({ assets: fetched, selectedIds: [] });
} catch (error: any) {
toast.error(error?.message || error?.error || "Failed to load assets");
} finally {
setLoadingAssets(false);
}
}, [appliedFilters]);
const fetchJobs = useCallback(async () => {
const requestId = ++jobsRequest.current;
try {
const data = await API.get("/api/compression/jobs");
if (requestId === jobsRequest.current) setJobs(data.jobs || []);
} catch {
// Polling errors are intentionally quiet; explicit actions still surface errors.
}
}, []);
useEffect(() => { void fetchAssets(); }, [fetchAssets]);
useEffect(() => {
void fetchJobs();
const resume = () => API.post("/api/compression/jobs/resume", {}).catch(() => undefined);
void resume();
const pollTimer = window.setInterval(() => {
if (!document.hidden) void fetchJobs();
}, 2000);
const resumeTimer = window.setInterval(resume, 5 * 60 * 1000);
return () => {
window.clearInterval(pollTimer);
window.clearInterval(resumeTimer);
};
}, [fetchJobs]);
const selectedAssets = useMemo(
() => contextState.assets.filter((asset) => contextState.selectedIds.includes(asset.id)),
[contextState.assets, contextState.selectedIds],
);
const selectedBytes = useMemo(
() => selectedAssets.reduce((sum, asset) => sum + (asset.fileSizeInByte || 0), 0),
[selectedAssets],
);
const queueSelected = async () => {
if (contextState.selectedIds.length === 0) return;
setQueueing(true);
try {
const result = await API.post("/api/compression/jobs", { assetIds: contextState.selectedIds });
const createdAssetIds = new Set<string>(result.createdAssetIds || []);
contextState.updateContext({ selectedIds: contextState.selectedIds.filter((id) => !createdAssetIds.has(id)) });
const rejectedCount = result.rejected?.length || 0;
if (result.jobIds?.length) toast.success(`Queued ${result.jobIds.length} compression job${result.jobIds.length === 1 ? "" : "s"}`);
if (rejectedCount) {
const reasons = [...new Set<string>(result.rejected.map((item: { reason: string }) => item.reason))].slice(0, 3).join("; ");
toast.error(`${rejectedCount} asset${rejectedCount === 1 ? " was" : "s were"} rejected: ${reasons}`);
}
await fetchJobs();
} catch (error: any) {
toast.error(error?.message || error?.error || "Failed to queue compression");
} finally {
setQueueing(false);
}
};
const runAction = async (job: CompressionJob, action: "cancel" | "retry" | "replace" | "discard") => {
if (pendingActions.has(job.id)) return;
if (action === "replace") {
const message = job.status === "NEEDS_ATTENTION"
? `Reconcile ${job.originalFilename}?\n\nPower Tools will inspect Immich and continue only from the last durable replacement stage.`
: `Replace ${job.originalFilename} in Immich?\n\nThe uploaded bytes and supported metadata will be verified before the original is moved to Trash.`;
if (!window.confirm(message)) return;
}
setPendingActions((current) => new Set(current).add(job.id));
try {
if (action === "discard") {
await API.delete(`/api/compression/jobs/${job.id}`);
} else {
await API.post(`/api/compression/jobs/${job.id}/${action}`, {});
}
await fetchJobs();
} catch (error: any) {
toast.error(error?.message || error?.error || `${action} failed`);
} finally {
setPendingActions((current) => {
const next = new Set(current);
next.delete(job.id);
return next;
});
}
};
const activeJobs = jobs.filter((job) => job.status !== "COMPLETED");
const completedJobs = jobs.filter((job) => job.status === "COMPLETED");
return (
<PageLayout className="!p-0 !mb-0 relative pb-24">
<Header
leftComponent="Compression"
rightComponent={
<Link href="/settings">
<Button variant="outline" size="sm"><Settings className="h-4 w-4 mr-1" /> FFmpeg settings</Button>
</Link>
}
/>
<PhotoSelectionContext.Provider value={{ ...contextState, updateContext: contextState.updateContext }}>
<div className="p-4 space-y-6">
<section className="border rounded-lg p-4 space-y-3">
<div className="flex items-center gap-2">
<Search className="h-4 w-4 text-muted-foreground" />
<h2 className="font-medium">Browse assets</h2>
<span className="text-xs text-muted-foreground ml-auto">Manual selection only · max 500 results</span>
</div>
<div className="grid grid-cols-1 md:grid-cols-4 gap-2">
<Select value={filters.type} onValueChange={(value: Filters["type"]) => setFilters((prev) => ({ ...prev, type: value }))}>
<SelectTrigger><SelectValue /></SelectTrigger>
<SelectContent>
<SelectItem value="ALL">Images + Videos</SelectItem>
<SelectItem value="VIDEO">Videos</SelectItem>
<SelectItem value="IMAGE">Images</SelectItem>
</SelectContent>
</Select>
<Input type="number" min="0" value={filters.minSizeMb} onChange={(e) => setFilters((prev) => ({ ...prev, minSizeMb: e.target.value }))} placeholder="Minimum size (MB)" />
<Input value={filters.filename} onChange={(e) => setFilters((prev) => ({ ...prev, filename: e.target.value }))} placeholder="Filename" />
<Select value={filters.sort} onValueChange={(value: Filters["sort"]) => setFilters((prev) => ({ ...prev, sort: value }))}>
<SelectTrigger><SelectValue /></SelectTrigger>
<SelectContent>
<SelectItem value="largest">Largest first</SelectItem>
<SelectItem value="smallest">Smallest first</SelectItem>
<SelectItem value="newest">Newest</SelectItem>
<SelectItem value="oldest">Oldest</SelectItem>
</SelectContent>
</Select>
</div>
<div className="flex justify-end">
<Button size="sm" onClick={() => setAppliedFilters({ ...filters })} disabled={loadingAssets}>
<RefreshCw className={`h-4 w-4 mr-1 ${loadingAssets ? "animate-spin" : ""}`} /> Apply filters
</Button>
</div>
</section>
{activeJobs.length > 0 && (
<section className="space-y-2">
<div className="flex items-center gap-2"><FileArchive className="h-4 w-4" /><h2 className="font-medium">Compression jobs</h2></div>
<div className="space-y-2">
{activeJobs.map((job) => {
const saved = savings(job);
const running = ["QUEUED", "DOWNLOADING", "COMPRESSING", "VALIDATING", "REPLACING", "CANCELLING", "RETRYING"].includes(job.status);
const actionPending = pendingActions.has(job.id);
return (
<div key={job.id} className="border rounded-lg p-3 space-y-2">
<div className="flex flex-wrap items-center gap-2">
<span className="font-medium truncate min-w-0 flex-1">{job.originalFilename}</span>
<Badge variant={job.status === "FAILED" || job.status === "NO_SAVING" ? "destructive" : "outline"}>{statusLabel[job.status] || job.status}</Badge>
{job.speed && job.status === "COMPRESSING" && <span className="text-xs text-muted-foreground">{job.speed}</span>}
</div>
{running && job.status !== "REPLACING" && (
<div className="h-2 rounded bg-muted overflow-hidden"><div className="h-full bg-primary transition-all" style={{ width: `${Math.max(2, job.progress || 0)}%` }} /></div>
)}
{job.outputSize != null && (
<div className="text-xs text-muted-foreground">
{humanizeBytes(job.originalSize)} → {humanizeBytes(job.outputSize)}
{saved && saved.saved > 0 ? ` · saved ${humanizeBytes(saved.saved)} (${saved.percent.toFixed(1)}%)` : ""}
</div>
)}
{job.errorMessage && <div className="text-xs text-destructive">{job.errorMessage}</div>}
{job.status === "READY" && (
<div className="rounded border border-yellow-500/30 bg-yellow-500/5 p-2 text-xs text-muted-foreground">
The source will be checked again for edits and unsupported relationships before replacement.
</div>
)}
<div className="flex flex-wrap gap-2 justify-end">
{["DOWNLOADING", "COMPRESSING", "QUEUED"].includes(job.status) && (
<Button size="sm" variant="outline" disabled={actionPending} onClick={() => runAction(job, "cancel")}><X className="h-3.5 w-3.5 mr-1" />Cancel</Button>
)}
{["FAILED", "CANCELLED", "NO_SAVING"].includes(job.status) && (
<Button size="sm" variant="outline" disabled={actionPending} onClick={() => runAction(job, "retry")}><RotateCcw className="h-3.5 w-3.5 mr-1" />Retry</Button>
)}
{["FAILED", "CANCELLED", "NO_SAVING", "READY"].includes(job.status) && (
<Button size="sm" variant="outline" onClick={() => window.open(`/api/compression/jobs/${job.id}/log`, "_blank")}><FileText className="h-3.5 w-3.5 mr-1" />Log</Button>
)}
{["READY", "NO_SAVING"].includes(job.status) && (
<Button size="sm" variant="outline" onClick={() => window.open(`/api/compression/jobs/${job.id}/preview`, "_blank")}><Play className="h-3.5 w-3.5 mr-1" />Preview</Button>
)}
{["FAILED", "CANCELLED", "NO_SAVING", "READY"].includes(job.status) && (
<Button size="sm" variant="outline" disabled={actionPending} onClick={() => runAction(job, "discard")}><Trash2 className="h-3.5 w-3.5 mr-1" />Discard</Button>
)}
{job.status === "READY" && (
<Button size="sm" disabled={actionPending} onClick={() => runAction(job, "replace")}><ArrowRightLeft className="h-3.5 w-3.5 mr-1" />Replace in Immich</Button>
)}
{job.status === "NEEDS_ATTENTION" && (
<Button size="sm" disabled={actionPending} onClick={() => runAction(job, "replace")}><ArrowRightLeft className="h-3.5 w-3.5 mr-1" />Reconcile</Button>
)}
</div>
</div>
);
})}
</div>
</section>
)}
{completedJobs.length > 0 && (
<section className="space-y-2">
<h2 className="font-medium">Recent completed replacements</h2>
<div className="border rounded-lg divide-y">
{completedJobs.map((job) => {
const saved = savings(job);
return (
<div key={job.id} className="p-3 flex items-center gap-3 text-sm">
<span className="truncate flex-1">{job.originalFilename}</span>
<span className="text-muted-foreground">{humanizeBytes(job.originalSize)} → {humanizeBytes(job.outputSize || 0)}</span>
{saved && <Badge variant="outline">-{saved.percent.toFixed(1)}%</Badge>}
</div>
);
})}
</div>
</section>
)}
<section>
{loadingAssets ? (
<div className="py-16 flex justify-center"><Loader /></div>
) : assets.length === 0 ? (
<div className="py-16 text-center text-muted-foreground">No assets match the current filters.</div>
) : (
<AssetGrid assets={assets} selectable showMetadataOverlay allowAssetMutations={false} onSelectionChange={(selectedIds) => contextState.updateContext({ selectedIds })} />
)}
</section>
</div>
{selectedAssets.length > 0 && (
<FloatingBar>
<div className="flex items-center gap-3 justify-between w-full">
<div className="text-sm">
<div>{selectedAssets.length} asset{selectedAssets.length === 1 ? "" : "s"} selected</div>
<div className="text-xs text-muted-foreground">{humanizeBytes(selectedBytes)}</div>
</div>
<div className="flex gap-2">
<Button variant="outline" size="sm" onClick={() => contextState.updateContext({ selectedIds: [] })}>Clear</Button>
<Button size="sm" disabled={queueing} onClick={queueSelected}>
<FileArchive className="h-4 w-4 mr-1" />{queueing ? "Queueing..." : "Compress selected"}
</Button>
</div>
</div>
</FloatingBar>
)}
</PhotoSelectionContext.Provider>
</PageLayout>
);
}
+74 -1
View File
@@ -5,7 +5,7 @@ import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import {
Save, Eye, EyeOff, ExternalLink,
Workflow, Shield, CheckCircle, XCircle, Server, Globe, Download, Wand2,
Workflow, Shield, CheckCircle, XCircle, Server, Globe, Download, Wand2, FileArchive,
} from "lucide-react";
import { useEffect, useState } from "react";
import toast from "react-hot-toast";
@@ -55,6 +55,11 @@ export default function SettingsPage() {
valid: boolean; error: string | null; permissions: string[]; missing: string[];
} | null>(null);
const [ffmpegArguments, setFfmpegArguments] = useState("");
const [ffmpegOutputExtension, setFfmpegOutputExtension] = useState(".mp4");
const [ffmpegExample, setFfmpegExample] = useState("");
const [compressionSettingsLoading, setCompressionSettingsLoading] = useState(false);
const fetchWorkflowApiKey = async () => {
try {
const data = await API.get("/api/settings/kv/workflow_api_key");
@@ -75,6 +80,32 @@ export default function SettingsPage() {
}
};
const fetchCompressionSettings = async () => {
try {
const data = await API.get("/api/settings/compression");
setFfmpegArguments(data.ffmpegArguments || "");
setFfmpegOutputExtension(data.outputExtension || ".mp4");
setFfmpegExample(data.example || "");
} catch {
toast.error("Failed to load compression settings");
}
};
const saveCompressionSettings = async () => {
setCompressionSettingsLoading(true);
try {
await API.put("/api/settings/compression", {
ffmpegArguments,
outputExtension: ffmpegOutputExtension,
});
toast.success("Compression settings saved");
} catch (error: any) {
toast.error(error?.message || error?.error || "Failed to save compression settings");
} finally {
setCompressionSettingsLoading(false);
}
};
const IMPORT_REQUIRED_PERMISSIONS = ["asset.read", "asset.upload", "album.read", "album.create", "album.update", "tag.create", "tag.asset"];
const saveImportApiKey = async () => {
@@ -174,6 +205,7 @@ export default function SettingsPage() {
useEffect(() => {
fetchWorkflowApiKey();
fetchImportApiKey();
fetchCompressionSettings();
}, []);
return (
@@ -292,6 +324,47 @@ export default function SettingsPage() {
</div>
</section>
{/* Compression */}
<section>
<div className="flex items-center gap-2 mb-3">
<FileArchive className="h-4 w-4 text-muted-foreground" />
<h2 className="text-sm font-semibold tracking-wide uppercase text-muted-foreground">Compression</h2>
</div>
<div className="border rounded-lg p-4 space-y-3">
<p className="text-xs text-muted-foreground">
FFmpeg is executed directly without a shell. Both <code>{`{input}`}</code> and <code>{`{output}`}</code> placeholders are required. No presets are applied automatically.
</p>
<div className="space-y-1.5">
<label htmlFor="ffmpegArguments" className="text-xs font-medium">FFmpeg arguments</label>
<textarea
id="ffmpegArguments"
rows={4}
className="flex w-full rounded-md border border-input bg-background px-3 py-2 text-sm font-mono ring-offset-background placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2"
placeholder={ffmpegExample || '-i {input} ... {output}'}
value={ffmpegArguments}
onChange={(e) => setFfmpegArguments(e.target.value)}
/>
</div>
<div className="space-y-1.5">
<label htmlFor="ffmpegOutputExtension" className="text-xs font-medium">Output extension</label>
<Input
id="ffmpegOutputExtension"
className="w-32 font-mono"
value={ffmpegOutputExtension}
onChange={(e) => setFfmpegOutputExtension(e.target.value)}
placeholder=".mp4"
/>
</div>
<div className="flex items-center justify-between gap-3">
<p className="text-[10px] text-muted-foreground">Each queued job stores a snapshot of these values.</p>
<Button size="sm" onClick={saveCompressionSettings} disabled={compressionSettingsLoading}>
<Save className="h-3.5 w-3.5 mr-1" />
{compressionSettingsLoading ? "Saving..." : "Save"}
</Button>
</div>
</div>
</section>
{/* Import API Key */}
<section>
<div className="flex items-center gap-2 mb-3">
+6
View File
@@ -0,0 +1,6 @@
import { pgTable, uuid } from "drizzle-orm/pg-core";
export const activities = pgTable("activity", {
id: uuid("id").primaryKey(),
assetId: uuid("assetId"),
});
+6
View File
@@ -0,0 +1,6 @@
import { pgTable, uuid } from "drizzle-orm/pg-core";
export const assetEdits = pgTable("asset_edit", {
id: uuid("id").primaryKey(),
assetId: uuid("assetId").notNull(),
});
+2 -2
View File
@@ -10,6 +10,7 @@ export const assets = pgTable('asset', {
isFavorite: boolean('isFavorite').default(false).notNull(),
duration: integer('duration'),
// checksum: bytea('checksum').notNull(),
libraryId: uuid('libraryId'),
visibility: varchar('visibility').default('timeline').notNull(),
livePhotoVideoId: uuid('livePhotoVideoId').references((): any => assets.id, { onDelete: 'set null', onUpdate: 'cascade' }),
updatedAt: timestamp('updatedAt', { withTimezone: true }).defaultNow().notNull(),
@@ -18,7 +19,6 @@ export const assets = pgTable('asset', {
originalFileName: varchar('originalFileName').notNull(),
// thumbhash: bytea('thumbhash'),
isOffline: boolean('isOffline').default(false).notNull(),
// libraryId: uuid('libraryId').references(() => libraries.id, { onDelete: 'cascade', onUpdate: 'cascade' }),
isExternal: boolean('isExternal').default(false).notNull(),
deletedAt: timestamp('deletedAt', { withTimezone: true }),
localDateTime: timestamp('localDateTime', { withTimezone: true }).notNull(),
@@ -40,4 +40,4 @@ export const assets = pgTable('asset', {
// export const assetStack = pgTable('asset_stack', {
// id: uuid('id').primaryKey(),
// // ... other columns
// });
// });
+2 -1
View File
@@ -31,6 +31,7 @@ export const exif = pgTable('asset_exif', {
bitsPerSample: integer('bitsPerSample'),
autoStackId: varchar('autoStackId'),
rating: integer('rating'),
lockedProperties: varchar('lockedProperties').array(),
});
export type IExifColumns = keyof typeof exif.$inferSelect
export type IExifColumns = keyof typeof exif.$inferSelect
+3
View File
@@ -5,4 +5,7 @@ export { assetFaces } from './assetFaces.schema';
export { users } from './users.schema';
export { albumsAssetsAssets } from './albumAssetsAssets.schema';
export { tags } from './tags.schema';
export { assetEdits } from './assetEdits.schema';
export { memoryAssets } from './memoryAssets.schema';
export { activities } from './activities.schema';
export * from './relationships';
+6
View File
@@ -0,0 +1,6 @@
import { pgTable, uuid } from "drizzle-orm/pg-core";
export const memoryAssets = pgTable("memory_asset", {
memoriesId: uuid("memoriesId").notNull(),
assetId: uuid("assetId").notNull(),
});
+8
View File
@@ -16,6 +16,14 @@ export interface IAsset {
dateTimeOriginal: string;
orientation?: number | null | string;
downloadUrl?: string;
fileSizeInByte?: number;
fileCreatedAt?: string | Date;
originalMimeType?: string;
livePhotoVideoId?: string | null;
isExternal?: boolean;
visibility?: string;
fps?: number | null;
compressionUnsupportedReason?: string;
}
export interface IAssetThumbhash {
+643
View File
@@ -0,0 +1,643 @@
import { openAsBlob } from "fs";
import { rm, stat } from "fs/promises";
import path from "path";
import { and, eq, inArray, isNull, lt, or } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { db } from "@/config/db";
import { ENV } from "@/config/environment";
import { assets } from "@/schema/assets.schema";
import { getRelationshipUnsupportedReasons } from "@/lib/compression/eligibility";
import { checksumsEqual, sha1File } from "@/lib/compression/files";
import {
getImmichAsset,
getImmichAssetMetadata,
getImmichAssetUnsupportedReason,
hashImmichOriginal,
immichJson,
type ImmichAsset,
type ImmichAssetMetadata,
} from "@/lib/compression/immich";
import { guessContentType } from "@/lib/compression/mime";
import { STALE_JOB_MS } from "@/lib/compression/constants";
type CompressionJob = typeof compressionJobs.$inferSelect;
type ReplacementStage =
| "STARTED"
| "SOURCE_VERIFIED"
| "UPLOAD_REQUESTED"
| "UPLOADED"
| "RELATIONSHIPS_COPIED"
| "TAGS_COPIED"
| "METADATA_COPIED"
| "VERIFIED"
| "TRASH_REQUESTED"
| "ORIGINAL_TRASHED";
interface ReplacementSnapshot {
asset: ImmichAsset;
metadata: ImmichAssetMetadata[];
fingerprint: string;
}
interface StageUpdate {
sourceMetadata?: string;
outputSize?: number;
outputChecksum?: string;
replacementAssetId?: string | null;
}
const METADATA_KEY = "immich-power-tools:compression";
const CANDIDATE_STAGES: ReplacementStage[] = [
"UPLOADED",
"RELATIONSHIPS_COPIED",
"TAGS_COPIED",
"METADATA_COPIED",
"VERIFIED",
];
function canonicalize(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonicalize);
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, item]) => [key, canonicalize(item)]),
);
}
return value;
}
function normalizeSource(asset: ImmichAsset, metadata: ImmichAssetMetadata[]) {
return canonicalize({
updatedAt: asset.updatedAt,
checksum: asset.checksum,
fileCreatedAt: asset.fileCreatedAt,
fileModifiedAt: asset.fileModifiedAt,
isFavorite: asset.isFavorite,
visibility: asset.visibility,
isEdited: asset.isEdited,
livePhotoVideoId: asset.livePhotoVideoId ?? null,
duplicateId: asset.duplicateId ?? null,
stack: asset.stack ?? null,
tags: (asset.tags ?? []).map((tag) => tag.id).sort(),
people: (asset.people ?? []).map((person) => person.personGroupId ?? person.id ?? "").sort(),
exifInfo: asset.exifInfo ?? null,
metadata: metadata
.filter((item) => item.key !== METADATA_KEY)
.map((item) => ({ key: item.key, value: item.value }))
.sort((left, right) => left.key.localeCompare(right.key)),
});
}
function sourceFingerprint(asset: ImmichAsset, metadata: ImmichAssetMetadata[]): string {
return JSON.stringify(normalizeSource(asset, metadata));
}
function metadataValueEqual(left: unknown, right: unknown): boolean {
return JSON.stringify(canonicalize(left)) === JSON.stringify(canonicalize(right));
}
async function assertSourceSupported(asset: ImmichAsset, userId: string, mediaType: string): Promise<void> {
const apiReason = getImmichAssetUnsupportedReason(asset, userId, mediaType);
if (apiReason) throw new Error(apiReason);
const relationshipReasons = await getRelationshipUnsupportedReasons([asset.id]);
const relationshipReason = relationshipReasons.get(asset.id);
if (relationshipReason) throw new Error(relationshipReason);
}
async function loadSourceSnapshot(job: CompressionJob, headers: Record<string, string>): Promise<ReplacementSnapshot> {
const [asset, metadata] = await Promise.all([
getImmichAsset(job.assetId, headers),
getImmichAssetMetadata(job.assetId, headers),
]);
await assertSourceSupported(asset, job.userId, job.mediaType);
return { asset, metadata, fingerprint: sourceFingerprint(asset, metadata) };
}
function readSourceSnapshot(job: CompressionJob): ReplacementSnapshot {
try {
const snapshot = JSON.parse(job.sourceMetadata) as ReplacementSnapshot;
if (!snapshot?.asset?.id || !snapshot.fingerprint || !Array.isArray(snapshot.metadata)) {
throw new Error("invalid snapshot");
}
return snapshot;
} catch {
throw new Error("Replacement source snapshot is missing or corrupt");
}
}
async function assertSourceUnchanged(
job: CompressionJob,
snapshot: ReplacementSnapshot,
headers: Record<string, string>,
): Promise<void> {
const current = await loadSourceSnapshot(job, headers);
if (current.fingerprint !== snapshot.fingerprint) {
throw new Error("The source asset changed after replacement started; the original was not trashed");
}
}
async function advanceStage(
jobId: string,
expected: ReplacementStage | ReplacementStage[],
next: ReplacementStage,
values: StageUpdate = {},
): Promise<CompressionJob> {
const expectedStages = Array.isArray(expected) ? expected : [expected];
const [updated] = await appDb.update(compressionJobs).set({
...values,
replacementStage: next,
heartbeatAt: new Date(),
}).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.status, "REPLACING"),
inArray(compressionJobs.replacementStage, expectedStages),
)).returning();
if (!updated) throw new Error(`Replacement state changed before ${next.toLowerCase().replaceAll("_", " ")}`);
return updated;
}
async function findCandidateByChecksum(
job: CompressionJob,
checksum: string,
headers: Record<string, string>,
): Promise<string | null> {
const result = await immichJson<{ results?: Array<{ id?: string; assetId?: string; action?: string; reason?: string }> }>(
"/assets/bulk-upload-check",
headers,
{ method: "POST", body: JSON.stringify({ assets: [{ id: job.id, checksum }] }) },
);
const match = result.results?.find((item) => item.id === job.id && item.action === "reject" && item.reason === "duplicate");
if (!match?.assetId) return null;
const metadata = await getImmichAssetMetadata(match.assetId, headers).catch(() => []);
const marker = metadata.find((item) => item.key === METADATA_KEY)?.value;
return marker?.compressionJobId === job.id && marker?.sourceAssetId === job.assetId ? match.assetId : null;
}
async function assertNoUnrelatedDuplicate(job: CompressionJob, checksum: string, headers: Record<string, string>): Promise<void> {
const result = await immichJson<{ results?: Array<{ id?: string; assetId?: string; action?: string; reason?: string }> }>(
"/assets/bulk-upload-check",
headers,
{ method: "POST", body: JSON.stringify({ assets: [{ id: job.id, checksum }] }) },
);
const duplicate = result.results?.find((item) => item.id === job.id && item.action === "reject" && item.reason === "duplicate");
if (!duplicate?.assetId) return;
const ownCandidate = await findCandidateByChecksum(job, checksum, headers);
if (!ownCandidate) throw new Error(`Replacement output already exists in Immich as asset ${duplicate.assetId}`);
}
async function uploadCandidate(
job: CompressionJob,
snapshot: ReplacementSnapshot,
checksum: string,
headers: Record<string, string>,
): Promise<string> {
const blob = await openAsBlob(job.outputPath, { type: guessContentType(job.outputPath) });
const outputName = `${path.parse(job.originalFilename).name}${job.outputExtension}`;
const formData = new FormData();
formData.set("fileCreatedAt", snapshot.asset.fileCreatedAt);
formData.set("fileModifiedAt", snapshot.asset.fileModifiedAt);
if (snapshot.asset.duration != null) formData.set("duration", String(snapshot.asset.duration));
formData.set("filename", outputName);
formData.set("isFavorite", String(snapshot.asset.isFavorite));
formData.set("visibility", snapshot.asset.visibility);
formData.set("metadata", JSON.stringify([
...snapshot.metadata.filter((item) => item.key !== METADATA_KEY).map(({ key, value }) => ({ key, value })),
{ key: METADATA_KEY, value: { compressionJobId: job.id, sourceAssetId: job.assetId } },
]));
formData.set("assetData", blob, outputName);
const uploadHeaders = { ...headers, "x-immich-checksum": checksum };
delete uploadHeaders["Content-Type"];
delete uploadHeaders["content-type"];
const response = await fetch(`${ENV.IMMICH_URL}/api/assets`, {
method: "POST",
headers: uploadHeaders,
body: formData,
});
const body = await response.json().catch(async () => ({ message: await response.text().catch(() => "") }));
if (response.status === 200 || String(body?.status ?? "").toLowerCase() === "duplicate") {
throw new Error("Replacement upload resolved to an existing duplicate");
}
if (response.status !== 201 || !body?.id) {
throw new Error(`Failed to upload replacement (${response.status}): ${body?.message || response.statusText}`);
}
return body.id as string;
}
async function ensureTags(sourceAsset: ImmichAsset, targetId: string, headers: Record<string, string>): Promise<void> {
for (const tag of sourceAsset.tags ?? []) {
if (!tag.id) continue;
await immichJson(`/tags/${tag.id}/assets`, headers, {
method: "PUT",
body: JSON.stringify({ ids: [targetId] }),
});
}
}
async function copyUserVisibleMetadata(source: ImmichAsset, targetId: string, headers: Record<string, string>): Promise<void> {
const exif = source.exifInfo;
const update: Record<string, unknown> = {
isFavorite: source.isFavorite,
visibility: source.visibility,
};
if (exif?.description != null) update.description = exif.description;
if (exif && "rating" in exif) update.rating = exif.rating ?? null;
if (exif?.dateTimeOriginal) update.dateTimeOriginal = exif.dateTimeOriginal;
if (exif?.latitude != null && exif?.longitude != null) {
update.latitude = exif.latitude;
update.longitude = exif.longitude;
}
await immichJson(`/assets/${targetId}`, headers, { method: "PUT", body: JSON.stringify(update) });
if (exif?.timeZone) {
await immichJson("/assets", headers, {
method: "PUT",
body: JSON.stringify({ ids: [targetId], timeZone: exif.timeZone }),
});
}
}
async function verifyCandidate(
job: CompressionJob,
snapshot: ReplacementSnapshot,
candidateId: string,
checksum: string,
headers: Record<string, string>,
): Promise<void> {
const [candidate, outputStat] = await Promise.all([
getImmichAsset(candidateId, headers),
stat(job.outputPath),
]);
if (candidate.id !== candidateId || candidate.ownerId !== job.userId) throw new Error("Immich returned an invalid replacement asset");
if (candidate.type !== job.mediaType) throw new Error(`Replacement media type changed (${job.mediaType} -> ${candidate.type})`);
if (path.extname(candidate.originalFileName).toLowerCase() !== job.outputExtension.toLowerCase()) {
throw new Error("Replacement filename extension does not match the validated output");
}
if (job.outputSize != null && outputStat.size !== job.outputSize) throw new Error("Compressed output size changed after validation");
if (candidate.exifInfo?.fileSizeInByte != null && candidate.exifInfo.fileSizeInByte !== outputStat.size) {
throw new Error("Replacement size in Immich does not match the compressed output");
}
if (!checksumsEqual(candidate.checksum, checksum)) throw new Error("Replacement checksum does not match the compressed output");
if (!checksumsEqual(await hashImmichOriginal(candidateId, headers), checksum)) {
throw new Error("Downloaded replacement bytes do not match the compressed output");
}
if (candidate.visibility !== snapshot.asset.visibility || candidate.isFavorite !== snapshot.asset.isFavorite) {
throw new Error("Replacement visibility or favorite state was not preserved");
}
const expectedTags = new Set((snapshot.asset.tags ?? []).map((tag) => tag.id));
const actualTags = new Set((candidate.tags ?? []).map((tag) => tag.id));
if (expectedTags.size !== actualTags.size || [...expectedTags].some((id) => !actualTags.has(id))) {
throw new Error("Replacement tags were not preserved exactly");
}
const sourceExif = snapshot.asset.exifInfo;
const candidateExif = candidate.exifInfo;
for (const key of ["description", "rating", "dateTimeOriginal", "latitude", "longitude"] as const) {
if ((sourceExif?.[key] ?? null) !== (candidateExif?.[key] ?? null)) {
throw new Error(`Replacement ${key} metadata was not preserved`);
}
}
const metadata = await getImmichAssetMetadata(candidateId, headers);
const marker = metadata.find((item) => item.key === METADATA_KEY)?.value;
if (marker?.compressionJobId !== job.id || marker?.sourceAssetId !== job.assetId) {
throw new Error("Replacement provenance metadata is missing");
}
for (const expected of snapshot.metadata.filter((item) => item.key !== METADATA_KEY)) {
const actual = metadata.find((item) => item.key === expected.key);
if (!actual || !metadataValueEqual(actual.value, expected.value)) {
throw new Error(`Replacement metadata key ${expected.key} was not preserved`);
}
}
}
async function sourceDisposalState(assetId: string): Promise<"active" | "disposed" | "unknown"> {
const [source] = await db.select({ status: assets.status, deletedAt: assets.deletedAt })
.from(assets)
.where(eq(assets.id, assetId));
if (!source) return "unknown";
if (source.deletedAt || ["trashed", "deleted"].includes(source.status)) return "disposed";
return source.status === "active" ? "active" : "unknown";
}
async function completeReplacement(job: CompressionJob): Promise<void> {
const now = new Date();
const [completed] = await appDb.update(compressionJobs).set({
status: "COMPLETED",
replacementStage: "ORIGINAL_TRASHED",
replacedAt: now,
finishedAt: now,
heartbeatAt: null,
errorMessage: null,
}).where(and(
eq(compressionJobs.id, job.id),
inArray(compressionJobs.status, ["REPLACING", "NEEDS_ATTENTION"]),
inArray(compressionJobs.replacementStage, ["TRASH_REQUESTED", "ORIGINAL_TRASHED"]),
)).returning({ id: compressionJobs.id });
if (!completed) {
const [current] = await appDb.select({ status: compressionJobs.status }).from(compressionJobs).where(eq(compressionJobs.id, job.id));
if (current?.status !== "COMPLETED") throw new Error("Replacement completion state changed unexpectedly");
}
await rm(path.dirname(job.inputPath), { recursive: true, force: true }).catch((error) => {
console.error(`Replacement ${job.id} completed but temporary files could not be removed`, error);
});
}
async function rollbackCandidate(candidateId: string | null, headers: Record<string, string>): Promise<boolean> {
if (!candidateId) return true;
try {
await immichJson("/assets", headers, {
method: "DELETE",
body: JSON.stringify({ ids: [candidateId], force: true }),
});
return true;
} catch (error) {
console.error(`Failed to roll back replacement asset ${candidateId}`, error);
return false;
}
}
async function setNeedsAttention(job: CompressionJob, message: string, candidateId = job.replacementAssetId): Promise<void> {
await appDb.update(compressionJobs).set({
status: "NEEDS_ATTENTION",
replacementAssetId: candidateId,
heartbeatAt: null,
errorMessage: message,
finishedAt: new Date(),
}).where(and(eq(compressionJobs.id, job.id), inArray(compressionJobs.status, ["REPLACING", "NEEDS_ATTENTION"])));
}
async function handleReplacementFailure(jobId: string, error: unknown, headers: Record<string, string>): Promise<void> {
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, jobId));
if (!job) return;
const message = error instanceof Error ? error.message : String(error);
const stage = job.replacementStage as ReplacementStage | null;
const disposalState = await sourceDisposalState(job.assetId).catch(() => "unknown" as const);
if (stage === "TRASH_REQUESTED" || stage === "ORIGINAL_TRASHED") {
if (disposalState === "disposed") {
await completeReplacement(job);
return;
}
await setNeedsAttention(
job,
`Original trashing has an ambiguous result. The verified replacement was kept and will not be deleted automatically. Reconcile the job after checking Immich. ${message}`,
);
return;
}
let candidateId = job.replacementAssetId;
if (!candidateId && job.outputChecksum && (stage === "UPLOAD_REQUESTED" || (stage != null && CANDIDATE_STAGES.includes(stage)))) {
candidateId = await findCandidateByChecksum(job, job.outputChecksum, headers).catch(() => null);
}
if (!candidateId && stage != null && CANDIDATE_STAGES.includes(stage)) {
await setNeedsAttention(
job,
`Replacement state says an uploaded candidate exists, but its asset ID could not be recovered safely. The job was not reset or discarded. ${message}`,
null,
);
return;
}
if (disposalState !== "active") {
await setNeedsAttention(
job,
`The original asset is not provably active, so the replacement candidate was kept to avoid data loss. Reconcile the job in Immich. ${message}`,
candidateId,
);
return;
}
const rollbackSucceeded = await rollbackCandidate(candidateId, headers);
await appDb.update(compressionJobs).set(rollbackSucceeded ? {
status: "READY",
replacementAssetId: null,
replacementStage: null,
heartbeatAt: null,
errorMessage: `Replacement stopped before the original was touched. ${message}`,
finishedAt: null,
} : {
status: "NEEDS_ATTENTION",
replacementAssetId: candidateId,
heartbeatAt: null,
errorMessage: `Replacement failed and candidate ${candidateId ?? "unknown"} could not be removed. Resolve it in Immich before continuing. ${message}`,
finishedAt: new Date(),
}).where(and(eq(compressionJobs.id, job.id), eq(compressionJobs.status, "REPLACING")));
}
async function finishTrash(job: CompressionJob, headers: Record<string, string>): Promise<void> {
let state = await sourceDisposalState(job.assetId);
if (state === "disposed") {
const updated = await advanceStage(job.id, "TRASH_REQUESTED", "ORIGINAL_TRASHED");
await completeReplacement(updated);
return;
}
if (state !== "active") throw new Error("Could not determine whether the original asset was already trashed");
let deleteError: unknown;
try {
await immichJson("/assets", headers, {
method: "DELETE",
body: JSON.stringify({ ids: [job.assetId], force: false }),
});
} catch (error) {
deleteError = error;
}
state = await sourceDisposalState(job.assetId);
if (state === "disposed") {
const updated = await advanceStage(job.id, "TRASH_REQUESTED", "ORIGINAL_TRASHED");
await completeReplacement(updated);
return;
}
if (deleteError) throw deleteError;
if (state === "unknown") throw new Error("Immich returned from trashing the original, but its state is unknown");
throw new Error("Immich did not confirm that the original was trashed");
}
async function continueReplacement(jobId: string, headers: Record<string, string>): Promise<void> {
let [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, jobId));
if (!job || job.status !== "REPLACING") return;
let stage = job.replacementStage as ReplacementStage | null;
if (stage === "ORIGINAL_TRASHED") {
await completeReplacement(job);
return;
}
if (stage === "TRASH_REQUESTED") {
await finishTrash(job, headers);
return;
}
if (!stage) throw new Error("Replacement stage is missing");
let snapshot: ReplacementSnapshot;
if (stage === "STARTED") {
snapshot = await loadSourceSnapshot(job, headers);
const outputStat = await stat(job.outputPath);
const outputChecksum = await sha1File(job.outputPath);
if (job.outputSize != null && job.outputSize !== outputStat.size) throw new Error("Compressed output size changed after validation");
if (job.outputChecksum && !checksumsEqual(job.outputChecksum, outputChecksum)) throw new Error("Compressed output changed after validation");
job = await advanceStage(job.id, "STARTED", "SOURCE_VERIFIED", {
sourceMetadata: JSON.stringify(snapshot),
outputSize: outputStat.size,
outputChecksum,
});
stage = "SOURCE_VERIFIED";
} else {
snapshot = readSourceSnapshot(job);
}
const checksum = job.outputChecksum;
if (!checksum) throw new Error("Compressed output checksum is missing");
await assertSourceUnchanged(job, snapshot, headers);
let candidateId = job.replacementAssetId;
if (!candidateId && ["SOURCE_VERIFIED", "UPLOAD_REQUESTED"].includes(stage)) {
candidateId = await findCandidateByChecksum(job, checksum, headers);
if (!candidateId) {
await assertNoUnrelatedDuplicate(job, checksum, headers);
if (stage === "SOURCE_VERIFIED") {
job = await advanceStage(job.id, "SOURCE_VERIFIED", "UPLOAD_REQUESTED");
stage = "UPLOAD_REQUESTED";
}
try {
candidateId = await uploadCandidate(job, snapshot, checksum, headers);
} catch (error) {
candidateId = await findCandidateByChecksum(job, checksum, headers);
if (!candidateId) throw error;
}
}
job = await advanceStage(job.id, ["SOURCE_VERIFIED", "UPLOAD_REQUESTED"], "UPLOADED", {
replacementAssetId: candidateId,
});
stage = "UPLOADED";
}
candidateId = job.replacementAssetId ?? candidateId;
if (!candidateId && CANDIDATE_STAGES.includes(stage)) {
candidateId = await findCandidateByChecksum(job, checksum, headers);
if (candidateId) {
const [recovered] = await appDb.update(compressionJobs).set({
replacementAssetId: candidateId,
heartbeatAt: new Date(),
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.status, "REPLACING"),
eq(compressionJobs.replacementStage, stage),
isNull(compressionJobs.replacementAssetId),
)).returning();
if (!recovered) throw new Error("Replacement state changed while recovering the uploaded candidate");
job = recovered;
}
}
if (!candidateId) throw new Error("Replacement candidate is missing from durable state");
if (stage === "UPLOADED") {
await immichJson("/assets/copy", headers, {
method: "PUT",
body: JSON.stringify({
sourceId: job.assetId,
targetId: candidateId,
albums: true,
sharedLinks: true,
sidecar: true,
favorite: true,
stack: false,
}),
});
job = await advanceStage(job.id, "UPLOADED", "RELATIONSHIPS_COPIED");
stage = "RELATIONSHIPS_COPIED";
}
if (stage === "RELATIONSHIPS_COPIED") {
await ensureTags(snapshot.asset, candidateId, headers);
job = await advanceStage(job.id, "RELATIONSHIPS_COPIED", "TAGS_COPIED");
stage = "TAGS_COPIED";
}
if (stage === "TAGS_COPIED") {
await copyUserVisibleMetadata(snapshot.asset, candidateId, headers);
job = await advanceStage(job.id, "TAGS_COPIED", "METADATA_COPIED");
stage = "METADATA_COPIED";
}
if (stage === "METADATA_COPIED") {
await verifyCandidate(job, snapshot, candidateId, checksum, headers);
job = await advanceStage(job.id, "METADATA_COPIED", "VERIFIED");
stage = "VERIFIED";
}
if (stage === "VERIFIED") {
await assertSourceUnchanged(job, snapshot, headers);
job = await advanceStage(job.id, "VERIFIED", "TRASH_REQUESTED");
stage = "TRASH_REQUESTED";
}
if (stage === "TRASH_REQUESTED") await finishTrash(job, headers);
}
export async function replaceCompressionJob(jobId: string, userId: string, headers: Record<string, string>): Promise<void> {
let [job] = await appDb.update(compressionJobs).set({
status: "REPLACING",
replacementStage: "STARTED",
replacementAssetId: null,
heartbeatAt: new Date(),
errorMessage: null,
finishedAt: null,
}).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.userId, userId),
eq(compressionJobs.status, "READY"),
)).returning();
if (!job) {
[job] = await appDb.update(compressionJobs).set({
status: "REPLACING",
heartbeatAt: new Date(),
errorMessage: null,
finishedAt: null,
}).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.userId, userId),
eq(compressionJobs.status, "NEEDS_ATTENTION"),
)).returning();
}
if (!job) throw new Error("Job is no longer ready for replacement or reconciliation");
try {
await continueReplacement(job.id, headers);
} catch (error) {
await handleReplacementFailure(job.id, error, headers);
throw error;
}
}
export async function resumeReplacementJobs(userId: string, headers: Record<string, string>): Promise<void> {
const staleBefore = new Date(Date.now() - STALE_JOB_MS);
const jobs = await appDb.select({ id: compressionJobs.id }).from(compressionJobs).where(and(
eq(compressionJobs.userId, userId),
eq(compressionJobs.status, "REPLACING"),
or(isNull(compressionJobs.heartbeatAt), lt(compressionJobs.heartbeatAt, staleBefore)),
));
for (const job of jobs) {
const [claimed] = await appDb.update(compressionJobs).set({ heartbeatAt: new Date() })
.where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.status, "REPLACING"),
or(isNull(compressionJobs.heartbeatAt), lt(compressionJobs.heartbeatAt, staleBefore)),
))
.returning({ id: compressionJobs.id });
if (!claimed) continue;
try {
await continueReplacement(job.id, headers);
} catch (error) {
await handleReplacementFailure(job.id, error, headers);
}
}
}
+607
View File
@@ -0,0 +1,607 @@
import { ChildProcessWithoutNullStreams, spawn } from "child_process";
import { createWriteStream } from "fs";
import { mkdir, readFile, rename, rm, stat, unlink } from "fs/promises";
import path from "path";
import { Readable } from "stream";
import { pipeline } from "stream/promises";
import { and, asc, eq, inArray, isNull, lt, or, sql } from "drizzle-orm";
import { appDb } from "@/db";
import { compressionJobs } from "@/db/schema";
import { buildFfmpegArguments, type CompressionSettings } from "@/lib/compression/arguments";
import {
AUTH_CACHE_TTL_MS,
CANCEL_GRACE_MS,
COMPRESSION_TIMEOUT_MS,
MAX_LOG_BYTES,
STALE_JOB_MS,
} from "@/lib/compression/constants";
import { checksumsEqual, sha1File } from "@/lib/compression/files";
import { ffprobe, mediaDurationSeconds, validateProbePair, verifyMediaDecodes } from "@/lib/compression/ffprobe";
import {
getImmichAsset,
getImmichAssetUnsupportedReason,
immichRequest,
type ImmichAsset,
} from "@/lib/compression/immich";
import { isOutputExtensionCompatible } from "@/lib/compression/mime";
export type CompressionJobStatus =
| "QUEUED"
| "DOWNLOADING"
| "COMPRESSING"
| "VALIDATING"
| "READY"
| "REPLACING"
| "CANCELLING"
| "RETRYING"
| "NEEDS_ATTENTION"
| "COMPLETED"
| "FAILED"
| "CANCELLED"
| "NO_SAVING";
type CompressionJob = typeof compressionJobs.$inferSelect;
interface ActiveAttempt {
attempt: number;
controller: AbortController;
child?: ChildProcessWithoutNullStreams;
done: Promise<void>;
resolveDone: () => void;
}
const authByUser = new Map<string, { headers: Record<string, string>; expiresAt: number }>();
const activeAttempts = new Map<string, ActiveAttempt>();
let runnerActive = false;
let runnerRequested = false;
function safeMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
async function existsAndNonZero(filePath: string): Promise<boolean> {
try {
return (await stat(filePath)).size > 0;
} catch {
return false;
}
}
async function cleanupPartialOutput(outputPath: string): Promise<void> {
await unlink(outputPath).catch(() => undefined);
}
async function cleanupAttemptFiles(job: CompressionJob, removeInput: boolean): Promise<void> {
const paths = [`${job.inputPath}.part`, job.outputPath];
if (removeInput) paths.push(job.inputPath);
await Promise.all(paths.map((filePath) => unlink(filePath).catch(() => undefined)));
}
class BoundedLogWriter {
private written = 0;
private truncated = false;
private streamError: Error | null = null;
private stream: ReturnType<typeof createWriteStream>;
constructor(filePath: string) {
this.stream = createWriteStream(filePath, { flags: "w" });
this.stream.on("error", (error) => { this.streamError = error; });
}
write(data: Buffer | string) {
if (this.truncated || this.streamError) return;
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data);
const remaining = MAX_LOG_BYTES - this.written;
if (remaining <= 0) {
this.stream.write("\n[Power Tools: FFmpeg log truncated at 1 MiB]\n");
this.truncated = true;
return;
}
const chunk = buffer.subarray(0, remaining);
this.stream.write(chunk);
this.written += chunk.length;
if (chunk.length < buffer.length) {
this.stream.write("\n[Power Tools: FFmpeg log truncated at 1 MiB]\n");
this.truncated = true;
}
}
async close() {
if (!this.stream.closed && !this.stream.destroyed) {
await new Promise<void>((resolve) => this.stream.end(resolve));
}
if (this.streamError) throw this.streamError;
}
}
async function getJob(jobId: string) {
const [job] = await appDb.select().from(compressionJobs).where(eq(compressionJobs.id, jobId));
return job;
}
async function isAttemptInStatus(job: CompressionJob, statuses: string[]): Promise<boolean> {
const [current] = await appDb.select({ status: compressionJobs.status })
.from(compressionJobs)
.where(and(eq(compressionJobs.id, job.id), eq(compressionJobs.attempt, job.attempt)));
return Boolean(current && statuses.includes(current.status));
}
async function storeProgress(job: CompressionJob, status: string, values: { progress?: number; speed?: string }) {
await appDb.update(compressionJobs).set({ ...values, heartbeatAt: new Date() }).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
eq(compressionJobs.status, status),
)).catch((error) => console.error(`Failed to store compression progress for ${job.id}`, error));
}
async function verifyReusableInput(job: CompressionJob, source: ImmichAsset): Promise<boolean> {
if (!(await existsAndNonZero(job.inputPath)) || !source.checksum) return false;
const inputStat = await stat(job.inputPath);
const expectedSize = source.exifInfo?.fileSizeInByte;
if (expectedSize && inputStat.size !== expectedSize) return false;
return checksumsEqual(await sha1File(job.inputPath), source.checksum);
}
async function downloadOriginal(
job: CompressionJob,
sourceAsset: ImmichAsset,
headers: Record<string, string>,
signal: AbortSignal,
) {
if (await verifyReusableInput(job, sourceAsset)) return;
await Promise.all([
unlink(job.inputPath).catch(() => undefined),
unlink(`${job.inputPath}.part`).catch(() => undefined),
]);
await mkdir(path.dirname(job.inputPath), { recursive: true });
const response = await immichRequest(`/assets/${job.assetId}/original`, headers, { signal });
if (!response.body) throw new Error("Immich returned an empty original asset response");
const contentLength = Number(response.headers.get("content-length") || 0);
const expectedSize = contentLength || sourceAsset.exifInfo?.fileSizeInByte || 0;
let downloaded = 0;
let lastStoredProgress = -1;
const source = Readable.fromWeb(response.body as any);
source.on("data", (chunk: Buffer) => {
downloaded += chunk.length;
if (expectedSize > 0) {
const progress = Math.min(99, Math.round((downloaded / expectedSize) * 100));
if (progress !== lastStoredProgress) {
lastStoredProgress = progress;
void storeProgress(job, "DOWNLOADING", { progress });
}
}
});
try {
await pipeline(source, createWriteStream(`${job.inputPath}.part`), { signal });
const downloadedStat = await stat(`${job.inputPath}.part`);
if (downloadedStat.size <= 0) throw new Error("Downloaded original is empty");
if (expectedSize > 0 && downloadedStat.size !== expectedSize) {
throw new Error(`Downloaded original size mismatch (${downloadedStat.size} != ${expectedSize})`);
}
const checksum = await sha1File(`${job.inputPath}.part`);
if (!checksumsEqual(checksum, sourceAsset.checksum)) {
throw new Error("Downloaded original checksum does not match Immich");
}
await rename(`${job.inputPath}.part`, job.inputPath);
const recorded = await appDb.update(compressionJobs).set({
originalSize: downloadedStat.size,
originalChecksum: checksum,
heartbeatAt: new Date(),
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
eq(compressionJobs.status, "DOWNLOADING"),
)).returning({ id: compressionJobs.id });
if (recorded.length !== 1) {
await unlink(job.inputPath).catch(() => undefined);
throw new Error("Compression attempt changed while the original was downloading");
}
} catch (error) {
await unlink(`${job.inputPath}.part`).catch(() => undefined);
throw error;
}
}
async function transitionToCompressing(job: CompressionJob): Promise<void> {
const updated = await appDb.update(compressionJobs).set({
status: "COMPRESSING",
progress: 0,
errorMessage: null,
heartbeatAt: new Date(),
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
eq(compressionJobs.status, "DOWNLOADING"),
)).returning({ id: compressionJobs.id });
if (updated.length !== 1) throw new Error("Compression attempt is no longer active");
}
async function runFfmpeg(job: CompressionJob, inputDuration: number | null, active: ActiveAttempt): Promise<number> {
await cleanupPartialOutput(job.outputPath);
await transitionToCompressing(job);
const configuredArgs = buildFfmpegArguments(job.ffmpegArguments, job.inputPath, job.outputPath);
const args = ["-progress", "pipe:1", "-nostats", ...configuredArgs];
const log = new BoundedLogWriter(job.logPath);
return new Promise<number>((resolve, reject) => {
const child = spawn("ffmpeg", args, { shell: false });
active.child = child;
let stdoutBuffer = "";
let settled = false;
let killTimer: NodeJS.Timeout | undefined;
let timedOut = false;
const abort = () => {
if (child.exitCode !== null || child.killed) return;
child.kill("SIGTERM");
killTimer = setTimeout(() => {
if (child.exitCode === null) child.kill("SIGKILL");
}, CANCEL_GRACE_MS);
killTimer.unref();
};
const compressionTimer = setTimeout(() => {
timedOut = true;
abort();
}, COMPRESSION_TIMEOUT_MS);
compressionTimer.unref();
active.controller.signal.addEventListener("abort", abort, { once: true });
if (active.controller.signal.aborted) abort();
child.stdout.on("data", (chunk: Buffer) => {
log.write(chunk);
stdoutBuffer += chunk.toString();
const lines = stdoutBuffer.split(/\r?\n/);
stdoutBuffer = lines.pop() ?? "";
for (const line of lines) {
const separator = line.indexOf("=");
if (separator < 0) continue;
const key = line.slice(0, separator);
const value = line.slice(separator + 1);
if (key === "speed") void storeProgress(job, "COMPRESSING", { speed: value });
if ((key === "out_time_ms" || key === "out_time_us") && inputDuration && inputDuration > 0) {
const outMicroseconds = Number(value);
if (Number.isFinite(outMicroseconds)) {
const progress = Math.max(0, Math.min(99, Math.round((outMicroseconds / 1_000_000 / inputDuration) * 100)));
void storeProgress(job, "COMPRESSING", { progress });
}
}
}
});
child.stderr.on("data", (chunk: Buffer) => log.write(chunk));
child.on("error", async (error) => {
if (settled) return;
settled = true;
active.controller.signal.removeEventListener("abort", abort);
clearTimeout(compressionTimer);
if (killTimer) clearTimeout(killTimer);
active.child = undefined;
await log.close().catch(() => undefined);
reject(error);
});
child.on("close", async (code, signal) => {
if (settled) return;
settled = true;
active.controller.signal.removeEventListener("abort", abort);
clearTimeout(compressionTimer);
if (killTimer) clearTimeout(killTimer);
active.child = undefined;
try {
await log.close();
} catch (error) {
reject(error);
return;
}
if (timedOut) {
reject(new Error("FFmpeg exceeded the 24-hour runtime limit"));
} else if (active.controller.signal.aborted) {
resolve(code ?? 143);
} else if (code !== 0) {
reject(new Error(`FFmpeg exited with code ${code ?? "unknown"}${signal ? ` (${signal})` : ""}`));
} else {
resolve(code ?? 0);
}
});
});
}
async function validateCompressionJob(job: CompressionJob, signal: AbortSignal): Promise<void> {
if (!(await existsAndNonZero(job.inputPath))) throw new Error("Original working file is missing");
if (!(await existsAndNonZero(job.outputPath))) throw new Error("Compressed output is missing or empty");
if (!isOutputExtensionCompatible(job.mediaType, job.outputExtension)) {
throw new Error(`${job.outputExtension} is not a valid output extension for ${job.mediaType.toLowerCase()} assets`);
}
const [inputStat, outputStat, inputProbe, outputProbe, outputChecksum] = await Promise.all([
stat(job.inputPath),
stat(job.outputPath),
ffprobe(job.inputPath, signal),
ffprobe(job.outputPath, signal),
sha1File(job.outputPath),
]);
const validationErrors = validateProbePair(inputProbe, outputProbe, job.mediaType, job.outputExtension);
if (validationErrors.length > 0) throw new Error(`Validation failed: ${validationErrors.join("; ")}`);
await verifyMediaDecodes(job.outputPath, signal);
const nextStatus = outputStat.size >= inputStat.size ? "NO_SAVING" : "READY";
const updated = await appDb.update(compressionJobs).set({
status: nextStatus,
outputSize: outputStat.size,
outputChecksum,
originalSize: inputStat.size,
progress: 100,
heartbeatAt: null,
finishedAt: new Date(),
errorMessage: nextStatus === "NO_SAVING" ? "Compressed output is not smaller than the original" : null,
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
eq(compressionJobs.status, "VALIDATING"),
)).returning({ id: compressionJobs.id });
if (updated.length !== 1) throw new Error("Compression attempt was cancelled during validation");
}
async function processJob(job: CompressionJob, headers: Record<string, string>): Promise<void> {
let resolveDone!: () => void;
const active: ActiveAttempt = {
attempt: job.attempt,
controller: new AbortController(),
done: new Promise<void>((resolve) => { resolveDone = resolve; }),
resolveDone: () => resolveDone(),
};
activeAttempts.set(job.id, active);
const heartbeat = setInterval(() => {
void appDb.update(compressionJobs).set({ heartbeatAt: new Date() }).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
inArray(compressionJobs.status, ["DOWNLOADING", "COMPRESSING", "VALIDATING"]),
)).catch(() => undefined);
}, 15_000);
heartbeat.unref();
try {
if (!(await isAttemptInStatus(job, ["DOWNLOADING"]))) return;
const sourceAsset = await getImmichAsset(job.assetId, headers);
const unsupportedReason = getImmichAssetUnsupportedReason(sourceAsset, job.userId, job.mediaType);
if (unsupportedReason) throw new Error(unsupportedReason);
await downloadOriginal(job, sourceAsset, headers, active.controller.signal);
if (!(await isAttemptInStatus(job, ["DOWNLOADING"]))) return;
const inputProbe = await ffprobe(job.inputPath, active.controller.signal);
const exitCode = await runFfmpeg(job, mediaDurationSeconds(inputProbe), active);
if (!(await isAttemptInStatus(job, ["COMPRESSING"]))) return;
const validating = await appDb.update(compressionJobs).set({
status: "VALIDATING",
progress: 100,
ffmpegExitCode: exitCode,
heartbeatAt: new Date(),
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
eq(compressionJobs.status, "COMPRESSING"),
)).returning({ id: compressionJobs.id });
if (validating.length !== 1) return;
await validateCompressionJob(job, active.controller.signal);
} catch (error) {
if (await isAttemptInStatus(job, ["CANCELLING", "CANCELLED"])) return;
await cleanupPartialOutput(job.outputPath);
await appDb.update(compressionJobs).set({
status: "FAILED",
errorMessage: safeMessage(error),
heartbeatAt: null,
finishedAt: new Date(),
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
inArray(compressionJobs.status, ["DOWNLOADING", "COMPRESSING", "VALIDATING"]),
));
} finally {
clearInterval(heartbeat);
if (activeAttempts.get(job.id) === active) activeAttempts.delete(job.id);
active.resolveDone();
}
}
async function claimNextJob(userIds: string[]): Promise<CompressionJob | undefined> {
const candidate = appDb.select({ id: compressionJobs.id })
.from(compressionJobs)
.where(and(eq(compressionJobs.status, "QUEUED"), inArray(compressionJobs.userId, userIds)))
.orderBy(asc(compressionJobs.createdAt), asc(compressionJobs.id))
.limit(1);
const [claimed] = await appDb.update(compressionJobs).set({
status: "DOWNLOADING",
attempt: sql`${compressionJobs.attempt} + 1`,
progress: 0,
speed: null,
startedAt: new Date(),
finishedAt: null,
heartbeatAt: new Date(),
errorMessage: null,
}).where(and(
eq(compressionJobs.status, "QUEUED"),
inArray(compressionJobs.id, candidate),
)).returning();
return claimed;
}
async function runQueueLoop(): Promise<void> {
if (runnerActive) {
runnerRequested = true;
return;
}
runnerActive = true;
runnerRequested = false;
try {
while (true) {
const now = Date.now();
for (const [userId, auth] of authByUser) {
if (auth.expiresAt <= now) authByUser.delete(userId);
}
const userIds = [...authByUser.keys()];
if (userIds.length === 0) break;
const job = await claimNextJob(userIds);
if (!job) break;
const auth = authByUser.get(job.userId);
if (!auth) {
await appDb.update(compressionJobs).set({ status: "QUEUED", heartbeatAt: null })
.where(and(eq(compressionJobs.id, job.id), eq(compressionJobs.attempt, job.attempt), eq(compressionJobs.status, "DOWNLOADING")));
continue;
}
await processJob(job, auth.headers);
}
} finally {
runnerActive = false;
if (runnerRequested) void runQueueLoop().catch((error) => console.error("Compression runner failed", error));
}
}
export function registerCompressionAuth(userId: string, headers: Record<string, string>): void {
authByUser.set(userId, { headers, expiresAt: Date.now() + AUTH_CACHE_TTL_MS });
runnerRequested = true;
void import("./replacement")
.then(({ resumeReplacementJobs }) => resumeReplacementJobs(userId, headers))
.catch((error) => console.error("Compression replacement recovery failed", error));
void runQueueLoop().catch((error) => console.error("Compression runner failed", error));
}
export async function cancelCompressionJob(jobId: string, userId: string, expectedAttempt: number): Promise<void> {
const [job] = await appDb.update(compressionJobs).set({
status: "CANCELLING",
heartbeatAt: new Date(),
}).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.userId, userId),
eq(compressionJobs.attempt, expectedAttempt),
inArray(compressionJobs.status, ["QUEUED", "DOWNLOADING", "COMPRESSING", "VALIDATING"]),
)).returning();
if (!job) throw new Error("Job is no longer cancellable");
const active = activeAttempts.get(jobId);
if (active?.attempt === expectedAttempt) {
active.controller.abort();
await Promise.race([
active.done,
new Promise<void>((resolve) => setTimeout(resolve, CANCEL_GRACE_MS * 2)),
]);
}
await cleanupAttemptFiles(job, true);
const cancelled = await appDb.update(compressionJobs).set({
status: "CANCELLED",
progress: 0,
speed: null,
heartbeatAt: null,
finishedAt: new Date(),
errorMessage: null,
}).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.attempt, expectedAttempt),
eq(compressionJobs.status, "CANCELLING"),
)).returning({ id: compressionJobs.id });
if (cancelled.length !== 1) throw new Error("Job cancellation did not complete cleanly");
}
export async function retryCompressionJob(
jobId: string,
userId: string,
expectedAttempt: number,
settings: CompressionSettings,
): Promise<void> {
const [job] = await appDb.update(compressionJobs).set({ status: "RETRYING", heartbeatAt: new Date() }).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.userId, userId),
eq(compressionJobs.attempt, expectedAttempt),
inArray(compressionJobs.status, ["FAILED", "CANCELLED", "NO_SAVING"]),
)).returning();
if (!job) throw new Error("Job is no longer retryable");
if (job.replacementAssetId || job.replacementStage) {
await appDb.update(compressionJobs).set({ status: "NEEDS_ATTENTION", heartbeatAt: null })
.where(and(eq(compressionJobs.id, job.id), eq(compressionJobs.status, "RETRYING")));
throw new Error("This job has unresolved replacement state and cannot be recompressed");
}
if (
job.status === "RETRYING"
&& job.errorMessage === "Compressed output is not smaller than the original"
&& job.ffmpegArguments === settings.ffmpegArguments
&& job.outputExtension === settings.outputExtension
) {
await appDb.update(compressionJobs).set({ status: "NO_SAVING", heartbeatAt: null }).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, expectedAttempt),
eq(compressionJobs.status, "RETRYING"),
));
throw new Error("Change the FFmpeg settings before retrying a job that produced no space saving");
}
const outputPath = path.join(path.dirname(job.outputPath), `output${settings.outputExtension}`);
await cleanupPartialOutput(job.outputPath);
if (outputPath !== job.outputPath) await cleanupPartialOutput(outputPath);
const queued = await appDb.update(compressionJobs).set({
status: "QUEUED",
outputPath,
outputSize: null,
outputChecksum: null,
ffmpegArguments: settings.ffmpegArguments,
outputExtension: settings.outputExtension,
progress: 0,
speed: null,
ffmpegExitCode: null,
errorMessage: null,
startedAt: null,
finishedAt: null,
heartbeatAt: null,
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, expectedAttempt),
eq(compressionJobs.status, "RETRYING"),
)).returning({ id: compressionJobs.id });
if (queued.length !== 1) throw new Error("Job retry did not complete cleanly");
runnerRequested = true;
void runQueueLoop().catch((error) => console.error("Compression retry runner failed", error));
}
export async function discardCompressionJob(jobId: string, userId: string): Promise<void> {
const [job] = await appDb.delete(compressionJobs).where(and(
eq(compressionJobs.id, jobId),
eq(compressionJobs.userId, userId),
inArray(compressionJobs.status, ["FAILED", "CANCELLED", "NO_SAVING", "READY", "COMPLETED"]),
)).returning();
if (!job) throw new Error("Job is active or requires replacement recovery and cannot be discarded");
await rm(path.dirname(job.inputPath), { recursive: true, force: true });
}
export async function readCompressionLog(jobId: string): Promise<string> {
const job = await getJob(jobId);
if (!job) throw new Error("Job not found");
return readFile(job.logPath, "utf8").catch(() => "No FFmpeg log is available for this job.");
}
export async function resumeCompressionJobs(): Promise<void> {
const staleBefore = new Date(Date.now() - STALE_JOB_MS);
const staleJobs = await appDb.select().from(compressionJobs).where(and(
inArray(compressionJobs.status, ["DOWNLOADING", "COMPRESSING", "VALIDATING", "CANCELLING", "RETRYING"]),
or(isNull(compressionJobs.heartbeatAt), lt(compressionJobs.heartbeatAt, staleBefore)),
));
for (const job of staleJobs) {
const wasCancelling = job.status === "CANCELLING";
await cleanupAttemptFiles(job, wasCancelling || job.status === "DOWNLOADING");
await appDb.update(compressionJobs).set({
status: wasCancelling ? "CANCELLED" : "QUEUED",
progress: 0,
speed: null,
heartbeatAt: null,
finishedAt: wasCancelling ? new Date() : null,
errorMessage: wasCancelling ? null : "Recovered after an interrupted compression attempt",
}).where(and(
eq(compressionJobs.id, job.id),
eq(compressionJobs.attempt, job.attempt),
eq(compressionJobs.status, job.status),
));
}
}