jfmcarreira/immich-power-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integrate OpenAI-compatible API for query parsing and remove Gemini references

Browse Source
This commit is contained in:
sai80082
2026-02-18 09:52:08 +05:30
parent 3f30e71b95
commit 774a0c480b
11 changed files with 130 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.sample
+3 -1
View File
@@ -10,7 +10,9 @@ DB_DATABASE_NAME="" # Name of the database
SECURE_COOKIE=false # Set to true to enable secure cookies
# Optional
GEMINI_API_KEY="" # Gemini API Key for parsing search query in "Find"
AI_API_KEY="" # API key for your OpenAI-compatible provider used by "Find"
AI_BASE_URL="https://api.openai.com/v1" # Base URL for OpenAI-compatible API
AI_MODEL="gpt-4o-mini" # Model used for parsing search queries in "Find"
# Immich Share Link
IMMICH_SHARE_LINK_KEY="" # Share link key for Immich
README.md
+6 -4
View File
@@ -58,7 +58,9 @@ DB_PORT = "5432"
DB_HOST = "immich_postgres"
# Optional
GEMINI_API_KEY= # Gemini API Key
AI_API_KEY= # API key for your OpenAI-compatible provider
AI_BASE_URL=https://api.openai.com/v1 # Base URL for OpenAI-compatible API
AI_MODEL=gpt-4o-mini # Model used to parse search queries in Find
```
Refer here for obtaining Immich API Key: https://immich.app/docs/features/command-line-interface#obtain-the-api-key
@@ -145,10 +147,10 @@ Google Maps Javascript API Key is used to render the maps layer for heatmap. Whe
> Code where heatmap data is plotted: [src/pages/assets/geo-heatmap.tsx:32](./src/pages/assets/geo-heatmap.tsx#L32-L35)
- [Gemini](https://gemini.google.com/) for rewind video generation
Google Gemini 1.5 Flash model is used for parsing your search query in "Find" page. We do not send any personal data to Gemini, only data that is sent to Gemini is your search query and rest of the querying happes using [Immich's Smart Search API](https://immich.app/docs/api/search-smart)
- OpenAI-compatible AI provider for smart query parsing in Find
Any OpenAI-compatible model can be used for parsing your search query in the "Find" page. We do not send any personal library data to the AI provider; only your search text is sent for parsing, and the actual asset querying uses [Immich's Smart Search API](https://immich.app/docs/api/search-smart)
> Code where Gemini is used: [src/helpers/gemini.helper.ts](./src/helpers/gemini.helper.ts)
> Code where AI parsing is used: [src/helpers/ai.helper.ts](./src/helpers/ai.helper.ts)
## Contributing
bun.lockb
BIN
View File
Binary file not shown.
package.json
+5 -2
View File
@@ -14,7 +14,9 @@
"url": "https://saicharang.com"
},
"dependencies": {
"@google/generative-ai": "^0.21.0",
"@ai-sdk/openai": "1.3.24",
"@ai-sdk/openai-compatible": "^2.0.30",
"@ai-sdk/react": "^3.0.88",
"@radix-ui/react-accordion": "^1.2.1",
"@radix-ui/react-alert-dialog": "^1.1.4",
"@radix-ui/react-avatar": "^1.1.0",
@@ -42,6 +44,7 @@
"@types/react-autosuggest": "^10.1.11",
"@types/react-calendar-heatmap": "^1.6.7",
"@types/react-window": "^1.8.8",
"ai": "^6.0.86",
"axios": "^1.7.4",
"chrono-node": "^1.4.9",
"class-variance-authority": "^0.7.0",
@@ -76,7 +79,7 @@
"tailwind-merge": "^2.5.2",
"tailwindcss-animate": "^1.0.7",
"yet-another-react-lightbox": "^3.21.4",
"zod": "^3.24.1"
"zod": "^4.3.6"
},
"devDependencies": {
"@semantic-release/commit-analyzer": "^13.0.0",
src/config/environment.ts
+3 -1
View File
@@ -12,7 +12,9 @@ export const ENV = {
JWT_SECRET: process.env.JWT_SECRET as string,
SECURE_COOKIE: process.env.SECURE_COOKIE === 'true',
VERSION: process.env.VERSION,
GEMINI_API_KEY: process.env.GEMINI_API_KEY as string,
AI_API_KEY: process.env.AI_API_KEY as string,
AI_BASE_URL: (process.env.AI_BASE_URL || 'https://api.openai.com/v1') as string,
AI_MODEL: process.env.AI_MODEL as string,
IMMICH_SHARE_LINK_KEY: process.env.IMMICH_SHARE_LINK_KEY as string,
POWER_TOOLS_ENDPOINT_URL: process.env.POWER_TOOLS_ENDPOINT_URL as string,
// OAuth configuration
src/contexts/ConfigContext.tsx
+2 -2
View File
@@ -4,7 +4,7 @@ export interface ConfigContextType {
immichURL: string;
exImmichUrl: string;
version?: string;
geminiEnabled: boolean;
aiEnabled: boolean;
oauthEnabled: boolean;
oauthButtonText: string;
}
@@ -13,7 +13,7 @@ const ConfigContext = createContext<ConfigContextType>({
immichURL: "",
exImmichUrl: "",
version: "",
geminiEnabled: false,
aiEnabled: false,
oauthEnabled: false,
oauthButtonText: "",
});
src/helpers/ai.helper.ts
+102
View File
@@ -0,0 +1,102 @@
import { ENV } from "@/config/environment";
import { generateText, Output } from "ai";
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
import { z } from "zod";
import { removeNullOrUndefinedProperties } from "./data.helper";
interface FindQuery {
query: string;
personIds?: string[];
city?: string;
country?: string;
size?: number;
model?: string;
state?: string;
takenAfter?: string;
takenBefore?: string;
type?: string;
}
const allowedTypes = ["IMAGE", "VIDEO", "AUDIO"];
const responseSchema = z.object({
query: z.string().describe("Gist of the query. Remove the details of tags"),
personIds: z
.array(z.string())
.nullish()
.describe("List of person ids that match the query which starts with @"),
city: z.string().nullish().describe("City of the query"),
country: z
.string()
.nullish()
.describe("Any reference to a country in the query, return full country name"),
state: z
.string()
.nullish()
.describe("Any reference to a state (California, New York, etc) in the query"),
size: z.number().int().nullish().describe("Size of the file in bytes"),
takenAfter: z
.string()
.nullish()
.describe("Extract a valid date from query in YYYY-MM-DD format"),
takenBefore: z
.string()
.nullish()
.describe("Extract a valid date from query in YYYY-MM-DD format"),
model: z.string().nullish().describe("Name of the device that the query is about"),
type: z
.string()
.nullish()
.describe("One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)"),
});
const getOpenAICompatibleModel = () => {
if (!ENV.AI_API_KEY || !ENV.AI_MODEL) {
throw new Error("AI is not configured. Please set AI_API_KEY and AI_MODEL.");
}
const openai = createOpenAICompatible({
name: "openai-compatible",
apiKey: ENV.AI_API_KEY,
baseURL: ENV.AI_BASE_URL,
});
return openai(ENV.AI_MODEL);
};
export const parseFindQuery = async (query: string): Promise<FindQuery> => {
const prompt = [
`Parse the following query and return extracted filters as JSON: ${query}.`,
"Do not include any information that is not intentionally provided in the query.",
`Today's date is ${new Date().toISOString().split("T")[0]}.`,
"Dates must be in YYYY-MM-DD format.",
"Return ONLY valid JSON object with keys: query, personIds, city, country, state, size, model, takenAfter, takenBefore, type.",
"Do not use null values. Omit a key when it is not present in the query.",
"Use type values only IMAGE, VIDEO, AUDIO when possible.",
].join("\n");
const { text } = await generateText({
model: getOpenAICompatibleModel(),
prompt,
output: Output.object({
name:"json",
schema: responseSchema,
}),
});
const parsedResponse = JSON.parse(text) as FindQuery;
if (parsedResponse.type) {
if (!allowedTypes.includes(parsedResponse.type)) {
delete parsedResponse.type;
}
}
const cleanedResponse = {
...parsedResponse
};
return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
};
src/helpers/gemini.helper.ts
-106
View File
@@ -1,106 +0,0 @@
import { ENV } from "@/config/environment";
import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai";
import { z } from "zod";
import { removeNullOrUndefinedProperties } from "./data.helper";
const gemini = new GoogleGenerativeAI(ENV.GEMINI_API_KEY);
const model = gemini.getGenerativeModel({ model: "gemini-2.0-flash" });
interface FindQuery {
query: string;
personIds?: string[];
city?: string;
size?: number;
model?: string;
createdAfter?: string;
state?: string;
type?: string;
}
const allowedTypes = [
'IMAGE',
'VIDEO',
'AUDIO'
]
const responseSchema = {
type: SchemaType.OBJECT,
properties: {
query: { type: SchemaType.STRING, description: "Gist of the query. Remove the details of tags" },
personIds: { type: SchemaType.ARRAY,
description: "List of person ids that match the query which starts with @",
items: { type: SchemaType.STRING }
},
city: { type: SchemaType.STRING,
description: "City of the query"
},
size: {
type: SchemaType.INTEGER,
description: "Size of the file in bytes"
},
country: {
type: SchemaType.STRING,
description: "Any reference to a country (United States, Canada, etc) in the query, return full country name"
},
state: {
type: SchemaType.STRING,
description: "Any reference to a state (California, New York, etc) in the query"
},
takenAfter: {
type: SchemaType.STRING,
description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
},
takenBefore: {
type: SchemaType.STRING,
description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
},
model: {
type: SchemaType.STRING,
description: "Name of the device that the query is about"
},
type: {
type: SchemaType.STRING,
description: "One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)",
}
},
required: ["query"],
};
export const parseFindQuery = async (query: string): Promise<FindQuery> => {
const prompt = `
Parse the following query and return the query and tags: ${query}.
Dont include any information that are not intentionally provided in the query.
Additional Information For Parsing:
today's date is ${new Date().toISOString().split('T')[0]}
`;
const { response } = await model.generateContent({
contents: [
{
role: "user",
parts: [
{
text: prompt,
},
],
},
],
generationConfig: {
responseMimeType: "application/json",
responseSchema,
}
});
const parsedResponse = JSON.parse(response.text()) as FindQuery;
if (parsedResponse.type) {
if (!allowedTypes.includes(parsedResponse.type)) {
delete parsedResponse.type;
}
}
const cleanedResponse = {
...parsedResponse
};
return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
}
src/pages/_app.tsx
+1 -1
View File
@@ -33,7 +33,7 @@ App.getInitialProps = async () => {
exImmichUrl: ENV.EXTERNAL_IMMICH_URL,
immichURL: ENV.IMMICH_URL,
version: ENV.VERSION,
geminiEnabled: !!ENV.GEMINI_API_KEY?.length,
aiEnabled: !!ENV.AI_API_KEY?.length && !!ENV.AI_MODEL?.length,
oauthEnabled: ENV.OAUTH_ENABLED,
oauthButtonText: ENV.OAUTH_BUTTON_TEXT,
},
src/pages/api/find/search.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { ENV } from "@/config/environment";
import { ASSET_VIDEO_PATH } from "@/config/routes";
import { getCurrentUser } from "@/handlers/serverUtils/user.utils";
import { cleanUpAsset, isFlipped } from "@/helpers/asset.helper";
import { parseFindQuery } from "@/helpers/gemini.helper";
import { parseFindQuery } from "@/helpers/ai.helper";
import { person } from "@/schema";
import { Person } from "@/schema/person.schema";
import { inArray } from "drizzle-orm";
src/pages/find/index.tsx
+7 -7
View File
@@ -49,7 +49,7 @@ export default function FindPage() {
// const [selectedIds, setSelectedIds] = useState<string[]>([]);
// const [assets, setAssets] = useState<IAsset[]>([]);
const { toast } = useToast(); // Initialize toast
const { geminiEnabled, exImmichUrl } = useConfig();
const { aiEnabled } = useConfig();
const [query, setQuery] = useState('');
const [loading, setLoading] = useState(false);
const [filters, setFilters] = useState<IFindFilters>({});
@@ -216,7 +216,7 @@ export default function FindPage() {
))}
</div>
<p className='text-xs text-muted-foreground text-center flex gap-1 items-center mt-4'> {/* Adjusted margin top */}
<span>Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini.</span>
<span>Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.</span>
</p>
</div>
)
@@ -272,7 +272,7 @@ export default function FindPage() {
return (
<PageLayout className="pb-20">
<Header leftComponent="Find" />
{geminiEnabled ? (
{aiEnabled ? (
<>
<div className="flex flex-col gap-4 p-2">
<FindInput
@@ -287,14 +287,14 @@ export default function FindPage() {
) : (
<div className="flex justify-center py-10 items-center h-full flex-col gap-2">
<WandSparkles className='w-10 h-10 text-muted-foreground' />
<p className='text-lg font-semibold'>Google Gemini is not enabled</p>
<p className='text-lg font-semibold'>AI parsing is not enabled</p>
<p className='text-sm text-muted-foreground max-w-md text-center'>
Currently, the Power Tools Find is relying on Google Gemini for parsing the query.
Please enable Gemini by adding <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>GEMINI_API_KEY</kbd> in the <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>.env</kbd> file.
Currently, the Power Tools Find relies on an OpenAI-compatible API for parsing the query.
Please configure <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>AI_API_KEY</kbd> and <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>AI_MODEL</kbd> in the <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>.env</kbd> file.
</p>
<div className="border border-l-4 border-zinc-200 dark:border-zinc-500 rounded-md p-2">
<p className='text-xs text-muted-foreground text-center flex gap-1 items-center'>
<span>Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini.</span>
<span>Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.</span>
</p>
</div>
</div>