feat: integrate OpenAI-compatible API for query parsing and remove Gemini references

This commit is contained in:
sai80082 committed 2026-02-18 09:52:08 +05:30
1 parent 3f30e71b95
commit 774a0c480b
11 files changed
+130 -125

No files matched your search

+3 -1
View File
@@ -12,7 +12,9 @@ export const ENV = {
JWT_SECRET: process.env.JWT_SECRET as string,
SECURE_COOKIE: process.env.SECURE_COOKIE === 'true',
VERSION: process.env.VERSION,
GEMINI_API_KEY: process.env.GEMINI_API_KEY as string,
AI_API_KEY: process.env.AI_API_KEY as string,
AI_BASE_URL: (process.env.AI_BASE_URL || 'https://api.openai.com/v1') as string,
AI_MODEL: process.env.AI_MODEL as string,
IMMICH_SHARE_LINK_KEY: process.env.IMMICH_SHARE_LINK_KEY as string,
POWER_TOOLS_ENDPOINT_URL: process.env.POWER_TOOLS_ENDPOINT_URL as string,
// OAuth configuration
+2 -2
View File
@@ -4,7 +4,7 @@ export interface ConfigContextType {
immichURL: string;
exImmichUrl: string;
version?: string;
geminiEnabled: boolean;
aiEnabled: boolean;
oauthEnabled: boolean;
oauthButtonText: string;
}
@@ -13,7 +13,7 @@ const ConfigContext = createContext<ConfigContextType>({
immichURL: "",
exImmichUrl: "",
version: "",
geminiEnabled: false,
aiEnabled: false,
oauthEnabled: false,
oauthButtonText: "",
});
+102
View File
@@ -0,0 +1,102 @@
import { ENV } from "@/config/environment";
import { generateText, Output } from "ai";
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
import { z } from "zod";
import { removeNullOrUndefinedProperties } from "./data.helper";
interface FindQuery {
query: string;
personIds?: string[];
city?: string;
country?: string;
size?: number;
model?: string;
state?: string;
takenAfter?: string;
takenBefore?: string;
type?: string;
}
const allowedTypes = ["IMAGE", "VIDEO", "AUDIO"];
const responseSchema = z.object({
query: z.string().describe("Gist of the query. Remove the details of tags"),
personIds: z
.array(z.string())
.nullish()
.describe("List of person ids that match the query which starts with @"),
city: z.string().nullish().describe("City of the query"),
country: z
.string()
.nullish()
.describe("Any reference to a country in the query, return full country name"),
state: z
.string()
.nullish()
.describe("Any reference to a state (California, New York, etc) in the query"),
size: z.number().int().nullish().describe("Size of the file in bytes"),
takenAfter: z
.string()
.nullish()
.describe("Extract a valid date from query in YYYY-MM-DD format"),
takenBefore: z
.string()
.nullish()
.describe("Extract a valid date from query in YYYY-MM-DD format"),
model: z.string().nullish().describe("Name of the device that the query is about"),
type: z
.string()
.nullish()
.describe("One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)"),
});
const getOpenAICompatibleModel = () => {
if (!ENV.AI_API_KEY || !ENV.AI_MODEL) {
throw new Error("AI is not configured. Please set AI_API_KEY and AI_MODEL.");
}
const openai = createOpenAICompatible({
name: "openai-compatible",
apiKey: ENV.AI_API_KEY,
baseURL: ENV.AI_BASE_URL,
});
return openai(ENV.AI_MODEL);
};
export const parseFindQuery = async (query: string): Promise<FindQuery> => {
const prompt = [
`Parse the following query and return extracted filters as JSON: ${query}.`,
"Do not include any information that is not intentionally provided in the query.",
`Today's date is ${new Date().toISOString().split("T")[0]}.`,
"Dates must be in YYYY-MM-DD format.",
"Return ONLY valid JSON object with keys: query, personIds, city, country, state, size, model, takenAfter, takenBefore, type.",
"Do not use null values. Omit a key when it is not present in the query.",
"Use type values only IMAGE, VIDEO, AUDIO when possible.",
].join("\n");
const { text } = await generateText({
model: getOpenAICompatibleModel(),
prompt,
output: Output.object({
name:"json",
schema: responseSchema,
}),
});
const parsedResponse = JSON.parse(text) as FindQuery;
if (parsedResponse.type) {
if (!allowedTypes.includes(parsedResponse.type)) {
delete parsedResponse.type;
}
}
const cleanedResponse = {
...parsedResponse
};
return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
};
-106
View File
@@ -1,106 +0,0 @@
import { ENV } from "@/config/environment";
import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai";
import { z } from "zod";
import { removeNullOrUndefinedProperties } from "./data.helper";
const gemini = new GoogleGenerativeAI(ENV.GEMINI_API_KEY);
const model = gemini.getGenerativeModel({ model: "gemini-2.0-flash" });
interface FindQuery {
query: string;
personIds?: string[];
city?: string;
size?: number;
model?: string;
createdAfter?: string;
state?: string;
type?: string;
}
const allowedTypes = [
'IMAGE',
'VIDEO',
'AUDIO'
]
const responseSchema = {
type: SchemaType.OBJECT,
properties: {
query: { type: SchemaType.STRING, description: "Gist of the query. Remove the details of tags" },
personIds: { type: SchemaType.ARRAY,
description: "List of person ids that match the query which starts with @",
items: { type: SchemaType.STRING }
},
city: { type: SchemaType.STRING,
description: "City of the query"
},
size: {
type: SchemaType.INTEGER,
description: "Size of the file in bytes"
},
country: {
type: SchemaType.STRING,
description: "Any reference to a country (United States, Canada, etc) in the query, return full country name"
},
state: {
type: SchemaType.STRING,
description: "Any reference to a state (California, New York, etc) in the query"
},
takenAfter: {
type: SchemaType.STRING,
description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
},
takenBefore: {
type: SchemaType.STRING,
description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
},
model: {
type: SchemaType.STRING,
description: "Name of the device that the query is about"
},
type: {
type: SchemaType.STRING,
description: "One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)",
}
},
required: ["query"],
};
export const parseFindQuery = async (query: string): Promise<FindQuery> => {
const prompt = `
Parse the following query and return the query and tags: ${query}.
Dont include any information that are not intentionally provided in the query.
Additional Information For Parsing:
today's date is ${new Date().toISOString().split('T')[0]}
`;
const { response } = await model.generateContent({
contents: [
{
role: "user",
parts: [
{
text: prompt,
},
],
},
],
generationConfig: {
responseMimeType: "application/json",
responseSchema,
}
});
const parsedResponse = JSON.parse(response.text()) as FindQuery;
if (parsedResponse.type) {
if (!allowedTypes.includes(parsedResponse.type)) {
delete parsedResponse.type;
}
}
const cleanedResponse = {
...parsedResponse
};
return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
}
+1 -1
View File
@@ -33,7 +33,7 @@ App.getInitialProps = async () => {
exImmichUrl: ENV.EXTERNAL_IMMICH_URL,
immichURL: ENV.IMMICH_URL,
version: ENV.VERSION,
geminiEnabled: !!ENV.GEMINI_API_KEY?.length,
aiEnabled: !!ENV.AI_API_KEY?.length && !!ENV.AI_MODEL?.length,
oauthEnabled: ENV.OAUTH_ENABLED,
oauthButtonText: ENV.OAUTH_BUTTON_TEXT,
},
+1 -1
View File
@@ -3,7 +3,7 @@ import { ENV } from "@/config/environment";
import { ASSET_VIDEO_PATH } from "@/config/routes";
import { getCurrentUser } from "@/handlers/serverUtils/user.utils";
import { cleanUpAsset, isFlipped } from "@/helpers/asset.helper";
import { parseFindQuery } from "@/helpers/gemini.helper";
import { parseFindQuery } from "@/helpers/ai.helper";
import { person } from "@/schema";
import { Person } from "@/schema/person.schema";
import { inArray } from "drizzle-orm";
+7 -7
View File
@@ -49,7 +49,7 @@ export default function FindPage() {
// const [selectedIds, setSelectedIds] = useState<string[]>([]);
// const [assets, setAssets] = useState<IAsset[]>([]);
const { toast } = useToast(); // Initialize toast
const { geminiEnabled, exImmichUrl } = useConfig();
const { aiEnabled } = useConfig();
const [query, setQuery] = useState('');
const [loading, setLoading] = useState(false);
const [filters, setFilters] = useState<IFindFilters>({});
@@ -216,7 +216,7 @@ export default function FindPage() {
))}
</div>
<p className='text-xs text-muted-foreground text-center flex gap-1 items-center mt-4'> {/* Adjusted margin top */}
<span>Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini.</span>
<span>Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.</span>
</p>
</div>
)
@@ -272,7 +272,7 @@ export default function FindPage() {
return (
<PageLayout className="pb-20">
<Header leftComponent="Find" />
{geminiEnabled ? (
{aiEnabled ? (
<>
<div className="flex flex-col gap-4 p-2">
<FindInput
@@ -287,14 +287,14 @@ export default function FindPage() {
) : (
<div className="flex justify-center py-10 items-center h-full flex-col gap-2">
<WandSparkles className='w-10 h-10 text-muted-foreground' />
<p className='text-lg font-semibold'>Google Gemini is not enabled</p>
<p className='text-lg font-semibold'>AI parsing is not enabled</p>
<p className='text-sm text-muted-foreground max-w-md text-center'>
Currently, the Power Tools Find is relying on Google Gemini for parsing the query.
Please enable Gemini by adding <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>GEMINI_API_KEY</kbd> in the <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>.env</kbd> file.
Currently, the Power Tools Find relies on an OpenAI-compatible API for parsing the query.
Please configure <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>AI_API_KEY</kbd> and <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>AI_MODEL</kbd> in the <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>.env</kbd> file.
</p>
<div className="border border-l-4 border-zinc-200 dark:border-zinc-500 rounded-md p-2">
<p className='text-xs text-muted-foreground text-center flex gap-1 items-center'>
<span>Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini.</span>
<span>Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.</span>
</p>
</div>
</div>