diff --git a/.env.sample b/.env.sample index 1b86051..071d49f 100644 --- a/.env.sample +++ b/.env.sample @@ -10,7 +10,9 @@ DB_DATABASE_NAME="" # Name of the database SECURE_COOKIE=false # Set to true to enable secure cookies # Optional -GEMINI_API_KEY="" # Gemini API Key for parsing search query in "Find" +AI_API_KEY="" # API key for your OpenAI-compatible provider used by "Find" +AI_BASE_URL="https://api.openai.com/v1" # Base URL for OpenAI-compatible API +AI_MODEL="gpt-4o-mini" # Model used for parsing search queries in "Find" # Immich Share Link IMMICH_SHARE_LINK_KEY="" # Share link key for Immich diff --git a/README.md b/README.md index b61b334..5b5e7c3 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,9 @@ DB_PORT = "5432" DB_HOST = "immich_postgres" # Optional -GEMINI_API_KEY= # Gemini API Key +AI_API_KEY= # API key for your OpenAI-compatible provider +AI_BASE_URL=https://api.openai.com/v1 # Base URL for OpenAI-compatible API +AI_MODEL=gpt-4o-mini # Model used to parse search queries in Find ``` Refer here for obtaining Immich API Key: https://immich.app/docs/features/command-line-interface#obtain-the-api-key @@ -145,10 +147,10 @@ Google Maps Javascript API Key is used to render the maps layer for heatmap. Whe > Code where heatmap data is plotted: [src/pages/assets/geo-heatmap.tsx:32](./src/pages/assets/geo-heatmap.tsx#L32-L35) -- [Gemini](https://gemini.google.com/) for rewind video generation -Google Gemini 1.5 Flash model is used for parsing your search query in "Find" page. We do not send any personal data to Gemini, only data that is sent to Gemini is your search query and rest of the querying happes using [Immich's Smart Search API](https://immich.app/docs/api/search-smart) +- OpenAI-compatible AI provider for smart query parsing in Find +Any OpenAI-compatible model can be used for parsing your search query in the "Find" page. We do not send any personal library data to the AI provider; only your search text is sent for parsing, and the actual asset querying uses [Immich's Smart Search API](https://immich.app/docs/api/search-smart) - > Code where Gemini is used: [src/helpers/gemini.helper.ts](./src/helpers/gemini.helper.ts) + > Code where AI parsing is used: [src/helpers/ai.helper.ts](./src/helpers/ai.helper.ts) ## Contributing diff --git a/bun.lockb b/bun.lockb index 7944ab4..ca9ac68 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 38d51c4..d83ebed 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,9 @@ "url": "https://saicharang.com" }, "dependencies": { - "@google/generative-ai": "^0.21.0", + "@ai-sdk/openai": "1.3.24", + "@ai-sdk/openai-compatible": "^2.0.30", + "@ai-sdk/react": "^3.0.88", "@radix-ui/react-accordion": "^1.2.1", "@radix-ui/react-alert-dialog": "^1.1.4", "@radix-ui/react-avatar": "^1.1.0", @@ -42,6 +44,7 @@ "@types/react-autosuggest": "^10.1.11", "@types/react-calendar-heatmap": "^1.6.7", "@types/react-window": "^1.8.8", + "ai": "^6.0.86", "axios": "^1.7.4", "chrono-node": "^1.4.9", "class-variance-authority": "^0.7.0", @@ -76,7 +79,7 @@ "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", "yet-another-react-lightbox": "^3.21.4", - "zod": "^3.24.1" + "zod": "^4.3.6" }, "devDependencies": { "@semantic-release/commit-analyzer": "^13.0.0", diff --git a/src/config/environment.ts b/src/config/environment.ts index 7f66aca..f3daf06 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -12,7 +12,9 @@ export const ENV = { JWT_SECRET: process.env.JWT_SECRET as string, SECURE_COOKIE: process.env.SECURE_COOKIE === 'true', VERSION: process.env.VERSION, - GEMINI_API_KEY: process.env.GEMINI_API_KEY as string, + AI_API_KEY: process.env.AI_API_KEY as string, + AI_BASE_URL: (process.env.AI_BASE_URL || 'https://api.openai.com/v1') as string, + AI_MODEL: process.env.AI_MODEL as string, IMMICH_SHARE_LINK_KEY: process.env.IMMICH_SHARE_LINK_KEY as string, POWER_TOOLS_ENDPOINT_URL: process.env.POWER_TOOLS_ENDPOINT_URL as string, // OAuth configuration diff --git a/src/contexts/ConfigContext.tsx b/src/contexts/ConfigContext.tsx index c4c8c0f..c6d8246 100644 --- a/src/contexts/ConfigContext.tsx +++ b/src/contexts/ConfigContext.tsx @@ -4,7 +4,7 @@ export interface ConfigContextType { immichURL: string; exImmichUrl: string; version?: string; - geminiEnabled: boolean; + aiEnabled: boolean; oauthEnabled: boolean; oauthButtonText: string; } @@ -13,7 +13,7 @@ const ConfigContext = createContext({ immichURL: "", exImmichUrl: "", version: "", - geminiEnabled: false, + aiEnabled: false, oauthEnabled: false, oauthButtonText: "", }); diff --git a/src/helpers/ai.helper.ts b/src/helpers/ai.helper.ts new file mode 100644 index 0000000..c4ecf92 --- /dev/null +++ b/src/helpers/ai.helper.ts @@ -0,0 +1,102 @@ +import { ENV } from "@/config/environment"; +import { generateText, Output } from "ai"; +import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; +import { z } from "zod"; +import { removeNullOrUndefinedProperties } from "./data.helper"; + +interface FindQuery { + query: string; + personIds?: string[]; + city?: string; + country?: string; + size?: number; + model?: string; + state?: string; + takenAfter?: string; + takenBefore?: string; + type?: string; +} + +const allowedTypes = ["IMAGE", "VIDEO", "AUDIO"]; + + +const responseSchema = z.object({ + query: z.string().describe("Gist of the query. Remove the details of tags"), + personIds: z + .array(z.string()) + .nullish() + .describe("List of person ids that match the query which starts with @"), + city: z.string().nullish().describe("City of the query"), + country: z + .string() + .nullish() + .describe("Any reference to a country in the query, return full country name"), + state: z + .string() + .nullish() + .describe("Any reference to a state (California, New York, etc) in the query"), + size: z.number().int().nullish().describe("Size of the file in bytes"), + takenAfter: z + .string() + .nullish() + .describe("Extract a valid date from query in YYYY-MM-DD format"), + takenBefore: z + .string() + .nullish() + .describe("Extract a valid date from query in YYYY-MM-DD format"), + model: z.string().nullish().describe("Name of the device that the query is about"), + type: z + .string() + .nullish() + .describe("One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)"), +}); + + +const getOpenAICompatibleModel = () => { + if (!ENV.AI_API_KEY || !ENV.AI_MODEL) { + throw new Error("AI is not configured. Please set AI_API_KEY and AI_MODEL."); + } + + const openai = createOpenAICompatible({ + name: "openai-compatible", + apiKey: ENV.AI_API_KEY, + baseURL: ENV.AI_BASE_URL, + }); + + return openai(ENV.AI_MODEL); +}; + +export const parseFindQuery = async (query: string): Promise => { + const prompt = [ + `Parse the following query and return extracted filters as JSON: ${query}.`, + "Do not include any information that is not intentionally provided in the query.", + `Today's date is ${new Date().toISOString().split("T")[0]}.`, + "Dates must be in YYYY-MM-DD format.", + "Return ONLY valid JSON object with keys: query, personIds, city, country, state, size, model, takenAfter, takenBefore, type.", + "Do not use null values. Omit a key when it is not present in the query.", + "Use type values only IMAGE, VIDEO, AUDIO when possible.", + ].join("\n"); + + const { text } = await generateText({ + model: getOpenAICompatibleModel(), + prompt, + output: Output.object({ + name:"json", + schema: responseSchema, + }), + + }); + + const parsedResponse = JSON.parse(text) as FindQuery; + + if (parsedResponse.type) { + if (!allowedTypes.includes(parsedResponse.type)) { + delete parsedResponse.type; + } + } + const cleanedResponse = { + ...parsedResponse + }; + + return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery; +}; diff --git a/src/helpers/gemini.helper.ts b/src/helpers/gemini.helper.ts deleted file mode 100644 index af54e9e..0000000 --- a/src/helpers/gemini.helper.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { ENV } from "@/config/environment"; -import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai"; -import { z } from "zod"; -import { removeNullOrUndefinedProperties } from "./data.helper"; - -const gemini = new GoogleGenerativeAI(ENV.GEMINI_API_KEY); -const model = gemini.getGenerativeModel({ model: "gemini-2.0-flash" }); - -interface FindQuery { - query: string; - personIds?: string[]; - city?: string; - size?: number; - model?: string; - createdAfter?: string; - state?: string; - type?: string; -} - -const allowedTypes = [ - 'IMAGE', - 'VIDEO', - 'AUDIO' -] -const responseSchema = { - type: SchemaType.OBJECT, - properties: { - query: { type: SchemaType.STRING, description: "Gist of the query. Remove the details of tags" }, - personIds: { type: SchemaType.ARRAY, - description: "List of person ids that match the query which starts with @", - items: { type: SchemaType.STRING } - }, - city: { type: SchemaType.STRING, - description: "City of the query" - }, - size: { - type: SchemaType.INTEGER, - description: "Size of the file in bytes" - }, - country: { - type: SchemaType.STRING, - description: "Any reference to a country (United States, Canada, etc) in the query, return full country name" - }, - state: { - type: SchemaType.STRING, - description: "Any reference to a state (California, New York, etc) in the query" - }, - takenAfter: { - type: SchemaType.STRING, - description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date" - }, - takenBefore: { - type: SchemaType.STRING, - description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date" - }, - model: { - type: SchemaType.STRING, - description: "Name of the device that the query is about" - }, - type: { - type: SchemaType.STRING, - description: "One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)", - } - }, - required: ["query"], -}; - -export const parseFindQuery = async (query: string): Promise => { - - const prompt = ` - Parse the following query and return the query and tags: ${query}. - Dont include any information that are not intentionally provided in the query. - Additional Information For Parsing: - today's date is ${new Date().toISOString().split('T')[0]} - `; - - const { response } = await model.generateContent({ - contents: [ - { - role: "user", - parts: [ - { - text: prompt, - }, - ], - }, - ], - generationConfig: { - responseMimeType: "application/json", - responseSchema, - } - }); - - const parsedResponse = JSON.parse(response.text()) as FindQuery; - - if (parsedResponse.type) { - if (!allowedTypes.includes(parsedResponse.type)) { - delete parsedResponse.type; - } - } - const cleanedResponse = { - ...parsedResponse - }; - - return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery; -} diff --git a/src/pages/_app.tsx b/src/pages/_app.tsx index 16c354f..a96cf90 100644 --- a/src/pages/_app.tsx +++ b/src/pages/_app.tsx @@ -33,7 +33,7 @@ App.getInitialProps = async () => { exImmichUrl: ENV.EXTERNAL_IMMICH_URL, immichURL: ENV.IMMICH_URL, version: ENV.VERSION, - geminiEnabled: !!ENV.GEMINI_API_KEY?.length, + aiEnabled: !!ENV.AI_API_KEY?.length && !!ENV.AI_MODEL?.length, oauthEnabled: ENV.OAUTH_ENABLED, oauthButtonText: ENV.OAUTH_BUTTON_TEXT, }, diff --git a/src/pages/api/find/search.ts b/src/pages/api/find/search.ts index 2bf858a..e5fead7 100644 --- a/src/pages/api/find/search.ts +++ b/src/pages/api/find/search.ts @@ -3,7 +3,7 @@ import { ENV } from "@/config/environment"; import { ASSET_VIDEO_PATH } from "@/config/routes"; import { getCurrentUser } from "@/handlers/serverUtils/user.utils"; import { cleanUpAsset, isFlipped } from "@/helpers/asset.helper"; -import { parseFindQuery } from "@/helpers/gemini.helper"; +import { parseFindQuery } from "@/helpers/ai.helper"; import { person } from "@/schema"; import { Person } from "@/schema/person.schema"; import { inArray } from "drizzle-orm"; diff --git a/src/pages/find/index.tsx b/src/pages/find/index.tsx index b7e72cf..0518064 100644 --- a/src/pages/find/index.tsx +++ b/src/pages/find/index.tsx @@ -49,7 +49,7 @@ export default function FindPage() { // const [selectedIds, setSelectedIds] = useState([]); // const [assets, setAssets] = useState([]); const { toast } = useToast(); // Initialize toast - const { geminiEnabled, exImmichUrl } = useConfig(); + const { aiEnabled } = useConfig(); const [query, setQuery] = useState(''); const [loading, setLoading] = useState(false); const [filters, setFilters] = useState({}); @@ -216,7 +216,7 @@ export default function FindPage() { ))}

{/* Adjusted margin top */} - Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini. + Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.

) @@ -272,7 +272,7 @@ export default function FindPage() { return (
- {geminiEnabled ? ( + {aiEnabled ? ( <>
-

Google Gemini is not enabled

+

AI parsing is not enabled

- Currently, the Power Tools Find is relying on Google Gemini for parsing the query. - Please enable Gemini by adding GEMINI_API_KEY in the .env file. + Currently, the Power Tools Find relies on an OpenAI-compatible API for parsing the query. + Please configure AI_API_KEY and AI_MODEL in the .env file.

- Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini. + Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.