feat: integrate OpenAI-compatible API for query parsing and remove Gemini references
This commit is contained in:
1 parent
3f30e71b95
commit
774a0c480b
11 files changed
+130
-125
No files matched your search
@@ -12,7 +12,9 @@ export const ENV = {
|
||||
JWT_SECRET: process.env.JWT_SECRET as string,
|
||||
SECURE_COOKIE: process.env.SECURE_COOKIE === 'true',
|
||||
VERSION: process.env.VERSION,
|
||||
GEMINI_API_KEY: process.env.GEMINI_API_KEY as string,
|
||||
AI_API_KEY: process.env.AI_API_KEY as string,
|
||||
AI_BASE_URL: (process.env.AI_BASE_URL || 'https://api.openai.com/v1') as string,
|
||||
AI_MODEL: process.env.AI_MODEL as string,
|
||||
IMMICH_SHARE_LINK_KEY: process.env.IMMICH_SHARE_LINK_KEY as string,
|
||||
POWER_TOOLS_ENDPOINT_URL: process.env.POWER_TOOLS_ENDPOINT_URL as string,
|
||||
// OAuth configuration
|
||||
|
||||
@@ -4,7 +4,7 @@ export interface ConfigContextType {
|
||||
immichURL: string;
|
||||
exImmichUrl: string;
|
||||
version?: string;
|
||||
geminiEnabled: boolean;
|
||||
aiEnabled: boolean;
|
||||
oauthEnabled: boolean;
|
||||
oauthButtonText: string;
|
||||
}
|
||||
@@ -13,7 +13,7 @@ const ConfigContext = createContext<ConfigContextType>({
|
||||
immichURL: "",
|
||||
exImmichUrl: "",
|
||||
version: "",
|
||||
geminiEnabled: false,
|
||||
aiEnabled: false,
|
||||
oauthEnabled: false,
|
||||
oauthButtonText: "",
|
||||
});
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import { ENV } from "@/config/environment";
|
||||
import { generateText, Output } from "ai";
|
||||
import { createOpenAICompatible } from '@ai-sdk/openai-compatible';
|
||||
import { z } from "zod";
|
||||
import { removeNullOrUndefinedProperties } from "./data.helper";
|
||||
|
||||
interface FindQuery {
|
||||
query: string;
|
||||
personIds?: string[];
|
||||
city?: string;
|
||||
country?: string;
|
||||
size?: number;
|
||||
model?: string;
|
||||
state?: string;
|
||||
takenAfter?: string;
|
||||
takenBefore?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
const allowedTypes = ["IMAGE", "VIDEO", "AUDIO"];
|
||||
|
||||
|
||||
const responseSchema = z.object({
|
||||
query: z.string().describe("Gist of the query. Remove the details of tags"),
|
||||
personIds: z
|
||||
.array(z.string())
|
||||
.nullish()
|
||||
.describe("List of person ids that match the query which starts with @"),
|
||||
city: z.string().nullish().describe("City of the query"),
|
||||
country: z
|
||||
.string()
|
||||
.nullish()
|
||||
.describe("Any reference to a country in the query, return full country name"),
|
||||
state: z
|
||||
.string()
|
||||
.nullish()
|
||||
.describe("Any reference to a state (California, New York, etc) in the query"),
|
||||
size: z.number().int().nullish().describe("Size of the file in bytes"),
|
||||
takenAfter: z
|
||||
.string()
|
||||
.nullish()
|
||||
.describe("Extract a valid date from query in YYYY-MM-DD format"),
|
||||
takenBefore: z
|
||||
.string()
|
||||
.nullish()
|
||||
.describe("Extract a valid date from query in YYYY-MM-DD format"),
|
||||
model: z.string().nullish().describe("Name of the device that the query is about"),
|
||||
type: z
|
||||
.string()
|
||||
.nullish()
|
||||
.describe("One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)"),
|
||||
});
|
||||
|
||||
|
||||
const getOpenAICompatibleModel = () => {
|
||||
if (!ENV.AI_API_KEY || !ENV.AI_MODEL) {
|
||||
throw new Error("AI is not configured. Please set AI_API_KEY and AI_MODEL.");
|
||||
}
|
||||
|
||||
const openai = createOpenAICompatible({
|
||||
name: "openai-compatible",
|
||||
apiKey: ENV.AI_API_KEY,
|
||||
baseURL: ENV.AI_BASE_URL,
|
||||
});
|
||||
|
||||
return openai(ENV.AI_MODEL);
|
||||
};
|
||||
|
||||
export const parseFindQuery = async (query: string): Promise<FindQuery> => {
|
||||
const prompt = [
|
||||
`Parse the following query and return extracted filters as JSON: ${query}.`,
|
||||
"Do not include any information that is not intentionally provided in the query.",
|
||||
`Today's date is ${new Date().toISOString().split("T")[0]}.`,
|
||||
"Dates must be in YYYY-MM-DD format.",
|
||||
"Return ONLY valid JSON object with keys: query, personIds, city, country, state, size, model, takenAfter, takenBefore, type.",
|
||||
"Do not use null values. Omit a key when it is not present in the query.",
|
||||
"Use type values only IMAGE, VIDEO, AUDIO when possible.",
|
||||
].join("\n");
|
||||
|
||||
const { text } = await generateText({
|
||||
model: getOpenAICompatibleModel(),
|
||||
prompt,
|
||||
output: Output.object({
|
||||
name:"json",
|
||||
schema: responseSchema,
|
||||
}),
|
||||
|
||||
});
|
||||
|
||||
const parsedResponse = JSON.parse(text) as FindQuery;
|
||||
|
||||
if (parsedResponse.type) {
|
||||
if (!allowedTypes.includes(parsedResponse.type)) {
|
||||
delete parsedResponse.type;
|
||||
}
|
||||
}
|
||||
const cleanedResponse = {
|
||||
...parsedResponse
|
||||
};
|
||||
|
||||
return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
|
||||
};
|
||||
@@ -1,106 +0,0 @@
|
||||
import { ENV } from "@/config/environment";
|
||||
import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai";
|
||||
import { z } from "zod";
|
||||
import { removeNullOrUndefinedProperties } from "./data.helper";
|
||||
|
||||
const gemini = new GoogleGenerativeAI(ENV.GEMINI_API_KEY);
|
||||
const model = gemini.getGenerativeModel({ model: "gemini-2.0-flash" });
|
||||
|
||||
interface FindQuery {
|
||||
query: string;
|
||||
personIds?: string[];
|
||||
city?: string;
|
||||
size?: number;
|
||||
model?: string;
|
||||
createdAfter?: string;
|
||||
state?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
const allowedTypes = [
|
||||
'IMAGE',
|
||||
'VIDEO',
|
||||
'AUDIO'
|
||||
]
|
||||
const responseSchema = {
|
||||
type: SchemaType.OBJECT,
|
||||
properties: {
|
||||
query: { type: SchemaType.STRING, description: "Gist of the query. Remove the details of tags" },
|
||||
personIds: { type: SchemaType.ARRAY,
|
||||
description: "List of person ids that match the query which starts with @",
|
||||
items: { type: SchemaType.STRING }
|
||||
},
|
||||
city: { type: SchemaType.STRING,
|
||||
description: "City of the query"
|
||||
},
|
||||
size: {
|
||||
type: SchemaType.INTEGER,
|
||||
description: "Size of the file in bytes"
|
||||
},
|
||||
country: {
|
||||
type: SchemaType.STRING,
|
||||
description: "Any reference to a country (United States, Canada, etc) in the query, return full country name"
|
||||
},
|
||||
state: {
|
||||
type: SchemaType.STRING,
|
||||
description: "Any reference to a state (California, New York, etc) in the query"
|
||||
},
|
||||
takenAfter: {
|
||||
type: SchemaType.STRING,
|
||||
description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
|
||||
},
|
||||
takenBefore: {
|
||||
type: SchemaType.STRING,
|
||||
description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
|
||||
},
|
||||
model: {
|
||||
type: SchemaType.STRING,
|
||||
description: "Name of the device that the query is about"
|
||||
},
|
||||
type: {
|
||||
type: SchemaType.STRING,
|
||||
description: "One of the allowed media types (IMAGE, VIDEO, AUDIO, OTHER)",
|
||||
}
|
||||
},
|
||||
required: ["query"],
|
||||
};
|
||||
|
||||
export const parseFindQuery = async (query: string): Promise<FindQuery> => {
|
||||
|
||||
const prompt = `
|
||||
Parse the following query and return the query and tags: ${query}.
|
||||
Dont include any information that are not intentionally provided in the query.
|
||||
Additional Information For Parsing:
|
||||
today's date is ${new Date().toISOString().split('T')[0]}
|
||||
`;
|
||||
|
||||
const { response } = await model.generateContent({
|
||||
contents: [
|
||||
{
|
||||
role: "user",
|
||||
parts: [
|
||||
{
|
||||
text: prompt,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
generationConfig: {
|
||||
responseMimeType: "application/json",
|
||||
responseSchema,
|
||||
}
|
||||
});
|
||||
|
||||
const parsedResponse = JSON.parse(response.text()) as FindQuery;
|
||||
|
||||
if (parsedResponse.type) {
|
||||
if (!allowedTypes.includes(parsedResponse.type)) {
|
||||
delete parsedResponse.type;
|
||||
}
|
||||
}
|
||||
const cleanedResponse = {
|
||||
...parsedResponse
|
||||
};
|
||||
|
||||
return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
|
||||
}
|
||||
+1
-1
@@ -33,7 +33,7 @@ App.getInitialProps = async () => {
|
||||
exImmichUrl: ENV.EXTERNAL_IMMICH_URL,
|
||||
immichURL: ENV.IMMICH_URL,
|
||||
version: ENV.VERSION,
|
||||
geminiEnabled: !!ENV.GEMINI_API_KEY?.length,
|
||||
aiEnabled: !!ENV.AI_API_KEY?.length && !!ENV.AI_MODEL?.length,
|
||||
oauthEnabled: ENV.OAUTH_ENABLED,
|
||||
oauthButtonText: ENV.OAUTH_BUTTON_TEXT,
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@ import { ENV } from "@/config/environment";
|
||||
import { ASSET_VIDEO_PATH } from "@/config/routes";
|
||||
import { getCurrentUser } from "@/handlers/serverUtils/user.utils";
|
||||
import { cleanUpAsset, isFlipped } from "@/helpers/asset.helper";
|
||||
import { parseFindQuery } from "@/helpers/gemini.helper";
|
||||
import { parseFindQuery } from "@/helpers/ai.helper";
|
||||
import { person } from "@/schema";
|
||||
import { Person } from "@/schema/person.schema";
|
||||
import { inArray } from "drizzle-orm";
|
||||
|
||||
@@ -49,7 +49,7 @@ export default function FindPage() {
|
||||
// const [selectedIds, setSelectedIds] = useState<string[]>([]);
|
||||
// const [assets, setAssets] = useState<IAsset[]>([]);
|
||||
const { toast } = useToast(); // Initialize toast
|
||||
const { geminiEnabled, exImmichUrl } = useConfig();
|
||||
const { aiEnabled } = useConfig();
|
||||
const [query, setQuery] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [filters, setFilters] = useState<IFindFilters>({});
|
||||
@@ -216,7 +216,7 @@ export default function FindPage() {
|
||||
))}
|
||||
</div>
|
||||
<p className='text-xs text-muted-foreground text-center flex gap-1 items-center mt-4'> {/* Adjusted margin top */}
|
||||
<span>Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini.</span>
|
||||
<span>Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.</span>
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
@@ -272,7 +272,7 @@ export default function FindPage() {
|
||||
return (
|
||||
<PageLayout className="pb-20">
|
||||
<Header leftComponent="Find" />
|
||||
{geminiEnabled ? (
|
||||
{aiEnabled ? (
|
||||
<>
|
||||
<div className="flex flex-col gap-4 p-2">
|
||||
<FindInput
|
||||
@@ -287,14 +287,14 @@ export default function FindPage() {
|
||||
) : (
|
||||
<div className="flex justify-center py-10 items-center h-full flex-col gap-2">
|
||||
<WandSparkles className='w-10 h-10 text-muted-foreground' />
|
||||
<p className='text-lg font-semibold'>Google Gemini is not enabled</p>
|
||||
<p className='text-lg font-semibold'>AI parsing is not enabled</p>
|
||||
<p className='text-sm text-muted-foreground max-w-md text-center'>
|
||||
Currently, the Power Tools Find is relying on Google Gemini for parsing the query.
|
||||
Please enable Gemini by adding <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>GEMINI_API_KEY</kbd> in the <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>.env</kbd> file.
|
||||
Currently, the Power Tools Find relies on an OpenAI-compatible API for parsing the query.
|
||||
Please configure <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>AI_API_KEY</kbd> and <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>AI_MODEL</kbd> in the <kbd className='bg-zinc-200 text-black dark:text-white px-1 py-0.5 rounded-md dark:bg-zinc-500'>.env</kbd> file.
|
||||
</p>
|
||||
<div className="border border-l-4 border-zinc-200 dark:border-zinc-500 rounded-md p-2">
|
||||
<p className='text-xs text-muted-foreground text-center flex gap-1 items-center'>
|
||||
<span>Power tools uses Google Gemini only for parsing your query. None of your data is sent to Gemini.</span>
|
||||
<span>Power tools uses your configured AI model only for parsing your query. None of your library data is sent to the AI provider.</span>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in new issue
Block a user