diff --git a/bun.lockb b/bun.lockb index 2f76eac..9e16cf7 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index aea774c..53e14c1 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "url": "https://varunraj.in" }, "dependencies": { + "@google/generative-ai": "^0.21.0", "@radix-ui/react-accordion": "^1.2.1", "@radix-ui/react-avatar": "^1.1.0", "@radix-ui/react-checkbox": "^1.1.2", @@ -44,24 +45,27 @@ "date-fns": "^3.6.0", "drizzle-orm": "^0.33.0", "leaflet": "^1.9.4", - "leaflet-defaulticon-compatibility":"0.1.2", + "leaflet-defaulticon-compatibility": "0.1.2", "lucide-react": "^0.428.0", "next": "14.2.5", "next-themes": "^0.3.0", "pg": "^8.12.0", "qs": "^6.13.0", + "rc-mentions": "^2.18.0", "react": "^18", "react-calendar-heatmap": "^1.9.0", "react-day-picker": "9.0.8", "react-dom": "^18", "react-grid-gallery": "^1.0.1", "react-leaflet": "^4.2.1", + "react-mentions": "^4.4.10", "react-query": "^3.39.3", "recharts": "^2.12.7", "sass": "^1.77.8", "tailwind-merge": "^2.5.2", "tailwindcss-animate": "^1.0.7", - "yet-another-react-lightbox": "^3.21.4" + "yet-another-react-lightbox": "^3.21.4", + "zod": "^3.24.1" }, "devDependencies": { "@semantic-release/commit-analyzer": "^13.0.0", @@ -70,11 +74,12 @@ "@semantic-release/npm": "^12.0.1", "@semantic-release/release-notes-generator": "^14.0.1", "@types/jsonwebtoken": "^9.0.6", - "@types/leaflet":"^1.9.15", + "@types/leaflet": "^1.9.15", "@types/node": "^20", "@types/pg": "^8.11.6", "@types/react": "^18", "@types/react-dom": "^18", + "@types/react-mentions": "^4.4.1", "autoprefixer": "^10.4.20", "drizzle-kit": "^0.24.0", "eslint": "^8", diff --git a/src/components/find/FindInput.tsx b/src/components/find/FindInput.tsx new file mode 100644 index 0000000..3fe6ec2 --- /dev/null +++ b/src/components/find/FindInput.tsx @@ -0,0 +1,58 @@ +import React, { useState } from 'react' +import style from './findInputStyle' +import { MentionsInput, Mention } from 'react-mentions' +import { searchPeople } from '@/handlers/api/people.handler'; +import mentionStyle from './findMentionStyle'; +import { PERSON_THUBNAIL_PATH } from '@/config/routes'; +import Image from 'next/image'; + +interface FindInputProps { + onSearch: (query: string) => void; +} + +export default function FindInput({ onSearch }: FindInputProps) { + + const [query, setQuery] = useState(""); + + const handleSearchPeople = async (e: any, callback: any) => { + if (!e.length) return []; + return searchPeople(e).then((people) => people.map((person: any) => ({ + id: person.id, + display: person.name, + }))).then((people) => callback(people)); + } + + return ( + { + if (e.key === 'Enter') { + onSearch(query); + } + }} + onChange={(e) => setQuery(e.target.value)}> + ( +
+ + {highlightedDisplay.display} +
+ )} + /> + + ) +} diff --git a/src/components/find/findInputStyle.ts b/src/components/find/findInputStyle.ts new file mode 100644 index 0000000..9e6a587 --- /dev/null +++ b/src/components/find/findInputStyle.ts @@ -0,0 +1,40 @@ +const findInputStyle = { + control: { + backgroundColor: '#fff', + fontSize: 14, + fontWeight: 'normal', + }, + + + '&singleLine': { + display: 'inline-block', + + highlighter: { + padding: '10px', + border: '2px inset transparent', + }, + input: { + width: "100%", + padding: '10px', + borderRadius: '4px', + border: '1px solid #e0e0e0', + }, + }, + + suggestions: { + list: { + backgroundColor: 'white', + border: '1px solid rgba(0,0,0,0.15)', + fontSize: 14, + }, + item: { + padding: '5px 15px', + borderBottom: '1px solid rgba(0,0,0,0.15)', + '&focused': { + backgroundColor: '#cee4e5', + }, + }, + }, +} + +export default findInputStyle; \ No newline at end of file diff --git a/src/components/find/findMentionStyle.ts b/src/components/find/findMentionStyle.ts new file mode 100644 index 0000000..802b1e4 --- /dev/null +++ b/src/components/find/findMentionStyle.ts @@ -0,0 +1,3 @@ +export default { + backgroundColor: '#cee4e5', +} \ No newline at end of file diff --git a/src/config/constants/sidebarNavs.tsx b/src/config/constants/sidebarNavs.tsx index 7f1eebe..7b35b65 100644 --- a/src/config/constants/sidebarNavs.tsx +++ b/src/config/constants/sidebarNavs.tsx @@ -1,4 +1,4 @@ -import { Album, GalleryHorizontal, GalleryVerticalEnd, Home, Image, LocateOff, MapPinX, User } from "lucide-react"; +import { Album, GalleryHorizontal, GalleryVerticalEnd, Home, Image, LocateOff, MapPinX, Search, User } from "lucide-react"; export const sidebarNavs = [ { @@ -26,4 +26,9 @@ export const sidebarNavs = [ link: "/albums", icon: , }, + { + title: "Find", + link: "/find", + icon: , + }, ]; diff --git a/src/config/environment.ts b/src/config/environment.ts index e68cc52..a6ac9d1 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -5,7 +5,8 @@ export const ENV = { DATABASE_URL: (process.env.DATABASE_URL || `postgresql://${process.env.DB_USERNAME}:${process.env.DB_PASSWORD}@${(process.env.DB_HOST || 'immich_postgres')}:${(process.env.DB_PORT || '5432')}/${process.env.DB_DATABASE_NAME}`), JWT_SECRET: process.env.JWT_SECRET as string, SECURE_COOKIE: process.env.SECURE_COOKIE === 'true', - VERSION: process.env.VERSION + VERSION: process.env.VERSION, + GEMINI_API_KEY: process.env.GEMINI_API_KEY as string, }; diff --git a/src/config/routes.ts b/src/config/routes.ts index 090483c..8d9bffb 100644 --- a/src/config/routes.ts +++ b/src/config/routes.ts @@ -48,4 +48,7 @@ export const HEATMAP_DATA = BASE_API_ENDPOINT + "/analytics/statistics/heatmap"; export const GET_FILTERS = BASE_API_ENDPOINT + "/filters/asset-filters"; // Person -export const GET_PERSON_INFO = (personId: string) => BASE_API_ENDPOINT + "/people/" + personId + "/info"; \ No newline at end of file +export const GET_PERSON_INFO = (personId: string) => BASE_API_ENDPOINT + "/people/" + personId + "/info"; + +// Find +export const FIND_ASSETS = BASE_API_ENDPOINT + "/find/search"; \ No newline at end of file diff --git a/src/handlers/api/asset.handler.ts b/src/handlers/api/asset.handler.ts index 664f4b5..b04cced 100644 --- a/src/handlers/api/asset.handler.ts +++ b/src/handlers/api/asset.handler.ts @@ -1,5 +1,6 @@ import { ADD_ASSETS_ALBUMS_PATH, + FIND_ASSETS, LIST_ALBUMS_PATH, LIST_MISSING_LOCATION_ASSETS_PATH, LIST_MISSING_LOCATION_DATES_PATH, @@ -45,4 +46,8 @@ export interface IUpdateAssetsParams { export const updateAssets = async (params: IUpdateAssetsParams) => { return API.put(UPDATE_ASSETS_PATH, params); } - \ No newline at end of file + + +export const findAssets = async (query: string) => { + return API.post(FIND_ASSETS, { query }); +} \ No newline at end of file diff --git a/src/helpers/asset.helper.ts b/src/helpers/asset.helper.ts index a8236fa..b36bc3d 100644 --- a/src/helpers/asset.helper.ts +++ b/src/helpers/asset.helper.ts @@ -11,4 +11,4 @@ export const cleanUpAsset = (asset: IAsset): IAsset => { previewUrl: ASSET_PREVIEW_PATH(asset.id), videoURL: ASSET_VIDEO_PATH(asset.id), } -} \ No newline at end of file +} diff --git a/src/helpers/data.helper.ts b/src/helpers/data.helper.ts index d201cb7..939493a 100644 --- a/src/helpers/data.helper.ts +++ b/src/helpers/data.helper.ts @@ -3,4 +3,18 @@ export const stringToBoolean = (value: string | boolean): boolean => { return value; } return value === 'true'; +} + +export const removeNullOrUndefinedProperties = (obj: any) => { + return Object.fromEntries(Object.entries(obj).filter(([_, v]) => { + if (Array.isArray(v)) { + return v.length > 0; + } + + if (typeof v === 'number') { + return v !== null && v !== undefined && v !== 0; + } + + return v !== null && v !== undefined && v !== '' && v !== 'null' && v !== 'undefined' && v !== 'null' && v !== 'undefined' + })); } \ No newline at end of file diff --git a/src/helpers/gemini.helper.ts b/src/helpers/gemini.helper.ts new file mode 100644 index 0000000..7eb89c0 --- /dev/null +++ b/src/helpers/gemini.helper.ts @@ -0,0 +1,87 @@ +import { ENV } from "@/config/environment"; +import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai"; +import { z } from "zod"; +import { removeNullOrUndefinedProperties } from "./data.helper"; + +const gemini = new GoogleGenerativeAI(ENV.GEMINI_API_KEY); +const model = gemini.getGenerativeModel({ model: "gemini-1.5-flash" }); + +interface FindQuery { + query: string; + personIds?: string[]; + city?: string; + size?: number; + model?: string; + createdAfter?: string; + state?: string; +} +const responseSchema = { + type: SchemaType.OBJECT, + properties: { + query: { type: SchemaType.STRING, description: "Gist of the query. Remove the details of tags" }, + personIds: { type: SchemaType.ARRAY, + description: "List of person ids that match the query which starts with @", + items: { type: SchemaType.STRING } + }, + city: { type: SchemaType.STRING, + description: "City of the query" + }, + size: { + type: SchemaType.INTEGER, + description: "Size of the file in bytes" + }, + country: { + type: SchemaType.STRING, + description: "Any reference to a country (United States, Canada, etc) in the query, return full country name" + }, + state: { + type: SchemaType.STRING, + description: "Any reference to a state (California, New York, etc) in the query" + }, + takenAfter: { + type: SchemaType.STRING, + description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date" + }, + takenBefore: { + type: SchemaType.STRING, + description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date" + }, + model: { + type: SchemaType.STRING, + description: "Name of the device that the query is about" + }, + }, + required: ["query"], +}; + +export const parseFindQuery = async (query: string): Promise => { + + const prompt = ` + Parse the following query and return the query and tags: ${query} + Additional Information For Parsing: + today's date is ${new Date().toISOString().split('T')[0]} + `; + + const { response } = await model.generateContent({ + contents: [ + { + role: "user", + parts: [ + { + text: prompt, + }, + ], + }, + ], + generationConfig: { + responseMimeType: "application/json", + responseSchema, + } + }); + + const parsedResponse = JSON.parse(response.text()) as FindQuery; + const cleanedResponse = { + ...parsedResponse + }; + return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery; +} diff --git a/src/pages/api/find/search.ts b/src/pages/api/find/search.ts new file mode 100644 index 0000000..b76fe55 --- /dev/null +++ b/src/pages/api/find/search.ts @@ -0,0 +1,42 @@ +import { db } from '@/config/db'; +import { ENV } from '@/config/environment'; +import { getCurrentUser } from '@/handlers/serverUtils/user.utils'; +import { parseFindQuery } from '@/helpers/gemini.helper'; +import { assetFaces, assets, person } from '@/schema'; +import { and, eq, inArray } from 'drizzle-orm'; +import { NextApiRequest, NextApiResponse } from 'next'; + +export default async function search(req: NextApiRequest, res: NextApiResponse) { + const { query } = req.body; + const currentUser = await getCurrentUser(req); + const parsedQuery = await parseFindQuery(query as string); + const url = ENV.IMMICH_URL + "/api/search/smart"; + + return fetch(url, { + method: 'POST', + body: JSON.stringify(parsedQuery), + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${currentUser?.accessToken}` + } + }).then(response => { + if (response.status !== 200) { + return res.json({ + assets: [], + filters: parsedQuery, + error: "Failed to fetch assets", + errorData: response.json() + }); + } + return response.json(); + }) + .then(data => { + res.status(200).json({ + assets: data.assets.items, + filters: parsedQuery + }); + }).catch(err => { + res.status(500).json({ error: err.message }); + }); + +} \ No newline at end of file diff --git a/src/pages/find/index.tsx b/src/pages/find/index.tsx new file mode 100644 index 0000000..88f5318 --- /dev/null +++ b/src/pages/find/index.tsx @@ -0,0 +1,67 @@ +import FindInput from '@/components/find/FindInput'; +import PageLayout from '@/components/layouts/PageLayout'; +import Header from '@/components/shared/Header'; +import Loader from '@/components/ui/loader'; +import { ASSET_THUMBNAIL_PATH } from '@/config/routes'; +import { findAssets } from '@/handlers/api/asset.handler'; +import { IAsset } from '@/types/asset'; +import Image from 'next/image'; +import React, { useState } from 'react' + + +export default function FindPage() { + const [assets, setAssets] = useState([]); + const [loading, setLoading] = useState(false); + const handleSearch = (query: string) => { + setLoading(true); + findAssets(query) + .then(({ assets }) => { + setAssets(assets); + }) + .finally(() => { + setLoading(false); + }); + } + + const renderContent = () => { + if (assets.length === 0) { + return
+ No results found +
+ } + else if (loading) { + return
+ +
+ } + return ( +
+ {assets.map((asset) => ( +
+ {asset.id} +
+ ))} +
+ ) + } + + return ( + +
+
+ +
+ {renderContent()} + + ) +}