feat: base setup for search photos

Signed-off-by: Varun Raj <varun@skcript.com>

package.json

@@ -14,6 +14,7 @@
     "url": "https://varunraj.in"
   },
   "dependencies": {
+    "@google/generative-ai": "^0.21.0",
     "@radix-ui/react-accordion": "^1.2.1",
     "@radix-ui/react-avatar": "^1.1.0",
     "@radix-ui/react-checkbox": "^1.1.2",
@@ -50,18 +51,21 @@
     "next-themes": "^0.3.0",
     "pg": "^8.12.0",
     "qs": "^6.13.0",
+    "rc-mentions": "^2.18.0",
     "react": "^18",
     "react-calendar-heatmap": "^1.9.0",
     "react-day-picker": "9.0.8",
     "react-dom": "^18",
     "react-grid-gallery": "^1.0.1",
     "react-leaflet": "^4.2.1",
+    "react-mentions": "^4.4.10",
     "react-query": "^3.39.3",
     "recharts": "^2.12.7",
     "sass": "^1.77.8",
     "tailwind-merge": "^2.5.2",
     "tailwindcss-animate": "^1.0.7",
-    "yet-another-react-lightbox": "^3.21.4"
+    "yet-another-react-lightbox": "^3.21.4",
+    "zod": "^3.24.1"
   },
   "devDependencies": {
     "@semantic-release/commit-analyzer": "^13.0.0",
@@ -75,6 +79,7 @@
     "@types/pg": "^8.11.6",
     "@types/react": "^18",
     "@types/react-dom": "^18",
+    "@types/react-mentions": "^4.4.1",
     "autoprefixer": "^10.4.20",
     "drizzle-kit": "^0.24.0",
     "eslint": "^8",

src/components/find/FindInput.tsx

@@ -0,0 +1,58 @@
+import React, { useState } from 'react'
+import style from './findInputStyle'
+import { MentionsInput, Mention } from 'react-mentions'
+import { searchPeople } from '@/handlers/api/people.handler';
+import mentionStyle from './findMentionStyle';
+import { PERSON_THUBNAIL_PATH } from '@/config/routes';
+import Image from 'next/image';
+
+interface FindInputProps {
+  onSearch: (query: string) => void;
+}
+
+export default function FindInput({ onSearch }: FindInputProps) {
+
+  const [query, setQuery] = useState("");
+
+  const handleSearchPeople = async (e: any, callback: any) => {
+    if (!e.length) return [];
+    return searchPeople(e).then((people) => people.map((person: any) => ({
+      id: person.id,
+      display: person.name,
+    }))).then((people) => callback(people));
+  }
+
+  return (
+    <MentionsInput
+      value={query}
+      singleLine={true}
+      style={style}
+      className='w-full border border-gray-200 rounded-md p-2'
+      classNames={{
+        control: 'w-full border border-gray-200 rounded-md p-2',
+        input: 'w-full border border-gray-200 rounded-md p-2 w-full',
+      }}
+      placeholder='Search for people or tags'
+      onKeyDown={(e) => {
+        if (e.key === 'Enter') {
+          onSearch(query);
+        }
+      }}
+      onChange={(e) => setQuery(e.target.value)}>
+      <Mention
+        trigger="@"
+        data={handleSearchPeople}
+        style={mentionStyle}
+        renderSuggestion={(
+          highlightedDisplay,
+          focused
+        ) => (
+          <div className={`user ${focused ? 'focused' : ''}`}>
+            
+            {highlightedDisplay.display}
+          </div>
+        )}
+      />
+    </MentionsInput>
+  )
+}

src/components/find/findInputStyle.ts

@@ -0,0 +1,40 @@
+const findInputStyle = {
+  control: {
+    backgroundColor: '#fff',
+    fontSize: 14,
+    fontWeight: 'normal',
+  },
+
+
+  '&singleLine': {
+    display: 'inline-block',
+    
+    highlighter: {
+      padding: '10px',
+      border: '2px inset transparent',
+    },
+    input: {
+      width: "100%",
+      padding: '10px',
+      borderRadius: '4px',
+      border: '1px solid #e0e0e0',
+    },
+  },
+
+  suggestions: {
+    list: {
+      backgroundColor: 'white',
+      border: '1px solid rgba(0,0,0,0.15)',
+      fontSize: 14,
+    },
+    item: {
+      padding: '5px 15px',
+      borderBottom: '1px solid rgba(0,0,0,0.15)',
+      '&focused': {
+        backgroundColor: '#cee4e5',
+      },
+    },
+  },
+}
+
+export default findInputStyle;

src/components/find/findMentionStyle.ts

@@ -0,0 +1,3 @@
+export default {
+  backgroundColor: '#cee4e5',
+}

src/config/constants/sidebarNavs.tsx

@@ -1,4 +1,4 @@
-import { Album, GalleryHorizontal, GalleryVerticalEnd, Home, Image, LocateOff, MapPinX, User } from "lucide-react";
+import { Album, GalleryHorizontal, GalleryVerticalEnd, Home, Image, LocateOff, MapPinX, Search, User } from "lucide-react";
 
 export const sidebarNavs = [
   {
@@ -26,4 +26,9 @@ export const sidebarNavs = [
     link: "/albums",
     icon: <GalleryHorizontal className="h-4 w-4" />,
   },
+  {
+    title: "Find",
+    link: "/find",
+    icon: <Search className="h-4 w-4" />,
+  },
 ];

src/config/environment.ts

@@ -5,7 +5,8 @@ export const ENV = {
   DATABASE_URL: (process.env.DATABASE_URL || `postgresql://${process.env.DB_USERNAME}:${process.env.DB_PASSWORD}@${(process.env.DB_HOST || 'immich_postgres')}:${(process.env.DB_PORT || '5432')}/${process.env.DB_DATABASE_NAME}`),
   JWT_SECRET: process.env.JWT_SECRET as string,
   SECURE_COOKIE: process.env.SECURE_COOKIE === 'true',
-  VERSION: process.env.VERSION
+  VERSION: process.env.VERSION,
+  GEMINI_API_KEY: process.env.GEMINI_API_KEY as string,
 };
 
 

src/config/routes.ts

@@ -49,3 +49,6 @@ export const GET_FILTERS = BASE_API_ENDPOINT + "/filters/asset-filters";
 
 // Person
 export const GET_PERSON_INFO = (personId: string) => BASE_API_ENDPOINT + "/people/" + personId + "/info";
+
+// Find
+export const FIND_ASSETS = BASE_API_ENDPOINT + "/find/search";

src/handlers/api/asset.handler.ts

@@ -1,5 +1,6 @@
 import {
   ADD_ASSETS_ALBUMS_PATH,
+  FIND_ASSETS,
   LIST_ALBUMS_PATH,
   LIST_MISSING_LOCATION_ASSETS_PATH,
   LIST_MISSING_LOCATION_DATES_PATH,
@@ -46,3 +47,7 @@ export const updateAssets = async (params: IUpdateAssetsParams) => {
   return API.put(UPDATE_ASSETS_PATH, params);
 }
   
+
+export const findAssets = async (query: string) => {
+  return API.post(FIND_ASSETS, { query });
+}

src/helpers/data.helper.ts

@@ -4,3 +4,17 @@ export const stringToBoolean = (value: string | boolean): boolean => {
   }
   return value === 'true';
 }
+
+export const removeNullOrUndefinedProperties = (obj: any) => {
+  return Object.fromEntries(Object.entries(obj).filter(([_, v]) => {
+    if (Array.isArray(v)) {
+      return v.length > 0;
+    }
+
+    if (typeof v === 'number') {
+      return v !== null && v !== undefined && v !== 0;
+    }
+
+    return v !== null && v !== undefined && v !== '' && v !== 'null' && v !== 'undefined' && v !== 'null' && v !== 'undefined'
+  }));
+}

src/helpers/gemini.helper.ts

@@ -0,0 +1,87 @@
+import { ENV } from "@/config/environment";
+import { GoogleGenerativeAI, SchemaType } from "@google/generative-ai";
+import { z } from "zod";
+import { removeNullOrUndefinedProperties } from "./data.helper";
+
+const gemini = new GoogleGenerativeAI(ENV.GEMINI_API_KEY);
+const model = gemini.getGenerativeModel({ model: "gemini-1.5-flash" });
+
+interface FindQuery {
+  query: string;
+  personIds?: string[];
+  city?: string;
+  size?: number;
+  model?: string;
+  createdAfter?: string;
+  state?: string;
+}
+const responseSchema = {
+  type: SchemaType.OBJECT,
+  properties: {
+    query: { type: SchemaType.STRING, description: "Gist of the query. Remove the details of tags" },
+    personIds: { type: SchemaType.ARRAY,
+      description: "List of person ids that match the query which starts with @",
+       items: { type: SchemaType.STRING } 
+    },
+    city: { type: SchemaType.STRING,
+      description: "City of the query"
+    },
+    size: {
+      type: SchemaType.INTEGER,
+      description: "Size of the file in bytes"
+    },
+    country: {
+      type: SchemaType.STRING,
+      description: "Any reference to a country (United States, Canada, etc) in the query, return full country name"
+    },
+    state: {
+      type: SchemaType.STRING,
+      description: "Any reference to a state (California, New York, etc) in the query"
+    },
+    takenAfter: {
+      type: SchemaType.STRING,
+      description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
+    },
+    takenBefore: {
+      type: SchemaType.STRING,
+      description: "Extract the date from the query and return it in the format YYYY-MM-DD but only if it is a valid date"
+    },
+    model: {
+      type: SchemaType.STRING,
+      description: "Name of the device that the query is about"
+    },
+  },
+  required: ["query"],
+};
+
+export const parseFindQuery = async (query: string): Promise<FindQuery>  => {
+
+  const prompt = `
+  Parse the following query and return the query and tags: ${query}
+  Additional Information For Parsing:
+  today's date is ${new Date().toISOString().split('T')[0]}
+  `;
+
+  const { response } = await model.generateContent({
+    contents: [
+      {
+        role: "user",
+        parts: [
+          {
+            text: prompt,
+          },
+        ],
+      },
+    ],
+    generationConfig: {
+      responseMimeType: "application/json",
+      responseSchema,
+    }
+  });
+
+  const parsedResponse = JSON.parse(response.text()) as FindQuery;
+  const cleanedResponse = {
+    ...parsedResponse
+  };
+  return removeNullOrUndefinedProperties(cleanedResponse) as any as FindQuery;
+}

src/pages/api/find/search.ts

@@ -0,0 +1,42 @@
+import { db } from '@/config/db';
+import { ENV } from '@/config/environment';
+import { getCurrentUser } from '@/handlers/serverUtils/user.utils';
+import { parseFindQuery } from '@/helpers/gemini.helper';
+import { assetFaces, assets, person } from '@/schema';
+import { and, eq, inArray } from 'drizzle-orm';
+import { NextApiRequest, NextApiResponse } from 'next';
+
+export default async function search(req: NextApiRequest, res: NextApiResponse) {
+  const { query } = req.body;
+  const currentUser = await getCurrentUser(req);  
+  const parsedQuery = await parseFindQuery(query as string);
+  const url = ENV.IMMICH_URL + "/api/search/smart";
+
+  return fetch(url, {
+    method: 'POST',
+    body: JSON.stringify(parsedQuery),
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${currentUser?.accessToken}`
+    }
+  }).then(response => {
+    if (response.status !== 200) {
+      return res.json({
+        assets: [],
+        filters: parsedQuery,
+        error: "Failed to fetch assets",
+        errorData: response.json()
+      });
+    }
+    return response.json();
+  })
+  .then(data => {
+    res.status(200).json({
+      assets: data.assets.items,
+      filters: parsedQuery
+    });
+  }).catch(err => {
+    res.status(500).json({ error: err.message });
+  });
+
+}

src/pages/find/index.tsx

@@ -0,0 +1,67 @@
+import FindInput from '@/components/find/FindInput';
+import PageLayout from '@/components/layouts/PageLayout';
+import Header from '@/components/shared/Header';
+import Loader from '@/components/ui/loader';
+import { ASSET_THUMBNAIL_PATH } from '@/config/routes';
+import { findAssets } from '@/handlers/api/asset.handler';
+import { IAsset } from '@/types/asset';
+import Image from 'next/image';
+import React, { useState } from 'react'
+
+
+export default function FindPage() {
+  const [assets, setAssets] = useState<IAsset[]>([]);
+  const [loading, setLoading] = useState(false);
+  const handleSearch = (query: string) => {
+    setLoading(true);
+    findAssets(query)
+      .then(({ assets }) => {
+        setAssets(assets);
+      })
+      .finally(() => {
+        setLoading(false);
+      });
+  }
+
+  const renderContent = () => {
+    if (assets.length === 0) {
+      return <div className="flex justify-center items-center h-full">
+        No results found
+      </div>
+    }
+    else if (loading) {
+      return <div className="flex justify-center items-center h-full">
+        <Loader />
+      </div>
+    }
+    return (
+      <div className="grid grid-cols-5 gap-4 p-2">
+        {assets.map((asset) => (
+          <div key={asset.id} className='w-full h-[200px] rounded-md overflow-hidden'>
+            <Image
+              src={ASSET_THUMBNAIL_PATH(asset.id)}
+              alt={asset.id}
+              style={{
+                objectFit: 'cover',
+                width: '100%',
+                height: '100%',
+              }}
+              width={200}
+              height={200}
+            />
+          </div>
+        ))}
+      </div>
+    )
+  }
+
+  return (
+    <PageLayout>
+      <Header leftComponent="Find" />
+      <div className="flex flex-col gap-4 p-2">
+        <FindInput onSearch={handleSearch} />
+      </div>
+      {renderContent()}
+    </PageLayout>
+  )
+}