add jellyfin, improvements

src/renderer/api/controller.ts

@@ -46,6 +46,8 @@ import type {
   AuthenticationResponse,
   SearchArgs,
   SearchResponse,
+  LyricsArgs,
+  SynchronizedLyricsArray,
 } from '/@/renderer/api/types';
 import { ServerType } from '/@/renderer/types';
 import { DeletePlaylistResponse, RandomSongListArgs } from './types';
@@ -76,6 +78,7 @@ export type ControllerEndpoint = Partial<{
   getFolderList: () => void;
   getFolderSongs: () => void;
   getGenreList: (args: GenreListArgs) => Promise<GenreListResponse>;
+  getLyrics: (args: LyricsArgs) => Promise<SynchronizedLyricsArray>;
   getMusicFolderList: (args: MusicFolderListArgs) => Promise<MusicFolderListResponse>;
   getPlaylistDetail: (args: PlaylistDetailArgs) => Promise<PlaylistDetailResponse>;
   getPlaylistList: (args: PlaylistListArgs) => Promise<PlaylistListResponse>;
@@ -119,6 +122,7 @@ const endpoints: ApiController = {
     getFolderList: undefined,
     getFolderSongs: undefined,
     getGenreList: jfController.getGenreList,
+    getLyrics: jfController.getLyrics,
     getMusicFolderList: jfController.getMusicFolderList,
     getPlaylistDetail: jfController.getPlaylistDetail,
     getPlaylistList: jfController.getPlaylistList,
@@ -154,6 +158,7 @@ const endpoints: ApiController = {
     getFolderList: undefined,
     getFolderSongs: undefined,
     getGenreList: ndController.getGenreList,
+    getLyrics: undefined,
     getMusicFolderList: ssController.getMusicFolderList,
     getPlaylistDetail: ndController.getPlaylistDetail,
     getPlaylistList: ndController.getPlaylistList,
@@ -188,6 +193,7 @@ const endpoints: ApiController = {
     getFolderList: undefined,
     getFolderSongs: undefined,
     getGenreList: undefined,
+    getLyrics: undefined,
     getMusicFolderList: ssController.getMusicFolderList,
     getPlaylistDetail: undefined,
     getPlaylistList: undefined,
@@ -448,6 +454,12 @@ const getRandomSongList = async (args: RandomSongListArgs) => {
   )?.(args);
 };
 
+const getLyrics = async (args: LyricsArgs) => {
+  return (
+    apiController('getLyrics', args.apiClientProps.server?.type) as ControllerEndpoint['getLyrics']
+  )?.(args);
+};
+
 export const controller = {
   addToPlaylist,
   authenticate,
@@ -461,6 +473,7 @@ export const controller = {
   getAlbumList,
   getArtistList,
   getGenreList,
+  getLyrics,
   getMusicFolderList,
   getPlaylistDetail,
   getPlaylistList,

src/renderer/api/jellyfin/jellyfin-api.ts

@@ -174,6 +174,14 @@ export const contract = c.router({
       400: jfType._response.error,
     },
   },
+  getSongLyrics: {
+    method: 'GET',
+    path: 'users/:userId/Items/:id/Lyrics',
+    responses: {
+      200: jfType._response.lyrics,
+      404: jfType._response.error,
+    },
+  },
   getTopSongsList: {
     method: 'GET',
     path: 'users/:userId/items',

src/renderer/api/jellyfin/jellyfin-controller.ts

@@ -44,6 +44,8 @@ import {
   SearchResponse,
   RandomSongListResponse,
   RandomSongListArgs,
+  LyricsArgs,
+  SynchronizedLyricsArray,
 } from '/@/renderer/api/types';
 import { jfApiClient } from '/@/renderer/api/jellyfin/jellyfin-api';
 import { jfNormalize } from './jellyfin-normalize';
@@ -846,6 +848,28 @@ const getRandomSongList = async (args: RandomSongListArgs): Promise<RandomSongLi
     totalRecordCount: res.body.Items.length || 0,
   };
 };
+
+const getLyrics = async (args: LyricsArgs): Promise<SynchronizedLyricsArray> => {
+  const { query, apiClientProps } = args;
+
+  if (!apiClientProps.server?.userId) {
+    throw new Error('No userId found');
+  }
+
+  const res = await jfApiClient(apiClientProps).getSongLyrics({
+    params: {
+      id: query.songId,
+      userId: apiClientProps.server?.userId,
+    },
+  });
+
+  if (res.status !== 200) {
+    throw new Error('Failed to get lyrics');
+  }
+
+  return res.body.Lyrics.map((lyric) => [lyric.Start / 1e4, lyric.Text]);
+};
+
 export const jfController = {
   addToPlaylist,
   authenticate,
@@ -859,6 +883,7 @@ export const jfController = {
   getAlbumList,
   getArtistList,
   getGenreList,
+  getLyrics,
   getMusicFolderList,
   getPlaylistDetail,
   getPlaylistList,

src/renderer/api/jellyfin/jellyfin-types.ts

@@ -631,6 +631,15 @@ const searchParameters = paginationParameters.merge(baseParameters);
 
 const search = z.any();
 
+const lyricText = z.object({
+  Start: z.number(),
+  Text: z.string(),
+});
+
+const lyrics = z.object({
+  Lyrics: z.array(lyricText),
+});
+
 export const jfType = {
   _enum: {
     collection: jfCollection,
@@ -670,6 +679,7 @@ export const jfType = {
     favorite,
     genre,
     genreList,
+    lyrics,
     musicFolderList,
     playlist,
     playlistList,

src/renderer/api/query-keys.ts

@@ -14,6 +14,7 @@ import type {
   SearchQuery,
   SongDetailQuery,
   RandomSongListQuery,
+  LyricsQuery,
 } from './types';
 
 export const queryKeys: Record<
@@ -102,6 +103,10 @@ export const queryKeys: Record<
       if (query) return [serverId, 'songs', 'list', query] as const;
       return [serverId, 'songs', 'list'] as const;
     },
+    lyrics: (serverId: string, query?: LyricsQuery) => {
+      if (query) return [serverId, 'song', 'lyrics', query] as const;
+      return [serverId, 'song', 'lyrics'] as const;
+    },
     randomSongList: (serverId: string, query?: RandomSongListQuery) => {
       if (query) return [serverId, 'songs', 'randomSongList', query] as const;
       return [serverId, 'songs', 'randomSongList'] as const;

src/renderer/api/types.ts

@@ -1017,6 +1017,16 @@ export type RandomSongListArgs = {
 
 export type RandomSongListResponse = SongListResponse;
 
+export type LyricsQuery = {
+  songId: string;
+};
+
+export type LyricsArgs = {
+  query: LyricsQuery;
+} & BaseEndpointArgs;
+
+export type SynchronizedLyricsArray = Array<[number, string]>;
+
 export const instanceOfCancellationError = (error: any) => {
   return 'revert' in error;
 };

src/renderer/features/lyrics/lyrics.tsx

@@ -3,9 +3,14 @@ import isElectron from 'is-electron';
 import { ErrorBoundary } from 'react-error-boundary';
 import { ErrorFallback } from '/@/renderer/features/action-required';
 import { useCurrentServer, useCurrentSong } from '/@/renderer/store';
-import { SynchronizedLyricsArray, SynchronizedLyrics } from './synchronized-lyrics';
+import { SynchronizedLyrics } from './synchronized-lyrics';
 import { UnsynchronizedLyrics } from '/@/renderer/features/lyrics/unsynchronized-lyrics';
 import { LyricLine } from '/@/renderer/features/lyrics/lyric-line';
+import { Center, Group } from '@mantine/core';
+import { RiInformationFill } from 'react-icons/ri';
+import { TextTitle } from '/@/renderer/components';
+import { SynchronizedLyricsArray } from '/@/renderer/api/types';
+import { useSongLyrics } from '/@/renderer/features/lyrics/queries/lyric-query';
 
 const lyrics = isElectron() ? window.electron.lyrics : null;
 
@@ -22,6 +27,11 @@ export const Lyrics = () => {
   const [source, setSource] = useState<string | null>(null);
   const [songLyrics, setSongLyrics] = useState<SynchronizedLyricsArray | string | null>(null);
 
+  const remoteLyrics = useSongLyrics({
+    query: { songId: currentSong?.id ?? '' },
+    serverId: currentServer?.id,
+  });
+
   const songRef = useRef<string | null>(null);
 
   useEffect(() => {
@@ -38,7 +48,7 @@ export const Lyrics = () => {
   }, []);
 
   useEffect(() => {
-    if (currentSong && !currentSong.lyrics) {
+    if (currentSong && !currentSong.lyrics && !remoteLyrics.isLoading && !remoteLyrics.isSuccess) {
       lyrics?.fetchLyrics(currentSong);
     }
 
@@ -46,7 +56,7 @@ export const Lyrics = () => {
 
     setOverride(null);
     setSource(null);
-  }, [currentSong]);
+  }, [currentSong, remoteLyrics.isLoading, remoteLyrics.isSuccess]);
 
   useEffect(() => {
     let lyrics: string | null = null;
@@ -57,6 +67,10 @@ export const Lyrics = () => {
       setSource(currentServer?.name ?? 'music server');
     } else if (override) {
       lyrics = override;
+    } else if (remoteLyrics.isSuccess) {
+      setSource(currentServer?.name ?? 'music server');
+      setSongLyrics(remoteLyrics.data!);
+      return;
     }
 
     if (lyrics) {
@@ -82,16 +96,23 @@ export const Lyrics = () => {
     } else {
       setSongLyrics(null);
     }
-  }, [currentServer?.name, currentSong, override]);
+  }, [currentServer?.name, currentSong, override, remoteLyrics.data, remoteLyrics.isSuccess]);
 
   return (
     <ErrorBoundary FallbackComponent={ErrorFallback}>
-      {songLyrics &&
+      {!songLyrics && (
-        (Array.isArray(songLyrics) ? (
+        <Center>
-          <SynchronizedLyrics lyrics={songLyrics} />
+          <Group>
-        ) : (
+            <RiInformationFill size="2rem" />
-          <UnsynchronizedLyrics lyrics={songLyrics} />
+            <TextTitle
-        ))}
+              order={3}
+              weight={700}
+            >
+              No lyrics found
+            </TextTitle>
+          </Group>
+        </Center>
+      )}
       {source && (
         <LyricLine
           key="provided-by"
@@ -99,6 +120,12 @@ export const Lyrics = () => {
           text={`Provided by: ${source}`}
         />
       )}
+      {songLyrics &&
+        (Array.isArray(songLyrics) ? (
+          <SynchronizedLyrics lyrics={songLyrics} />
+        ) : (
+          <UnsynchronizedLyrics lyrics={songLyrics} />
+        ))}
     </ErrorBoundary>
   );
 };

src/renderer/features/lyrics/queries/lyric-query.ts

@@ -0,0 +1,25 @@
+import { useQuery } from '@tanstack/react-query';
+import { LyricsQuery } from '/@/renderer/api/types';
+import { QueryHookArgs } from '/@/renderer/lib/react-query';
+import { getServerById } from '/@/renderer/store';
+import { controller } from '/@/renderer/api/controller';
+import { queryKeys } from '/@/renderer/api/query-keys';
+import { ServerType } from '/@/renderer/types';
+
+export const useSongLyrics = (args: QueryHookArgs<LyricsQuery>) => {
+  const { query, serverId } = args;
+  const server = getServerById(serverId);
+
+  return useQuery({
+    // Note: This currently fetches for every song, even if it shouldn't have
+    // lyrics, because for some reason HasLyrics is not exposed. Thus, ignore the error
+    onError: () => {},
+    queryFn: ({ signal }) => {
+      if (!server) throw new Error('Server not found');
+      // This should only be called for Jellyfin. Return null to ignore errors
+      if (server.type !== ServerType.JELLYFIN) return null;
+      return controller.getLyrics({ apiClientProps: { server, signal }, query });
+    },
+    queryKey: queryKeys.songs.lyrics(server?.id || '', query),
+  });
+};

src/renderer/features/lyrics/synchronized-lyrics.tsx

@@ -10,11 +10,10 @@ import { PlaybackType, PlayerStatus } from '/@/renderer/types';
 import { LyricLine } from '/@/renderer/features/lyrics/lyric-line';
 import isElectron from 'is-electron';
 import { PlayersRef } from '/@/renderer/features/player/ref/players-ref';
+import { SynchronizedLyricsArray } from '/@/renderer/api/types';
 
 const mpvPlayer = isElectron() ? window.electron.mpvPlayer : null;
 
-export type SynchronizedLyricsArray = Array<[number, string]>;
-
 interface SynchronizedLyricsProps {
   lyrics: SynchronizedLyricsArray;
 }
@@ -40,7 +39,12 @@ export const SynchronizedLyrics = ({ lyrics }: SynchronizedLyricsProps) => {
   // whether to proceed or stop
   const timerEpoch = useRef(0);
 
-  const followRef = useRef<boolean>(settings.follow);
+  const delayMsRef = useRef(settings.delayMs);
+  const followRef = useRef(settings.follow);
+
+  useEffect(() => {
+    delayMsRef.current = settings.delayMs;
+  }, [settings.delayMs]);
 
   useEffect(() => {
     // Copy the follow settings into a ref that can be accessed in the timeout
@@ -127,7 +131,7 @@ export const SynchronizedLyrics = ({ lyrics }: SynchronizedLyricsProps) => {
     }
 
     if (index !== lyricRef.current!.length - 1) {
-      const [nextTime] = lyricRef.current![index + 1];
+      const nextTime = lyricRef.current![index + 1][0];
 
       const elapsed = performance.now() - start;
 
@@ -149,7 +153,7 @@ export const SynchronizedLyrics = ({ lyrics }: SynchronizedLyricsProps) => {
             return false;
           }
 
-          setCurrentLyric(timeInSec * 1000);
+          setCurrentLyric(timeInSec * 1000 + delayMsRef.current);
 
           return true;
         })
@@ -185,7 +189,7 @@ export const SynchronizedLyrics = ({ lyrics }: SynchronizedLyricsProps) => {
       clearTimeout(lyricTimer.current);
     }
 
-    setCurrentLyric(now * 1000);
+    setCurrentLyric(now * 1000 + delayMsRef.current);
   }, [now, seeked, setCurrentLyric, status]);
 
   useEffect(() => {

src/renderer/features/settings/components/playback/lyric-settings.tsx

@@ -1,4 +1,4 @@
-import { Switch } from '@mantine/core';
+import { NumberInput, Switch } from '@mantine/core';
 import {
   SettingOption,
   SettingsSection,
@@ -82,6 +82,28 @@ export const LyricSettings = () => {
       isHidden: !isElectron(),
       title: 'Providers to fetch music',
     },
+    {
+      control: (
+        <NumberInput
+          defaultValue={settings.delayMs}
+          step={10}
+          width={100}
+          onBlur={(e) => {
+            const value = Number(e.currentTarget.value);
+            setSettings({
+              lyrics: {
+                ...settings,
+                delayMs: value,
+              },
+            });
+          }}
+        />
+      ),
+      description:
+        'Lyric offset (in milliseconds). Positive values mean that lyrics are shown later, and negative mean that lyrics are shown earlier',
+      isHidden: !isElectron(),
+      title: 'Lyric offset',
+    },
   ];
 
   return <SettingsSection options={lyricOptions} />;

src/renderer/store/settings.store.ts

@@ -123,6 +123,7 @@ export interface SettingsState {
     globalMediaHotkeys: boolean;
   };
   lyrics: {
+    delayMs: number;
     fetch: boolean;
     follow: boolean;
     sources: LyricSource[];
@@ -209,6 +210,7 @@ const initialState: SettingsState = {
     globalMediaHotkeys: true,
   },
   lyrics: {
+    delayMs: 0,
     fetch: false,
     follow: true,
     sources: [],