Replaygain support for Web Player (#243)

* replaygain!

* resume context

* don't fire both players

* replaygain for jellyfin

* actually remove console.log

---------

Co-authored-by: Jeff <42182408+jeffvli@users.noreply.github.com>
Co-authored-by: jeffvli <jeffvictorli@gmail.com>

src/renderer/api/jellyfin/jellyfin-normalize.ts

@@ -151,6 +151,11 @@ const normalizeSong = (
         createdAt: item.DateCreated,
         discNumber: (item.ParentIndexNumber && item.ParentIndexNumber) || 1,
         duration: item.RunTimeTicks / 10000,
+        gain: item.LUFS
+            ? {
+                  track: -18 - item.LUFS,
+              }
+            : null,
         genres: item.GenreItems?.map((entry) => ({
             id: entry.Id,
             imageUrl: null,
@@ -165,6 +170,7 @@ const normalizeSong = (
         lyrics: null,
         name: item.Name,
         path: (item.MediaSources && item.MediaSources[0]?.Path) || null,
+        peak: null,
         playCount: (item.UserData && item.UserData.PlayCount) || 0,
         playlistItemId: item.PlaylistItemId,
         // releaseDate: (item.ProductionYear && new Date(item.ProductionYear, 0, 1).toISOString()) || null,

src/renderer/api/jellyfin/jellyfin-types.ts

@@ -406,6 +406,7 @@ const song = z.object({
     ImageTags: imageTags,
     IndexNumber: z.number(),
     IsFolder: z.boolean(),
+    LUFS: z.number().optional(),
     LocationType: z.string(),
     MediaSources: z.array(mediaSources),
     MediaType: z.string(),

src/renderer/api/navidrome/navidrome-normalize.ts

@@ -74,7 +74,6 @@ const normalizeSong = (
     });
 
     const imagePlaceholderUrl = null;
-
     return {
         album: item.album,
         albumArtists: [{ id: item.artistId, imageUrl: null, name: item.artist }],
@@ -90,6 +89,10 @@ const normalizeSong = (
         createdAt: item.createdAt.split('T')[0],
         discNumber: item.discNumber,
         duration: item.duration * 1000,
+        gain:
+            item.rgAlbumGain || item.rgTrackGain
+                ? { album: item.rgAlbumGain, track: item.rgTrackGain }
+                : null,
         genres: item.genres?.map((genre) => ({
             id: genre.id,
             imageUrl: null,
@@ -104,6 +107,10 @@ const normalizeSong = (
         lyrics: item.lyrics ? item.lyrics : null,
         name: item.title,
         path: item.path,
+        peak:
+            item.rgAlbumPeak || item.rgTrackPeak
+                ? { album: item.rgAlbumPeak, track: item.rgTrackPeak }
+                : null,
         playCount: item.playCount,
         playlistItemId,
         releaseDate: new Date(item.year, 0, 1).toISOString(),

src/renderer/api/navidrome/navidrome-types.ts

@@ -205,6 +205,10 @@ const song = z.object({
     playCount: z.number(),
     playDate: z.string(),
     rating: z.number().optional(),
+    rgAlbumGain: z.number().optional(),
+    rgAlbumPeak: z.number().optional(),
+    rgTrackGain: z.number().optional(),
+    rgTrackPeak: z.number().optional(),
     size: z.number(),
     sortAlbumArtistName: z.string(),
     sortArtistName: z.string(),

src/renderer/api/subsonic/subsonic-normalize.ts

@@ -68,6 +68,7 @@ const normalizeSong = (
         createdAt: item.created,
         discNumber: item.discNumber || 1,
         duration: item.duration || 0,
+        gain: null,
         genres: item.genre
             ? [
                   {
@@ -86,6 +87,7 @@ const normalizeSong = (
         lyrics: null,
         name: item.title,
         path: item.path,
+        peak: null,
         playCount: item?.playCount || 0,
         releaseDate: null,
         releaseYear: item.year ? String(item.year) : null,

src/renderer/api/types.ts

@@ -171,6 +171,11 @@ export type Album = {
     userRating: number | null;
 } & { songs?: Song[] };
 
+export type GainInfo = {
+    album?: number;
+    track?: number;
+};
+
 export type Song = {
     album: string | null;
     albumArtists: RelatedArtist[];
@@ -186,6 +191,7 @@ export type Song = {
     createdAt: string;
     discNumber: number;
     duration: number;
+    gain: GainInfo | null;
     genres: Genre[];
     id: string;
     imagePlaceholderUrl: string | null;
@@ -195,6 +201,7 @@ export type Song = {
     lyrics: string | null;
     name: string;
     path: string | null;
+    peak: GainInfo | null;
     playCount: number;
     playlistItemId?: string;
     releaseDate: string | null;

src/renderer/components/audio-player/index.tsx

@@ -1,7 +1,7 @@
 import { useImperativeHandle, forwardRef, useRef, useState, useCallback, useEffect } from 'react';
 import isElectron from 'is-electron';
 import type { ReactPlayerProps } from 'react-player';
-import ReactPlayer from 'react-player';
+import ReactPlayer from 'react-player/lazy';
 import type { Song } from '/@/renderer/api/types';
 import {
     crossfadeHandler,
@@ -33,6 +33,11 @@ const getDuration = (ref: any) => {
     return ref.current?.player?.player?.player?.duration;
 };
 
+type WebAudio = {
+    context: AudioContext;
+    gain: GainNode;
+};
+
 export const AudioPlayer = forwardRef(
     (
         {
@@ -49,10 +54,86 @@ export const AudioPlayer = forwardRef(
         }: AudioPlayerProps,
         ref: any,
     ) => {
-        const player1Ref = useRef<any>(null);
+        const player1Ref = useRef<ReactPlayer>(null);
-        const player2Ref = useRef<any>(null);
+        const player2Ref = useRef<ReactPlayer>(null);
         const [isTransitioning, setIsTransitioning] = useState(false);
         const audioDeviceId = useSettingsStore((state) => state.playback.audioDeviceId);
+        const playback = useSettingsStore((state) => state.playback.mpvProperties);
+
+        const [webAudio, setWebAudio] = useState<WebAudio | null>(null);
+        const [player1Source, setPlayer1Source] = useState<MediaElementAudioSourceNode | null>(
+            null,
+        );
+        const [player2Source, setPlayer2Source] = useState<MediaElementAudioSourceNode | null>(
+            null,
+        );
+        const calculateReplayGain = useCallback(
+            (song: Song): number => {
+                if (playback.replayGainMode === 'no') {
+                    return 1;
+                }
+
+                let gain: number | undefined;
+                let peak: number | undefined;
+
+                if (playback.replayGainMode === 'track') {
+                    gain = song.gain?.track ?? song.gain?.album;
+                    peak = song.peak?.track ?? song.peak?.album;
+                } else {
+                    gain = song.gain?.album ?? song.gain?.track;
+                    peak = song.peak?.album ?? song.peak?.track;
+                }
+
+                if (gain === undefined) {
+                    gain = playback.replayGainFallbackDB;
+
+                    if (!gain) {
+                        return 1;
+                    }
+                }
+
+                if (peak === undefined) {
+                    peak = 1;
+                }
+
+                const preAmp = playback.replayGainPreampDB ?? 0;
+
+                // https://wiki.hydrogenaud.io/index.php?title=ReplayGain_1.0_specification&section=19
+                // Normalized to max gain
+                const expectedGain = 10 ** ((gain + preAmp) / 20);
+
+                if (playback.replayGainClip) {
+                    return Math.min(expectedGain, 1 / peak);
+                }
+                return expectedGain;
+            },
+            [
+                playback.replayGainClip,
+                playback.replayGainFallbackDB,
+                playback.replayGainMode,
+                playback.replayGainPreampDB,
+            ],
+        );
+
+        useEffect(() => {
+            if ('AudioContext' in window) {
+                const context = new AudioContext({
+                    latencyHint: 'playback',
+                    sampleRate: playback.audioSampleRateHz || undefined,
+                });
+                const gain = context.createGain();
+                gain.connect(context.destination);
+
+                setWebAudio({ context, gain });
+
+                return () => {
+                    return context.close();
+                };
+            }
+            return () => {};
+            // Intentionally ignore the sample rate dependency, as it makes things really messy
+            // eslint-disable-next-line react-hooks/exhaustive-deps
+        }, []);
 
         useImperativeHandle(ref, () => ({
             get player1() {
@@ -159,10 +240,71 @@ export const AudioPlayer = forwardRef(
             }
         }, [audioDeviceId]);
 
+        useEffect(() => {
+            if (webAudio && player1Source) {
+                if (player1 === undefined) {
+                    player1Source.disconnect();
+                    setPlayer1Source(null);
+                } else if (currentPlayer === 1) {
+                    webAudio.gain.gain.setValueAtTime(calculateReplayGain(player1), 0);
+                }
+            }
+        }, [calculateReplayGain, currentPlayer, player1, player1Source, webAudio]);
+
+        useEffect(() => {
+            if (webAudio && player2Source) {
+                if (player2 === undefined) {
+                    player2Source.disconnect();
+                    setPlayer2Source(null);
+                } else if (currentPlayer === 2) {
+                    webAudio.gain.gain.setValueAtTime(calculateReplayGain(player2), 0);
+                }
+            }
+        }, [calculateReplayGain, currentPlayer, player2, player2Source, webAudio]);
+
+        const handlePlayer1Start = useCallback(
+            async (player: ReactPlayer) => {
+                if (!webAudio || player1Source) return;
+                if (webAudio.context.state !== 'running') {
+                    await webAudio.context.resume();
+                }
+
+                const internal = player.getInternalPlayer() as HTMLMediaElement | undefined;
+                if (internal) {
+                    const { context, gain } = webAudio;
+                    const source = context.createMediaElementSource(internal);
+                    source.connect(gain);
+                    setPlayer1Source(source);
+                }
+            },
+            [player1Source, webAudio],
+        );
+
+        const handlePlayer2Start = useCallback(
+            async (player: ReactPlayer) => {
+                if (!webAudio || player2Source) return;
+                if (webAudio.context.state !== 'running') {
+                    await webAudio.context.resume();
+                }
+
+                const internal = player.getInternalPlayer() as HTMLMediaElement | undefined;
+                if (internal) {
+                    const { context, gain } = webAudio;
+                    const source = context.createMediaElementSource(internal);
+                    source.connect(gain);
+                    setPlayer2Source(source);
+                }
+            },
+            [player2Source, webAudio],
+        );
+
         return (
             <>
                 <ReactPlayer
                     ref={player1Ref}
+                    config={{
+                        file: { attributes: { crossOrigin: 'anonymous' }, forceAudio: true },
+                    }}
                     height={0}
                     muted={muted}
                     playing={currentPlayer === 1 && status === PlayerStatus.PLAYING}
@@ -174,9 +316,13 @@ export const AudioPlayer = forwardRef(
                     onProgress={
                         playbackStyle === PlaybackStyle.GAPLESS ? handleGapless1 : handleCrossfade1
                     }
+                    onReady={handlePlayer1Start}
                 />
                 <ReactPlayer
                     ref={player2Ref}
+                    config={{
+                        file: { attributes: { crossOrigin: 'anonymous' }, forceAudio: true },
+                    }}
                     height={0}
                     muted={muted}
                     playing={currentPlayer === 2 && status === PlayerStatus.PLAYING}
@@ -188,6 +334,7 @@ export const AudioPlayer = forwardRef(
                     onProgress={
                         playbackStyle === PlaybackStyle.GAPLESS ? handleGapless2 : handleCrossfade2
                     }
+                    onReady={handlePlayer2Start}
                 />
             </>
         );

src/renderer/features/settings/components/playback/mpv-settings.tsx

@@ -195,7 +195,7 @@ export const MpvSettings = () => {
             ),
             description:
                 'Select the output sample rate to be used if the sample frequency selected is different from that of the current media',
-            isHidden: settings.type !== PlaybackType.LOCAL,
+            note: 'Page refresh required for web player',
             title: 'Sample rate',
         },
         {
@@ -233,7 +233,6 @@ export const MpvSettings = () => {
             ),
             description:
                 'Adjust volume gain according to replaygain values stored in the file metadata (--replaygain)',
-            isHidden: settings.type !== PlaybackType.LOCAL,
             note: 'Restart required',
             title: 'ReplayGain mode',
         },
@@ -247,7 +246,6 @@ export const MpvSettings = () => {
             ),
             description:
                 'Pre-amplification gain in dB to apply to the selected replaygain gain (--replaygain-preamp)',
-            isHidden: settings.type !== PlaybackType.LOCAL,
             title: 'ReplayGain preamp (dB)',
         },
         {
@@ -261,7 +259,6 @@ export const MpvSettings = () => {
             ),
             description:
                 'Prevent clipping caused by replaygain by automatically lowering the gain (--replaygain-clip)',
-            isHidden: settings.type !== PlaybackType.LOCAL,
             title: 'ReplayGain clipping',
         },
         {
@@ -274,7 +271,6 @@ export const MpvSettings = () => {
             ),
             description:
                 'Gain in dB to apply if the file has no replay gain tags. This option is always applied if the replaygain logic is somehow inactive. If this is applied, no other replaygain options are applied',
-            isHidden: settings.type !== PlaybackType.LOCAL,
             title: 'ReplayGain fallback (dB)',
         },
     ];

src/renderer/features/settings/components/playback/playback-tab.tsx

@@ -1,4 +1,4 @@
-import { lazy, Suspense } from 'react';
+import { lazy, Suspense, useMemo } from 'react';
 import { Divider, Stack } from '@mantine/core';
 import { AudioSettings } from '/@/renderer/features/settings/components/playback/audio-settings';
 import { ScrobbleSettings } from '/@/renderer/features/settings/components/playback/scrobble-settings';
@@ -12,13 +12,20 @@ const MpvSettings = lazy(() =>
 );
 
 export const PlaybackTab = () => {
+    const hasFancyAudio = useMemo(() => {
+        return isElectron() || 'AudioContext' in window;
+    }, []);
     return (
         <Stack spacing="md">
             <AudioSettings />
-            <Suspense fallback={<></>}>{isElectron() && <MpvSettings />}</Suspense>
+            <Suspense fallback={<></>}>{hasFancyAudio && <MpvSettings />}</Suspense>
             <Divider />
+            {isElectron() && (
+                <>
                     <ScrobbleSettings />
                     <Divider />
+                </>
+            )}
             <LyricSettings />
         </Stack>
     );

src/renderer/features/settings/components/playback/scrobble-settings.tsx

@@ -1,4 +1,3 @@
-import isElectron from 'is-electron';
 import { NumberInput, Slider, Switch, Text } from '/@/renderer/components';
 import { usePlaybackSettings, useSettingsStoreActions } from '/@/renderer/store/settings.store';
 import { SettingOption, SettingsSection } from '../settings-section';
@@ -27,7 +26,6 @@ export const ScrobbleSettings = () => {
                 />
             ),
             description: 'Enable or disable scrobbling to your media server',
-            isHidden: !isElectron(),
             title: 'Scrobble',
         },
         {
@@ -54,7 +52,6 @@ export const ScrobbleSettings = () => {
             ),
             description:
                 'The percentage of the song that must be played before submitting a scrobble',
-            isHidden: !isElectron(),
             title: 'Minimum scrobble percentage*',
         },
         {
@@ -81,7 +78,6 @@ export const ScrobbleSettings = () => {
             ),
             description:
                 'The duration in seconds of a song that must be played before submitting a scrobble',
-            isHidden: !isElectron(),
             title: 'Minimum scrobble duration (seconds)*',
         },
     ];