Explicit sample_rate in owhisper client (#1651)

Author: yujonglee

.cursor/rules/simple.mdc

@@ -9,6 +9,11 @@ alwaysApply: true
 # Typescript
 
 - Avoid creating a bunch of types/interfaces if they are not shared. Especially for function props. Just inline them.
+- After some amount of TypeScript changes, run `pnpm -r typecheck`.
+
+# Rust
+
+- After some amount of Rust changes, run `cargo check`.
 
 # Mutation
 - Never do manual state management for form/mutation. Things like setError is anti-pattern. use useForm(from tanstack-form) and useQuery/useMutation(from tanstack-query) for 99% cases.
@@ -19,7 +24,6 @@ alwaysApply: true
 
 # Misc
 - Do not create summary docs or example code file if not requested. Plan is ok.
-- After a significant amount of TypeScript changes, run `pnpm -r typecheck`.
 - If there are many classNames and they have conditional logic, use `cn` (import it with `import { cn } from "@hypr/utils"`). It is similar to `clsx`. Always pass an array. Split by logical grouping.
 - Use `motion/react` instead of `framer-motion`.
 

Cargo.lock

@@ -132,7 +132,8 @@ async-stream = "0.3.6"
 futures-channel = "0.3.31"
 futures-core = "0.3.31"
 futures-util = "0.3.31"
-ractor = "0.15"
+ractor = { version = "0.14.3" }
+ractor-supervisor = "0.1.9"
 reqwest = "0.12"
 reqwest-streams = "0.10.0"
 tokio = "1"

Cargo.toml

@@ -72,3 +72,12 @@ tasks:
     cmds:
       - chmod +x ./apps/desktop/src-tauri/resources/stt-aarch64-apple-darwin
       - chmod +x ./apps/desktop/src-tauri/resources/passthrough-aarch64-apple-darwin
+
+  db:
+    env:
+      DB: /Users/yujonglee/Library/Application Support/com.hyprnote.nightly/db.sqlite
+    cmds:
+      - |
+          sqlite3 -json "$DB" 'SELECT store FROM main LIMIT 1;' |
+          jq -r '.[0].store' |
+          jless

Taskfile.yaml

@@ -200,7 +200,7 @@ function OptionsMenu({
             queryClient.invalidateQueries({ queryKey: ["audio", sessionId, "url"] });
           })
         ),
-        Effect.flatMap((importedPath) => Effect.promise(() => runBatch(importedPath, { channels: 1 }))),
+        Effect.flatMap((importedPath) => Effect.promise(() => runBatch(importedPath))),
       );
     },
     [queryClient, runBatch, sessionId],

apps/desktop/src/components/main/body/sessions/floating/listen.tsx

@@ -13,9 +13,11 @@ export function useFinalWords(transcriptId: string): (main.Word & { id: string }
       return [];
     }
 
-    return Object.entries(resultTable)
+    const ret = Object.entries(resultTable)
       .map(([wordId, row]) => ({ ...(row as unknown as main.Word), id: wordId }))
       .sort((a, b) => a.start_ms - b.start_ms);
+
+    return ret;
   }, [resultTable]);
 }
 

apps/desktop/src/components/main/body/sessions/note-input/transcript/shared/hooks.ts

@@ -59,12 +59,7 @@ export function useAutoEnhance(tab: Extract<Tab, { type: "sessions" }>) {
 
     if (listenerJustStopped) {
       startEnhance();
-    }
-  }, [listenerStatus, prevListenerStatus, startEnhance]);
-
-  useEffect(() => {
-    if (enhanceTask.status === "generating" && tab.state.editor !== "enhanced") {
       updateSessionTabState(tab, { editor: "enhanced" });
     }
-  }, [enhanceTask.status, tab, updateSessionTabState]);
+  }, [listenerStatus, prevListenerStatus, startEnhance]);
 }

apps/desktop/src/hooks/useAutoEnhance.ts

@@ -13,7 +13,6 @@ import { useSTTConnection } from "./useSTTConnection";
 
 type RunOptions = {
   handlePersist?: HandlePersistCallback;
-  channels?: number;
   model?: string;
   baseUrl?: string;
   apiKey?: string;
@@ -138,7 +137,6 @@ export const useRunBatch = (sessionId: string) => {
         api_key: options?.apiKey ?? conn.apiKey,
         keywords: options?.keywords ?? keywords ?? [],
         languages: options?.languages ?? languages ?? [],
-        channels: options?.channels,
       };
 
       await runBatch(params, { handlePersist: persist, sessionId });

apps/desktop/src/hooks/useRunBatch.ts

@@ -1,3 +1,5 @@
+use std::convert::TryFrom;
+
 use bytes::{BufMut, Bytes, BytesMut};
 use futures_util::{Stream, StreamExt};
 use kalosm_sound::AsyncSource;
@@ -11,6 +13,12 @@ pub use rodio::Source;
 
 const I16_SCALE: f32 = 32768.0;
 
+#[derive(Debug, Clone, Copy)]
+pub struct AudioMetadata {
+    pub sample_rate: u32,
+    pub channels: u8,
+}
+
 impl<T: AsyncSource> AudioFormatExt for T {}
 
 pub trait AudioFormatExt: AsyncSource {
@@ -81,6 +89,40 @@ pub fn source_from_path(
     Ok(decoder)
 }
 
+fn metadata_from_source<S>(source: &S) -> Result<AudioMetadata, crate::Error>
+where
+    S: Source,
+    S::Item: rodio::Sample,
+{
+    let sample_rate = source.sample_rate();
+    if sample_rate == 0 {
+        return Err(crate::Error::InvalidSampleRate(sample_rate));
+    }
+
+    let channels_u16 = source.channels();
+    if channels_u16 == 0 {
+        return Err(crate::Error::UnsupportedChannelCount {
+            count: channels_u16,
+        });
+    }
+    let channels =
+        u8::try_from(channels_u16).map_err(|_| crate::Error::UnsupportedChannelCount {
+            count: channels_u16,
+        })?;
+
+    Ok(AudioMetadata {
+        sample_rate,
+        channels,
+    })
+}
+
+pub fn audio_file_metadata(
+    path: impl AsRef<std::path::Path>,
+) -> Result<AudioMetadata, crate::Error> {
+    let source = source_from_path(path)?;
+    metadata_from_source(&source)
+}
+
 pub fn resample_audio<S, T>(source: S, to_rate: u32) -> Result<Vec<f32>, crate::Error>
 where
     S: rodio::Source<Item = T> + Iterator<Item = T>,
@@ -136,32 +178,48 @@ where
 pub struct ChunkedAudio {
     pub chunks: Vec<Bytes>,
     pub sample_count: usize,
+    pub frame_count: usize,
+    pub metadata: AudioMetadata,
 }
 
 pub fn chunk_audio_file(
     path: impl AsRef<std::path::Path>,
-    sample_rate: u32,
-    chunk_size: usize,
+    chunk_ms: u64,
 ) -> Result<ChunkedAudio, crate::Error> {
     let source = source_from_path(path)?;
-    let samples = resample_audio(source, sample_rate)?;
+    let metadata = metadata_from_source(&source)?;
+    let samples = resample_audio(source, metadata.sample_rate)?;
 
     if samples.is_empty() {
         return Ok(ChunkedAudio {
             chunks: Vec::new(),
             sample_count: 0,
+            frame_count: 0,
+            metadata,
         });
     }
 
-    let chunk_size = chunk_size.max(1);
+    let channels = metadata.channels.max(1) as usize;
+    let frames_per_chunk = {
+        let frames = ((chunk_ms as u128).saturating_mul(metadata.sample_rate as u128) + 999) / 1000;
+        frames.max(1).min(usize::MAX as u128) as usize
+    };
+    let samples_per_chunk = frames_per_chunk
+        .saturating_mul(channels)
+        .max(1)
+        .min(usize::MAX);
+
     let sample_count = samples.len();
+    let frame_count = sample_count / channels;
     let chunks = samples
-        .chunks(chunk_size)
+        .chunks(samples_per_chunk)
         .map(|chunk| f32_to_i16_bytes(chunk.iter().copied()))
         .collect();
 
     Ok(ChunkedAudio {
         chunks,
         sample_count,
+        frame_count,
+        metadata,
     })
 }

crates/audio-utils/src/lib.rs

@@ -65,6 +65,10 @@ impl MicInput {
             config,
         })
     }
+
+    pub fn sample_rate(&self) -> u32 {
+        self.config.sample_rate().0
+    }
 }
 
 impl MicInput {