Skip to content

Commit 54a08bd

Browse files
litongjavalitongmacosbobqianic
authored
examples : add whisper.android.java for compatibility with older Android versions using Java (#1382)
* save the recorded audio to a file * Alignment -help * Save the correct audio * chage to a consistent coding style * Correct typo * Update examples/stream/stream.cpp * Update examples/stream/stream.cpp * Correct variable misuse * Update examples/stream/stream.cpp * Update examples/stream/stream.cpp * Update examples/stream/stream.cpp * Update examples/stream/stream.cpp * add *.bin .cxx/ .gradle/ cmake-build-debug/ to gitignore * add whisper.android.java * Added support for older versions of Android of Java * add examples for android java * add README.md for android java * add fullTranscribeWithTime * 增加 toString()方法和测试 * change return type to void * update to v1.4.1 * add WhisperService * chage to whisper_full_get_segment_t1 * add method transcribeDataWithTime * modified toString ``` return "[" + start + " --> " + end + "]:" + sentence; ``` * Optimize code logic * update text view on handle * set max lines * change Chinese to English * Update bindings/java/build.gradle * Update .gitignore * add android.java to github action * chage android.java to android_java in build.yml * remove gradle * chage jdk to temurin in android_java of CI * chage jdk to temurin 11 in android_java of CI * add x to gradlew * set api-level for android_java of CI * Update examples/whisper.android.java/app/src/main/jni/whisper/CMakeLists.txt * add ndk version in build.gradle * remove local.properties * add testFullTranscribeWithTime --------- Co-authored-by: litongmacos <[email protected]> Co-authored-by: bobqianic <[email protected]>
1 parent 9f8bbd3 commit 54a08bd

File tree

59 files changed

+2299
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+2299
-1
lines changed

‎.github/workflows/build.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,32 @@ jobs:
396396
cd examples/whisper.android
397397
./gradlew assembleRelease --no-daemon
398398
399+
android_java:
400+
runs-on: ubuntu-latest
401+
402+
steps:
403+
- name: Clone
404+
uses: actions/checkout@v3
405+
406+
- name: set up JDK 11
407+
uses: actions/setup-java@v3
408+
with:
409+
java-version: '11'
410+
distribution: 'temurin'
411+
cache: gradle
412+
413+
- name: Setup Android SDK
414+
uses: android-actions/setup-android@v2
415+
with:
416+
api-level: 30
417+
build-tools-version: 30.0.3
418+
419+
- name: Build
420+
run: |
421+
cd examples/whisper.android.java
422+
chmod +x ./gradlew
423+
./gradlew assembleRelease
424+
399425
java:
400426
needs: [ 'windows' ]
401427
runs-on: windows-latest

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,7 @@ bindings/java/.idea/
5454
.idea/
5555

5656
benchmark_results.csv
57+
cmake-build-debug/
58+
.cxx/
59+
.gradle/
60+
local.properties

‎bindings/java/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ archivesBaseName = 'whispercpp'
99
group = 'io.github.ggerganov'
1010
version = '1.4.0'
1111

12+
1213
sourceCompatibility = 1.8
1314
targetCompatibility = 1.8
1415

‎bindings/java/src/main/java/io/github/ggerganov/whispercpp/WhisperCpp.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@
22

33
import com.sun.jna.Native;
44
import com.sun.jna.Pointer;
5+
import io.github.ggerganov.whispercpp.bean.WhisperSegment;
56
import io.github.ggerganov.whispercpp.params.WhisperContextParams;
67
import io.github.ggerganov.whispercpp.params.WhisperFullParams;
78
import io.github.ggerganov.whispercpp.params.WhisperSamplingStrategy;
89

910
import java.io.File;
1011
import java.io.FileNotFoundException;
1112
import java.io.IOException;
13+
import java.util.ArrayList;
14+
import java.util.List;
1215

1316
/**
1417
* Before calling most methods, you must call `initContext(modelPath)` to initialise the `ctx` Pointer.
@@ -160,6 +163,28 @@ public String fullTranscribe(WhisperFullParams whisperParams, float[] audioData)
160163

161164
return str.toString().trim();
162165
}
166+
public List<WhisperSegment> fullTranscribeWithTime(WhisperFullParams whisperParams, float[] audioData) throws IOException {
167+
if (ctx == null) {
168+
throw new IllegalStateException("Model not initialised");
169+
}
170+
171+
if (lib.whisper_full(ctx, whisperParams, audioData, audioData.length) != 0) {
172+
throw new IOException("Failed to process audio");
173+
}
174+
175+
int nSegments = lib.whisper_full_n_segments(ctx);
176+
List<WhisperSegment> segments= new ArrayList<>(nSegments);
177+
178+
179+
for (int i = 0; i < nSegments; i++) {
180+
long t0 = lib.whisper_full_get_segment_t0(ctx, i);
181+
String text = lib.whisper_full_get_segment_text(ctx, i);
182+
long t1 = lib.whisper_full_get_segment_t1(ctx, i);
183+
segments.add(new WhisperSegment(t0,t1,text));
184+
}
185+
186+
return segments;
187+
}
163188

164189
// public int getTextSegmentCount(Pointer ctx) {
165190
// return lib.whisper_full_n_segments(ctx);

‎bindings/java/src/main/java/io/github/ggerganov/whispercpp/bean/WhisperSegment.java

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package io.github.ggerganov.whispercpp.bean;
2+
3+
/**
4+
* Created by [email protected] on 10/21/2023_7:48 AM
5+
*/
6+
public class WhisperSegment {
7+
private long start, end;
8+
private String sentence;
9+
10+
public WhisperSegment() {
11+
}
12+
13+
public WhisperSegment(long start, long end, String sentence) {
14+
this.start = start;
15+
this.end = end;
16+
this.sentence = sentence;
17+
}
18+
19+
public long getStart() {
20+
return start;
21+
}
22+
23+
public long getEnd() {
24+
return end;
25+
}
26+
27+
public String getSentence() {
28+
return sentence;
29+
}
30+
31+
public void setStart(long start) {
32+
this.start = start;
33+
}
34+
35+
public void setEnd(long end) {
36+
this.end = end;
37+
}
38+
39+
public void setSentence(String sentence) {
40+
this.sentence = sentence;
41+
}
42+
43+
@Override
44+
public String toString() {
45+
return "[" + start + " --> " + end + "]:" + sentence;
46+
}
47+
}

‎bindings/java/src/test/java/io/github/ggerganov/whispercpp/WhisperCppTest.java

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import static org.junit.jupiter.api.Assertions.*;
44

5+
import io.github.ggerganov.whispercpp.bean.WhisperSegment;
56
import io.github.ggerganov.whispercpp.params.CBool;
67
import io.github.ggerganov.whispercpp.params.WhisperFullParams;
78
import io.github.ggerganov.whispercpp.params.WhisperSamplingStrategy;
@@ -11,6 +12,7 @@
1112
import javax.sound.sampled.AudioSystem;
1213
import java.io.File;
1314
import java.io.FileNotFoundException;
15+
import java.util.List;
1416

1517
class WhisperCppTest {
1618
private static WhisperCpp whisper = new WhisperCpp();
@@ -20,7 +22,8 @@ class WhisperCppTest {
2022
static void init() throws FileNotFoundException {
2123
// By default, models are loaded from ~/.cache/whisper/ and are usually named "ggml-${name}.bin"
2224
// or you can provide the absolute path to the model file.
23-
String modelName = "../../models/ggml-tiny.en.bin";
25+
String modelName = "../../models/ggml-tiny.bin";
26+
// String modelName = "../../models/ggml-tiny.en.bin";
2427
try {
2528
whisper.initContext(modelName);
2629
// whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);
@@ -99,4 +102,44 @@ void testFullTranscribe() throws Exception {
99102
audioInputStream.close();
100103
}
101104
}
105+
106+
@Test
107+
void testFullTranscribeWithTime() throws Exception {
108+
if (!modelInitialised) {
109+
System.out.println("Model not initialised, skipping test");
110+
return;
111+
}
112+
113+
// Given
114+
File file = new File(System.getProperty("user.dir"), "../../samples/jfk.wav");
115+
AudioInputStream audioInputStream = AudioSystem.getAudioInputStream(file);
116+
117+
byte[] b = new byte[audioInputStream.available()];
118+
float[] floats = new float[b.length / 2];
119+
120+
// WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_GREEDY);
121+
WhisperFullParams params = whisper.getFullDefaultParams(WhisperSamplingStrategy.WHISPER_SAMPLING_BEAM_SEARCH);
122+
params.setProgressCallback((ctx, state, progress, user_data) -> System.out.println("progress: " + progress));
123+
params.print_progress = CBool.FALSE;
124+
// params.initial_prompt = "and so my fellow Americans um, like";
125+
126+
127+
try {
128+
audioInputStream.read(b);
129+
130+
for (int i = 0, j = 0; i < b.length; i += 2, j++) {
131+
int intSample = (int) (b[i + 1]) << 8 | (int) (b[i]) & 0xFF;
132+
floats[j] = intSample / 32767.0f;
133+
}
134+
135+
List<WhisperSegment> segments = whisper.fullTranscribeWithTime(params, floats);
136+
assertTrue(segments.size() > 0, "The size of segments should be greater than 0");
137+
for (WhisperSegment segment : segments) {
138+
System.out.println(segment);
139+
}
140+
} finally {
141+
audioInputStream.close();
142+
}
143+
}
144+
102145
}

‎examples/whisper.android.java/.gitignore

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
*.iml
2+
.gradle
3+
/local.properties
4+
/.idea/caches
5+
/.idea/libraries
6+
/.idea/modules.xml
7+
/.idea/workspace.xml
8+
/.idea/navEditor.xml
9+
/.idea/assetWizardSettings.xml
10+
.DS_Store
11+
/build
12+
/captures
13+
.externalNativeBuild
14+
.cxx
15+
local.properties

‎examples/whisper.android.java/README.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
A sample Android app using java code and [whisper.cpp](https://github.com/ggerganov/whisper.cpp/) to do voice-to-text transcriptions.
2+
3+
To use:
4+
5+
1. Select a model from the [whisper.cpp repository](https://github.com/ggerganov/whisper.cpp/tree/master/models).[^1]
6+
2. Copy the model to the "app/src/main/assets/models" folder.
7+
3. Select a sample audio file (for example, [jfk.wav](https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav)).
8+
4. Copy the sample to the "app/src/main/assets/samples" folder.
9+
5. Modify the modelFilePath in the WhisperService.java
10+
6. Modify the sampleFilePath in the WhisperService.java
11+
7. Select the "release" active build variant, and use Android Studio to run and deploy to your device.
12+
[^1]: I recommend the tiny or base models for running on an Android device.
13+
14+
PS:
15+
1. Do not move this android project folder individually to other folders, because this android project folder depends on the files of the whole project.
16+
2. The cpp code is compiled during the build process
17+
3. If you want to import a compiled cpp project in your Android project, please refer to the https://github.com/litongjava/whisper.cpp.android.java.demo
18+
19+
![](README_files/1.jpg)
20+

‎examples/whisper.android.java/README_files/1.jpg

67.4 KB
Loading

‎examples/whisper.android.java/app/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/build

‎examples/whisper.android.java/app/build.gradle

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
plugins {
2+
id 'com.android.application'
3+
}
4+
5+
android {
6+
compileSdkVersion 30
7+
buildToolsVersion '30.0.3'
8+
9+
defaultConfig {
10+
applicationId "com.litongjava.whisper.android.java"
11+
minSdkVersion 21
12+
targetSdkVersion 30
13+
versionCode 1
14+
versionName "1.0"
15+
16+
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
17+
externalNativeBuild {
18+
cmake {
19+
cppFlags ""
20+
}
21+
}
22+
ndk {
23+
abiFilters 'arm64-v8a', 'armeabi-v7a', 'x86', 'x86_64'
24+
}
25+
}
26+
27+
buildTypes {
28+
release {
29+
signingConfig signingConfigs.debug
30+
minifyEnabled true
31+
proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro'
32+
}
33+
}
34+
externalNativeBuild {
35+
cmake {
36+
path "src/main/jni/whisper/CMakeLists.txt"
37+
}
38+
}
39+
ndkVersion "25.2.9519653"
40+
compileOptions {
41+
sourceCompatibility JavaVersion.VERSION_1_8
42+
targetCompatibility JavaVersion.VERSION_1_8
43+
}
44+
}
45+
46+
dependencies {
47+
implementation 'androidx.appcompat:appcompat:1.1.0'
48+
implementation 'com.google.android.material:material:1.1.0'
49+
implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
50+
testImplementation 'junit:junit:4.+'
51+
androidTestImplementation 'androidx.test.ext:junit:1.1.5'
52+
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
53+
54+
//litongjava
55+
implementation 'com.litongjava:android-view-inject:1.0'
56+
implementation 'com.litongjava:jfinal-aop:1.0.1'
57+
implementation 'com.litongjava:litongjava-android-utils:1.0.0'
58+
}

‎examples/whisper.android.java/app/proguard-rules.pro

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Add project specific ProGuard rules here.
2+
# You can control the set of applied configuration files using the
3+
# proguardFiles setting in build.gradle.
4+
#
5+
# For more details, see
6+
# http://developer.android.com/guide/developing/tools/proguard.html
7+
8+
# If your project uses WebView with JS, uncomment the following
9+
# and specify the fully qualified class name to the JavaScript interface
10+
# class:
11+
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
12+
# public *;
13+
#}
14+
15+
# Uncomment this to preserve the line number information for
16+
# debugging stack traces.
17+
#-keepattributes SourceFile,LineNumberTable
18+
19+
# If you keep the line number information, uncomment this to
20+
# hide the original source file name.
21+
#-renamesourcefileattribute SourceFile

‎examples/whisper.android.java/app/src/androidTest/java/com/litongjava/whisper/android/java/ExampleInstrumentedTest.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package com.litongjava.whisper.android.java;
2+
3+
import android.content.Context;
4+
5+
import androidx.test.platform.app.InstrumentationRegistry;
6+
import androidx.test.ext.junit.runners.AndroidJUnit4;
7+
8+
import org.junit.Test;
9+
import org.junit.runner.RunWith;
10+
11+
import static org.junit.Assert.*;
12+
13+
/**
14+
* Instrumented test, which will execute on an Android device.
15+
*
16+
* @see <a href="http://d.android.com/tools/testing">Testing documentation</a>
17+
*/
18+
@RunWith(AndroidJUnit4.class)
19+
public class ExampleInstrumentedTest {
20+
@Test
21+
public void useAppContext() {
22+
// Context of the app under test.
23+
Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext();
24+
assertEquals("com.litongjava.whisper.android.java", appContext.getPackageName());
25+
}
26+
}

‎examples/whisper.android.java/app/src/main/AndroidManifest.xml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
3+
package="com.litongjava.whisper.android.java">
4+
5+
<application
6+
android:allowBackup="true"
7+
android:name=".app.App"
8+
android:icon="@mipmap/ic_launcher"
9+
android:label="@string/app_name"
10+
android:roundIcon="@mipmap/ic_launcher_round"
11+
android:supportsRtl="true"
12+
android:theme="@style/Theme.Whisperandroidjava">
13+
<activity android:name=".MainActivity">
14+
<intent-filter>
15+
<action android:name="android.intent.action.MAIN" />
16+
17+
<category android:name="android.intent.category.LAUNCHER" />
18+
</intent-filter>
19+
</activity>
20+
</application>
21+
22+
</manifest>

0 commit comments

Comments
 (0)