From 5300117471da12ea33378f0ae68585fa205c392d Mon Sep 17 00:00:00 2001 From: Jhen-Jie Hong Date: Sun, 14 May 2023 14:47:02 +0800 Subject: [PATCH] whisper.objc : enable Core ML in example & fix segmentation fault (#910) * coreml : update endcoder header import path * coreml : force objc_arc in whisper-encoder.mm * whisper.objc : create coreml/ group link * whisper.objc : add coreml model link * whisper.objc : update readme * coreml : use -fobjc-arc for coreml/whisper-encoder.mm * ci: create dummy .mlmodelc for pass ios build * whisper.objc : update readme --------- Co-authored-by: Georgi Gerganov --- .github/workflows/build.yml | 4 ++- Makefile | 2 +- coreml/whisper-encoder.mm | 10 +++--- examples/whisper.objc/README.md | 11 ++++++- .../whisper.objc.xcodeproj/project.pbxproj | 33 ++++++++++++++++++- 5 files changed, 52 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 02c8900..6575700 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,7 +333,9 @@ jobs: uses: actions/checkout@v1 - name: Configure - run: cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin + run: | + cp models/for-tests-ggml-base.en.bin models/ggml-base.en.bin + mkdir models/ggml-base.en-encoder.mlmodelc - name: Build objc example run: xcodebuild -project examples/whisper.objc/whisper.objc.xcodeproj -scheme whisper.objc -configuration ${{ matrix.build }} -sdk iphonesimulator build diff --git a/Makefile b/Makefile index 0787136..7bb7e31 100644 --- a/Makefile +++ b/Makefile @@ -240,7 +240,7 @@ ifndef WHISPER_COREML WHISPER_OBJ += whisper.o else whisper-encoder.o: coreml/whisper-encoder.mm coreml/whisper-encoder.h - $(CXX) -O3 -I . -c coreml/whisper-encoder.mm -o whisper-encoder.o + $(CXX) -O3 -I . -fobjc-arc -c coreml/whisper-encoder.mm -o whisper-encoder.o whisper-encoder-impl.o: coreml/whisper-encoder-impl.m coreml/whisper-encoder-impl.h $(CXX) -O3 -I . -fobjc-arc -c coreml/whisper-encoder-impl.m -o whisper-encoder-impl.o diff --git a/coreml/whisper-encoder.mm b/coreml/whisper-encoder.mm index 2fcccea..09e44fc 100644 --- a/coreml/whisper-encoder.mm +++ b/coreml/whisper-encoder.mm @@ -1,5 +1,9 @@ -#import "coreml/whisper-encoder.h" -#import "coreml/whisper-encoder-impl.h" +#if !__has_feature(objc_arc) +#error This file must be compiled with automatic reference counting enabled (-fobjc-arc) +#endif + +#import "whisper-encoder.h" +#import "whisper-encoder-impl.h" #import @@ -52,8 +56,6 @@ void whisper_coreml_encode( whisper_encoder_implOutput * outCoreML = [(__bridge id) ctx->data predictionFromLogmel_data:inMultiArray error:nil]; memcpy(out, outCoreML.output.dataPointer, outCoreML.output.count * sizeof(float)); - - [inMultiArray release]; } #if __cplusplus diff --git a/examples/whisper.objc/README.md b/examples/whisper.objc/README.md index 46174d2..6833ebb 100644 --- a/examples/whisper.objc/README.md +++ b/examples/whisper.objc/README.md @@ -14,15 +14,24 @@ https://user-images.githubusercontent.com/1991296/204126266-ce4177c6-6eca-4bd9-b ```java git clone https://github.com/ggerganov/whisper.cpp open whisper.cpp/examples/whisper.objc/whisper.objc.xcodeproj/ + +// If you don't want to convert a Core ML model, you can skip this step by create dummy model +mkdir models/ggml-base.en-encoder.mlmodelc ``` Make sure to build the project in `Release`: image -Also, don't forget to add the `-DGGML_USE_ACCELERATE` compiler flag in Build Phases. +Also, don't forget to add the `-DGGML_USE_ACCELERATE` compiler flag for `ggml.c` in Build Phases. This can significantly improve the performance of the transcription: image +If you want to enable Core ML support, you can add the `-DWHISPER_USE_COREML -DWHISPER_COREML_ALLOW_FALLBACK` compiler flag for `whisper.cpp` in Build Phases: + +image + +Then follow the [`Core ML support` section of readme](../../README.md#core-ml-support) for convert the model. + In this project, it also added `-O3 -DNDEBUG` to `Other C Flags`, but adding flags to app proj is not ideal in real world (applies to all C/C++ files), consider splitting xcodeproj in workspace in your own project. diff --git a/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj b/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj index 2c93de7..49bd74e 100644 --- a/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj +++ b/examples/whisper.objc/whisper.objc.xcodeproj/project.pbxproj @@ -14,9 +14,13 @@ 18627C8629052BE000BD2A04 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8529052BE000BD2A04 /* Assets.xcassets */; }; 18627C8929052BE000BD2A04 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 18627C8729052BE000BD2A04 /* LaunchScreen.storyboard */; }; 18627C8C29052BE000BD2A04 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 18627C8B29052BE000BD2A04 /* main.m */; }; - 18627C9429052C4900BD2A04 /* whisper.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 18627C9329052C4900BD2A04 /* whisper.cpp */; }; + 18627C9429052C4900BD2A04 /* whisper.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 18627C9329052C4900BD2A04 /* whisper.cpp */; settings = {COMPILER_FLAGS = "-DWHISPER_USE_COREML -DWHISPER_COREML_ALLOW_FALLBACK"; }; }; 18627C9629052C5800BD2A04 /* ggml.c in Sources */ = {isa = PBXBuildFile; fileRef = 18627C9529052C5800BD2A04 /* ggml.c */; settings = {COMPILER_FLAGS = "-DGGML_USE_ACCELERATE"; }; }; 18627C9B29052CFF00BD2A04 /* ggml-base.en.bin in Resources */ = {isa = PBXBuildFile; fileRef = 18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */; }; + 7FE3424B2A0C3FA20015A058 /* whisper-encoder-impl.m in Sources */ = {isa = PBXBuildFile; fileRef = 7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */; }; + 7FE3424C2A0C3FA20015A058 /* whisper-encoder.mm in Sources */ = {isa = PBXBuildFile; fileRef = 7FE342472A0C3FA20015A058 /* whisper-encoder.mm */; }; + 7FE3424D2A0C3FA20015A058 /* whisper-decoder-impl.m in Sources */ = {isa = PBXBuildFile; fileRef = 7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */; }; + 7FE3424F2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc in Resources */ = {isa = PBXBuildFile; fileRef = 7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -37,6 +41,13 @@ 18627C9529052C5800BD2A04 /* ggml.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = ggml.c; path = ../../../ggml.c; sourceTree = ""; }; 18627C9729052C6600BD2A04 /* ggml.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ggml.h; path = ../../../ggml.h; sourceTree = ""; }; 18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */ = {isa = PBXFileReference; lastKnownFileType = archive.macbinary; name = "ggml-base.en.bin"; path = "../../../models/ggml-base.en.bin"; sourceTree = ""; }; + 7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "whisper-encoder-impl.m"; sourceTree = ""; }; + 7FE342462A0C3FA20015A058 /* whisper-encoder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "whisper-encoder.h"; sourceTree = ""; }; + 7FE342472A0C3FA20015A058 /* whisper-encoder.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "whisper-encoder.mm"; sourceTree = ""; }; + 7FE342482A0C3FA20015A058 /* whisper-decoder-impl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "whisper-decoder-impl.h"; sourceTree = ""; }; + 7FE342492A0C3FA20015A058 /* whisper-encoder-impl.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "whisper-encoder-impl.h"; sourceTree = ""; }; + 7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "whisper-decoder-impl.m"; sourceTree = ""; }; + 7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = "ggml-base.en-encoder.mlmodelc"; path = "../../../models/ggml-base.en-encoder.mlmodelc"; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -69,6 +80,8 @@ 18627C7829052BDF00BD2A04 /* whisper.objc */ = { isa = PBXGroup; children = ( + 7FE3424E2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc */, + 7FE342442A0C3FA20015A058 /* coreml */, 18627C9A29052CFF00BD2A04 /* ggml-base.en.bin */, 18627C9729052C6600BD2A04 /* ggml.h */, 18627C9529052C5800BD2A04 /* ggml.c */, @@ -89,6 +102,20 @@ path = whisper.objc; sourceTree = ""; }; + 7FE342442A0C3FA20015A058 /* coreml */ = { + isa = PBXGroup; + children = ( + 7FE342452A0C3FA20015A058 /* whisper-encoder-impl.m */, + 7FE342462A0C3FA20015A058 /* whisper-encoder.h */, + 7FE342472A0C3FA20015A058 /* whisper-encoder.mm */, + 7FE342482A0C3FA20015A058 /* whisper-decoder-impl.h */, + 7FE342492A0C3FA20015A058 /* whisper-encoder-impl.h */, + 7FE3424A2A0C3FA20015A058 /* whisper-decoder-impl.m */, + ); + name = coreml; + path = ../../../coreml; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -147,6 +174,7 @@ buildActionMask = 2147483647; files = ( 18627C8929052BE000BD2A04 /* LaunchScreen.storyboard in Resources */, + 7FE3424F2A0C418A0015A058 /* ggml-base.en-encoder.mlmodelc in Resources */, 18627C8629052BE000BD2A04 /* Assets.xcassets in Resources */, 18627C8429052BDF00BD2A04 /* Main.storyboard in Resources */, 18627C9B29052CFF00BD2A04 /* ggml-base.en.bin in Resources */, @@ -161,11 +189,14 @@ buildActionMask = 2147483647; files = ( 18627C8129052BDF00BD2A04 /* ViewController.m in Sources */, + 7FE3424C2A0C3FA20015A058 /* whisper-encoder.mm in Sources */, 18627C9429052C4900BD2A04 /* whisper.cpp in Sources */, 18627C9629052C5800BD2A04 /* ggml.c in Sources */, 18627C7B29052BDF00BD2A04 /* AppDelegate.m in Sources */, + 7FE3424D2A0C3FA20015A058 /* whisper-decoder-impl.m in Sources */, 18627C8C29052BE000BD2A04 /* main.m in Sources */, 18627C7E29052BDF00BD2A04 /* SceneDelegate.m in Sources */, + 7FE3424B2A0C3FA20015A058 /* whisper-encoder-impl.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; };