diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index acf4933..246addd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,13 +7,15 @@ jobs: name: "x86_64" runs-on: ubuntu-latest container: - image: bilelmoussaoui/flatpak-github-actions:gnome-nightly + # Keep in step with runtime-version in the manifest + image: ghcr.io/flathub-infra/flatpak-github-actions:gnome-50 options: --privileged steps: - uses: actions/checkout@v4 with: submodules: recursive - - uses: flatpak/flatpak-github-actions/flatpak-builder@master + - uses: flatpak/flatpak-github-actions/flatpak-builder@v6 with: + bundle: livecaptions.flatpak manifest-path: build-aux/flatpak/net.sapples.LiveCaptions.json cache-key: flatpak-builder-${{ github.sha }} diff --git a/.gitignore b/.gitignore index 02114a1..2bd3cc5 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ build-dir .flatpak-builder result aprilv0_en-us.april +april-english-dev-01110_en.april +april-english-dev-01110_en.april.part diff --git a/README.md b/README.md index 75fd886..f2569a2 100644 --- a/README.md +++ b/README.md @@ -52,74 +52,73 @@ If you forgot, you can initialize submodules like so: $ git submodule update --init --recursive ``` -## Option 1: Building with GNOME Builder (easy) -You can build this easily with GNOME Builder. After cloning, open the project directory in GNOME Builder, download the SDK if it asks you, and click the play button to build and run. +## Option 1: Building from the terminal (Linux) -If you are using Flatpak GNOME Builder and experience issues running this (for example, some cryptic X Window System error), please try using your distro's native packaged version of GNOME Builder instead of Flatpak (e.g. `sudo apt install gnome-builder`). - -## Option 2: Building from the terminal (not as easy) -First you must [download ONNXRuntime v1.14.1 (Linux)](https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-linux-x64-1.14.1.tgz) or [ONNXRuntime v.1.14.1 (Mac OS x86)](https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-osx-x86_64-1.14.1.tgz), extract it somewhere, and set the environment variables to point to it. - -### Make ONNX available +### 1. Install the dependencies +You need GTK 4.12 or newer and libadwaita 1.5 or newer. Ubuntu 22.04 and other older releases are too old; use Nix (Option 2) there. -Linux: +Arch Linux: ``` -$ export ONNX_ROOT=/path/to/onnxruntime-linux-x64-1.14.1/ -$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/onnxruntime-linux-x64-1.14.1/lib +$ sudo pacman -S --needed base-devel git meson ninja cmake pkgconf gtk4 libadwaita libpulse libx11 gettext glib2 desktop-file-utils curl pipewire-pulse ``` -Alternatively you should also be able to locally build and install ONNXRuntime, in which case that step shouldn't be necessary. - - -Mac OS: +Ubuntu 24.04 or newer, Debian 13 or newer: ``` -$ export ONNX_ROOT=/path/to/onnxruntime-osx-x86_64-1.14.1/ -$ cp /path/to/onnxruntime-osx-x86_64-1.14.1/lib/libonnxruntime.1.14.1.dylib /usr/local/lib/libonnxruntime.1.14.1.dylib -$ cp /path/to/onnxruntime-osx-x86_64-1.14.1/lib/libonnxruntime.dylib /usr/local/lib/libonnxruntime.dylib +$ sudo apt install build-essential git meson ninja-build cmake pkg-config libgtk-4-dev libadwaita-1-dev libpulse-dev libx11-dev gettext libglib2.0-bin desktop-file-utils curl ``` -### Other dependencies -You will also need the following prerequisites: +Fedora: ``` -pulseaudio -libadwaita -meson -ninja +$ sudo dnf install gcc gcc-c++ git meson ninja-build cmake pkgconf-pkg-config gtk4-devel libadwaita-devel pulseaudio-libs-devel libX11-devel gettext glib2-devel desktop-file-utils curl ``` -If you're on macOS, you may have to run the following command for Pulseaudio to be registered as a background service: +Live Captions records audio through the PulseAudio interface. On PipeWire desktops this is provided by `pipewire-pulse`, which most distributions install by default. + +### 2. Build ``` -brew services restart pulseaudio +$ ./setup.sh ``` -and to stop it, just change `restart` for stop. +This builds into `builddir/` and downloads the speech model (about 320 MB). If your distribution packages ONNX Runtime, the system copy is used; otherwise the official ONNX Runtime SDK is downloaded into `subprojects/april-asr/lib`. To use another SDK, set `ONNX_ROOT` to its directory before running `setup.sh`. Run `./setup.sh` again after `git pull`. - -### Set up the build - -To set up a build, run these commands: +### 3. Run ``` -$ meson setup builddir --buildtype release -$ meson devenv -C builddir +$ ./run.sh ``` +On first launch, a short benchmark checks that your computer can caption in real time. It needs a result of at least 1.2. -### Build and run +Recognition uses 2 CPU threads by default. To try another number, set `APRIL_ENCODER_THREADS`, for example `APRIL_ENCODER_THREADS=4 ./run.sh`, then re-run the benchmark from the settings window to compare. To use a different model, set `APRIL_MODEL_PATH` to its file. -Now you can build the application. Run the following: +## Option 2: Building with Nix +The flake pins a nixpkgs revision that provides GTK 4.22 and libadwaita 1.9, and downloads the default model. It builds the `april-asr` submodule from the checkout, which requires Nix 2.27 or newer: ``` -$ ninja +$ nix build +$ ./result/bin/livecaptions ``` -Before being able to run the app, you must also download the model and export `APRIL_MODEL_PATH` to where the model is. For example: +## Option 3: Building a Flatpak +With `flatpak-builder` installed and the Flathub remote added: ``` -$ wget https://april.sapples.net/april-english-dev-01110_en.april -$ export APRIL_MODEL_PATH=`pwd`/april-english-dev-01110_en.april +$ flatpak install --user flathub org.gnome.Sdk//50 org.gnome.Platform//50 +$ flatpak-builder --user --install --force-clean build-dir build-aux/flatpak/net.sapples.LiveCaptions.json +$ flatpak run net.sapples.LiveCaptions ``` -You should now be able to run the app with: +The manifest builds this checkout, and bundles ONNX Runtime and the default model. + +## Option 4: Building with GNOME Builder +You can build this with GNOME Builder. After cloning, open the project directory in GNOME Builder, download the SDK if it asks you, and click the play button to build and run. + +If you are using Flatpak GNOME Builder and experience issues running this (for example, some cryptic X Window System error), please try using your distro's native packaged version of GNOME Builder instead of Flatpak (e.g. `sudo apt install gnome-builder`). + +## macOS +`setup.sh` downloads ONNX Runtime for Linux only. Download [ONNX Runtime for macOS](https://github.com/microsoft/onnxruntime/releases/tag/v1.30.0), extract it, and point `ONNX_ROOT` at it. Install the other dependencies with Homebrew (`gtk4 libadwaita pulseaudio meson ninja cmake pkg-config`) and register PulseAudio as a background service with `brew services restart pulseaudio`. Then: ``` -$ src/livecaptions +$ export ONNX_ROOT=/path/to/onnxruntime-osx-arm64-1.30.0 +$ cp $ONNX_ROOT/lib/libonnxruntime*.dylib /usr/local/lib/ +$ ./setup.sh +$ ./run.sh ``` -If you're on MacOS, now, go to security settings and confirm the prompt which asks to allow `libonnxruntime`, then, the application should work as intended. +The first time, confirm the security prompt that asks to allow `libonnxruntime`. diff --git a/build-aux/flatpak/net.sapples.LiveCaptions.json b/build-aux/flatpak/net.sapples.LiveCaptions.json index 2d53fb1..e395998 100644 --- a/build-aux/flatpak/net.sapples.LiveCaptions.json +++ b/build-aux/flatpak/net.sapples.LiveCaptions.json @@ -1,7 +1,7 @@ { - "app-id" : "net.sapples.LiveCaptions", + "id" : "net.sapples.LiveCaptions", "runtime" : "org.gnome.Platform", - "runtime-version" : "master", + "runtime-version" : "50", "sdk" : "org.gnome.Sdk", "command" : "livecaptions", "finish-args" : [ @@ -19,15 +19,13 @@ "/share/gtk-doc", "/share/man", "/share/pkgconfig", - "/onnx_root", + "/onnxruntime/include", + "/onnxruntime/lib/cmake", + "/onnxruntime/lib/pkgconfig", + "/lib/libaprilasr.so*", "*.la", "*.a" ], - "build-options" : { - "env" : { - "ONNX_ROOT" : "/app/onnx_root" - } - }, "modules" : [ { "name" : "onnxruntime", @@ -36,30 +34,23 @@ { "type" : "archive", "only-arches" : ["x86_64"], - "dest" : "onnxruntime-1", - "url" : "https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-linux-x64-1.14.1.tgz", - "sha256" : "010585f534d822bf02d54c7cfc53b57aa3eb0cf34f8fb7503ee519ed18e25d29" + "url" : "https://github.com/microsoft/onnxruntime/releases/download/v1.30.0/onnxruntime-linux-x64-1.30.0.tgz", + "sha256" : "a5ed5a3cac51fbb2e90da632ae43d19212faaa20e76484e62bcb7c23ddb3b3fd" }, - { "type" : "archive", "only-arches" : ["aarch64"], - "dest" : "onnxruntime-1", - "url" : "https://github.com/microsoft/onnxruntime/releases/download/v1.14.1/onnxruntime-linux-aarch64-1.14.1.tgz", - "sha256" : "20965e05d2da6a0e6acedda8aca1a8f75c3f4cb1d7932403a9977b753b1b2120" + "url" : "https://github.com/microsoft/onnxruntime/releases/download/v1.30.0/onnxruntime-linux-aarch64-1.30.0.tgz", + "sha256" : "e16a27a8ed330bbc698df7330b0cf56e722f354e3bcc92118682c74ef3c3e3da" } ], "build-commands" : [ - "mkdir -p $ONNX_ROOT", - "mv onnxruntime-1/* $ONNX_ROOT", - - "mkdir -p $FLATPAK_DEST/lib", - "cp $ONNX_ROOT/lib/* $FLATPAK_DEST/lib" + "mkdir -p ${FLATPAK_DEST}/onnxruntime", + "cp -a include lib LICENSE ThirdPartyNotices.txt ${FLATPAK_DEST}/onnxruntime/" ] }, - { - "name": "aprilmodel", + "name" : "aprilmodel", "buildsystem" : "simple", "sources" : [ { @@ -73,16 +64,28 @@ "install -Dm644 aprilv0_en-us.april -t ${FLATPAK_DEST}/LiveCaptions/models" ] }, - { "name" : "livecaptions", "builddir" : true, "buildsystem" : "meson", + "config-opts" : ["--buildtype=release"], + "build-options" : { + "env" : { + "ONNX_ROOT" : "/app/onnxruntime" + } + }, "sources" : [ { - "type" : "git", - "url" : "https://github.com/abb128/LiveCaptions", - "branch" : "main" + "type" : "dir", + "path" : "../..", + "skip" : [ + "builddir", + "_build", + "result", + ".flatpak-builder", + "subprojects/april-asr/lib", + "april-english-dev-01110_en.april" + ] } ] } diff --git a/data/meson.build b/data/meson.build index 8d4d86f..7afdea3 100644 --- a/data/meson.build +++ b/data/meson.build @@ -15,11 +15,11 @@ if desktop_utils.found() endif appstream_file = i18n.merge_file( - input: 'net.sapples.LiveCaptions.appdata.xml.in', - output: 'net.sapples.LiveCaptions.appdata.xml', + input: 'net.sapples.LiveCaptions.metainfo.xml.in', + output: 'net.sapples.LiveCaptions.metainfo.xml', po_dir: '../po', install: true, - install_dir: join_paths(get_option('datadir'), 'appdata') + install_dir: join_paths(get_option('datadir'), 'metainfo') ) appstream_util = find_program('appstream-util', required: false) @@ -31,7 +31,7 @@ endif gnome.compile_schemas(build_by_default: true, depend_files: 'net.sapples.LiveCaptions.gschema.xml') devenv = environment() -devenv.set('GSETTINGS_SCHEMA_DIR', meson.current_build_dir() / 'data') +devenv.set('GSETTINGS_SCHEMA_DIR', meson.current_build_dir()) meson.add_devenv(devenv) install_data('net.sapples.LiveCaptions.gschema.xml', diff --git a/data/net.sapples.LiveCaptions.appdata.xml.in b/data/net.sapples.LiveCaptions.metainfo.xml.in similarity index 74% rename from data/net.sapples.LiveCaptions.appdata.xml.in rename to data/net.sapples.LiveCaptions.metainfo.xml.in index ff03ab3..8ca6fa3 100644 --- a/data/net.sapples.LiveCaptions.appdata.xml.in +++ b/data/net.sapples.LiveCaptions.metainfo.xml.in @@ -4,7 +4,9 @@ Live Captions Live Captioning for the desktop - abb128 + + abb128 +

Live Captions is an application that provides realtime automatic subtitles for the Linux desktop.

@@ -34,17 +36,32 @@ https://github.com/abb128/LiveCaptions https://github.com/abb128/LiveCaptions/issues + https://github.com/abb128/LiveCaptions - https://github.com/abb128/LiveCaptions/blob/main/screenshots/0.png?raw=true + https://raw.githubusercontent.com/abb128/LiveCaptions/main/screenshots/0.png + The caption window, with less confident words faded - https://github.com/abb128/LiveCaptions/blob/main/screenshots/1.png?raw=true + https://raw.githubusercontent.com/abb128/LiveCaptions/main/screenshots/1.png + Captioning a recorded talk over a video + + https://github.com/abb128/LiveCaptions/releases/tag/v0.5.0 + +

Ported to current GTK 4 and libadwaita APIs, requiring libadwaita 1.5

+

Recognition no longer speeds up audio on slow computers, which reduced accuracy; the setup benchmark now requires a result of at least 1.2

+

Slow-processing warnings now show how far behind recognition is

+

Sentences are finalized after 1.2 seconds of silence instead of 2.2 seconds

+

Fixed recognition state carrying over between sentences

+

Recognition uses two CPU threads by default

+

Updated ONNX Runtime to 1.30.0

+ + https://github.com/abb128/LiveCaptions/releases/tag/v0.4.1 @@ -97,4 +114,11 @@ net.sapples.LiveCaptions.desktop GPL-3.0-or-later + + 360 + + + pointing + keyboard + diff --git a/flake.lock b/flake.lock index fc40eb9..5830f3a 100644 --- a/flake.lock +++ b/flake.lock @@ -32,11 +32,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1693663421, - "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=", + "lastModified": 1791456040, + "narHash": "sha256-H83yz/do+hjOO0uujHyH8RgtIQP2v7Mw+cLwcZK/xKc=", "owner": "nixos", "repo": "nixpkgs", - "rev": "e56990880811a451abd32515698c712788be5720", + "rev": "e7439b6b14ad3cc35d05608ebca9bce01a25f5f8", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a176c5c..49b49c6 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,8 @@ { description = "LiveCaptions flake"; inputs = { + # april-asr is a git submodule built by meson's cmake subproject + self.submodules = true; flake-utils.url = "github:numtide/flake-utils"; nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; april-asr = { @@ -25,19 +27,23 @@ nativeBuildInputs = with pkgs; [ meson ninja - wrapGAppsHook - ]; - buildInputs = with pkgs; [ cmake - pkgconfig + pkg-config + wrapGAppsHook4 appstream-glib # appstream-util desktop-file-utils # desktop-file-utils gettext # msgfmt glib # glib-compile-schemas - libadwaita # libadwaita-1 + ]; + buildInputs = with pkgs; [ + gtk4 # gtk4 >= 4.12 + libadwaita # libadwaita-1 >= 1.5 libpulseaudio # libpulse + libx11 # x11 onnxruntime ]; + # cmake is only used for the april-asr meson subproject + dontUseCmakeConfigure = true; preFixup = '' gappsWrapperArgs+=(--prefix APRIL_MODEL_PATH : "${april-asr}") ''; diff --git a/meson.build b/meson.build index 8dfd7b3..505b4ad 100644 --- a/meson.build +++ b/meson.build @@ -1,9 +1,10 @@ -project('livecaptions', 'c', - version: '0.1.0', +project('livecaptions', 'c', 'cpp', + version: '0.5.0', meson_version: '>= 0.59.0', default_options: [ 'warning_level=2', 'werror=false', 'c_std=gnu11', + 'cpp_std=c++17', ], ) diff --git a/po/POTFILES b/po/POTFILES index 7889454..fb2c0e3 100644 --- a/po/POTFILES +++ b/po/POTFILES @@ -1,5 +1,5 @@ data/net.sapples.LiveCaptions.desktop.in -data/net.sapples.LiveCaptions.appdata.xml.in +data/net.sapples.LiveCaptions.metainfo.xml.in data/net.sapples.LiveCaptions.gschema.xml src/main.c src/livecaptions-window.ui diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..2032df4 --- /dev/null +++ b/run.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Runs Live Captions from builddir/ without installing it. Build with ./setup.sh. +# Extra arguments are passed to the app. +set -eu +cd "$(dirname "$0")" + +if [ ! -x builddir/src/livecaptions ]; then + echo "Live Captions is not built yet. Run ./setup.sh first." >&2 + exit 1 +fi + +export GSETTINGS_SCHEMA_DIR="$PWD/builddir/data" +export APRIL_MODEL_PATH="${APRIL_MODEL_PATH:-$PWD/april-english-dev-01110_en.april}" +export ORT_DISABLE_TELEMETRY=1 +if [ ! -f "$APRIL_MODEL_PATH" ]; then + echo "Model not found at $APRIL_MODEL_PATH. Run ./setup.sh to download it." >&2 + exit 1 +fi + +exec ./builddir/src/livecaptions "$@" diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..6fbeed2 --- /dev/null +++ b/setup.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Builds Live Captions into builddir/ and downloads the speech model. +# Install the system dependencies listed in README.md first. +set -eu +cd "$(dirname "$0")" + +model=april-english-dev-01110_en.april +ort_sdk=subprojects/april-asr/lib + +# Check out the april-asr version this commit expects, also after git pull +if [ -d .git ] || [ -f .git ]; then + git submodule update --init --recursive +elif [ ! -f subprojects/april-asr/CMakeLists.txt ]; then + echo "subprojects/april-asr is missing. Clone with git clone --recursive." >&2 + exit 1 +fi + +# Use a distro ONNX Runtime if there is one, else download the official SDK. +# April links whichever it finds and records the path of a downloaded SDK in +# the program, so no LD_LIBRARY_PATH is needed either way. +if [ -n "${ONNX_ROOT:-}" ]; then + echo "Using ONNX Runtime from ONNX_ROOT=$ONNX_ROOT" +elif [ -e "$ort_sdk" ]; then + echo "Using the downloaded ONNX Runtime in $ort_sdk" +elif pkg-config --exists libonnxruntime 2>/dev/null; then + echo "Using the system ONNX Runtime $(pkg-config --modversion libonnxruntime)" +else + echo "No system ONNX Runtime found, downloading it..." + ./subprojects/april-asr/download_onnx_linux_x64.sh +fi + +# Configure from scratch so a changed ONNX Runtime choice is never mixed +# with a cached one. +if [ -d builddir ]; then + meson setup --wipe builddir --buildtype release +else + meson setup builddir --buildtype release +fi +ninja -C builddir + +if [ ! -f "$model" ]; then + echo "Downloading the speech model (about 320 MB)..." + curl -fL --retry 3 -o "$model.part" "https://april.sapples.net/$model" + mv "$model.part" "$model" +fi + +echo +echo "Done. Start Live Captions with ./run.sh" diff --git a/src/asrproc.c b/src/asrproc.c index 5548f38..dac139e 100644 --- a/src/asrproc.c +++ b/src/asrproc.c @@ -116,11 +116,22 @@ static gboolean main_thread_update_label(void *userdata){ return G_SOURCE_REMOVE; } +static gboolean main_thread_warn_slow(void *userdata){ + asr_thread data = userdata; + + if(data->window != NULL) livecaptions_window_warn_slow(data->window); + + return G_SOURCE_REMOVE; +} + static void april_result_handler(void* userdata, AprilResultType result, size_t count, const AprilToken* tokens) { asr_thread data = userdata; if((data->window == NULL) || (data->pause)) return; switch(result) { + case APRIL_RESULT_UNKNOWN: + break; + case APRIL_RESULT_RECOGNITION_PARTIAL: case APRIL_RESULT_RECOGNITION_FINAL: { @@ -148,7 +159,8 @@ static void april_result_handler(void* userdata, AprilResultType result, size_t } case APRIL_RESULT_ERROR_CANT_KEEP_UP: { - livecaptions_window_warn_slow(data->window); + // Called on April's worker thread; GTK must be used from the main one + g_idle_add(main_thread_warn_slow, data); break; } @@ -183,6 +195,10 @@ void asr_thread_enqueue_audio(asr_thread thread, short *data, size_t num_shorts) if(thread->silence_counter >= 24000){ thread->silence_counter = 24000; + // April queues the flush in order with the audio, so speech fed after + // this starts a new utterance even if recognition is still behind. + // Waiting here would block audio capture. Repeated flushes during the + // same silence are no-ops. return aas_flush(thread->session); } @@ -257,7 +273,7 @@ bool asr_thread_update_model(asr_thread data, const char *model_path) { AprilConfig config = { .handler = april_result_handler, - .flags = APRIL_CONFIG_FLAG_ASYNC_RT_BIT, + .flags = APRIL_CONFIG_FLAG_ASYNC_NO_RT_BIT, .userdata = data }; diff --git a/src/audiocap-pa.c b/src/audiocap-pa.c index 14a7bfe..7d8da93 100644 --- a/src/audiocap-pa.c +++ b/src/audiocap-pa.c @@ -55,6 +55,7 @@ static void stream_success_cb(pa_stream *stream, int success, void *userdata); static void stream_read_cb(pa_stream *stream, size_t nbytes, void *userdata); static void server_info_callback(pa_context *c, const pa_server_info *i, void *userdata){ + (void)c; audio_thread_pa data = (audio_thread_pa)userdata; data->source_name = (char *)calloc(1, strlen(i->default_source_name) + 1); @@ -154,11 +155,13 @@ void *run_audio_thread_pa(void *userdata) { } static void context_state_cb(pa_context* context, void* userdata) { + (void)context; audio_thread_pa data = userdata; pa_threaded_mainloop_signal(data->mainloop, 0); } static void stream_state_cb(pa_stream *s, void *userdata) { + (void)s; audio_thread_pa data = userdata; pa_threaded_mainloop_signal(data->mainloop, 0); } @@ -196,6 +199,8 @@ static void stream_read_cb(pa_stream *stream, size_t nbytes, void *userdata) { } static void stream_success_cb(pa_stream *stream, int success, void *userdata) { + (void)stream; + (void)success; audio_thread_pa data = userdata; pa_threaded_mainloop_signal(data->mainloop, 0); } diff --git a/src/audiocap.c b/src/audiocap.c index 675f502..ab3f283 100644 --- a/src/audiocap.c +++ b/src/audiocap.c @@ -41,12 +41,6 @@ struct audio_thread_i { }; -static void *run_audio_thread(void *userdata) { - audio_thread data = (audio_thread)userdata; - - return NULL; -} - audio_thread create_audio_thread(bool microphone, asr_thread asr){ audio_thread data = calloc(1, sizeof(struct audio_thread_i)); diff --git a/src/common.h b/src/common.h index 20ac8be..d16ad22 100644 --- a/src/common.h +++ b/src/common.h @@ -20,7 +20,13 @@ #define SWEAR_REPLACEMENT " [__]" -#define LIVECAPTIONS_VERSION "0.4.1" +#include "livecaptions-config.h" +#define LIVECAPTIONS_VERSION PACKAGE_VERSION -#define MINIMUM_BENCHMARK_RESULT (0.6) +// Recognition runs without time compression (ASYNC_NO_RT), so anything below +// 1.0x real time accumulates a backlog until audio is dropped. Require +// headroom for the load of the rest of the desktop. +#define MINIMUM_BENCHMARK_RESULT (1.2) +// Below this, captions keep up only with little headroom +#define GOOD_BENCHMARK_RESULT (1.5) #define GET_MODEL_PATH() (getenv("APRIL_MODEL_PATH") == NULL) ? "/app/LiveCaptions/models/aprilv0_en-us.april" : getenv("APRIL_MODEL_PATH") diff --git a/src/history.c b/src/history.c index 5ca722a..7c21765 100644 --- a/src/history.c +++ b/src/history.c @@ -77,7 +77,7 @@ void commit_tokens_to_current_history(const AprilToken *tokens, struct history_token *token = &entry->tokens[i]; if(strlen(tokens[i].token) >= HISTORY_TOKEN_MAX_CHARS){ - printf("Token %s is too long! (%d)\n", tokens[i].token, strlen(tokens[i].token)); + printf("Token %s is too long! (%zu)\n", tokens[i].token, strlen(tokens[i].token)); g_assert(false); } diff --git a/src/line-gen.c b/src/line-gen.c index ddca228..4b26743 100644 --- a/src/line-gen.c +++ b/src/line-gen.c @@ -240,7 +240,7 @@ void line_generator_update(struct line_generator *lg, size_t num_tokens, const A // break line if too long if(i == lg->current_line){ curr->len += line_generator_get_text_width(lg, token); - if(curr->len >= lg->max_text_width) { + if (lg->max_text_width > 0 && curr->len >= (size_t)lg->max_text_width) { size_t tgt_brk = j; // find previous word boundary while((!(tokens[tgt_brk].flags & APRIL_TOKEN_FLAG_WORD_BOUNDARY_BIT)) && (tgt_brk > start_of_line)) tgt_brk--; @@ -342,7 +342,7 @@ const char *line_generator_get_plaintext(struct line_generator *lg) { } else { // HACK: We need to remove the tags bool inside_markup = false; - for(int j=0; jhead; j++) { + for(size_t j=0; jhead; j++) { if(curr->text[j] == '<') { inside_markup = true; continue; diff --git a/src/livecaptions-application.c b/src/livecaptions-application.c index 2517f2c..4295337 100644 --- a/src/livecaptions-application.c +++ b/src/livecaptions-application.c @@ -37,10 +37,13 @@ static void deinit_audio(LiveCaptionsApplication *self){ static void init_audio(LiveCaptionsApplication *self) { deinit_audio(self); + // End the previous source's utterance while no capture thread is feeding, + // so the session keeps a single feeder and the boundary falls between + // the two sources. + asr_thread_flush(self->asr); + gboolean use_microphone = g_settings_get_boolean(self->settings, "microphone"); self->audio = create_audio_thread(use_microphone, self->asr); - - asr_thread_flush(self->asr); } LiveCaptionsApplication *livecaptions_application_new (gchar *application_id, GApplicationFlags flags) { @@ -119,7 +122,7 @@ static gboolean on_handle_allow_keep_above(DBLCapNetSapplesLiveCaptionsExternal GDBusMethodInvocation *invocation, gpointer user_data) { - LiveCaptionsApplication *self = LIVECAPTIONS_APPLICATION(user_data); + (void)user_data; override_keep_above_system(true); @@ -130,7 +133,7 @@ static gboolean on_handle_allow_keep_above(DBLCapNetSapplesLiveCaptionsExternal static gboolean livecaptions_application_dbus_register(GApplication *app, GDBusConnection *connection, - const gchar *object_path, + G_GNUC_UNUSED const gchar *object_path, GError **error) { LiveCaptionsApplication *self = LIVECAPTIONS_APPLICATION(app); @@ -172,6 +175,8 @@ livecaptions_application_dbus_unregister(GApplication *app, GDBusConnection *connection, const gchar *object_path) { + (void)connection; + (void)object_path; LiveCaptionsApplication *self = LIVECAPTIONS_APPLICATION(app); g_dbus_interface_skeleton_unexport(G_DBUS_INTERFACE_SKELETON(self->dbus_external)); @@ -242,12 +247,11 @@ livecaptions_application_show_preferences(G_GNUC_UNUSED GSimpleAction *action, LiveCaptionsApplication *self = LIVECAPTIONS_APPLICATION(user_data); if(self->welcome != NULL) return; - GtkApplication *app = GTK_APPLICATION(user_data); - GtkWindow *window = gtk_application_get_active_window (app); - LiveCaptionsSettings *preferences = g_object_new(LIVECAPTIONS_TYPE_SETTINGS, "application", GTK_APPLICATION(self), NULL); + LiveCaptionsSettings *preferences = g_object_new(LIVECAPTIONS_TYPE_SETTINGS, NULL); preferences->application = self; - gtk_window_present (GTK_WINDOW (preferences)); + // No parent, so the preferences open in their own window instead of inside the captions bar + adw_dialog_present(ADW_DIALOG(preferences), NULL); } diff --git a/src/livecaptions-history-window.c b/src/livecaptions-history-window.c index 3f4edbd..275937a 100644 --- a/src/livecaptions-history-window.c +++ b/src/livecaptions-history-window.c @@ -39,7 +39,8 @@ static gboolean close_self_window(gpointer userdata) { return G_SOURCE_REMOVE; } -static void message_cb(AdwMessageDialog *dialog, gchar *response, gpointer userdata){ +static void message_cb(AdwAlertDialog *dialog, gchar *response, gpointer userdata){ + (void)dialog; if(g_str_equal(response, "delete")){ erase_all_history(); @@ -48,26 +49,24 @@ static void message_cb(AdwMessageDialog *dialog, gchar *response, gpointer userd } static void warn_deletion_cb(LiveCaptionsHistoryWindow *self){ - GtkWindow *parent = GTK_WINDOW(gtk_widget_get_root(GTK_WIDGET(self))); - GtkWidget *dialog; + AdwDialog *dialog; - dialog = adw_message_dialog_new(parent, - _("Erase History?"), - _("Everything in history will be erased. You may wish to export your history before erasing!")); + dialog = adw_alert_dialog_new(_("Erase History?"), + _("Everything in history will be erased. You may wish to export your history before erasing!")); - adw_message_dialog_add_responses(ADW_MESSAGE_DIALOG(dialog), - "cancel", _("_Cancel"), - "delete", _("_Erase Everything"), - NULL); + adw_alert_dialog_add_responses(ADW_ALERT_DIALOG(dialog), + "cancel", _("_Cancel"), + "delete", _("_Erase Everything"), + NULL); - adw_message_dialog_set_response_appearance(ADW_MESSAGE_DIALOG(dialog), "delete", ADW_RESPONSE_DESTRUCTIVE); + adw_alert_dialog_set_response_appearance(ADW_ALERT_DIALOG(dialog), "delete", ADW_RESPONSE_DESTRUCTIVE); - adw_message_dialog_set_default_response(ADW_MESSAGE_DIALOG(dialog), "cancel"); - adw_message_dialog_set_close_response(ADW_MESSAGE_DIALOG(dialog), "cancel"); + adw_alert_dialog_set_default_response(ADW_ALERT_DIALOG(dialog), "cancel"); + adw_alert_dialog_set_close_response(ADW_ALERT_DIALOG(dialog), "cancel"); - g_signal_connect(ADW_MESSAGE_DIALOG(dialog), "response", G_CALLBACK(message_cb), self); + g_signal_connect(ADW_ALERT_DIALOG(dialog), "response", G_CALLBACK(message_cb), self); - gtk_window_present(GTK_WINDOW(dialog)); + adw_dialog_present(dialog, GTK_WIDGET(self)); } @@ -232,50 +231,35 @@ static void load_more_cb(LiveCaptionsHistoryWindow *self) { } -static void on_save_response(GtkNativeDialog *native, - int response, - LiveCaptionsHistoryWindow *self) +static void on_save_response(GObject *source, + GAsyncResult *result, + gpointer user_data) { - if(response == GTK_RESPONSE_ACCEPT){ - GtkFileChooser *chooser = GTK_FILE_CHOOSER(native); + LiveCaptionsHistoryWindow *self = LIVECAPTIONS_HISTORY_WINDOW(user_data); - g_autoptr(GFile) file = gtk_file_chooser_get_file(chooser); - + g_autoptr(GFile) file = gtk_file_dialog_save_finish(GTK_FILE_DIALOG(source), result, NULL); + if(file != NULL){ char *path = g_file_get_path(file); export_history_as_text(path); - char *uri = g_file_get_uri(file); - - gtk_show_uri(GTK_WINDOW(self), uri, GDK_CURRENT_TIME); + g_autoptr(GtkFileLauncher) launcher = gtk_file_launcher_new(file); + gtk_file_launcher_launch(launcher, GTK_WINDOW(self), NULL, NULL, NULL); g_free(path); - g_free(uri); } - g_object_unref(native); + g_object_unref(self); } static void export_cb(LiveCaptionsHistoryWindow *self) { - GtkFileChooserNative *native; - GtkFileChooser *chooser; - GtkFileChooserAction action = GTK_FILE_CHOOSER_ACTION_SAVE; - - native = gtk_file_chooser_native_new("Export History", - GTK_WINDOW(self), - action, - _("_Export"), - _("_Cancel")); - - chooser = GTK_FILE_CHOOSER(native); - - gtk_file_chooser_set_current_name(chooser, _("Live Captions History.txt")); + g_autoptr(GtkFileDialog) dialog = gtk_file_dialog_new(); - gtk_native_dialog_show(GTK_NATIVE_DIALOG(native)); + gtk_file_dialog_set_title(dialog, "Export History"); + gtk_file_dialog_set_accept_label(dialog, _("_Export")); + gtk_file_dialog_set_initial_name(dialog, _("Live Captions History.txt")); - g_signal_connect(native, "response", - G_CALLBACK (on_save_response), - self); + gtk_file_dialog_save(dialog, GTK_WINDOW(self), NULL, on_save_response, g_object_ref(self)); } // The window needs to be kept on top if the main window is kept on top, diff --git a/src/livecaptions-settings.c b/src/livecaptions-settings.c index d23a8ab..627f240 100644 --- a/src/livecaptions-settings.c +++ b/src/livecaptions-settings.c @@ -27,19 +27,16 @@ #include "livecaptions-history-window.h" #include "asrproc.h" -G_DEFINE_TYPE(LiveCaptionsSettings, livecaptions_settings, ADW_TYPE_PREFERENCES_WINDOW) +G_DEFINE_TYPE(LiveCaptionsSettings, livecaptions_settings, ADW_TYPE_PREFERENCES_DIALOG) static void rerun_benchmark_cb (LiveCaptionsSettings *self) { - gtk_window_close(GTK_WINDOW(self)); + g_action_group_activate_action(G_ACTION_GROUP(self->application), "setup", NULL); - gtk_widget_activate_action(GTK_WIDGET(self), "app.setup", NULL); - - gtk_window_destroy(GTK_WINDOW(self)); + adw_dialog_close(ADW_DIALOG(self)); } static void about_cb(LiveCaptionsSettings *self) { - GtkRoot *root = gtk_widget_get_root (GTK_WIDGET (self)); - GtkWidget *about; + AdwDialog *about; const char *developers[] = { "abb128", @@ -52,8 +49,7 @@ static void about_cb(LiveCaptionsSettings *self) { }; about = - g_object_new(ADW_TYPE_ABOUT_WINDOW, - "transient-for", root, + g_object_new(ADW_TYPE_ABOUT_DIALOG, "application-icon", "net.sapples.LiveCaptions", "application-name", _("Live Captions"), "developer-name", _("abb128"), @@ -67,69 +63,67 @@ static void about_cb(LiveCaptionsSettings *self) { "translator-credits", _("translator-credits"), NULL); - adw_about_window_add_legal_section(ADW_ABOUT_WINDOW(about), + adw_about_dialog_add_legal_section(ADW_ABOUT_DIALOG(about), _("Model"), NULL, GTK_LICENSE_CUSTOM, "The ASR model was originally trained by Fangjun Kuang (@csukuangfj), and has been finetuned on extra data."); - adw_about_window_add_legal_section(ADW_ABOUT_WINDOW(about), + adw_about_dialog_add_legal_section(ADW_ABOUT_DIALOG(about), "april-asr", "Copyright © 2022 abb128", GTK_LICENSE_GPL_3_0_ONLY, NULL); - adw_about_window_add_legal_section(ADW_ABOUT_WINDOW(about), + adw_about_dialog_add_legal_section(ADW_ABOUT_DIALOG(about), "ONNXRuntime", "Copyright © 2021 Microsoft Corporation", GTK_LICENSE_MIT_X11, NULL); - adw_about_window_add_legal_section(ADW_ABOUT_WINDOW(about), + adw_about_dialog_add_legal_section(ADW_ABOUT_DIALOG(about), "pocketfft", "Copyright © 2010-2019 Max-Planck-Society", GTK_LICENSE_BSD_3, NULL); - adw_about_window_add_legal_section(ADW_ABOUT_WINDOW(about), + adw_about_dialog_add_legal_section(ADW_ABOUT_DIALOG(about), "sonic", "Copyright © 2010 Bill Cox", GTK_LICENSE_APACHE_2_0, NULL); - adw_about_window_add_acknowledgement_section(ADW_ABOUT_WINDOW(about), + adw_about_dialog_add_acknowledgement_section(ADW_ABOUT_DIALOG(about), _("Special thanks to"), special_thanks); - gtk_window_present(GTK_WINDOW(about)); + adw_dialog_present(about, GTK_WIDGET(self)); +} + + +static GtkWindow *get_parent_window(LiveCaptionsSettings *self) { + return GTK_WINDOW(gtk_widget_get_root(GTK_WIDGET(self))); } +static void launch_uri(LiveCaptionsSettings *self, const char *uri) { + g_autoptr(GtkUriLauncher) launcher = gtk_uri_launcher_new(uri); + gtk_uri_launcher_launch(launcher, get_parent_window(self), NULL, NULL, NULL); +} static void report_cb(LiveCaptionsSettings *self) { - gtk_show_uri( - GTK_WINDOW(self), - "https://github.com/abb128/LiveCaptions/issues/48", - GDK_CURRENT_TIME - ); + launch_uri(self, "https://github.com/abb128/LiveCaptions/issues/48"); } static void download_models_cb(LiveCaptionsSettings *self) { - gtk_show_uri( - GTK_WINDOW(self), - "https://abb128.github.io/april-asr/models.html", - GDK_CURRENT_TIME - ); + launch_uri(self, "https://abb128.github.io/april-asr/models.html"); } static void open_history(LiveCaptionsSettings *self) { - GtkRoot *root = gtk_widget_get_root(GTK_WIDGET(self)); - - LiveCaptionsHistoryWindow *window = g_object_new(LIVECAPTIONS_TYPE_HISTORY_WINDOW, "transient-for", root, NULL); + LiveCaptionsHistoryWindow *window = g_object_new(LIVECAPTIONS_TYPE_HISTORY_WINDOW, "application", self->application, NULL); gtk_window_present(GTK_WINDOW(window)); - gtk_window_close(GTK_WINDOW(self)); - gtk_window_destroy(GTK_WINDOW(self)); + adw_dialog_close(ADW_DIALOG(self)); } static const char *get_always_on_top_tip_text(){ @@ -146,29 +140,18 @@ static const char *get_always_on_top_tip_text(){ } -static void on_add_model_response(GtkNativeDialog *native, - int response, - LiveCaptionsSettings *self); +static void on_add_model_response(GObject *source, + GAsyncResult *result, + gpointer user_data); static void add_model_cb(LiveCaptionsSettings *self) { - GtkFileChooserNative *native; - GtkFileChooser *chooser; - GtkFileChooserAction action = GTK_FILE_CHOOSER_ACTION_OPEN; + g_autoptr(GtkFileDialog) dialog = gtk_file_dialog_new(); - native = gtk_file_chooser_native_new("Load Model", - GTK_WINDOW(self), - action, - _("_Load"), - _("_Cancel")); + gtk_file_dialog_set_title(dialog, "Load Model"); + gtk_file_dialog_set_accept_label(dialog, _("_Load")); + gtk_file_dialog_set_default_filter(dialog, self->file_filter); - chooser = GTK_FILE_CHOOSER(native); - gtk_file_chooser_add_filter(chooser, self->file_filter); - - gtk_native_dialog_show(GTK_NATIVE_DIALOG(native)); - - g_signal_connect(native, "response", - G_CALLBACK (on_add_model_response), - self); + gtk_file_dialog_open(dialog, get_parent_window(self), NULL, on_add_model_response, g_object_ref(self)); } static void on_builtin_toggled(LiveCaptionsSettings *self); @@ -217,7 +200,7 @@ static void livecaptions_settings_class_init(LiveCaptionsSettingsClass *klass) { static gboolean deferred_update_keep_above(void *userdata) { LiveCaptionsSettings *self = userdata; - set_window_keep_above(GTK_WINDOW(self), g_settings_get_boolean(self->settings, "keep-on-top")); + set_window_keep_above(get_parent_window(self), g_settings_get_boolean(self->settings, "keep-on-top")); return G_SOURCE_REMOVE; } @@ -226,7 +209,7 @@ static void model_load_failsafe(LiveCaptionsSettings *self, bool load_default); static void on_model_selected(GtkCheckButton* button, LiveCaptionsSettings *self){ if(!gtk_check_button_get_active(button)) return; - const char *model = g_quark_to_string((GQuark)g_object_get_data(G_OBJECT(button), "lcap-model-path")); + const char *model = g_quark_to_string(GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(button), "lcap-model-path"))); if(!asr_thread_update_model(self->application->asr, model)) { model_load_failsafe(self, false); return; @@ -244,7 +227,7 @@ static void on_builtin_toggled(LiveCaptionsSettings *self) { } static void on_model_deleted(GtkButton *button, LiveCaptionsSettings *self) { - const char *model = g_quark_to_string((GQuark)g_object_get_data(G_OBJECT(button), "lcap-model-path")); + const char *model = g_quark_to_string(GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(button), "lcap-model-path"))); if(g_str_equal(model, "/app/LiveCaptions/models/aprilv0_en-us.april")) return; char *active_model = g_settings_get_string(self->settings, "active-model"); @@ -306,7 +289,7 @@ static void insert_model_to_list(LiveCaptionsSettings *self, gchar *model) { g_free(active_model); - gpointer quark = (gpointer)g_quark_from_string(model); + gpointer quark = GUINT_TO_POINTER(g_quark_from_string(model)); g_object_set_data(G_OBJECT(button), "lcap-model-path", quark); g_signal_connect(button, "toggled", G_CALLBACK(on_model_selected), self); @@ -344,18 +327,10 @@ static void add_new_model(LiveCaptionsSettings *self, gchar *model) { } static void model_load_failsafe(LiveCaptionsSettings *self, bool load_default) { - GtkDialogFlags flags = GTK_DIALOG_DESTROY_WITH_PARENT | GTK_DIALOG_MODAL; - GtkWidget *dialog = gtk_message_dialog_new (GTK_WINDOW(self), - flags, - GTK_MESSAGE_ERROR, - GTK_BUTTONS_CLOSE, - "Failed to load model!"); - - gtk_window_present(GTK_WINDOW(dialog)); + AdwDialog *dialog = adw_alert_dialog_new("Failed to load model!", NULL); + adw_alert_dialog_add_response(ADW_ALERT_DIALOG(dialog), "close", _("_Close")); - g_signal_connect(dialog, "response", - G_CALLBACK (gtk_window_destroy), - NULL); + adw_dialog_present(dialog, GTK_WIDGET(self)); char *prev_model; if(load_default) { @@ -369,15 +344,14 @@ static void model_load_failsafe(LiveCaptionsSettings *self, bool load_default) { if(load_default) gtk_check_button_set_active(self->radio_button_1, true); } -static void on_add_model_response(GtkNativeDialog *native, - int response, - LiveCaptionsSettings *self) +static void on_add_model_response(GObject *source, + GAsyncResult *result, + gpointer user_data) { - if(response == GTK_RESPONSE_ACCEPT){ - GtkFileChooser *chooser = GTK_FILE_CHOOSER(native); + LiveCaptionsSettings *self = LIVECAPTIONS_SETTINGS(user_data); - g_autoptr(GFile) file = gtk_file_chooser_get_file(chooser); - + g_autoptr(GFile) file = gtk_file_dialog_open_finish(GTK_FILE_DIALOG(source), result, NULL); + if(file != NULL){ char *model = g_file_get_path(file); @@ -385,7 +359,7 @@ static void on_add_model_response(GtkNativeDialog *native, model_load_failsafe(self, false); g_free(model); - g_object_unref(native); + g_object_unref(self); return; } @@ -398,7 +372,7 @@ static void on_add_model_response(GtkNativeDialog *native, g_free(model); } - g_object_unref(native); + g_object_unref(self); } static void livecaptions_settings_init(LiveCaptionsSettings *self) { @@ -427,7 +401,7 @@ static void livecaptions_settings_init(LiveCaptionsSettings *self) { sprintf(benchmark_result, "%.2f", (float)benchmark_result_v); gtk_label_set_text(self->benchmark_label, benchmark_result); - if(is_keep_above_supported(GTK_WINDOW(self))) { + if(is_keep_above_supported(GTK_WIDGET(self))) { g_settings_bind(self->settings, "keep-on-top", self->keep_above_switch, "active", G_SETTINGS_BIND_DEFAULT); gtk_widget_set_sensitive(GTK_WIDGET(self->keep_above_switch), true); diff --git a/src/livecaptions-settings.h b/src/livecaptions-settings.h index 0c68318..9ec5a74 100644 --- a/src/livecaptions-settings.h +++ b/src/livecaptions-settings.h @@ -22,7 +22,7 @@ #include "livecaptions-application.h" struct _LiveCaptionsSettings { - AdwPreferencesWindow parent_instance; + AdwPreferencesDialog parent_instance; LiveCaptionsApplication *application; @@ -59,6 +59,6 @@ G_BEGIN_DECLS #define LIVECAPTIONS_TYPE_SETTINGS (livecaptions_settings_get_type()) -G_DECLARE_FINAL_TYPE(LiveCaptionsSettings, livecaptions_settings, LIVECAPTIONS, SETTINGS, AdwPreferencesWindow) +G_DECLARE_FINAL_TYPE(LiveCaptionsSettings, livecaptions_settings, LIVECAPTIONS, SETTINGS, AdwPreferencesDialog) G_END_DECLS diff --git a/src/livecaptions-settings.ui b/src/livecaptions-settings.ui index 9f3ac91..d36e1ac 100644 --- a/src/livecaptions-settings.ui +++ b/src/livecaptions-settings.ui @@ -1,9 +1,8 @@ - -