diff --git a/CLAUDE.md b/CLAUDE.md index ef005ec..af0b34c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,11 +28,12 @@ npm run deploy # Deploy to Firebase Functions (asia-south1) > [!NOTE] > `:app:testDebugUnitTest` now compiles and runs (the stale `AIGrammarViewModelTest`, which > referenced deleted Grammar classes, was removed). It holds **pure JVM tests** for the -> conversion-critical decision logic — `ProStripState` (free-vs-PRO strip), `UnlockRouter` -> (auth→paywall routing), `PostLoginDestination` (post-login routing), and `KeyboardLayoutTest` -> (guards the removed "+"/`-20` key across all `res/xml` layouts). Keep view/IME/RevenueCat-SDK -> behaviour out of these (no mockito/Robolectric); test it by extracting the decision into a pure -> unit (see those classes) or via on-device verification. `:botok:test` remains the tokenizer suite. +> conversion-critical decision logic — `ProStripState` (free-vs-PRO strip), `WordComposer` +> (suggestion tap / tsheg rules), `UnlockRouter` (auth→paywall routing), `PostLoginDestination` +> (post-login routing), and `KeyboardLayoutTest` (guards the removed "+"/`-20` key across all +> `res/xml` layouts). Keep view/IME/RevenueCat-SDK behaviour out of these (no mockito/Robolectric); +> test it by extracting the decision into a pure unit (see those classes) or via on-device +> verification. `:botok:test` remains the tokenizer suite. ## Architecture Overview @@ -114,6 +115,12 @@ preference changes apply the next time the keyboard opens. - `TibetanKeyboardView extends KeyboardView` — the typing surface. Custom `onDraw` paints the **Enter key gold** on top of any layout (visual only; key codes untouched). - `SuggestionStripView` — Botok autocomplete; top suggestion gold, rest cream. +- `WordComposer` (pure, unit-tested) decides what a suggestion tap does: it replaces only the part + of the typed run the engine matched (`SuggestionEngine.suggest(...).matched`: the whole run, or + the longest tail of it starting after a tsheg), writes the next tsheg (`བོ` → `བོད་`), swallows + a tsheg typed onto it and lets a shad take its place (`བོད།`, but `རང་།` after nga). Every edit + is checked against the text before the cursor (`onUpdateSelection` re-syncs), so a cursor move + can never make a tap delete the wrong characters. Enter (`KEYCODE_DONE`) closes the run too. - `KeyboardMode` enum: `NORMAL`, `AI_GRAMMAR`, `AI_REPHRASE`. **Keyboard layouts & theming** are driven by `PreferenceManager.getDefaultSharedPreferences` diff --git a/app/build.gradle b/app/build.gradle index 6455cc0..ac175a9 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -31,10 +31,10 @@ android { // installed devices on API 23 (0.03%). minSdkVersion 24 targetSdkVersion 36 - versionCode project.hasProperty('versionCode') ? project.property('versionCode').toInteger() : 58 + versionCode project.hasProperty('versionCode') ? project.property('versionCode').toInteger() : 59 // RevenueCat targeting rules match on this (App version >= 2.3.0). - versionName "2.3.1" + versionName "2.3.2" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" multiDexEnabled true diff --git a/app/src/main/java/com/kharagedition/tibetankeyboard/service/TibetanKeyboard.kt b/app/src/main/java/com/kharagedition/tibetankeyboard/service/TibetanKeyboard.kt index e26d217..528f62d 100644 --- a/app/src/main/java/com/kharagedition/tibetankeyboard/service/TibetanKeyboard.kt +++ b/app/src/main/java/com/kharagedition/tibetankeyboard/service/TibetanKeyboard.kt @@ -26,6 +26,7 @@ import android.view.inputmethod.InputConnection import android.widget.FrameLayout import androidx.preference.PreferenceManager import com.google.firebase.auth.FirebaseAuth +import com.kharagedition.tibetankeyboard.BuildConfig import com.kharagedition.tibetankeyboard.analytics.AppAnalytics import com.kharagedition.tibetankeyboard.analytics.UserActivityTracker import com.kharagedition.tibetankeyboard.data.local.TypingStatsStore @@ -41,6 +42,7 @@ import com.kharagedition.tibetankeyboard.ui.journey.WordSegmenter import com.kharagedition.tibetankeyboard.ui.keyboard.AIKeyboardInterface import com.kharagedition.tibetankeyboard.ui.keyboard.KeyboardLayoutHint import com.kharagedition.tibetankeyboard.ui.keyboard.TibetanKeyboardView +import com.kharagedition.tibetankeyboard.ui.keyboard.WordComposer import com.kharagedition.tibetankeyboard.ui.keyboard.AIKeyboardView import com.kharagedition.tibetankeyboard.ui.keyboard.EmojiKeyboardView import com.kharagedition.tibetankeyboard.ui.keyboard.AIKeyboardCodes @@ -66,11 +68,10 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa // honour the user's own saved language choice. Set on every onStartInputView. private var forcedTibetan: Boolean? = null - // Tracks how many Unicode code points the user has typed since the last word boundary. - // Used to know exactly what to delete when a suggestion is selected. - // Resets on: shad (།), space, newline, or suggestion commit. - // Tshek (་) is NOT a boundary — it is part of the Tibetan word. - private var currentWordLength = 0 + // The word being typed (the suggestion prefix) and the rules for accepting a suggestion: + // what a tap replaces, the tsheg it writes, and how the next key treats that tsheg. Pure and + // unit-tested; this service only feeds it editor text and applies the edits it returns. + private val composer = WordComposer() // Journey streak/stats. PRIVACY: the store only ever receives counts and one-way word // hashes — no typed text is persisted or transmitted (see TypingStatsStore's contract). @@ -125,7 +126,7 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa } override fun onStartInputView(info: EditorInfo?, restarting: Boolean) { - currentWordLength = 0 + composer.reset() // Read the focused field's requested layout before (re)building the view, // so onCreateInputView can open on the matching language. forcedTibetan = KeyboardLayoutHint.forcedTibetan(info?.privateImeOptions) @@ -259,9 +260,9 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa when (i) { Keyboard.KEYCODE_DELETE -> { + composer.onDelete(textBeforeCursor(inputConnection)) inputConnection.deleteSurroundingText(1, 0) - if (currentWordLength > 0) currentWordLength-- - aiKeyboardView?.updateSuggestions(currentPrefix(inputConnection)) + aiKeyboardView?.updateSuggestions(composer.composing) } Keyboard.KEYCODE_SHIFT -> { isCaps = !isCaps @@ -269,6 +270,12 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa keyboardView!!.invalidateAllKeys() } Keyboard.KEYCODE_DONE -> { + // Enter ends the run like a shad does (word stats, suggestion state); the editor + // itself gets the action / key event below, not the composer's text. + val edit = composer.onChar('\n', textBeforeCursor(inputConnection)) + if (edit.finishedChunk.isNotEmpty()) { + recordChunkAsWords(edit.finishedChunk) + } sendDefaultEditorAction(true) inputConnection.sendKeyEvent( KeyEvent( @@ -276,6 +283,7 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa KeyEvent.KEYCODE_ENTER ) ) + aiKeyboardView?.updateSuggestions("") } // AI Feature Codes - These are handled by AIKeyboardView now @@ -324,28 +332,23 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa else -> { var code = i.toChar() if (Character.isLetter(code) && isCaps) code = Character.toUpperCase(code) - // Shad (།) and space are sentence/word boundaries — reset the word tracker. - // Tshek (་) is a syllable separator WITHIN a word, so it increments the counter. - if (code == '།' || code == '༎' || code == ' ' || code == '\n') { - // A chunk just finished: segment it into real dictionary words (Tibetan has - // no spaces between words, so the whole chunk can be a full clause) and fold - // each into the Journey stats. Only one-way hashes survive (vocabulary size), - // never the text. - if (currentWordLength > 0) { - recordChunkAsWords(currentPrefix(inputConnection)) - } - currentWordLength = 0 - } else { - currentWordLength++ + val edit = composer.onChar(code, textBeforeCursor(inputConnection)) + // A shad (།), space or newline just finished a chunk: segment it into real + // dictionary words (Tibetan has no spaces between words, so the whole chunk can + // be a full clause) and fold each into the Journey stats. Only one-way hashes + // survive (vocabulary size), never the text. + if (edit.finishedChunk.isNotEmpty()) { + recordChunkAsWords(edit.finishedChunk) } - inputConnection.commitText(code.toString(), 1) - // Journey streak: count Tibetan code points typed (a number, nothing else). - if (TypingStatsStore.isTibetanCodePoint(code.code)) { + applyEdit(inputConnection, edit) + // Journey streak: count Tibetan code points typed (a number, nothing else). A + // tsheg typed onto the one an accepted suggestion already wrote inserts nothing. + if (edit.text.isNotEmpty() && TypingStatsStore.isTibetanCodePoint(code.code)) { typingStats.recordTibetanChars(1)?.let { milestone -> AppAnalytics.logStreakMilestone(milestone) } } - aiKeyboardView?.updateSuggestions(currentPrefix(inputConnection)) + aiKeyboardView?.updateSuggestions(composer.composing) } } } @@ -490,6 +493,7 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa inputConnection.deleteSurroundingText(textLength, 0) inputConnection.commitText(correctedText, 1) } + composer.reset() currentMode = KeyboardMode.NORMAL aiKeyboardView?.showNormalKeyboard() } @@ -503,6 +507,7 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa inputConnection.deleteSurroundingText(textLength, 0) inputConnection.commitText(translatedText, 1) } + composer.reset() currentMode = KeyboardMode.NORMAL aiKeyboardView?.showNormalKeyboard() } @@ -516,6 +521,7 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa inputConnection.deleteSurroundingText(textLength, 0) inputConnection.commitText(rephrasedText, 1) } + composer.reset() currentMode = KeyboardMode.NORMAL aiKeyboardView?.showNormalKeyboard() } @@ -525,20 +531,47 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa aiKeyboardView?.showNormalKeyboard() } - override fun onSuggestionSelected(word: String) { + override fun onSuggestionSelected(word: String, matched: String) { val ic = currentInputConnection ?: return AppAnalytics.logKeyboardSuggestionSelected() - Log.d("TibetanKeyboard", "onSuggestionSelected: '$word', deleting $currentWordLength chars") - if (currentWordLength > 0) ic.deleteSurroundingText(currentWordLength, 0) - ic.commitText(word, 1) - currentWordLength = 0 - // Journey stats: an accepted suggestion completes a word (hash-only, see store contract). + val edit = composer.onAccept( + word, + matched, + textBeforeCursor(ic, extra = matched.length), + ic.getTextAfterCursor(1, 0), + ) + if (BuildConfig.DEBUG) { + Log.d( + "TibetanKeyboard", + "onSuggestionSelected: '$word' for '$matched' → delete ${edit.deleteBefore}, insert '${edit.text}'" + ) + } + applyEdit(ic, edit) + // Journey stats (hash-only, see store contract): the accepted word completes a word, and + // the syllables it leaves in place before it are typed words too. + if (edit.finishedChunk.isNotEmpty()) { + recordChunkAsWords(edit.finishedChunk) + } typingStats.recordWordTyped(word)?.let { milestone -> AppAnalytics.logStreakMilestone(milestone) } aiKeyboardView?.updateSuggestions("") } + override fun onUpdateSelection( + oldSelStart: Int, oldSelEnd: Int, newSelStart: Int, newSelEnd: Int, + candidatesStart: Int, candidatesEnd: Int, + ) { + super.onUpdateSelection(oldSelStart, oldSelEnd, newSelStart, newSelEnd, candidatesStart, candidatesEnd) + // Our own edits echo back here too. The composer keeps state the editor still confirms + // and drops the rest (the user tapped elsewhere, the app changed the text), so the strip + // never offers to replace characters that are no longer before the cursor. + if (!composer.hasState) return + val ic = currentInputConnection ?: return + val before = if (newSelStart == newSelEnd) textBeforeCursor(ic) else null + if (composer.sync(before)) aiKeyboardView?.updateSuggestions("") + } + override fun onOpenChat() { startActivity( Intent(this, ChatActivity::class.java).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) @@ -570,14 +603,20 @@ class TibetanKeyboard : InputMethodService(), OnKeyboardActionListener, AIKeyboa } } - // Returns the Unicode code points the user has typed since the last word boundary, - // verified against actual text before the cursor. - private fun currentPrefix(ic: InputConnection): String { - if (currentWordLength == 0) return "" - val textBefore = ic.getTextBeforeCursor(currentWordLength + 5, 0)?.toString() ?: "" - val prefix = textBefore.takeLast(currentWordLength) - Log.d("TibetanKeyboard", "prefix='$prefix' (wordLen=$currentWordLength)") - return prefix + /** + * The text before the cursor, as much as the composer needs to check its state (plus + * [extra]); null when the editor cannot say, which the composer treats as "trust nothing". + */ + private fun textBeforeCursor(ic: InputConnection, extra: Int = 0): CharSequence? = + ic.getTextBeforeCursor(composer.lookback + extra, 0) + + /** Applies one composer decision as a single editor change. */ + private fun applyEdit(ic: InputConnection, edit: WordComposer.Edit) { + if (edit.deleteBefore == 0 && edit.text.isEmpty()) return + ic.beginBatchEdit() + if (edit.deleteBefore > 0) ic.deleteSurroundingText(edit.deleteBefore, 0) + if (edit.text.isNotEmpty()) ic.commitText(edit.text, 1) + ic.endBatchEdit() } private fun vibratePhone() { diff --git a/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardInterface.kt b/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardInterface.kt index e07ad5f..47cbf77 100644 --- a/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardInterface.kt +++ b/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardInterface.kt @@ -8,7 +8,12 @@ interface AIKeyboardInterface { fun onTranslateReplace(originalText: String, translatedText: String) fun onRephraseReplace(originalText: String, rephrasedText: String) fun onAICancel() - fun onSuggestionSelected(word: String) + + /** + * The user tapped [word] in the suggestion strip. [matched] is the end of the typed prefix + * the word completes (see `SuggestionEngine.suggest`): that, and only that, gets replaced. + */ + fun onSuggestionSelected(word: String, matched: String) /** Open the AI chat screen (PRO users). */ fun onOpenChat() diff --git a/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardView.kt b/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardView.kt index d9a9610..fa7ebe4 100644 --- a/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardView.kt +++ b/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/AIKeyboardView.kt @@ -1,5 +1,6 @@ package com.kharagedition.tibetankeyboard.ui.keyboard +import com.kharagedition.tibetankeyboard.BuildConfig import com.kharagedition.tibetankeyboard.R import android.animation.AnimatorSet @@ -70,6 +71,8 @@ class AIKeyboardView @JvmOverloads constructor( private var themeColor: String = "#FF704C04" private lateinit var suggestionStrip: SuggestionStripView private var suggestionEngine: SuggestionEngine? = null + /** The end of the typed prefix the chips now showing complete (what a tap replaces). */ + private var suggestionsMatched = "" private var currentOriginalText = "" private var currentSuggestedText = "" private var currentSourceLang = "en" // English @@ -230,7 +233,7 @@ class AIKeyboardView @JvmOverloads constructor( AppAnalytics.logSuggestionQuotaExhausted(dailyLimit()) refreshStrip() } - aiKeyboardInterface?.onSuggestionSelected(word) + aiKeyboardInterface?.onSuggestionSelected(word, suggestionsMatched) } suggestionStrip.onLockedSuggestionClick = { aiKeyboardInterface?.onUnlockPro(AppAnalytics.UpgradeSource.KEYBOARD_SUGGESTIONS) @@ -632,8 +635,12 @@ class AIKeyboardView @JvmOverloads constructor( val mode = suggestionMode() val strip = ProStripState.forUser(isPremiumUser, mode) applyStrip(strip) - val suggestions = if (prefix.isNotEmpty()) engine.getSuggestions(prefix, strip.maxSuggestions) else emptyList() - Log.d(TAG, "updateSuggestions: prefix='$prefix' → ${suggestions.size} results: $suggestions") + val result = if (prefix.isNotEmpty()) engine.suggest(prefix, strip.maxSuggestions) else SuggestionEngine.Suggestions.NONE + val suggestions = result.words + suggestionsMatched = result.matched + if (BuildConfig.DEBUG) { + Log.d(TAG, "updateSuggestions: prefix='$prefix' → ${suggestions.size} results for '${result.matched}': $suggestions") + } // Only swap the chips inside the single top bar — never change its height. Empty list → // blank middle; the bar stays one fixed height so the keyboard never jumps (Gboard flow). if (mode == SuggestionMode.LIVE) { diff --git a/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/WordComposer.kt b/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/WordComposer.kt new file mode 100644 index 0000000..ecdeac6 --- /dev/null +++ b/app/src/main/java/com/kharagedition/tibetankeyboard/ui/keyboard/WordComposer.kt @@ -0,0 +1,156 @@ +package com.kharagedition.tibetankeyboard.ui.keyboard + +/** + * The word being typed, and what a suggestion tap and the keys around it do to the editor. + * + * Pure (no Android) so the rules are unit-tested; the IME applies the returned [Edit]s to its + * InputConnection and asks the engine for suggestions on [composing]. + * + * How Tibetan is typed: syllables are joined with a tsheg (་) and words are not separated at all, + * so a run of text only ends at a shad (།), a space or a newline. The word in progress is + * everything typed since the last run end or the last accepted suggestion. The engine matches it + * whole (བོད་ཡ → བོད་ཡུལ) or, failing that, a tail of it starting after a tsheg (ང་བོད་ར → + * བོད་རིགས, ང་བོད་པ → པད་མ), and an accepted word replaces exactly the part that was matched, + * never the whole run. + * + * Accepting a word also writes the tsheg the user would type next (བོ → བོད་). That tsheg stays + * provisional until the next key: a typed tsheg is swallowed (never ་་) and changes nothing, a + * shad takes the tsheg's place (བོད་ + ། → བོད།) except after nga, where the tsheg is kept so ང། + * cannot be read as ད (རང་།), and any other key makes it an ordinary tsheg. + * + * Every decision is checked against the text actually before the cursor: state the editor no + * longer confirms (the user tapped elsewhere, the app changed the text, a callback was lost) is + * dropped, so a tap can never delete characters that are not the ones it completes. + */ +class WordComposer { + + /** What the user has typed since the last run end or accepted suggestion. */ + var composing: String = "" + private set + + /** The word + tsheg just accepted, while that tsheg is still provisional. */ + private var provisional: String = "" + + /** An editor change: delete [deleteBefore] chars before the cursor, then insert [text]. */ + data class Edit( + val deleteBefore: Int = 0, + val text: String = "", + /** + * Typed text that is now final, for the Journey word stats: the run a boundary key just + * closed, or the syllables an accepted word leaves in place before it. Empty otherwise. + */ + val finishedChunk: String = "", + ) + + val hasState: Boolean get() = composing.isNotEmpty() || provisional.isNotEmpty() + + /** How much editor text before the cursor [sync] needs to check this state. */ + val lookback: Int get() = maxOf(composing.length, provisional.length) + LOOKBACK_MARGIN + + fun reset() { + composing = "" + provisional = "" + } + + /** + * Drops whatever [before] (the text before the cursor; null when the editor cannot say) no + * longer confirms. Returns true when the word in progress was dropped, so the strip can be + * cleared. + */ + fun sync(before: CharSequence?): Boolean { + val hadWord = composing.isNotEmpty() + if (before == null || !before.endsWith(composing)) composing = "" + if (provisional.isNotEmpty() && (before == null || !before.endsWith(provisional))) provisional = "" + return hadWord && composing.isEmpty() + } + + /** The user typed [c]; returns what to write, which may differ from [c] after an accepted word. */ + fun onChar(c: Char, before: CharSequence?): Edit { + sync(before) + var deleteBefore = 0 + if (provisional.isNotEmpty()) { + // The tsheg is already there: nothing to write, and it stays provisional. + if (isTsheg(c)) return Edit() + val accepted = provisional.dropLast(1) + provisional = "" + if (isShad(c) && !endsWithNga(accepted)) deleteBefore = 1 + } + if (endsRun(c)) { + val chunk = composing + composing = "" + return Edit(deleteBefore, c.toString(), chunk) + } + // An editor that cannot report its text gets no suggestions: a replacement could never + // be checked against what is really before the cursor. + if (before != null) composing += c + return Edit(deleteBefore, c.toString()) + } + + /** The user is deleting the char before the cursor. */ + fun onDelete(before: CharSequence?) { + sync(before) + provisional = "" + if (composing.isNotEmpty()) composing = composing.dropLast(1) + } + + /** + * The user accepted [word], which the engine offered for [matched] (the end of the prefix it + * completes). [after] is the text after the cursor (null when the editor cannot say), so no + * tsheg is written onto one that is already there or in front of a shad. + */ + fun onAccept(word: String, matched: String, before: CharSequence?, after: CharSequence?): Edit { + sync(before) + val deleteBefore = + if (matched.isNotEmpty() && before != null && before.endsWith(matched)) matched.length else 0 + val text = if (wantsTsheg(word, after)) word + TSHEG else word + // Whatever the word does not replace stays in the editor as typed words. + val kept = composing.dropLast(deleteBefore) + composing = "" + provisional = if (text.length > word.length) text else "" + return Edit(deleteBefore, text, kept) + } + + private fun wantsTsheg(word: String, after: CharSequence?): Boolean { + if (word.isEmpty() || !isLetterOrVowel(word.last())) return false + // Unknown text after the cursor: a missing tsheg is cheaper than a doubled one. + if (after == null) return false + val next = after.firstOrNull() ?: return true + return when { + isTsheg(next) -> false + isShad(next) -> endsWithNga(word) + else -> true + } + } + + companion object { + const val TSHEG = '་' + private const val TSHEG_NO_BREAK = '༌' + private const val NGA = 'ང' + private const val SUBJOINED_NGA = 'ྔ' + private const val LOOKBACK_MARGIN = 8 + + fun isTsheg(c: Char): Boolean = c == TSHEG || c == TSHEG_NO_BREAK + + /** Shad (།), nyis shad (༎) and the other clause-ending marks, plus gter tsheg (༔). */ + fun isShad(c: Char): Boolean = c in '།'..'༒' || c == '༔' + + /** A key that ends the run the word in progress belongs to. */ + fun endsRun(c: Char): Boolean = isShad(c) || c == ' ' || c == '\n' + + /** + * True when the syllable's last letter is nga (ང, or subjoined ྔ in a stack), vowel signs + * aside. Such a syllable keeps its tsheg before a shad. + */ + fun endsWithNga(syllable: CharSequence): Boolean { + var i = syllable.length - 1 + while (i >= 0 && isVowelOrMark(syllable[i])) i-- + return i >= 0 && (syllable[i] == NGA || syllable[i] == SUBJOINED_NGA) + } + + /** Vowel signs and the marks that sit on a syllable's last letter. */ + private fun isVowelOrMark(c: Char): Boolean = c in 'ཱ'..'྄' || c == '྆' || c == '྇' + + private fun isLetterOrVowel(c: Char): Boolean = + c in 'ཀ'..'ཬ' || c in 'ྐ'..'ྼ' || isVowelOrMark(c) + } +} diff --git a/app/src/test/java/com/kharagedition/tibetankeyboard/ui/keyboard/SuggestionFlowTest.kt b/app/src/test/java/com/kharagedition/tibetankeyboard/ui/keyboard/SuggestionFlowTest.kt new file mode 100644 index 0000000..d83299e --- /dev/null +++ b/app/src/test/java/com/kharagedition/tibetankeyboard/ui/keyboard/SuggestionFlowTest.kt @@ -0,0 +1,123 @@ +package com.kharagedition.tibetankeyboard.ui.keyboard + +import com.kharagedition.botok.autocomplete.SuggestionEngine +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Assume.assumeTrue +import org.junit.BeforeClass +import org.junit.Test +import java.io.File + +/** + * The strip and the tap together, on the real dictionary: the engine's `matched` feeds the + * composer exactly as the IME wires them. Replays the keystrokes of the reported recording + * (pick བོད, type ར, pick རང) and checks both the chips shown and the text that results. + * + * Reads the botok assets from the sibling module like botok's own SmokeTest does; skipped when + * they are not on disk. Asserts on the words that matter, not on the dictionary's exact ranking. + */ +class SuggestionFlowTest { + + /** A text field with the cursor at the end. */ + private class Editor(var text: String = "") { + fun apply(edit: WordComposer.Edit) { + text = text.dropLast(edit.deleteBefore) + edit.text + } + } + + private val composer = WordComposer() + private val editor = Editor() + + /** The chips the strip would show for the word in progress (PRO width, four chips). */ + private fun strip(): SuggestionEngine.Suggestions = engine.suggest(composer.composing, ProStripState.PRO_SUGGESTIONS) + + private fun type(keys: String) { + for (c in keys) editor.apply(composer.onChar(c, editor.text)) + } + + private fun tap(word: String) { + val shown = strip() + assertTrue("'$word' must be on the strip, got ${shown.words}", word in shown.words) + editor.apply(composer.onAccept(word, shown.matched, editor.text, after = "")) + } + + @Test + fun theRecordedFlow_endsWithBothWordsAndTheirTshegs() { + type("བ") + assertTrue(strip().words.isNotEmpty()) + + type("ོ") + assertTrue("བོད" in strip().words) + + tap("བོད") + assertEquals("བོད་", editor.text) + assertEquals("nothing to suggest right after a pick", 0, strip().words.size) + + type("་") + assertEquals("the habitual tsheg is swallowed", "བོད་", editor.text) + + type("ར") + assertTrue("རང" in strip().words) + assertEquals("ར", strip().matched) + + tap("རང") + assertEquals("བོད་རང་", editor.text) + + type("།") + assertEquals("after nga the tsheg stays before the shad", "བོད་རང་།", editor.text) + } + + @Test + fun handTypedRun_tapReplacesOnlyWhatTheStripMatched() { + val run = "ང་བོད་པ" + type(run) + val shown = strip() + assertTrue(shown.words.isNotEmpty()) + // What the chips complete is the run itself or a tail of it starting after a tsheg. + assertTrue(run.endsWith(shown.matched)) + val cut = run.length - shown.matched.length + assertTrue(cut == 0 || run[cut - 1] == WordComposer.TSHEG) + + tap(shown.words.first()) + + assertEquals(run.dropLast(shown.matched.length) + shown.words.first() + "་", editor.text) + } + + @Test + fun compoundTypedAfterAnotherWord_isOffered_andReplacedWhole() { + type("ང་བོད་ར") + val shown = strip() + assertTrue("བོད་རིགས" in shown.words) + assertEquals("བོད་ར", shown.matched) + + tap("བོད་རིགས") + + assertEquals("ང་བོད་རིགས་", editor.text) + } + + @Test + fun handTypedCompound_tapReplacesTheWholeCompound() { + type("བོད་ར") + assertEquals("བོད་ར", strip().matched) + + tap("བོད་རིགས") + + assertEquals("བོད་རིགས་", editor.text) + } + + companion object { + private val assets = File("../botok/src/main/assets/botok/general/dictionary") + private lateinit var engine: SuggestionEngine + + @BeforeClass + @JvmStatic + fun loadDictionary() { + assumeTrue("botok dictionary assets not found at $assets", assets.isDirectory) + engine = SuggestionEngine() + for (name in listOf("words/tsikchen.tsv", "words/uncompound_lexicon.tsv", "words_non_inflected/particles.tsv")) { + File(assets, name).bufferedReader(Charsets.UTF_8).useLines { engine.addLines(it) } + } + engine.ready() + } + } +} diff --git a/app/src/test/java/com/kharagedition/tibetankeyboard/ui/keyboard/WordComposerTest.kt b/app/src/test/java/com/kharagedition/tibetankeyboard/ui/keyboard/WordComposerTest.kt new file mode 100644 index 0000000..04c4334 --- /dev/null +++ b/app/src/test/java/com/kharagedition/tibetankeyboard/ui/keyboard/WordComposerTest.kt @@ -0,0 +1,257 @@ +package com.kharagedition.tibetankeyboard.ui.keyboard + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The suggestion-tap rules: what a tap replaces, the tsheg it writes, and what the keys after it + * do. Driven through a tiny editor so each case reads like the typing it describes. + */ +class WordComposerTest { + + /** A text field with the cursor at the end, applying the composer's edits. */ + private class Editor(var text: String = "") { + fun apply(edit: WordComposer.Edit): Editor { + text = text.dropLast(edit.deleteBefore) + edit.text + return this + } + } + + private val composer = WordComposer() + private val editor = Editor() + + private fun type(keys: String) { + for (c in keys) editor.apply(composer.onChar(c, editor.text)) + } + + private fun accept(word: String, matched: String = "") = + editor.apply(composer.onAccept(word, matched, editor.text, after = "")) + + private fun backspace() { + composer.onDelete(editor.text) + editor.text = editor.text.dropLast(1) + } + + @Test + fun acceptedWord_replacesWhatItCompletes_andWritesTheTsheg() { + type("བོ") + assertEquals("བོ", composer.composing) + + accept("བོད", matched = "བོ") + + assertEquals("བོད་", editor.text) + assertEquals("", composer.composing) + } + + @Test + fun theReportedFlow_secondTapKeepsTheTsheg() { + // Pick བོད, type ར, pick རང: used to give བོདརང (the tsheg deleted with the ར). + type("བོ") + accept("བོད", matched = "བོ") + type("ར") + assertEquals("ར", composer.composing) + + accept("རང", matched = "ར") + + assertEquals("བོད་རང་", editor.text) + } + + @Test + fun typedTshegAfterAcceptedWord_isSwallowed_andChangesNothing() { + accept("བོད") + type("་") + assertEquals("one tsheg, not ་་", "བོད་", editor.text) + + type("།") + assertEquals("the shad still takes the tsheg's place", "བོད།", editor.text) + } + + @Test + fun acceptedWord_reportsTheSyllablesItKeeps_forStats() { + type("ང་བོད་པ") + val tail = composer.onAccept("པད་མ", "པ", editor.text, "") + assertEquals("ང་བོད་", tail.finishedChunk) + + val e = Editor() + val fresh = WordComposer() + for (c in "བོད་ར") e.apply(fresh.onChar(c, e.text)) + val whole = fresh.onAccept("བོད་རིགས", "བོད་ར", e.text, "") + assertEquals("", whole.finishedChunk) + } + + @Test + fun fallbackMatch_replacesOnlyTheLastSyllable() { + // The engine found nothing for the whole run and matched the syllable after the last tsheg. + type("ང་བོད་པ") + + accept("པད་མ", matched = "པ") + + assertEquals("ང་བོད་པད་མ་", editor.text) + } + + @Test + fun directMatch_replacesTheWholeRun() { + type("བོད་ར") + + accept("བོད་རིགས", matched = "བོད་ར") + + assertEquals("བོད་རིགས་", editor.text) + } + + @Test + fun shadAfterAcceptedWord_takesTheTshegsPlace() { + type("བོ") + accept("བོད", matched = "བོ") + + type("།") + + assertEquals("བོད།", editor.text) + assertEquals("", composer.composing) + } + + @Test + fun shadAfterNga_keepsTheTsheg() { + // ང། would read as ད, so the tsheg stays: after a bare nga, nga with a vowel, and a stack. + for (word in listOf("རང", "ངོ", "ལྔ")) { + val e = Editor() + val c = WordComposer() + e.apply(c.onAccept(word, "", e.text, "")) + e.apply(c.onChar('།', e.text)) + assertEquals("$word་།", e.text) + } + } + + @Test + fun otherClauseMarks_countAsShad() { + for (mark in listOf('༎', '༔', '༑')) { + val e = Editor() + val c = WordComposer() + e.apply(c.onAccept("བོད", "", e.text, "")) + e.apply(c.onChar(mark, e.text)) + assertEquals("བོད$mark", e.text) + } + } + + @Test + fun anyOtherKey_makesTheTshegOrdinary() { + accept("བོད") + type("ཀ") + backspace() + type("།") + + assertEquals("the tsheg is no longer provisional", "བོད་།", editor.text) + } + + @Test + fun backspaceAfterAccept_deletesJustTheTsheg() { + accept("བོད") + backspace() + assertEquals("བོད", editor.text) + + type("།") + assertEquals("བོད།", editor.text) + } + + @Test + fun backspace_shortensTheWord() { + type("བོད") + backspace() + assertEquals("བོ", composer.composing) + + backspace() + backspace() + backspace() + assertEquals("", composer.composing) + } + + @Test + fun noTsheg_whenTheWordAlreadyEndsWithOne() { + accept("བོང་བུ་") + assertEquals("བོང་བུ་", editor.text) + } + + @Test + fun noTsheg_whenOneFollowsTheCursor_orTheEditorCannotSay() { + assertEquals(WordComposer.Edit(0, "བོད"), WordComposer().onAccept("བོད", "", "", after = "་ཡུལ")) + assertEquals(WordComposer.Edit(0, "བོད"), WordComposer().onAccept("བོད", "", "", after = null)) + assertEquals(WordComposer.Edit(0, "བོད་"), WordComposer().onAccept("བོད", "", "", after = "")) + } + + @Test + fun beforeAShad_onlyNgaGetsATsheg() { + assertEquals(WordComposer.Edit(0, "བོད"), WordComposer().onAccept("བོད", "", "", after = "།")) + assertEquals(WordComposer.Edit(0, "རང་"), WordComposer().onAccept("རང", "", "", after = "།")) + } + + @Test + fun boundaryKeys_endTheRun_andReportItForStats() { + type("བོད་ཡིག") + + val edit = composer.onChar('།', editor.text) + + assertEquals("བོད་ཡིག", edit.finishedChunk) + assertEquals("", composer.composing) + for (c in listOf('།', '༎', '༔', ' ', '\n')) assertTrue("$c ends a run", WordComposer.endsRun(c)) + assertFalse("a tsheg joins syllables of one word", WordComposer.endsRun('་')) + } + + @Test + fun cursorMovedElsewhere_dropsTheWord_soATapCannotDeleteThere() { + type("བོ") + + // The text before the cursor no longer ends with what was typed. + assertTrue(composer.sync("ཁྱེད་རང")) + assertEquals("", composer.composing) + + // A tap on a stale chip inserts, but deletes nothing. + assertEquals(WordComposer.Edit(0, "བོད་"), composer.onAccept("བོད", "བོ", "ཁྱེད་རང", "")) + } + + @Test + fun ownEditsEchoedBack_keepTheWord() { + type("བོ") + + assertFalse(composer.sync("ཁྱེད་རང་བོ")) + + assertEquals("བོ", composer.composing) + } + + @Test + fun cursorMovedAfterAccept_makesTheTshegOrdinary() { + accept("བོད") + composer.sync("ཁྱེད་རང") + + val edit = composer.onChar('།', "ཁྱེད་རང") + + assertEquals(WordComposer.Edit(0, "།"), edit) + } + + @Test + fun editorThatCannotReportText_keepsNoState() { + assertEquals(WordComposer.Edit(0, "བ"), composer.onChar('བ', before = null)) + assertEquals("", composer.composing) + assertFalse(composer.hasState) + } + + @Test + fun lookback_coversTheStateBeingChecked() { + type("བོད་ཡི") + assertTrue(composer.lookback >= "བོད་ཡི".length) + + accept("བོད་ཡིག", matched = "བོད་ཡི") + assertTrue(composer.lookback >= "བོད་ཡིག་".length) + } + + @Test + fun endsWithNga_ignoresVowelSigns() { + assertTrue(WordComposer.endsWithNga("ང")) + assertTrue(WordComposer.endsWithNga("སྣང")) + assertTrue(WordComposer.endsWithNga("ངོ")) + assertTrue(WordComposer.endsWithNga("ལྔ")) + assertFalse(WordComposer.endsWithNga("བོད")) + assertFalse(WordComposer.endsWithNga("ངག")) + assertFalse(WordComposer.endsWithNga("")) + } +} diff --git a/botok/src/main/java/com/kharagedition/botok/autocomplete/SuggestionEngine.kt b/botok/src/main/java/com/kharagedition/botok/autocomplete/SuggestionEngine.kt index fccea69..70b9ae9 100644 --- a/botok/src/main/java/com/kharagedition/botok/autocomplete/SuggestionEngine.kt +++ b/botok/src/main/java/com/kharagedition/botok/autocomplete/SuggestionEngine.kt @@ -7,7 +7,7 @@ package com.kharagedition.botok.autocomplete * and stored in a sorted array for O(log N) prefix lookup via binary search. * * Call [addLines] for each TSV file, then [ready] once to sort and activate. - * After [ready], [getSuggestions] is thread-safe and allocation-light. + * After [ready], [suggest] is thread-safe and allocation-light. */ class SuggestionEngine { @@ -19,6 +19,11 @@ class SuggestionEngine { private data class WordEntry(val form: String, val freq: Int) + private companion object { + /** The syllable separator, and its no-break form (used before a shad after nga). */ + val TSHEGS = charArrayOf('་', '༌') + } + fun addLines(lines: Sequence) { for (line in lines) { val trimmed = line.trimStart('\uFEFF') @@ -38,20 +43,36 @@ class SuggestionEngine { isReady = true } - fun getSuggestions(prefix: String, max: Int = 4): List { - if (!isReady || prefix.isEmpty()) return emptyList() + /** + * Completions for a prefix, with [matched]: the part of the prefix the words complete — the + * whole prefix, or the tail of it (starting after a tsheg) that found them. Whoever inserts a + * chosen word must replace exactly [matched]; what precedes it is earlier words the user + * already typed (Tibetan puts no spaces between words). + */ + data class Suggestions(val words: List, val matched: String) { + companion object { + val NONE = Suggestions(emptyList(), "") + } + } - val direct = lookup(prefix, max) - if (direct.isNotEmpty()) return direct + /** + * Tries the whole prefix, then each tail that starts after a tsheg, longest first: a run has + * no word boundaries, so "ང་བོད་ར" is first read as one word, then as the compound in + * progress (བོད་ར → བོད་རིགས), then as a new word (ར → རང). + */ + fun suggest(prefix: String, max: Int = 4): Suggestions { + if (!isReady || prefix.isEmpty()) return Suggestions.NONE - // Fallback: when typing a new word after a tshek without a space, the prefix is - // "previousWord་newPartial". Strip back to the text after the last tshek and retry. - val lastTshek = prefix.lastIndexOf('་') - if (lastTshek >= 0) { - val tail = prefix.substring(lastTshek + 1) - if (tail.isNotEmpty()) return lookup(tail, max) + var start = 0 + while (start < prefix.length) { + val tail = prefix.substring(start) + val words = lookup(tail, max) + if (words.isNotEmpty()) return Suggestions(words, tail) + val tsheg = prefix.indexOfAny(TSHEGS, start) + if (tsheg < 0) break + start = tsheg + 1 } - return emptyList() + return Suggestions.NONE } /** True if [word] is an exact dictionary entry (used for greedy word-boundary segmentation). */ diff --git a/botok/src/test/java/com/kharagedition/botok/autocomplete/SuggestionEngineTest.kt b/botok/src/test/java/com/kharagedition/botok/autocomplete/SuggestionEngineTest.kt new file mode 100644 index 0000000..ee78f2e --- /dev/null +++ b/botok/src/test/java/com/kharagedition/botok/autocomplete/SuggestionEngineTest.kt @@ -0,0 +1,97 @@ +package com.kharagedition.botok.autocomplete + +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * Prefix lookup on a small in-memory dictionary, and above all `matched`: the part of the prefix + * a suggestion completes, which is what the keyboard replaces when one is tapped. + */ +class SuggestionEngineTest { + + private val engine = SuggestionEngine().apply { + addLines( + """ + # form pos lemma sense freq + བོད NOUN བོད 900 + བོད་ཡུལ NOUN བོད་ཡུལ 300 + བོད་རིགས NOUN བོད་རིགས 200 + བོགས NOUN བོགས 100 + རང PRON རང 800 + རགས་པ ADJ རགས་པ 50 + པད་མ NOUN པད་མ 400 + """.trimIndent().lineSequence() + ) + ready() + } + + @Test + fun directHit_completesTheWholePrefix() { + val s = engine.suggest("བོ") + + assertEquals(listOf("བོད", "བོད་ཡུལ", "བོད་རིགས", "བོགས"), s.words) + assertEquals("བོ", s.matched) + } + + @Test + fun multiSyllablePrefix_isMatchedWhole() { + val s = engine.suggest("བོད་ར") + + assertEquals(listOf("བོད་རིགས"), s.words) + assertEquals("བོད་ར", s.matched) + } + + @Test + fun fallback_matchesTheLastSyllable_whenNothingLongerDoes() { + // Nothing starts with ང་བོད་པ or བོད་པ: the words complete just the པ. + val s = engine.suggest("ང་བོད་པ") + + assertEquals(listOf("པད་མ"), s.words) + assertEquals("པ", s.matched) + } + + @Test + fun compoundInProgress_isFoundAfterEarlierWords() { + // ང་བོད་ར is not a word, but the run's tail བོད་ར is a compound being typed. + val s = engine.suggest("ང་བོད་ར") + + assertEquals(listOf("བོད་རིགས"), s.words) + assertEquals("བོད་ར", s.matched) + } + + @Test + fun fallback_alsoSplitsAtTheNoBreakTsheg() { + val s = engine.suggest("བོད༌ར") + + assertEquals(listOf("རང", "རགས་པ"), s.words) + assertEquals("ར", s.matched) + } + + @Test + fun aTshegAfterAWord_offersItsContinuations() { + val s = engine.suggest("བོད་") + + assertEquals(listOf("བོད་ཡུལ", "བོད་རིགས"), s.words) + assertEquals("བོད་", s.matched) + } + + @Test + fun noMatch_matchesNothing() { + assertEquals(SuggestionEngine.Suggestions.NONE, engine.suggest("ཀ")) + // Nothing continues བོགས, and there is no syllable after its tsheg to fall back to. + assertEquals(SuggestionEngine.Suggestions.NONE, engine.suggest("བོགས་")) + assertEquals(SuggestionEngine.Suggestions.NONE, engine.suggest("")) + } + + @Test + fun resultsAreCapped_byFrequency() { + assertEquals(listOf("བོད", "བོད་ཡུལ"), engine.suggest("བོ", max = 2).words) + } + + @Test + fun notReady_suggestsNothing() { + val cold = SuggestionEngine().apply { addLines(sequenceOf("བོད\tNOUN\tབོད\t\t1")) } + + assertEquals(SuggestionEngine.Suggestions.NONE, cold.suggest("བོ")) + } +}