diff --git a/apps/common-app/orchestra-server/.gitignore b/apps/common-app/orchestra-server/.gitignore new file mode 100644 index 000000000..dbcba36d8 --- /dev/null +++ b/apps/common-app/orchestra-server/.gitignore @@ -0,0 +1,4 @@ +samples/* +!samples/.gitkeep +midi/* +!midi/.gitkeep diff --git a/apps/common-app/orchestra-server/README.md b/apps/common-app/orchestra-server/README.md new file mode 100644 index 000000000..156942cc2 --- /dev/null +++ b/apps/common-app/orchestra-server/README.md @@ -0,0 +1,97 @@ +# Phone orchestra server + +Companion server for the **Orchestra** demo (`src/demos/Orchestra/`). Phones +connect over WebSocket, estimate the server clock (NTP-style ping/pong) and +schedule their part of the score with `start(when)`, so a room full of phones +plays one piece together. + +```sh +node server.ts # Node 22.18+ (type stripping), no dependencies +PORT=9000 node server.ts # custom port (default 8765) +``` + +The server prints its LAN addresses: open the `http://…` one in a browser for +the dashboard (start/stop, tempo, score, per-phone part assignment, +sync quality). Phones join by pointing their camera at the dashboard's QR +code: it encodes `audioapi-example://orchestra?url=ws://:8765`, which +opens the app on the Orchestra screen and connects. The `ws://…` address can +also be typed in by hand. + +The QR code is rendered by `qrcodegen.ts`, vendored from +[nayuki/QR-Code-generator](https://github.com/nayuki/QR-Code-generator) (MIT) +and adapted to the erasable-syntax subset Node's type stripping accepts; its +output was checked identical to the upstream build for inputs spanning +versions 1–8. + +Phones and the laptop must share a network. Conference Wi-Fi is the usual +failure point — bring a router, and watch the RTT column on the dashboard. + +Scores live in `score.ts`; the wire format in `../src/demos/Orchestra/protocol.ts`. + +## Recorded instruments + +Parts with an `instrument` play recorded notes instead of the synth: every +General MIDI instrument of the FluidR3_GM soundfont, one mp3 per semitone +(~22 KB), or `drums`, an acoustic kit (kick, snare, clap, closed and open +hi-hat, two toms, crash, ride; a few takes each, picked at random) for parts +whose notes are General MIDI percussion keys. The kit's key mapping lives in +`drumKit.ts`. At startup the server downloads every note the scores use into +`samples/` (kept out of git) and serves it at `/samples//.mp3`, +so phones need only the local network; notes nobody asked for yet are +downloaded on first request. Run the server once with internet access before +a show. Without it, phones fall back to the synth voices. + +Phones load the notes of their own part when it is dealt, and play the synth +for any note that has not loaded yet. The phone's Voice row can switch any +part to piano, strings, flute, trumpet, harp or marimba, or back to the synth. + +Sample credits: FluidR3_GM by Frank Wen (MIT), rendered to per-note mp3 by +Benjamin Gleitzman in [midi-js-soundfonts](https://github.com/gleitz/midi-js-soundfonts) +(CC BY 3.0). Drums from the [Versilian Community Sample Library](https://github.com/sgossner/VCSL) +by Versilian Studios (CC0). + +## Importing MIDI files + +Drop Standard MIDI Files (`.mid`, format 0 or 1) into `midi/` and restart the +server (or run `node --watch server.ts`). Each file becomes a score named after +the file. Only import music you have the right to use. + +At startup the server prints every imported part, for example: + +``` +MIDI My Song.mid → "My Song" (midi-my-song), 120 bpm, 128 beats + part "Lead": 212 notes, midi 64-84, pluck + part "Bass": 160 notes, midi 33-45, transposed +24, pluck + part "Drums (ch 10)": 410 notes, midi 36-51, kit +``` + +- Each track (or each channel of a multi-channel track) becomes one part. +- Each part plays the recorded instrument its General MIDI program selects + (piano when the file sets none). +- Channel 10 becomes a `kit` part on the recorded drum kit; keys the kit has + no piece for (cowbell, congas, …) are dropped. +- Parts lower than midi 55 move up whole octaves so phones can play them. +- Only the first tempo is used; tempo changes are reported as a warning. + +To choose voices and how many phones play each part, add `My Song.json` next to +`My Song.mid`, using the part names the server printed: + +```json +{ + "title": "My Song", + "bpm": 118, + "parts": { + "Lead": { "instrument": "violin", "maxPlayers": 1 }, + "Bass": { "timbre": "pluck", "transpose": 12 }, + "Strings": { "skip": true } + } +} +``` + +`instrument` takes any name from +[the FluidR3_GM list](https://gleitz.github.io/midi-js-soundfonts/FluidR3_GM/names.json) +and replaces the one the file selects. Timbres (the synth fallback): `bell`, +`pluck`, `pad`, `lead`, `hat`, `clap`, `kit`, `click`. +`maxPlayers` caps how many phones are dealt the part automatically (the rest +fill the uncapped parts). `transpose` (semitones) replaces the automatic octave +shift. Files that fail to parse are reported and skipped. diff --git a/apps/common-app/orchestra-server/dashboard.html b/apps/common-app/orchestra-server/dashboard.html new file mode 100644 index 000000000..bf459b5e4 --- /dev/null +++ b/apps/common-app/orchestra-server/dashboard.html @@ -0,0 +1,412 @@ + + + + + +Phone Orchestra + + + +
+
+

Phone Orchestra

+ connecting +
+ +
+
+
+ stopped + +
+
+ + + + + +
+
+ +
+ Join QR code +
+

Join

+ +
Scan with the phone camera, or type the address into the app.
+
+
+
+ +
+
+

Phones

+ +
+ + + + + + + + + + + + + +
PhonePartsRTTSyncLatencySeen
+
+
+ + + + diff --git a/apps/common-app/orchestra-server/drumKit.ts b/apps/common-app/orchestra-server/drumKit.ts new file mode 100644 index 000000000..7e956360f --- /dev/null +++ b/apps/common-app/orchestra-server/drumKit.ts @@ -0,0 +1,55 @@ +// The recorded drum kit: General MIDI percussion keys (channel 10) mapped to +// the kit pieces this server mirrors from VCSL. The phones import it too, so +// it stays free of imports and of TypeScript that Node cannot erase; it lives +// here because only this package is an ES module for Node. + +/** `ScorePart.instrument` of a part played on the recorded kit. */ +export const DRUMS = 'drums'; + +export type DrumPiece = + | 'kick' + | 'snare' + | 'clap' + | 'closedHat' + | 'openHat' + | 'lowTom' + | 'highTom' + | 'crash' + | 'ride'; + +interface DrumPieceInfo { + label: string; + /** General MIDI percussion keys this piece plays. */ + keys: readonly number[]; + /** Recorded takes, played in random order so repeated hits differ. */ + takes: number; + /** Cymbals fall back to the synth hat, skins to the synth clap. */ + metal: boolean; +} + +export const DRUM_PIECES: Record = { + kick: { label: 'kick', keys: [35, 36], takes: 2, metal: false }, + snare: { label: 'snare', keys: [37, 38, 40], takes: 2, metal: false }, + clap: { label: 'clap', keys: [39], takes: 3, metal: false }, + closedHat: { label: 'hi-hat', keys: [42, 44], takes: 2, metal: true }, + openHat: { label: 'open hat', keys: [46], takes: 2, metal: true }, + lowTom: { label: 'low tom', keys: [41, 43, 45], takes: 2, metal: false }, + highTom: { label: 'high tom', keys: [47, 48, 50], takes: 2, metal: false }, + crash: { label: 'crash', keys: [49, 52, 55, 57], takes: 2, metal: true }, + ride: { label: 'ride', keys: [51, 53, 59], takes: 1, metal: true }, +}; + +const PIECE_BY_KEY = new Map(); +for (const [piece, info] of Object.entries(DRUM_PIECES)) { + for (const key of info.keys) { + PIECE_BY_KEY.set(key, piece as DrumPiece); + } +} + +/** + * The kit piece a General MIDI percussion key plays; null when the kit has + * none. + */ +export function drumPieceFor(key: number): DrumPiece | null { + return PIECE_BY_KEY.get(key) ?? null; +} diff --git a/apps/common-app/orchestra-server/midi.ts b/apps/common-app/orchestra-server/midi.ts new file mode 100644 index 000000000..bc436932b --- /dev/null +++ b/apps/common-app/orchestra-server/midi.ts @@ -0,0 +1,596 @@ +// Standard MIDI File (SMF) import: turns a .mid file into an orchestra score. +// Dependency-free. Supports format 0 and 1 with ticks-per-quarter timing. +// +// Each (track, channel) pair with notes becomes one part. Channel 10 (drums) +// becomes a 'kit' part on the recorded drum kit. Parts too low for phone speakers move up whole +// octaves. An optional JSON config next to the file overrides the title, +// tempo and per-part timbre / phone cap / transposition. +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { basename, extname, join } from 'node:path'; + +import { DRUMS, drumPieceFor } from './drumKit.ts'; + +import type { + Score, + ScoreNote, + ScorePart, + Timbre, +} from '../src/demos/Orchestra/protocol.ts'; + +export interface MidiNote { + startTick: number; + endTick: number; + key: number; + velocity: number; +} + +export interface MidiTrack { + name: string | null; + /** Notes per MIDI channel (0-15). */ + notesByChannel: Map; + /** The first General MIDI program each channel selects in this track. */ + programByChannel: Map; +} + +export interface MidiFile { + format: number; + ticksPerQuarter: number; + /** Microseconds per quarter note from the first Set Tempo event, if any. */ + tempo: number | null; + /** Numerator of the first Time Signature event, if any. */ + beatsPerBar: number | null; + /** Number of Set Tempo events; only the first is honoured. */ + tempoChanges: number; + /** + * The first program per channel anywhere in the file, for tracks that set + * none. + */ + programByChannel: Map; + tracks: MidiTrack[]; +} + +class ByteReader { + private readonly bytes: Uint8Array; + position: number; + readonly end: number; + + constructor(bytes: Uint8Array, start = 0, end = bytes.length) { + this.bytes = bytes; + this.position = start; + this.end = end; + } + + get done(): boolean { + return this.position >= this.end; + } + + u8(): number { + if (this.position >= this.end) { + throw new Error('Unexpected end of MIDI data'); + } + return this.bytes[this.position++]; + } + + u16(): number { + return (this.u8() << 8) | this.u8(); + } + + u32(): number { + return ( + ((this.u8() << 24) >>> 0) + + (this.u8() << 16) + + (this.u8() << 8) + + this.u8() + ); + } + + ascii(length: number): string { + let text = ''; + for (let i = 0; i < length; i += 1) { + text += String.fromCharCode(this.u8()); + } + return text; + } + + /** Variable-length quantity: 7 bits per byte, high bit means "more". */ + vlq(): number { + let value = 0; + for (let i = 0; i < 4; i += 1) { + const byte = this.u8(); + value = (value << 7) | (byte & 0x7f); + if ((byte & 0x80) === 0) { + return value; + } + } + throw new Error('Variable-length quantity longer than 4 bytes'); + } + + skip(length: number): void { + if (this.position + length > this.end) { + throw new Error('Unexpected end of MIDI data'); + } + this.position += length; + } +} + +export function parseMidi(bytes: Uint8Array): MidiFile { + const reader = new ByteReader(bytes); + if (reader.ascii(4) !== 'MThd') { + throw new Error('Not a MIDI file (missing MThd header)'); + } + const headerLength = reader.u32(); + const headerEnd = reader.position + headerLength; + const format = reader.u16(); + const trackCount = reader.u16(); + const division = reader.u16(); + reader.position = headerEnd; + + if (format > 1) { + throw new Error(`MIDI format ${format} is not supported (only 0 and 1)`); + } + if (division & 0x8000) { + throw new Error('SMPTE time division is not supported'); + } + + const midi: MidiFile = { + format, + ticksPerQuarter: division, + tempo: null, + beatsPerBar: null, + tempoChanges: 0, + programByChannel: new Map(), + tracks: [], + }; + + for (let index = 0; index < trackCount && !reader.done; index += 1) { + const chunkType = reader.ascii(4); + const chunkLength = reader.u32(); + const chunkEnd = reader.position + chunkLength; + if (chunkType === 'MTrk') { + midi.tracks.push( + parseTrack(new ByteReader(bytes, reader.position, chunkEnd), midi) + ); + } else { + index -= 1; // Unknown chunks do not count as tracks. + } + reader.position = chunkEnd; + } + + return midi; +} + +function parseTrack(reader: ByteReader, midi: MidiFile): MidiTrack { + const track: MidiTrack = { + name: null, + notesByChannel: new Map(), + programByChannel: new Map(), + }; + // Sounding notes per channel+key; a FIFO so overlapping repeats pair up in order. + const open = new Map(); + let tick = 0; + let runningStatus = 0; + + const noteOff = (channel: number, key: number) => { + const started = open.get(channel * 128 + key)?.shift(); + if (!started) { + return; + } + const notes = track.notesByChannel.get(channel) ?? []; + notes.push({ + startTick: started.startTick, + endTick: tick, + key, + velocity: started.velocity, + }); + track.notesByChannel.set(channel, notes); + }; + + while (!reader.done) { + tick += reader.vlq(); + let status = reader.u8(); + + if (status === 0xff) { + const type = reader.u8(); + const length = reader.vlq(); + if (type === 0x2f) { + break; // End of track. + } else if (type === 0x03 && track.name === null) { + track.name = reader.ascii(length).trim() || null; + } else if (type === 0x51 && length === 3) { + const tempo = (reader.u8() << 16) | (reader.u8() << 8) | reader.u8(); + midi.tempoChanges += 1; + midi.tempo ??= tempo; + } else if (type === 0x58 && length >= 1) { + const numerator = reader.u8(); + midi.beatsPerBar ??= numerator; + reader.skip(length - 1); + } else { + reader.skip(length); + } + continue; + } + + if (status === 0xf0 || status === 0xf7) { + reader.skip(reader.vlq()); // SysEx. + continue; + } + + if (status < 0x80) { + // Running status: this byte is already the first data byte. + if (runningStatus === 0) { + throw new Error('Running status used before any channel message'); + } + reader.position -= 1; + status = runningStatus; + } else { + runningStatus = status; + } + + const kind = status & 0xf0; + const channel = status & 0x0f; + if (kind === 0x90 || kind === 0x80) { + const key = reader.u8(); + const velocity = reader.u8(); + if (kind === 0x90 && velocity > 0) { + const slot = channel * 128 + key; + const sounding = open.get(slot) ?? []; + sounding.push({ startTick: tick, velocity }); + open.set(slot, sounding); + } else { + noteOff(channel, key); + } + } else if (kind === 0xc0) { + const program = reader.u8(); + if (!track.programByChannel.has(channel)) { + track.programByChannel.set(channel, program); + } + if (!midi.programByChannel.has(channel)) { + midi.programByChannel.set(channel, program); + } + } else if (kind === 0xd0) { + reader.skip(1); + } else { + reader.skip(2); // Aftertouch, control change, pitch bend. + } + } + + // Notes never released end with the track. + for (const [slot, sounding] of open) { + for (let i = sounding.length; i > 0; i -= 1) { + noteOff(Math.floor(slot / 128), slot % 128); + } + } + return track; +} + +// ---- MIDI → orchestra score ------------------------------------------------- + +export interface MidiPartConfig { + timbre?: Timbre; + /** A FluidR3_GM instrument name; replaces the one the file's program selects. */ + instrument?: string; + maxPlayers?: number; + /** Semitones; replaces the automatic octave shift. */ + transpose?: number; + skip?: boolean; + color?: string; +} + +export interface MidiScoreConfig { + title?: string; + /** Overrides the file's tempo. */ + bpm?: number; + /** Keyed by the part names the server prints at startup. */ + parts?: Record; +} + +export const TIMBRES: readonly Timbre[] = [ + 'bell', + 'pluck', + 'pad', + 'lead', + 'hat', + 'clap', + 'kit', + 'click', +]; + +const DRUM_CHANNEL = 9; +const LOWEST_PHONE_MIDI = 55; +const HIGHEST_PHONE_MIDI = 96; +const PART_COLORS = [ + '#38ACDD', + '#FFD61E', + '#FF7774', + '#8BE38B', + '#7B5CFF', + '#FF9F1C', + '#F4F1DE', + '#E8443A', +]; + +export interface ImportedPart { + name: string; + noteCount: number; + lowest: number; + highest: number; + transpose: number; + timbre: Timbre; + instrument?: string; + maxPlayers?: number; + skipped: boolean; +} + +export interface MidiImport { + score: Score; + parts: ImportedPart[]; + warnings: string[]; +} + +/** + * `instrumentNames` lists the FluidR3_GM instruments by General MIDI program; + * when it is empty, parts get no recorded instrument and phones use the synth. + */ +export function midiToScore( + id: string, + midi: MidiFile, + config: MidiScoreConfig = {}, + instrumentNames: readonly string[] = [] +): MidiImport { + const warnings: string[] = []; + const quarterTicks = midi.ticksPerQuarter; + const fileBpm = midi.tempo ? 60_000_000 / midi.tempo : 120; + if (midi.tempoChanges > 1) { + warnings.push( + `${midi.tempoChanges} tempo changes; only the first (${fileBpm.toFixed(1)} bpm) is used` + ); + } + + const rawParts = collectRawParts(midi); + const knownNames = new Set(rawParts.map((part) => part.name)); + for (const name of Object.keys(config.parts ?? {})) { + if (!knownNames.has(name)) { + warnings.push( + `config names part "${name}", which the file does not have` + ); + } + } + + const parts: ScorePart[] = []; + const imported: ImportedPart[] = []; + let lastBeat = 0; + + rawParts.forEach((raw, index) => { + const partConfig = config.parts?.[raw.name] ?? {}; + const isDrums = raw.channel === DRUM_CHANNEL; + const timbre = + validTimbre(partConfig.timbre, warnings, raw.name) ?? + (isDrums ? 'kit' : 'pluck'); + const instrument = + partConfig.instrument ?? (isDrums ? DRUMS : instrumentNames[raw.program]); + + const keys = raw.notes.map((note) => note.key); + const lowest = Math.min(...keys); + const highest = Math.max(...keys); + const transpose = isDrums + ? 0 + : (partConfig.transpose ?? phoneTranspose(lowest, highest)); + + const notes: ScoreNote[] = []; + for (const note of raw.notes) { + const midiKey = isDrums ? drumKey(note.key) : note.key + transpose; + if (midiKey === null) { + continue; + } + const beat = note.startTick / quarterTicks; + const duration = Math.max( + 0.05, + (note.endTick - note.startTick) / quarterTicks + ); + lastBeat = Math.max(lastBeat, beat + duration); + notes.push({ + beat, + midi: midiKey, + duration, + velocity: note.velocity / 127, + }); + } + + imported.push({ + name: raw.name, + noteCount: notes.length, + lowest, + highest, + transpose, + timbre, + instrument, + maxPlayers: partConfig.maxPlayers, + skipped: Boolean(partConfig.skip) || notes.length === 0, + }); + if (partConfig.skip || notes.length === 0) { + return; + } + + parts.push({ + name: raw.name, + color: partConfig.color ?? PART_COLORS[index % PART_COLORS.length], + timbre, + ...(instrument ? { instrument } : {}), + notes, + ...(partConfig.maxPlayers ? { maxPlayers: partConfig.maxPlayers } : {}), + }); + }); + + if (parts.length === 0) { + throw new Error('no playable notes'); + } + + const beatsPerBar = midi.beatsPerBar ?? 4; + return { + score: { + id, + title: config.title ?? id, + bpm: Math.round((config.bpm ?? fileBpm) * 10) / 10, + lengthBeats: Math.ceil(lastBeat / beatsPerBar) * beatsPerBar, + parts, + }, + parts: imported, + warnings, + }; +} + +interface RawPart { + name: string; + channel: number; + /** General MIDI program, 0 (piano) when the file never sets one. */ + program: number; + notes: MidiNote[]; +} + +function collectRawParts(midi: MidiFile): RawPart[] { + const parts: RawPart[] = []; + midi.tracks.forEach((track, trackIndex) => { + const channels = [...track.notesByChannel.keys()].sort((a, b) => a - b); + for (const channel of channels) { + const base = track.name ?? `Track ${trackIndex + 1}`; + const name = + channels.length > 1 || channel === DRUM_CHANNEL + ? `${base} (ch ${channel + 1})` + : base; + const notes = [...track.notesByChannel.get(channel)!].sort( + (a, b) => a.startTick - b.startTick + ); + const program = + track.programByChannel.get(channel) ?? + midi.programByChannel.get(channel) ?? + 0; + parts.push({ name: uniqueName(name, parts), channel, program, notes }); + } + }); + // Phones are dealt parts in order, so the parts that open the song come + // first; among parts entering together, the busier one wins. + return parts.sort( + (a, b) => + a.notes[0].startTick - b.notes[0].startTick || + b.notes.length - a.notes.length + ); +} + +function uniqueName(name: string, existing: RawPart[]): string { + let candidate = name; + for (let n = 2; existing.some((part) => part.name === candidate); n += 1) { + candidate = `${name} #${n}`; + } + return candidate; +} + +/** + * Whole octaves up until the part clears the phone floor, without passing the + * ceiling. + */ +function phoneTranspose(lowest: number, highest: number): number { + let shift = 0; + while ( + lowest + shift < LOWEST_PHONE_MIDI && + highest + shift + 12 <= HIGHEST_PHONE_MIDI + ) { + shift += 12; + } + return shift; +} + +// Drum keys stay General MIDI keys; ones the kit has no piece for are dropped. +function drumKey(key: number): number | null { + return drumPieceFor(key) ? key : null; +} + +function validTimbre( + timbre: string | undefined, + warnings: string[], + partName: string +): Timbre | undefined { + if (timbre === undefined) { + return undefined; + } + if ((TIMBRES as readonly string[]).includes(timbre)) { + return timbre as Timbre; + } + warnings.push( + `part "${partName}": unknown timbre "${timbre}" (use ${TIMBRES.join(', ')})` + ); + return undefined; +} + +// ---- Loading a folder of .mid files ----------------------------------------- + +/** + * Imports every `.mid` in `directory` as score `midi-`, with + * `.json` as its optional config. A broken file is reported and skipped. + */ +export function loadMidiFolder( + directory: string, + instrumentNames: readonly string[] = [] +): Record { + if (!existsSync(directory)) { + return {}; + } + + const scores: Record = {}; + for (const file of readdirSync(directory).sort()) { + const extension = extname(file).toLowerCase(); + if (extension !== '.mid' && extension !== '.midi') { + continue; + } + const name = basename(file, extname(file)); + const slug = name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); + const id = `midi-${slug}`; + try { + const configPath = join(directory, `${name}.json`); + const config = existsSync(configPath) + ? (JSON.parse(readFileSync(configPath, 'utf8')) as MidiScoreConfig) + : {}; + const midi = parseMidi( + new Uint8Array(readFileSync(join(directory, file))) + ); + const result = midiToScore( + id, + midi, + { title: name, ...config }, + instrumentNames + ); + scores[id] = result.score; + printImport(file, result); + } catch (error) { + console.warn(`MIDI ${file}: skipped — ${(error as Error).message}`); + } + } + return scores; +} + +function printImport( + file: string, + { score, parts, warnings }: MidiImport +): void { + console.log( + `MIDI ${file} → "${score.title}" (${score.id}), ${score.bpm} bpm, ${score.lengthBeats} beats` + ); + for (const part of parts) { + const details = [ + `${part.noteCount} notes`, + `midi ${part.lowest}-${part.highest}`, + part.transpose + ? `transposed ${part.transpose > 0 ? '+' : ''}${part.transpose}` + : null, + part.instrument ?? part.timbre, + part.maxPlayers ? `max ${part.maxPlayers} phone(s)` : null, + part.skipped ? 'SKIPPED' : null, + ].filter(Boolean); + console.log(` part "${part.name}": ${details.join(', ')}`); + } + for (const warning of warnings) { + console.warn(` warning: ${warning}`); + } +} diff --git a/apps/common-app/orchestra-server/midi/.gitkeep b/apps/common-app/orchestra-server/midi/.gitkeep new file mode 100644 index 000000000..4f1fbade0 --- /dev/null +++ b/apps/common-app/orchestra-server/midi/.gitkeep @@ -0,0 +1 @@ +Drop Standard MIDI Files here to add scores; the contents stay out of git. diff --git a/apps/common-app/orchestra-server/package.json b/apps/common-app/orchestra-server/package.json new file mode 100644 index 000000000..1f53549e0 --- /dev/null +++ b/apps/common-app/orchestra-server/package.json @@ -0,0 +1,9 @@ +{ + "name": "orchestra-server", + "private": true, + "type": "module", + "scripts": { + "start": "node server.ts", + "typecheck": "tsc --noEmit -p ." + } +} diff --git a/apps/common-app/orchestra-server/qr.ts b/apps/common-app/orchestra-server/qr.ts new file mode 100644 index 000000000..5ddfa3dca --- /dev/null +++ b/apps/common-app/orchestra-server/qr.ts @@ -0,0 +1,27 @@ +// Renders a QR code as a standalone SVG, for the dashboard's join code. +import { Ecc, QrCode } from './qrcodegen.ts'; + +const QUIET_ZONE_MODULES = 4; + +export function qrSvg(text: string): string { + const qr = QrCode.encodeText(text, Ecc.MEDIUM); + const span = qr.size + QUIET_ZONE_MODULES * 2; + + const squares: string[] = []; + for (let y = 0; y < qr.size; y += 1) { + for (let x = 0; x < qr.size; x += 1) { + if (qr.getModule(x, y)) { + squares.push( + `M${x + QUIET_ZONE_MODULES},${y + QUIET_ZONE_MODULES}h1v1h-1z` + ); + } + } + } + + return ( + `` + + `` + + `` + + '' + ); +} diff --git a/apps/common-app/orchestra-server/qrcodegen.ts b/apps/common-app/orchestra-server/qrcodegen.ts new file mode 100644 index 000000000..340f1eae6 --- /dev/null +++ b/apps/common-app/orchestra-server/qrcodegen.ts @@ -0,0 +1,1037 @@ +/* + * QR Code generator library (TypeScript) + * + * Copyright (c) Project Nayuki. (MIT License) + * https://www.nayuki.io/page/qr-code-generator-library + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * - The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * - The Software is provided "as is", without warranty of any kind, express or + * implied, including but not limited to the warranties of merchantability, + * fitness for a particular purpose and noninfringement. In no event shall the + * authors or copyright holders be liable for any claim, damages or other + * liability, whether in an action of contract, tort or otherwise, arising from, + * out of or in connection with the Software or the use or other dealings in the + * Software. + */ + +// Vendored from https://github.com/nayuki/QR-Code-generator (typescript-javascript/ +// qrcodegen.ts) and adapted for Node's type stripping: the `qrcodegen` namespaces +// are flattened into module exports and parameter properties expanded into +// explicit fields. Behaviour is unchanged (output verified identical to upstream). + +type bit = number; +type byte = number; +type int = number; + +/*---- QR Code symbol class ----*/ + +/* + * A QR Code symbol, which is a type of two-dimension barcode. + * Invented by Denso Wave and described in the ISO/IEC 18004 standard. + * Instances of this class represent an immutable square grid of dark and light cells. + * The class provides static factory functions to create a QR Code from text or binary data. + * The class covers the QR Code Model 2 specification, supporting all versions (sizes) + * from 1 to 40, all 4 error correction levels, and 4 character encoding modes. + * + * Ways to create a QR Code object: + * - High level: Take the payload data and call QrCode.encodeText() or QrCode.encodeBinary(). + * - Mid level: Custom-make the list of segments and call QrCode.encodeSegments(). + * - Low level: Custom-make the array of data codeword bytes (including + * segment headers and final padding, excluding error correction codewords), + * supply the appropriate version number, and call the QrCode() constructor. + * (Note that all ways require supplying the desired error correction level.) + */ +export class QrCode { + /*-- Static factory functions (high level) --*/ + + // Returns a QR Code representing the given Unicode text string at the given error correction level. + // As a conservative upper bound, this function is guaranteed to succeed for strings that have 738 or fewer + // Unicode code points (not UTF-16 code units) if the low error correction level is used. The smallest possible + // QR Code version is automatically chosen for the output. The ECC level of the result may be higher than the + // ecl argument if it can be done without increasing the version. + public static encodeText(text: string, ecl: Ecc): QrCode { + const segs: Array = QrSegment.makeSegments(text); + return QrCode.encodeSegments(segs, ecl); + } + + // Returns a QR Code representing the given binary data at the given error correction level. + // This function always encodes using the binary segment mode, not any text mode. The maximum number of + // bytes allowed is 2953. The smallest possible QR Code version is automatically chosen for the output. + // The ECC level of the result may be higher than the ecl argument if it can be done without increasing the version. + public static encodeBinary(data: Readonly>, ecl: Ecc): QrCode { + const seg: QrSegment = QrSegment.makeBytes(data); + return QrCode.encodeSegments([seg], ecl); + } + + /*-- Static factory functions (mid level) --*/ + + // Returns a QR Code representing the given segments with the given encoding parameters. + // The smallest possible QR Code version within the given range is automatically + // chosen for the output. Iff boostEcl is true, then the ECC level of the result + // may be higher than the ecl argument if it can be done without increasing the + // version. The mask number is either between 0 to 7 (inclusive) to force that + // mask, or -1 to automatically choose an appropriate mask (which may be slow). + // This function allows the user to create a custom sequence of segments that switches + // between modes (such as alphanumeric and byte) to encode text in less space. + // This is a mid-level API; the high-level API is encodeText() and encodeBinary(). + public static encodeSegments( + segs: Readonly>, + ecl: Ecc, + minVersion: int = 1, + maxVersion: int = 40, + mask: int = -1, + boostEcl: boolean = true + ): QrCode { + if ( + !( + QrCode.MIN_VERSION <= minVersion && + minVersion <= maxVersion && + maxVersion <= QrCode.MAX_VERSION + ) || + mask < -1 || + mask > 7 + ) + throw new RangeError('Invalid value'); + + // Find the minimal version number to use + let version: int; + let dataUsedBits: int; + for (version = minVersion; ; version++) { + const dataCapacityBits: int = + QrCode.getNumDataCodewords(version, ecl) * 8; // Number of data bits available + const usedBits: number = QrSegment.getTotalBits(segs, version); + if (usedBits <= dataCapacityBits) { + dataUsedBits = usedBits; + break; // This version number is found to be suitable + } + if (version >= maxVersion) + // All versions in the range could not fit the given data + throw new RangeError('Data too long'); + } + + // Increase the error correction level while the data still fits in the current version number + for (const newEcl of [Ecc.MEDIUM, Ecc.QUARTILE, Ecc.HIGH]) { + // From low to high + if ( + boostEcl && + dataUsedBits <= QrCode.getNumDataCodewords(version, newEcl) * 8 + ) + ecl = newEcl; + } + + // Concatenate all segments to create the data bit string + let bb: Array = []; + for (const seg of segs) { + appendBits(seg.mode.modeBits, 4, bb); + appendBits(seg.numChars, seg.mode.numCharCountBits(version), bb); + for (const b of seg.getData()) bb.push(b); + } + assert(bb.length == dataUsedBits); + + // Add terminator and pad up to a byte if applicable + const dataCapacityBits: int = QrCode.getNumDataCodewords(version, ecl) * 8; + assert(bb.length <= dataCapacityBits); + appendBits(0, Math.min(4, dataCapacityBits - bb.length), bb); + appendBits(0, (8 - (bb.length % 8)) % 8, bb); + assert(bb.length % 8 == 0); + + // Pad with alternating bytes until data capacity is reached + for ( + let padByte = 0xec; + bb.length < dataCapacityBits; + padByte ^= 0xec ^ 0x11 + ) + appendBits(padByte, 8, bb); + + // Pack bits into bytes in big endian + let dataCodewords: Array = []; + while (dataCodewords.length * 8 < bb.length) dataCodewords.push(0); + bb.forEach( + (b: bit, i: int) => (dataCodewords[i >>> 3] |= b << (7 - (i & 7))) + ); + + // Create the QR Code object + return new QrCode(version, ecl, dataCodewords, mask); + } + + /*-- Fields --*/ + + // The width and height of this QR Code, measured in modules, between + // 21 and 177 (inclusive). This is equal to version * 4 + 17. + public readonly size: int; + + // The index of the mask pattern used in this QR Code, which is between 0 and 7 (inclusive). + // Even if a QR Code is created with automatic masking requested (mask = -1), + // the resulting object still has a mask value between 0 and 7. + public readonly mask: int; + + // The modules of this QR Code (false = light, true = dark). + // Immutable after constructor finishes. Accessed through getModule(). + private readonly modules: Array> = []; + + // Indicates function modules that are not subjected to masking. Discarded when constructor finishes. + private readonly isFunction: Array> = []; + + /*-- Constructor (low level) and fields --*/ + + // Creates a new QR Code with the given version number, + // error correction level, data codeword bytes, and mask number. + // This is a low-level API that most users should not use directly. + // A mid-level API is the encodeSegments() function. + // The version number of this QR Code, which is between 1 and 40 (inclusive). + // This determines the size of this barcode. + public readonly version: int; + + // The error correction level used in this QR Code. + public readonly errorCorrectionLevel: Ecc; + + public constructor( + version: int, + errorCorrectionLevel: Ecc, + dataCodewords: Readonly>, + msk: int + ) { + this.version = version; + this.errorCorrectionLevel = errorCorrectionLevel; + + // Check scalar arguments + if (version < QrCode.MIN_VERSION || version > QrCode.MAX_VERSION) + throw new RangeError('Version value out of range'); + if (msk < -1 || msk > 7) throw new RangeError('Mask value out of range'); + this.size = version * 4 + 17; + + // Initialize both grids to be size*size arrays of Boolean false + let row: Array = []; + for (let i = 0; i < this.size; i++) row.push(false); + for (let i = 0; i < this.size; i++) { + this.modules.push(row.slice()); // Initially all light + this.isFunction.push(row.slice()); + } + + // Compute ECC, draw modules + this.drawFunctionPatterns(); + const allCodewords: Array = this.addEccAndInterleave(dataCodewords); + this.drawCodewords(allCodewords); + + // Do masking + if (msk == -1) { + // Automatically choose best mask + let minPenalty: int = 1000000000; + for (let i = 0; i < 8; i++) { + this.applyMask(i); + this.drawFormatBits(i); + const penalty: int = this.getPenaltyScore(); + if (penalty < minPenalty) { + msk = i; + minPenalty = penalty; + } + this.applyMask(i); // Undoes the mask due to XOR + } + } + assert(0 <= msk && msk <= 7); + this.mask = msk; + this.applyMask(msk); // Apply the final choice of mask + this.drawFormatBits(msk); // Overwrite old format bits + + this.isFunction = []; + } + + /*-- Accessor methods --*/ + + // Returns the color of the module (pixel) at the given coordinates, which is false + // for light or true for dark. The top left corner has the coordinates (x=0, y=0). + // If the given coordinates are out of bounds, then false (light) is returned. + public getModule(x: int, y: int): boolean { + return ( + 0 <= x && x < this.size && 0 <= y && y < this.size && this.modules[y][x] + ); + } + + /*-- Private helper methods for constructor: Drawing function modules --*/ + + // Reads this object's version field, and draws and marks all function modules. + private drawFunctionPatterns(): void { + // Draw horizontal and vertical timing patterns + for (let i = 0; i < this.size; i++) { + this.setFunctionModule(6, i, i % 2 == 0); + this.setFunctionModule(i, 6, i % 2 == 0); + } + + // Draw 3 finder patterns (all corners except bottom right; overwrites some timing modules) + this.drawFinderPattern(3, 3); + this.drawFinderPattern(this.size - 4, 3); + this.drawFinderPattern(3, this.size - 4); + + // Draw numerous alignment patterns + const alignPatPos: Array = this.getAlignmentPatternPositions(); + const numAlign: int = alignPatPos.length; + for (let i = 0; i < numAlign; i++) { + for (let j = 0; j < numAlign; j++) { + // Don't draw on the three finder corners + if ( + !( + (i == 0 && j == 0) || + (i == 0 && j == numAlign - 1) || + (i == numAlign - 1 && j == 0) + ) + ) + this.drawAlignmentPattern(alignPatPos[i], alignPatPos[j]); + } + } + + // Draw configuration data + this.drawFormatBits(0); // Dummy mask value; overwritten later in the constructor + this.drawVersion(); + } + + // Draws two copies of the format bits (with its own error correction code) + // based on the given mask and this object's error correction level field. + private drawFormatBits(mask: int): void { + // Calculate error correction code and pack bits + const data: int = (this.errorCorrectionLevel.formatBits << 3) | mask; // errCorrLvl is uint2, mask is uint3 + let rem: int = data; + for (let i = 0; i < 10; i++) rem = (rem << 1) ^ ((rem >>> 9) * 0x537); + const bits = ((data << 10) | rem) ^ 0x5412; // uint15 + assert(bits >>> 15 == 0); + + // Draw first copy + for (let i = 0; i <= 5; i++) this.setFunctionModule(8, i, getBit(bits, i)); + this.setFunctionModule(8, 7, getBit(bits, 6)); + this.setFunctionModule(8, 8, getBit(bits, 7)); + this.setFunctionModule(7, 8, getBit(bits, 8)); + for (let i = 9; i < 15; i++) + this.setFunctionModule(14 - i, 8, getBit(bits, i)); + + // Draw second copy + for (let i = 0; i < 8; i++) + this.setFunctionModule(this.size - 1 - i, 8, getBit(bits, i)); + for (let i = 8; i < 15; i++) + this.setFunctionModule(8, this.size - 15 + i, getBit(bits, i)); + this.setFunctionModule(8, this.size - 8, true); // Always dark + } + + // Draws two copies of the version bits (with its own error correction code), + // based on this object's version field, iff 7 <= version <= 40. + private drawVersion(): void { + if (this.version < 7) return; + + // Calculate error correction code and pack bits + let rem: int = this.version; // version is uint6, in the range [7, 40] + for (let i = 0; i < 12; i++) rem = (rem << 1) ^ ((rem >>> 11) * 0x1f25); + const bits: int = (this.version << 12) | rem; // uint18 + assert(bits >>> 18 == 0); + + // Draw two copies + for (let i = 0; i < 18; i++) { + const color: boolean = getBit(bits, i); + const a: int = this.size - 11 + (i % 3); + const b: int = Math.floor(i / 3); + this.setFunctionModule(a, b, color); + this.setFunctionModule(b, a, color); + } + } + + // Draws a 9*9 finder pattern including the border separator, + // with the center module at (x, y). Modules can be out of bounds. + private drawFinderPattern(x: int, y: int): void { + for (let dy = -4; dy <= 4; dy++) { + for (let dx = -4; dx <= 4; dx++) { + const dist: int = Math.max(Math.abs(dx), Math.abs(dy)); // Chebyshev/infinity norm + const xx: int = x + dx; + const yy: int = y + dy; + if (0 <= xx && xx < this.size && 0 <= yy && yy < this.size) + this.setFunctionModule(xx, yy, dist != 2 && dist != 4); + } + } + } + + // Draws a 5*5 alignment pattern, with the center module + // at (x, y). All modules must be in bounds. + private drawAlignmentPattern(x: int, y: int): void { + for (let dy = -2; dy <= 2; dy++) { + for (let dx = -2; dx <= 2; dx++) + this.setFunctionModule( + x + dx, + y + dy, + Math.max(Math.abs(dx), Math.abs(dy)) != 1 + ); + } + } + + // Sets the color of a module and marks it as a function module. + // Only used by the constructor. Coordinates must be in bounds. + private setFunctionModule(x: int, y: int, isDark: boolean): void { + this.modules[y][x] = isDark; + this.isFunction[y][x] = true; + } + + /*-- Private helper methods for constructor: Codewords and masking --*/ + + // Returns a new byte string representing the given data with the appropriate error correction + // codewords appended to it, based on this object's version and error correction level. + private addEccAndInterleave(data: Readonly>): Array { + const ver: int = this.version; + const ecl: Ecc = this.errorCorrectionLevel; + if (data.length != QrCode.getNumDataCodewords(ver, ecl)) + throw new RangeError('Invalid argument'); + + // Calculate parameter numbers + const numBlocks: int = QrCode.NUM_ERROR_CORRECTION_BLOCKS[ecl.ordinal][ver]; + const blockEccLen: int = QrCode.ECC_CODEWORDS_PER_BLOCK[ecl.ordinal][ver]; + const rawCodewords: int = Math.floor(QrCode.getNumRawDataModules(ver) / 8); + const numShortBlocks: int = numBlocks - (rawCodewords % numBlocks); + const shortBlockLen: int = Math.floor(rawCodewords / numBlocks); + + // Split data into blocks and append ECC to each block + let blocks: Array> = []; + const rsDiv: Array = QrCode.reedSolomonComputeDivisor(blockEccLen); + for (let i = 0, k = 0; i < numBlocks; i++) { + let dat: Array = data.slice( + k, + k + shortBlockLen - blockEccLen + (i < numShortBlocks ? 0 : 1) + ); + k += dat.length; + const ecc: Array = QrCode.reedSolomonComputeRemainder(dat, rsDiv); + if (i < numShortBlocks) dat.push(0); + blocks.push(dat.concat(ecc)); + } + + // Interleave (not concatenate) the bytes from every block into a single sequence + let result: Array = []; + for (let i = 0; i < blocks[0].length; i++) { + blocks.forEach((block, j) => { + // Skip the padding byte in short blocks + if (i != shortBlockLen - blockEccLen || j >= numShortBlocks) + result.push(block[i]); + }); + } + assert(result.length == rawCodewords); + return result; + } + + // Draws the given sequence of 8-bit codewords (data and error correction) onto the entire + // data area of this QR Code. Function modules need to be marked off before this is called. + private drawCodewords(data: Readonly>): void { + if ( + data.length != Math.floor(QrCode.getNumRawDataModules(this.version) / 8) + ) + throw new RangeError('Invalid argument'); + let i: int = 0; // Bit index into the data + // Do the funny zigzag scan + for (let right = this.size - 1; right >= 1; right -= 2) { + // Index of right column in each column pair + if (right == 6) right = 5; + for (let vert = 0; vert < this.size; vert++) { + // Vertical counter + for (let j = 0; j < 2; j++) { + const x: int = right - j; // Actual x coordinate + const upward: boolean = ((right + 1) & 2) == 0; + const y: int = upward ? this.size - 1 - vert : vert; // Actual y coordinate + if (!this.isFunction[y][x] && i < data.length * 8) { + this.modules[y][x] = getBit(data[i >>> 3], 7 - (i & 7)); + i++; + } + // If this QR Code has any remainder bits (0 to 7), they were assigned as + // 0/false/light by the constructor and are left unchanged by this method + } + } + } + assert(i == data.length * 8); + } + + // XORs the codeword modules in this QR Code with the given mask pattern. + // The function modules must be marked and the codeword bits must be drawn + // before masking. Due to the arithmetic of XOR, calling applyMask() with + // the same mask value a second time will undo the mask. A final well-formed + // QR Code needs exactly one (not zero, two, etc.) mask applied. + private applyMask(mask: int): void { + if (mask < 0 || mask > 7) throw new RangeError('Mask value out of range'); + for (let y = 0; y < this.size; y++) { + for (let x = 0; x < this.size; x++) { + let invert: boolean; + switch (mask) { + case 0: + invert = (x + y) % 2 == 0; + break; + case 1: + invert = y % 2 == 0; + break; + case 2: + invert = x % 3 == 0; + break; + case 3: + invert = (x + y) % 3 == 0; + break; + case 4: + invert = (Math.floor(x / 3) + Math.floor(y / 2)) % 2 == 0; + break; + case 5: + invert = ((x * y) % 2) + ((x * y) % 3) == 0; + break; + case 6: + invert = (((x * y) % 2) + ((x * y) % 3)) % 2 == 0; + break; + case 7: + invert = (((x + y) % 2) + ((x * y) % 3)) % 2 == 0; + break; + default: + throw new Error('Unreachable'); + } + if (!this.isFunction[y][x] && invert) + this.modules[y][x] = !this.modules[y][x]; + } + } + } + + // Calculates and returns the penalty score based on state of this QR Code's current modules. + // This is used by the automatic mask choice algorithm to find the mask pattern that yields the lowest score. + private getPenaltyScore(): int { + let result: int = 0; + + // Adjacent modules in row having same color, and finder-like patterns + for (let y = 0; y < this.size; y++) { + let runColor = false; + let runX = 0; + let runHistory = [0, 0, 0, 0, 0, 0, 0]; + for (let x = 0; x < this.size; x++) { + if (this.modules[y][x] == runColor) { + runX++; + if (runX == 5) result += QrCode.PENALTY_N1; + else if (runX > 5) result++; + } else { + this.finderPenaltyAddHistory(runX, runHistory); + if (!runColor) + result += + this.finderPenaltyCountPatterns(runHistory) * QrCode.PENALTY_N3; + runColor = this.modules[y][x]; + runX = 1; + } + } + result += + this.finderPenaltyTerminateAndCount(runColor, runX, runHistory) * + QrCode.PENALTY_N3; + } + // Adjacent modules in column having same color, and finder-like patterns + for (let x = 0; x < this.size; x++) { + let runColor = false; + let runY = 0; + let runHistory = [0, 0, 0, 0, 0, 0, 0]; + for (let y = 0; y < this.size; y++) { + if (this.modules[y][x] == runColor) { + runY++; + if (runY == 5) result += QrCode.PENALTY_N1; + else if (runY > 5) result++; + } else { + this.finderPenaltyAddHistory(runY, runHistory); + if (!runColor) + result += + this.finderPenaltyCountPatterns(runHistory) * QrCode.PENALTY_N3; + runColor = this.modules[y][x]; + runY = 1; + } + } + result += + this.finderPenaltyTerminateAndCount(runColor, runY, runHistory) * + QrCode.PENALTY_N3; + } + + // 2*2 blocks of modules having same color + for (let y = 0; y < this.size - 1; y++) { + for (let x = 0; x < this.size - 1; x++) { + const color: boolean = this.modules[y][x]; + if ( + color == this.modules[y][x + 1] && + color == this.modules[y + 1][x] && + color == this.modules[y + 1][x + 1] + ) + result += QrCode.PENALTY_N2; + } + } + + // Balance of dark and light modules + let dark: int = 0; + for (const row of this.modules) + dark = row.reduce((sum, color) => sum + (color ? 1 : 0), dark); + const total: int = this.size * this.size; // Note that size is odd, so dark/total != 1/2 + // Compute the smallest integer k >= 0 such that (45-5k)% <= dark/total <= (55+5k)% + const k: int = Math.ceil(Math.abs(dark * 20 - total * 10) / total) - 1; + assert(0 <= k && k <= 9); + result += k * QrCode.PENALTY_N4; + assert(0 <= result && result <= 2568888); // Non-tight upper bound based on default values of PENALTY_N1, ..., N4 + return result; + } + + /*-- Private helper functions --*/ + + // Returns an ascending list of positions of alignment patterns for this version number. + // Each position is in the range [0,177), and are used on both the x and y axes. + // This could be implemented as lookup table of 40 variable-length lists of integers. + private getAlignmentPatternPositions(): Array { + if (this.version == 1) return []; + else { + const numAlign: int = Math.floor(this.version / 7) + 2; + const step: int = + Math.floor((this.version * 8 + numAlign * 3 + 5) / (numAlign * 4 - 4)) * + 2; + let result: Array = [6]; + for (let pos = this.size - 7; result.length < numAlign; pos -= step) + result.splice(1, 0, pos); + return result; + } + } + + // Returns the number of data bits that can be stored in a QR Code of the given version number, after + // all function modules are excluded. This includes remainder bits, so it might not be a multiple of 8. + // The result is in the range [208, 29648]. This could be implemented as a 40-entry lookup table. + private static getNumRawDataModules(ver: int): int { + if (ver < QrCode.MIN_VERSION || ver > QrCode.MAX_VERSION) + throw new RangeError('Version number out of range'); + let result: int = (16 * ver + 128) * ver + 64; + if (ver >= 2) { + const numAlign: int = Math.floor(ver / 7) + 2; + result -= (25 * numAlign - 10) * numAlign - 55; + if (ver >= 7) result -= 36; + } + assert(208 <= result && result <= 29648); + return result; + } + + // Returns the number of 8-bit data (i.e. not error correction) codewords contained in any + // QR Code of the given version number and error correction level, with remainder bits discarded. + // This stateless pure function could be implemented as a (40*4)-cell lookup table. + private static getNumDataCodewords(ver: int, ecl: Ecc): int { + return ( + Math.floor(QrCode.getNumRawDataModules(ver) / 8) - + QrCode.ECC_CODEWORDS_PER_BLOCK[ecl.ordinal][ver] * + QrCode.NUM_ERROR_CORRECTION_BLOCKS[ecl.ordinal][ver] + ); + } + + // Returns a Reed-Solomon ECC generator polynomial for the given degree. This could be + // implemented as a lookup table over all possible parameter values, instead of as an algorithm. + private static reedSolomonComputeDivisor(degree: int): Array { + if (degree < 1 || degree > 255) throw new RangeError('Degree out of range'); + // Polynomial coefficients are stored from highest to lowest power, excluding the leading term which is always 1. + // For example the polynomial x^3 + 255x^2 + 8x + 93 is stored as the uint8 array [255, 8, 93]. + let result: Array = []; + for (let i = 0; i < degree - 1; i++) result.push(0); + result.push(1); // Start off with the monomial x^0 + + // Compute the product polynomial (x - r^0) * (x - r^1) * (x - r^2) * ... * (x - r^{degree-1}), + // and drop the highest monomial term which is always 1x^degree. + // Note that r = 0x02, which is a generator element of this field GF(2^8/0x11D). + let root = 1; + for (let i = 0; i < degree; i++) { + // Multiply the current product by (x - r^i) + for (let j = 0; j < result.length; j++) { + result[j] = QrCode.reedSolomonMultiply(result[j], root); + if (j + 1 < result.length) result[j] ^= result[j + 1]; + } + root = QrCode.reedSolomonMultiply(root, 0x02); + } + return result; + } + + // Returns the Reed-Solomon error correction codeword for the given data and divisor polynomials. + private static reedSolomonComputeRemainder( + data: Readonly>, + divisor: Readonly> + ): Array { + let result: Array = divisor.map((_) => 0); + for (const b of data) { + // Polynomial division + const factor: byte = b ^ (result.shift() as byte); + result.push(0); + divisor.forEach( + (coef, i) => (result[i] ^= QrCode.reedSolomonMultiply(coef, factor)) + ); + } + return result; + } + + // Returns the product of the two given field elements modulo GF(2^8/0x11D). The arguments and result + // are unsigned 8-bit integers. This could be implemented as a lookup table of 256*256 entries of uint8. + private static reedSolomonMultiply(x: byte, y: byte): byte { + if (x >>> 8 != 0 || y >>> 8 != 0) throw new RangeError('Byte out of range'); + // Russian peasant multiplication + let z: int = 0; + for (let i = 7; i >= 0; i--) { + z = (z << 1) ^ ((z >>> 7) * 0x11d); + z ^= ((y >>> i) & 1) * x; + } + assert(z >>> 8 == 0); + return z as byte; + } + + // Can only be called immediately after a light run is added, and + // returns either 0, 1, or 2. A helper function for getPenaltyScore(). + private finderPenaltyCountPatterns(runHistory: Readonly>): int { + const n: int = runHistory[1]; + assert(n <= this.size * 3); + const core: boolean = + n > 0 && + runHistory[2] == n && + runHistory[3] == n * 3 && + runHistory[4] == n && + runHistory[5] == n; + return ( + (core && runHistory[0] >= n * 4 && runHistory[6] >= n ? 1 : 0) + + (core && runHistory[6] >= n * 4 && runHistory[0] >= n ? 1 : 0) + ); + } + + // Must be called at the end of a line (row or column) of modules. A helper function for getPenaltyScore(). + private finderPenaltyTerminateAndCount( + currentRunColor: boolean, + currentRunLength: int, + runHistory: Array + ): int { + if (currentRunColor) { + // Terminate dark run + this.finderPenaltyAddHistory(currentRunLength, runHistory); + currentRunLength = 0; + } + currentRunLength += this.size; // Add light border to final run + this.finderPenaltyAddHistory(currentRunLength, runHistory); + return this.finderPenaltyCountPatterns(runHistory); + } + + // Pushes the given value to the front and drops the last value. A helper function for getPenaltyScore(). + private finderPenaltyAddHistory( + currentRunLength: int, + runHistory: Array + ): void { + if (runHistory[0] == 0) currentRunLength += this.size; // Add light border to initial run + runHistory.pop(); + runHistory.unshift(currentRunLength); + } + + /*-- Constants and tables --*/ + + // The minimum version number supported in the QR Code Model 2 standard. + public static readonly MIN_VERSION: int = 1; + // The maximum version number supported in the QR Code Model 2 standard. + public static readonly MAX_VERSION: int = 40; + + // For use in getPenaltyScore(), when evaluating which mask is best. + private static readonly PENALTY_N1: int = 3; + private static readonly PENALTY_N2: int = 3; + private static readonly PENALTY_N3: int = 40; + private static readonly PENALTY_N4: int = 10; + + private static readonly ECC_CODEWORDS_PER_BLOCK: Array> = [ + // Version: (note that index 0 is for padding, and is set to an illegal value) + //0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 Error correction level + [ + -1, 7, 10, 15, 20, 26, 18, 20, 24, 30, 18, 20, 24, 26, 30, 22, 24, 28, 30, + 28, 28, 28, 28, 30, 30, 26, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, + ], // Low + [ + -1, 10, 16, 26, 18, 24, 16, 18, 22, 22, 26, 30, 22, 22, 24, 24, 28, 28, + 26, 26, 26, 26, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 28, + ], // Medium + [ + -1, 13, 22, 18, 26, 18, 24, 18, 22, 20, 24, 28, 26, 24, 20, 30, 24, 28, + 28, 26, 30, 28, 30, 30, 30, 30, 28, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, + ], // Quartile + [ + -1, 17, 28, 22, 16, 22, 28, 26, 26, 24, 28, 24, 28, 22, 24, 24, 30, 28, + 28, 26, 28, 30, 24, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, + 30, 30, 30, 30, 30, + ], // High + ]; + + private static readonly NUM_ERROR_CORRECTION_BLOCKS: Array> = [ + // Version: (note that index 0 is for padding, and is set to an illegal value) + //0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 Error correction level + [ + -1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 4, 4, 4, 4, 4, 6, 6, 6, 6, 7, 8, 8, 9, 9, + 10, 12, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 24, 25, + ], // Low + [ + -1, 1, 1, 1, 2, 2, 4, 4, 4, 5, 5, 5, 8, 9, 9, 10, 10, 11, 13, 14, 16, 17, + 17, 18, 20, 21, 23, 25, 26, 28, 29, 31, 33, 35, 37, 38, 40, 43, 45, 47, + 49, + ], // Medium + [ + -1, 1, 1, 2, 2, 4, 4, 6, 6, 8, 8, 8, 10, 12, 16, 12, 17, 16, 18, 21, 20, + 23, 23, 25, 27, 29, 34, 34, 35, 38, 40, 43, 45, 48, 51, 53, 56, 59, 62, + 65, 68, + ], // Quartile + [ + -1, 1, 1, 2, 4, 4, 4, 5, 6, 8, 8, 11, 11, 16, 16, 18, 16, 19, 21, 25, 25, + 25, 34, 30, 32, 35, 37, 40, 42, 45, 48, 51, 54, 57, 60, 63, 66, 70, 74, + 77, 81, + ], // High + ]; +} + +// Appends the given number of low-order bits of the given value +// to the given buffer. Requires 0 <= len <= 31 and 0 <= val < 2^len. +function appendBits(val: int, len: int, bb: Array): void { + if (len < 0 || len > 31 || val >>> len != 0) + throw new RangeError('Value out of range'); + for ( + let i = len - 1; + i >= 0; + i-- // Append bit by bit + ) + bb.push((val >>> i) & 1); +} + +// Returns true iff the i'th bit of x is set to 1. +function getBit(x: int, i: int): boolean { + return ((x >>> i) & 1) != 0; +} + +// Throws an exception if the given condition is false. +function assert(cond: boolean): void { + if (!cond) throw new Error('Assertion error'); +} + +/*---- Data segment class ----*/ + +/* + * A segment of character/binary/control data in a QR Code symbol. + * Instances of this class are immutable. + * The mid-level way to create a segment is to take the payload data + * and call a static factory function such as QrSegment.makeNumeric(). + * The low-level way to create a segment is to custom-make the bit buffer + * and call the QrSegment() constructor with appropriate values. + * This segment class imposes no length restrictions, but QR Codes have restrictions. + * Even in the most favorable conditions, a QR Code can only hold 7089 characters of data. + * Any segment longer than this is meaningless for the purpose of generating QR Codes. + */ +export class QrSegment { + /*-- Static factory functions (mid level) --*/ + + // Returns a segment representing the given binary data encoded in + // byte mode. All input byte arrays are acceptable. Any text string + // can be converted to UTF-8 bytes and encoded as a byte mode segment. + public static makeBytes(data: Readonly>): QrSegment { + let bb: Array = []; + for (const b of data) appendBits(b, 8, bb); + return new QrSegment(Mode.BYTE, data.length, bb); + } + + // Returns a segment representing the given string of decimal digits encoded in numeric mode. + public static makeNumeric(digits: string): QrSegment { + if (!QrSegment.isNumeric(digits)) + throw new RangeError('String contains non-numeric characters'); + let bb: Array = []; + for (let i = 0; i < digits.length; ) { + // Consume up to 3 digits per iteration + const n: int = Math.min(digits.length - i, 3); + appendBits(parseInt(digits.substring(i, i + n), 10), n * 3 + 1, bb); + i += n; + } + return new QrSegment(Mode.NUMERIC, digits.length, bb); + } + + // Returns a segment representing the given text string encoded in alphanumeric mode. + // The characters allowed are: 0 to 9, A to Z (uppercase only), space, + // dollar, percent, asterisk, plus, hyphen, period, slash, colon. + public static makeAlphanumeric(text: string): QrSegment { + if (!QrSegment.isAlphanumeric(text)) + throw new RangeError( + 'String contains unencodable characters in alphanumeric mode' + ); + let bb: Array = []; + let i: int; + for (i = 0; i + 2 <= text.length; i += 2) { + // Process groups of 2 + let temp: int = + QrSegment.ALPHANUMERIC_CHARSET.indexOf(text.charAt(i)) * 45; + temp += QrSegment.ALPHANUMERIC_CHARSET.indexOf(text.charAt(i + 1)); + appendBits(temp, 11, bb); + } + if (i < text.length) + // 1 character remaining + appendBits(QrSegment.ALPHANUMERIC_CHARSET.indexOf(text.charAt(i)), 6, bb); + return new QrSegment(Mode.ALPHANUMERIC, text.length, bb); + } + + // Returns a new mutable list of zero or more segments to represent the given Unicode text string. + // The result may use various segment modes and switch modes to optimize the length of the bit stream. + public static makeSegments(text: string): Array { + // Select the most efficient segment encoding automatically + if (text == '') return []; + else if (QrSegment.isNumeric(text)) return [QrSegment.makeNumeric(text)]; + else if (QrSegment.isAlphanumeric(text)) + return [QrSegment.makeAlphanumeric(text)]; + else return [QrSegment.makeBytes(QrSegment.toUtf8ByteArray(text))]; + } + + // Returns a segment representing an Extended Channel Interpretation + // (ECI) designator with the given assignment value. + public static makeEci(assignVal: int): QrSegment { + let bb: Array = []; + if (assignVal < 0) + throw new RangeError('ECI assignment value out of range'); + else if (assignVal < 1 << 7) appendBits(assignVal, 8, bb); + else if (assignVal < 1 << 14) { + appendBits(0b10, 2, bb); + appendBits(assignVal, 14, bb); + } else if (assignVal < 1000000) { + appendBits(0b110, 3, bb); + appendBits(assignVal, 21, bb); + } else throw new RangeError('ECI assignment value out of range'); + return new QrSegment(Mode.ECI, 0, bb); + } + + // Tests whether the given string can be encoded as a segment in numeric mode. + // A string is encodable iff each character is in the range 0 to 9. + public static isNumeric(text: string): boolean { + return QrSegment.NUMERIC_REGEX.test(text); + } + + // Tests whether the given string can be encoded as a segment in alphanumeric mode. + // A string is encodable iff each character is in the following set: 0 to 9, A to Z + // (uppercase only), space, dollar, percent, asterisk, plus, hyphen, period, slash, colon. + public static isAlphanumeric(text: string): boolean { + return QrSegment.ALPHANUMERIC_REGEX.test(text); + } + + /*-- Constructor (low level) and fields --*/ + + // Creates a new QR Code segment with the given attributes and data. + // The character count (numChars) must agree with the mode and the bit buffer length, + // but the constraint isn't checked. The given bit buffer is cloned and stored. + // The mode indicator of this segment. + public readonly mode: Mode; + + // The length of this segment's unencoded data. Measured in characters for + // numeric/alphanumeric/kanji mode, bytes for byte mode, and 0 for ECI mode. + // Always zero or positive. Not the same as the data's bit length. + public readonly numChars: int; + + // The data bits of this segment. Accessed through getData(). + private readonly bitData: Array; + + public constructor(mode: Mode, numChars: int, bitData: Array) { + if (numChars < 0) throw new RangeError('Invalid argument'); + this.mode = mode; + this.numChars = numChars; + this.bitData = bitData.slice(); // Make defensive copy + } + + /*-- Methods --*/ + + // Returns a new copy of the data bits of this segment. + public getData(): Array { + return this.bitData.slice(); // Make defensive copy + } + + // (Package-private) Calculates and returns the number of bits needed to encode the given segments at + // the given version. The result is infinity if a segment has too many characters to fit its length field. + public static getTotalBits( + segs: Readonly>, + version: int + ): number { + let result: number = 0; + for (const seg of segs) { + const ccbits: int = seg.mode.numCharCountBits(version); + if (seg.numChars >= 1 << ccbits) return Infinity; // The segment's length doesn't fit the field's bit width + result += 4 + ccbits + seg.bitData.length; + } + return result; + } + + // Returns a new array of bytes representing the given string encoded in UTF-8. + private static toUtf8ByteArray(str: string): Array { + str = encodeURI(str); + let result: Array = []; + for (let i = 0; i < str.length; i++) { + if (str.charAt(i) != '%') result.push(str.charCodeAt(i)); + else { + result.push(parseInt(str.substring(i + 1, i + 3), 16)); + i += 2; + } + } + return result; + } + + /*-- Constants --*/ + + // Describes precisely all strings that are encodable in numeric mode. + private static readonly NUMERIC_REGEX: RegExp = /^[0-9]*$/; + + // Describes precisely all strings that are encodable in alphanumeric mode. + private static readonly ALPHANUMERIC_REGEX: RegExp = /^[A-Z0-9 $%*+.\/:-]*$/; + + // The set of all legal characters in alphanumeric mode, + // where each character value maps to the index in the string. + private static readonly ALPHANUMERIC_CHARSET: string = + '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ $%*+-./:'; +} + +/*---- Public helper enumeration ----*/ + +/* + * The error correction level in a QR Code symbol. Immutable. + */ +export class Ecc { + /*-- Constants --*/ + + public static readonly LOW = new Ecc(0, 1); // The QR Code can tolerate about 7% erroneous codewords + public static readonly MEDIUM = new Ecc(1, 0); // The QR Code can tolerate about 15% erroneous codewords + public static readonly QUARTILE = new Ecc(2, 3); // The QR Code can tolerate about 25% erroneous codewords + public static readonly HIGH = new Ecc(3, 2); // The QR Code can tolerate about 30% erroneous codewords + + /*-- Constructor and fields --*/ + + // In the range 0 to 3 (unsigned 2-bit integer). + public readonly ordinal: int; + // (Package-private) In the range 0 to 3 (unsigned 2-bit integer). + public readonly formatBits: int; + + private constructor(ordinal: int, formatBits: int) { + this.ordinal = ordinal; + this.formatBits = formatBits; + } +} + +/*---- Public helper enumeration ----*/ + +/* + * Describes how a segment's data bits are interpreted. Immutable. + */ +export class Mode { + /*-- Constants --*/ + + public static readonly NUMERIC = new Mode(0x1, [10, 12, 14]); + public static readonly ALPHANUMERIC = new Mode(0x2, [9, 11, 13]); + public static readonly BYTE = new Mode(0x4, [8, 16, 16]); + public static readonly KANJI = new Mode(0x8, [8, 10, 12]); + public static readonly ECI = new Mode(0x7, [0, 0, 0]); + + /*-- Constructor and fields --*/ + + // The mode indicator bits, which is a uint4 value (range 0 to 15). + public readonly modeBits: int; + // Number of character count bits for three different version ranges. + private readonly numBitsCharCount: [int, int, int]; + + private constructor(modeBits: int, numBitsCharCount: [int, int, int]) { + this.modeBits = modeBits; + this.numBitsCharCount = numBitsCharCount; + } + + /*-- Method --*/ + + // (Package-private) Returns the bit width of the character count field for a segment in + // this mode in a QR Code at the given version number. The result is in the range [0, 16]. + public numCharCountBits(ver: int): int { + return this.numBitsCharCount[Math.floor((ver + 7) / 17)]; + } +} diff --git a/apps/common-app/orchestra-server/samples.ts b/apps/common-app/orchestra-server/samples.ts new file mode 100644 index 000000000..06048dda6 --- /dev/null +++ b/apps/common-app/orchestra-server/samples.ts @@ -0,0 +1,233 @@ +// Recorded instruments for the phones: the FluidR3_GM soundfont rendered to +// one mp3 per note by gleitz/midi-js-soundfonts, and a drum kit from VCSL +// (credits in the README). The server mirrors the notes the scores use into +// ./samples, so phones only need the local network, and fetches anything else +// on first request. +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +import { + DRUM_PIECES, + DRUMS, + drumPieceFor, + type DrumPiece, +} from './drumKit.ts'; +import type { Score } from '../src/demos/Orchestra/protocol.ts'; + +const UPSTREAM = 'https://gleitz.github.io/midi-js-soundfonts/FluidR3_GM'; +const VCSL = 'https://raw.githubusercontent.com/sgossner/VCSL/master'; +const SKINS = 'Membranophones/Struck Membranophones'; +const METALS = 'Idiophones/Struck Idiophones'; + +// One mid-loud velocity layer per piece; its round robins are the takes. +const DRUM_SOURCES: Record = { + kick: [ + `${SKINS}/Bass Drum 1/BDrumNew_hit_v5_rr1_Sum.wav`, + `${SKINS}/Bass Drum 1/BDrumNew_hit_v5_rr2_Sum.wav`, + ], + snare: [ + `${SKINS}/Snare Drum, Modern 1/Snare2_HitSN_v6_rr1_Mid.wav`, + `${SKINS}/Snare Drum, Modern 1/Snare2_HitSN_v6_rr2_Mid.wav`, + ], + clap: [ + `${METALS}/Claps/Clap_rr1.wav`, + `${METALS}/Claps/Clap_rr2.wav`, + `${METALS}/Claps/Clap_rr3.wav`, + ], + closedHat: [ + `${METALS}/Hi-Hat Cymbal/HiHat_HitC_v3_rr1_Mid.wav`, + `${METALS}/Hi-Hat Cymbal/HiHat_HitC_v3_rr2_Mid.wav`, + ], + openHat: [ + `${METALS}/Hi-Hat Cymbal/HiHat_HitO_rr1_Mid.wav`, + `${METALS}/Hi-Hat Cymbal/HiHat_HitO_rr2_Mid.wav`, + ], + lowTom: [ + `${SKINS}/Tom 2/Stick/TomL_HitS_v3_rr1_Mid.wav`, + `${SKINS}/Tom 2/Stick/TomL_HitS_v3_rr2_Mid.wav`, + ], + highTom: [ + `${SKINS}/Tom 1/Stick/TomH_HitS_v3_rr1_Mid.wav`, + `${SKINS}/Tom 1/Stick/TomH_HitS_v3_rr2_Mid.wav`, + ], + crash: [ + `${METALS}/Clash Cymbals 1/cymbal_crash1_mf1.wav`, + `${METALS}/Clash Cymbals 1/cymbal_crash1_mf2.wav`, + ], + ride: [`${METALS}/Suspended Cymbal 1/susCymb1_hit_stick_mp1.wav`], +}; + +for (const [piece, info] of Object.entries(DRUM_PIECES)) { + if (DRUM_SOURCES[piece as DrumPiece].length !== info.takes) { + throw new Error(`drum kit: ${piece} lists a different number of takes`); + } +} +const UPSTREAM_TIMEOUT_MS = 8000; +const PREFETCH_CONCURRENCY = 6; +/** FluidR3_GM is rendered from A0 to C8. */ +export const LOWEST_SAMPLED_MIDI = 21; +export const HIGHEST_SAMPLED_MIDI = 108; + +// The renders spell every black key as a flat. +const NOTE_NAMES = [ + 'C', + 'Db', + 'D', + 'Eb', + 'E', + 'F', + 'Gb', + 'G', + 'Ab', + 'A', + 'Bb', + 'B', +]; +const INSTRUMENT_PATTERN = /^[a-z0-9_]+$/; +const NOTE_PATTERN = /^[A-G]b?[0-8]$/; + +export function sampleNoteName(midi: number): string { + return `${NOTE_NAMES[midi % 12]}${Math.floor(midi / 12) - 1}`; +} + +export class SampleStore { + private readonly directory: string; + private readonly downloads = new Map>(); + + constructor(directory: string) { + this.directory = directory; + } + + /** + * The General MIDI program names, index = program. Cached next to the + * samples; empty when the server has never been online. + */ + async instrumentNames(): Promise { + const cached = join(this.directory, 'names.json'); + try { + return JSON.parse(await readFile(cached, 'utf8')) as string[]; + } catch { + // Not cached yet. + } + const body = await download(`${UPSTREAM}/names.json`); + if (!body) { + console.warn( + 'Samples: no instrument list (offline?); parts use synth voices' + ); + return []; + } + await mkdir(this.directory, { recursive: true }); + await writeFile(cached, body); + return JSON.parse(body.toString('utf8')) as string[]; + } + + /** One note's mp3, from the mirror or, the first time, from upstream. */ + get(instrument: string, note: string): Promise { + if (!INSTRUMENT_PATTERN.test(instrument) || !NOTE_PATTERN.test(note)) { + return Promise.resolve(null); + } + return this.mirrored( + join(instrument, `${note}.mp3`), + `${UPSTREAM}/${instrument}-mp3/${note}.mp3` + ); + } + + /** One take (1-based) of a drum kit piece, as wav. */ + getDrum(piece: string, take: number): Promise { + const source = DRUM_SOURCES[piece as DrumPiece]?.[take - 1]; + if (!source) { + return Promise.resolve(null); + } + const url = `${VCSL}/${source.split('/').map(encodeURIComponent).join('/')}`; + return this.mirrored(join(DRUMS, piece, `${take}.wav`), url); + } + + private async mirrored( + relativePath: string, + url: string + ): Promise { + const path = join(this.directory, relativePath); + try { + return await readFile(path); + } catch { + // Not mirrored yet. + } + + let pending = this.downloads.get(relativePath); + if (!pending) { + pending = download(url).then(async (body) => { + if (body) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, body); + } + this.downloads.delete(relativePath); + return body; + }); + this.downloads.set(relativePath, pending); + } + return pending; + } + + /** Mirrors every note any part of these scores plays, a few at a time. */ + async prefetch(scores: readonly Score[]): Promise { + const wanted = new Set(); + for (const score of scores) { + for (const part of score.parts) { + if (part.instrument === DRUMS) { + for (const note of part.notes) { + const piece = drumPieceFor(note.midi); + for ( + let take = 1; + piece && take <= DRUM_PIECES[piece].takes; + take += 1 + ) { + wanted.add(`${DRUMS}/${piece}/${take}`); + } + } + continue; + } + if (!part.instrument) { + continue; + } + for (const note of part.notes) { + const midi = Math.min( + HIGHEST_SAMPLED_MIDI, + Math.max(LOWEST_SAMPLED_MIDI, note.midi) + ); + wanted.add(`${part.instrument}/${sampleNoteName(midi)}`); + } + } + } + + const queue = [...wanted]; + let missing = 0; + const worker = async () => { + for (let key = queue.pop(); key; key = queue.pop()) { + const [instrument, note, take] = key.split('/'); + const body = + instrument === DRUMS + ? await this.getDrum(note, Number(take)) + : await this.get(instrument, note); + if (!body) { + missing += 1; + } + } + }; + await Promise.all(Array.from({ length: PREFETCH_CONCURRENCY }, worker)); + console.log( + `Samples: ${wanted.size - missing}/${wanted.size} notes mirrored in ./samples` + + (missing ? ` (${missing} unavailable)` : '') + ); + } +} + +async function download(url: string): Promise { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + return response.ok ? Buffer.from(await response.arrayBuffer()) : null; + } catch { + return null; + } +} diff --git a/apps/common-app/orchestra-server/samples/.gitkeep b/apps/common-app/orchestra-server/samples/.gitkeep new file mode 100644 index 000000000..6c39a03ab --- /dev/null +++ b/apps/common-app/orchestra-server/samples/.gitkeep @@ -0,0 +1 @@ +The server downloads instrument samples here at startup; the contents stay out of git. diff --git a/apps/common-app/orchestra-server/score.ts b/apps/common-app/orchestra-server/score.ts new file mode 100644 index 000000000..cfade471c --- /dev/null +++ b/apps/common-app/orchestra-server/score.ts @@ -0,0 +1,685 @@ +// The conductor's scores: the built-in "Relay", plus every MIDI file in ./midi. +// Scores are sent to every phone in the `state` message, so the server is the +// single source of truth for what gets played. Phone speakers reproduce almost +// nothing below ~300 Hz, so Relay keeps every part at midi 55 or above. +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { loadMidiFolder } from './midi.ts'; +import { SampleStore } from './samples.ts'; +import { DRUMS } from './drumKit.ts'; +import type { + Score, + ScoreNote, + ScorePart, +} from '../src/demos/Orchestra/protocol.ts'; + +const BEATS_PER_BAR = 4; + +// "Relay": six parts in D minor. Intro → theme → a dialogue where one motif +// hops from phone to phone → a build over a rising progression → the theme +// again → a held final chord, whose last bar sits on the dominant (A) so the +// loop falls back into the intro. + +/** [beat offset within the bar, midi, duration in beats] */ +type Figure = readonly [number, number, number]; + +interface Harmony { + /** Voice-led three-note pad chord around middle C. */ + pad: readonly number[]; + /** Kept at midi 55+ so phone speakers can still carry it. */ + bass: number; + /** The four-note dialogue motif spelled over this chord. */ + motif: readonly number[]; + counter: readonly Figure[]; +} + +const D_MINOR: Harmony = { + pad: [62, 65, 69], + bass: 62, + motif: [69, 74, 76, 77], + counter: [ + [0, 65, 2], + [2, 69, 2], + ], +}; +const B_FLAT: Harmony = { + pad: [62, 65, 70], + bass: 58, + motif: [65, 70, 72, 74], + counter: [ + [0, 70, 2], + [2, 65, 2], + ], +}; +const F_MAJOR: Harmony = { + pad: [60, 65, 69], + bass: 65, + motif: [72, 77, 79, 81], + counter: [ + [0, 69, 1.5], + [1.5, 67, 0.5], + [2, 65, 2], + ], +}; +const C_MAJOR: Harmony = { + pad: [60, 64, 67], + bass: 60, + motif: [67, 72, 74, 76], + counter: [ + [0, 64, 2], + [2, 67, 1], + [3, 64, 1], + ], +}; +const G_MINOR: Harmony = { + pad: [62, 67, 70], + bass: 55, + motif: [67, 70, 72, 74], + counter: [ + [0, 70, 2], + [2, 74, 2], + ], +}; +const A_MAJOR: Harmony = { + pad: [61, 64, 69], + bass: 57, + motif: [64, 69, 71, 73], + counter: [ + [0, 73, 2], + [2, 76, 2], + ], +}; + +// i – VI – III – VII, and the build's iv – VI – VII – V that pulls back home. +const HOME = [D_MINOR, B_FLAT, F_MAJOR, C_MAJOR]; +const RISE = [G_MINOR, B_FLAT, C_MAJOR, A_MAJOR]; + +type Section = 'intro' | 'theme' | 'dialogue' | 'build' | 'reprise' | 'coda'; + +const FORM: readonly (readonly [Section, readonly Harmony[]])[] = [ + ['intro', HOME], + ['theme', [...HOME, ...HOME]], + ['dialogue', [...HOME, ...HOME]], + ['build', [...RISE, ...RISE]], + ['reprise', [...HOME, ...HOME]], + ['coda', [D_MINOR, D_MINOR, D_MINOR, A_MAJOR]], +]; + +interface Bar { + start: number; + section: Section; + barInSection: number; + harmony: Harmony; +} + +function relayBars(): Bar[] { + const bars: Bar[] = []; + for (const [section, harmonies] of FORM) { + harmonies.forEach((harmony, barInSection) => { + bars.push({ + start: bars.length * BEATS_PER_BAR, + section, + barInSection, + harmony, + }); + }); + } + return bars; +} + +// One phrase per bar of the theme; the motif (A D E F) opens bars 1, 3 and 5. +const THEME_MELODY: readonly (readonly Figure[])[] = [ + [ + [0, 69, 0.5], + [0.5, 74, 0.5], + [1, 76, 0.5], + [1.5, 77, 1.5], + [3, 76, 0.5], + [3.5, 74, 0.5], + ], + [ + [0, 74, 1.5], + [1.5, 72, 0.5], + [2, 70, 1], + [3, 69, 1], + ], + [ + [0, 72, 0.5], + [0.5, 77, 0.5], + [1, 79, 0.5], + [1.5, 81, 1.5], + [3, 79, 0.5], + [3.5, 77, 0.5], + ], + [ + [0, 76, 2], + [2, 79, 1], + [3, 76, 1], + ], + [ + [0, 69, 0.5], + [0.5, 74, 0.5], + [1, 76, 0.5], + [1.5, 77, 1], + [2.5, 81, 0.5], + [3, 79, 0.5], + [3.5, 77, 0.5], + ], + [ + [0, 74, 1], + [1, 77, 0.5], + [1.5, 74, 0.5], + [2, 70, 2], + ], + [ + [0, 72, 1], + [1, 69, 0.5], + [1.5, 72, 0.5], + [2, 77, 1], + [3, 76, 0.5], + [3.5, 77, 0.5], + ], + [ + [0, 76, 3], + [3, 72, 1], + ], +]; + +const BUILD_MELODY: readonly (readonly Figure[])[] = [ + [ + [0, 67, 0.5], + [0.5, 70, 0.5], + [1, 72, 0.5], + [1.5, 74, 2.5], + ], + [ + [0, 70, 0.5], + [0.5, 72, 0.5], + [1, 74, 0.5], + [1.5, 77, 2.5], + ], + [ + [0, 72, 0.5], + [0.5, 76, 0.5], + [1, 79, 3], + ], + [ + [0, 76, 0.5], + [0.5, 81, 3.5], + ], + [[0, 74, 4]], + [[0, 77, 4]], + [[0, 79, 4]], + [ + [0, 81, 2], + [2, 79, 1], + [3, 76, 0.5], + [3.5, 73, 0.5], + ], +]; + +type DialogueVoice = 'lead' | 'counter' | 'arpeggio' | 'bass'; + +const DIALOGUE_ORDER: readonly DialogueVoice[] = [ + 'lead', + 'counter', + 'arpeggio', + 'bass', +]; + +function dialogueRegister(voice: DialogueVoice, motif: readonly number[]) { + switch (voice) { + case 'counter': + return motif.map((midi) => midi + 12); + case 'bass': { + const octavesDown = Math.floor((Math.min(...motif) - 55) / 12); + return motif.map((midi) => midi - 12 * octavesDown); + } + default: + return motif; + } +} + +// The motif changes hands every bar, then every half bar, and in the last bar +// all four voices that carried it sound it together. +function dialogueTurn(voice: DialogueVoice, bar: Bar): ScoreNote[] { + const motif = dialogueRegister(voice, bar.harmony.motif); + const turn = DIALOGUE_ORDER.indexOf(voice); + const isLastBar = bar.barInSection === 7; + + if (bar.barInSection < 4 || isLastBar) { + if (!isLastBar && bar.barInSection !== turn) { + return []; + } + return figureNotes( + bar.start, + [ + [0, motif[0], 0.5], + [0.5, motif[1], 0.5], + [1, motif[2], 0.5], + [1.5, motif[3], 2], + ], + isLastBar ? 0.95 : 0.85 + ); + } + + const notes: ScoreNote[] = []; + for (const half of [0, 1]) { + const handoff = (bar.barInSection - 4) * 2 + half; + if (handoff % DIALOGUE_ORDER.length === turn) { + notes.push( + ...figureNotes( + bar.start + half * 2, + [ + [0, motif[0], 0.5], + [0.5, motif[1], 0.5], + [1, motif[2], 0.5], + [1.5, motif[3], 0.5], + ], + 0.85 + ) + ); + } + } + return notes; +} + +/** Downbeats get a little accent so phrases breathe. */ +function figureNotes( + barStart: number, + figures: readonly Figure[], + velocity: number +): ScoreNote[] { + return figures.map(([offset, midi, duration]) => ({ + beat: barStart + offset, + midi, + duration, + velocity: Math.min( + 1, + Number.isInteger(offset) ? velocity + 0.08 : velocity + ), + })); +} + +/** 0 → 1 across a section, for crescendos. */ +function progress(bar: Bar, barsInSection: number): number { + return bar.barInSection / (barsInSection - 1); +} + +function leadPart(bars: Bar[]): ScoreNote[] { + return bars.flatMap((bar) => { + switch (bar.section) { + case 'theme': + return figureNotes(bar.start, THEME_MELODY[bar.barInSection], 0.75); + case 'dialogue': + return dialogueTurn('lead', bar); + case 'build': + return figureNotes( + bar.start, + BUILD_MELODY[bar.barInSection], + 0.65 + 0.3 * progress(bar, 8) + ); + case 'reprise': + return figureNotes(bar.start, THEME_MELODY[bar.barInSection], 0.9); + case 'coda': + return bar.barInSection === 0 + ? figureNotes(bar.start, [[0, 74, 8]], 0.85) + : []; + default: + return []; + } + }); +} + +function counterPart(bars: Bar[]): ScoreNote[] { + return bars.flatMap((bar) => { + switch (bar.section) { + case 'theme': + return bar.barInSection >= 4 + ? figureNotes(bar.start, bar.harmony.counter, 0.6) + : []; + case 'dialogue': + return dialogueTurn('counter', bar); + case 'build': + return bar.barInSection >= 4 + ? figureNotes(bar.start, bar.harmony.counter, 0.7) + : []; + case 'reprise': + return figureNotes(bar.start, bar.harmony.counter, 0.7); + case 'coda': + return bar.barInSection === 0 + ? figureNotes( + bar.start, + [ + [0, 81, 2], + [2, 77, 2], + ], + 0.7 + ) + : []; + default: + return []; + } + }); +} + +function padPart(bars: Bar[]): ScoreNote[] { + const chord = (bar: Bar, duration: number, velocity: number) => + bar.harmony.pad.map((midi) => ({ + beat: bar.start, + midi, + duration, + velocity, + })); + + return bars.flatMap((bar) => { + switch (bar.section) { + case 'intro': + return chord(bar, 4, 0.45 + 0.1 * progress(bar, 4)); + case 'dialogue': + return chord(bar, 4, 0.35); + case 'build': + return chord(bar, 4, 0.45 + 0.3 * progress(bar, 8)); + case 'coda': + if (bar.barInSection === 0) { + return chord(bar, 12, 0.6); + } + return bar.barInSection === 3 ? chord(bar, 4, 0.4) : []; + default: + return chord(bar, 4, 0.5); + } + }); +} + +function arpeggioPart(bars: Bar[]): ScoreNote[] { + const pattern = [0, 1, 2, 3, 2, 1, 2, 1]; + const run = (bar: Bar, stepBeats: number, velocity: number) => { + const [root, third, fifth] = bar.harmony.pad.map((midi) => midi + 12); + const tones = [root, third, fifth, root + 12]; + const notes: ScoreNote[] = []; + for (let step = 0; step * stepBeats < BEATS_PER_BAR; step += 1) { + const offset = step * stepBeats; + notes.push({ + beat: bar.start + offset, + midi: tones[pattern[step % pattern.length]], + duration: stepBeats, + velocity: Number.isInteger(offset) ? velocity + 0.15 : velocity, + }); + } + return notes; + }; + + return bars.flatMap((bar) => { + switch (bar.section) { + case 'intro': + return run(bar, 0.5, 0.55); + case 'dialogue': + return dialogueTurn('arpeggio', bar); + case 'build': + return bar.barInSection < 4 + ? run(bar, 0.5, 0.5) + : run(bar, 0.25, 0.45 + 0.35 * progress(bar, 8)); + case 'coda': + if (bar.barInSection === 0) { + return figureNotes( + bar.start, + [ + [0, 86, 1], + [1, 81, 1], + [2, 77, 1], + [3, 74, 1], + ], + 0.55 + ); + } + // C# rising to the D that opens the intro. + return bar.barInSection === 3 + ? figureNotes( + bar.start, + [69, 73, 76, 81, 85, 81, 76, 73].map( + (midi, step): Figure => [step / 2, midi, 0.5] + ), + 0.45 + ) + : []; + default: + return run(bar, 0.5, 0.5); + } + }); +} + +function bassPart(bars: Bar[]): ScoreNote[] { + const groove = (bar: Bar, velocity: number) => { + const root = bar.harmony.bass; + return figureNotes( + bar.start, + [ + [0, root, 1.25], + [1.5, root, 0.5], + [2, root + 12, 0.5], + [2.5, root, 1], + [3.5, root + 7, 0.5], + ], + velocity + ); + }; + + return bars.flatMap((bar) => { + const root = bar.harmony.bass; + switch (bar.section) { + case 'theme': + return groove(bar, 0.75); + case 'dialogue': + return dialogueTurn('bass', bar); + case 'build': + if (bar.barInSection < 4) { + return figureNotes( + bar.start, + [ + [0, root, 2], + [2, root, 2], + ], + 0.7 + ); + } + return Array.from({ length: 8 }, (_, step) => ({ + beat: bar.start + step / 2, + midi: step % 2 === 0 ? root : root + 12, + duration: 0.5, + velocity: 0.6 + 0.35 * progress(bar, 8), + })); + case 'reprise': + return groove(bar, 0.85); + case 'coda': + if (bar.barInSection === 0) { + return figureNotes(bar.start, [[0, root, 4]], 0.85); + } + return bar.barInSection === 3 + ? figureNotes( + bar.start, + [ + [0, root, 2], + [2, root + 7, 1], + [3, root + 4, 1], + ], + 0.6 + ) + : []; + default: + return []; + } + }); +} + +// General MIDI percussion keys, played on the recorded kit (drumKit.ts). +const KICK = 36; +const SNARE = 38; +const CLAP = 39; +const CLOSED_HAT = 42; +const OPEN_HAT = 46; +const CRASH = 49; + +function percussionPart(bars: Bar[]): ScoreNote[] { + const hats = (bar: Bar, stepBeats: number, velocity: number) => { + const notes: ScoreNote[] = []; + for (let offset = 0; offset < BEATS_PER_BAR; offset += stepBeats) { + notes.push({ + beat: bar.start + offset, + midi: CLOSED_HAT, + duration: 0.1, + velocity: Number.isInteger(offset) ? velocity + 0.15 : velocity, + }); + } + return notes; + }; + const hits = + (midi: number) => (bar: Bar, offsets: number[], velocity: number) => + offsets.map((offset) => ({ + beat: bar.start + offset, + midi, + duration: 0.25, + velocity, + })); + const claps = hits(CLAP); + const snares = hits(SNARE); + const kicks = hits(KICK); + // An open hat on the last off-beat of every other bar. + const backbeat = (bar: Bar, velocity: number) => { + const openHatBeat = + bar.barInSection % 2 === 1 ? bar.start + 3.5 : Number.NaN; + const hatLine = hats(bar, 0.5, velocity - 0.2).map((note) => + note.beat === openHatBeat + ? { ...note, midi: OPEN_HAT, duration: 0.5 } + : note + ); + return [ + ...hatLine, + ...kicks(bar, [0, 2], velocity), + ...snares(bar, [1, 3], velocity), + ]; + }; + + return bars.flatMap((bar) => { + switch (bar.section) { + case 'theme': + return backbeat(bar, 0.75); + case 'dialogue': + return bar.barInSection === 7 + ? [...hats(bar, 1, 0.25), ...claps(bar, [0], 1)] + : hats(bar, 1, 0.2); + case 'build': { + const swell = progress(bar, 8); + if (bar.barInSection < 4) { + return backbeat(bar, 0.65); + } + if (bar.barInSection < 6) { + return [ + ...hats(bar, 0.25, 0.3 + 0.3 * swell), + ...kicks(bar, [0, 2], 0.8), + ...snares(bar, [1, 3], 0.8), + ]; + } + if (bar.barInSection === 6) { + return [ + ...hats(bar, 0.25, 0.55), + ...kicks(bar, [0, 1, 2, 3], 0.85), + ...claps(bar, [1, 3], 0.85), + ]; + } + // A snare roll swelling into the reprise. + return snares( + bar, + [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.25, 3.5, 3.75], + 0.9 + ).map((note, index, roll) => ({ + ...note, + velocity: 0.55 + (0.45 * index) / (roll.length - 1), + })); + } + case 'reprise': + return backbeat(bar, 0.85); + case 'coda': + if (bar.barInSection === 0) { + return [ + ...kicks(bar, [0], 1), + { beat: bar.start, midi: CRASH, duration: 2, velocity: 0.8 }, + ]; + } + return bar.barInSection === 3 ? hats(bar, 0.5, 0.2).slice(4) : []; + default: + return []; + } + }); +} + +function relayScore(): Score { + const bars = relayBars(); + const parts: ScorePart[] = [ + { + name: 'Lead', + color: '#FF5E7A', + timbre: 'lead', + instrument: 'violin', + notes: leadPart(bars), + }, + { + name: 'Pad', + color: '#9B7BFF', + timbre: 'pad', + instrument: 'string_ensemble_1', + notes: padPart(bars), + }, + { + name: 'Arpeggio', + color: '#38ACDD', + timbre: 'pluck', + instrument: 'orchestral_harp', + notes: arpeggioPart(bars), + }, + { + name: 'Bass', + color: '#FF9F43', + timbre: 'pluck', + instrument: 'pizzicato_strings', + notes: bassPart(bars), + }, + { + name: 'Counter-melody', + color: '#FFD61E', + timbre: 'bell', + instrument: 'flute', + notes: counterPart(bars), + }, + { + name: 'Percussion', + color: '#8BE38B', + timbre: 'kit', + instrument: DRUMS, + notes: percussionPart(bars), + }, + ]; + + return { + id: 'relay', + title: 'Relay in D minor', + bpm: 100, + lengthBeats: bars.length * BEATS_PER_BAR, + parts, + }; +} + +const here = dirname(fileURLToPath(import.meta.url)); + +export const sampleStore = new SampleStore(join(here, 'samples')); + +// Imported parts get the instrument their General MIDI program names. +const instrumentNames = await sampleStore.instrumentNames(); + +export const scores: Record = { + relay: relayScore(), + // Every .mid file in ./midi (see README). + ...loadMidiFolder(join(here, 'midi'), instrumentNames), +}; + +export const DEFAULT_SCORE_ID = 'relay'; diff --git a/apps/common-app/orchestra-server/server.ts b/apps/common-app/orchestra-server/server.ts new file mode 100644 index 000000000..a30be3300 --- /dev/null +++ b/apps/common-app/orchestra-server/server.ts @@ -0,0 +1,346 @@ +// Phone orchestra server: one WebSocket server every phone syncs its clock +// against and takes the score + transport from. Run with `node server.ts` +// (Node 22.18+, no dependencies) and open http://:8765 for the +// dashboard. The wire format is documented in `src/demos/Orchestra/protocol.ts`. +import { createServer } from 'node:http'; +import { readFileSync } from 'node:fs'; +import { networkInterfaces } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +import { acceptUpgrade, type WebSocketConnection } from './websocket.ts'; +import { scores, sampleStore, DEFAULT_SCORE_ID } from './score.ts'; +import { qrSvg } from './qr.ts'; +import type { + ClientMessage, + PlayerInfo, + Score, + ServerMessage, + Transport, +} from '../src/demos/Orchestra/protocol.ts'; + +const PORT = Number(process.env.PORT ?? 8765); +const CONDUCTOR_REFRESH_MS = 1000; +const MIN_COUNT_IN_MS = 500; + +interface Client extends PlayerInfo { + role: 'player' | 'conductor'; + connection: WebSocketConnection; +} + +const here = dirname(fileURLToPath(import.meta.url)); +const dashboardHtml = readFileSync(join(here, 'dashboard.html')); + +const serverNow = (): number => performance.now(); + +const transport: Transport = { + playing: false, + startAt: 0, + bpm: scores[DEFAULT_SCORE_ID].bpm, + epoch: 0, +}; +let scoreId = DEFAULT_SCORE_ID; + +const clients = new Map(); +let nextClientId = 1; + +function currentScore(): Score { + return scores[scoreId]; +} + +function players(): Client[] { + return [...clients.values()].filter((client) => client.role === 'player'); +} + +function playersSnapshot(): PlayerInfo[] { + return players().map( + ({ connection: _connection, role: _role, ...info }) => info + ); +} + +function stateMessage(): string { + const message: ServerMessage = { + type: 'state', + serverTime: serverNow(), + transport, + score: currentScore(), + players: playersSnapshot(), + scores: Object.values(scores).map(({ id, title }) => ({ id, title })), + }; + return JSON.stringify(message); +} + +function send(client: Client, message: ServerMessage): void { + client.connection.send(JSON.stringify(message)); +} + +function broadcastState(): void { + const message = stateMessage(); + for (const client of clients.values()) { + client.connection.send(message); + } +} + +function sendToConductors(message: string): void { + for (const client of clients.values()) { + if (client.role === 'conductor') { + client.connection.send(message); + } + } +} + +// The emptiest part that still has room, earliest part on ties, so every part +// gets a phone before any part gets two, and a part capped at `maxPlayers` is +// skipped once full. If every part is full, the cap is ignored. +function leastPopulatedPart(counts: number[]): number { + const { parts } = currentScore(); + const hasRoom = (index: number) => + counts[index] < (parts[index].maxPlayers ?? Number.POSITIVE_INFINITY); + + let best = -1; + for (let index = 0; index < parts.length; index += 1) { + if (hasRoom(index) && (best < 0 || counts[index] < counts[best])) { + best = index; + } + } + return best >= 0 ? best : counts.indexOf(Math.min(...counts)); +} + +function partCounts(except?: Client): number[] { + const counts = new Array(currentScore().parts.length).fill(0); + for (const player of players()) { + if (player !== except) { + for (const index of player.partIndices) { + counts[index] += 1; + } + } + } + return counts; +} + +function rebalanceParts(): void { + const counts = new Array(currentScore().parts.length).fill(0); + for (const player of players()) { + const part = leastPopulatedPart(counts); + player.partIndices = [part]; + counts[part] += 1; + } +} + +// Changing tempo mid-piece keeps the current beat position so phones do not +// jump back to the start. +function setBpm(bpm: number): void { + if (!(bpm > 20 && bpm < 400)) { + return; + } + if (transport.playing) { + const now = serverNow(); + const beatNow = ((now - transport.startAt) * transport.bpm) / 60000; + transport.startAt = now - (beatNow * 60000) / bpm; + } + transport.bpm = bpm; +} + +function handleMessage(client: Client, raw: string): void { + let message: ClientMessage; + try { + message = JSON.parse(raw) as ClientMessage; + } catch { + return; + } + + switch (message.type) { + case 'ping': { + const t1 = serverNow(); + send(client, { type: 'pong', t0: message.t0, t1, t2: serverNow() }); + return; + } + case 'hello': { + client.role = message.role; + if (message.role === 'player') { + client.name = message.name.slice(0, 24); + client.platform = message.platform; + client.outputLatencyMs = message.outputLatencyMs; + client.partIndices = [leastPopulatedPart(partCounts(client))]; + client.joinedAt = serverNow(); + console.log( + `+ ${client.name} (${client.platform}) → part ${client.partIndices[0]}` + ); + } + send(client, { + type: 'welcome', + playerId: client.id, + serverTime: serverNow(), + }); + broadcastState(); + return; + } + case 'report': { + client.offsetMs = message.offsetMs; + client.rttMs = message.rttMs; + client.uncertaintyMs = message.uncertaintyMs; + client.jitterMs = message.jitterMs; + client.outputLatencyMs = message.outputLatencyMs; + client.trimMs = message.trimMs; + client.reportedAt = serverNow(); + return; + } + default: + break; + } + + if (client.role !== 'conductor') { + return; + } + + switch (message.type) { + case 'start': + transport.playing = true; + transport.startAt = serverNow() + Math.max(MIN_COUNT_IN_MS, message.inMs); + transport.epoch += 1; + console.log( + `▶ start in ${(transport.startAt - serverNow()).toFixed(0)} ms, ${transport.bpm} bpm, ${scoreId}` + ); + break; + case 'stop': + transport.playing = false; + console.log('■ stop'); + break; + case 'setScore': + if (scores[message.scoreId] && message.scoreId !== scoreId) { + scoreId = message.scoreId; + transport.playing = false; + transport.bpm = currentScore().bpm; + rebalanceParts(); + } + break; + case 'setBpm': + setBpm(message.bpm); + break; + case 'assignParts': { + const target = clients.get(message.playerId); + const partCount = currentScore().parts.length; + const valid = message.partIndices.filter( + (index) => Number.isInteger(index) && index >= 0 && index < partCount + ); + if (target?.role === 'player') { + target.partIndices = [...new Set(valid)].sort((a, b) => a - b); + } + break; + } + case 'setMuted': { + const target = clients.get(message.playerId); + if (target?.role === 'player') { + target.muted = message.muted; + } + break; + } + case 'rebalance': + rebalanceParts(); + break; + default: + return; + } + + broadcastState(); +} + +const httpServer = createServer((request, response) => { + const url = new URL(request.url ?? '/', 'http://localhost'); + + // The dashboard's join code: the phone's camera opens the app on the + // Orchestra screen with the server URL filled in. + if (url.pathname === '/qr.svg') { + const data = url.searchParams.get('data') ?? ''; + response.writeHead(200, { + 'Content-Type': 'image/svg+xml', + 'Cache-Control': 'no-store', + }); + response.end(qrSvg(data)); + return; + } + const drum = url.pathname.match( + /^\/samples\/drums\/([a-zA-Z]+)\/(\d+)\.wav$/ + ); + if (drum) { + sampleStore.getDrum(drum[1], Number(drum[2])).then((body) => { + if (!body) { + response.writeHead(404); + response.end(); + return; + } + response.writeHead(200, { + 'Content-Type': 'audio/wav', + 'Cache-Control': 'public, max-age=86400', + }); + response.end(body); + }); + return; + } + // Phones load the notes of their recorded instrument from here. + const sample = url.pathname.match(/^\/samples\/([^/]+)\/([^/]+)\.mp3$/); + if (sample) { + sampleStore.get(sample[1], sample[2]).then((body) => { + if (!body) { + response.writeHead(404); + response.end(); + return; + } + response.writeHead(200, { + 'Content-Type': 'audio/mpeg', + 'Cache-Control': 'public, max-age=86400', + }); + response.end(body); + }); + return; + } + if (url.pathname === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end( + JSON.stringify({ players: players().length, transport, scoreId }) + ); + return; + } + response.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + response.end(dashboardHtml); +}); + +httpServer.on('upgrade', (request, socket) => { + const connection = acceptUpgrade(request, socket); + if (!connection) { + return; + } + + const client: Client = { + id: nextClientId++, + role: 'player', + connection, + partIndices: [0], + }; + clients.set(client.id, client); + + connection.on('message', (raw) => handleMessage(client, raw)); + connection.on('close', () => { + clients.delete(client.id); + if (client.name) { + console.log(`- ${client.name}`); + } + broadcastState(); + }); +}); + +setInterval(() => sendToConductors(stateMessage()), CONDUCTOR_REFRESH_MS); + +httpServer.listen(PORT, '0.0.0.0', () => { + const addresses = Object.values(networkInterfaces()) + .flat() + .filter((iface) => iface?.family === 'IPv4' && !iface.internal) + .map((iface) => iface!.address); + + console.log('Phone orchestra'); + for (const address of addresses) { + console.log(` phones: ws://${address}:${PORT}`); + console.log(` dashboard: http://${address}:${PORT}`); + } + void sampleStore.prefetch(Object.values(scores)); +}); diff --git a/apps/common-app/orchestra-server/tsconfig.json b/apps/common-app/orchestra-server/tsconfig.json new file mode 100644 index 000000000..04ca9ffb6 --- /dev/null +++ b/apps/common-app/orchestra-server/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "target": "es2022", + "lib": ["es2022", "dom"], + "types": ["node"] + }, + "include": ["*.ts", "../src/demos/Orchestra/protocol.ts"] +} diff --git a/apps/common-app/orchestra-server/websocket.ts b/apps/common-app/orchestra-server/websocket.ts new file mode 100644 index 000000000..47d2d9de6 --- /dev/null +++ b/apps/common-app/orchestra-server/websocket.ts @@ -0,0 +1,180 @@ +// Minimal RFC 6455 WebSocket server over node:http — enough for small JSON +// text messages (no extensions, no fragmentation across frames). Written +// instead of depending on `ws` so the conductor runs with a bare `node`. +import { createHash } from 'node:crypto'; +import { EventEmitter } from 'node:events'; +import type { IncomingMessage } from 'node:http'; +import type { Duplex } from 'node:stream'; + +const HANDSHAKE_GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; + +const OPCODE_TEXT = 0x1; +const OPCODE_CLOSE = 0x8; +const OPCODE_PING = 0x9; +const OPCODE_PONG = 0xa; + +interface Frame { + opcode: number; + payload: Buffer; + consumed: number; +} + +export class WebSocketConnection extends EventEmitter<{ + message: [text: string]; + close: []; +}> { + private readonly socket: Duplex; + private open = true; + private pending: Buffer = Buffer.alloc(0); + + constructor(socket: Duplex) { + super(); + this.socket = socket; + + socket.on('data', (chunk: Buffer) => this.consume(chunk)); + socket.on('close', () => this.finish()); + socket.on('error', () => this.finish()); + } + + send(text: string): void { + if (!this.open) { + return; + } + this.socket.write(encodeFrame(OPCODE_TEXT, Buffer.from(text, 'utf8'))); + } + + close(): void { + if (!this.open) { + return; + } + this.socket.end(encodeFrame(OPCODE_CLOSE, Buffer.alloc(0))); + this.finish(); + } + + private consume(chunk: Buffer): void { + this.pending = Buffer.concat([this.pending, chunk]); + + for (;;) { + const frame = decodeFrame(this.pending); + if (!frame) { + return; + } + this.pending = this.pending.subarray(frame.consumed); + + switch (frame.opcode) { + case OPCODE_TEXT: + this.emit('message', frame.payload.toString('utf8')); + break; + case OPCODE_PING: + this.socket.write(encodeFrame(OPCODE_PONG, frame.payload)); + break; + case OPCODE_CLOSE: + this.close(); + return; + default: + break; + } + } + } + + private finish(): void { + if (!this.open) { + return; + } + this.open = false; + this.socket.destroy(); + this.emit('close'); + } +} + +export function acceptUpgrade( + request: IncomingMessage, + socket: Duplex +): WebSocketConnection | null { + const key = request.headers['sec-websocket-key']; + if (!key || request.headers.upgrade?.toLowerCase() !== 'websocket') { + socket.end('HTTP/1.1 400 Bad Request\r\n\r\n'); + return null; + } + + const accept = createHash('sha1') + .update(key + HANDSHAKE_GUID) + .digest('base64'); + + socket.write( + [ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Accept: ${accept}`, + '', + '', + ].join('\r\n') + ); + + return new WebSocketConnection(socket); +} + +function encodeFrame(opcode: number, payload: Buffer): Buffer { + const length = payload.length; + let header: Buffer; + + if (length < 126) { + header = Buffer.from([0x80 | opcode, length]); + } else if (length < 0x10000) { + header = Buffer.alloc(4); + header[0] = 0x80 | opcode; + header[1] = 126; + header.writeUInt16BE(length, 2); + } else { + header = Buffer.alloc(10); + header[0] = 0x80 | opcode; + header[1] = 127; + header.writeBigUInt64BE(BigInt(length), 2); + } + + return Buffer.concat([header, payload]); +} + +function decodeFrame(buffer: Buffer): Frame | null { + if (buffer.length < 2) { + return null; + } + + const opcode = buffer[0] & 0x0f; + const masked = (buffer[1] & 0x80) !== 0; + let length = buffer[1] & 0x7f; + let offset = 2; + + if (length === 126) { + if (buffer.length < 4) { + return null; + } + length = buffer.readUInt16BE(2); + offset = 4; + } else if (length === 127) { + if (buffer.length < 10) { + return null; + } + length = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + + const maskLength = masked ? 4 : 0; + if (buffer.length < offset + maskLength + length) { + return null; + } + + const mask = masked ? buffer.subarray(offset, offset + 4) : null; + const payload = Buffer.from( + buffer.subarray(offset + maskLength, offset + maskLength + length) + ); + + if (mask) { + for (let i = 0; i < payload.length; i += 1) { + payload[i] ^= mask[i % 4]; + } + } + + return { opcode, payload, consumed: offset + maskLength + length }; +} diff --git a/apps/common-app/src/App.tsx b/apps/common-app/src/App.tsx index 966090bd1..0e984a766 100644 --- a/apps/common-app/src/App.tsx +++ b/apps/common-app/src/App.tsx @@ -17,11 +17,7 @@ import { createBottomTabNavigator } from '@react-navigation/bottom-tabs'; import { Spacer } from './components'; import Container from './components/Container'; import { demos, DemoScreen } from './demos'; -import { - Example, - Examples, - MainStackProps, -} from './examples'; +import { Example, Examples, MainStackProps } from './examples'; import { otherScreens } from './other'; import { colors, layout } from './styles'; @@ -186,6 +182,9 @@ const linking = { config: { screens: { RecordDemo: 'record', + // audioapi-example://orchestra?url=ws://host:8765 — what the conductor + // dashboard's QR code encodes. + Orchestra: 'orchestra', }, }, }; @@ -204,7 +203,8 @@ const App: FC = () => { headerTintColor: colors.white, headerBackTitle: 'Back', headerBackAccessibilityLabel: 'Go back', - }}> + }} + > | null = null; + + constructor(context: AudioContext, preferred: AudioClockSource) { + this.context = context; + this.preferred = preferred; + } + + setPreferredSource(source: AudioClockSource): void { + this.preferred = source; + } + + start(): void { + if (this.timer) { + return; + } + this.timer = setInterval(() => this.sample(), SAMPLE_INTERVAL_MS); + } + + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + this.currentTimeLeads = []; + this.outputTimestampLeads = []; + this.estimates = []; + } + + get isReady(): boolean { + return this.currentTimeLeads.length >= MIN_SAMPLES; + } + + get quality(): AudioClockQuality { + return { source: this.activeSource(), jitterMs: this.estimateJitter() }; + } + + /** Context time to pass to `start()` for the sound to be heard at `perfMs`. */ + startTimeHeardAt(perfMs: number): number { + if (!this.currentTimeLeads.length) { + this.sample(); + } + return (perfMs + this.heardLeadMs()) / 1000; + } + + private activeSource(): AudioClockSource { + const outputTimestampReady = + this.outputTimestampSupported && + this.outputTimestampLeads.length >= MIN_SAMPLES; + return this.preferred === 'outputTimestamp' && outputTimestampReady + ? 'outputTimestamp' + : 'currentTime'; + } + + // contextTime (ms) − perf of the sound being *heard*. + private heardLeadMs(): number { + if (this.activeSource() === 'outputTimestamp') { + return median(this.outputTimestampLeads.map((s) => s.leadMs)); + } + const renderedLeadMs = this.currentTimeLeads.reduce( + (best, s) => Math.max(best, s.leadMs), + Number.NEGATIVE_INFINITY + ); + return renderedLeadMs - this.context.outputLatency * 1000; + } + + private sample(): void { + const perf = performance.now(); + const horizon = perf - WINDOW_MS; + + this.currentTimeLeads.push({ + perf, + leadMs: this.context.currentTime * 1000 - perf, + }); + trim(this.currentTimeLeads, horizon); + + if (this.outputTimestampSupported) { + try { + const { contextTime, performanceTime } = + this.context.getOutputTimestamp(); + // Zero until the first block has been rendered. + if (contextTime > 0) { + this.outputTimestampLeads.push({ + perf, + leadMs: contextTime * 1000 - performanceTime, + }); + } + } catch { + // A native build that predates getOutputTimestamp(). + this.outputTimestampSupported = false; + } + trim(this.outputTimestampLeads, horizon); + } + + this.recordEstimate(perf); + } + + private recordEstimate(perf: number): void { + if (perf - this.lastEstimateAt < ESTIMATE_INTERVAL_MS || !this.isReady) { + return; + } + const source = this.activeSource(); + if (source !== this.estimateSource) { + this.estimates = []; + this.estimateSource = source; + } + this.lastEstimateAt = perf; + this.estimates.push(this.heardLeadMs()); + if (this.estimates.length > ESTIMATE_HISTORY) { + this.estimates.shift(); + } + } + + private estimateJitter(): number { + if (this.estimates.length < 2) { + return Number.NaN; + } + const center = median(this.estimates); + return median(this.estimates.map((e) => Math.abs(e - center))); + } +} + +function trim(samples: LeadSample[], horizon: number): void { + while (samples.length && samples[0].perf < horizon) { + samples.shift(); + } +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +} diff --git a/apps/common-app/src/demos/Orchestra/ClockSync.ts b/apps/common-app/src/demos/Orchestra/ClockSync.ts new file mode 100644 index 000000000..e69206678 --- /dev/null +++ b/apps/common-app/src/demos/Orchestra/ClockSync.ts @@ -0,0 +1,148 @@ +// NTP-style estimate of the server clock as seen from this phone. +// +// Each ping/pong gives one sample: offset = server − local and the round-trip +// time. Samples taken while the network was congested are useless, so only +// the lowest-RTT samples in the window vote (median), and a slow linear drift +// between the two clocks is fitted once the window spans enough time. +import type { ServerMessage } from './protocol'; + +interface Sample { + localTime: number; + offset: number; + rtt: number; +} + +export interface ClockSyncQuality { + /** + * Server − local. Meaningless on its own: the two clocks start at different + * moments. + */ + offsetMs: number; + rttMs: number; + /** + * Worst-case error of the offset: the reply may have spent anywhere in the + * round trip in flight. + */ + uncertaintyMs: number; + /** Median absolute deviation of the voting samples' offsets. */ + jitterMs: number; + driftPpm: number; + samples: number; +} + +const WINDOW_SIZE = 64; +/** Samples slower than best RTT + this margin do not vote. */ +const RTT_MARGIN_MS = 3; +/** Drift is only trusted over a span long enough to rise above RTT jitter. */ +const MIN_DRIFT_SPAN_MS = 20000; +const MAX_PLAUSIBLE_DRIFT_PPM = 500; + +export default class ClockSync { + private samples: Sample[] = []; + private offsetMs = 0; + private jitterMs = 0; + private driftPpm = 0; + private anchorLocalTime = 0; + + makePing(): { type: 'ping'; t0: number } { + return { type: 'ping', t0: performance.now() }; + } + + acceptPong(pong: Extract): void { + const t3 = performance.now(); + const { t0, t1, t2 } = pong; + + this.samples.push({ + localTime: t3, + offset: (t1 - t0 + (t2 - t3)) / 2, + rtt: t3 - t0 - (t2 - t1), + }); + if (this.samples.length > WINDOW_SIZE) { + this.samples.shift(); + } + + this.refit(); + } + + get isSynced(): boolean { + return this.samples.length >= 4; + } + + get quality(): ClockSyncQuality { + const rttMs = this.bestRtt(); + return { + offsetMs: this.offsetMs, + rttMs, + uncertaintyMs: rttMs / 2, + jitterMs: this.jitterMs, + driftPpm: this.driftPpm, + samples: this.samples.length, + }; + } + + localToServer(localMs: number): number { + return localMs + this.offsetAt(localMs); + } + + serverToLocal(serverMs: number): number { + // offset changes by at most ~1 ms over any realistic session, so one + // fixed-point step is enough to invert localToServer. + const approximateLocal = serverMs - this.offsetMs; + return serverMs - this.offsetAt(approximateLocal); + } + + private offsetAt(localMs: number): number { + return ( + this.offsetMs + ((localMs - this.anchorLocalTime) * this.driftPpm) / 1e6 + ); + } + + private bestRtt(): number { + return this.samples.reduce( + (best, sample) => Math.min(best, sample.rtt), + Number.POSITIVE_INFINITY + ); + } + + private refit(): void { + const threshold = this.bestRtt() + RTT_MARGIN_MS; + const voters = this.samples.filter((sample) => sample.rtt <= threshold); + + const offsets = voters.map((sample) => sample.offset).sort((a, b) => a - b); + const latest = voters[voters.length - 1]; + this.offsetMs = median(offsets); + this.jitterMs = median( + offsets.map((offset) => Math.abs(offset - this.offsetMs)) + ); + this.anchorLocalTime = latest.localTime; + + const span = latest.localTime - voters[0].localTime; + if (voters.length >= 8 && span >= MIN_DRIFT_SPAN_MS) { + const slope = leastSquaresSlope(voters); + this.driftPpm = + Math.abs(slope * 1e6) <= MAX_PLAUSIBLE_DRIFT_PPM ? slope * 1e6 : 0; + } else { + this.driftPpm = 0; + } + } +} + +function median(values: number[]): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +} + +function leastSquaresSlope(samples: Sample[]): number { + const n = samples.length; + const meanX = samples.reduce((sum, s) => sum + s.localTime, 0) / n; + const meanY = samples.reduce((sum, s) => sum + s.offset, 0) / n; + + let covariance = 0; + let variance = 0; + for (const { localTime, offset } of samples) { + covariance += (localTime - meanX) * (offset - meanY); + variance += (localTime - meanX) ** 2; + } + + return variance === 0 ? 0 : covariance / variance; +} diff --git a/apps/common-app/src/demos/Orchestra/Console.ts b/apps/common-app/src/demos/Orchestra/Console.ts new file mode 100644 index 000000000..a752dfc8d --- /dev/null +++ b/apps/common-app/src/demos/Orchestra/Console.ts @@ -0,0 +1,57 @@ +// The phone's performance console. The filter acts on the synth's output +// chain instantly; note controls (octave, stutter, voice) reshape each note +// as it is scheduled, up to one lookahead (~400 ms) later. +import type { Timbre } from './protocol'; + +export type Stutter = 'off' | '1/8' | '1/16'; +/** Recorded instruments the phone can switch to, by FluidR3_GM name. */ +export const INSTRUMENT_CHOICES = { + piano: 'acoustic_grand_piano', + strings: 'string_ensemble_1', + flute: 'flute', + trumpet: 'trumpet', + harp: 'orchestral_harp', + marimba: 'marimba', +} as const; + +/** + * `score` plays each part as written (its recorded instrument when it has one), + * `synth` forces the part's synthesized timbre. + */ +export type VoiceOverride = 'score' | 'synth' | keyof typeof INSTRUMENT_CHOICES; + +export interface ConsoleSettings { + /** 0..1, log-scaled to 150 Hz – 18 kHz. */ + cutoff: number; + /** 0..1. */ + resonance: number; + /** Whole octaves, -2..2. */ + octave: number; + stutter: Stutter; + voice: VoiceOverride; +} + +/** The score as written. */ +export const NEUTRAL_CONSOLE: ConsoleSettings = { + cutoff: 1, + resonance: 0, + octave: 0, + stutter: 'off', + voice: 'score', +}; + +export const STUTTER_STEP_BEATS: Record = { + off: null, + '1/8': 0.5, + '1/16': 0.25, +}; + +/** Percussion keeps its sound and pitch: a hi-hat as a pad is just noise. */ +export function isPercussion(timbre: Timbre): boolean { + return ( + timbre === 'kit' || + timbre === 'hat' || + timbre === 'clap' || + timbre === 'click' + ); +} diff --git a/apps/common-app/src/demos/Orchestra/ConsolePanel.tsx b/apps/common-app/src/demos/Orchestra/ConsolePanel.tsx new file mode 100644 index 000000000..14f891986 --- /dev/null +++ b/apps/common-app/src/demos/Orchestra/ConsolePanel.tsx @@ -0,0 +1,156 @@ +import React, { FC, memo, useMemo, useRef, useState } from 'react'; +import { + LayoutChangeEvent, + PanResponder, + Pressable, + StyleSheet, + Text, + View, +} from 'react-native'; + +import { colors, layout } from '../../styles'; +import { NEUTRAL_CONSOLE, type ConsoleSettings } from './Console'; + +interface ConsolePanelProps { + settings: ConsoleSettings; + onChange: (settings: ConsoleSettings) => void; + accent: string; +} + +const ConsolePanel: FC = ({ + settings, + onChange, + accent, +}) => { + const update = (patch: Partial) => + onChange({ ...settings, ...patch }); + + return ( + + + Console + onChange(NEUTRAL_CONSOLE)} hitSlop={8}> + Reset + + + + update({ cutoff, resonance })} + /> + + Filter: left darker, right brighter, up more resonance + + + ); +}; + +interface XYPadProps { + /** 0..1, left to right. */ + x: number; + /** 0..1, bottom to top. */ + y: number; + accent: string; + onChange: (x: number, y: number) => void; +} + +const PAD_HEIGHT = 120; +const KNOB_SIZE = 28; + +const XYPad: FC = ({ x, y, accent, onChange }) => { + const [width, setWidth] = useState(0); + const sizeRef = useRef({ width: 0 }); + const onChangeRef = useRef(onChange); + onChangeRef.current = onChange; + + const responder = useMemo(() => { + const report = (locationX: number, locationY: number) => { + const padWidth = sizeRef.current.width; + if (padWidth === 0) { + return; + } + // The knob's centre travels inset by its radius, so it never leaves the pad. + onChangeRef.current( + clamp01((locationX - KNOB_SIZE / 2) / (padWidth - KNOB_SIZE)), + clamp01(1 - (locationY - KNOB_SIZE / 2) / (PAD_HEIGHT - KNOB_SIZE)) + ); + }; + return PanResponder.create({ + onStartShouldSetPanResponder: () => true, + onMoveShouldSetPanResponder: () => true, + // Keep the gesture when the finger drifts, instead of scrolling the page. + onPanResponderTerminationRequest: () => false, + onPanResponderGrant: (event) => + report(event.nativeEvent.locationX, event.nativeEvent.locationY), + onPanResponderMove: (event) => + report(event.nativeEvent.locationX, event.nativeEvent.locationY), + }); + }, []); + + const onLayout = (event: LayoutChangeEvent) => { + sizeRef.current.width = event.nativeEvent.layout.width; + setWidth(event.nativeEvent.layout.width); + }; + + return ( + + + + ); +}; + +function clamp01(value: number): number { + return Math.min(1, Math.max(0, value)); +} + +// Memoised: the screen above re-renders on every status tick. +export default memo(ConsolePanel); + +const styles = StyleSheet.create({ + header: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + marginBottom: layout.spacing, + }, + heading: { + color: colors.white, + fontSize: 16, + fontWeight: '600', + }, + hint: { + color: colors.white, + opacity: 0.5, + fontSize: 12, + marginTop: 4, + }, + pad: { + height: PAD_HEIGHT, + borderRadius: layout.radius, + borderWidth: 1, + borderColor: colors.border, + backgroundColor: colors.backgroundLight, + overflow: 'hidden', + }, + knob: { + position: 'absolute', + width: KNOB_SIZE, + height: KNOB_SIZE, + borderRadius: KNOB_SIZE / 2, + }, + resetText: { + color: colors.main, + }, +}); diff --git a/apps/common-app/src/demos/Orchestra/InstrumentBank.ts b/apps/common-app/src/demos/Orchestra/InstrumentBank.ts new file mode 100644 index 000000000..5b06b1bfe --- /dev/null +++ b/apps/common-app/src/demos/Orchestra/InstrumentBank.ts @@ -0,0 +1,148 @@ +// Recorded instrument notes, fetched from the orchestra server's /samples +// mirror and decoded once: FluidR3_GM records every semitone, so a note plays +// its own sample at its natural speed; the drum kit has a few takes per piece, +// picked at random so repeated hits differ. +import type { AudioBuffer, BaseAudioContext } from 'react-native-audio-api'; + +import { DRUM_PIECES, DRUMS, drumPieceFor } from '../../../orchestra-server/drumKit'; + +const LOWEST_SAMPLED_MIDI = 21; +const HIGHEST_SAMPLED_MIDI = 108; +/** Decodes in flight at once: enough to fill a part quickly on a LAN. */ +const CONCURRENT_LOADS = 4; +// The renders spell every black key as a flat. +const NOTE_NAMES = [ + 'C', + 'Db', + 'D', + 'Eb', + 'E', + 'F', + 'Gb', + 'G', + 'Ab', + 'A', + 'Bb', + 'B', +]; + +export interface InstrumentRequest { + /** FluidR3_GM name, e.g. `violin`. */ + instrument: string; + midi: number; +} + +export function sampledMidi(midi: number): number { + return Math.min(HIGHEST_SAMPLED_MIDI, Math.max(LOWEST_SAMPLED_MIDI, midi)); +} + +interface Take { + sound: string; + url: string; +} + +// What a note sounds with: one buffer list per pitch, or per kit piece. +function soundKey(instrument: string, midi: number): string | null { + if (instrument !== DRUMS) { + return `${instrument}:${sampledMidi(midi)}`; + } + const piece = drumPieceFor(midi); + return piece ? `${DRUMS}:${piece}` : null; +} + +function takesOf(baseUrl: string, request: InstrumentRequest): Take[] { + const sound = soundKey(request.instrument, request.midi); + if (!sound) { + return []; + } + if (request.instrument === DRUMS) { + const piece = drumPieceFor(request.midi)!; + return Array.from({ length: DRUM_PIECES[piece].takes }, (_, index) => ({ + sound, + url: `${baseUrl}/samples/${DRUMS}/${piece}/${index + 1}.wav`, + })); + } + const note = sampledMidi(request.midi); + const name = `${NOTE_NAMES[note % 12]}${Math.floor(note / 12) - 1}`; + return [ + { sound, url: `${baseUrl}/samples/${request.instrument}/${name}.mp3` }, + ]; +} + +export default class InstrumentBank { + private readonly context: BaseAudioContext; + private readonly baseUrl: string | null; + private readonly sounds = new Map(); + /** + * Requested once; a failed load is not retried, so the synth keeps playing + * it. + */ + private readonly requested = new Set(); + private queue: Take[] = []; + private loading = 0; + private disposed = false; + + /** `baseUrl`: the orchestra server over http, or null to play synth only. */ + constructor(context: BaseAudioContext, baseUrl: string | null) { + this.context = context; + this.baseUrl = baseUrl; + } + + /** A loaded take for this note, or null while none has loaded. */ + get(instrument: string, midi: number): AudioBuffer | null { + const sound = soundKey(instrument, midi); + const takes = sound ? this.sounds.get(sound) : undefined; + if (!takes || takes.length === 0) { + return null; + } + return takes[Math.floor(Math.random() * takes.length)]; + } + + /** Fetches and decodes, in the background, every take not loaded yet. */ + prepare(requests: readonly InstrumentRequest[]): void { + if (!this.baseUrl) { + return; + } + for (const request of requests) { + for (const take of takesOf(this.baseUrl, request)) { + if (!this.requested.has(take.url)) { + this.requested.add(take.url); + this.queue.push(take); + } + } + } + while (this.loading < CONCURRENT_LOADS && this.queue.length > 0) { + this.loadNext(); + } + } + + dispose(): void { + this.disposed = true; + this.queue = []; + this.sounds.clear(); + } + + private async loadNext(): Promise { + const take = this.queue.shift(); + if (!take) { + return; + } + this.loading += 1; + try { + const buffer = await this.context.decodeAudioData(take.url); + if (!this.disposed) { + this.sounds.set(take.sound, [ + ...(this.sounds.get(take.sound) ?? []), + buffer, + ]); + } + } catch (error) { + console.warn(`Orchestra: could not load ${take.url}`, error); + } finally { + this.loading -= 1; + if (!this.disposed && this.queue.length > 0) { + this.loadNext(); + } + } + } +} diff --git a/apps/common-app/src/demos/Orchestra/NoteRoll.tsx b/apps/common-app/src/demos/Orchestra/NoteRoll.tsx new file mode 100644 index 000000000..87b261ad3 --- /dev/null +++ b/apps/common-app/src/demos/Orchestra/NoteRoll.tsx @@ -0,0 +1,506 @@ +// What this phone plays, made readable on camera: a piano roll of its parts +// scrolling past a fixed playhead, with the notes it is sounding lit. Pitch +// runs up the screen one row per semitone, so leaps look like leaps. +import React, { FC, memo, useEffect, useMemo, useState } from 'react'; +import { LayoutChangeEvent, StyleSheet, Text, View } from 'react-native'; +import Animated, { + cancelAnimation, + Easing, + SharedValue, + useAnimatedStyle, + useSharedValue, + withSequence, + withTiming, +} from 'react-native-reanimated'; + +import { colors } from '../../styles'; +import { DRUM_PIECES, drumPieceFor, type DrumPiece } from '../../../orchestra-server/drumKit'; +import type { NoteListener, PartNote } from './Scheduler'; +import type { ScorePart } from './protocol'; + +const BEAT_WIDTH = 56; +const BEATS_PER_BAR = 4; +/** Where "now" sits, as a fraction of the roll's width. */ +const PLAYHEAD_AT = 0.3; +const GUTTER_WIDTH = 52; +/** Labels keep this height however thin the rows get. */ +const LABEL_HEIGHT = 12; +/** Even a staccato note stays lit long enough to be seen. */ +const MIN_LIT_MS = 120; + +const PAST_OPACITY = 0.25; +const UPCOMING_OPACITY = 0.6; + +const NOTE_NAMES = [ + 'C', + 'C♯', + 'D', + 'D♯', + 'E', + 'F', + 'F♯', + 'G', + 'G♯', + 'A', + 'A♯', + 'B', +]; +export function noteName(midi: number): string { + return `${NOTE_NAMES[midi % 12]}${Math.floor(midi / 12) - 1}`; +} + +interface NoteRollProps { + notes: readonly PartNote[]; + parts: readonly ScorePart[]; + lengthBeats: number; + bpm: number; + playing: boolean; + /** Changes on every transport start. */ + epoch: number; + subscribeToBeats: (listener: (beat: number) => void) => () => void; + subscribeToNotes: (listener: NoteListener) => () => void; +} + +const NoteRoll: FC = ({ + notes, + parts, + lengthBeats, + bpm, + playing, + epoch, + subscribeToBeats, + subscribeToNotes, +}) => { + const [size, setSize] = useState({ width: 0, height: 0 }); + const [beat, setBeat] = useState(0); + const [lit, setLit] = useState>(new Set()); + const position = useSharedValue(0); + const msPerBeat = 60000 / bpm; + + // Every transport start begins from the top of the score. + useEffect(() => { + cancelAnimation(position); + position.value = 0; + setBeat(0); + setLit(new Set()); + }, [epoch, position]); + + useEffect(() => { + if (!playing) { + cancelAnimation(position); + } + }, [playing, position]); + + // Each beat glides the roll to the next one; a phone that joined late, or a + // late timer, snaps first so the playhead never drifts off the music. + useEffect( + () => + subscribeToBeats((heardBeat) => { + setBeat(heardBeat); + const glide = withTiming(heardBeat + 1, { + duration: msPerBeat, + easing: Easing.linear, + }); + position.value = + Math.abs(position.value - heardBeat) > 0.5 + ? withSequence(withTiming(heardBeat, { duration: 0 }), glide) + : glide; + }), + [subscribeToBeats, msPerBeat, position] + ); + + useEffect(() => { + const timers = new Set>(); + const unsubscribe = subscribeToNotes((cue) => { + setLit((current) => new Set(current).add(cue.key)); + const timer = setTimeout( + () => { + timers.delete(timer); + setLit((current) => { + const next = new Set(current); + next.delete(cue.key); + return next; + }); + }, + Math.max(MIN_LIT_MS, cue.durationMs) + ); + timers.add(timer); + }); + return () => { + unsubscribe(); + timers.forEach(clearTimeout); + }; + }, [subscribeToNotes]); + + const onLayout = (event: LayoutChangeEvent) => { + const { width, height } = event.nativeEvent.layout; + setSize({ width, height }); + }; + + const rollWidth = Math.max(0, size.width - GUTTER_WIDTH); + const playheadX = rollWidth * PLAYHEAD_AT; + const scrollStyle = useAnimatedStyle(() => ({ + transform: [{ translateX: playheadX - position.value * BEAT_WIDTH }], + })); + + const grid = useMemo(() => gridFor(notes), [notes]); + const rowHeight = grid ? size.height / grid.rows.length : 0; + const rowTop = (midi: number) => (grid?.rowOf.get(midi) ?? 0) * rowHeight; + + const firstBeat = Math.floor(beat - playheadX / BEAT_WIDTH) - 1; + const lastBeat = Math.ceil(beat + (rollWidth - playheadX) / BEAT_WIDTH) + 2; + const visible = useMemo( + () => notesBetween(notes, lengthBeats, firstBeat, lastBeat), + [notes, lengthBeats, firstBeat, lastBeat] + ); + + return ( + + + {grid && size.height > 0 && ( + <> + + + + {beatLines(firstBeat, lastBeat, size.height)} + {visible.map(({ note, key, startBeat }) => ( + + ))} + + + + + )} + {!grid && no notes in this part} + + + ); +}; + +interface NoteBarProps { + left: number; + width: number; + top: number; + height: number; + color: string; + endBeat: number; + isLit: boolean; + position: SharedValue; +} + +// Dims on the UI thread the frame its end crosses the playhead, rather than +// on the next beat cue. +const NoteBar = memo(function NoteBar({ + left, + width, + top, + height, + color, + endBeat, + isLit, + position, +}) { + const dimStyle = useAnimatedStyle(() => ({ + opacity: isLit + ? 1 + : position.value >= endBeat + ? PAST_OPACITY + : UPCOMING_OPACITY, + })); + return ( + + ); +}); + +// Faint rows, labelled on the left: semitones (stronger on every C, black +// keys shaded) so intervals can be read off the grid, or one row per drum. +const RowGrid = memo<{ + rows: readonly GridRow[]; + rowHeight: number; + width: number; +}>(function RowGrid({ rows, rowHeight, width }) { + // Every label fits once rows are as tall as the text; thinner rows label + // only their octaves (Cs), which sit twelve rows apart, centred on the row + // and spilling over its neighbours. + const labelEveryRow = rowHeight >= LABEL_HEIGHT; + const height = rows.length * rowHeight; + return ( + + {rows.map((row, index) => ( + + ))} + {rows.map((row, index) => + row.label !== null && (labelEveryRow || row.shade === 'octave') ? ( + + {row.label} + + ) : null + )} + + ); +}); + +function beatLines(firstBeat: number, lastBeat: number, height: number) { + const lines = []; + for (let beat = Math.max(0, firstBeat); beat <= lastBeat; beat += 1) { + const isBar = beat % BEATS_PER_BAR === 0; + lines.push( + + {isBar && ( + {beat / BEATS_PER_BAR + 1} + )} + + ); + } + return lines; +} + +interface GridRow { + id: string; + label: string | null; + shade?: 'blackKey' | 'octave'; +} + +interface Grid { + /** Top to bottom. */ + rows: GridRow[]; + /** Row index of each midi value (a drum piece's keys share one row). */ + rowOf: Map; +} + +// Cymbals on top, kick at the bottom, like a drum chart. +const DRUM_ROW_ORDER: readonly DrumPiece[] = [ + 'crash', + 'ride', + 'openHat', + 'closedHat', + 'highTom', + 'lowTom', + 'clap', + 'snare', + 'kick', +]; + +function gridFor(notes: readonly PartNote[]): Grid | null { + if (notes.length === 0) { + return null; + } + return notes.every((note) => note.timbre === 'kit') + ? drumGrid(notes) + : pitchGrid(notes); +} + +function pitchGrid(notes: readonly PartNote[]): Grid { + let low = Number.POSITIVE_INFINITY; + let high = Number.NEGATIVE_INFINITY; + for (const note of notes) { + low = Math.min(low, note.midi - 1); + high = Math.max(high, note.midi + 1); + } + const labelOctavesOnly = high - low > 24; + const rows: GridRow[] = []; + const rowOf = new Map(); + for (let midi = high; midi >= low; midi -= 1) { + const isC = midi % 12 === 0; + rowOf.set(midi, rows.length); + rows.push({ + id: `pitch-${midi}`, + label: !labelOctavesOnly || isC ? noteName(midi) : null, + shade: isC + ? 'octave' + : [1, 3, 6, 8, 10].includes(midi % 12) + ? 'blackKey' + : undefined, + }); + } + return { rows, rowOf }; +} + +function drumGrid(notes: readonly PartNote[]): Grid { + const used = new Set( + notes.map((note) => drumPieceFor(note.midi)).filter((piece) => piece) + ); + const pieces = DRUM_ROW_ORDER.filter((piece) => used.has(piece)); + const rowOf = new Map(); + pieces.forEach((piece, index) => { + for (const key of DRUM_PIECES[piece].keys) { + rowOf.set(key, index); + } + }); + return { + rows: pieces.map((piece) => ({ + id: `drum-${piece}`, + label: DRUM_PIECES[piece].label, + })), + rowOf, + }; +} + +// The notes that overlap [firstBeat, lastBeat] on the transport's beat line, +// across as many loops of the score as the window spans. +function notesBetween( + notes: readonly PartNote[], + lengthBeats: number, + firstBeat: number, + lastBeat: number +) { + const visible: { note: PartNote; key: string; startBeat: number }[] = []; + if (lengthBeats <= 0) { + return visible; + } + const firstCycle = Math.max(0, Math.floor(firstBeat / lengthBeats) - 1); + const lastCycle = Math.floor(lastBeat / lengthBeats); + for (let cycle = firstCycle; cycle <= lastCycle; cycle += 1) { + notes.forEach((note, index) => { + const startBeat = cycle * lengthBeats + note.beat; + if (startBeat <= lastBeat && startBeat + note.duration >= firstBeat) { + visible.push({ note, key: `${cycle}:${index}`, startBeat }); + } + }); + } + return visible; +} + +export default NoteRoll; + +const styles = StyleSheet.create({ + container: { + flex: 1, + }, + roll: { + flex: 1, + borderRadius: 8, + overflow: 'hidden', + backgroundColor: colors.backgroundDark, + }, + window: { + position: 'absolute', + top: 0, + right: 0, + bottom: 0, + left: GUTTER_WIDTH, + overflow: 'hidden', + }, + scroller: { + position: 'absolute', + top: 0, + right: 0, + bottom: 0, + left: 0, + }, + row: { + position: 'absolute', + left: 0, + borderBottomWidth: StyleSheet.hairlineWidth, + borderBottomColor: '#ffffff14', + }, + blackKeyRow: { + backgroundColor: '#00000033', + }, + cRow: { + borderBottomColor: '#ffffff40', + }, + rowLabel: { + position: 'absolute', + left: 0, + width: GUTTER_WIDTH - 4, + height: LABEL_HEIGHT, + lineHeight: LABEL_HEIGHT, + fontSize: 10, + color: colors.white, + opacity: 0.55, + textAlign: 'right', + }, + beatLine: { + position: 'absolute', + top: 0, + width: StyleSheet.hairlineWidth, + backgroundColor: '#ffffff1a', + }, + barLine: { + width: 1, + backgroundColor: '#ffffff40', + }, + barNumber: { + position: 'absolute', + top: 2, + left: 4, + width: 30, + color: colors.white, + opacity: 0.5, + fontSize: 10, + }, + note: { + position: 'absolute', + borderRadius: 3, + }, + litNote: { + borderWidth: 1, + borderColor: colors.white, + }, + playhead: { + position: 'absolute', + top: 0, + bottom: 0, + width: 2, + backgroundColor: colors.white, + opacity: 0.9, + }, + empty: { + color: colors.white, + opacity: 0.5, + textAlign: 'center', + marginTop: 24, + }, +}); diff --git a/apps/common-app/src/demos/Orchestra/Orchestra.tsx b/apps/common-app/src/demos/Orchestra/Orchestra.tsx new file mode 100644 index 000000000..d3f832ec0 --- /dev/null +++ b/apps/common-app/src/demos/Orchestra/Orchestra.tsx @@ -0,0 +1,351 @@ +import React, { + FC, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { + NativeModules, + Platform, + ScrollView, + StyleSheet, + Text, + TextInput, + View, +} from 'react-native'; +import { AudioContext, AudioManager } from 'react-native-audio-api'; +import { RouteProp, useRoute } from '@react-navigation/native'; + +import { Button, Container, Spacer } from '../../components'; +import { colors, layout } from '../../styles'; +import AudioClockMap from './AudioClockMap'; +import OrchestraClient, { OrchestraSnapshot } from './OrchestraClient'; +import Scheduler from './Scheduler'; +import Synth from './Synth'; +import { NEUTRAL_CONSOLE, type ConsoleSettings } from './Console'; +import ConsolePanel from './ConsolePanel'; +import NoteRoll from './NoteRoll'; +import { mergeParts, type NoteListener } from './Scheduler'; +import type { Score, ScorePart } from './protocol'; + +const SERVER_PORT = 8765; + +// The server usually runs on the laptop that serves the JS bundle, so its +// host is the best default. Inputs survive navigating away within a session. +let lastServerUrl = `ws://${metroHost() ?? '192.168.0.10'}:${SERVER_PORT}`; +let lastName = `${Platform.OS}-${Math.random().toString(36).slice(2, 6)}`; + +function metroHost(): string | null { + const sourceCode = NativeModules.SourceCode as + | { scriptURL?: string; getConstants?: () => { scriptURL?: string } } + | undefined; + const scriptURL = + sourceCode?.getConstants?.().scriptURL ?? sourceCode?.scriptURL; + const match = scriptURL?.match(/^https?:\/\/([^/:]+)/); + return match ? match[1] : null; +} + +interface Engine { + context: AudioContext; + clockMap: AudioClockMap; + synth: Synth; + scheduler: Scheduler; + client: OrchestraClient; +} + +type OrchestraRoute = RouteProp< + { Orchestra: { url?: string; name?: string } | undefined }, + 'Orchestra' +>; + +const IDLE_SNAPSHOT: OrchestraSnapshot = { + status: 'disconnected', + playerId: null, + state: null, + me: null, +}; + +const Orchestra: FC = () => { + const route = useRoute(); + const scannedUrl = route.params?.url; + const [serverUrl, setServerUrl] = useState(scannedUrl ?? lastServerUrl); + const [name, setName] = useState(route.params?.name ?? lastName); + const [snapshot, setSnapshot] = useState(IDLE_SNAPSHOT); + // Beat and note cues go straight to the roll, so they never + // re-render the whole screen. + const beatListeners = useRef(new Set<(beat: number) => void>()); + const noteListeners = useRef(new Set()); + const [consoleSettings, setConsoleSettings] = + useState(NEUTRAL_CONSOLE); + + const engine = useRef(null); + const nameRef = useRef(name); + nameRef.current = name; + + useEffect(() => { + lastServerUrl = serverUrl; + lastName = name; + }, [serverUrl, name]); + + const disconnect = useCallback(() => { + const current = engine.current; + engine.current = null; + if (!current) { + return; + } + current.client.disconnect(); + current.scheduler.stop(); + current.clockMap.stop(); + current.synth.dispose(); + current.context.close(); + setSnapshot(IDLE_SNAPSHOT); + }, []); + + const connectTo = useCallback( + async (url: string) => { + disconnect(); + + AudioManager.setSystemOptions({ + ios: { + category: 'playback', + mode: 'default', + options: [], + }, + }); + await AudioManager.setAudioSessionActivity(true); + + // Small hardware buffers keep the gap between rendering a note and + // hearing it short, and getOutputTimestamp() reports that gap exactly. + const context = new AudioContext({ latencyHint: 'balanced' }); + if (context.state === 'suspended') { + await context.resume(); + } + + const client = new OrchestraClient({ + url: url.trim(), + name: nameRef.current.trim() || lastName, + platform: Platform.OS, + readLocal: () => ({ + outputLatencyMs: context.outputLatency * 1000, + trimMs: 0, + }), + onChange: setSnapshot, + }); + const clockMap = new AudioClockMap(context, 'outputTimestamp'); + // Recorded instruments come from the same server, over http. + const synth = new Synth( + context, + url + .trim() + .replace(/^ws(s?):\/\//, 'http$1://') + .replace(/\/+$/, '') + ); + const scheduler = new Scheduler( + context, + client.clockSync, + clockMap, + synth + ); + + scheduler.onBeat((beat) => { + beatListeners.current.forEach((listener) => listener(beat)); + }); + scheduler.onNote((cue) => { + noteListeners.current.forEach((listener) => listener(cue)); + }); + + engine.current = { context, clockMap, synth, scheduler, client }; + clockMap.start(); + scheduler.start(); + client.connect(); + }, + [disconnect] + ); + + const connect = useCallback( + () => connectTo(serverUrl), + [connectTo, serverUrl] + ); + + // A scanned QR code deep-links here with the server URL: connect without a + // tap, and again whenever a new code is scanned. + useEffect(() => { + if (scannedUrl) { + setServerUrl(scannedUrl); + connectTo(scannedUrl); + } + }, [scannedUrl, connectTo]); + + useEffect(() => disconnect, [disconnect]); + + // Hand every state change (transport, parts, tempo) and the console's note + // controls to the scheduler. + useEffect(() => { + const current = engine.current; + const { state, me } = snapshot; + if (!current || !state || !me) { + return; + } + current.scheduler.configure({ + transport: state.transport, + score: state.score, + partIndices: me.partIndices, + console: consoleSettings, + muted: me.muted ?? false, + trimMs: 0, + }); + }, [snapshot, consoleSettings]); + + // Output controls act on the synth directly, so they are heard at once. + const { cutoff, resonance } = consoleSettings; + useEffect(() => { + engine.current?.synth.setFilter(cutoff, resonance); + }, [cutoff, resonance, snapshot.status]); + + const muted = snapshot.me?.muted ?? false; + useEffect(() => { + engine.current?.synth.setMuted(muted); + }, [muted]); + + const transport = snapshot.state?.transport; + const subscribeToBeats = useCallback((listener: (beat: number) => void) => { + const listeners = beatListeners.current; + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, []); + const subscribeToNotes = useCallback((listener: NoteListener) => { + const listeners = noteListeners.current; + listeners.add(listener); + return () => { + listeners.delete(listener); + }; + }, []); + + const connected = snapshot.status !== 'disconnected'; + const parts = + snapshot.state && snapshot.me + ? myParts(snapshot.state.score, snapshot.me.partIndices) + : []; + const accent = parts[0]?.color ?? colors.main; + const score = snapshot.state?.score; + const partIndicesKey = snapshot.me?.partIndices.join(',') ?? ''; + const myNotes = useMemo( + () => + score && partIndicesKey + ? mergeParts(score, partIndicesKey.split(',').map(Number)) + : [], + [score, partIndicesKey] + ); + if (connected) { + return ( + + + + + {/* Not in a ScrollView: dragging the XY pad would fight the scroll. */} + + + + ); + } + + return ( + + + Server + + + Name + + +