diff --git a/.github/workflows/cpu-unit-tests.yml b/.github/workflows/cpu-unit-tests.yml index d8efd53..449af90 100644 --- a/.github/workflows/cpu-unit-tests.yml +++ b/.github/workflows/cpu-unit-tests.yml @@ -19,7 +19,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install torch==2.11.0 --index-url https://download.pytorch.org/whl/cpu - python -m pip install numpy==1.26.4 huggingface-hub==1.27.0 soundfile==0.14.0 + python -m pip install numpy==1.26.4 scipy==1.13.1 huggingface-hub==1.27.0 soundfile==0.14.0 - name: Compile and run unit tests run: | python -m py_compile inference.py tests/test_inference.py diff --git a/TEXTURE_SOFTENING.md b/TEXTURE_SOFTENING.md new file mode 100644 index 0000000..78fdbe3 --- /dev/null +++ b/TEXTURE_SOFTENING.md @@ -0,0 +1,39 @@ +# Optional experimental texture softening + +Apollo remains by Kai Li and Yi Luo / Look2Hear. This Neuracoust contribution +is adapted from Sound Build-up post-processing; the repository CC BY-SA 4.0 +license and upstream attribution apply. It changes neither weights nor training. + +```sh +python inference.py --in_wav=input.wav --out_wav=output.wav --texture-reduction-db=2 +``` + +The option defaults to **0 (off)** and accepts 0–3 dB. When enabled, a +stereo-linked NumPy/SciPy processor runs once on the existing Apollo prediction, +after either full-file or chunked inference; it adds no model forward passes. +Install SciPy in the inference environment (`python -m pip install scipy`). +Enabled processing checks SciPy before audio/checkpoint loading and rejects +inputs shorter than 2048 frames before checkpoint/model loading. With the +option disabled, SciPy is not required and the existing short-input path remains. + +The processor attenuates short positive spectral excursions against a roughly +40 ms temporal median, tapering between 1.2 and 14 kHz with a broad attack gate. +The cap bounds STFT gain reduction, not every time-domain sample difference. +This is a heuristic, not an artifact/vocal detector: wanted detail, modulation, +and transients can be suppressed. Keep this experimental and opt-in. + +A full-output STFT makes memory **O(duration)** even with chunked model +inference. Historical model-excluded probes measured 388.74 MiB tracked peak / +648.77 MiB process RSS for 30 s, and 2328.58 / 3042.25 MiB for a 180 s repeated +Apollo prediction. The latter is an allocation probe, not a full-song listening +test; these host-dependent figures are not new measurements of this review copy. + +Run `python -m unittest discover -s tests -v` for DSP and CLI regression tests. +The CLI tests mock model loading; they do not validate a checkpoint or listening +quality. [Existing texture-only audio bundle](https://github.com/hemanpapa-coder/Apollo/releases/tag/texture-evaluation-20261016) +and [fixed detailed evidence](https://github.com/hemanpapa-coder/Apollo/blob/f160a8a1e267626da2136fc20a487eef99f8cf28/evaluation/texture-20261016/REPORT.md) +provide eight aligned 15 s excerpts, raw 0/1/2/3 dB WAVs, loudness-matched blind +copies, commands and provenance. Sources are lossy proxies, not lossless clean +references; controlled listening and maintainer evaluation remain outstanding. +No universal quality improvement is claimed. The full enhancement chain is +outside this feature's scope and remains separate fork experiment material. diff --git a/inference.py b/inference.py index 356bda4..936d2d2 100644 --- a/inference.py +++ b/inference.py @@ -1,4 +1,5 @@ import argparse +import importlib from pathlib import Path from huggingface_hub import hf_hub_download @@ -221,6 +222,20 @@ def run_model( return output_sum / weight_sum +def load_texture_processor(): + """Check optional dependencies before checkpoint download/model execution.""" + try: + importlib.import_module("scipy") + return importlib.import_module("texture_softening").soften_texture + except ModuleNotFoundError as exc: + if exc.name == "scipy" or (exc.name or "").startswith("scipy."): + raise RuntimeError( + "Texture softening requires SciPy. Install scipy>=1.13,<2 " + "or use --texture-reduction-db=0. No model was loaded." + ) from exc + raise + + def run_inference( input_wav, output_wav, @@ -230,7 +245,10 @@ def run_inference( overlap_seconds=1.0, chunk_batch_size=1, chunk_pad_seconds=0.0, + texture_reduction_db=0.0, ): + if not np.isfinite(texture_reduction_db) or not 0 <= texture_reduction_db <= 3: + raise ValueError("Texture reduction must be finite and between 0 and 3 dB.") input_path = Path(input_wav) output_path = Path(output_wav) if input_path.resolve() == output_path.resolve(): @@ -242,8 +260,14 @@ def run_inference( chunk_seconds, overlap_seconds, chunk_pad_seconds ) chunk_batch_size = validate_chunk_batch_size(chunk_batch_size) + texture_processor = load_texture_processor() if texture_reduction_db else None device = select_device(requested_device) test_data, sample_rate = load_audio(input_path) + if texture_processor is not None and test_data.shape[-1] < 2048: + raise ValueError( + "Texture softening requires at least 2048 frames; " + "use --texture-reduction-db=0 for shorter clips. No model was loaded." + ) checkpoint_path = resolve_checkpoint(checkpoint) model = look2hear.models.BaseModel.from_pretrain( @@ -263,6 +287,11 @@ def run_inference( chunk_batch_size=chunk_batch_size, chunk_pad_samples=chunk_pad_samples, ) + if texture_reduction_db: + softened, _ = texture_processor( + output.squeeze(0).numpy(), sample_rate, texture_reduction_db + ) + output = torch.from_numpy(softened).unsqueeze(0) save_audio(output_path, output, sample_rate) return device @@ -310,6 +339,10 @@ def main(): default=1, help="Chunks per model forward pass (default: 1; higher uses more memory)", ) + parser.add_argument( + "--texture-reduction-db", type=float, default=0.0, + help="Experimental texture softening cap, 0-3 dB (default: 0, disabled)", + ) args = parser.parse_args() device = run_inference( @@ -321,6 +354,7 @@ def main(): args.overlap_seconds, args.chunk_batch_size, args.chunk_pad_seconds, + args.texture_reduction_db, ) print(f"Inference completed on {device}: {args.out_wav}") diff --git a/tests/test_texture_inference.py b/tests/test_texture_inference.py new file mode 100644 index 0000000..c39307c --- /dev/null +++ b/tests/test_texture_inference.py @@ -0,0 +1,76 @@ +"""CLI integration without checkpoint download or real model inference.""" +import tempfile +from pathlib import Path +import unittest +from unittest import mock +import numpy as np +import soundfile as sf +import torch +import inference + + +class IntegrationTests(unittest.TestCase): + def test_optional_postprocessing_and_default_bypass(self): + t = np.arange(4410)/44100 + x = np.array([.4*np.sin(2*np.pi*220*t) + .02*(.1+(np.mod(t,.08)<.008))*np.sin(2*np.pi*4000*t)], dtype='float32') + from texture_softening import soften_texture + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source, dest, checkpoint = root/'in.wav', root/'out.wav', root/'model.pt' + checkpoint.touch() + sf.write(source, x.T, 44100, subtype='FLOAT') + with mock.patch.object(inference.look2hear.models.BaseModel, 'from_pretrain', return_value=torch.nn.Identity()): + with mock.patch('texture_softening.soften_texture', wraps=soften_texture) as process: + inference.run_inference(source, dest, checkpoint, 'cpu') + process.assert_not_called() + y, _ = sf.read(dest, dtype='float32', always_2d=True) + np.testing.assert_array_equal(y.T, x) + inference.run_inference(source, dest, checkpoint, 'cpu', texture_reduction_db=2) + process.assert_called_once() + y, _ = sf.read(dest, dtype='float32', always_2d=True) + expected, _ = soften_texture(x) + np.testing.assert_array_equal(y.T, expected) + original, _ = sf.read(source, dtype='float32', always_2d=True) + np.testing.assert_array_equal(original.T, x) + + def test_invalid_cap_rejected_before_model_load(self): + for cap in (-1, 4, np.nan, np.inf): + with self.assertRaises(ValueError): + inference.run_inference('unused.wav', 'out.wav', 'unused.pt', texture_reduction_db=cap) + + def test_missing_scipy_fails_before_audio_checkpoint_or_model(self): + missing = ModuleNotFoundError("No module named scipy", name="scipy") + with mock.patch.object(inference.importlib, 'import_module', side_effect=missing), \ + mock.patch.object(inference, 'load_audio') as audio, \ + mock.patch.object(inference, 'resolve_checkpoint') as checkpoint, \ + mock.patch.object(inference.look2hear.models.BaseModel, 'from_pretrain') as model: + with self.assertRaisesRegex(RuntimeError, 'requires SciPy'): + inference.run_inference('in.wav', 'out.wav', 'model.pt', 'cpu', texture_reduction_db=2) + audio.assert_not_called() + checkpoint.assert_not_called() + model.assert_not_called() + + def test_short_clip_rejected_before_checkpoint_but_disabled_still_works(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source, dest = root/'short.wav', root/'out.wav' + x = np.zeros((1024, 1), dtype='float32') + sf.write(source, x, 44100, subtype='FLOAT') + with mock.patch.object(inference, 'resolve_checkpoint') as checkpoint, \ + mock.patch.object(inference.look2hear.models.BaseModel, 'from_pretrain', return_value=torch.nn.Identity()) as model: + with self.assertRaisesRegex(ValueError, 'at least 2048'): + inference.run_inference(source, dest, 'model.pt', 'cpu', texture_reduction_db=2) + checkpoint.assert_not_called() + model.assert_not_called() + self.assertFalse(dest.exists()) + with mock.patch.object(inference, 'load_texture_processor') as optional: + inference.run_inference(source, dest, 'model.pt', 'cpu') + optional.assert_not_called() + y, _ = sf.read(dest, dtype='float32', always_2d=True) + np.testing.assert_array_equal(y, x) + + def test_unrelated_import_errors_are_not_misreported(self): + missing = ModuleNotFoundError('missing other', name='other') + with mock.patch.object(inference.importlib, 'import_module', side_effect=missing): + with self.assertRaises(ModuleNotFoundError): + inference.load_texture_processor() diff --git a/tests/test_texture_softening.py b/tests/test_texture_softening.py new file mode 100644 index 0000000..8740c29 --- /dev/null +++ b/tests/test_texture_softening.py @@ -0,0 +1,45 @@ +import unittest +import numpy as np +from texture_softening import soften_texture + + +class TextureTests(unittest.TestCase): + def test_exact_bypass(self): + x = np.random.default_rng(4).normal(0, .1, (2, 16000)).astype('float32') + y, _ = soften_texture(x, max_reduction_db=0) + np.testing.assert_array_equal(x, y) + + def test_steady_tone_stereo_and_length(self): + a = .2*np.sin(2*np.pi*3000*np.arange(44101)/44100) + x = np.array([a, -.3*a], dtype='float32') + y, _ = soften_texture(x) + self.assertEqual(y.shape, x.shape) + np.testing.assert_allclose(y[1], -.3*y[0], atol=1e-7) + self.assertLess(np.sqrt(np.mean((x[:,2048:-2048]-y[:,2048:-2048])**2)), 1e-5) + + def test_silence(self): + y, _ = soften_texture(np.zeros((2, 4410), dtype='float32')) + self.assertEqual(float(abs(y).max()), 0) + + def test_actual_modulation_is_bounded(self): + t = np.arange(44100)/44100 + envelope = .02*(.1 + (np.mod(t, .08) < .008)) + x = np.array([.4*np.sin(2*np.pi*220*t) + envelope*np.sin(2*np.pi*4000*t)], dtype='float32') + y, stats = soften_texture(x) + self.assertGreater(stats['max_reduction_db'], .01) + self.assertLessEqual(stats['max_reduction_db'], 2) + self.assertTrue(np.isfinite(y).all()) + self.assertGreater(np.sqrt(np.mean((x-y)**2)), 1e-5) + + def test_invalid(self): + for x in (np.zeros(4410), np.zeros((0,4410)), np.zeros((2,100)), + np.full((2,4410), np.nan), np.zeros((2,4410), dtype='complex64')): + with self.assertRaises(ValueError): + soften_texture(x) + for cap in (-1, 4, np.nan, np.inf): + with self.assertRaises(ValueError): + soften_texture(np.zeros((1,4410)), max_reduction_db=cap) + + +if __name__ == '__main__': + unittest.main() diff --git a/texture_softening.py b/texture_softening.py new file mode 100644 index 0000000..8f5f024 --- /dev/null +++ b/texture_softening.py @@ -0,0 +1,43 @@ +# Adapted from Neuracoust Sound Build-up experimental post-processing. +# Changes: explicit zero-strength bypass and input validation. +# Distributed under the repository's CC BY-SA 4.0 license. +"""Offline candidate: bounded temporal spectral excursion reduction, no dry input. +Not a defect classifier. Musical fluctuations may also trigger the detector. +""" +import numpy as np +from scipy.signal import stft, istft +from scipy.ndimage import median_filter, gaussian_filter1d, maximum_filter1d + + +def soften_texture(audio, sample_rate=44100, max_reduction_db=2.0): + x = np.asarray(audio) + if (x.ndim != 2 or x.shape[0] < 1 or x.shape[-1] < 2048 + or not np.issubdtype(x.dtype, np.floating) or not np.isfinite(x).all()): + raise ValueError('Expected finite channels-by-frames audio, at least 2048 frames') + if not np.isfinite(sample_rate) or sample_rate < 16000 or not 0 <= max_reduction_db <= 3: + raise ValueError('Unsupported rate or attenuation') + if max_reduction_db == 0: + return x.copy(), {'max_reduction_db': 0.0, + 'power_weighted_reduction_db': 0.0, + 'attack_frame_fraction': 0.0} + kw = dict(fs=sample_rate, nperseg=1024, noverlap=768) + f, _, z = stft(np.pad(x, ((0,0),(1024,1024)), mode='reflect'), **kw) + power = np.mean(abs(z)**2, axis=0) + level = 10*np.log10(np.maximum(power, 1e-16)) + # About 40 ms neighborhood: only positive, short-lived excursions. + frames = max(3, int(round(.04*sample_rate/256)) | 1) + baseline = median_filter(level, size=(1,frames), mode='nearest') + excess = np.maximum(level-baseline-3, 0) + # Broad attack gate protects simultaneous rising energy (not guaranteed perfect). + energy = power.sum(axis=0) + attack = np.r_[False, energy[1:] > 1.4*np.maximum(energy[:-1],1e-16)] + gate = 1-maximum_filter1d(attack.astype(float), size=5) + band = np.clip((f-1200)/800,0,1)*np.clip((14000-f)/4000,0,1) + active = power > max(float(power.max())*1e-7,1e-15) + reduction = np.minimum(excess*.35,max_reduction_db)*band[:,None]*gate*active + reduction = gaussian_filter1d(gaussian_filter1d(reduction,.7,axis=0),.8,axis=1) + _, y = istft(z*10**(-reduction[None,:,:]/20), **kw) + y = y[:,1024:1024+x.shape[-1]].astype(np.float32) + return y, {'max_reduction_db':float(reduction.max()), + 'power_weighted_reduction_db':float((reduction*power).sum()/max(power.sum(),1e-30)), + 'attack_frame_fraction':float(np.mean(gate==0))}