diff --git a/Source/Util/AVWrapper.cpp b/Source/Util/AVWrapper.cpp index f5e81185..c698d18f 100644 --- a/Source/Util/AVWrapper.cpp +++ b/Source/Util/AVWrapper.cpp @@ -22,13 +22,10 @@ size_t AVWrapperFrame::getBufferSize() const { if (type == AVWrapperType::Video) { AVPixelFormat format = static_cast(frame->format); return av_image_get_buffer_size(format, frame->width, frame->height, 1); - } else if (AVWrapperType::Audio) { + } else if (type == AVWrapperType::Audio) { AVSampleFormat format = static_cast(frame->format); - int channels = 1; - if (av_sample_fmt_is_planar(format)) { - channels = frame->ch_layout.nb_channels; - } - return frame->linesize[0] * channels; + return static_cast(frame->nb_samples) * frame->ch_layout.nb_channels + * av_get_bytes_per_sample(format); } else { xassert(0); return 0; @@ -44,12 +41,12 @@ size_t AVWrapperFrame::copyBuffer(uint8_t** buffer) const { *buffer = new uint8_t[buffer_size]; } if (type == AVWrapperType::Video) { - AVPixelFormat format = static_cast(frame->format); - int written = av_image_copy_to_buffer(*buffer, static_cast(buffer_size), frame->data, - frame->linesize, format, frame->width, frame->height, 1); - xassert(written == buffer_size); + int written = av_image_copy_to_buffer(*buffer, static_cast(buffer_size), + frame->data, frame->linesize, static_cast(frame->format), + frame->width, frame->height, 1); + if (written < 0) ErrH.Abort("Cannot copy video frame: " + wrapper->file_path); return buffer_size; - } else if (AVWrapperType::Audio) { + } else if (type == AVWrapperType::Audio) { AVSampleFormat format = static_cast(frame->format); int channels = 1; if (av_sample_fmt_is_planar(format)) { @@ -58,7 +55,7 @@ size_t AVWrapperFrame::copyBuffer(uint8_t** buffer) const { if (1 < channels) { //We need to pack planar audio size_t bytes_sample = av_get_bytes_per_sample(format); - xassert(buffer_size == frame->linesize[0] * channels); + //Store each plane sample as interleaved for (int si = 0; si < frame->nb_samples; ++si) { size_t srcoff = si * bytes_sample; @@ -66,12 +63,12 @@ size_t AVWrapperFrame::copyBuffer(uint8_t** buffer) const { for (int pi = 0; pi < channels; pi++) { size_t dstoff = (si * channels + pi) * bytes_sample; xassert(dstoff + bytes_sample <= buffer_size); - memcpy(*buffer + dstoff, frame->data[pi] + srcoff, bytes_sample); + memcpy(*buffer + dstoff, frame->extended_data[pi] + srcoff, bytes_sample); } } } else { //Copy all buffer directly - xassert(buffer_size == frame->linesize[0]); + memcpy(*buffer, frame->data[0], buffer_size); } return buffer_size; @@ -100,6 +97,17 @@ int64_t AVWrapperFrame::getPTS() const { void AVWrapper::init() { if (avwrapper_init) return; + if ((avcodec_version() >> 16) != LIBAVCODEC_VERSION_MAJOR || + (avformat_version() >> 16) != LIBAVFORMAT_VERSION_MAJOR || + (avutil_version() >> 16) != LIBAVUTIL_VERSION_MAJOR) { + ErrH.Abort("FFmpeg headers and runtime libraries have incompatible versions"); + } +#ifdef PERIMETER_FFMPEG_MOVIE + if ((swscale_version() >> 16) != LIBSWSCALE_VERSION_MAJOR || + (avfilter_version() >> 16) != LIBAVFILTER_VERSION_MAJOR) { + ErrH.Abort("FFmpeg scaler/filter headers and runtime libraries are incompatible"); + } +#endif avwrapper_init = true; } @@ -122,7 +130,8 @@ int AVWrapper::close() { if (filterGraph) { avfilter_graph_free(&filterGraph); } - swrChannelLayout = {}; + av_channel_layout_uninit(&swrChannelLayout); + buffersrcCtx = buffersinkCtx = nullptr; swrSampleRate = 0; swrFormat = AV_SAMPLE_FMT_NONE; #endif @@ -185,6 +194,12 @@ void AVWrapper::pullFilterAudio() { void AVWrapper::onEnd() { readPacket(true); +#ifdef PERIMETER_FFMPEG_MOVIE + if (filterGraph) { + int ret = av_buffersrc_add_frame_flags(buffersrcCtx, nullptr, 0); + if (ret < 0) fprintf(stderr, "Cannot flush audio filter: %d\n", ret); + } +#endif pullFilterAudio(); } @@ -311,18 +326,12 @@ bool AVWrapper::setupVideoScaler(int width, int height, AVPixelFormat format, in swsWidth = width != 0 ? width : videoCodecCtx->width; swsHeight = height != 0 ? height : videoCodecCtx->height; swsFormat = format != AV_PIX_FMT_NONE ? format : videoCodecCtx->pix_fmt; - if (swsCtx) { - sws_freeContext(swsCtx); - } - swsCtx = sws_getContext( - //Source - videoCodecCtx->width, videoCodecCtx->height, videoCodecCtx->pix_fmt, - //Destination - swsWidth, swsHeight, swsFormat, - //Params - flags, nullptr, nullptr, nullptr - ); - return true; + swsFlags = flags; + // The decoder may not know its pixel format until the first frame arrives. + // Construct the scaler from that frame in handleFrame(). + if (swsCtx) sws_freeContext(swsCtx); + swsCtx = nullptr; + return swsWidth > 0 && swsHeight > 0 && swsFormat != AV_PIX_FMT_NONE; #else return false; #endif @@ -432,15 +441,16 @@ bool AVWrapper::setupAudioConverter(AVSampleFormat dst_format, int dst_channels, //Nothing to setup since there is no codec return false; } +#ifdef PERIMETER_FFMPEG_MOVIE if (filterGraph) { avfilter_graph_free(&filterGraph); } filterGraph = nullptr; buffersrcCtx = nullptr; buffersinkCtx = nullptr; -#ifdef PERIMETER_FFMPEG_MOVIE - swrFormat = dst_format != 0 ? dst_format : audioCodecCtx->sample_fmt; + swrFormat = dst_format != AV_SAMPLE_FMT_NONE ? dst_format : audioCodecCtx->sample_fmt; int num_channel = dst_channels != 0 ? dst_channels : audioCodecCtx->ch_layout.nb_channels; + av_channel_layout_uninit(&swrChannelLayout); av_channel_layout_default(&swrChannelLayout, num_channel); swrSampleRate = dst_frequency != 0 ? dst_frequency : audioCodecCtx->sample_rate; if (swrFormat == AV_SAMPLE_FMT_NONE) { @@ -451,7 +461,7 @@ bool AVWrapper::setupAudioConverter(AVSampleFormat dst_format, int dst_channels, AVFilterInOut *inputs = avfilter_inout_alloc(); const AVFilterLink *inlink; const AVFilterLink *outlink; - AVRational time_base = audioCodecCtx->time_base; + AVRational time_base = av_make_q(1, audioCodecCtx->sample_rate); char inChannelLayout_str[128]; char swrChannelLayout_str[128]; @@ -539,90 +549,68 @@ bool AVWrapper::setupAudioConverter(AVSampleFormat dst_format, int dst_channels, #endif } -void AVWrapper::readPacket(bool drain) { - int ret = 0; - AVPacket* packet = nullptr; +void AVWrapper::readPacket(bool drain) { + AVPacket* packet = drain ? nullptr : streamPacket; if (!drain) { - packet = streamPacket; - //Attempt to get a packet - ret = av_read_frame(formatCtx, packet); - if (0 <= ret) { - end = false; + int ret = av_read_frame(formatCtx, packet); + if (ret < 0) { + if (ret != AVERROR_EOF) { + char error[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(ret, error, sizeof(error)); + fprintf(stderr, "Video read error in %s: %s\n", file_path.c_str(), error); + } + // A failed local-file read must not spin in the player's decode loop. + if (!end) onEnd(); + end = true; + return; } } - if (0 <= ret) { - if (0 <= videoStream && (drain || packet->stream_index == videoStream)) { - // Extract video frame - ret = avcodec_send_packet(videoCodecCtx, packet); - if (ret != 0) { - if (ret == AVERROR_EOF) { - fprintf(stderr, "Already reached video EOF - file: %s\n", file_path.c_str()); - } else if (ret != AVERROR(EAGAIN)) { - fprintf(stderr, "Can't send video packet %d - file: %s\n", ret, file_path.c_str()); - } else { - ret = 0; - } - } - while (ret == 0) { + + auto decode = [&](AVCodecContext* codec, AVWrapperType type) { + auto receive = [&]() { + int count = 0; + for (;;) { AVWrapperFrame* frame = new AVWrapperFrame(); - frame->type = AVWrapperType::Video; + frame->type = type; frame->frame = AV_FRAME_ALLOC(); - ret = avcodec_receive_frame(videoCodecCtx, frame->frame); - if (ret == 0) { - handleFrame(frame); - ret = 0; - continue; - } else if (ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) { - fprintf(stderr, "Can't receive the video frame - file: %s\n", file_path.c_str()); - } - delete frame; - break; - } - } - - if (0 <= audioStream && (drain || packet->stream_index == audioStream)) { - // Extract audio frame - ret = avcodec_send_packet(audioCodecCtx, packet); - if (ret != 0) { - if (ret == AVERROR_EOF) { - fprintf(stderr, "Already reached audio EOF - file: %s\n", file_path.c_str()); - } else if (ret != AVERROR(EAGAIN)) { - fprintf(stderr, "Can't send audio packet %d - file: %s\n", ret, file_path.c_str()); - } else { - ret = 0; + int ret = avcodec_receive_frame(codec, frame->frame); + if (ret < 0) { + delete frame; + if (ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) + fprintf(stderr, "Decode error %d in %s\n", ret, file_path.c_str()); + return count; } + handleFrame(frame); + ++count; } - while (true) { - AVWrapperFrame* frame = new AVWrapperFrame(); - frame->type = AVWrapperType::Audio; - frame->frame = AV_FRAME_ALLOC(); - ret = avcodec_receive_frame(audioCodecCtx, frame->frame); - if (ret == 0) { - handleFrame(frame); - ret = 0; - continue; - } else if (ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) { - fprintf(stderr, "Can't receive the audio frame - file: %s\n", file_path.c_str()); - } - delete frame; - break; + }; + int ret = avcodec_send_packet(codec, packet); + if (ret == AVERROR(EAGAIN)) { + // Receive pending output before retrying the SAME packet. + // No progress is a decoder error, not a reason to loop forever. + if (!receive()) { + fprintf(stderr, "Decoder made no progress in %s\n", file_path.c_str()); + end = true; + return; } + ret = avcodec_send_packet(codec, packet); } - - if (!drain) { - AV_PACKET_UNREF(packet); - } - } else { - if (!end) { - onEnd(); + if (ret < 0 && ret != AVERROR_EOF) { + fprintf(stderr, "Cannot send packet (%d) in %s\n", ret, file_path.c_str()); + end = true; + return; } - end = true; - } - - //pull filtered audio from the filtergraph + receive(); + }; + if (videoCodecCtx && (drain || packet->stream_index == videoStream)) + decode(videoCodecCtx, AVWrapperType::Video); + if (audioCodecCtx && (drain || packet->stream_index == audioStream)) + decode(audioCodecCtx, AVWrapperType::Audio); + if (packet) AV_PACKET_UNREF(packet); pullFilterAudio(); } + void AVWrapper::handleFrame(AVWrapperFrame* frame) { frame->wrapper = this; @@ -637,7 +625,7 @@ void AVWrapper::handleFrame(AVWrapperFrame* frame) { #ifdef PERIMETER_FFMPEG_MOVIE //Do video scaling if scaler is setup - if (swsCtx) { + if (swsWidth > 0 && swsHeight > 0) { AVWrapperFrame original; original.frame = frame->frame; frame->frame = AV_FRAME_ALLOC(); @@ -645,17 +633,16 @@ void AVWrapper::handleFrame(AVWrapperFrame* frame) { frame->frame->width = swsWidth; frame->frame->height = swsHeight; frame->frame->format = swsFormat; - av_frame_make_writable(frame->frame); - int ret = av_frame_get_buffer(frame->frame, 1); - if (ret < 0) { - ErrH.Abort("Can't alloc sws image - file: " + file_path); - } - sws_scale( - swsCtx, - original.frame->data, original.frame->linesize, - 0, videoCodecCtx->height, - frame->frame->data, frame->frame->linesize - ); + swsCtx = sws_getCachedContext(swsCtx, + original.frame->width, original.frame->height, + static_cast(original.frame->format), + swsWidth, swsHeight, swsFormat, swsFlags, nullptr, nullptr, nullptr); + if (!swsCtx || av_frame_get_buffer(frame->frame, 32) < 0) + ErrH.Abort("Cannot allocate video scaler/frame: " + file_path); + int rows = sws_scale(swsCtx, original.frame->data, original.frame->linesize, + 0, original.frame->height, frame->frame->data, frame->frame->linesize); + if (rows != swsHeight) + ErrH.Abort("Incomplete scaled video frame: " + file_path); } #endif @@ -668,7 +655,7 @@ void AVWrapper::handleFrame(AVWrapperFrame* frame) { if (frame->frame->pts != AV_NOPTS_VALUE) { frame->frame->pts = av_rescale_q(frame->frame->pts, frame->time_base, tb); } else if (audioNextPTS != AV_NOPTS_VALUE) { - frame->frame->pts = av_rescale_q(audioNextPTS, frame->time_base, tb); + frame->frame->pts = audioNextPTS; } if (frame->frame->pts != AV_NOPTS_VALUE) { @@ -677,6 +664,7 @@ void AVWrapper::handleFrame(AVWrapperFrame* frame) { fprintf(stderr, "Audio frame without PTS\n"); } + frame->time_base = tb; //Do audio conversion #ifdef PERIMETER_FFMPEG_MOVIE if (filterGraph) { @@ -724,11 +712,19 @@ int AVWrapper::getVideoCodecHeight() const { } int AVWrapper::getVideoWidth() const { - return swsCtx ? swsWidth : videoCodecCtx->width; +#ifdef PERIMETER_FFMPEG_MOVIE + return swsWidth > 0 ? swsWidth : videoCodecCtx->width; +#else + return videoCodecCtx->width; +#endif } int AVWrapper::getVideoHeight() const { - return swsCtx ? swsHeight : videoCodecCtx->height; +#ifdef PERIMETER_FFMPEG_MOVIE + return swsHeight > 0 ? swsHeight : videoCodecCtx->height; +#else + return videoCodecCtx->height; +#endif } int AVWrapper::getVideoBPP() const { diff --git a/Source/Util/AVWrapper.h b/Source/Util/AVWrapper.h index c1bc341f..e966a779 100644 --- a/Source/Util/AVWrapper.h +++ b/Source/Util/AVWrapper.h @@ -109,6 +109,7 @@ class AVWrapper int swsWidth = 0; ///Software scaler output frame height int swsHeight = 0; + int swsFlags = 0; ///Software scaler output frame format AVPixelFormat swsFormat = AV_PIX_FMT_NONE; ///Audio converter output channel layout diff --git a/tests/avwrapper/regression.cpp b/tests/avwrapper/regression.cpp new file mode 100644 index 00000000..e0a39153 --- /dev/null +++ b/tests/avwrapper/regression.cpp @@ -0,0 +1,80 @@ +#include "AVWrapper.h" +#include "xerrhand.h" +#include +#include +#include + +XErrorHandler ErrH; +XErrorHandler::XErrorHandler() = default; +XErrorHandler::~XErrorHandler() = default; +void XErrorHandler::Abort(const std::string& s, int, int, const char*) { std::fprintf(stderr, "%s\n", s.c_str()); std::exit(2); } +void XErrorHandler::Abort(const char* s, int, int, const char*) { std::fprintf(stderr, "%s\n", s); std::exit(2); } +void ErrH_RTC(const char*, unsigned, const char*) { std::exit(3); } +static void check(bool ok, const char* what) { + if (!ok) { std::fprintf(stderr, "FAIL: %s\n", what); std::exit(1); } +} + +// Expose fixture setup only; copying and sizing use the production implementation. +struct AudioFixture : AVWrapperFrame { + AudioFixture(AVSampleFormat format, int channels) { + type = AVWrapperType::Audio; + frame = av_frame_alloc(); + frame->format = format; + frame->nb_samples = 7; // Deliberately shorter than an aligned FFmpeg plane. + av_channel_layout_default(&frame->ch_layout, channels); + check(av_frame_get_buffer(frame, 32) == 0, "audio allocation"); + if (av_sample_fmt_is_planar(format)) { + for (int c = 0; c < channels; ++c) + for (int s = 0; s < 7; ++s) + reinterpret_cast(frame->extended_data[c])[s] = s * channels + c; + } else { + for (int i = 0; i < 7 * channels; ++i) + reinterpret_cast(frame->data[0])[i] = i; + } + } +}; +int main(int argc, char** argv) { + AVWrapper::init(); + for (auto format : {AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S16P}) { + for (int channels : {1, 2, 10}) { + AudioFixture f(format, channels); + uint8_t* data = nullptr; + check(f.getBufferSize() == 7u * channels * 2, "no alignment bytes in audio size"); + check(f.copyBuffer(&data) == 7u * channels * 2, "audio copied size"); + for (int i = 0; i < 7 * channels; ++i) + check(reinterpret_cast(data)[i] == i, "interleaved samples"); + delete[] data; + } + } + check(argc == 2, "expected generated fixture path"); + AVWrapper w; + check(w.open(argv[1], AVWrapperType::Video) == 0, "open fixture"); + check(w.setupVideoScaler(65, 33, AV_PIX_FMT_BGRA, SWS_BILINEAR), "odd-sized scaler"); + check(w.setupAudioConverter(AV_SAMPLE_FMT_S16, 2, 48000), "resampler"); + unsigned frames = 0; + size_t audioBytes = 0; + double videoPTS = -1, audioPTS = -1; + while (!w.end) { + w.readPacket(); + for (auto* f : w.videoFrames) { + check(f->getPresentationTime() >= videoPTS, "video PTS monotonic"); + videoPTS = f->getPresentationTime(); + uint8_t* data = nullptr; + check(f->copyBuffer(&data) == 65 * 33 * 4, "full odd-sized frame"); + check(data[(65 * 33 - 1) * 4 + 3] == 255, "last pixel written"); + delete[] data; + ++frames; + } + for (auto* f : w.audioFrames) { + check(f->getPresentationTime() >= audioPTS, "audio PTS monotonic"); + audioPTS = f->getPresentationTime(); + uint8_t* data = nullptr; + audioBytes += f->copyBuffer(&data); + delete[] data; + } + w.clearFrames(); + } + check(frames == 30, "all 30 video frames including EOF"); + check(audioBytes == 48000 * 2 * 2, "one second resampled audio including filter tail"); + std::printf("PASS: packed/planar audio, 10 channels, odd dimensions, timestamps, EOF (%u frames, %zu audio bytes)\n", frames, audioBytes); +} diff --git a/tests/avwrapper/run.py b/tests/avwrapper/run.py new file mode 100644 index 00000000..10e5a272 --- /dev/null +++ b/tests/avwrapper/run.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Standalone regression test. Requires a C++17 compiler, pkg-config, FFmpeg and SDL2 headers.""" +import argparse +import os +from pathlib import Path +import shlex +import subprocess +import tempfile + +parser = argparse.ArgumentParser() +parser.add_argument('--pevents-include', required=True, type=Path, help='directory containing pevents.h') +parser.add_argument('--sanitize', action='store_true') +args = parser.parse_args() +root = Path(__file__).resolve().parents[2] + +def run(cmd): + subprocess.run([str(x) for x in cmd], check=True) + +def pkg(*args): + return shlex.split(subprocess.check_output(['pkg-config', *args], text=True)) + +with tempfile.TemporaryDirectory(prefix='perimeter-avwrapper-') as tmp: + tmp = Path(tmp) + flags = ['-fsanitize=address,undefined', '-fno-omit-frame-pointer'] if args.sanitize else [] + run([os.environ.get('CXX', 'c++'), '-std=c++17', '-DPERIMETER_EXODUS', '-DPERIMETER_FFMPEG', + '-DPERIMETER_FFMPEG_MOVIE', '-DNASSERT', *flags, + '-I' + str(root / 'Source/XTool'), '-I' + str(root / 'Source/Util'), + '-I' + str(args.pevents_include.resolve()), *pkg('--cflags', 'sdl2'), + root / 'tests/avwrapper/regression.cpp', root / 'Source/Util/AVWrapper.cpp', + *pkg('--cflags', '--libs', 'libavcodec', 'libavformat', 'libavutil', 'libswscale', 'libavfilter'), + '-o', tmp / 'regression']) + run(['ffmpeg', '-v', 'error', '-f', 'lavfi', '-i', 'testsrc=size=65x33:rate=30:duration=1', + '-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=44100:duration=1', + '-c:v', 'ffv1', '-pix_fmt', 'bgr0', '-c:a', 'pcm_s16le', tmp / 'fixture.mkv']) + run([tmp / 'regression', tmp / 'fixture.mkv'])