writing .avi files via custom IO context (AVIOContext) yields incomplete header

[Gustav]

Summary of the bug:
I am trying to utilise AVIOContext to write an .avi file using a custom IO context. Doing so, however, yields .avi files that have a header without any information on the number of frames or file size, which in turn leads to an incorrect duration.

How to reproduce:

• compile the following code (taken from here taken from here: [here](https://ffmpeg.org/pipermail/ffmpeg-devel/2014-November/165014.html) using a C compiler (I used GCC 4.8.5 on CentOS 7) dynamically linking to the ffmpeg libraries

  /* avio_writing.c */
  /*
   * Copyright (c) 2014 Stefano Sabatini
   *
   * Permission is hereby granted, free of charge, to any person obtaining a copy
   * of this software and associated documentation files (the "Software"), to deal
   * in the Software without restriction, including without limitation the rights
   * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   * copies of the Software, and to permit persons to whom the Software is
   * furnished to do so, subject to the following conditions:
   *
   * The above copyright notice and this permission notice shall be included in
   * all copies or substantial portions of the Software.
   *
   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
   * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   * THE SOFTWARE.
   */
  
  #include <libavcodec/avcodec.h>
  #include <libavformat/avformat.h>
  #include <libavformat/avio.h>
  #include <libavutil/file.h>
  #include <libavutil/timestamp.h>
  
  struct buffer_data
  {
      uint8_t* buf;
      size_t size;
      uint8_t* ptr;
      size_t room; ///< size left in the buffer
  };
  
  static void
  log_packet(const AVFormatContext* fmt_ctx, const AVPacket* pkt, const char* tag)
  {
      AVRational* time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
  
      printf("%s: pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n",
             tag,
             av_ts2str(pkt->pts),
             av_ts2timestr(pkt->pts, time_base),
             av_ts2str(pkt->dts),
             av_ts2timestr(pkt->dts, time_base),
             av_ts2str(pkt->duration),
             av_ts2timestr(pkt->duration, time_base),
             pkt->stream_index);
  }
  
  static int
  write_packet(void* opaque, uint8_t* buf, int buf_size)
  {
      struct buffer_data* bd = (struct buffer_data*)opaque;
      while (buf_size > bd->room) {
          int64_t offset = bd->ptr - bd->buf;
          bd->buf = av_realloc_f(bd->buf, 2, bd->size);
          if (!bd->buf) return AVERROR(ENOMEM);
          bd->size *= 2;
          bd->ptr = bd->buf + offset;
          bd->room = bd->size - offset;
      }
      printf("write packet pkt_size:%d used_buf_size:%zu buf_size:%zu buf_room:%zu\n", buf_size, bd->ptr - bd->buf, bd->size, bd->room);
  
      /* copy buffer data to buffer_data buffer */
      memcpy(bd->ptr, buf, buf_size);
      bd->ptr += buf_size;
      bd->room -= buf_size;
  
      return buf_size;
  }
  
  int
  main(int argc, char* argv[])
  {
      AVFormatContext* ifmt_ctx = NULL;
      AVFormatContext* ofmt_ctx = NULL;
      AVIOContext* avio_ctx = NULL;
      uint8_t* avio_ctx_buffer = NULL;
      size_t avio_ctx_buffer_size = 4096;
      char* in_filename = NULL;
      char* out_filename = NULL;
      int i, ret = 0;
      struct buffer_data bd = {0};
      const size_t bd_buf_size = 1024;
      AVPacket pkt;
  
      if (argc != 3) {
          fprintf(stderr,
                  "usage: %s input_file output_file\n"
                  "API example program to show how to write to a custom buffer "
                  "accessed through AVIOContext.\n"
                  "Remux the content of input_file to output_file, writing to a custom buffer.\n"
                  "The output file must support the same codec as the input file.\n",
                  argv[0]);
          return 1;
      }
      in_filename = argv[1];
      out_filename = argv[2];
  
      if ((ret = avformat_open_input(&ifmt_ctx, in_filename, 0, 0)) < 0) {
          fprintf(stderr, "Could not open input file '%s'", in_filename);
          goto end;
      }
  
      if ((ret = avformat_find_stream_info(ifmt_ctx, 0)) < 0) {
          fprintf(stderr, "Failed to retrieve input stream information");
          goto end;
      }
  
      av_dump_format(ifmt_ctx, 0, in_filename, 0);
  
      /* fill opaque structure used by the AVIOContext write callback */
      bd.ptr = bd.buf = av_malloc(bd_buf_size);
      if (!bd.buf) {
          ret = AVERROR(ENOMEM);
          goto end;
      }
      bd.size = bd.room = bd_buf_size;
  
      avio_ctx_buffer = av_malloc(avio_ctx_buffer_size);
      if (!avio_ctx_buffer) {
          ret = AVERROR(ENOMEM);
          goto end;
      }
      avio_ctx = avio_alloc_context(avio_ctx_buffer, avio_ctx_buffer_size, 1, &bd, NULL, &write_packet, NULL);
      if (!avio_ctx) {
          ret = AVERROR(ENOMEM);
          goto end;
      }
  
      ret = avformat_alloc_output_context2(&ofmt_ctx, NULL, "avi", NULL);
      if (ret < 0) {
          fprintf(stderr, "Could not create output context\n");
          goto end;
      }
      ofmt_ctx->pb = avio_ctx;
  
      for (i = 0; i < ifmt_ctx->nb_streams; i++) {
          AVStream* in_stream = ifmt_ctx->streams[i];
          AVStream* out_stream = avformat_new_stream(ofmt_ctx, NULL);
          if (!out_stream) {
              fprintf(stderr, "Failed creating output stream\n");
              ret = AVERROR_UNKNOWN;
              goto end;
          }
  
          AVCodecContext *codec_ctx = avcodec_alloc_context3(avcodec_find_encoder(in_stream->codecpar->codec_id));
          ret = avcodec_parameters_to_context(codec_ctx, in_stream->codecpar);
          if (ret < 0){
              printf("Failed to copy in_stream codecpar to codec context\n");
              goto end;
          }
          
          codec_ctx->codec_tag = 0;
          if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER)
              codec_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
          
          ret = avcodec_parameters_from_context(out_stream->codecpar, codec_ctx);
          if (ret < 0){
              printf("Failed to copy codec context to out_stream codecpar context\n");
              goto end;
          }
  
      }
      av_dump_format(ofmt_ctx, 0, out_filename, 1);
  
      ret = avformat_write_header(ofmt_ctx, NULL);
      if (ret < 0) {
          fprintf(stderr, "Error occurred when opening output file\n");
          goto end;
      }
  
      while (1) {
          AVStream *in_stream, *out_stream;
  
          ret = av_read_frame(ifmt_ctx, &pkt);
          if (ret < 0) break;
  
          in_stream = ifmt_ctx->streams[pkt.stream_index];
          out_stream = ofmt_ctx->streams[pkt.stream_index];
          log_packet(ifmt_ctx, &pkt, "in");
  
          /* copy packet */
          av_packet_rescale_ts(&pkt, in_stream->time_base, out_stream->time_base);
          pkt.pos = -1;
          log_packet(ofmt_ctx, &pkt, "out");
  
          ret = av_interleaved_write_frame(ofmt_ctx, &pkt);
          if (ret < 0) {
              fprintf(stderr, "Error muxing packet\n");
              break;
          }
          av_packet_unref(&pkt);
      }
  
      av_write_trailer(ofmt_ctx);
  end:
  
      avformat_close_input(&ifmt_ctx);
  
      /* close output */
      avformat_free_context(ofmt_ctx);
      av_freep(&avio_ctx->buffer);
      av_free(avio_ctx);
  
      /* write buffer to file */
      {
          FILE* out_file = fopen(out_filename, "w");
          if (!out_file) {
              fprintf(stderr, "Could not open file '%s'\n", out_filename);
              ret = AVERROR(errno);
          } else {
              fwrite(bd.buf, bd.size, 1, out_file);
              fclose(out_file);
          }
      }
  
      av_free(bd.buf);
  
      if (ret < 0 && ret != AVERROR_EOF) {
          fprintf(stderr, "Error occurred: %s\n", av_err2str(ret));
          return 1;
      }
  
      return 0;
  }
  

• run compiled program as ./avio_writing ../data/forest_video.avi forest_video_rewritten.avi

While the input file has information on the file size (byte offset 4) and number of frames (byte offset of 48) this is FF FF FF FF and 00 00 00 00 respectively in the output file. Which I think leads to a wrong bitrate and duration for the files. Both of these things can cause issues in tools consuming such files downstream.

Bytes in the header of the input file (../data/forest_video.avi):

00000000  52 49 46 46 a6 c5 0d 00  41 56 49 20 4c 49 53 54  |RIFF....AVI LIST|
00000010  ec 11 00 00 68 64 72 6c  61 76 69 68 38 00 00 00  |....hdrlavih8...|
00000020  56 82 00 00 98 3a 00 00  00 00 00 00 10 09 00 00  |V....:..........|
00000030  74 06 00 00 00 00 00 00  01 00 00 00 00 00 10 00  |t...............|
00000040  c0 03 00 00 1c 02 00 00  00 00 00 00 00 00 00 00  |................|
00000050  00 00 00 00 00 00 00 00                           |........|

and ffprobe output on that file:

Input #0, avi, from '../data/forest_video.avi':
  Metadata:
    software        : Lavf58.76.100
  Duration: 00:00:55.12, start: 0.000000, bitrate: 130 kb/s
  Stream #0:0: Video: h264 (High) (H264 / 0x34363248), yuv420p(tv, smpte170m, progressive), 960x540, 124 kb/s, 29.97 fps, 29.97 tbr, 29.97 tbn

And hexdump of the file that went through AVIOContext (forest_video_rewritten.avi):

00000000  52 49 46 46 ff ff ff ff  41 56 49 20 4c 49 53 54  |RIFF....AVI LIST|
00000010  e8 00 00 00 68 64 72 6c  61 76 69 68 38 00 00 00  |....hdrlavih8...|
00000020  0b 00 00 00 ba 3c 00 00  00 00 00 00 00 09 00 00  |.....<..........|
00000030  00 00 00 00 00 00 00 00  01 00 00 00 00 00 10 00  |................|
00000040  c0 03 00 00 1c 02 00 00  00 00 00 00 00 00 00 00  |................|
00000050  00 00 00 00 00 00 00 00                           |........|
00000058

and ffprobe output:

Input #0, avi, from 'forest_video_rewritten.avi':
  Metadata:
    software        : Lavf60.3.100
  Duration: 00:14:33.81, start: 0.000000, bitrate: 19 kb/s
  Stream #0:0: Video: h264 (High) (H264 / 0x34363248), yuv420p(tv, smpte170m, progressive), 960x540, 600 fps, 29.97 tbr, 600 tbn

Maybe something is done incorrectly in this example file, but the documentation and examples on using AVIOContext for writing a file "in-memory" are quite scarce. I feel like this is the correct way of using AVIOContext to write the contents of an .avi file into a buffer (which is then written to a file to check the results), but it isn't working correctly, as the header that is written to this file does not correspond to the data the file contains.

[Gustav]

sample input file

[Gustav]

output of example code, using forest_video.api as input

[mkver]

The AVI muxer (as well as many other muxers) update some stats (or other stuff) at the end of muxing, but they can only do that if the output is seekable; your custom I/O is not, because you do not use a seek function.

[Gustav]

Thank you so much for the answer! I have just verified that adding a seek function makes the resulting .avi file written via the custom I/O complete.
Knowing what the solution is, it makes perfect sense that a seekable output is a requirement for the way the AVI muxer works. Unfortunately I hadn't come across that information before and the documentation on custom I/O is quite scarce, so thanks for the tip.