const std = @import("std"); const c = @import("c.zig").c; const gl = @import("gl.zig"); const src = @import("source.zig"); const ctrl = @import("control.zig"); const build_config = @import("build_config"); pub const Errors = error{ AVGenericError, AVAllocationError, HAPBadArguments, HAPBufferTooSmall, HAPBadFrame, HAPInternalError, UnsupportedCodec, OutOfMemory, NoFrames, EOF, TryLater, }; pub fn check(err: c_int) Errors!void { if (err >= 0) return; var buf: [c.AV_ERROR_MAX_STRING_SIZE]u8 = undefined; _ = c.av_make_error_string(&buf, buf.len, err); std.debug.print("libav error: {s}\n", .{buf}); return error.AVGenericError; } fn get_format_context( filename: [*:0]const u8, format_name: ?[*:0]const u8, format_options: []const [*c]c.lo_arg, ) !*c.AVFormatContext { c.avdevice_register_all(); var format: ?*const c.AVInputFormat = null; if (format_name) |name| { format = c.av_find_input_format(name); } var options: ?*c.AVDictionary = null; var i: usize = 0; std.debug.assert(format_options.len % 2 == 0); while (i < format_options.len) : (i += 2) { try check(c.av_dict_set( &options, &format_options[i].*.s, &format_options[i + 1].*.s, 0, )); } var format_ctx: ?*c.AVFormatContext = null; try check(c.avformat_open_input(&format_ctx, filename, format, &options)); errdefer c.avformat_close_input(&format_ctx); return format_ctx orelse error.AVGenericError; } pub const Decoder = struct { num_frames: i32, format: *c.AVFormatContext, stream: *c.AVStream, packet: [*c]c.AVPacket, vtable: VTable, pub const VTable = struct { process_next_frame_fn: *const fn (self: *Decoder, texture: ?*const gl.Texture, z: i32) Errors!?f64, init_texture_fn: *const fn (self: *Decoder, texture: *const gl.Texture) void, reset_fn: ?*const fn (self: *Decoder) void, deinit_fn: *const fn (self: *Decoder, allocator: std.mem.Allocator) void, }; pub const StreamState = enum { done, more_now, more_later, }; pub fn init(format: *c.AVFormatContext, stream: *c.AVStream, vtable: VTable) Decoder { return .{ .num_frames = 0, .format = format, .stream = stream, .packet = c.av_packet_alloc(), .vtable = vtable, }; } pub fn getNextPacket(self: *Decoder) Errors!void { return res: switch (c.av_read_frame(self.format, self.packet)) { c.AVERROR_EOF => error.EOF, c.AVERROR(c.EAGAIN) => error.TryLater, else => |res| { try check(res); if (self.packet.*.stream_index == self.stream.index) { break :res; } else { c.av_packet_unref(self.packet); continue :res c.av_read_frame(self.format, self.packet); } }, }; } fn processStream( self: *Decoder, progress: std.Progress.Node, texture: ?*const gl.Texture, ) Errors!void { defer progress.end(); var just_one = false; if (texture) |tex| { if (tex.type == .TEXTURE_2D) { just_one = true; } } try self.rewind(); self.num_frames = 0; while (true) { const res = self.processFrame(texture, self.num_frames); if (res == error.EOF) break; _ = try res; self.num_frames += 1; progress.setCompletedItems(@intCast(self.num_frames)); if (just_one) break; } } pub fn processFrame( self: *Decoder, texture: ?*const gl.Texture, z: i32, ) Errors!?f64 { return try self.vtable.process_next_frame_fn(self, texture, z); } pub fn rewind(self: *Decoder) !void { try check(c.avformat_seek_file(self.format, self.stream.index, 0, 0, 0, 0)); if (self.vtable.reset_fn) |reset_fn| reset_fn(self); } pub fn createTexture( self: *Decoder, outer_progress: std.Progress.Node, texture_type: gl.Texture.Type, ) Errors!gl.Texture { try self.processStream( outer_progress.start("scanning video frames", @intCast(self.stream.nb_frames)), null, ); if (self.num_frames <= 0) return error.NoFrames; const texture = gl.Texture.create(texture_type); errdefer texture.destroy(); self.vtable.init_texture_fn(self, &texture); try self.processStream( outer_progress.start("loading video frames", @intCast(self.num_frames)), &texture, ); return texture; } pub fn deinit(self: *Decoder, allocator: std.mem.Allocator) void { c.av_packet_free(&self.packet); self.vtable.deinit_fn(self, allocator); } }; pub const AVDecoder = struct { decoder: Decoder, sws_ctx: *c.SwsContext, codec_par: c.AVCodecParameters, codec_ctx: [*c]c.AVCodecContext, raw_frame: [*c]c.AVFrame, rgba_frame: [*c]c.AVFrame, pub fn init(allocator: std.mem.Allocator, format: *c.AVFormatContext, stream: *c.AVStream) Errors!*Decoder { const self = try allocator.create(AVDecoder); errdefer allocator.destroy(self); const codec_par = stream.codecpar.*; const codec = c.avcodec_find_decoder(codec_par.codec_id) orelse return error.UnsupportedCodec; var codec_ctx = c.avcodec_alloc_context3(codec) orelse return error.AVAllocationError; errdefer c.avcodec_free_context(&codec_ctx); var raw_frame = c.av_frame_alloc() orelse return error.AVAllocationError; errdefer c.av_frame_free(&raw_frame); var rgba_frame = c.av_frame_alloc() orelse return error.AVAllocationError; errdefer c.av_frame_free(&rgba_frame); try check(c.avcodec_parameters_to_context(codec_ctx, &codec_par)); try check(c.avcodec_open2(codec_ctx, codec, null)); const sws_ctx = c.sws_getContext( codec_par.width, codec_par.height, codec_par.format, codec_par.width, codec_par.height, c.AV_PIX_FMT_RGBA, c.SWS_POINT, null, null, 0, ) orelse return error.AVGenericError; errdefer c.sws_freeContext(sws_ctx); self.* = .{ .decoder = Decoder.init( format, stream, .{ .process_next_frame_fn = processNextFrame, .init_texture_fn = initTexture, .reset_fn = reset, .deinit_fn = deinit, }, ), .sws_ctx = sws_ctx, .codec_par = codec_par, .codec_ctx = codec_ctx, .raw_frame = raw_frame, .rgba_frame = rgba_frame, }; return &self.decoder; } fn deinit(decoder: *Decoder, allocator: std.mem.Allocator) void { const self: *AVDecoder = @fieldParentPtr("decoder", decoder); const raw_ptr: [*c][*c]c.AVFrame = &self.raw_frame; const rgba_ptr: [*c][*c]c.AVFrame = &self.rgba_frame; c.av_frame_free(raw_ptr); c.av_frame_free(rgba_ptr); c.sws_freeContext(self.sws_ctx); c.avcodec_free_context(&self.codec_ctx); allocator.destroy(self); } fn reset(decoder: *Decoder) void { const self: *AVDecoder = @fieldParentPtr("decoder", decoder); c.avcodec_flush_buffers(self.codec_ctx); } fn initTexture(decoder: *Decoder, texture: *const gl.Texture) void { const self: *AVDecoder = @fieldParentPtr("decoder", decoder); texture.allocate( self.codec_par.width, self.codec_par.height, self.decoder.num_frames, c.GL_COMPRESSED_RGBA_S3TC_DXT1_EXT, ); } fn getNextFrame(self: *AVDecoder) Errors!void { while (true) { switch (c.avcodec_receive_frame(self.codec_ctx, self.raw_frame)) { c.AVERROR_EOF => return error.EOF, c.AVERROR(c.EAGAIN) => { try self.decoder.getNextPacket(); defer c.av_packet_unref(self.decoder.packet); try check(c.avcodec_send_packet(self.codec_ctx, self.decoder.packet)); continue; }, else => |res| { try check(res); return; }, } } } fn processNextFrame(decoder: *Decoder, texture: ?*const gl.Texture, z: i32) Errors!?f64 { const self: *AVDecoder = @fieldParentPtr("decoder", decoder); try self.getNextFrame(); if (texture) |tex| { try check(c.sws_scale_frame(self.sws_ctx, self.rgba_frame, self.raw_frame)); c.glPixelStorei(c.GL_UNPACK_ROW_LENGTH, @divFloor(self.rgba_frame.*.linesize[0], @sizeOf(u8) * 4)); tex.setLayer( self.rgba_frame.*.width, self.rgba_frame.*.height, z, c.GL_RGBA, self.rgba_frame.*.data[0], ); } const pts: f64 = @floatFromInt(self.raw_frame.*.best_effort_timestamp); return pts * c.av_q2d(self.decoder.stream.*.time_base); } }; pub const VideoSource = struct { source: src.Source, pub fn init( allocator: std.mem.Allocator, progress_root: std.Progress.Node, texture_type: gl.Texture.Type, filename: [*:0]const u8, format_name: ?[*:0]const u8, format_options: []const [*c]c.lo_arg, ) !*src.Source { const self = try allocator.create(VideoSource); errdefer allocator.destroy(self); const progress = progress_root.start("loading texture", 2); defer progress.end(); var format = try get_format_context(filename, format_name, format_options); defer c.avformat_close_input(@ptrCast(&format)); try check(c.avformat_find_stream_info(format, null)); const video_stream = for (format.streams, 0..format.nb_streams) |c_stream, _| { const stream = @as(*c.AVStream, c_stream orelse unreachable); if (stream.codecpar.*.codec_type == c.AVMEDIA_TYPE_VIDEO) break stream; } else unreachable; const codec_par = video_stream.codecpar.*; const is_hap = codec_par.codec_tag == c.MKTAG('H', 'a', 'p', '1') or codec_par.codec_tag == c.MKTAG('H', 'a', 'p', '5'); const decoder = if (is_hap and build_config.have_hap) try @import("hap.zig").HAPDecoder.init(allocator, format, video_stream) else try AVDecoder.init(allocator, format, video_stream); defer decoder.deinit(allocator); self.* = .{ .source = .{ .texture = try decoder.createTexture(progress, texture_type), .deinit_fn = deinit, }, }; return &self.source; } fn deinit(source: *const src.Source, allocator: std.mem.Allocator) void { const self: *const VideoSource = @fieldParentPtr("source", source); allocator.destroy(self); } }; pub const StreamSource = struct { source: src.Source, flags: src.StreamFlags, decoder: *Decoder, format: *c.AVFormatContext, depth: i32, frame: i32, start_time: ?f64, // last rendered glfw time thread: *gl.Thread, pub fn init( allocator: std.mem.Allocator, constants: *const gl.Constants, texture_type: gl.Texture.Type, depth: i32, filename: [*:0]const u8, format_name: ?[*:0]const u8, format_options: []const [*c]c.lo_arg, ) !*src.Source { const self = try allocator.create(StreamSource); errdefer allocator.destroy(self); var format = try get_format_context(filename, format_name, format_options); errdefer c.avformat_close_input(@ptrCast(&format)); format.flags |= c.AVFMT_FLAG_NONBLOCK; try check(c.avformat_find_stream_info(format, null)); const video_stream = for (format.streams, 0..format.nb_streams) |c_stream, _| { const stream = @as(*c.AVStream, c_stream orelse unreachable); if (stream.codecpar.*.codec_type == c.AVMEDIA_TYPE_VIDEO) break stream; } else unreachable; const codec_par = video_stream.codecpar.*; switch (texture_type) { .TEXTURE_2D, .TEXTURE_RECTANGLE => { std.debug.assert(depth == 1); }, else => {}, } const texture = switch (texture_type) { .TEXTURE_3D, .TEXTURE_2D_ARRAY, .TEXTURE_2D, .TEXTURE_RECTANGLE => |t| gl.Texture.create(t), else => return error.textureTypeInvalid, }; texture.allocate( codec_par.width, codec_par.height, depth, c.GL_RGBA8, ); const decoder = try AVDecoder.init(allocator, format, video_stream); errdefer decoder.deinit(allocator); self.* = .{ .source = .{ .texture = texture, .update_fn = update_source, .deinit_fn = deinit, .register_fn = register_methods, .unregister_fn = unregister_methods, .update_uniform_fn = update_uniform, }, .flags = .{}, .decoder = decoder, .format = format, .depth = depth, .frame = 0, .start_time = null, .thread = undefined, }; _ = try self.update(false); self.thread = try gl.Thread.init(allocator, constants, update_loop, .{self}); return &self.source; } fn update_source(source: *src.Source) !bool { const self: *StreamSource = @fieldParentPtr("source", source); if (self.thread.has_quit) return true; return false; } fn register_methods(source: *src.Source, name: []const u8, control: *ctrl.ControlServer) void { const self: *StreamSource = @fieldParentPtr("source", source); self.flags.register(name, control); } fn unregister_methods(source: *src.Source, name: []const u8, control: *ctrl.ControlServer) void { const self: *StreamSource = @fieldParentPtr("source", source); self.flags.unregister(name, control); } fn update_uniform(source: *src.Source, param: []const u8, value: gl.UniformPointer) !void { const self: *StreamSource = @fieldParentPtr("source", source); if (!std.mem.eql(u8, param, "offset")) return error.unknownSourceParam; const frame = @rem(self.frame + self.depth - 1, self.depth); if (value.arrayLength() != 1) return error.uniformNotSupported; try switch (value) { .FLOAT => |val| val[0] = @as(f32, @floatFromInt(frame)) / @as(f32, @floatFromInt(self.depth)), .DOUBLE => |val| val[0] = @as(f64, @floatFromInt(frame)) / @as(f64, @floatFromInt(self.depth)), .INT => |val| val[0] = frame, .UNSIGNED_INT => |val| val[0] = @intCast(frame), else => error.uniformNotSupported, }; } fn update_loop(self: *StreamSource) !void { var skip = false; while (!self.thread.quit) { if (self.update(skip)) |maybe_pts| { if (maybe_pts) |pts| { if (self.start_time) |start_time| { while (true) { const position = c.glfwGetTime() - start_time; const delta = pts - position; if (delta > 0.001) { // we are EARLY const nanos: u64 = @intFromFloat(delta * 1_000_000_000 - 100000); std.Thread.sleep(nanos); continue; } else if (delta < -0.5) { // we are LATE skip = true; } break; } } else { self.start_time = c.glfwGetTime() - pts; } } if (skip) continue; c.glFlush(); try std.Thread.yield(); } else |err| { switch (err) { error.TryLater => std.Thread.sleep(1000), error.EOF => { try self.decoder.rewind(); self.start_time = null; }, else => return err, } } } } fn update(self: *StreamSource, skip: bool) Errors!?f64 { if (self.flags.shouldStep() and !skip) { const new_pos = try self.decoder.processFrame( &self.source.texture, self.frame, ); self.frame = @rem(self.frame + 1, self.depth); return new_pos; } else { return try self.decoder.processFrame(null, 0); } } fn deinit(source: *src.Source, allocator: std.mem.Allocator) void { const self: *StreamSource = @fieldParentPtr("source", source); self.thread.deinit(allocator); self.decoder.deinit(allocator); c.avformat_close_input(@ptrCast(&self.format)); allocator.destroy(self); } };