implement HAP decoding straight to compressed GPU texture

Supports HAP and HAP Alpha (Hap1/Hap5) formats

4 files changed, 400 insertions, 125 deletions

diff --git a/src/c.zig b/src/c.zig
index 7b46914..54c156e 100644
--- a/src/c.zig
+++ b/src/c.zig
@@ -11,4 +11,8 @@ pub usingnamespace @cImport({
         @cInclude("libavformat/avformat.h");
         @cInclude("libswscale/swscale.h");
     }
+
+    if (build_config.have_hap) {
+        @cInclude("hap.h");
+    }
 });
diff --git a/src/control.zig b/src/control.zig
index 0449b3d..dfdd55d 100644
--- a/src/control.zig
+++ b/src/control.zig
@@ -108,8 +108,7 @@ fn set_texture(
     const filepath = try project_root.realpathZ(@ptrCast(&argv[0].*.s), buffer[0..]);
     buffer[filepath.len] = 0;
 
-    dest.* = gl.Texture.create(texture_type);
-    try video.loadVideo(progress, @ptrCast(filepath), &dest.*.?);
+    dest.* = try video.loadVideo(progress, @ptrCast(filepath), texture_type);
 }
 
 pub const ControlServer = struct {
diff --git a/src/gl.zig b/src/gl.zig
index c1543dc..22a2ebe 100644
--- a/src/gl.zig
+++ b/src/gl.zig
@@ -73,54 +73,95 @@ pub const Texture = struct {
         c.glDeleteTextures(1, &self.id);
     }
 
-    pub fn setData2D(self: *const Texture, width: i32, height: i32, data: [*]const u8) void {
+    pub fn allocate(self: *const Texture, width: i32, height: i32, depth: i32, format: c.GLenum) void {
         c.glBindTexture(@intFromEnum(self.type), self.id);
         defer c.glBindTexture(@intFromEnum(self.type), 0);
 
-        c.glTexImage2D(
-            @intFromEnum(self.type),
-            0,
-            c.GL_COMPRESSED_RGBA_S3TC_DXT1_EXT, // c.GL_RGBA,
-            width,
-            height,
-            0,
-            c.GL_RGBA,
-            c.GL_UNSIGNED_BYTE,
-            data,
-        );
+        switch (self.type) {
+            .TEXTURE_2D => c.glTexStorage2D(
+                @intFromEnum(self.type),
+                1,
+                format,
+                width,
+                height,
+            ),
+
+            .TEXTURE_2D_ARRAY, .TEXTURE_3D => c.glTexStorage3D(
+                @intFromEnum(self.type),
+                1,
+                format,
+                width,
+                height,
+                depth,
+            ),
+            else => unreachable,
+        }
     }
 
-    pub fn allocate3D(self: *const Texture, width: i32, height: i32, depth: i32) void {
+    pub fn setLayer(self: *const Texture, width: i32, height: i32, layer: i32, format: c.GLenum, data: [*]const u8) void {
         c.glBindTexture(@intFromEnum(self.type), self.id);
         defer c.glBindTexture(@intFromEnum(self.type), 0);
 
-        c.glTexStorage3D(
-            @intFromEnum(self.type),
-            1,
-            c.GL_COMPRESSED_RGBA_S3TC_DXT1_EXT, // c.GL_RGBA,
-            width,
-            height,
-            depth,
-        );
+        switch (self.type) {
+            .TEXTURE_2D => c.glTexSubImage2D(
+                @intFromEnum(self.type),
+                0,
+                0,
+                0,
+                width,
+                height,
+                format,
+                c.GL_UNSIGNED_BYTE,
+                data,
+            ),
+            .TEXTURE_2D_ARRAY, .TEXTURE_3D => c.glTexSubImage3D(
+                @intFromEnum(self.type),
+                0,
+                0,
+                0,
+                layer,
+                width,
+                height,
+                1,
+                format,
+                c.GL_UNSIGNED_BYTE,
+                data,
+            ),
+            else => unreachable,
+        }
     }
 
-    pub fn setLayer3D(self: *const Texture, width: i32, height: i32, layer: i32, data: [*]const u8) void {
+    pub fn setLayerCompressed(self: *const Texture, width: i32, height: i32, layer: i32, format: c.GLenum, data: []const u8) void {
         c.glBindTexture(@intFromEnum(self.type), self.id);
         defer c.glBindTexture(@intFromEnum(self.type), 0);
 
-        c.glTexSubImage3D(
-            @intFromEnum(self.type),
-            0,
-            0,
-            0,
-            layer,
-            width,
-            height,
-            1,
-            c.GL_RGBA,
-            c.GL_UNSIGNED_BYTE,
-            data,
-        );
+        switch (self.type) {
+            .TEXTURE_2D => c.glCompressedTexSubImage2D(
+                @intFromEnum(self.type),
+                0,
+                0,
+                0,
+                width,
+                height,
+                format,
+                @intCast(data.len),
+                data.ptr,
+            ),
+            .TEXTURE_2D_ARRAY, .TEXTURE_3D => c.glCompressedTexSubImage3D(
+                @intFromEnum(self.type),
+                0,
+                0,
+                0,
+                layer,
+                width,
+                height,
+                1,
+                format,
+                @intCast(data.len),
+                data.ptr,
+            ),
+            else => unreachable,
+        }
     }
 };
 
diff --git a/src/video.zig b/src/video.zig
index 368d46f..98772b6 100644
--- a/src/video.zig
+++ b/src/video.zig
@@ -2,40 +2,133 @@ const std = @import("std");
 const c = @import("c.zig");
 const gl = @import("gl.zig");
 
-fn check(err: c_int) !void {
+pub const Errors = error{
+    AVGenericError,
+    AVAllocationError,
+    HAPBadArguments,
+    HAPBufferTooSmall,
+    HAPBadFrame,
+    HAPInternalError,
+    UnsupportedCodec,
+    OutOfMemory,
+    NoFrames,
+};
+
+fn check(err: c_int) Errors!void {
     if (err >= 0) return;
     var buf: [c.AV_ERROR_MAX_STRING_SIZE]u8 = undefined;
     _ = c.av_make_error_string(&buf, buf.len, err);
     std.debug.print("libav error: {s}\n", .{buf});
-    return error.AVError;
+    return error.AVGenericError;
 }
 
-pub fn loadVideo(progress_root: std.Progress.Node, filename: [*:0]const u8, texture: *gl.Texture) !void {
-    const outer_progress = progress_root.start("load video", 0);
-    defer outer_progress.end();
+fn hapCheck(err: c_uint) Errors!void {
+    return switch (err) {
+        c.HapResult_No_Error => return,
+        c.HapResult_Bad_Arguments => return error.HAPBadArguments,
+        c.HapResult_Buffer_Too_Small => return error.HAPBufferTooSmall,
+        c.HapResult_Bad_Frame => return error.HAPBadFrame,
+        c.HapResult_Internal_Error => return error.HAPInternalError,
+        else => unreachable,
+    };
+}
 
-    var format_ctx: ?*c.AVFormatContext = null;
-    try check(c.avformat_open_input(&format_ctx, filename, null, null));
-    defer c.avformat_close_input(&format_ctx);
+const Decoder = struct {
+    num_frames: i32,
 
-    if (format_ctx) |format| {
-        try check(c.avformat_find_stream_info(format, null));
+    process_packet_fn: *const fn (self: *Decoder, packet: *c.AVPacket, texture: ?*const gl.Texture) Errors!bool,
+    init_texture_fn: *const fn (self: *Decoder, texture: *const gl.Texture) void,
+    reset_fn: ?*const fn (self: *Decoder) void,
+    deinit_fn: *const fn (self: *Decoder, allocator: std.mem.Allocator) void,
 
-        const video_stream = for (format.streams, 0..format.nb_streams) |c_stream, _| {
-            const stream = @as(*c.AVStream, c_stream orelse unreachable);
-            if (stream.codecpar.*.codec_type == c.AVMEDIA_TYPE_VIDEO) break stream;
-        } else unreachable;
+    fn processStream(
+        self: *Decoder,
+        progress: std.Progress.Node,
+        format: *c.AVFormatContext,
+        stream: *c.AVStream,
+        texture: ?*const gl.Texture,
+    ) Errors!void {
+        defer progress.end();
 
-        const codec_par = video_stream.codecpar.*;
-        const codec = c.avcodec_find_decoder(codec_par.codec_id) orelse unreachable;
-        var codec_ctx = c.avcodec_alloc_context3(codec) orelse unreachable;
-        defer c.avcodec_free_context(&codec_ctx);
+        try check(c.avformat_seek_file(format, stream.index, 0, 0, 0, 0));
+        if (self.reset_fn) |reset_fn| reset_fn(self);
+
+        var packet = c.av_packet_alloc();
+        defer c.av_packet_free(&packet);
+
+        self.num_frames = 0;
+        var res = c.av_read_frame(format, packet);
+        while (res >= 0) : (res = c.av_read_frame(format, packet)) {
+            if (packet.*.stream_index == stream.index) {
+                const more = try self.process_packet_fn(self, packet, texture);
+                progress.setCompletedItems(@intCast(self.num_frames));
+                if (!more) break;
+            }
+            c.av_packet_unref(packet);
+        } else {
+            if (res != c.AVERROR_EOF) try check(res);
+        }
+    }
+
+    pub fn createTexture(
+        self: *Decoder,
+        outer_progress: std.Progress.Node,
+        format: *c.AVFormatContext,
+        stream: *c.AVStream,
+        texture_type: gl.Texture.Type,
+    ) Errors!gl.Texture {
+        try self.processStream(
+            outer_progress.start("scanning video frames", @intCast(stream.nb_frames)),
+            format,
+            stream,
+            null,
+        );
+        if (self.num_frames <= 0) return error.NoFrames;
+
+        const texture = gl.Texture.create(texture_type);
+        errdefer texture.destroy();
+
+        self.init_texture_fn(self, &texture);
+        try self.processStream(
+            outer_progress.start("loading video frames", @intCast(self.num_frames)),
+            format,
+            stream,
+            &texture,
+        );
+        return texture;
+    }
+
+    pub fn deinit(self: *Decoder, allocator: std.mem.Allocator) void {
+        self.deinit_fn(self, allocator);
+    }
+};
+
+const AVDecoder = struct {
+    decoder: Decoder,
+
+    sws_ctx: *c.SwsContext,
+    codec_par: c.AVCodecParameters,
+    codec_ctx: [*c]c.AVCodecContext,
+
+    raw_frame: [*c]c.AVFrame,
+    rgba_frame: [*c]c.AVFrame,
+
+    pub fn init(allocator: std.mem.Allocator, codec_par: c.AVCodecParameters) Errors!*Decoder {
+        const self = try allocator.create(AVDecoder);
+        errdefer allocator.destroy(self);
+
+        const codec = c.avcodec_find_decoder(codec_par.codec_id) orelse return error.UnsupportedCodec;
+
+        var codec_ctx = c.avcodec_alloc_context3(codec) orelse return error.AVAllocationError;
+        errdefer c.avcodec_free_context(&codec_ctx);
+        var raw_frame = c.av_frame_alloc() orelse return error.AVAllocationError;
+        errdefer c.av_frame_free(&raw_frame);
+        var rgba_frame = c.av_frame_alloc() orelse return error.AVAllocationError;
+        errdefer c.av_frame_free(&rgba_frame);
 
         try check(c.avcodec_parameters_to_context(codec_ctx, &codec_par));
         try check(c.avcodec_open2(codec_ctx, codec, null));
-        defer check(c.avcodec_close(codec_ctx)) catch unreachable;
-
-        std.debug.print("opened codec, size {}x{}, ~{} frames\n", .{ codec_par.width, codec_par.height, video_stream.nb_frames });
+        errdefer check(c.avcodec_close(codec_ctx)) catch unreachable;
 
         const sws_ctx = c.sws_getContext(
             codec_par.width,
@@ -48,91 +141,229 @@ pub fn loadVideo(progress_root: std.Progress.Node, filename: [*:0]const u8, text
             null,
             null,
             0,
-        ) orelse unreachable;
-        defer c.sws_freeContext(sws_ctx);
+        ) orelse return error.AVGenericError;
+        errdefer c.sws_freeContext(sws_ctx);
 
-        var packet = c.av_packet_alloc();
-        defer c.av_packet_free(&packet);
+        self.* = .{
+            .decoder = .{
+                .process_packet_fn = processPacket,
+                .init_texture_fn = initTexture,
+                .reset_fn = reset,
+                .deinit_fn = deinit,
+                .num_frames = 0,
+            },
+            .sws_ctx = sws_ctx,
+            .codec_par = codec_par,
+            .codec_ctx = codec_ctx,
+            .raw_frame = raw_frame,
+            .rgba_frame = rgba_frame,
+        };
+        return &self.decoder;
+    }
+
+    pub fn deinit(decoder: *Decoder, allocator: std.mem.Allocator) void {
+        const self: *AVDecoder = @fieldParentPtr("decoder", decoder);
+
+        const raw_ptr: [*c][*c]c.AVFrame = &self.raw_frame;
+        const rgba_ptr: [*c][*c]c.AVFrame = &self.rgba_frame;
+        c.av_frame_free(raw_ptr);
+        c.av_frame_free(rgba_ptr);
+
+        c.sws_freeContext(self.sws_ctx);
+        check(c.avcodec_close(self.codec_ctx)) catch unreachable;
+        c.avcodec_free_context(&self.codec_ctx);
+
+        allocator.destroy(self);
+    }
+
+    pub fn reset(decoder: *Decoder) void {
+        const self: *AVDecoder = @fieldParentPtr("decoder", decoder);
+        c.avcodec_flush_buffers(self.codec_ctx);
+    }
 
-        var raw_frame = c.av_frame_alloc();
-        defer c.av_frame_free(&raw_frame);
-        var rgba_frame = c.av_frame_alloc();
-        defer c.av_frame_free(&rgba_frame);
-
-        const frame_count = blk: {
-            const progress = outer_progress.start("scanning video frames", @intCast(video_stream.nb_frames));
-            defer progress.end();
-
-            var i: usize = 0;
-            var res = c.av_read_frame(format, packet);
-            while (res >= 0) : (res = c.av_read_frame(format, packet)) {
-                if (packet.*.stream_index == video_stream.index) {
-                    try check(c.avcodec_send_packet(codec_ctx, packet));
-
-                    const ires = c.avcodec_receive_frame(codec_ctx, raw_frame);
-                    if (ires == c.AVERROR_EOF) break;
-                    if (ires == c.AVERROR(c.EAGAIN)) continue;
-                    try check(ires);
-
-                    i += 1;
-                    progress.completeOne();
-                }
-                c.av_packet_unref(packet);
+    pub fn initTexture(decoder: *Decoder, texture: *const gl.Texture) void {
+        const self: *AVDecoder = @fieldParentPtr("decoder", decoder);
+
+        texture.allocate(
+            self.codec_par.width,
+            self.codec_par.height,
+            self.decoder.num_frames,
+            c.GL_COMPRESSED_RGBA_S3TC_DXT1_EXT,
+        );
+    }
+
+    pub fn processPacket(decoder: *Decoder, packet: *c.AVPacket, texture: ?*const gl.Texture) Errors!bool {
+        const self: *AVDecoder = @fieldParentPtr("decoder", decoder);
+
+        try check(c.avcodec_send_packet(self.codec_ctx, packet));
+        while (true) {
+            const ires = c.avcodec_receive_frame(self.codec_ctx, self.raw_frame);
+            if (ires == c.AVERROR_EOF) return false;
+            if (ires == c.AVERROR(c.EAGAIN)) return true;
+            try check(ires);
+            try check(c.sws_scale_frame(self.sws_ctx, self.rgba_frame, self.raw_frame));
+
+            c.glPixelStorei(c.GL_UNPACK_ROW_LENGTH, @divFloor(self.rgba_frame.*.linesize[0], @sizeOf(u8) * 4));
+
+            if (texture) |tex| {
+                tex.setLayer(
+                    self.rgba_frame.*.width,
+                    self.rgba_frame.*.height,
+                    self.decoder.num_frames,
+                    c.GL_RGBA,
+                    self.rgba_frame.*.data[0],
+                );
+
+                if (tex.type == .TEXTURE_2D) return false;
             }
 
-            try check(c.avformat_seek_file(format, video_stream.index, 0, 0, 0, 0));
-            c.avcodec_flush_buffers(codec_ctx);
+            self.decoder.num_frames += 1;
+        }
+    }
+};
+
+const HAPDecoder = struct {
+    decoder: Decoder,
+
+    codec_par: c.AVCodecParameters,
+    format: ?c.HapTextureFormat,
+    gl_format: c.GLenum,
+
+    pub fn init(allocator: std.mem.Allocator, codec_par: c.AVCodecParameters) Errors!*Decoder {
+        const self = try allocator.create(HAPDecoder);
 
-            break :blk i;
+        self.* = .{
+            .decoder = .{
+                .process_packet_fn = processPacket,
+                .init_texture_fn = initTexture,
+                .reset_fn = null,
+                .deinit_fn = deinit,
+                .num_frames = 0,
+            },
+            .codec_par = codec_par,
+            .format = null,
+            .gl_format = undefined,
         };
+        return &self.decoder;
+    }
 
-        std.debug.print("counted frames, found {}\n", .{frame_count});
-        switch (texture.type) {
-            .TEXTURE_2D => {},
-            .TEXTURE_3D, .TEXTURE_2D_ARRAY => texture.allocate3D(codec_par.width, codec_par.height, @intCast(frame_count)),
-            else => unreachable,
-        }
+    pub fn deinit(decoder: *Decoder, allocator: std.mem.Allocator) void {
+        const self: *HAPDecoder = @fieldParentPtr("decoder", decoder);
+        allocator.destroy(self);
+    }
 
-        {
-            const progress = outer_progress.start("loading video frames", frame_count);
-            defer progress.end();
+    pub fn initTexture(decoder: *Decoder, texture: *const gl.Texture) void {
+        const self: *HAPDecoder = @fieldParentPtr("decoder", decoder);
 
-            defer c.glPixelStorei(c.GL_UNPACK_ROW_LENGTH, 0);
+        texture.allocate(
+            self.codec_par.width,
+            self.codec_par.height,
+            self.decoder.num_frames,
+            self.gl_format,
+        );
+    }
 
-            var i: i32 = 0;
-            var res = c.av_read_frame(format, packet);
-            while (res >= 0) : (res = c.av_read_frame(format, packet)) {
-                if (packet.*.stream_index == video_stream.index) {
-                    try check(c.avcodec_send_packet(codec_ctx, packet));
+    fn launchThread(workfn: c.HapDecodeWorkFunction, p: ?*anyopaque, count: c_uint, info: ?*anyopaque) callconv(.C) void {
+        _ = info;
 
-                    const ires = c.avcodec_receive_frame(codec_ctx, raw_frame);
-                    if (ires == c.AVERROR_EOF) break;
-                    if (ires == c.AVERROR(c.EAGAIN)) continue;
-                    try check(ires);
-                    try check(c.sws_scale_frame(sws_ctx, rgba_frame, raw_frame));
+        var i: c_uint = 0;
+        while (i < count) : (i += 1) {
+            workfn.?(p, i);
+        }
+    }
 
-                    c.glPixelStorei(c.GL_UNPACK_ROW_LENGTH, @divFloor(rgba_frame.*.linesize[0], @sizeOf(u8) * 4));
+    pub fn processPacket(decoder: *Decoder, packet: *c.AVPacket, texture: ?*const gl.Texture) Errors!bool {
+        const self: *HAPDecoder = @fieldParentPtr("decoder", decoder);
 
-                    switch (texture.type) {
-                        .TEXTURE_2D => {
-                            texture.setData2D(codec_par.width, codec_par.height, rgba_frame.*.data[0]);
-                            break;
-                        },
+        var textureCount: u32 = undefined;
+        try hapCheck(c.HapGetFrameTextureCount(packet.data, @intCast(packet.size), &textureCount));
+        if (textureCount != 1) {
+            return error.UnsupportedCodec;
+        }
 
-                        .TEXTURE_3D,
-                        .TEXTURE_2D_ARRAY,
-                        => texture.setLayer3D(codec_par.width, codec_par.height, i, rgba_frame.*.data[0]),
+        var format: c.HapTextureFormat = undefined;
 
-                        else => unreachable,
-                    }
+        if (texture) |tex| {
+            var buffer: [16 * 1024 * 1024]u8 = undefined;
+            var length: c_ulong = undefined;
+            try hapCheck(c.HapDecode(
+                packet.data,
+                @intCast(packet.size),
+                0,
+                launchThread,
+                self,
+                &buffer,
+                buffer.len,
+                &length,
+                &format,
+            ));
 
-                    i += 1;
-                    progress.completeOne();
-                }
-                c.av_packet_unref(packet);
+            tex.setLayerCompressed(
+                self.codec_par.width,
+                self.codec_par.height,
+                self.decoder.num_frames,
+                self.gl_format,
+                buffer[0..length],
+            );
+
+            if (tex.type == .TEXTURE_2D) return false;
+        } else {
+            try hapCheck(c.HapGetFrameTextureFormat(packet.data, @intCast(packet.size), 0, &format));
+            if (self.format) |fmt| {
+                if (fmt != format) return error.UnsupportedCodec;
             } else {
-                if (res != c.AVERROR_EOF) try check(res);
+                self.format = format;
+                self.gl_format = switch (format) {
+                    c.HapTextureFormat_RGB_DXT1 => c.GL_COMPRESSED_RGB_S3TC_DXT1_EXT,
+                    c.HapTextureFormat_RGBA_DXT5 => c.GL_COMPRESSED_RGBA_S3TC_DXT5_EXT,
+                    c.HapTextureFormat_YCoCg_DXT5,
+                    c.HapTextureFormat_A_RGTC1,
+                    c.HapTextureFormat_RGBA_BPTC_UNORM,
+                    c.HapTextureFormat_RGB_BPTC_UNSIGNED_FLOAT,
+                    c.HapTextureFormat_RGB_BPTC_SIGNED_FLOAT,
+                    => return error.UnsupportedCodec,
+                    else => unreachable,
+                };
             }
         }
+
+        self.decoder.num_frames += 1;
+        return true;
+    }
+};
+
+pub fn loadVideo(progress_root: std.Progress.Node, filename: [*:0]const u8, texture_type: gl.Texture.Type) !gl.Texture {
+    const progress = progress_root.start("loading texture", 2);
+    defer progress.end();
+
+    var format_ctx: ?*c.AVFormatContext = null;
+    try check(c.avformat_open_input(&format_ctx, filename, null, null));
+    defer c.avformat_close_input(&format_ctx);
+
+    if (format_ctx) |format| {
+        try check(c.avformat_find_stream_info(format, null));
+
+        const video_stream = for (format.streams, 0..format.nb_streams) |c_stream, _| {
+            const stream = @as(*c.AVStream, c_stream orelse unreachable);
+            if (stream.codecpar.*.codec_type == c.AVMEDIA_TYPE_VIDEO) break stream;
+        } else unreachable;
+
+        const codec_par = video_stream.codecpar.*;
+        std.debug.print("loading codec, size {}x{}, ~{} frames\n", .{ codec_par.width, codec_par.height, video_stream.nb_frames });
+
+        var packet = c.av_packet_alloc();
+        defer c.av_packet_free(&packet);
+
+        const allocator = std.heap.c_allocator;
+
+        const decoder = switch (codec_par.codec_tag) {
+            c.MKTAG('H', 'a', 'p', '1'),
+            c.MKTAG('H', 'a', 'p', '5'),
+            => try HAPDecoder.init(allocator, codec_par),
+            else => try AVDecoder.init(allocator, codec_par),
+        };
+        defer decoder.deinit(allocator);
+
+        return try decoder.createTexture(progress, format, video_stream, texture_type);
     } else unreachable;
 }