[deflate] Add support for DEFLATE decompression

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/include/ipxe/deflate.h

@@ -0,0 +1,283 @@
+#ifndef _IPXE_DEFLATE_H
+#define _IPXE_DEFLATE_H
+
+/** @file
+ *
+ * DEFLATE decompression algorithm
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include <stdint.h>
+#include <string.h>
+#include <ipxe/uaccess.h>
+
+/** Compression formats */
+enum deflate_format {
+	/** Raw DEFLATE data (no header or footer) */
+	DEFLATE_RAW,
+	/** ZLIB header and footer */
+	DEFLATE_ZLIB,
+};
+
+/** Block header length (in bits) */
+#define DEFLATE_HEADER_BITS 3
+
+/** Block header final block flags bit */
+#define DEFLATE_HEADER_BFINAL_BIT 0
+
+/** Block header type LSB */
+#define DEFLATE_HEADER_BTYPE_LSB 1
+
+/** Block header type mask */
+#define DEFLATE_HEADER_BTYPE_MASK 0x03
+
+/** Block header type: literal data */
+#define DEFLATE_HEADER_BTYPE_LITERAL 0
+
+/** Block header type: static Huffman alphabet */
+#define DEFLATE_HEADER_BTYPE_STATIC 1
+
+/** Block header type: dynamic Huffman alphabet */
+#define DEFLATE_HEADER_BTYPE_DYNAMIC 2
+
+/** Literal header LEN/NLEN field length (in bits) */
+#define DEFLATE_LITERAL_LEN_BITS 16
+
+/** Dynamic header length (in bits) */
+#define DEFLATE_DYNAMIC_BITS 14
+
+/** Dynamic header HLIT field LSB */
+#define DEFLATE_DYNAMIC_HLIT_LSB 0
+
+/** Dynamic header HLIT field mask */
+#define DEFLATE_DYNAMIC_HLIT_MASK 0x1f
+
+/** Dynamic header HDIST field LSB */
+#define DEFLATE_DYNAMIC_HDIST_LSB 5
+
+/** Dynamic header HDIST field mask */
+#define DEFLATE_DYNAMIC_HDIST_MASK 0x1f
+
+/** Dynamic header HCLEN field LSB */
+#define DEFLATE_DYNAMIC_HCLEN_LSB 10
+
+/** Dynamic header HCLEN field mask */
+#define DEFLATE_DYNAMIC_HCLEN_MASK 0x0f
+
+/** Dynamic header code length length (in bits) */
+#define DEFLATE_CODELEN_BITS 3
+
+/** Maximum length of a Huffman symbol (in bits) */
+#define DEFLATE_HUFFMAN_BITS 15
+
+/** Quick lookup length for a Huffman symbol (in bits)
+ *
+ * This is a policy decision.
+ */
+#define DEFLATE_HUFFMAN_QL_BITS 7
+
+/** Quick lookup shift */
+#define DEFLATE_HUFFMAN_QL_SHIFT ( 16 - DEFLATE_HUFFMAN_QL_BITS )
+
+/** Literal/length end of block code */
+#define DEFLATE_LITLEN_END 256
+
+/** Maximum value of a literal/length code */
+#define DEFLATE_LITLEN_MAX_CODE 287
+
+/** Maximum value of a distance code */
+#define DEFLATE_DISTANCE_MAX_CODE 31
+
+/** Maximum value of a code length code */
+#define DEFLATE_CODELEN_MAX_CODE 18
+
+/** ZLIB header length (in bits) */
+#define ZLIB_HEADER_BITS 16
+
+/** ZLIB header compression method LSB */
+#define ZLIB_HEADER_CM_LSB 0
+
+/** ZLIB header compression method mask */
+#define ZLIB_HEADER_CM_MASK 0x0f
+
+/** ZLIB header compression method: DEFLATE */
+#define ZLIB_HEADER_CM_DEFLATE 8
+
+/** ZLIB header preset dictionary flag bit */
+#define ZLIB_HEADER_FDICT_BIT 13
+
+/** ZLIB ADLER32 length (in bits) */
+#define ZLIB_ADLER32_BITS 32
+
+/** A Huffman-coded set of symbols of a given length */
+struct deflate_huf_symbols {
+	/** Length of Huffman-coded symbols */
+	uint8_t bits;
+	/** Shift to normalise symbols of this length to 16 bits */
+	uint8_t shift;
+	/** Number of Huffman-coded symbols having this length */
+	uint16_t freq;
+	/** First symbol of this length (normalised to 16 bits)
+	 *
+	 * Stored as a 32-bit value to allow the value 0x10000 to be
+	 * used for empty sets of symbols longer than the maximum
+	 * utilised length.
+	 */
+	uint32_t start;
+	/** Raw symbols having this length */
+	uint16_t *raw;
+};
+
+/** A Huffman-coded alphabet */
+struct deflate_alphabet {
+	/** Huffman-coded symbol set for each length */
+	struct deflate_huf_symbols huf[DEFLATE_HUFFMAN_BITS];
+	/** Quick lookup table */
+	uint8_t lookup[ 1 << DEFLATE_HUFFMAN_QL_BITS ];
+	/** Raw symbols
+	 *
+	 * Ordered by Huffman-coded symbol length, then by symbol
+	 * value.  This field has a variable length.
+	 */
+	uint16_t raw[0];
+};
+
+/** A static Huffman alphabet length pattern */
+struct deflate_static_length_pattern {
+	/** Length pair */
+	uint8_t fill;
+	/** Repetition count */
+	uint8_t count;
+} __attribute__ (( packed ));
+
+/** Decompressor */
+struct deflate {
+	/** Resume point
+	 *
+	 * Used as the target of a computed goto to jump to the
+	 * appropriate point within the state machine.
+	 */
+	void *resume;
+	/** Format */
+	enum deflate_format format;
+
+	/** Accumulator */
+	uint32_t accumulator;
+	/** Bit-reversed accumulator
+	 *
+	 * Don't ask.
+	 */
+	uint32_t rotalumucca;
+	/** Number of bits within the accumulator */
+	unsigned int bits;
+
+	/** Current block header */
+	unsigned int header;
+	/** Remaining length of data (e.g. within a literal block) */
+	size_t remaining;
+	/** Current length index within a set of code lengths */
+	unsigned int length_index;
+	/** Target length index within a set of code lengths */
+	unsigned int length_target;
+	/** Current length within a set of code lengths */
+	unsigned int length;
+	/** Number of extra bits required */
+	unsigned int extra_bits;
+	/** Length of a duplicated string */
+	size_t dup_len;
+	/** Distance of a duplicated string */
+	size_t dup_distance;
+
+	/** Literal/length Huffman alphabet */
+	struct deflate_alphabet litlen;
+	/** Literal/length raw symbols
+	 *
+	 * Must immediately follow the literal/length Huffman alphabet.
+	 */
+	uint16_t litlen_raw[ DEFLATE_LITLEN_MAX_CODE + 1 ];
+	/** Number of symbols in the literal/length Huffman alphabet */
+	unsigned int litlen_count;
+
+	/** Distance and code length Huffman alphabet
+	 *
+	 * The code length Huffman alphabet has a maximum Huffman
+	 * symbol length of 7 and a maximum code value of 18, and is
+	 * thus strictly smaller than the distance Huffman alphabet.
+	 * Since we never need both alphabets simultaneously, we can
+	 * reuse the storage space for the distance alphabet to
+	 * temporarily hold the code length alphabet.
+	 */
+	struct deflate_alphabet distance_codelen;
+	/** Distance and code length raw symbols
+	 *
+	 * Must immediately follow the distance and code length
+	 * Huffman alphabet.
+	 */
+	uint16_t distance_codelen_raw[ DEFLATE_DISTANCE_MAX_CODE + 1 ];
+	/** Number of symbols in the distance Huffman alphabet */
+	unsigned int distance_count;
+
+	/** Huffman code lengths
+	 *
+	 * The literal/length and distance code lengths are
+	 * constructed as a single set of lengths.
+	 *
+	 * The code length Huffman alphabet has a maximum code value
+	 * of 18 and the set of lengths is thus strictly smaller than
+	 * the combined literal/length and distance set of lengths.
+	 * Since we never need both alphabets simultaneously, we can
+	 * reuse the storage space for the literal/length and distance
+	 * code lengths to temporarily hold the code length code
+	 * lengths.
+	 */
+	uint8_t lengths[ ( ( DEFLATE_LITLEN_MAX_CODE + 1 ) +
+			   ( DEFLATE_DISTANCE_MAX_CODE + 1 ) +
+			   1 /* round up */ ) / 2 ];
+};
+
+/** A chunk of data */
+struct deflate_chunk {
+	/** Data */
+	userptr_t data;
+	/** Current offset */
+	size_t offset;
+	/** Length of data */
+	size_t len;
+};
+
+/**
+ * Initialise chunk of data
+ *
+ * @v chunk		Chunk of data to initialise
+ * @v data		Data
+ * @v offset		Starting offset
+ * @v len		Length
+ */
+static inline __attribute__ (( always_inline )) void
+deflate_chunk_init ( struct deflate_chunk *chunk, userptr_t data,
+		     size_t offset, size_t len ) {
+
+	chunk->data = data;
+	chunk->offset = offset;
+	chunk->len = len;
+}
+
+/**
+ * Check if decompression has finished
+ *
+ * @v deflate		Decompressor
+ * @ret finished	Decompression has finished
+ */
+static inline int deflate_finished ( struct deflate *deflate ) {
+	return ( deflate->resume == NULL );
+}
+
+extern void deflate_init ( struct deflate *deflate,
+			   enum deflate_format format );
+extern int deflate_inflate ( struct deflate *deflate,
+			     struct deflate_chunk *in,
+			     struct deflate_chunk *out );
+
+#endif /* _IPXE_DEFLATE_H */

src/include/ipxe/errfile.h

@@ -297,6 +297,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #define ERRFILE_efi_reboot	      ( ERRFILE_OTHER | 0x003e0000 )
 #define ERRFILE_memmap_settings	      ( ERRFILE_OTHER | 0x003f0000 )
 #define ERRFILE_param_cmd	      ( ERRFILE_OTHER | 0x00400000 )
+#define ERRFILE_deflate		      ( ERRFILE_OTHER | 0x00410000 )
 
 /** @} */
 

src/tests/deflate_test.c

@@ -0,0 +1,239 @@
+/*
+ * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+/** @file
+ *
+ * DEFLATE tests
+ *
+ */
+
+/* Forcibly enable assertions */
+#undef NDEBUG
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ipxe/deflate.h>
+#include <ipxe/test.h>
+
+/** A DEFLATE test */
+struct deflate_test {
+	/** Compression format */
+	enum deflate_format format;
+	/** Compressed data */
+	const void *compressed;
+	/** Length of compressed data */
+	size_t compressed_len;
+	/** Expected uncompressed data */
+	const void *expected;
+	/** Length of expected uncompressed data */
+	size_t expected_len;
+};
+
+/** A DEFLATE fragment list */
+struct deflate_test_fragments {
+	/** Fragment lengths */
+	size_t len[8];
+};
+
+/** Define inline data */
+#define DATA(...) { __VA_ARGS__ }
+
+/** Define a DEFLATE test */
+#define DEFLATE( name, FORMAT, COMPRESSED, EXPECTED )			\
+	static const uint8_t name ## _compressed[] = COMPRESSED;	\
+	static const uint8_t name ## _expected[] = EXPECTED;		\
+	static struct deflate_test name = {				\
+		.format = FORMAT,					\
+		.compressed = name ## _compressed,			\
+		.compressed_len = sizeof ( name ## _compressed ),	\
+		.expected = name ## _expected,				\
+		.expected_len = sizeof ( name ## _expected ),		\
+	};
+
+/* Empty file, no compression */
+DEFLATE ( empty_literal, DEFLATE_RAW,
+	  DATA ( 0x01, 0x00, 0x00, 0xff, 0xff ), DATA() );
+
+/* "iPXE" string, no compression */
+DEFLATE ( literal, DEFLATE_RAW,
+	  DATA ( 0x01, 0x04, 0x00, 0xfb, 0xff, 0x69, 0x50, 0x58, 0x45 ),
+	  DATA ( 0x69, 0x50, 0x58, 0x45 ) );
+
+/* Empty file */
+DEFLATE ( empty, DEFLATE_RAW, DATA ( 0x03, 0x00 ), DATA() );
+
+/* "Hello world" */
+DEFLATE ( hello_world, DEFLATE_RAW,
+	  DATA ( 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca,
+		 0x49, 0x01, 0x00 ),
+	  DATA ( 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c,
+		 0x64 ) );
+
+/* "Hello hello world" */
+DEFLATE ( hello_hello_world, DEFLATE_RAW,
+	  DATA ( 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0xc8, 0x00, 0x93, 0xe5,
+		 0xf9, 0x45, 0x39, 0x29, 0x00 ),
+	  DATA ( 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x68, 0x65, 0x6c, 0x6c,
+		 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64 ) );
+
+/* "This specification defines a lossless compressed data format" */
+DEFLATE ( rfc_sentence, DEFLATE_RAW,
+	  DATA ( 0x0d, 0xc6, 0xdb, 0x09, 0x00, 0x21, 0x0c, 0x04, 0xc0, 0x56,
+		 0xb6, 0x28, 0x1b, 0x08, 0x79, 0x70, 0x01, 0x35, 0xe2, 0xa6,
+		 0x7f, 0xce, 0xf9, 0x9a, 0xf1, 0x25, 0xc1, 0xe3, 0x9a, 0x91,
+		 0x2a, 0x9d, 0xb5, 0x61, 0x1e, 0xb9, 0x9d, 0x10, 0xcc, 0x22,
+		 0xa7, 0x93, 0xd0, 0x5a, 0xe7, 0xbe, 0xb8, 0xc1, 0xa4, 0x05,
+		 0x51, 0x77, 0x49, 0xff ),
+	  DATA ( 0x54, 0x68, 0x69, 0x73, 0x20, 0x73, 0x70, 0x65, 0x63, 0x69,
+		 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x64,
+		 0x65, 0x66, 0x69, 0x6e, 0x65, 0x73, 0x20, 0x61, 0x20, 0x6c,
+		 0x6f, 0x73, 0x73, 0x6c, 0x65, 0x73, 0x73, 0x20, 0x63, 0x6f,
+		 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x20, 0x64,
+		 0x61, 0x74, 0x61, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74 ) );
+
+/* "ZLIB Compressed Data Format Specification" */
+DEFLATE ( zlib, DEFLATE_ZLIB,
+	  DATA ( 0x78, 0x01, 0x8b, 0xf2, 0xf1, 0x74, 0x52, 0x70, 0xce, 0xcf,
+		 0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0x4e, 0x4d, 0x51, 0x70, 0x49,
+		 0x2c, 0x49, 0x54, 0x70, 0xcb, 0x2f, 0xca, 0x4d, 0x2c, 0x51,
+		 0x08, 0x2e, 0x48, 0x4d, 0xce, 0x4c, 0xcb, 0x4c, 0x4e, 0x2c,
+		 0xc9, 0xcc, 0xcf, 0x03, 0x00, 0x2c, 0x0e, 0x0e, 0xeb ),
+	  DATA ( 0x5a, 0x4c, 0x49, 0x42, 0x20, 0x43, 0x6f, 0x6d, 0x70, 0x72,
+		 0x65, 0x73, 0x73, 0x65, 0x64, 0x20, 0x44, 0x61, 0x74, 0x61,
+		 0x20, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x53, 0x70,
+		 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+		 0x6e ) );
+
+/* "ZLIB Compressed Data Format Specification" fragment list */
+static struct deflate_test_fragments zlib_fragments[] = {
+	{ { -1UL, } },
+	{ { 0, 1, 5, -1UL, } },
+	{ { 0, 0, 1, 0, 0, 1, -1UL } },
+	{ { 10, 8, 4, 7, 11, -1UL } },
+	{ { 45, -1UL } },
+	{ { 48, -1UL } },
+};
+
+/**
+ * Report DEFLATE test result
+ *
+ * @v deflate		Decompressor
+ * @v test		Deflate test
+ * @v frags		Fragment list, or NULL
+ * @v file		Test code file
+ * @v line		Test code line
+ */
+static void deflate_okx ( struct deflate *deflate,
+			  struct deflate_test *test,
+			  struct deflate_test_fragments *frags,
+			  const char *file, unsigned int line ) {
+	uint8_t data[ test->expected_len ];
+	struct deflate_chunk in;
+	struct deflate_chunk out;
+	size_t frag_len = -1UL;
+	size_t offset = 0;
+	size_t remaining = test->compressed_len;
+	unsigned int i;
+
+	/* Initialise decompressor */
+	deflate_init ( deflate, test->format );
+
+	/* Initialise output chunk */
+	deflate_chunk_init ( &out, virt_to_user ( data ), 0, sizeof ( data ) );
+
+	/* Process input (in fragments, if applicable) */
+	for ( i = 0 ; i < ( sizeof ( frags->len ) /
+			    sizeof ( frags->len[0] ) ) ; i++ ) {
+
+		/* Initialise input chunk */
+		if ( frags )
+			frag_len = frags->len[i];
+		if ( frag_len > remaining )
+			frag_len = remaining;
+		deflate_chunk_init ( &in, virt_to_user ( test->compressed ),
+				     offset, ( offset + frag_len ) );
+
+		/* Decompress this fragment */
+		okx ( deflate_inflate ( deflate, &in, &out ) == 0, file, line );
+		okx ( in.len == ( offset + frag_len ), file, line );
+		okx ( in.offset == in.len, file, line );
+
+		/* Move to next fragment */
+		offset = in.offset;
+		remaining -= frag_len;
+		if ( ! remaining )
+			break;
+
+		/* Check that decompression has not terminated early */
+		okx ( ! deflate_finished ( deflate ), file, line );
+	}
+
+	/* Check decompression has terminated as expected */
+	okx ( deflate_finished ( deflate ), file, line );
+	okx ( offset == test->compressed_len, file, line );
+	okx ( out.offset == test->expected_len, file, line );
+	okx ( memcmp ( data, test->expected, test->expected_len ) == 0,
+	     file, line );
+}
+#define deflate_ok( deflate, test, frags ) \
+	deflate_okx ( deflate, test, frags, __FILE__, __LINE__ )
+
+/**
+ * Perform DEFLATE self-test
+ *
+ */
+static void deflate_test_exec ( void ) {
+	struct deflate *deflate;
+	unsigned int i;
+
+	/* Allocate shared structure */
+	deflate = malloc ( sizeof ( *deflate ) );
+	ok ( deflate != NULL );
+
+	/* Perform self-tests */
+	if ( deflate ) {
+
+		/* Test as a single pass */
+		deflate_ok ( deflate, &empty_literal, NULL );
+		deflate_ok ( deflate, &literal, NULL );
+		deflate_ok ( deflate, &empty, NULL );
+		deflate_ok ( deflate, &hello_world, NULL );
+		deflate_ok ( deflate, &hello_hello_world, NULL );
+		deflate_ok ( deflate, &rfc_sentence, NULL );
+		deflate_ok ( deflate, &zlib, NULL );
+
+		/* Test fragmentation */
+		for ( i = 0 ; i < ( sizeof ( zlib_fragments ) /
+				    sizeof ( zlib_fragments[0] ) ) ; i++ ) {
+			deflate_ok ( deflate, &zlib, &zlib_fragments[i] );
+		}
+	}
+
+	/* Free shared structure */
+	free ( deflate );
+}
+
+/** DEFLATE self-test */
+struct self_test deflate_test __self_test = {
+	.name = "deflate",
+	.exec = deflate_test_exec,
+};

src/tests/tests.c

@@ -50,3 +50,4 @@ REQUIRE_OBJECT ( x509_test );
 REQUIRE_OBJECT ( ocsp_test );
 REQUIRE_OBJECT ( cms_test );
 REQUIRE_OBJECT ( pnm_test );
+REQUIRE_OBJECT ( deflate_test );