[arm] Add support for 64-bit ARM (Aarch64)

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/arch/arm/core/arm_io.c

@@ -84,5 +84,10 @@ PROVIDE_IOAPI_INLINE ( arm, writew );
 PROVIDE_IOAPI_INLINE ( arm, writel );
 PROVIDE_IOAPI_INLINE ( arm, iodelay );
 PROVIDE_IOAPI_INLINE ( arm, mb );
+#ifdef __aarch64__
+PROVIDE_IOAPI_INLINE ( arm, readq );
+PROVIDE_IOAPI_INLINE ( arm, writeq );
+#else
 PROVIDE_IOAPI ( arm, readq, arm32_readq );
 PROVIDE_IOAPI ( arm, writeq, arm32_writeq );
+#endif

src/arch/arm/include/bits/xen.h

@@ -10,12 +10,21 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 /* Hypercall registers */
+#ifdef __aarch64__
+#define XEN_HC "x16"
+#define XEN_REG1 "x0"
+#define XEN_REG2 "x1"
+#define XEN_REG3 "x2"
+#define XEN_REG4 "x3"
+#define XEN_REG5 "x4"
+#else
 #define XEN_HC "r12"
 #define XEN_REG1 "r0"
 #define XEN_REG2 "r1"
 #define XEN_REG3 "r2"
 #define XEN_REG4 "r3"
 #define XEN_REG5 "r4"
+#endif
 
 /**
  * Issue hypercall with one argument

src/arch/arm/include/ipxe/arm_io.h

@@ -43,34 +43,46 @@ IOAPI_INLINE ( arm, bus_to_phys ) ( unsigned long bus_addr ) {
  *
  */
 
-#define ARM_READX( _api_func, _type, _insn_suffix )			      \
+#define ARM_READX( _api_func, _type, _insn_suffix, _reg_prefix )	      \
 static inline __always_inline _type					      \
 IOAPI_INLINE ( arm, _api_func ) ( volatile _type *io_addr ) {		      \
 	_type data;							      \
-	__asm__ __volatile__ ( "ldr" _insn_suffix " %0, %1"		      \
+	__asm__ __volatile__ ( "ldr" _insn_suffix " %" _reg_prefix "0, %1"    \
 			       : "=r" ( data ) : "Qo" ( *io_addr ) );	      \
 	return data;							      \
 }
-ARM_READX ( readb, uint8_t, "b" );
-ARM_READX ( readw, uint16_t, "h" );
-ARM_READX ( readl, uint32_t, "" );
+#ifdef __aarch64__
+ARM_READX ( readb, uint8_t, "b", "w" );
+ARM_READX ( readw, uint16_t, "h", "w" );
+ARM_READX ( readl, uint32_t, "", "w" );
+ARM_READX ( readq, uint64_t, "", "" );
+#else
+ARM_READX ( readb, uint8_t, "b", "" );
+ARM_READX ( readw, uint16_t, "h", "" );
+ARM_READX ( readl, uint32_t, "", "" );
+#endif
 
-#define ARM_WRITEX( _api_func, _type, _insn_suffix )			      \
+#define ARM_WRITEX( _api_func, _type, _insn_suffix, _reg_prefix )			\
 static inline __always_inline void					      \
-IOAPI_INLINE ( arm, _api_func ) ( _type data,				      \
-				  volatile _type *io_addr ) {		      \
-	__asm__ __volatile__ ( "str" _insn_suffix " %0, %1"		      \
+IOAPI_INLINE ( arm, _api_func ) ( _type data, volatile _type *io_addr ) {     \
+	__asm__ __volatile__ ( "str" _insn_suffix " %" _reg_prefix "0, %1"    \
 			       : : "r" ( data ), "Qo" ( *io_addr ) );	      \
 }
-ARM_WRITEX ( writeb, uint8_t, "b" );
-ARM_WRITEX ( writew, uint16_t, "h" );
-ARM_WRITEX ( writel, uint32_t, "" );
+#ifdef __aarch64__
+ARM_WRITEX ( writeb, uint8_t, "b", "w" );
+ARM_WRITEX ( writew, uint16_t, "h", "w" );
+ARM_WRITEX ( writel, uint32_t, "", "w" );
+ARM_WRITEX ( writeq, uint64_t, "", "" );
+#else
+ARM_WRITEX ( writeb, uint8_t, "b", "" );
+ARM_WRITEX ( writew, uint16_t, "h", "" );
+ARM_WRITEX ( writel, uint32_t, "", "" );
+#endif
 
 /*
  * Slow down I/O
  *
  */
-
 static inline __always_inline void
 IOAPI_INLINE ( arm, iodelay ) ( void ) {
 	/* Nothing to do */
@@ -80,10 +92,14 @@ IOAPI_INLINE ( arm, iodelay ) ( void ) {
  * Memory barrier
  *
  */
-
 static inline __always_inline void
 IOAPI_INLINE ( arm, mb ) ( void ) {
+
+#ifdef __aarch64__
+	__asm__ __volatile__ ( "dmb sy" );
+#else
 	__asm__ __volatile__ ( "dmb" );
+#endif
 }
 
 #endif /* _IPXE_ARM_IO_H */

src/arch/arm64/Makefile

@@ -0,0 +1,22 @@
+# ARM64-specific directories containing source files
+#
+SRCDIRS		+= arch/arm64/core
+
+# ARM64-specific flags
+#
+CFLAGS		+= -mabi=lp64 -mlittle-endian -mcmodel=small
+CFLAGS		+= -fomit-frame-pointer
+ASFLAGS		+= -mabi=lp64 -EL
+
+# EFI requires -fshort-wchar, and nothing else currently uses wchar_t
+#
+CFLAGS		+= -fshort-wchar
+
+# Include common ARM Makefile
+MAKEDEPS	+= arch/arm/Makefile
+include arch/arm/Makefile
+
+# Include platform-specific Makefile
+#
+MAKEDEPS	+= arch/arm64/Makefile.$(PLATFORM)
+include arch/arm64/Makefile.$(PLATFORM)

src/arch/arm64/Makefile.efi

@@ -0,0 +1,14 @@
+# -*- makefile -*- : Force emacs to use Makefile mode
+
+# Specify EFI image builder
+#
+ELF2EFI		= $(ELF2EFI64)
+
+# Specify EFI boot file
+#
+EFI_BOOT_FILE	= bootaa64.efi
+
+# Include generic EFI Makefile
+#
+MAKEDEPS	+= arch/arm/Makefile.efi
+include arch/arm/Makefile.efi

src/arch/arm64/core/arm64_bigint.c

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <ipxe/bigint.h>
+
+/** @file
+ *
+ * Big integer support
+ */
+
+/**
+ * Multiply big integers
+ *
+ * @v multiplicand0	Element 0 of big integer to be multiplied
+ * @v multiplier0	Element 0 of big integer to be multiplied
+ * @v result0		Element 0 of big integer to hold result
+ * @v size		Number of elements
+ */
+void bigint_multiply_raw ( const uint64_t *multiplicand0,
+			   const uint64_t *multiplier0,
+			   uint64_t *result0, unsigned int size ) {
+	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplicand =
+		( ( const void * ) multiplicand0 );
+	const bigint_t ( size ) __attribute__ (( may_alias )) *multiplier =
+		( ( const void * ) multiplier0 );
+	bigint_t ( size * 2 ) __attribute__ (( may_alias )) *result =
+		( ( void * ) result0 );
+	unsigned int i;
+	unsigned int j;
+	uint64_t multiplicand_element;
+	uint64_t multiplier_element;
+	uint64_t *result_elements;
+	uint64_t discard_low;
+	uint64_t discard_high;
+	uint64_t discard_temp_low;
+	uint64_t discard_temp_high;
+
+	/* Zero result */
+	memset ( result, 0, sizeof ( *result ) );
+
+	/* Multiply integers one element at a time */
+	for ( i = 0 ; i < size ; i++ ) {
+		multiplicand_element = multiplicand->element[i];
+		for ( j = 0 ; j < size ; j++ ) {
+			multiplier_element = multiplier->element[j];
+			result_elements = &result->element[ i + j ];
+			/* Perform a single multiply, and add the
+			 * resulting double-element into the result,
+			 * carrying as necessary.  The carry can
+			 * never overflow beyond the end of the
+			 * result, since:
+			 *
+			 *     a < 2^{n}, b < 2^{n} => ab < 2^{2n}
+			 */
+			__asm__ __volatile__ ( "mul %1, %6, %7\n\t"
+					       "umulh %2, %6, %7\n\t"
+					       "ldp %3, %4, [%0]\n\t"
+					       "adds %3, %3, %1\n\t"
+					       "adcs %4, %4, %2\n\t"
+					       "stp %3, %4, [%0], #16\n\t"
+					       "bcc 2f\n\t"
+					       "\n1:\n\t"
+					       "ldr %3, [%0]\n\t"
+					       "adcs %3, %3, xzr\n\t"
+					       "str %3, [%0], #8\n\t"
+					       "bcs 1b\n\t"
+					       "\n2:\n\t"
+					       : "+r" ( result_elements ),
+						 "=&r" ( discard_low ),
+						 "=&r" ( discard_high ),
+						 "=r" ( discard_temp_low ),
+						 "=r" ( discard_temp_high ),
+						 "+m" ( *result )
+					       : "r" ( multiplicand_element ),
+						 "r" ( multiplier_element )
+					       : "cc" );
+		}
+	}
+}

src/arch/arm64/core/setjmp.S

@@ -0,0 +1,56 @@
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )
+
+	.text
+
+	/* Must match jmp_buf structure layout */
+	.struct	0
+env_x19_x20:	.quad	0, 0
+env_x21_x22:	.quad	0, 0
+env_x23_x24:	.quad	0, 0
+env_x25_x26:	.quad	0, 0
+env_x27_x28:	.quad	0, 0
+env_x29_x30:	.quad	0, 0
+env_sp:		.quad	0
+	.previous
+
+/*
+ * Save stack context for non-local goto
+ */
+	.globl	setjmp
+	.type	setjmp, %function
+setjmp:
+	/* Store registers */
+	stp	x19, x20, [x0, #env_x19_x20]
+	stp	x21, x22, [x0, #env_x21_x22]
+	stp	x23, x24, [x0, #env_x23_x24]
+	stp	x25, x26, [x0, #env_x25_x26]
+	stp	x27, x28, [x0, #env_x27_x28]
+	stp	x29, x30, [x0, #env_x29_x30]
+	mov	x16, sp
+	str	x16, [x0, #env_sp]
+	/* Return 0 when returning as setjmp() */
+	mov	x0, #0
+	ret
+	.size	setjmp, . - setjmp
+
+/*
+ * Non-local jump to a saved stack context
+ */
+	.globl	longjmp
+	.type	longjmp, %function
+longjmp:
+	/* Restore registers */
+	ldp	x19, x20, [x0, #env_x19_x20]
+	ldp	x21, x22, [x0, #env_x21_x22]
+	ldp	x23, x24, [x0, #env_x23_x24]
+	ldp	x25, x26, [x0, #env_x25_x26]
+	ldp	x27, x28, [x0, #env_x27_x28]
+	ldp	x29, x30, [x0, #env_x29_x30]
+	ldr	x16, [x0, #env_sp]
+	mov	sp, x16
+	/* Force result to non-zero */
+	cmp	w1, #0
+	csinc	w0, w1, w1, ne
+	/* Return to setjmp() caller */
+	br	x30
+	.size	longjmp, . - longjmp

src/arch/arm64/include/bits/bigint.h

@@ -0,0 +1,317 @@
+#ifndef _BITS_BIGINT_H
+#define _BITS_BIGINT_H
+
+/** @file
+ *
+ * Big integer support
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+
+/** Element of a big integer */
+typedef uint64_t bigint_element_t;
+
+/**
+ * Initialise big integer
+ *
+ * @v value0		Element 0 of big integer to initialise
+ * @v size		Number of elements
+ * @v data		Raw data
+ * @v len		Length of raw data
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_init_raw ( uint64_t *value0, unsigned int size,
+		  const void *data, size_t len ) {
+	size_t pad_len = ( sizeof ( bigint_t ( size ) ) - len );
+	uint8_t *value_byte = ( ( void * ) value0 );
+	const uint8_t *data_byte = ( data + len );
+
+	/* Copy raw data in reverse order, padding with zeros */
+	while ( len-- )
+		*(value_byte++) = *(--data_byte);
+	while ( pad_len-- )
+		*(value_byte++) = 0;
+}
+
+/**
+ * Add big integers
+ *
+ * @v addend0		Element 0 of big integer to add
+ * @v value0		Element 0 of big integer to be added to
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_add_raw ( const uint64_t *addend0, uint64_t *value0,
+		 unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	uint64_t *discard_addend;
+	uint64_t *discard_value;
+	uint64_t discard_addend_i;
+	uint64_t discard_value_i;
+	unsigned int discard_size;
+
+	__asm__ __volatile__ ( "cmn xzr, xzr\n\t" /* clear CF */
+			       "\n1:\n\t"
+			       "ldr %3, [%0], #8\n\t"
+			       "ldr %4, [%1]\n\t"
+			       "adcs %4, %4, %3\n\t"
+			       "str %4, [%1], #8\n\t"
+			       "sub %w2, %w2, #1\n\t"
+			       "cbnz %w2, 1b\n\t"
+			       : "=r" ( discard_addend ),
+				 "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_addend_i ),
+				 "=r" ( discard_value_i ),
+				 "+m" ( *value )
+			       : "0" ( addend0 ), "1" ( value0 ), "2" ( size )
+			       : "cc" );
+}
+
+/**
+ * Subtract big integers
+ *
+ * @v subtrahend0	Element 0 of big integer to subtract
+ * @v value0		Element 0 of big integer to be subtracted from
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0,
+		      unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	uint64_t *discard_subtrahend;
+	uint64_t *discard_value;
+	uint64_t discard_subtrahend_i;
+	uint64_t discard_value_i;
+	unsigned int discard_size;
+
+	__asm__ __volatile__ ( "cmp xzr, xzr\n\t" /* set CF */
+			       "\n1:\n\t"
+			       "ldr %3, [%0], #8\n\t"
+			       "ldr %4, [%1]\n\t"
+			       "sbcs %4, %4, %3\n\t"
+			       "str %4, [%1], #8\n\t"
+			       "sub %w2, %w2, #1\n\t"
+			       "cbnz %w2, 1b\n\t"
+			       : "=r" ( discard_subtrahend ),
+				 "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_subtrahend_i ),
+				 "=r" ( discard_value_i ),
+				 "+m" ( *value )
+			       : "0" ( subtrahend0 ), "1" ( value0 ),
+				 "2" ( size )
+			       : "cc" );
+}
+
+/**
+ * Rotate big integer left
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_rol_raw ( uint64_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	uint64_t *discard_value;
+	uint64_t discard_value_i;
+	unsigned int discard_size;
+
+	__asm__ __volatile__ ( "cmn xzr, xzr\n\t" /* clear CF */
+			       "\n1:\n\t"
+			       "ldr %2, [%0]\n\t"
+			       "adcs %2, %2, %2\n\t"
+			       "str %2, [%0], #8\n\t"
+			       "sub %w1, %w1, #1\n\t"
+			       "cbnz %w1, 1b\n\t"
+			       : "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_value_i ),
+				 "+m" ( *value )
+			       : "0" ( value0 ), "1" ( size )
+			       : "cc" );
+}
+
+/**
+ * Rotate big integer right
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_ror_raw ( uint64_t *value0, unsigned int size ) {
+	bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( void * ) value0 );
+	uint64_t *discard_value;
+	uint64_t discard_value_i;
+	uint64_t discard_value_j;
+	unsigned int discard_size;
+
+	__asm__ __volatile__ ( "mov %3, #0\n\t"
+			       "\n1:\n\t"
+			       "sub %w1, %w1, #1\n\t"
+			       "ldr %2, [%0, %1, lsl #3]\n\t"
+			       "extr %3, %3, %2, #1\n\t"
+			       "str %3, [%0, %1, lsl #3]\n\t"
+			       "mov %3, %2\n\t"
+			       "cbnz %w1, 1b\n\t"
+			       : "=r" ( discard_value ),
+				 "=r" ( discard_size ),
+				 "=r" ( discard_value_i ),
+				 "=r" ( discard_value_j ),
+				 "+m" ( *value )
+			       : "0" ( value0 ), "1" ( size ) );
+}
+
+/**
+ * Test if big integer is equal to zero
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @ret is_zero		Big integer is equal to zero
+ */
+static inline __attribute__ (( always_inline, pure )) int
+bigint_is_zero_raw ( const uint64_t *value0, unsigned int size ) {
+	const uint64_t *value = value0;
+	uint64_t value_i;
+
+	do {
+		value_i = *(value++);
+		if ( value_i )
+			break;
+	} while ( --size );
+
+	return ( value_i == 0 );
+}
+
+/**
+ * Compare big integers
+ *
+ * @v value0		Element 0 of big integer
+ * @v reference0	Element 0 of reference big integer
+ * @v size		Number of elements
+ * @ret geq		Big integer is greater than or equal to the reference
+ */
+static inline __attribute__ (( always_inline, pure )) int
+bigint_is_geq_raw ( const uint64_t *value0, const uint64_t *reference0,
+		    unsigned int size ) {
+	const uint64_t *value = ( value0 + size );
+	const uint64_t *reference = ( reference0 + size );
+	uint64_t value_i;
+	uint64_t reference_i;
+
+	do {
+		value_i = *(--value);
+		reference_i = *(--reference);
+		if ( value_i != reference_i )
+			break;
+	} while ( --size );
+
+	return ( value_i >= reference_i );
+}
+
+/**
+ * Test if bit is set in big integer
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @v bit		Bit to test
+ * @ret is_set		Bit is set
+ */
+static inline __attribute__ (( always_inline )) int
+bigint_bit_is_set_raw ( const uint64_t *value0, unsigned int size,
+			unsigned int bit ) {
+	const bigint_t ( size ) __attribute__ (( may_alias )) *value =
+		( ( const void * ) value0 );
+	unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) );
+	unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) );
+
+	return ( !! ( value->element[index] & ( 1UL << subindex ) ) );
+}
+
+/**
+ * Find highest bit set in big integer
+ *
+ * @v value0		Element 0 of big integer
+ * @v size		Number of elements
+ * @ret max_bit		Highest bit set + 1 (or 0 if no bits set)
+ */
+static inline __attribute__ (( always_inline )) int
+bigint_max_set_bit_raw ( const uint64_t *value0, unsigned int size ) {
+	const uint64_t *value = ( value0 + size );
+	int max_bit = ( 8 * sizeof ( bigint_t ( size ) ) );
+	uint64_t value_i;
+
+	do {
+		value_i = *(--value);
+		max_bit -= ( 64 - fls ( value_i ) );
+		if ( value_i )
+			break;
+	} while ( --size );
+
+	return max_bit;
+}
+
+/**
+ * Grow big integer
+ *
+ * @v source0		Element 0 of source big integer
+ * @v source_size	Number of elements in source big integer
+ * @v dest0		Element 0 of destination big integer
+ * @v dest_size		Number of elements in destination big integer
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_grow_raw ( const uint64_t *source0, unsigned int source_size,
+		  uint64_t *dest0, unsigned int dest_size ) {
+	unsigned int pad_size = ( dest_size - source_size );
+
+	memcpy ( dest0, source0, sizeof ( bigint_t ( source_size ) ) );
+	memset ( ( dest0 + source_size ), 0, sizeof ( bigint_t ( pad_size ) ) );
+}
+
+/**
+ * Shrink big integer
+ *
+ * @v source0		Element 0 of source big integer
+ * @v source_size	Number of elements in source big integer
+ * @v dest0		Element 0 of destination big integer
+ * @v dest_size		Number of elements in destination big integer
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_shrink_raw ( const uint64_t *source0, unsigned int source_size __unused,
+		    uint64_t *dest0, unsigned int dest_size ) {
+
+	memcpy ( dest0, source0, sizeof ( bigint_t ( dest_size ) ) );
+}
+
+/**
+ * Finalise big integer
+ *
+ * @v value0		Element 0 of big integer to finalise
+ * @v size		Number of elements
+ * @v out		Output buffer
+ * @v len		Length of output buffer
+ */
+static inline __attribute__ (( always_inline )) void
+bigint_done_raw ( const uint64_t *value0, unsigned int size __unused,
+		  void *out, size_t len ) {
+	const uint8_t *value_byte = ( ( const void * ) value0 );
+	uint8_t *out_byte = ( out + len );
+
+	/* Copy raw data in reverse order */
+	while ( len-- )
+		*(--out_byte) = *(value_byte++);
+}
+
+extern void bigint_multiply_raw ( const uint64_t *multiplicand0,
+				  const uint64_t *multiplier0,
+				  uint64_t *value0, unsigned int size );
+
+#endif /* _BITS_BIGINT_H */

src/arch/arm64/include/bits/bitops.h

@@ -0,0 +1,100 @@
+#ifndef _BITS_BITOPS_H
+#define _BITS_BITOPS_H
+
+/** @file
+ *
+ * ARM bit operations
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/**
+ * Test and set bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ * @ret old		Old value of bit (zero or non-zero)
+ */
+static inline __attribute__ (( always_inline )) int
+test_and_set_bit ( unsigned int bit, volatile void *bits ) {
+	unsigned int index = ( bit / 64 );
+	unsigned int offset = ( bit % 64 );
+	volatile uint64_t *qword = ( ( ( volatile uint64_t * ) bits ) + index );
+	uint64_t mask = ( 1UL << offset );
+	uint64_t old;
+	uint64_t new;
+	uint32_t flag;
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       "ldxr %0, %3\n\t"
+			       "orr %1, %0, %4\n\t"
+			       "stxr %w2, %1, %3\n\t"
+			       "tst %w2, %w2\n\t"
+			       "bne 1b\n\t"
+			       : "=&r" ( old ), "=&r" ( new ), "=&r" ( flag ),
+				 "+Q" ( *qword )
+			       : "r" ( mask )
+			       : "cc" );
+
+	return ( !! ( old & mask ) );
+}
+
+/**
+ * Test and clear bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ * @ret old		Old value of bit (zero or non-zero)
+ */
+static inline __attribute__ (( always_inline )) int
+test_and_clear_bit ( unsigned int bit, volatile void *bits ) {
+	unsigned int index = ( bit / 64 );
+	unsigned int offset = ( bit % 64 );
+	volatile uint64_t *qword = ( ( ( volatile uint64_t * ) bits ) + index );
+	uint64_t mask = ( 1UL << offset );
+	uint64_t old;
+	uint64_t new;
+	uint32_t flag;
+
+	__asm__ __volatile__ ( "\n1:\n\t"
+			       "ldxr %0, %3\n\t"
+			       "bic %1, %0, %4\n\t"
+			       "stxr %w2, %1, %3\n\t"
+			       "tst %w2, %w2\n\t"
+			       "bne 1b\n\t"
+			       : "=&r" ( old ), "=&r" ( new ), "=&r" ( flag ),
+				 "+Q" ( *qword )
+			       : "r" ( mask )
+			       : "cc" );
+
+	return ( !! ( old & mask ) );
+}
+
+/**
+ * Set bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ */
+static inline __attribute__ (( always_inline )) void
+set_bit ( unsigned int bit, volatile void *bits ) {
+
+	test_and_set_bit ( bit, bits );
+}
+
+/**
+ * Clear bit atomically
+ *
+ * @v bit		Bit to set
+ * @v bits		Bit field
+ */
+static inline __attribute__ (( always_inline )) void
+clear_bit ( unsigned int bit, volatile void *bits ) {
+
+	test_and_clear_bit ( bit, bits );
+}
+
+#endif /* _BITS_BITOPS_H */

src/arch/arm64/include/bits/byteswap.h

@@ -0,0 +1,47 @@
+#ifndef _BITS_BYTESWAP_H
+#define _BITS_BYTESWAP_H
+
+/** @file
+ *
+ * Byte-order swapping functions
+ *
+ */
+
+#include <stdint.h>
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+static inline __attribute__ (( always_inline, const )) uint16_t
+__bswap_variable_16 ( uint16_t x ) {
+	__asm__ ( "rev16 %0, %1" : "=r" ( x ) : "r" ( x ) );
+	return x;
+}
+
+static inline __attribute__ (( always_inline )) void
+__bswap_16s ( uint16_t *x ) {
+	*x = __bswap_variable_16 ( *x );
+}
+
+static inline __attribute__ (( always_inline, const )) uint32_t
+__bswap_variable_32 ( uint32_t x ) {
+	__asm__ ( "rev32 %0, %1" : "=r" ( x ) : "r" ( x ) );
+	return x;
+}
+
+static inline __attribute__ (( always_inline )) void
+__bswap_32s ( uint32_t *x ) {
+	*x = __bswap_variable_32 ( *x );
+}
+
+static inline __attribute__ (( always_inline, const )) uint64_t
+__bswap_variable_64 ( uint64_t x ) {
+	__asm__ ( "rev %0, %1" : "=r" ( x ) : "r" ( x ) );
+	return x;
+}
+
+static inline __attribute__ (( always_inline )) void
+__bswap_64s ( uint64_t *x ) {
+	*x = __bswap_variable_64 ( *x );
+}
+
+#endif

src/arch/arm64/include/bits/compiler.h

@@ -0,0 +1,16 @@
+#ifndef _BITS_COMPILER_H
+#define _BITS_COMPILER_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/** Dummy relocation type */
+#define RELOC_TYPE_NONE R_AARCH64_NULL
+
+#ifndef ASSEMBLY
+
+#define __asmcall
+#define __libgcc
+
+#endif /* ASSEMBLY */
+
+#endif /*_BITS_COMPILER_H */

src/arch/arm64/include/bits/profile.h

@@ -0,0 +1,30 @@
+#ifndef _BITS_PROFILE_H
+#define _BITS_PROFILE_H
+
+/** @file
+ *
+ * Profiling
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/**
+ * Get profiling timestamp
+ *
+ * @ret timestamp	Timestamp
+ */
+static inline __attribute__ (( always_inline )) uint64_t
+profile_timestamp ( void ) {
+	uint64_t cycles;
+
+	/* Read cycle counter */
+	__asm__ __volatile__ ( "msr PMCR_EL0, %1\n\t"
+			       "mrs %0, PMCCNTR_EL0\n\t"
+			       : "=r" ( cycles ) : "r" ( 1 ) );
+	return cycles;
+}
+
+#endif /* _BITS_PROFILE_H */

src/arch/arm64/include/bits/stdint.h

@@ -0,0 +1,21 @@
+#ifndef _BITS_STDINT_H
+#define _BITS_STDINT_H
+
+typedef __SIZE_TYPE__		size_t;
+typedef signed long		ssize_t;
+typedef signed long		off_t;
+
+typedef unsigned char		uint8_t;
+typedef unsigned short		uint16_t;
+typedef unsigned int		uint32_t;
+typedef unsigned long long	uint64_t;
+
+typedef signed char		int8_t;
+typedef signed short		int16_t;
+typedef signed int		int32_t;
+typedef signed long long	int64_t;
+
+typedef unsigned long		physaddr_t;
+typedef unsigned long		intptr_t;
+
+#endif /* _BITS_STDINT_H */

src/arch/arm64/include/bits/strings.h

@@ -0,0 +1,69 @@
+#ifndef _BITS_STRINGS_H
+#define _BITS_STRINGS_H
+
+/** @file
+ *
+ * String functions
+ *
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){
+	unsigned long long bits = value;
+	unsigned long long lsb;
+	unsigned int lz;
+
+	/* Extract least significant set bit */
+	lsb = ( bits & -bits );
+
+	/* Count number of leading zeroes before LSB */
+	__asm__ ( "clz %0, %1" : "=r" ( lz ) : "r" ( lsb ) );
+
+	return ( 64 - lz );
+}
+
+/**
+ * Find first (i.e. least significant) set bit
+ *
+ * @v value		Value
+ * @ret lsb		Least significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __ffsl ( long value ) {
+
+	return __ffsll ( value );
+}
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v value		Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __flsll ( long long value ){
+	unsigned int lz;
+
+	/* Count number of leading zeroes */
+	__asm__ ( "clz %0, %1" : "=r" ( lz ) : "r" ( value ) );
+
+	return ( 64 - lz );
+}
+
+/**
+ * Find last (i.e. most significant) set bit
+ *
+ * @v value		Value
+ * @ret msb		Most significant bit set in value (LSB=1), or zero
+ */
+static inline __attribute__ (( always_inline )) int __flsl ( long value ) {
+
+	return __flsll ( value );
+}
+
+#endif /* _BITS_STRINGS_H */

src/arch/arm64/include/efi/ipxe/dhcp_arch.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2015 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ *
+ * You can also choose to distribute this program under the terms of
+ * the Unmodified Binary Distribution Licence (as given in the file
+ * COPYING.UBDL), provided that you have satisfied its requirements.
+ */
+
+#ifndef _DHCP_ARCH_H
+#define _DHCP_ARCH_H
+
+/** @file
+ *
+ * Architecture-specific DHCP options
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <ipxe/dhcp.h>
+
+#define DHCP_ARCH_VENDOR_CLASS_ID \
+	DHCP_STRING ( 'P', 'X', 'E', 'C', 'l', 'i', 'e', 'n', 't', ':',      \
+		      'A', 'r', 'c', 'h', ':', '0', '0', '0', '0', '7', ':', \
+		      'U', 'N', 'D', 'I', ':', '0', '0', '3', '0', '1', '0' )
+
+#define DHCP_ARCH_CLIENT_ARCHITECTURE \
+	DHCP_WORD ( DHCP_CLIENT_ARCHITECTURE_EFI )
+
+#define DHCP_ARCH_CLIENT_NDI DHCP_OPTION ( 1 /* UNDI */ , 3, 10 /* v3.10 */ )
+
+#endif

src/arch/arm64/include/gdbmach.h

@@ -0,0 +1,45 @@
+#ifndef GDBMACH_H
+#define GDBMACH_H
+
+/** @file
+ *
+ * GDB architecture specifics
+ *
+ * This file declares functions for manipulating the machine state and
+ * debugging context.
+ *
+ */
+
+#include <stdint.h>
+
+typedef unsigned long gdbreg_t;
+
+/* Register snapshot */
+enum {
+	/* Not yet implemented */
+	GDBMACH_NREGS,
+};
+
+#define GDBMACH_SIZEOF_REGS ( GDBMACH_NREGS * sizeof ( gdbreg_t ) )
+
+static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) {
+	/* Not yet implemented */
+	( void ) regs;
+	( void ) pc;
+}
+
+static inline void gdbmach_set_single_step ( gdbreg_t *regs, int step ) {
+	/* Not yet implemented */
+	( void ) regs;
+	( void ) step;
+}
+
+static inline void gdbmach_breakpoint ( void ) {
+	/* Not yet implemented */
+}
+
+extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len,
+				    int enable );
+extern void gdbmach_init ( void );
+
+#endif /* GDBMACH_H */

src/arch/arm64/include/limits.h

@@ -0,0 +1,59 @@
+#ifndef LIMITS_H
+#define LIMITS_H	1
+
+/* Number of bits in a `char' */
+#define CHAR_BIT	8
+
+/* Minimum and maximum values a `signed char' can hold */
+#define SCHAR_MIN	(-128)
+#define SCHAR_MAX	127
+
+/* Maximum value an `unsigned char' can hold. (Minimum is 0.) */
+#define UCHAR_MAX	255
+
+/* Minimum and maximum values a `char' can hold */
+#define CHAR_MIN	SCHAR_MIN
+#define CHAR_MAX	SCHAR_MAX
+
+/* Minimum and maximum values a `signed short int' can hold */
+#define SHRT_MIN	(-32768)
+#define SHRT_MAX	32767
+
+/* Maximum value an `unsigned short' can hold. (Minimum is 0.) */
+#define USHRT_MAX	65535
+
+
+/* Minimum and maximum values a `signed int' can hold */
+#define INT_MIN		(-INT_MAX - 1)
+#define INT_MAX		2147483647
+
+/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */
+#define UINT_MAX	4294967295U
+
+
+/* Minimum and maximum values a `signed int' can hold */
+#define INT_MAX		2147483647
+#define INT_MIN		(-INT_MAX - 1)
+
+
+/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */
+#define UINT_MAX	4294967295U
+
+
+/* Minimum and maximum values a `signed long' can hold */
+#define LONG_MAX	9223372036854775807L
+#define LONG_MIN	(-LONG_MAX - 1L)
+
+/* Maximum value an `unsigned long' can hold. (Minimum is 0.) */
+#define ULONG_MAX	18446744073709551615UL
+
+/* Minimum and maximum values a `signed long long' can hold */
+#define LLONG_MAX	9223372036854775807LL
+#define LLONG_MIN	(-LONG_MAX - 1LL)
+
+
+/* Maximum value an `unsigned long long' can hold. (Minimum is 0.) */
+#define ULLONG_MAX	18446744073709551615ULL
+
+
+#endif /* LIMITS_H */

src/arch/arm64/include/setjmp.h

@@ -0,0 +1,44 @@
+#ifndef _SETJMP_H
+#define _SETJMP_H
+
+FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
+
+#include <stdint.h>
+
+/** A jump buffer */
+typedef struct {
+	/** Saved x19 */
+	uint64_t x19;
+	/** Saved x20 */
+	uint64_t x20;
+	/** Saved x21 */
+	uint64_t x21;
+	/** Saved x22 */
+	uint64_t x22;
+	/** Saved x23 */
+	uint64_t x23;
+	/** Saved x24 */
+	uint64_t x24;
+	/** Saved x25 */
+	uint64_t x25;
+	/** Saved x26 */
+	uint64_t x26;
+	/** Saved x27 */
+	uint64_t x27;
+	/** Saved x28 */
+	uint64_t x28;
+	/** Saved frame pointer (x29) */
+	uint64_t x29;
+	/** Saved link register (x30) */
+	uint64_t x30;
+	/** Saved stack pointer (x31) */
+	uint64_t sp;
+} jmp_buf[1];
+
+extern int __asmcall __attribute__ (( returns_twice ))
+setjmp ( jmp_buf env );
+
+extern void __asmcall __attribute__ (( noreturn ))
+longjmp ( jmp_buf env, int val );
+
+#endif /* _SETJMP_H */

src/config/defaults/efi.h

@@ -40,7 +40,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define	CPUID_CMD		/* x86 CPU feature detection command */
 #endif
 
-#if defined ( __arm__ )
+#if defined ( __arm__ ) || defined ( __aarch64__ )
 #define IOAPI_ARM
 #define NAP_EFIARM
 #endif

src/util/efirom.c

@@ -85,6 +85,7 @@ static void read_pe_info ( void *pe, uint16_t *machine,
 		*subsystem = nt->nt32.OptionalHeader.Subsystem;
 		break;
 	case EFI_IMAGE_MACHINE_X64:
+	case EFI_IMAGE_MACHINE_AARCH64:
 		*subsystem = nt->nt64.OptionalHeader.Subsystem;
 		break;
 	default:

src/util/elf2efi.c

@@ -72,6 +72,11 @@
 
 #define ELF_MREL( mach, type ) ( (mach) | ( (type) << 16 ) )
 
+/* Seems to be missing from elf.h */
+#ifndef R_AARCH64_NULL
+#define R_AARCH64_NULL 256
+#endif
+
 #define EFI_FILE_ALIGN 0x20
 
 struct elf_file {
@@ -403,6 +408,9 @@ static void set_machine ( struct elf_file *elf, struct pe_header *pe_header ) {
 	case EM_ARM:
 		machine = EFI_IMAGE_MACHINE_ARMTHUMB_MIXED;
 		break;
+	case EM_AARCH64:
+		machine = EFI_IMAGE_MACHINE_AARCH64;
+		break;
 	default:
 		eprintf ( "Unknown ELF architecture %d\n", ehdr->e_machine );
 		exit ( 1 );
@@ -582,6 +590,8 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_386, R_386_NONE ) :
 		case ELF_MREL ( EM_ARM, R_ARM_NONE ) :
 		case ELF_MREL ( EM_X86_64, R_X86_64_NONE ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_NONE ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_NULL ) :
 			/* Ignore dummy relocations used by REQUIRE_SYMBOL() */
 			break;
 		case ELF_MREL ( EM_386, R_386_32 ) :
@@ -590,6 +600,7 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 			generate_pe_reloc ( pe_reltab, offset, 4 );
 			break;
 		case ELF_MREL ( EM_X86_64, R_X86_64_64 ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_ABS64 ) :
 			/* Generate an 8-byte PE relocation */
 			generate_pe_reloc ( pe_reltab, offset, 8 );
 			break;
@@ -598,6 +609,15 @@ static void process_reloc ( struct elf_file *elf, const Elf_Shdr *shdr,
 		case ELF_MREL ( EM_ARM, R_ARM_THM_PC22 ) :
 		case ELF_MREL ( EM_ARM, R_ARM_THM_JUMP24 ) :
 		case ELF_MREL ( EM_X86_64, R_X86_64_PC32 ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_CALL26 ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_JUMP26 ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_ADR_PREL_LO21 ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_ADR_PREL_PG_HI21 ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_ADD_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST8_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST16_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST32_ABS_LO12_NC ) :
+		case ELF_MREL ( EM_AARCH64, R_AARCH64_LDST64_ABS_LO12_NC ) :
 			/* Skip PC-relative relocations; all relative
 			 * offsets remain unaltered when the object is
 			 * loaded.