[librm] Allow interrupts in protected mode

When running in a virtual machine, switching to real mode may be
expensive.  Allow interrupts to be enabled while in protected mode and
reflected down to the real-mode interrupt handlers.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/arch/i386/core/gdbidt.S

@@ -1,101 +1,10 @@
 /*
- * Interrupt Descriptor Table (IDT) setup and interrupt handlers for GDB stub.
+ * Interrupt handlers for GDB stub
  */
 
-#include <librm.h>
-
 #define SIZEOF_I386_REGS	32
 #define SIZEOF_I386_FLAGS	4
 
-/****************************************************************************
- * Interrupt Descriptor Table
- ****************************************************************************
- */
-	.section ".data16", "aw", @progbits
-	.globl idtr
-idtr:
-idt_limit:
-	.word	idt_length - 1
-idt_base:
-	.long	0
-
-/* IDT entries have the following format:
- * offset_lo, segment selector, flags, offset_hi
- *
- * Since it is not possible to specify relocations in arbitrary
- * expressions like (int_overflow & 0xffff), we initialise the
- * IDT with entries in an incorrect format.
- *
- * The entries are shuffled into the correct format in init_librm().
- */
-#define IDT_ENTRY_EMPTY(name) .word 0, 0, 0, 0
-#define IDT_ENTRY_PRESENT(name) \
-	.long	int_##name; \
-	.word	0x8e00, VIRTUAL_CS
-
-.align 16
-idt:
-	IDT_ENTRY_PRESENT(divide_error)
-	IDT_ENTRY_PRESENT(debug_trap)
-	IDT_ENTRY_EMPTY(non_maskable_interrupt)
-	IDT_ENTRY_PRESENT(breakpoint)
-	IDT_ENTRY_PRESENT(overflow)
-	IDT_ENTRY_PRESENT(bound_range_exceeded)
-	IDT_ENTRY_PRESENT(invalid_opcode)
-	IDT_ENTRY_EMPTY(device_not_available)
-	IDT_ENTRY_PRESENT(double_fault)
-	IDT_ENTRY_EMPTY(coprocessor_segment_overrun)
-	IDT_ENTRY_PRESENT(invalid_tss)
-	IDT_ENTRY_PRESENT(segment_not_present)
-	IDT_ENTRY_PRESENT(stack_segment_fault)
-	IDT_ENTRY_PRESENT(general_protection)
-	IDT_ENTRY_PRESENT(page_fault)
-idt_end:
-	.equ	idt_length, idt_end - idt
-
-/* The IDT entries are fixed up (once) in init_librm() */
-idt_fixed:
-	.byte	0
-
-/****************************************************************************
- * idt_init (real-mode near call, 16-bit real-mode near return address)
- *
- * Initialise the IDT, called from init_librm.
- *
- * Parameters:
- *   %eax : IDT base address
- *
- * Destroys %ax, %bx, and %di.
- ****************************************************************************
- */
-	.section ".text16", "ax", @progbits
-	.code16
-	.globl idt_init
-idt_init:
-	movl	%eax, idt_base
-	addl	$idt, idt_base
-
-	/* IDT entries are only fixed up once */
-	movb	idt_fixed, %al
-	orb	%al, %al
-	jnz	2f
-	movb	$1, idt_fixed
-
-	/* Shuffle IDT entries into the correct format */
-	movb	$(idt_length / 8), %al
-	movw	$idt, %bx
-	or	%al, %al
-	jz	2f
-1:
-	movw	2(%bx), %di
-	xchg	%di, 6(%bx)
-	movw	%di, 2(%bx)
-	addw	$8, %bx
-	dec	%al
-	jnz	1b
-2:
-	ret
-
 /****************************************************************************
  * Interrupt handlers
  ****************************************************************************
@@ -111,35 +20,35 @@ idt_init:
 #define SIGSEGV 11
 #define SIGSTKFLT 16
 
-int_divide_error:
+	.globl	gdbmach_nocode_sigfpe
+gdbmach_nocode_sigfpe:
 	pushl	$SIGFPE
-	jmp	do_interrupt
+	jmp	gdbmach_interrupt
 
-int_debug_trap:
-int_breakpoint:
+	.globl	gdbmach_nocode_sigtrap
+gdbmach_nocode_sigtrap:
 	pushl	$SIGTRAP
-	jmp	do_interrupt
+	jmp	gdbmach_interrupt
 
-int_overflow:
-int_bound_range_exceeded:
+	.globl	gdbmach_nocode_sigstkflt
+gdbmach_nocode_sigstkflt:
 	pushl	$SIGSTKFLT
-	jmp	do_interrupt
+	jmp	gdbmach_interrupt
 
-int_invalid_opcode:
+	.globl	gdbmach_nocode_sigill
+gdbmach_nocode_sigill:
 	pushl	$SIGILL
-	jmp	do_interrupt
+	jmp	gdbmach_interrupt
 
-int_double_fault:
+	.globl	gdbmach_withcode_sigbus
+gdbmach_withcode_sigbus:
 	movl	$SIGBUS, (%esp)
-	jmp	do_interrupt
+	jmp	gdbmach_interrupt
 
-int_invalid_tss:
-int_segment_not_present:
-int_stack_segment_fault:
-int_general_protection:
-int_page_fault:
+	.globl	gdbmach_withcode_sigsegv
+gdbmach_withcode_sigsegv:
 	movl	$SIGSEGV, (%esp)
-	jmp	do_interrupt
+	jmp	gdbmach_interrupt
 
 /* When invoked, the stack contains: eflags, cs, eip, signo. */
 #define IH_OFFSET_GDB_REGS ( 0 )
@@ -161,7 +70,7 @@ int_page_fault:
 #define IH_OFFSET_FLUX_OLD_EFLAGS ( IH_OFFSET_OLD_EFLAGS - 40 )
 #define IH_OFFSET_FLUX_OLD_EIP ( IH_OFFSET_OLD_EIP - 36 )
 #define IH_OFFSET_FLUX_END ( IH_OFFSET_END - 20 )
-do_interrupt:
+gdbmach_interrupt:
 	/* Store CPU state in GDB register snapshot */
 	pushw	$0
 	pushw	%gs
@@ -187,25 +96,41 @@ do_interrupt:
 	pushl	%ecx
 	pushl	%eax
 
+	/* Switch to virtual addressing */
+	call	_intr_to_virt
+
 	/* Call GDB stub exception handler */
 	pushl	%esp
 	pushl	(IH_OFFSET_SIGNO + 4)(%esp)
 	call	gdbmach_handler
 	addl	$8, %esp
 
+	/* Copy register snapshot to new stack and switch to new stack */
+	movl	%esp, %esi
+	movl	(IH_OFFSET_GDB_SEG_REGS + 4)(%esp), %eax
+	movl	%eax, %es
+	movl	(IH_OFFSET_GDB_REGS + 16)(%esp), %edi
+	subl	$IH_OFFSET_END, %edi
+	movl	$(IH_OFFSET_END / 4), %ecx
+	pushl	%edi
+	ss rep movsl
+	popl	%edi
+	movl	%eax, %ss
+	movl	%edi, %esp
+
 	/* Restore CPU state from GDB register snapshot */
 	popl	%eax
 	popl	%ecx
 	popl	%edx
 	popl	%ebx
-	addl	$4, %esp /* Changing ESP currently not supported */
+	popl	%ebp /* Skip %esp: already loaded */
 	popl	%ebp
 	popl	%esi
 	popl	%edi
 	popl	IH_OFFSET_FLUX_OLD_EIP(%esp)
 	popl	IH_OFFSET_FLUX_OLD_EFLAGS(%esp)
 	popl	IH_OFFSET_FLUX_OLD_CS(%esp)
-	popl	%ss
+	popl	%ds /* Skip %ss: already loaded */
 	popl	%ds
 	popl	%es
 	popl	%fs

src/arch/i386/core/gdbmach.c

@@ -24,6 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER );
 #include <assert.h>
 #include <ipxe/uaccess.h>
 #include <ipxe/gdbstub.h>
+#include <librm.h>
 #include <gdbmach.h>
 
 /** @file
@@ -150,3 +151,30 @@ __asmcall void gdbmach_handler ( int signo, gdbreg_t *regs ) {
 	gdbstub_handler ( signo, regs );
 	gdbmach_enable_hwbps();
 }
+
+static void * gdbmach_interrupt_vectors[] = {
+	gdbmach_nocode_sigfpe,		/* Divide by zero */
+	gdbmach_nocode_sigtrap,		/* Debug trap */
+	NULL,				/* Non-maskable interrupt */
+	gdbmach_nocode_sigtrap,		/* Breakpoint */
+	gdbmach_nocode_sigstkflt,	/* Overflow */
+	gdbmach_nocode_sigstkflt,	/* Bound range exceeded */
+	gdbmach_nocode_sigill,		/* Invalid opcode */
+	NULL,				/* Device not available */
+	gdbmach_withcode_sigbus,	/* Double fault */
+	NULL,				/* Coprocessor segment overrun */
+	gdbmach_withcode_sigsegv,	/* Invalid TSS */
+	gdbmach_withcode_sigsegv,	/* Segment not present */
+	gdbmach_withcode_sigsegv,	/* Stack segment fault */
+	gdbmach_withcode_sigsegv,	/* General protection fault */
+	gdbmach_withcode_sigsegv,	/* Page fault */
+};
+
+void gdbmach_init ( void ) {
+	unsigned int i;
+
+	for ( i = 0 ; i < ( sizeof ( gdbmach_interrupt_vectors ) /
+			    sizeof ( gdbmach_interrupt_vectors[0] ) ) ; i++ ) {
+		set_interrupt_vector ( i, gdbmach_interrupt_vectors[i] );
+	}
+}

src/arch/i386/core/virtaddr.S

@@ -36,6 +36,7 @@ _virt_to_phys:
 	addl	%ebp, 12(%esp)
 
 	/* Switch to physical code segment */
+	cli
 	pushl	$PHYSICAL_CS
 	leal	1f(%ebp), %eax
 	pushl	%eax
@@ -44,10 +45,10 @@ _virt_to_phys:
 	/* Reload other segment registers and adjust %esp */
 	movl	$PHYSICAL_DS, %eax
 	movl	%eax, %ds
-	movl	%eax, %es	
-	movl	%eax, %fs	
+	movl	%eax, %es
+	movl	%eax, %fs
 	movl	%eax, %gs
-	movl	%eax, %ss	
+	movl	%eax, %ss
 	addl	%ebp, %esp
 
 	/* Restore registers and flags, and return */
@@ -64,9 +65,6 @@ _virt_to_phys:
  * selectors.  All other registers are preserved.  Flags are
  * preserved.
  *
- * Note that this depends on the GDT already being correctly set up
- * (e.g. by a call to run_here()).
- *
  * Parameters: none
  * Returns: none
  ****************************************************************************
@@ -79,18 +77,19 @@ _phys_to_virt:
 	pushl	%ebp
 
 	/* Switch to virtual code segment */
+	cli
 	ljmp	$VIRTUAL_CS, $1f
-1:	
+1:
 	/* Reload data segment registers */
 	movl	$VIRTUAL_DS, %eax
 	movl	%eax, %ds
-	movl	%eax, %es	
-	movl	%eax, %fs	
+	movl	%eax, %es
+	movl	%eax, %fs
 	movl	%eax, %gs
 
 	/* Reload stack segment and adjust %esp */
 	movl	virt_offset, %ebp
-	movl	%eax, %ss	
+	movl	%eax, %ss
 	subl	%ebp, %esp
 
 	/* Change the return address to a virtual address */
@@ -101,3 +100,46 @@ _phys_to_virt:
 	popl	%eax
 	popfl
 	ret
+
+/****************************************************************************
+ * _intr_to_virt (virtual code segment, virtual or physical stack segment)
+ *
+ * Switch from virtual code segment with either a virtual or physical
+ * stack segment to using virtual addressing.  %esp is adjusted if
+ * necessary to a virtual value.  Segment registers are set to virtual
+ * selectors.  All other registers are preserved.  Flags are
+ * preserved.
+ *
+ * Parameters: none
+ * Returns: none
+ ****************************************************************************
+ */
+	.globl _intr_to_virt
+_intr_to_virt:
+	/* Preserve registers and flags */
+	pushfl
+	pushl	%eax
+	pushl	%ebp
+
+	/* Check whether stack segment is physical or virtual */
+	movl	%ss, %eax
+	cmpw	$VIRTUAL_DS, %ax
+	movl	$VIRTUAL_DS, %eax
+
+	/* Reload data segment registers */
+	movl	%eax, %ds
+	movl	%eax, %es
+	movl	%eax, %fs
+	movl	%eax, %gs
+
+	/* Reload stack segment and adjust %esp if necessary */
+	je	1f
+	movl	virt_offset, %ebp
+	movl	%eax, %ss
+	subl	%ebp, %esp
+1:
+	/* Restore registers and flags, and return */
+	popl	%ebp
+	popl	%eax
+	popfl
+	ret

src/arch/i386/include/gdbmach.h

@@ -46,6 +46,14 @@ enum {
 	GDBMACH_AWATCH,
 };
 
+/* Interrupt vectors */
+extern void gdbmach_nocode_sigfpe ( void );
+extern void gdbmach_nocode_sigtrap ( void );
+extern void gdbmach_nocode_sigstkflt ( void );
+extern void gdbmach_nocode_sigill ( void );
+extern void gdbmach_withcode_sigbus ( void );
+extern void gdbmach_withcode_sigsegv ( void );
+
 static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) {
 	regs [ GDBMACH_EIP ] = pc;
 }
@@ -61,4 +69,6 @@ static inline void gdbmach_breakpoint ( void ) {
 
 extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len, int enable );
 
+extern void gdbmach_init ( void );
+
 #endif /* GDBMACH_H */

src/arch/i386/include/librm.h

@@ -209,6 +209,71 @@ extern void remove_user_from_rm_stack ( userptr_t data, size_t size );
 	asm_code_str					\
 	"call _phys_to_virt\n\t"
 
+/** Number of interrupts */
+#define NUM_INT 256
+
+/** An interrupt descriptor table register */
+struct idtr {
+	/** Limit */
+	uint16_t limit;
+	/** Base */
+	uint32_t base;
+} __attribute__ (( packed ));
+
+/** An interrupt descriptor table entry */
+struct interrupt_descriptor {
+	/** Low 16 bits of address */
+	uint16_t low;
+	/** Code segment */
+	uint16_t segment;
+	/** Unused */
+	uint8_t unused;
+	/** Type and attributes */
+	uint8_t attr;
+	/** High 16 bits of address */
+	uint16_t high;
+} __attribute__ (( packed ));
+
+/** Interrupt descriptor is present */
+#define IDTE_PRESENT 0x80
+
+/** Interrupt descriptor 32-bit interrupt gate type */
+#define IDTE_TYPE_IRQ32 0x0e
+
+/** An interrupt vector
+ *
+ * Each interrupt vector comprises an eight-byte fragment of code:
+ *
+ *   60			pushal
+ *   b0 xx		movb $INT, %al
+ *   e9 xx xx xx xx	jmp interrupt_wrapper
+ */
+struct interrupt_vector {
+	/** "pushal" instruction */
+	uint8_t pushal;
+	/** "movb" instruction */
+	uint8_t movb;
+	/** Interrupt number */
+	uint8_t intr;
+	/** "jmp" instruction */
+	uint8_t jmp;
+	/** Interrupt wrapper address offset */
+	uint32_t offset;
+	/** Next instruction after jump */
+	uint8_t next[0];
+} __attribute__ (( packed ));
+
+/** "pushal" instruction */
+#define PUSHAL_INSN 0x60
+
+/** "movb" instruction */
+#define MOVB_INSN 0xb0
+
+/** "jmp" instruction */
+#define JMP_INSN 0xe9
+
+extern void set_interrupt_vector ( unsigned int intr, void *vector );
+
 #endif /* ASSEMBLY */
 
 #endif /* LIBRM_H */

src/arch/i386/transitions/librm.S

@@ -128,10 +128,15 @@ init_librm:
 	addr32 leal	(%eax, %edi), %ebx
 	movl	%ebx, rm_data16
 
-	/* Set GDT and IDT base */
+	/* Set GDT base */
 	movl	%eax, gdt_base
 	addl	$gdt, gdt_base
-	call	idt_init
+
+	/* Initialise IDT */
+	pushl	$init_idt
+	pushw	%cs
+	call	prot_call
+	popl	%eax /* discard */
 
 	/* Restore registers */
 	negl	%edi
@@ -141,14 +146,12 @@ init_librm:
 
 	.section ".text16", "ax", @progbits
 	.code16
-	.weak idt_init
 set_seg_base:
 1:	movw	%ax, 2(%bx)
 	rorl	$16, %eax
 	movb	%al, 4(%bx)
 	movb	%ah, 7(%bx)
 	roll	$16, %eax
-idt_init: /* Reuse the return opcode here */
 	ret
 
 /****************************************************************************
@@ -237,10 +240,8 @@ real_to_prot:
 	/* Return to virtual address */
 	ret
 
-	/* Default IDTR with no interrupts */
+	/* Default real-mode interrupt descriptor table */
 	.section ".data16", "aw", @progbits
-	.weak idtr
-idtr:
 rm_idtr:
 	.word 0xffff /* limit */
 	.long 0 /* base */
@@ -536,6 +537,46 @@ flatten_real_mode:
 flatten_dummy:
 	ret
 
+/****************************************************************************
+ * Interrupt wrapper
+ *
+ * Used by the protected-mode interrupt vectors to call the
+ * interrupt() function.
+ *
+ * May be entered with either physical or virtual stack segment.
+ ****************************************************************************
+ */
+	.globl interrupt_wrapper
+interrupt_wrapper:
+	/* Preserve segment registers and original %esp */
+	pushl	%ds
+	pushl	%es
+	pushl	%fs
+	pushl	%gs
+	pushl	%ss
+	pushl	%esp
+
+	/* Switch to virtual addressing */
+	call	_intr_to_virt
+
+	/* Expand IRQ number to whole %eax register */
+	movzbl	%al, %eax
+
+	/* Call interrupt handler */
+	call	interrupt
+
+	/* Restore original stack and segment registers */
+	lss	(%esp), %esp
+	popl	%ss
+	popl	%gs
+	popl	%fs
+	popl	%es
+	popl	%ds
+
+	/* Restore registers and return */
+	popal
+	iret
+
 /****************************************************************************
  * Stored real-mode and protected-mode stack pointers
  *

src/arch/i386/transitions/librm_mgmt.c

@@ -9,19 +9,35 @@ FILE_LICENCE ( GPL2_OR_LATER );
 
 #include <stdint.h>
 #include <realmode.h>
+#include <pic8259.h>
 
 /*
  * This file provides functions for managing librm.
  *
  */
 
+/** The interrupt wrapper */
+extern char interrupt_wrapper[];
+
+/** The interrupt vectors */
+static struct interrupt_vector intr_vec[ IRQ_MAX + 1 ];
+
+/** The interrupt descriptor table */
+struct interrupt_descriptor idt[NUM_INT] __attribute__ (( aligned ( 16 ) ));
+
+/** The interrupt descriptor table register */
+struct idtr __data16 ( idtr ) = {
+	.limit = ( sizeof ( idt ) - 1 ),
+};
+#define idtr __use_data16 ( idtr )
+
 /**
  * Allocate space on the real-mode stack and copy data there from a
  * user buffer
  *
- * @v data			User buffer
- * @v size			Size of stack data
- * @ret sp			New value of real-mode stack pointer
+ * @v data		User buffer
+ * @v size		Size of stack data
+ * @ret sp		New value of real-mode stack pointer
  */
 uint16_t copy_user_to_rm_stack ( userptr_t data, size_t size ) {
 	userptr_t rm_stack;
@@ -35,8 +51,8 @@ uint16_t copy_user_to_rm_stack ( userptr_t data, size_t size ) {
  * Deallocate space on the real-mode stack, optionally copying back
  * data to a user buffer.
  *
- * @v data			User buffer
- * @v size			Size of stack data
+ * @v data		User buffer
+ * @v size		Size of stack data
  */
 void remove_user_from_rm_stack ( userptr_t data, size_t size ) {
 	if ( data ) {
@@ -46,6 +62,63 @@ void remove_user_from_rm_stack ( userptr_t data, size_t size ) {
 	rm_sp += size;
 };
 
+/**
+ * Set interrupt vector
+ *
+ * @v intr		Interrupt number
+ * @v vector		Interrupt vector, or NULL to disable
+ */
+void set_interrupt_vector ( unsigned int intr, void *vector ) {
+	struct interrupt_descriptor *idte;
+
+	idte = &idt[intr];
+	idte->segment = VIRTUAL_CS;
+	idte->attr = ( vector ? ( IDTE_PRESENT | IDTE_TYPE_IRQ32 ) : 0 );
+	idte->low = ( ( ( uint32_t ) vector ) & 0xffff );
+	idte->high = ( ( ( uint32_t ) vector ) >> 16 );
+}
+
+/**
+ * Initialise interrupt descriptor table
+ *
+ */
+void init_idt ( void ) {
+	struct interrupt_vector *vec;
+	unsigned int irq;
+	unsigned int intr;
+
+	/* Initialise the interrupt descriptor table and interrupt vectors */
+	for ( irq = 0 ; irq <= IRQ_MAX ; irq++ ) {
+		intr = IRQ_INT ( irq );
+		vec = &intr_vec[irq];
+		vec->pushal = PUSHAL_INSN;
+		vec->movb = MOVB_INSN;
+		vec->intr = intr;
+		vec->jmp = JMP_INSN;
+		vec->offset = ( ( uint32_t ) interrupt_wrapper -
+				( uint32_t ) vec->next );
+		set_interrupt_vector ( intr, vec );
+	}
+
+	/* Initialise the interrupt descriptor table register */
+	idtr.base = virt_to_phys ( idt );
+}
+
+/**
+ * Interrupt handler
+ *
+ * @v irq		Interrupt number
+ */
+void __attribute__ (( cdecl, regparm ( 1 ) )) interrupt ( int irq ) {
+	uint32_t discard_eax;
+
+	/* Reissue interrupt in real mode */
+	__asm__ __volatile__ ( REAL_CODE ( "movb %%al, %%cs:(1f + 1)\n\t"
+					   "\n1:\n\t"
+					   "int $0x00\n\t" )
+			       : "=a" ( discard_eax ) : "0" ( irq ) );
+}
+
 PROVIDE_UACCESS_INLINE ( librm, phys_to_user );
 PROVIDE_UACCESS_INLINE ( librm, user_to_phys );
 PROVIDE_UACCESS_INLINE ( librm, virt_to_user );

src/arch/x86_64/include/gdbmach.h

@@ -48,4 +48,6 @@ static inline void gdbmach_breakpoint ( void ) {
 
 extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len, int enable );
 
+extern void gdbmach_init ( void );
+
 #endif /* GDBMACH_H */

src/core/gdbstub.c

@@ -396,5 +396,6 @@ struct gdb_transport *find_gdb_transport ( const char *name ) {
 void gdbstub_start ( struct gdb_transport *trans ) {
 	stub.trans = trans;
 	stub.payload = &stub.buf [ 1 ];
+	gdbmach_init();
 	gdbmach_breakpoint();
 }