Browse Source

[librm] Add support for running in 64-bit long mode

Add support for running the BIOS version of iPXE in 64-bit long mode.
A 64-bit BIOS version of iPXE can be built using e.g.

  make bin-x86_64-pcbios/ipxe.usb
  make bin-x86_64-pcbios/8086100e.mrom

The 64-bit BIOS version should appear to function identically to the
normal 32-bit BIOS version.  The physical memory layout is unaltered:
iPXE is still relocated to the top of the available 32-bit address
space.  The code is linked to a virtual address of 0xffffffffeb000000
(in the negative 2GB as required by -mcmodel=kernel), with 4kB pages
created to cover the whole of .textdata.  2MB pages are created to
cover the whole of the 32-bit address space.

The 32-bit portions of the code run with VIRTUAL_CS and VIRTUAL_DS
configured such that truncating a 64-bit virtual address gives a
32-bit virtual address pointing to the same physical location.

The stack pointer remains as a physical address when running in long
mode (although the .stack section is accessible via the negative 2GB
virtual address); this is done in order to simplify the handling of
interrupts occurring while executing a portion of 32-bit code with
flat physical addressing via PHYS_CODE().

Interrupts may be enabled in either 64-bit long mode, 32-bit protected
mode with virtual addresses, 32-bit protected mode with physical
addresses, or 16-bit real mode.  Interrupts occurring in any mode
other than real mode will be reflected down to real mode and handled
by whichever ISR is hooked into the BIOS interrupt vector table.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
tags/v1.20.1
Michael Brown 8 years ago
parent
commit
6143057430
4 changed files with 498 additions and 61 deletions
  1. 3
    0
      src/Makefile
  2. 39
    9
      src/arch/x86/include/librm.h
  3. 412
    38
      src/arch/x86/transitions/librm.S
  4. 44
    14
      src/arch/x86/transitions/librm_mgmt.c

+ 3
- 0
src/Makefile View File

151
 everything :
151
 everything :
152
 	$(Q)$(MAKE) --no-print-directory $(ALL) \
152
 	$(Q)$(MAKE) --no-print-directory $(ALL) \
153
 		bin/3c509.rom bin/intel.rom bin/intel.mrom \
153
 		bin/3c509.rom bin/intel.rom bin/intel.mrom \
154
+		bin-x86_64-pcbios/8086100e.mrom bin-x86_64-pcbios/intel.rom \
155
+		bin-x86_64-pcbios/ipxe.usb bin-x86_64-pcbios/ipxe.pxe \
156
+		bin-x86_64-pcbios/undionly.kpxe \
154
 		bin-i386-efi/ipxe.efi bin-i386-efi/ipxe.efidrv \
157
 		bin-i386-efi/ipxe.efi bin-i386-efi/ipxe.efidrv \
155
 		bin-i386-efi/ipxe.efirom \
158
 		bin-i386-efi/ipxe.efirom \
156
 		bin-x86_64-efi/ipxe.efi bin-x86_64-efi/ipxe.efidrv \
159
 		bin-x86_64-efi/ipxe.efi bin-x86_64-efi/ipxe.efidrv \

+ 39
- 9
src/arch/x86/include/librm.h View File

14
 #define REAL_CS 0x28
14
 #define REAL_CS 0x28
15
 #define REAL_DS 0x30
15
 #define REAL_DS 0x30
16
 #define P2R_DS 0x38
16
 #define P2R_DS 0x38
17
+#define LONG_CS 0x40
17
 
18
 
18
 /* Calculate symbol address within VIRTUAL_CS or VIRTUAL_DS
19
 /* Calculate symbol address within VIRTUAL_CS or VIRTUAL_DS
19
  *
20
  *
286
 /** Number of interrupts */
287
 /** Number of interrupts */
287
 #define NUM_INT 256
288
 #define NUM_INT 256
288
 
289
 
289
-/** An interrupt descriptor table register */
290
-struct idtr {
290
+/** A 32-bit interrupt descriptor table register */
291
+struct idtr32 {
291
 	/** Limit */
292
 	/** Limit */
292
 	uint16_t limit;
293
 	uint16_t limit;
293
 	/** Base */
294
 	/** Base */
294
 	uint32_t base;
295
 	uint32_t base;
295
 } __attribute__ (( packed ));
296
 } __attribute__ (( packed ));
296
 
297
 
297
-/** An interrupt descriptor table entry */
298
-struct interrupt_descriptor {
298
+/** A 64-bit interrupt descriptor table register */
299
+struct idtr64 {
300
+	/** Limit */
301
+	uint16_t limit;
302
+	/** Base */
303
+	uint64_t base;
304
+} __attribute__ (( packed ));
305
+
306
+/** A 32-bit interrupt descriptor table entry */
307
+struct interrupt32_descriptor {
299
 	/** Low 16 bits of address */
308
 	/** Low 16 bits of address */
300
 	uint16_t low;
309
 	uint16_t low;
301
 	/** Code segment */
310
 	/** Code segment */
308
 	uint16_t high;
317
 	uint16_t high;
309
 } __attribute__ (( packed ));
318
 } __attribute__ (( packed ));
310
 
319
 
320
+/** A 64-bit interrupt descriptor table entry */
321
+struct interrupt64_descriptor {
322
+	/** Low 16 bits of address */
323
+	uint16_t low;
324
+	/** Code segment */
325
+	uint16_t segment;
326
+	/** Unused */
327
+	uint8_t unused;
328
+	/** Type and attributes */
329
+	uint8_t attr;
330
+	/** Middle 16 bits of address */
331
+	uint16_t mid;
332
+	/** High 32 bits of address */
333
+	uint32_t high;
334
+	/** Reserved */
335
+	uint32_t reserved;
336
+} __attribute__ (( packed ));
337
+
311
 /** Interrupt descriptor is present */
338
 /** Interrupt descriptor is present */
312
 #define IDTE_PRESENT 0x80
339
 #define IDTE_PRESENT 0x80
313
 
340
 
314
 /** Interrupt descriptor 32-bit interrupt gate type */
341
 /** Interrupt descriptor 32-bit interrupt gate type */
315
 #define IDTE_TYPE_IRQ32 0x0e
342
 #define IDTE_TYPE_IRQ32 0x0e
316
 
343
 
344
+/** Interrupt descriptor 64-bit interrupt gate type */
345
+#define IDTE_TYPE_IRQ64 0x0e
346
+
317
 /** An interrupt vector
347
 /** An interrupt vector
318
  *
348
  *
319
  * Each interrupt vector comprises an eight-byte fragment of code:
349
  * Each interrupt vector comprises an eight-byte fragment of code:
320
  *
350
  *
321
- *   60			pushal
351
+ *   50			pushl %eax (or pushq %rax in long mode)
322
  *   b0 xx		movb $INT, %al
352
  *   b0 xx		movb $INT, %al
323
  *   e9 xx xx xx xx	jmp interrupt_wrapper
353
  *   e9 xx xx xx xx	jmp interrupt_wrapper
324
  */
354
  */
325
 struct interrupt_vector {
355
 struct interrupt_vector {
326
-	/** "pushal" instruction */
327
-	uint8_t pushal;
356
+	/** "push" instruction */
357
+	uint8_t push;
328
 	/** "movb" instruction */
358
 	/** "movb" instruction */
329
 	uint8_t movb;
359
 	uint8_t movb;
330
 	/** Interrupt number */
360
 	/** Interrupt number */
337
 	uint8_t next[0];
367
 	uint8_t next[0];
338
 } __attribute__ (( packed ));
368
 } __attribute__ (( packed ));
339
 
369
 
340
-/** "pushal" instruction */
341
-#define PUSHAL_INSN 0x60
370
+/** "push %eax" instruction */
371
+#define PUSH_INSN 0x50
342
 
372
 
343
 /** "movb" instruction */
373
 /** "movb" instruction */
344
 #define MOVB_INSN 0xb0
374
 #define MOVB_INSN 0xb0

+ 412
- 38
src/arch/x86/transitions/librm.S View File

19
 /* CR4: physical address extensions */
19
 /* CR4: physical address extensions */
20
 #define CR4_PAE ( 1 << 5 )
20
 #define CR4_PAE ( 1 << 5 )
21
 
21
 
22
+/* Extended feature enable MSR (EFER) */
23
+#define MSR_EFER 0xc0000080
24
+
25
+/* EFER: long mode enable */
26
+#define EFER_LME ( 1 << 8 )
27
+
22
 /* Page: present */
28
 /* Page: present */
23
 #define PG_P 0x01
29
 #define PG_P 0x01
24
 
30
 
49
 #define SIZEOF_REAL_MODE_REGS	( SIZEOF_I386_SEG_REGS + SIZEOF_I386_REGS )
55
 #define SIZEOF_REAL_MODE_REGS	( SIZEOF_I386_SEG_REGS + SIZEOF_I386_REGS )
50
 #define SIZEOF_I386_FLAGS	4
56
 #define SIZEOF_I386_FLAGS	4
51
 #define SIZEOF_I386_ALL_REGS	( SIZEOF_REAL_MODE_REGS + SIZEOF_I386_FLAGS )
57
 #define SIZEOF_I386_ALL_REGS	( SIZEOF_REAL_MODE_REGS + SIZEOF_I386_FLAGS )
58
+#define SIZEOF_X86_64_REGS	128
52
 
59
 
53
 /* Size of an address */
60
 /* Size of an address */
54
 #ifdef __x86_64__
61
 #ifdef __x86_64__
57
 #define SIZEOF_ADDR 4
64
 #define SIZEOF_ADDR 4
58
 #endif
65
 #endif
59
 
66
 
67
+/* Default code size */
68
+#ifdef __x86_64__
69
+#define CODE_DEFAULT code64
70
+#else
71
+#define CODE_DEFAULT code32
72
+#endif
73
+
60
 /* Selectively assemble code for 32-bit/64-bit builds */
74
 /* Selectively assemble code for 32-bit/64-bit builds */
61
 #ifdef __x86_64__
75
 #ifdef __x86_64__
62
 #define if32 if 0
76
 #define if32 if 0
124
 	.word	0xffff, ( P2R_DS << 4 )
138
 	.word	0xffff, ( P2R_DS << 4 )
125
 	.byte	0, 0x93, 0x00, 0
139
 	.byte	0, 0x93, 0x00, 0
126
 
140
 
141
+	.org	gdt + LONG_CS, 0
142
+long_cs:	/* 64 bit long mode code segment */
143
+	.word	0, 0
144
+	.byte	0, 0x9a, 0x20, 0
145
+
127
 gdt_end:
146
 gdt_end:
128
 	.equ	gdt_length, gdt_end - gdt
147
 	.equ	gdt_length, gdt_end - gdt
129
 
148
 
256
 .if32 ;	subl	%edi, %eax ; .endif
275
 .if32 ;	subl	%edi, %eax ; .endif
257
 	movl	%eax, rm_data16
276
 	movl	%eax, rm_data16
258
 
277
 
259
-.if64 ;	/* Reset page tables, if applicable */
260
-	xorl	%eax, %eax
261
-	movl	%eax, pml4
262
-.endif
278
+	/* Configure virt_call for protected mode, if applicable */
279
+.if64 ;	movl	$VIRTUAL(vc_pmode), %cs:vc_jmp_offset ; .endif
280
+
263
 	/* Switch to protected mode */
281
 	/* Switch to protected mode */
264
 	virtcall init_librm_pmode
282
 	virtcall init_librm_pmode
265
 	.section ".text.init_librm", "ax", @progbits
283
 	.section ".text.init_librm", "ax", @progbits
276
 	rep movsl
294
 	rep movsl
277
 	popw	%ds
295
 	popw	%ds
278
 
296
 
279
-.if64 ;	/* Initialise page tables, if applicable */
297
+.if64 ;	/* Initialise long mode, if applicable */
280
 	movl	VIRTUAL(virt_offset), %edi
298
 	movl	VIRTUAL(virt_offset), %edi
299
+	leal	VIRTUAL(p2l_ljmp_target)(%edi), %eax
300
+	movl	%eax, VIRTUAL(p2l_ljmp_offset)
281
 	call	init_pages
301
 	call	init_pages
282
 .endif
302
 .endif
283
 	/* Return to real mode */
303
 	/* Return to real mode */
286
 	.code16
306
 	.code16
287
 init_librm_rmode:
307
 init_librm_rmode:
288
 
308
 
309
+	/* Configure virt_call for long mode, if applicable */
310
+.if64 ;	movl	$VIRTUAL(vc_lmode), %cs:vc_jmp_offset ; .endif
311
+
289
 	/* Initialise IDT */
312
 	/* Initialise IDT */
290
 	virtcall init_idt
313
 	virtcall init_idt
291
 
314
 
361
 	movw	%ax, %gs
384
 	movw	%ax, %gs
362
 	movw	%ax, %ss
385
 	movw	%ax, %ss
363
 
386
 
364
-	/* Switch to protected mode */
387
+	/* Switch to protected mode (with paging disabled if applicable) */
365
 	cli
388
 	cli
366
 	movl	%cr0, %eax
389
 	movl	%cr0, %eax
390
+.if64 ;	andl	$~CR0_PG, %eax ; .endif
367
 	orb	$CR0_PE, %al
391
 	orb	$CR0_PE, %al
368
 	movl	%eax, %cr0
392
 	movl	%eax, %cr0
369
 	data32 ljmp	$VIRTUAL_CS, $VIRTUAL(r2p_pmode)
393
 	data32 ljmp	$VIRTUAL_CS, $VIRTUAL(r2p_pmode)
380
 	movl	VIRTUAL(pm_esp), %esp
404
 	movl	VIRTUAL(pm_esp), %esp
381
 
405
 
382
 	/* Load protected-mode interrupt descriptor table */
406
 	/* Load protected-mode interrupt descriptor table */
383
-	lidt	VIRTUAL(idtr)
407
+	lidt	VIRTUAL(idtr32)
384
 
408
 
385
 	/* Record real-mode %ss:sp (after removal of data) */
409
 	/* Record real-mode %ss:sp (after removal of data) */
386
 	movw	%bp, VIRTUAL(rm_ss)
410
 	movw	%bp, VIRTUAL(rm_ss)
639
 	.globl	_intr_to_virt
663
 	.globl	_intr_to_virt
640
 	.equ	_intr_to_virt, intr_to_prot
664
 	.equ	_intr_to_virt, intr_to_prot
641
 
665
 
666
+/****************************************************************************
667
+ * prot_to_long (protected-mode near call, 32-bit virtual return address)
668
+ *
669
+ * Switch from 32-bit protected mode with virtual addresses to 64-bit
670
+ * long mode.  The protected-mode %esp is adjusted to a physical
671
+ * address.  All other registers are preserved.
672
+ *
673
+ * The return address for this function should be a 32-bit (sic)
674
+ * virtual address.
675
+ *
676
+ ****************************************************************************
677
+ */
678
+	.if64
679
+
680
+	.section ".text.prot_to_long", "ax", @progbits
681
+	.code32
682
+prot_to_long:
683
+	/* Preserve registers */
684
+	pushl	%eax
685
+	pushl	%ecx
686
+	pushl	%edx
687
+
688
+	/* Set up PML4 */
689
+	movl	VIRTUAL(pml4), %eax
690
+	movl	%eax, %cr3
691
+
692
+	/* Enable PAE */
693
+	movl	%cr4, %eax
694
+	orb	$CR4_PAE, %al
695
+	movl	%eax, %cr4
696
+
697
+	/* Enable long mode */
698
+	movl	$MSR_EFER, %ecx
699
+	rdmsr
700
+	orw	$EFER_LME, %ax
701
+	wrmsr
702
+
703
+	/* Enable paging */
704
+	movl	%cr0, %eax
705
+	orl	$CR0_PG, %eax
706
+	movl	%eax, %cr0
707
+
708
+	/* Restore registers */
709
+	popl	%edx
710
+	popl	%ecx
711
+	popl	%eax
712
+
713
+	/* Construct 64-bit return address */
714
+	pushl	(%esp)
715
+	movl	$0xffffffff, 4(%esp)
716
+p2l_ljmp:
717
+	/* Switch to long mode (using a physical %rip) */
718
+	ljmp	$LONG_CS, $0
719
+	.code64
720
+p2l_lmode:
721
+	/* Adjust and zero-extend %esp to a physical address */
722
+	addl	virt_offset, %esp
723
+
724
+	/* Use long-mode IDT */
725
+	lidt	idtr64
726
+
727
+	/* Return to virtual address */
728
+	ret
729
+
730
+	/* Long mode jump offset and target.  Required since an ljmp
731
+	 * in protected mode will zero-extend the offset, and so
732
+	 * cannot reach an address within the negative 2GB as used by
733
+	 * -mcmodel=kernel.  Assigned by the call to init_librm.
734
+	 */
735
+	.equ	p2l_ljmp_offset, ( p2l_ljmp + 1 )
736
+	.equ	p2l_ljmp_target, p2l_lmode
737
+
738
+	.endif
739
+
740
+/****************************************************************************
741
+ * long_to_prot (long-mode near call, 64-bit virtual return address)
742
+ *
743
+ * Switch from 64-bit long mode to 32-bit protected mode with virtual
744
+ * addresses.  The long-mode %rsp is adjusted to a virtual address.
745
+ * All other registers are preserved.
746
+ *
747
+ * The return address for this function should be a 64-bit (sic)
748
+ * virtual address.
749
+ *
750
+ ****************************************************************************
751
+ */
752
+	.if64
753
+
754
+	.section ".text.long_to_prot", "ax", @progbits
755
+	.code64
756
+long_to_prot:
757
+	/* Switch to protected mode */
758
+	ljmp	*l2p_vector
759
+	.code32
760
+l2p_pmode:
761
+	/* Adjust %esp to a virtual address */
762
+	subl	VIRTUAL(virt_offset), %esp
763
+
764
+	/* Preserve registers */
765
+	pushl	%eax
766
+	pushl	%ecx
767
+	pushl	%edx
768
+
769
+	/* Disable paging */
770
+	movl	%cr0, %eax
771
+	andl	$~CR0_PG, %eax
772
+	movl	%eax, %cr0
773
+
774
+	/* Disable PAE (in case external non-PAE-aware code enables paging) */
775
+	movl	%cr4, %eax
776
+	andb	$~CR4_PAE, %al
777
+	movl	%eax, %cr4
778
+
779
+	/* Disable long mode */
780
+	movl	$MSR_EFER, %ecx
781
+	rdmsr
782
+	andw	$~EFER_LME, %ax
783
+	wrmsr
784
+
785
+	/* Restore registers */
786
+	popl	%edx
787
+	popl	%ecx
788
+	popl	%eax
789
+
790
+	/* Use protected-mode IDT */
791
+	lidt	VIRTUAL(idtr32)
792
+
793
+	/* Return */
794
+	ret	$4
795
+
796
+	/* Long mode jump vector.  Required since there is no "ljmp
797
+	 * immediate" instruction in long mode.
798
+	 */
799
+	.section ".data.l2p_vector", "aw", @progbits
800
+l2p_vector:
801
+	.long	VIRTUAL(l2p_pmode), VIRTUAL_CS
802
+
803
+	.endif
804
+
805
+/****************************************************************************
806
+ * long_save_regs (long-mode near call, 64-bit virtual return address)
807
+ *
808
+ * Preserve registers that are accessible only in long mode.  This
809
+ * includes %r8-%r15 and the upper halves of %rax, %rbx, %rcx, %rdx,
810
+ * %rsi, %rdi, and %rbp.
811
+ *
812
+ ****************************************************************************
813
+ */
814
+	.if64
815
+
816
+	.section ".text.long_preserve_regs", "ax", @progbits
817
+	.code64
818
+long_preserve_regs:
819
+	/* Preserve registers */
820
+	pushq	%rax
821
+	pushq	%rcx
822
+	pushq	%rdx
823
+	pushq	%rbx
824
+	pushq	%rsp
825
+	pushq	%rbp
826
+	pushq	%rsi
827
+	pushq	%rdi
828
+	pushq	%r8
829
+	pushq	%r9
830
+	pushq	%r10
831
+	pushq	%r11
832
+	pushq	%r12
833
+	pushq	%r13
834
+	pushq	%r14
835
+	pushq	%r15
836
+
837
+	/* Return */
838
+	jmp	*SIZEOF_X86_64_REGS(%rsp)
839
+
840
+	.endif
841
+
842
+/****************************************************************************
843
+ * long_restore_regs (long-mode near call, 64-bit virtual return address)
844
+ *
845
+ * Restore registers that are accessible only in long mode.  This
846
+ * includes %r8-%r15 and the upper halves of %rax, %rbx, %rcx, %rdx,
847
+ * %rsi, %rdi, and %rbp.
848
+ *
849
+ ****************************************************************************
850
+ */
851
+	.if64
852
+
853
+	.section ".text.long_restore_regs", "ax", @progbits
854
+	.code64
855
+long_restore_regs:
856
+	/* Move return address above register dump */
857
+	popq	SIZEOF_X86_64_REGS(%rsp)
858
+
859
+	/* Restore registers */
860
+	popq	%r15
861
+	popq	%r14
862
+	popq	%r13
863
+	popq	%r12
864
+	popq	%r11
865
+	popq	%r10
866
+	popq	%r9
867
+	popq	%r8
868
+	movl	%edi, (%rsp)
869
+	popq	%rdi
870
+	movl	%esi, (%rsp)
871
+	popq	%rsi
872
+	movl	%ebp, (%rsp)
873
+	popq	%rbp
874
+	leaq	8(%rsp), %rsp /* discard */
875
+	movl	%ebx, (%rsp)
876
+	popq	%rbx
877
+	movl	%edx, (%rsp)
878
+	popq	%rdx
879
+	movl	%ecx, (%rsp)
880
+	popq	%rcx
881
+	movl	%eax, (%rsp)
882
+	popq	%rax
883
+
884
+	/* Return */
885
+	ret
886
+
887
+	.endif
888
+
642
 /****************************************************************************
889
 /****************************************************************************
643
  * virt_call (real-mode near call, 16-bit real-mode near return address)
890
  * virt_call (real-mode near call, 16-bit real-mode near return address)
644
  *
891
  *
645
- * Call a specific C function in the protected-mode code.  The
646
- * prototype of the C function must be
892
+ * Call a specific C function in 32-bit protected mode or 64-bit long
893
+ * mode (as applicable).  The prototype of the C function must be
647
  *   void function ( struct i386_all_regs *ix86 ); 
894
  *   void function ( struct i386_all_regs *ix86 ); 
648
  * ix86 will point to a struct containing the real-mode registers
895
  * ix86 will point to a struct containing the real-mode registers
649
  * at entry to virt_call().
896
  * at entry to virt_call().
662
  * critical data to registers before calling main()).
909
  * critical data to registers before calling main()).
663
  *
910
  *
664
  * Parameters:
911
  * Parameters:
665
- *   function : virtual address of protected-mode function to call
912
+ *   function : 32-bit virtual address of function to call
666
  *
913
  *
667
  * Example usage:
914
  * Example usage:
668
  *	pushl	$pxe_api_call
915
  *	pushl	$pxe_api_call
674
 	.struct	0
921
 	.struct	0
675
 VC_OFFSET_GDT:		.space	6
922
 VC_OFFSET_GDT:		.space	6
676
 VC_OFFSET_IDT:		.space	6
923
 VC_OFFSET_IDT:		.space	6
924
+.if64
925
+VC_OFFSET_PADDING64:	.space	4 /* for alignment */
926
+VC_OFFSET_CR3:		.space	4
927
+VC_OFFSET_CR4:		.space	4
928
+VC_OFFSET_EMER:		.space	8
929
+.endif
677
 VC_OFFSET_IX86:		.space	SIZEOF_I386_ALL_REGS
930
 VC_OFFSET_IX86:		.space	SIZEOF_I386_ALL_REGS
678
 VC_OFFSET_PADDING:	.space	2 /* for alignment */
931
 VC_OFFSET_PADDING:	.space	2 /* for alignment */
679
 VC_OFFSET_RETADDR:	.space	2
932
 VC_OFFSET_RETADDR:	.space	2
701
 	sidt	VC_OFFSET_IDT(%bp)
954
 	sidt	VC_OFFSET_IDT(%bp)
702
 	sgdt	VC_OFFSET_GDT(%bp)
955
 	sgdt	VC_OFFSET_GDT(%bp)
703
 
956
 
957
+.if64 ;	/* Preserve control registers, if applicable */
958
+	movl	$MSR_EFER, %ecx
959
+	rdmsr
960
+	movl	%eax, (VC_OFFSET_EMER+0)(%bp)
961
+	movl	%edx, (VC_OFFSET_EMER+4)(%bp)
962
+	movl	%cr4, %eax
963
+	movl	%eax, VC_OFFSET_CR4(%bp)
964
+	movl	%cr3, %eax
965
+	movl	%eax, VC_OFFSET_CR3(%bp)
966
+.endif
704
 	/* For sanity's sake, clear the direction flag as soon as possible */
967
 	/* For sanity's sake, clear the direction flag as soon as possible */
705
 	cld
968
 	cld
706
 
969
 
707
 	/* Switch to protected mode and move register dump to PM stack */
970
 	/* Switch to protected mode and move register dump to PM stack */
708
 	movl	$VC_OFFSET_END, %ecx
971
 	movl	$VC_OFFSET_END, %ecx
709
 	pushl	$VIRTUAL(vc_pmode)
972
 	pushl	$VIRTUAL(vc_pmode)
710
-	jmp	real_to_prot
973
+vc_jmp:	jmp	real_to_prot
711
 	.section ".text.virt_call", "ax", @progbits
974
 	.section ".text.virt_call", "ax", @progbits
712
 	.code32
975
 	.code32
713
 vc_pmode:
976
 vc_pmode:
714
-	/* Call function */
977
+	/* Call function (in protected mode) */
715
 	leal	VC_OFFSET_IX86(%esp), %eax
978
 	leal	VC_OFFSET_IX86(%esp), %eax
716
 	pushl	%eax
979
 	pushl	%eax
717
 	call	*(VC_OFFSET_FUNCTION+4)(%esp)
980
 	call	*(VC_OFFSET_FUNCTION+4)(%esp)
718
 	popl	%eax /* discard */
981
 	popl	%eax /* discard */
719
 
982
 
983
+.if64 ; /* Switch to long mode */
984
+	jmp	1f
985
+vc_lmode:
986
+	call	prot_to_long
987
+	.code64
988
+
989
+	/* Call function (in long mode) */
990
+	leaq	VC_OFFSET_IX86(%rsp), %rdi
991
+	pushq	%rdi
992
+	movslq	(VC_OFFSET_FUNCTION+8)(%rsp), %rax
993
+	callq	*%rax
994
+	popq	%rdi /* discard */
995
+
996
+	/* Switch to protected mode */
997
+	call	long_to_prot
998
+1:	.code32
999
+.endif
720
 	/* Switch to real mode and move register dump back to RM stack */
1000
 	/* Switch to real mode and move register dump back to RM stack */
721
 	movl	$VC_OFFSET_END, %ecx
1001
 	movl	$VC_OFFSET_END, %ecx
722
 	movl	%esp, %esi
1002
 	movl	%esp, %esi
725
 	.section ".text16.virt_call", "ax", @progbits
1005
 	.section ".text16.virt_call", "ax", @progbits
726
 	.code16
1006
 	.code16
727
 vc_rmode:
1007
 vc_rmode:
1008
+.if64 ;	/* Restore control registers, if applicable */
1009
+	movw	%sp, %bp
1010
+	movl	VC_OFFSET_CR3(%bp), %eax
1011
+	movl	%eax, %cr3
1012
+	movl	VC_OFFSET_CR4(%bp), %eax
1013
+	movl	%eax, %cr4
1014
+	movl	(VC_OFFSET_EMER+0)(%bp), %eax
1015
+	movl	(VC_OFFSET_EMER+4)(%bp), %edx
1016
+	movl	$MSR_EFER, %ecx
1017
+	wrmsr
1018
+.endif
728
 	/* Restore registers and flags and return */
1019
 	/* Restore registers and flags and return */
729
 	addw	$( VC_OFFSET_IX86 + 4 /* also skip %cs and %ss */ ), %sp
1020
 	addw	$( VC_OFFSET_IX86 + 4 /* also skip %cs and %ss */ ), %sp
730
 	popw	%ds
1021
 	popw	%ds
744
 	/* Return and discard function parameters */
1035
 	/* Return and discard function parameters */
745
 	ret	$( VC_OFFSET_END - VC_OFFSET_PARAMS )
1036
 	ret	$( VC_OFFSET_END - VC_OFFSET_PARAMS )
746
 
1037
 
1038
+
1039
+	/* Protected-mode jump target */
1040
+	.equ	vc_jmp_offset, ( vc_jmp - 4 )
1041
+
747
 /****************************************************************************
1042
 /****************************************************************************
748
  * real_call (protected-mode near call, 32-bit virtual return address)
1043
  * real_call (protected-mode near call, 32-bit virtual return address)
1044
+ * real_call (long-mode near call, 64-bit virtual return address)
749
  *
1045
  *
750
- * Call a real-mode function from protected-mode code.
1046
+ * Call a real-mode function from protected-mode or long-mode code.
751
  *
1047
  *
752
  * The non-segment register values will be passed directly to the
1048
  * The non-segment register values will be passed directly to the
753
  * real-mode code.  The segment registers will be set as per
1049
  * real-mode code.  The segment registers will be set as per
754
  * prot_to_real.  The non-segment register values set by the real-mode
1050
  * prot_to_real.  The non-segment register values set by the real-mode
755
- * function will be passed back to the protected-mode caller.  A
756
- * result of this is that this routine cannot be called directly from
757
- * C code, since it clobbers registers that the C ABI expects the
758
- * callee to preserve.
1051
+ * function will be passed back to the protected-mode or long-mode
1052
+ * caller.  A result of this is that this routine cannot be called
1053
+ * directly from C code, since it clobbers registers that the C ABI
1054
+ * expects the callee to preserve.
759
  *
1055
  *
760
  * librm.h defines a convenient macro REAL_CODE() for using real_call.
1056
  * librm.h defines a convenient macro REAL_CODE() for using real_call.
761
  * See librm.h and realmode.h for details and examples.
1057
  * See librm.h and realmode.h for details and examples.
769
 	.struct	0
1065
 	.struct	0
770
 RC_OFFSET_REGS:		.space	SIZEOF_I386_REGS
1066
 RC_OFFSET_REGS:		.space	SIZEOF_I386_REGS
771
 RC_OFFSET_REGS_END:
1067
 RC_OFFSET_REGS_END:
772
-RC_OFFSET_RETADDR:	.space	4
1068
+.if64
1069
+RC_OFFSET_LREGS:	.space	SIZEOF_X86_64_REGS
1070
+RC_OFFSET_LREG_RETADDR:	.space	SIZEOF_ADDR
1071
+.endif
1072
+RC_OFFSET_RETADDR:	.space	SIZEOF_ADDR
773
 RC_OFFSET_PARAMS:
1073
 RC_OFFSET_PARAMS:
774
-RC_OFFSET_FUNCTION:	.space	4
1074
+RC_OFFSET_FUNCTION:	.space	SIZEOF_ADDR
775
 RC_OFFSET_END:
1075
 RC_OFFSET_END:
776
 	.previous
1076
 	.previous
777
 
1077
 
778
 	.section ".text.real_call", "ax", @progbits
1078
 	.section ".text.real_call", "ax", @progbits
779
-	.code32
1079
+	.CODE_DEFAULT
780
 	.globl real_call
1080
 	.globl real_call
781
 real_call:
1081
 real_call:
1082
+.if64 ;	/* Preserve registers and switch to protected mode, if applicable */
1083
+	call	long_preserve_regs
1084
+	call	long_to_prot
1085
+	.code32
1086
+.endif
782
 	/* Create register dump and function pointer copy on PM stack */
1087
 	/* Create register dump and function pointer copy on PM stack */
783
 	pushal
1088
 	pushal
784
 	pushl	RC_OFFSET_FUNCTION(%esp)
1089
 	pushl	RC_OFFSET_FUNCTION(%esp)
810
 	/* Restore registers */
1115
 	/* Restore registers */
811
 	popal
1116
 	popal
812
 
1117
 
1118
+.if64 ; /* Switch to long mode and restore registers, if applicable */
1119
+	call	prot_to_long
1120
+	.code64
1121
+	call	long_restore_regs
1122
+.endif
813
 	/* Return and discard function parameters */
1123
 	/* Return and discard function parameters */
814
 	ret	$( RC_OFFSET_END - RC_OFFSET_PARAMS )
1124
 	ret	$( RC_OFFSET_END - RC_OFFSET_PARAMS )
815
 
1125
 
830
 
1140
 
831
 /****************************************************************************
1141
 /****************************************************************************
832
  * phys_call (protected-mode near call, 32-bit virtual return address)
1142
  * phys_call (protected-mode near call, 32-bit virtual return address)
1143
+ * phys_call (long-mode near call, 64-bit virtual return address)
833
  *
1144
  *
834
  * Call a function with flat 32-bit physical addressing
1145
  * Call a function with flat 32-bit physical addressing
835
  *
1146
  *
846
  ****************************************************************************
1157
  ****************************************************************************
847
  */
1158
  */
848
 	.struct 0
1159
 	.struct 0
849
-PHC_OFFSET_RETADDR:	.space	4
1160
+.if64
1161
+PHC_OFFSET_LREGS:	.space	SIZEOF_X86_64_REGS
1162
+PHC_OFFSET_LREG_RETADDR:.space	SIZEOF_ADDR
1163
+.endif
1164
+PHC_OFFSET_RETADDR:	.space	SIZEOF_ADDR
850
 PHC_OFFSET_PARAMS:
1165
 PHC_OFFSET_PARAMS:
851
-PHC_OFFSET_FUNCTION:	.space	4
1166
+PHC_OFFSET_FUNCTION:	.space	SIZEOF_ADDR
852
 PHC_OFFSET_END:
1167
 PHC_OFFSET_END:
853
 	.previous
1168
 	.previous
854
 
1169
 
855
 	.section ".text.phys_call", "ax", @progbits
1170
 	.section ".text.phys_call", "ax", @progbits
856
-	.code32
1171
+	.CODE_DEFAULT
857
 	.globl phys_call
1172
 	.globl phys_call
858
 phys_call:
1173
 phys_call:
1174
+.if64 ;	/* Preserve registers and switch to protected mode, if applicable */
1175
+	call	long_preserve_regs
1176
+	call	long_to_prot
1177
+	.code32
1178
+.endif
859
 	/* Adjust function pointer to a physical address */
1179
 	/* Adjust function pointer to a physical address */
860
 	pushl	%ebp
1180
 	pushl	%ebp
861
 	movl	VIRTUAL(virt_offset), %ebp
1181
 	movl	VIRTUAL(virt_offset), %ebp
874
 	/* Switch to virtual addresses */
1194
 	/* Switch to virtual addresses */
875
 	call	phys_to_prot
1195
 	call	phys_to_prot
876
 
1196
 
1197
+.if64 ; /* Switch to long mode and restore registers, if applicable */
1198
+	call	prot_to_long
1199
+	.code64
1200
+	call	long_restore_regs
1201
+.endif
877
 	/* Return and discard function parameters */
1202
 	/* Return and discard function parameters */
878
 	ret	$( PHC_OFFSET_END - PHC_OFFSET_PARAMS )
1203
 	ret	$( PHC_OFFSET_END - PHC_OFFSET_PARAMS )
879
 
1204
 
900
 	ret
1225
 	ret
901
 
1226
 
902
 	.section ".text.flatten_dummy", "ax", @progbits
1227
 	.section ".text.flatten_dummy", "ax", @progbits
903
-	.code32
1228
+	.CODE_DEFAULT
904
 flatten_dummy:
1229
 flatten_dummy:
905
 	ret
1230
 	ret
906
 
1231
 
907
 /****************************************************************************
1232
 /****************************************************************************
908
  * Interrupt wrapper
1233
  * Interrupt wrapper
909
  *
1234
  *
910
- * Used by the protected-mode interrupt vectors to call the
911
- * interrupt() function.
1235
+ * Used by the protected-mode and long-mode interrupt vectors to call
1236
+ * the interrupt() function.
912
  *
1237
  *
913
  * May be entered with either physical or virtual stack segment.
1238
  * May be entered with either physical or virtual stack segment.
914
  ****************************************************************************
1239
  ****************************************************************************
917
 	.code32
1242
 	.code32
918
 	.globl interrupt_wrapper
1243
 	.globl interrupt_wrapper
919
 interrupt_wrapper:
1244
 interrupt_wrapper:
1245
+	/* Preserve registers (excluding already-saved %eax and
1246
+	 * otherwise unused registers which are callee-save for both
1247
+	 * 32-bit and 64-bit ABIs).
1248
+	 */
1249
+	pushl	%ebx
1250
+	pushl	%ecx
1251
+	pushl	%edx
1252
+	pushl	%esi
1253
+	pushl	%edi
1254
+
1255
+	/* Expand IRQ number to whole %eax register */
1256
+	movzbl	%al, %eax
1257
+
1258
+.if64 ; /* Skip transition to long mode, if applicable */
1259
+	movw	%cs, %bx
1260
+	cmpw	$LONG_CS, %bx
1261
+	je	1f
1262
+.endif
920
 	/* Preserve segment registers and original %esp */
1263
 	/* Preserve segment registers and original %esp */
921
 	pushl	%ds
1264
 	pushl	%ds
922
 	pushl	%es
1265
 	pushl	%es
927
 
1270
 
928
 	/* Switch to virtual addressing */
1271
 	/* Switch to virtual addressing */
929
 	call	intr_to_prot
1272
 	call	intr_to_prot
930
-
931
-	/* Expand IRQ number to whole %eax register */
932
-	movzbl	%al, %eax
933
-
1273
+.if64
1274
+	/* Switch to long mode */
1275
+	call	prot_to_long
1276
+	.code64
1277
+
1278
+1:	/* Preserve long-mode caller-save registers */
1279
+	pushq	%r8
1280
+	pushq	%r9
1281
+	pushq	%r10
1282
+	pushq	%r11
1283
+
1284
+	/* Expand IRQ number to whole %rdi register */
1285
+	movl	%eax, %edi
1286
+.endif
934
 	/* Call interrupt handler */
1287
 	/* Call interrupt handler */
935
 	call	interrupt
1288
 	call	interrupt
1289
+.if64
1290
+	/* Restore long-mode caller-save registers */
1291
+	popq	%r11
1292
+	popq	%r10
1293
+	popq	%r9
1294
+	popq	%r8
1295
+
1296
+	/* Skip transition back to protected mode, if applicable */
1297
+	cmpw	$LONG_CS, %bx
1298
+	je	1f
936
 
1299
 
937
-	/* Restore original stack and segment registers */
1300
+	/* Switch to protected mode */
1301
+	call	long_to_prot
1302
+	.code32
1303
+	cmpw	$LONG_CS, %bx
1304
+.endif
1305
+	/* Restore segment registers and original %esp */
938
 	lss	(%esp), %esp
1306
 	lss	(%esp), %esp
939
 	popl	%ss
1307
 	popl	%ss
940
 	popl	%gs
1308
 	popl	%gs
942
 	popl	%es
1310
 	popl	%es
943
 	popl	%ds
1311
 	popl	%ds
944
 
1312
 
945
-	/* Restore registers and return */
946
-	popal
947
-	iret
1313
+1:	/* Restore registers */
1314
+	popl	%edi
1315
+	popl	%esi
1316
+	popl	%edx
1317
+	popl	%ecx
1318
+	popl	%ebx
1319
+	popl	%eax
1320
+
1321
+	/* Return from interrupt (with REX prefix if required) */
1322
+.if64 ; jne 1f ; .byte 0x48 ; .endif
1323
+1:	iret
948
 
1324
 
949
 /****************************************************************************
1325
 /****************************************************************************
950
  * Page tables
1326
  * Page tables
1022
 pte_textdata:
1398
 pte_textdata:
1023
 	/* Allocated by linker script; must be at the end of .textdata */
1399
 	/* Allocated by linker script; must be at the end of .textdata */
1024
 
1400
 
1025
-	.section ".bss16.pml4", "aw", @nobits
1401
+	.section ".bss.pml4", "aw", @nobits
1026
 pml4:	.long	0
1402
 pml4:	.long	0
1027
 
1403
 
1028
 /****************************************************************************
1404
 /****************************************************************************
1080
 
1456
 
1081
 	/* Record PML4 physical address */
1457
 	/* Record PML4 physical address */
1082
 	leal	VIRTUAL(pml4e)(%edi), %eax
1458
 	leal	VIRTUAL(pml4e)(%edi), %eax
1083
-	movl	VIRTUAL(data16), %ebx
1084
-	subl	%edi, %ebx
1085
-	movl	%eax, pml4(%ebx)
1459
+	movl	%eax, VIRTUAL(pml4)
1086
 
1460
 
1087
 	/* Return */
1461
 	/* Return */
1088
 	ret
1462
 	ret

+ 44
- 14
src/arch/x86/transitions/librm_mgmt.c View File

23
 /** The interrupt vectors */
23
 /** The interrupt vectors */
24
 static struct interrupt_vector intr_vec[NUM_INT];
24
 static struct interrupt_vector intr_vec[NUM_INT];
25
 
25
 
26
-/** The interrupt descriptor table */
27
-struct interrupt_descriptor idt[NUM_INT] __attribute__ (( aligned ( 16 ) ));
26
+/** The 32-bit interrupt descriptor table */
27
+static struct interrupt32_descriptor
28
+idt32[NUM_INT] __attribute__ (( aligned ( 16 ) ));
29
+
30
+/** The 32-bit interrupt descriptor table register */
31
+struct idtr32 idtr32 = {
32
+	.limit = ( sizeof ( idt32 ) - 1 ),
33
+};
34
+
35
+/** The 64-bit interrupt descriptor table */
36
+static struct interrupt64_descriptor
37
+idt64[NUM_INT] __attribute__ (( aligned ( 16 ) ));
28
 
38
 
29
 /** The interrupt descriptor table register */
39
 /** The interrupt descriptor table register */
30
-struct idtr idtr = {
31
-	.limit = ( sizeof ( idt ) - 1 ),
40
+struct idtr64 idtr64 = {
41
+	.limit = ( sizeof ( idt64 ) - 1 ),
32
 };
42
 };
33
 
43
 
34
 /** Timer interrupt profiler */
44
 /** Timer interrupt profiler */
75
  * @v vector		Interrupt vector, or NULL to disable
85
  * @v vector		Interrupt vector, or NULL to disable
76
  */
86
  */
77
 void set_interrupt_vector ( unsigned int intr, void *vector ) {
87
 void set_interrupt_vector ( unsigned int intr, void *vector ) {
78
-	struct interrupt_descriptor *idte;
79
-
80
-	idte = &idt[intr];
81
-	idte->segment = VIRTUAL_CS;
82
-	idte->attr = ( vector ? ( IDTE_PRESENT | IDTE_TYPE_IRQ32 ) : 0 );
83
-	idte->low = ( ( ( intptr_t ) vector ) & 0xffff );
84
-	idte->high = ( ( ( intptr_t ) vector ) >> 16 );
88
+	struct interrupt32_descriptor *idte32;
89
+	struct interrupt64_descriptor *idte64;
90
+	intptr_t addr = ( ( intptr_t ) vector );
91
+
92
+	/* Populate 32-bit interrupt descriptor */
93
+	idte32 = &idt32[intr];
94
+	idte32->segment = VIRTUAL_CS;
95
+	idte32->attr = ( vector ? ( IDTE_PRESENT | IDTE_TYPE_IRQ32 ) : 0 );
96
+	idte32->low = ( addr >> 0 );
97
+	idte32->high = ( addr >> 16 );
98
+
99
+	/* Populate 64-bit interrupt descriptor, if applicable */
100
+	if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) {
101
+		idte64 = &idt64[intr];
102
+		idte64->segment = LONG_CS;
103
+		idte64->attr = ( vector ?
104
+				 ( IDTE_PRESENT | IDTE_TYPE_IRQ64 ) : 0 );
105
+		idte64->low = ( addr >> 0 );
106
+		idte64->mid = ( addr >> 16 );
107
+		idte64->high = ( ( ( uint64_t ) addr ) >> 32 );
108
+	}
85
 }
109
 }
86
 
110
 
87
 /**
111
 /**
95
 	/* Initialise the interrupt descriptor table and interrupt vectors */
119
 	/* Initialise the interrupt descriptor table and interrupt vectors */
96
 	for ( intr = 0 ; intr < NUM_INT ; intr++ ) {
120
 	for ( intr = 0 ; intr < NUM_INT ; intr++ ) {
97
 		vec = &intr_vec[intr];
121
 		vec = &intr_vec[intr];
98
-		vec->pushal = PUSHAL_INSN;
122
+		vec->push = PUSH_INSN;
99
 		vec->movb = MOVB_INSN;
123
 		vec->movb = MOVB_INSN;
100
 		vec->intr = intr;
124
 		vec->intr = intr;
101
 		vec->jmp = JMP_INSN;
125
 		vec->jmp = JMP_INSN;
107
 	       intr_vec, sizeof ( intr_vec[0] ),
131
 	       intr_vec, sizeof ( intr_vec[0] ),
108
 	       virt_to_phys ( intr_vec ), sizeof ( intr_vec[0] ) );
132
 	       virt_to_phys ( intr_vec ), sizeof ( intr_vec[0] ) );
109
 
133
 
110
-	/* Initialise the interrupt descriptor table register */
111
-	idtr.base = virt_to_phys ( idt );
134
+	/* Initialise the 32-bit interrupt descriptor table register */
135
+	idtr32.base = virt_to_phys ( idt32 );
136
+
137
+	/* Initialise the 64-bit interrupt descriptor table register,
138
+	 * if applicable.
139
+	 */
140
+	if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) )
141
+		idtr64.base = virt_to_phys ( idt64 );
112
 }
142
 }
113
 
143
 
114
 /**
144
 /**

Loading…
Cancel
Save