Browse Source

[arbel] Ensure hardware is quiescent when no interfaces are open

Signed-off-by: Michael Brown <mcb30@ipxe.org>
tags/v1.20.1
Michael Brown 13 years ago
parent
commit
8ef5f6065d
2 changed files with 271 additions and 92 deletions
  1. 247
    90
      src/drivers/infiniband/arbel.c
  2. 24
    2
      src/drivers/infiniband/arbel.h

+ 247
- 90
src/drivers/infiniband/arbel.c View File

31
 #include <byteswap.h>
31
 #include <byteswap.h>
32
 #include <ipxe/io.h>
32
 #include <ipxe/io.h>
33
 #include <ipxe/pci.h>
33
 #include <ipxe/pci.h>
34
+#include <ipxe/pcibackup.h>
34
 #include <ipxe/malloc.h>
35
 #include <ipxe/malloc.h>
35
 #include <ipxe/umalloc.h>
36
 #include <ipxe/umalloc.h>
36
 #include <ipxe/iobuf.h>
37
 #include <ipxe/iobuf.h>
1991
 	struct arbelprm_query_fw fw;
1992
 	struct arbelprm_query_fw fw;
1992
 	struct arbelprm_access_lam lam;
1993
 	struct arbelprm_access_lam lam;
1993
 	unsigned int fw_pages;
1994
 	unsigned int fw_pages;
1994
-	size_t fw_size;
1995
+	size_t fw_len;
1995
 	physaddr_t fw_base;
1996
 	physaddr_t fw_base;
1996
 	uint64_t eq_set_ci_base_addr;
1997
 	uint64_t eq_set_ci_base_addr;
1997
 	int rc;
1998
 	int rc;
2019
 	arbel_cmd_enable_lam ( arbel, &lam );
2020
 	arbel_cmd_enable_lam ( arbel, &lam );
2020
 
2021
 
2021
 	/* Allocate firmware pages and map firmware area */
2022
 	/* Allocate firmware pages and map firmware area */
2022
-	fw_size = ( fw_pages * ARBEL_PAGE_SIZE );
2023
-	arbel->firmware_area = umalloc ( fw_size );
2023
+	fw_len = ( fw_pages * ARBEL_PAGE_SIZE );
2024
 	if ( ! arbel->firmware_area ) {
2024
 	if ( ! arbel->firmware_area ) {
2025
-		rc = -ENOMEM;
2026
-		goto err_alloc_fa;
2025
+		arbel->firmware_len = fw_len;
2026
+		arbel->firmware_area = umalloc ( arbel->firmware_len );
2027
+		if ( ! arbel->firmware_area ) {
2028
+			rc = -ENOMEM;
2029
+			goto err_alloc_fa;
2030
+		}
2031
+	} else {
2032
+		assert ( arbel->firmware_len == fw_len );
2027
 	}
2033
 	}
2028
 	fw_base = user_to_phys ( arbel->firmware_area, 0 );
2034
 	fw_base = user_to_phys ( arbel->firmware_area, 0 );
2029
 	DBGC ( arbel, "Arbel %p firmware area at [%08lx,%08lx)\n",
2035
 	DBGC ( arbel, "Arbel %p firmware area at [%08lx,%08lx)\n",
2030
-	       arbel, fw_base, ( fw_base + fw_size ) );
2036
+	       arbel, fw_base, ( fw_base + fw_len ) );
2031
 	if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_fa,
2037
 	if ( ( rc = arbel_map_vpm ( arbel, arbel_cmd_map_fa,
2032
-				    0, fw_base, fw_size ) ) != 0 ) {
2038
+				    0, fw_base, fw_len ) ) != 0 ) {
2033
 		DBGC ( arbel, "Arbel %p could not map firmware: %s\n",
2039
 		DBGC ( arbel, "Arbel %p could not map firmware: %s\n",
2034
 		       arbel, strerror ( rc ) );
2040
 		       arbel, strerror ( rc ) );
2035
 		goto err_map_fa;
2041
 		goto err_map_fa;
2048
  err_run_fw:
2054
  err_run_fw:
2049
 	arbel_cmd_unmap_fa ( arbel );
2055
 	arbel_cmd_unmap_fa ( arbel );
2050
  err_map_fa:
2056
  err_map_fa:
2051
-	ufree ( arbel->firmware_area );
2052
-	arbel->firmware_area = UNULL;
2053
  err_alloc_fa:
2057
  err_alloc_fa:
2054
  err_query_fw:
2058
  err_query_fw:
2055
 	return rc;
2059
 	return rc;
2067
 		DBGC ( arbel, "Arbel %p FATAL could not stop firmware: %s\n",
2071
 		DBGC ( arbel, "Arbel %p FATAL could not stop firmware: %s\n",
2068
 		       arbel, strerror ( rc ) );
2072
 		       arbel, strerror ( rc ) );
2069
 		/* Leak memory and return; at least we avoid corruption */
2073
 		/* Leak memory and return; at least we avoid corruption */
2074
+		arbel->firmware_area = UNULL;
2070
 		return;
2075
 		return;
2071
 	}
2076
 	}
2072
-	ufree ( arbel->firmware_area );
2073
-	arbel->firmware_area = UNULL;
2074
 }
2077
 }
2075
 
2078
 
2076
 /***************************************************************************
2079
 /***************************************************************************
2180
 	unsigned int log_num_uars, log_num_qps, log_num_srqs, log_num_ees;
2183
 	unsigned int log_num_uars, log_num_qps, log_num_srqs, log_num_ees;
2181
 	unsigned int log_num_cqs, log_num_mtts, log_num_mpts, log_num_rdbs;
2184
 	unsigned int log_num_cqs, log_num_mtts, log_num_mpts, log_num_rdbs;
2182
 	unsigned int log_num_eqs, log_num_mcs;
2185
 	unsigned int log_num_eqs, log_num_mcs;
2186
+	size_t icm_len, icm_aux_len;
2183
 	size_t len;
2187
 	size_t len;
2184
 	physaddr_t icm_phys;
2188
 	physaddr_t icm_phys;
2185
 	int rc;
2189
 	int rc;
2351
 
2355
 
2352
 	/* Record amount of ICM to be allocated */
2356
 	/* Record amount of ICM to be allocated */
2353
 	icm_offset = icm_align ( icm_offset, ARBEL_PAGE_SIZE );
2357
 	icm_offset = icm_align ( icm_offset, ARBEL_PAGE_SIZE );
2354
-	arbel->icm_len = icm_offset;
2358
+	icm_len = icm_offset;
2355
 
2359
 
2356
 	/* User access region contexts
2360
 	/* User access region contexts
2357
 	 *
2361
 	 *
2376
 
2380
 
2377
 	/* Get ICM auxiliary area size */
2381
 	/* Get ICM auxiliary area size */
2378
 	memset ( &icm_size, 0, sizeof ( icm_size ) );
2382
 	memset ( &icm_size, 0, sizeof ( icm_size ) );
2379
-	MLX_FILL_1 ( &icm_size, 1, value, arbel->icm_len );
2383
+	MLX_FILL_1 ( &icm_size, 1, value, icm_len );
2380
 	if ( ( rc = arbel_cmd_set_icm_size ( arbel, &icm_size,
2384
 	if ( ( rc = arbel_cmd_set_icm_size ( arbel, &icm_size,
2381
 					     &icm_aux_size ) ) != 0 ) {
2385
 					     &icm_aux_size ) ) != 0 ) {
2382
 		DBGC ( arbel, "Arbel %p could not set ICM size: %s\n",
2386
 		DBGC ( arbel, "Arbel %p could not set ICM size: %s\n",
2383
 		       arbel, strerror ( rc ) );
2387
 		       arbel, strerror ( rc ) );
2384
 		goto err_set_icm_size;
2388
 		goto err_set_icm_size;
2385
 	}
2389
 	}
2386
-	arbel->icm_aux_len =
2387
-		( MLX_GET ( &icm_aux_size, value ) * ARBEL_PAGE_SIZE );
2390
+	icm_aux_len = ( MLX_GET ( &icm_aux_size, value ) * ARBEL_PAGE_SIZE );
2388
 
2391
 
2389
 	/* Allocate ICM data and auxiliary area */
2392
 	/* Allocate ICM data and auxiliary area */
2390
 	DBGC ( arbel, "Arbel %p requires %zd kB ICM and %zd kB AUX ICM\n",
2393
 	DBGC ( arbel, "Arbel %p requires %zd kB ICM and %zd kB AUX ICM\n",
2391
-	       arbel, ( arbel->icm_len / 1024 ),
2392
-	       ( arbel->icm_aux_len / 1024 ) );
2393
-	arbel->icm = umalloc ( arbel->icm_len + arbel->icm_aux_len );
2394
+	       arbel, ( icm_len / 1024 ), ( icm_aux_len / 1024 ) );
2394
 	if ( ! arbel->icm ) {
2395
 	if ( ! arbel->icm ) {
2395
-		rc = -ENOMEM;
2396
-		goto err_alloc_icm;
2396
+		arbel->icm_len = icm_len;
2397
+		arbel->icm_aux_len = icm_aux_len;
2398
+		arbel->icm = umalloc ( arbel->icm_len + arbel->icm_aux_len );
2399
+		if ( ! arbel->icm ) {
2400
+			rc = -ENOMEM;
2401
+			goto err_alloc_icm;
2402
+		}
2403
+	} else {
2404
+		assert ( arbel->icm_len == icm_len );
2405
+		assert ( arbel->icm_aux_len == icm_aux_len );
2397
 	}
2406
 	}
2398
 	icm_phys = user_to_phys ( arbel->icm, 0 );
2407
 	icm_phys = user_to_phys ( arbel->icm, 0 );
2399
 
2408
 
2459
 	free_dma ( arbel->db_rec, ARBEL_PAGE_SIZE );
2468
 	free_dma ( arbel->db_rec, ARBEL_PAGE_SIZE );
2460
 	arbel->db_rec= NULL;
2469
 	arbel->db_rec= NULL;
2461
  err_alloc_doorbell:
2470
  err_alloc_doorbell:
2462
-	ufree ( arbel->icm );
2463
-	arbel->icm = UNULL;
2464
  err_alloc_icm:
2471
  err_alloc_icm:
2465
  err_set_icm_size:
2472
  err_set_icm_size:
2466
 	return rc;
2473
 	return rc;
2483
 	arbel_cmd_unmap_icm_aux ( arbel );
2490
 	arbel_cmd_unmap_icm_aux ( arbel );
2484
 	free_dma ( arbel->db_rec, ARBEL_PAGE_SIZE );
2491
 	free_dma ( arbel->db_rec, ARBEL_PAGE_SIZE );
2485
 	arbel->db_rec = NULL;
2492
 	arbel->db_rec = NULL;
2486
-	ufree ( arbel->icm );
2487
-	arbel->icm = UNULL;
2488
 }
2493
 }
2489
 
2494
 
2490
 /***************************************************************************
2495
 /***************************************************************************
2491
  *
2496
  *
2492
- * Initialisation
2497
+ * Initialisation and teardown
2493
  *
2498
  *
2494
  ***************************************************************************
2499
  ***************************************************************************
2495
  */
2500
  */
2496
 
2501
 
2502
+/**
2503
+ * Reset device
2504
+ *
2505
+ * @v arbel		Arbel device
2506
+ */
2507
+static void arbel_reset ( struct arbel *arbel ) {
2508
+	struct pci_device *pci = arbel->pci;
2509
+	struct pci_config_backup backup;
2510
+	static const uint8_t backup_exclude[] =
2511
+		PCI_CONFIG_BACKUP_EXCLUDE ( 0x58, 0x5c );
2512
+	uint16_t vendor;
2513
+	unsigned int i;
2514
+
2515
+	/* Perform device reset and preserve PCI configuration */
2516
+	pci_backup ( pci, &backup, backup_exclude );
2517
+	writel ( ARBEL_RESET_MAGIC,
2518
+		 ( arbel->config + ARBEL_RESET_OFFSET ) );
2519
+	for ( i = 0 ; i < ARBEL_RESET_WAIT_TIME_MS ; i++ ) {
2520
+		mdelay ( 1 );
2521
+		pci_read_config_word ( pci, PCI_VENDOR_ID, &vendor );
2522
+		if ( vendor != 0xffff )
2523
+			break;
2524
+	}
2525
+	pci_restore ( pci, &backup, backup_exclude );
2526
+}
2527
+
2497
 /**
2528
 /**
2498
  * Set up memory protection table
2529
  * Set up memory protection table
2499
  *
2530
  *
2572
 	return 0;
2603
 	return 0;
2573
 }
2604
 }
2574
 
2605
 
2606
+/**
2607
+ * Start Arbel device
2608
+ *
2609
+ * @v arbel		Arbel device
2610
+ * @v running		Firmware is already running
2611
+ * @ret rc		Return status code
2612
+ */
2613
+static int arbel_start ( struct arbel *arbel, int running ) {
2614
+	struct arbelprm_init_hca init_hca;
2615
+	unsigned int i;
2616
+	int rc;
2617
+
2618
+	/* Start firmware if not already running */
2619
+	if ( ! running ) {
2620
+		if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
2621
+			goto err_start_firmware;
2622
+	}
2623
+
2624
+	/* Allocate ICM */
2625
+	memset ( &init_hca, 0, sizeof ( init_hca ) );
2626
+	if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
2627
+		goto err_alloc_icm;
2628
+
2629
+	/* Initialise HCA */
2630
+	if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
2631
+		DBGC ( arbel, "Arbel %p could not initialise HCA: %s\n",
2632
+		       arbel, strerror ( rc ) );
2633
+		goto err_init_hca;
2634
+	}
2635
+
2636
+	/* Set up memory protection */
2637
+	if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
2638
+		goto err_setup_mpt;
2639
+	for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
2640
+		arbel->ibdev[i]->rdma_key = arbel->lkey;
2641
+
2642
+	/* Set up event queue */
2643
+	if ( ( rc = arbel_create_eq ( arbel ) ) != 0 )
2644
+		goto err_create_eq;
2645
+
2646
+	/* Configure special QPs */
2647
+	if ( ( rc = arbel_configure_special_qps ( arbel ) ) != 0 )
2648
+		goto err_conf_special_qps;
2649
+
2650
+	return 0;
2651
+
2652
+ err_conf_special_qps:
2653
+	arbel_destroy_eq ( arbel );
2654
+ err_create_eq:
2655
+ err_setup_mpt:
2656
+	arbel_cmd_close_hca ( arbel );
2657
+ err_init_hca:
2658
+	arbel_free_icm ( arbel );
2659
+ err_alloc_icm:
2660
+	arbel_stop_firmware ( arbel );
2661
+ err_start_firmware:
2662
+	return rc;
2663
+}
2664
+
2665
+/**
2666
+ * Stop Arbel device
2667
+ *
2668
+ * @v arbel		Arbel device
2669
+ */
2670
+static void arbel_stop ( struct arbel *arbel ) {
2671
+	arbel_destroy_eq ( arbel );
2672
+	arbel_cmd_close_hca ( arbel );
2673
+	arbel_free_icm ( arbel );
2674
+	arbel_stop_firmware ( arbel );
2675
+	arbel_reset ( arbel );
2676
+}
2677
+
2678
+/**
2679
+ * Open Arbel device
2680
+ *
2681
+ * @v arbel		Arbel device
2682
+ * @ret rc		Return status code
2683
+ */
2684
+static int arbel_open ( struct arbel *arbel ) {
2685
+	int rc;
2686
+
2687
+	/* Start device if applicable */
2688
+	if ( arbel->open_count == 0 ) {
2689
+		if ( ( rc = arbel_start ( arbel, 0 ) ) != 0 )
2690
+			return rc;
2691
+	}
2692
+
2693
+	/* Increment open counter */
2694
+	arbel->open_count++;
2695
+
2696
+	return 0;
2697
+}
2698
+
2699
+/**
2700
+ * Close Arbel device
2701
+ *
2702
+ * @v arbel		Arbel device
2703
+ */
2704
+static void arbel_close ( struct arbel *arbel ) {
2705
+
2706
+	/* Decrement open counter */
2707
+	assert ( arbel->open_count != 0 );
2708
+	arbel->open_count--;
2709
+
2710
+	/* Stop device if applicable */
2711
+	if ( arbel->open_count == 0 )
2712
+		arbel_stop ( arbel );
2713
+}
2714
+
2575
 /***************************************************************************
2715
 /***************************************************************************
2576
  *
2716
  *
2577
  * Infiniband link-layer operations
2717
  * Infiniband link-layer operations
2585
  * @v ibdev		Infiniband device
2725
  * @v ibdev		Infiniband device
2586
  * @ret rc		Return status code
2726
  * @ret rc		Return status code
2587
  */
2727
  */
2588
-static int arbel_open ( struct ib_device *ibdev ) {
2728
+static int arbel_ib_open ( struct ib_device *ibdev ) {
2589
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
2729
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
2590
 	struct arbelprm_init_ib init_ib;
2730
 	struct arbelprm_init_ib init_ib;
2591
 	int rc;
2731
 	int rc;
2592
 
2732
 
2733
+	/* Open hardware */
2734
+	if ( ( rc = arbel_open ( arbel ) ) != 0 )
2735
+		goto err_open;
2736
+
2737
+	/* Initialise IB */
2593
 	memset ( &init_ib, 0, sizeof ( init_ib ) );
2738
 	memset ( &init_ib, 0, sizeof ( init_ib ) );
2594
 	MLX_FILL_3 ( &init_ib, 0,
2739
 	MLX_FILL_3 ( &init_ib, 0,
2595
 		     mtu_cap, ARBEL_MTU_2048,
2740
 		     mtu_cap, ARBEL_MTU_2048,
2601
 					&init_ib ) ) != 0 ) {
2746
 					&init_ib ) ) != 0 ) {
2602
 		DBGC ( arbel, "Arbel %p port %d could not intialise IB: %s\n",
2747
 		DBGC ( arbel, "Arbel %p port %d could not intialise IB: %s\n",
2603
 		       arbel, ibdev->port, strerror ( rc ) );
2748
 		       arbel, ibdev->port, strerror ( rc ) );
2604
-		return rc;
2749
+		goto err_init_ib;
2605
 	}
2750
 	}
2606
 
2751
 
2607
 	/* Update MAD parameters */
2752
 	/* Update MAD parameters */
2608
 	ib_smc_update ( ibdev, arbel_mad );
2753
 	ib_smc_update ( ibdev, arbel_mad );
2609
 
2754
 
2610
 	return 0;
2755
 	return 0;
2756
+
2757
+ err_init_ib:
2758
+	arbel_close ( arbel );
2759
+ err_open:
2760
+	return rc;
2611
 }
2761
 }
2612
 
2762
 
2613
 /**
2763
 /**
2615
  *
2765
  *
2616
  * @v ibdev		Infiniband device
2766
  * @v ibdev		Infiniband device
2617
  */
2767
  */
2618
-static void arbel_close ( struct ib_device *ibdev ) {
2768
+static void arbel_ib_close ( struct ib_device *ibdev ) {
2619
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
2769
 	struct arbel *arbel = ib_get_drvdata ( ibdev );
2620
 	int rc;
2770
 	int rc;
2621
 
2771
 
2772
+	/* Close IB */
2622
 	if ( ( rc = arbel_cmd_close_ib ( arbel, ibdev->port ) ) != 0 ) {
2773
 	if ( ( rc = arbel_cmd_close_ib ( arbel, ibdev->port ) ) != 0 ) {
2623
 		DBGC ( arbel, "Arbel %p port %d could not close IB: %s\n",
2774
 		DBGC ( arbel, "Arbel %p port %d could not close IB: %s\n",
2624
 		       arbel, ibdev->port, strerror ( rc ) );
2775
 		       arbel, ibdev->port, strerror ( rc ) );
2625
 		/* Nothing we can do about this */
2776
 		/* Nothing we can do about this */
2626
 	}
2777
 	}
2778
+
2779
+	/* Close hardware */
2780
+	arbel_close ( arbel );
2627
 }
2781
 }
2628
 
2782
 
2629
 /**
2783
 /**
2753
 	.post_recv	= arbel_post_recv,
2907
 	.post_recv	= arbel_post_recv,
2754
 	.poll_cq	= arbel_poll_cq,
2908
 	.poll_cq	= arbel_poll_cq,
2755
 	.poll_eq	= arbel_poll_eq,
2909
 	.poll_eq	= arbel_poll_eq,
2756
-	.open		= arbel_open,
2757
-	.close		= arbel_close,
2910
+	.open		= arbel_ib_open,
2911
+	.close		= arbel_ib_close,
2758
 	.mcast_attach	= arbel_mcast_attach,
2912
 	.mcast_attach	= arbel_mcast_attach,
2759
 	.mcast_detach	= arbel_mcast_detach,
2913
 	.mcast_detach	= arbel_mcast_detach,
2760
 	.set_port_info	= arbel_inform_sma,
2914
 	.set_port_info	= arbel_inform_sma,
2768
  ***************************************************************************
2922
  ***************************************************************************
2769
  */
2923
  */
2770
 
2924
 
2925
+/**
2926
+ * Allocate Arbel device
2927
+ *
2928
+ * @ret arbel		Arbel device
2929
+ */
2930
+static struct arbel * arbel_alloc ( void ) {
2931
+	struct arbel *arbel;
2932
+
2933
+	/* Allocate Arbel device */
2934
+	arbel = zalloc ( sizeof ( *arbel ) );
2935
+	if ( ! arbel )
2936
+		goto err_arbel;
2937
+
2938
+	/* Allocate space for mailboxes */
2939
+	arbel->mailbox_in = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
2940
+	if ( ! arbel->mailbox_in )
2941
+		goto err_mailbox_in;
2942
+	arbel->mailbox_out = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
2943
+	if ( ! arbel->mailbox_out )
2944
+		goto err_mailbox_out;
2945
+
2946
+	return arbel;
2947
+
2948
+	free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
2949
+ err_mailbox_out:
2950
+	free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
2951
+ err_mailbox_in:
2952
+	free ( arbel );
2953
+ err_arbel:
2954
+	return NULL;
2955
+}
2956
+
2957
+/**
2958
+ * Free Arbel device
2959
+ *
2960
+ * @v arbel		Arbel device
2961
+ */
2962
+static void arbel_free ( struct arbel *arbel ) {
2963
+
2964
+	ufree ( arbel->icm );
2965
+	ufree ( arbel->firmware_area );
2966
+	free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
2967
+	free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
2968
+	free ( arbel );
2969
+}
2970
+
2771
 /**
2971
 /**
2772
  * Probe PCI device
2972
  * Probe PCI device
2773
  *
2973
  *
2778
 static int arbel_probe ( struct pci_device *pci ) {
2978
 static int arbel_probe ( struct pci_device *pci ) {
2779
 	struct arbel *arbel;
2979
 	struct arbel *arbel;
2780
 	struct ib_device *ibdev;
2980
 	struct ib_device *ibdev;
2781
-	struct arbelprm_init_hca init_hca;
2782
 	int i;
2981
 	int i;
2783
 	int rc;
2982
 	int rc;
2784
 
2983
 
2785
 	/* Allocate Arbel device */
2984
 	/* Allocate Arbel device */
2786
-	arbel = zalloc ( sizeof ( *arbel ) );
2985
+	arbel = arbel_alloc();
2787
 	if ( ! arbel ) {
2986
 	if ( ! arbel ) {
2788
 		rc = -ENOMEM;
2987
 		rc = -ENOMEM;
2789
-		goto err_alloc_arbel;
2988
+		goto err_alloc;
2790
 	}
2989
 	}
2791
 	pci_set_drvdata ( pci, arbel );
2990
 	pci_set_drvdata ( pci, arbel );
2991
+	arbel->pci = pci;
2792
 
2992
 
2793
 	/* Allocate Infiniband devices */
2993
 	/* Allocate Infiniband devices */
2794
 	for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) {
2994
 	for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ ) {
2814
 				 ARBEL_PCI_UAR_IDX * ARBEL_PCI_UAR_SIZE ),
3014
 				 ARBEL_PCI_UAR_IDX * ARBEL_PCI_UAR_SIZE ),
2815
 			       ARBEL_PCI_UAR_SIZE );
3015
 			       ARBEL_PCI_UAR_SIZE );
2816
 
3016
 
2817
-	/* Allocate space for mailboxes */
2818
-	arbel->mailbox_in = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
2819
-	if ( ! arbel->mailbox_in ) {
2820
-		rc = -ENOMEM;
2821
-		goto err_mailbox_in;
2822
-	}
2823
-	arbel->mailbox_out = malloc_dma ( ARBEL_MBOX_SIZE, ARBEL_MBOX_ALIGN );
2824
-	if ( ! arbel->mailbox_out ) {
2825
-		rc = -ENOMEM;
2826
-		goto err_mailbox_out;
2827
-	}
3017
+	/* Reset device */
3018
+	arbel_reset ( arbel );
2828
 
3019
 
2829
 	/* Start firmware */
3020
 	/* Start firmware */
2830
 	if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
3021
 	if ( ( rc = arbel_start_firmware ( arbel ) ) != 0 )
2834
 	if ( ( rc = arbel_get_limits ( arbel ) ) != 0 )
3025
 	if ( ( rc = arbel_get_limits ( arbel ) ) != 0 )
2835
 		goto err_get_limits;
3026
 		goto err_get_limits;
2836
 
3027
 
2837
-	/* Allocate ICM */
2838
-	memset ( &init_hca, 0, sizeof ( init_hca ) );
2839
-	if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
2840
-		goto err_alloc_icm;
2841
-
2842
-	/* Initialise HCA */
2843
-	if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
2844
-		DBGC ( arbel, "Arbel %p could not initialise HCA: %s\n",
2845
-		       arbel, strerror ( rc ) );
2846
-		goto err_init_hca;
2847
-	}
2848
-
2849
-	/* Set up memory protection */
2850
-	if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
2851
-		goto err_setup_mpt;
2852
-	for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
2853
-		arbel->ibdev[i]->rdma_key = arbel->lkey;
2854
-
2855
-	/* Set up event queue */
2856
-	if ( ( rc = arbel_create_eq ( arbel ) ) != 0 )
2857
-		goto err_create_eq;
2858
-
2859
-	/* Configure special QPs */
2860
-	if ( ( rc = arbel_configure_special_qps ( arbel ) ) != 0 )
2861
-		goto err_conf_special_qps;
3028
+	/* Start device */
3029
+	if ( ( rc = arbel_start ( arbel, 1 ) ) != 0 )
3030
+		goto err_start;
2862
 
3031
 
2863
 	/* Initialise parameters using SMC */
3032
 	/* Initialise parameters using SMC */
2864
 	for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
3033
 	for ( i = 0 ; i < ARBEL_NUM_PORTS ; i++ )
2874
 		}
3043
 		}
2875
 	}
3044
 	}
2876
 
3045
 
3046
+	/* Leave device quiescent until opened */
3047
+	if ( arbel->open_count == 0 )
3048
+		arbel_stop ( arbel );
3049
+
2877
 	return 0;
3050
 	return 0;
2878
 
3051
 
2879
 	i = ARBEL_NUM_PORTS;
3052
 	i = ARBEL_NUM_PORTS;
2880
  err_register_ibdev:
3053
  err_register_ibdev:
2881
 	for ( i-- ; i >= 0 ; i-- )
3054
 	for ( i-- ; i >= 0 ; i-- )
2882
 		unregister_ibdev ( arbel->ibdev[i] );
3055
 		unregister_ibdev ( arbel->ibdev[i] );
2883
- err_conf_special_qps:
2884
-	arbel_destroy_eq ( arbel );
2885
- err_create_eq:
2886
- err_setup_mpt:
2887
-	arbel_cmd_close_hca ( arbel );
2888
- err_init_hca:
2889
-	arbel_free_icm ( arbel );
2890
- err_alloc_icm:
3056
+	arbel_stop ( arbel );
3057
+ err_start:
2891
  err_get_limits:
3058
  err_get_limits:
2892
 	arbel_stop_firmware ( arbel );
3059
 	arbel_stop_firmware ( arbel );
2893
  err_start_firmware:
3060
  err_start_firmware:
2894
-	free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
2895
- err_mailbox_out:
2896
-	free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
2897
- err_mailbox_in:
2898
 	i = ARBEL_NUM_PORTS;
3061
 	i = ARBEL_NUM_PORTS;
2899
  err_alloc_ibdev:
3062
  err_alloc_ibdev:
2900
 	for ( i-- ; i >= 0 ; i-- )
3063
 	for ( i-- ; i >= 0 ; i-- )
2901
 		ibdev_put ( arbel->ibdev[i] );
3064
 		ibdev_put ( arbel->ibdev[i] );
2902
-	free ( arbel );
2903
- err_alloc_arbel:
3065
+	arbel_free ( arbel );
3066
+ err_alloc:
2904
 	return rc;
3067
 	return rc;
2905
 }
3068
 }
2906
 
3069
 
2915
 
3078
 
2916
 	for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
3079
 	for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
2917
 		unregister_ibdev ( arbel->ibdev[i] );
3080
 		unregister_ibdev ( arbel->ibdev[i] );
2918
-	arbel_destroy_eq ( arbel );
2919
-	arbel_cmd_close_hca ( arbel );
2920
-	arbel_free_icm ( arbel );
2921
-	arbel_stop_firmware ( arbel );
2922
-	free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
2923
-	free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
2924
 	for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
3081
 	for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
2925
 		ibdev_put ( arbel->ibdev[i] );
3082
 		ibdev_put ( arbel->ibdev[i] );
2926
-	free ( arbel );
3083
+	arbel_free ( arbel );
2927
 }
3084
 }
2928
 
3085
 
2929
 static struct pci_device_id arbel_nics[] = {
3086
 static struct pci_device_id arbel_nics[] = {

+ 24
- 2
src/drivers/infiniband/arbel.h View File

31
 #define ARBEL_PCI_UAR_IDX		1
31
 #define ARBEL_PCI_UAR_IDX		1
32
 #define ARBEL_PCI_UAR_SIZE		0x1000
32
 #define ARBEL_PCI_UAR_SIZE		0x1000
33
 
33
 
34
+/* Device reset */
35
+#define ARBEL_RESET_OFFSET		0x0f0010
36
+#define ARBEL_RESET_MAGIC		0x01000000UL
37
+#define ARBEL_RESET_WAIT_TIME_MS	1000
38
+
34
 /* UAR context table (UCE) resource types */
39
 /* UAR context table (UCE) resource types */
35
 #define ARBEL_UAR_RES_NONE		0x00
40
 #define ARBEL_UAR_RES_NONE		0x00
36
 #define ARBEL_UAR_RES_CQ_CI		0x01
41
 #define ARBEL_UAR_RES_CQ_CI		0x01
458
 
463
 
459
 /** An Arbel device */
464
 /** An Arbel device */
460
 struct arbel {
465
 struct arbel {
466
+	/** PCI device */
467
+	struct pci_device *pci;
461
 	/** PCI configuration registers */
468
 	/** PCI configuration registers */
462
 	void *config;
469
 	void *config;
463
 	/** PCI user Access Region */
470
 	/** PCI user Access Region */
470
 	/** Command output mailbox */
477
 	/** Command output mailbox */
471
 	void *mailbox_out;
478
 	void *mailbox_out;
472
 
479
 
473
-	/** Firmware area in external memory */
480
+	/** Device open request counter */
481
+	unsigned int open_count;
482
+
483
+	/** Firmware size */
484
+	size_t firmware_len;
485
+	/** Firmware area in external memory
486
+	 *
487
+	 * This is allocated when first needed, and freed only on
488
+	 * final teardown, in order to avoid memory map changes at
489
+	 * runtime.
490
+	 */
474
 	userptr_t firmware_area;
491
 	userptr_t firmware_area;
475
 	/** ICM size */
492
 	/** ICM size */
476
 	size_t icm_len;
493
 	size_t icm_len;
477
 	/** ICM AUX size */
494
 	/** ICM AUX size */
478
 	size_t icm_aux_len;
495
 	size_t icm_aux_len;
479
-	/** ICM area */
496
+	/** ICM area
497
+	 *
498
+	 * This is allocated when first needed, and freed only on
499
+	 * final teardown, in order to avoid memory map changes at
500
+	 * runtime.
501
+	 */
480
 	userptr_t icm;
502
 	userptr_t icm;
481
 	/** Offset within ICM of doorbell records */
503
 	/** Offset within ICM of doorbell records */
482
 	size_t db_rec_offset;
504
 	size_t db_rec_offset;

Loading…
Cancel
Save