瀏覽代碼

[Infiniband] Add preliminary multiple port support for Hermon cards

Infiniband devices no longer block waiting for link-up in
register_ibdev().

Hermon driver needs to create an event queue and poll for link-up events.

Infiniband core needs to reread MAD parameters when link state changes.

IPoIB needs to cope with Infiniband link parameters being only partially
available at probe and open time.
tags/v0.9.4
Michael Brown 16 年之前
父節點
當前提交
a176a24ac0
共有 6 個檔案被更改,包括 577 行新增141 行删除
  1. 276
    6
      src/drivers/infiniband/hermon.c
  2. 64
    1
      src/drivers/infiniband/hermon.h
  3. 164
    93
      src/drivers/net/ipoib.c
  4. 20
    0
      src/include/gpxe/infiniband.h
  5. 1
    0
      src/include/gpxe/ipoib.h
  6. 52
    41
      src/net/infiniband.c

+ 276
- 6
src/drivers/infiniband/hermon.c 查看文件

@@ -30,6 +30,7 @@
30 30
 #include <gpxe/umalloc.h>
31 31
 #include <gpxe/iobuf.h>
32 32
 #include <gpxe/netdevice.h>
33
+#include <gpxe/process.h>
33 34
 #include <gpxe/infiniband.h>
34 35
 #include "hermon.h"
35 36
 
@@ -316,20 +317,31 @@ hermon_cmd_write_mtt ( struct hermon *hermon,
316 317
 			    0, write_mtt, 1, NULL );
317 318
 }
318 319
 
320
+static inline int
321
+hermon_cmd_map_eq ( struct hermon *hermon, unsigned long index_map,
322
+		    const struct hermonprm_event_mask *mask ) {
323
+	return hermon_cmd ( hermon,
324
+			    HERMON_HCR_IN_CMD ( HERMON_HCR_MAP_EQ,
325
+						0, sizeof ( *mask ) ),
326
+			    0, mask, index_map, NULL );
327
+}
328
+
319 329
 static inline int
320 330
 hermon_cmd_sw2hw_eq ( struct hermon *hermon, unsigned int index,
321
-		      const struct hermonprm_eqc *eqc ) {
331
+		      const struct hermonprm_eqc *eqctx ) {
322 332
 	return hermon_cmd ( hermon,
323 333
 			    HERMON_HCR_IN_CMD ( HERMON_HCR_SW2HW_EQ,
324
-						1, sizeof ( *eqc ) ),
325
-			    0, eqc, index, NULL );
334
+						1, sizeof ( *eqctx ) ),
335
+			    0, eqctx, index, NULL );
326 336
 }
327 337
 
328 338
 static inline int
329
-hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index ) {
339
+hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index,
340
+		      struct hermonprm_eqc *eqctx ) {
330 341
 	return hermon_cmd ( hermon,
331
-			    HERMON_HCR_VOID_CMD ( HERMON_HCR_HW2SW_EQ ),
332
-			    1, NULL, index, NULL );
342
+			    HERMON_HCR_OUT_CMD ( HERMON_HCR_HW2SW_EQ,
343
+						 1, sizeof ( *eqctx ) ),
344
+			    1, NULL, index, eqctx );
333 345
 }
334 346
 
335 347
 static inline int
@@ -377,6 +389,15 @@ hermon_cmd_rtr2rts_qp ( struct hermon *hermon, unsigned long qpn,
377 389
 			    0, ctx, qpn, NULL );
378 390
 }
379 391
 
392
+static inline int
393
+hermon_cmd_rts2rts_qp ( struct hermon *hermon, unsigned long qpn,
394
+			const struct hermonprm_qp_ee_state_transitions *ctx ) {
395
+	return hermon_cmd ( hermon,
396
+			    HERMON_HCR_IN_CMD ( HERMON_HCR_RTS2RTS_QP,
397
+						1, sizeof ( *ctx ) ),
398
+			    0, ctx, qpn, NULL );
399
+}
400
+
380 401
 static inline int
381 402
 hermon_cmd_2rst_qp ( struct hermon *hermon, unsigned long qpn ) {
382 403
 	return hermon_cmd ( hermon,
@@ -859,6 +880,39 @@ static int hermon_create_qp ( struct ib_device *ibdev,
859 880
 	return rc;
860 881
 }
861 882
 
883
+/**
884
+ * Modify queue pair
885
+ *
886
+ * @v ibdev		Infiniband device
887
+ * @v qp		Queue pair
888
+ * @v mod_list		Modification list
889
+ * @ret rc		Return status code
890
+ */
891
+static int hermon_modify_qp ( struct ib_device *ibdev,
892
+			      struct ib_queue_pair *qp,
893
+			      unsigned long mod_list ) {
894
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
895
+	struct hermonprm_qp_ee_state_transitions qpctx;
896
+	unsigned long optparammask = 0;
897
+	int rc;
898
+
899
+	/* Construct optparammask */
900
+	if ( mod_list & IB_MODIFY_QKEY )
901
+		optparammask |= HERMON_QP_OPT_PARAM_QKEY;
902
+
903
+	/* Issue RTS2RTS_QP */
904
+	memset ( &qpctx, 0, sizeof ( qpctx ) );
905
+	MLX_FILL_1 ( &qpctx, 0, opt_param_mask, optparammask );
906
+	MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
907
+	if ( ( rc = hermon_cmd_rts2rts_qp ( hermon, qp->qpn, &qpctx ) ) != 0 ){
908
+		DBGC ( hermon, "Hermon %p RTS2RTS_QP failed: %s\n",
909
+		       hermon, strerror ( rc ) );
910
+		return rc;
911
+	}
912
+
913
+	return 0;
914
+}
915
+
862 916
 /**
863 917
  * Destroy queue pair
864 918
  *
@@ -1356,6 +1410,7 @@ static struct ib_device_operations hermon_ib_operations = {
1356 1410
 	.create_cq	= hermon_create_cq,
1357 1411
 	.destroy_cq	= hermon_destroy_cq,
1358 1412
 	.create_qp	= hermon_create_qp,
1413
+	.modify_qp	= hermon_modify_qp,
1359 1414
 	.destroy_qp	= hermon_destroy_qp,
1360 1415
 	.post_send	= hermon_post_send,
1361 1416
 	.post_recv	= hermon_post_recv,
@@ -1367,6 +1422,211 @@ static struct ib_device_operations hermon_ib_operations = {
1367 1422
 	.mad		= hermon_mad,
1368 1423
 };
1369 1424
 
1425
+/***************************************************************************
1426
+ *
1427
+ * Event queues
1428
+ *
1429
+ ***************************************************************************
1430
+ */
1431
+
1432
+/**
1433
+ * Create event queue
1434
+ *
1435
+ * @v hermon		Hermon device
1436
+ * @ret rc		Return status code
1437
+ */
1438
+static int hermon_create_eq ( struct hermon *hermon ) {
1439
+	struct hermon_event_queue *hermon_eq = &hermon->eq;
1440
+	struct hermonprm_eqc eqctx;
1441
+	struct hermonprm_event_mask mask;
1442
+	unsigned int i;
1443
+	int rc;
1444
+
1445
+	/* Allocate event queue itself */
1446
+	hermon_eq->eqe_size =
1447
+		( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) );
1448
+	hermon_eq->eqe = malloc_dma ( hermon_eq->eqe_size,
1449
+				      sizeof ( hermon_eq->eqe[0] ) );
1450
+	if ( ! hermon_eq->eqe ) {
1451
+		rc = -ENOMEM;
1452
+		goto err_eqe;
1453
+	}
1454
+	memset ( hermon_eq->eqe, 0, hermon_eq->eqe_size );
1455
+	for ( i = 0 ; i < HERMON_NUM_EQES ; i++ ) {
1456
+		MLX_FILL_1 ( &hermon_eq->eqe[i].generic, 7, owner, 1 );
1457
+	}
1458
+	barrier();
1459
+
1460
+	/* Allocate MTT entries */
1461
+	if ( ( rc = hermon_alloc_mtt ( hermon, hermon_eq->eqe,
1462
+				       hermon_eq->eqe_size,
1463
+				       &hermon_eq->mtt ) ) != 0 )
1464
+		goto err_alloc_mtt;
1465
+
1466
+	/* Hand queue over to hardware */
1467
+	memset ( &eqctx, 0, sizeof ( eqctx ) );
1468
+	MLX_FILL_1 ( &eqctx, 0, st, 0xa /* "Fired" */ );
1469
+	MLX_FILL_1 ( &eqctx, 2,
1470
+		     page_offset, ( hermon_eq->mtt.page_offset >> 5 ) );
1471
+	MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) );
1472
+	MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l,
1473
+		     ( hermon_eq->mtt.mtt_base_addr >> 3 ) );
1474
+	if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, 0, &eqctx ) ) != 0 ) {
1475
+		DBGC ( hermon, "Hermon %p SW2HW_EQ failed: %s\n",
1476
+		       hermon, strerror ( rc ) );
1477
+		goto err_sw2hw_eq;
1478
+	}
1479
+
1480
+	/* Map events to this event queue */
1481
+	memset ( &mask, 0, sizeof ( mask ) );
1482
+	MLX_FILL_1 ( &mask, 1, port_state_change, 1 );
1483
+	if ( ( rc = hermon_cmd_map_eq ( hermon, ( HERMON_MAP_EQ_MAP | 0 ),
1484
+					&mask ) ) != 0 ) {
1485
+		DBGC ( hermon, "Hermon %p MAP_EQ failed: %s\n",
1486
+		       hermon, strerror ( rc )  );
1487
+		goto err_map_eq;
1488
+	}
1489
+
1490
+	return 0;
1491
+
1492
+ err_map_eq:
1493
+	hermon_cmd_hw2sw_eq ( hermon, 0, &eqctx );
1494
+ err_sw2hw_eq:
1495
+	hermon_free_mtt ( hermon, &hermon_eq->mtt );
1496
+ err_alloc_mtt:
1497
+	free_dma ( hermon_eq->eqe, hermon_eq->eqe_size );
1498
+ err_eqe:
1499
+	memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
1500
+	return rc;
1501
+}
1502
+
1503
+/**
1504
+ * Destroy event queue
1505
+ *
1506
+ * @v hermon		Hermon device
1507
+ */
1508
+static void hermon_destroy_eq ( struct hermon *hermon ) {
1509
+	struct hermon_event_queue *hermon_eq = &hermon->eq;
1510
+	struct hermonprm_eqc eqctx;
1511
+	struct hermonprm_event_mask mask;
1512
+	int rc;
1513
+
1514
+	/* Unmap events from event queue */
1515
+	memset ( &mask, 0, sizeof ( mask ) );
1516
+	MLX_FILL_1 ( &mask, 1, port_state_change, 1 );
1517
+	if ( ( rc = hermon_cmd_map_eq ( hermon, ( HERMON_MAP_EQ_UNMAP | 0 ),
1518
+					&mask ) ) != 0 ) {
1519
+		DBGC ( hermon, "Hermon %p FATAL MAP_EQ failed to unmap: %s\n",
1520
+		       hermon, strerror ( rc ) );
1521
+		/* Continue; HCA may die but system should survive */
1522
+	}
1523
+
1524
+	/* Take ownership back from hardware */
1525
+	if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, 0, &eqctx ) ) != 0 ) {
1526
+		DBGC ( hermon, "Hermon %p FATAL HW2SW_EQ failed: %s\n",
1527
+		       hermon, strerror ( rc ) );
1528
+		/* Leak memory and return; at least we avoid corruption */
1529
+		return;
1530
+	}
1531
+
1532
+	/* Free MTT entries */
1533
+	hermon_free_mtt ( hermon, &hermon_eq->mtt );
1534
+
1535
+	/* Free memory */
1536
+	free_dma ( hermon_eq->eqe, hermon_eq->eqe_size );
1537
+	memset ( hermon_eq, 0, sizeof ( *hermon_eq ) );
1538
+}
1539
+
1540
+/**
1541
+ * Handle port state event
1542
+ *
1543
+ * @v hermon		Hermon device
1544
+ * @v eqe		Port state change event queue entry
1545
+ */
1546
+static void hermon_event_port_state_change ( struct hermon *hermon,
1547
+					     union hermonprm_event_entry *eqe){
1548
+	unsigned int port;
1549
+	int link_up;
1550
+
1551
+	/* Get port and link status */
1552
+	port = ( MLX_GET ( &eqe->port_state_change, data.p ) - 1 );
1553
+	link_up = ( MLX_GET ( &eqe->generic, event_sub_type ) & 0x04 );
1554
+	DBGC ( hermon, "Hermon %p port %d link %s\n", hermon, ( port + 1 ),
1555
+	       ( link_up ? "up" : "down" ) );
1556
+
1557
+	/* Sanity check */
1558
+	if ( port >= HERMON_NUM_PORTS ) {
1559
+		DBGC ( hermon, "Hermon %p port %d does not exist!\n",
1560
+		       hermon, ( port + 1 ) );
1561
+		return;
1562
+	}
1563
+
1564
+	/* Notify Infiniband core of link state change */
1565
+	ib_link_state_changed ( hermon->ibdev[port] );
1566
+}
1567
+
1568
+/**
1569
+ * Poll event queue
1570
+ *
1571
+ * @v hermon		Hermon device
1572
+ */
1573
+static void hermon_poll_eq ( struct hermon *hermon ) {
1574
+	struct hermon_event_queue *hermon_eq = &hermon->eq;
1575
+	union hermonprm_event_entry *eqe;
1576
+	union hermonprm_doorbell_register db_reg;
1577
+	unsigned int eqe_idx_mask;
1578
+	unsigned int event_type;
1579
+
1580
+	while ( 1 ) {
1581
+		eqe_idx_mask = ( HERMON_NUM_EQES - 1 );
1582
+		eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask];
1583
+		if ( MLX_GET ( &eqe->generic, owner ) ^
1584
+		     ( ( hermon_eq->next_idx & HERMON_NUM_EQES ) ? 1 : 0 ) ) {
1585
+			/* Entry still owned by hardware; end of poll */
1586
+			break;
1587
+		}
1588
+		DBGCP ( hermon, "Hermon %p event:\n", hermon );
1589
+		DBGCP_HD ( hermon, eqe, sizeof ( *eqe ) );
1590
+
1591
+		/* Handle event */
1592
+		event_type = MLX_GET ( &eqe->generic, event_type );
1593
+		switch ( event_type ) {
1594
+		case HERMON_EV_PORT_STATE_CHANGE:
1595
+			hermon_event_port_state_change ( hermon, eqe );
1596
+			break;
1597
+		default:
1598
+			DBGC ( hermon, "Hermon %p unrecognised event type "
1599
+			       "%#x:\n", hermon, event_type );
1600
+			DBGC_HD ( hermon, eqe, sizeof ( *eqe ) );
1601
+			break;
1602
+		}
1603
+
1604
+		/* Update event queue's index */
1605
+		hermon_eq->next_idx++;
1606
+
1607
+		/* Ring doorbell */
1608
+		memset ( &db_reg, 0, sizeof ( db_reg ) );
1609
+		MLX_FILL_1 ( &db_reg.event, 0, ci, hermon_eq->next_idx );
1610
+		DBGCP ( hermon, "Ringing doorbell %08lx with %08lx\n",
1611
+			virt_to_phys ( hermon->uar + HERMON_DB_EQ0_OFFSET ),
1612
+			db_reg.dword[0] );
1613
+		writel ( db_reg.dword[0],
1614
+			 ( hermon->uar + HERMON_DB_EQ0_OFFSET ) );
1615
+	}
1616
+}
1617
+
1618
+/**
1619
+ * Event queue poll processor
1620
+ *
1621
+ * @v process		Hermon event queue process
1622
+ */
1623
+static void hermon_step ( struct process *process ) {
1624
+	struct hermon *hermon =
1625
+		container_of ( process, struct hermon, event_process );
1626
+
1627
+	hermon_poll_eq ( hermon );
1628
+}
1629
+
1370 1630
 /***************************************************************************
1371 1631
  *
1372 1632
  * Firmware control
@@ -1879,6 +2139,7 @@ static int hermon_probe ( struct pci_device *pci,
1879 2139
 		goto err_alloc_hermon;
1880 2140
 	}
1881 2141
 	pci_set_drvdata ( pci, hermon );
2142
+	process_init ( &hermon->event_process, hermon_step, NULL );
1882 2143
 
1883 2144
 	/* Allocate Infiniband devices */
1884 2145
 	for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
@@ -1945,6 +2206,10 @@ static int hermon_probe ( struct pci_device *pci,
1945 2206
 	if ( ( rc = hermon_setup_mpt ( hermon ) ) != 0 )
1946 2207
 		goto err_setup_mpt;
1947 2208
 
2209
+	/* Set up event queue */
2210
+	if ( ( rc = hermon_create_eq ( hermon ) ) != 0 )
2211
+		goto err_create_eq;
2212
+
1948 2213
 	/* Register Infiniband devices */
1949 2214
 	for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
1950 2215
 		if ( ( rc = register_ibdev ( hermon->ibdev[i] ) ) != 0 ) {
@@ -1960,6 +2225,8 @@ static int hermon_probe ( struct pci_device *pci,
1960 2225
  err_register_ibdev:
1961 2226
 	for ( ; i >= 0 ; i-- )
1962 2227
 		unregister_ibdev ( hermon->ibdev[i] );
2228
+	hermon_destroy_eq ( hermon );
2229
+ err_create_eq:
1963 2230
  err_setup_mpt:
1964 2231
 	hermon_cmd_close_hca ( hermon );
1965 2232
  err_init_hca:
@@ -1976,6 +2243,7 @@ static int hermon_probe ( struct pci_device *pci,
1976 2243
  err_alloc_ibdev:
1977 2244
 	for ( ; i >= 0 ; i-- )
1978 2245
 		free_ibdev ( hermon->ibdev[i] );
2246
+	process_del ( &hermon->event_process );
1979 2247
 	free ( hermon );
1980 2248
  err_alloc_hermon:
1981 2249
 	return rc;
@@ -1992,6 +2260,7 @@ static void hermon_remove ( struct pci_device *pci ) {
1992 2260
 
1993 2261
 	for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
1994 2262
 		unregister_ibdev ( hermon->ibdev[i] );
2263
+	hermon_destroy_eq ( hermon );
1995 2264
 	hermon_cmd_close_hca ( hermon );
1996 2265
 	hermon_free_icm ( hermon );
1997 2266
 	hermon_stop_firmware ( hermon );
@@ -2000,6 +2269,7 @@ static void hermon_remove ( struct pci_device *pci ) {
2000 2269
 	free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
2001 2270
 	for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
2002 2271
 		free_ibdev ( hermon->ibdev[i] );
2272
+	process_del ( &hermon->event_process );
2003 2273
 	free ( hermon );
2004 2274
 }
2005 2275
 

+ 64
- 1
src/drivers/infiniband/hermon.h 查看文件

@@ -9,6 +9,7 @@
9 9
 
10 10
 #include <stdint.h>
11 11
 #include <gpxe/uaccess.h>
12
+#include <gpxe/process.h>
12 13
 #include "mlx_bitops.h"
13 14
 #include "MT25408_PRM.h"
14 15
 
@@ -18,7 +19,7 @@
18 19
  */
19 20
 
20 21
 /* Ports in existence */
21
-#define HERMON_NUM_PORTS		1
22
+#define HERMON_NUM_PORTS		2
22 23
 #define HERMON_PORT_BASE		1
23 24
 
24 25
 /* PCI BARs */
@@ -48,6 +49,7 @@
48 49
 #define HERMON_HCR_RST2INIT_QP		0x0019
49 50
 #define HERMON_HCR_INIT2RTR_QP		0x001a
50 51
 #define HERMON_HCR_RTR2RTS_QP		0x001b
52
+#define HERMON_HCR_RTS2RTS_QP		0x001c
51 53
 #define HERMON_HCR_2RST_QP		0x0021
52 54
 #define HERMON_HCR_MAD_IFC		0x0024
53 55
 #define HERMON_HCR_READ_MCG		0x0025
@@ -75,6 +77,14 @@
75 77
 #define HERMON_PAGE_SIZE		4096
76 78
 
77 79
 #define HERMON_DB_POST_SND_OFFSET	0x14
80
+#define HERMON_DB_EQ0_OFFSET		0x800
81
+
82
+#define HERMON_QP_OPT_PARAM_QKEY	0x00000020UL
83
+
84
+#define HERMON_MAP_EQ_MAP		( 0UL << 31 )
85
+#define HERMON_MAP_EQ_UNMAP		( 1UL << 31 )
86
+
87
+#define HERMON_EV_PORT_STATE_CHANGE	0x09
78 88
 
79 89
 /*
80 90
  * Datatypes that seem to be missing from the autogenerated documentation
@@ -108,12 +118,32 @@ struct hermonprm_send_db_register_st {
108 118
 	pseudo_bit_t qn[0x00018];
109 119
 } __attribute__ (( packed ));
110 120
 
121
+struct hermonprm_event_db_register_st {
122
+	pseudo_bit_t ci[0x00018];
123
+	pseudo_bit_t reserver[0x00007];
124
+	pseudo_bit_t a[0x00001];
125
+} __attribute__ (( packed ));
126
+
111 127
 struct hermonprm_scalar_parameter_st {
112 128
 	pseudo_bit_t value_hi[0x00020];
113 129
 /* -------------- */
114 130
 	pseudo_bit_t value[0x00020];
115 131
 } __attribute__ (( packed ));
116 132
 
133
+struct hermonprm_event_mask_st {
134
+	pseudo_bit_t reserved0[0x00020];
135
+/* -------------- */
136
+	pseudo_bit_t completion[0x00001];
137
+	pseudo_bit_t reserved1[0x0008];
138
+	pseudo_bit_t port_state_change[0x00001];
139
+	pseudo_bit_t reserved2[0x00016];
140
+} __attribute__ (( packed ));
141
+
142
+struct hermonprm_port_state_change_event_st {
143
+	pseudo_bit_t reserved[0x00020];
144
+	struct hermonprm_port_state_change_st data;
145
+} __attribute__ (( packed ));
146
+
117 147
 /*
118 148
  * Wrapper structures for hardware datatypes
119 149
  *
@@ -124,6 +154,9 @@ struct MLX_DECLARE_STRUCT ( hermonprm_completion_queue_entry );
124 154
 struct MLX_DECLARE_STRUCT ( hermonprm_completion_with_error );
125 155
 struct MLX_DECLARE_STRUCT ( hermonprm_cq_db_record );
126 156
 struct MLX_DECLARE_STRUCT ( hermonprm_eqc );
157
+struct MLX_DECLARE_STRUCT ( hermonprm_event_db_register );
158
+struct MLX_DECLARE_STRUCT ( hermonprm_event_mask );
159
+struct MLX_DECLARE_STRUCT ( hermonprm_event_queue_entry );
127 160
 struct MLX_DECLARE_STRUCT ( hermonprm_hca_command_register );
128 161
 struct MLX_DECLARE_STRUCT ( hermonprm_init_hca );
129 162
 struct MLX_DECLARE_STRUCT ( hermonprm_init_port );
@@ -132,6 +165,7 @@ struct MLX_DECLARE_STRUCT ( hermonprm_mcg_entry );
132 165
 struct MLX_DECLARE_STRUCT ( hermonprm_mgm_hash );
133 166
 struct MLX_DECLARE_STRUCT ( hermonprm_mpt );
134 167
 struct MLX_DECLARE_STRUCT ( hermonprm_mtt );
168
+struct MLX_DECLARE_STRUCT ( hermonprm_port_state_change_event );
135 169
 struct MLX_DECLARE_STRUCT ( hermonprm_qp_db_record );
136 170
 struct MLX_DECLARE_STRUCT ( hermonprm_qp_ee_state_transitions );
137 171
 struct MLX_DECLARE_STRUCT ( hermonprm_query_dev_cap );
@@ -175,8 +209,14 @@ union hermonprm_completion_entry {
175 209
 	struct hermonprm_completion_with_error error;
176 210
 } __attribute__ (( packed ));
177 211
 
212
+union hermonprm_event_entry {
213
+	struct hermonprm_event_queue_entry generic;
214
+	struct hermonprm_port_state_change_event port_state_change;
215
+} __attribute__ (( packed ));
216
+
178 217
 union hermonprm_doorbell_register {
179 218
 	struct hermonprm_send_db_register send;
219
+	struct hermonprm_event_db_register event;
180 220
 	uint32_t dword[1];
181 221
 } __attribute__ (( packed ));
182 222
 
@@ -362,6 +402,24 @@ struct hermon_completion_queue {
362 402
  */
363 403
 #define HERMON_MAX_EQS		4
364 404
 
405
+/** A Hermon event queue */
406
+struct hermon_event_queue {
407
+	/** Event queue entries */
408
+	union hermonprm_event_entry *eqe;
409
+	/** Size of event queue */
410
+	size_t eqe_size;
411
+	/** MTT descriptor */
412
+	struct hermon_mtt mtt;
413
+	/** Next event queue entry index */
414
+	unsigned long next_idx;
415
+};
416
+
417
+/** Number of event queue entries
418
+ *
419
+ * This is a policy decision.
420
+ */
421
+#define HERMON_NUM_EQES		4
422
+
365 423
 /** A Hermon resource bitmask */
366 424
 typedef uint32_t hermon_bitmask_t;
367 425
 
@@ -397,6 +455,11 @@ struct hermon {
397 455
 	 */
398 456
 	unsigned long reserved_lkey;
399 457
 
458
+	/** Event queue */
459
+	struct hermon_event_queue eq;
460
+	/** Event queue process */
461
+	struct process event_process;
462
+
400 463
 	/** Completion queue in-use bitmask */
401 464
 	hermon_bitmask_t cq_inuse[ HERMON_BITMASK_SIZE ( HERMON_MAX_CQS ) ];
402 465
 	/** Queue pair in-use bitmask */

+ 164
- 93
src/drivers/net/ipoib.c 查看文件

@@ -80,10 +80,14 @@ struct ipoib_device {
80 80
 	struct ib_gid broadcast_gid;
81 81
 	/** Broadcast LID */
82 82
 	unsigned int broadcast_lid;
83
-	/** Joined to broadcast group */
84
-	int broadcast_joined;
85 83
 	/** Data queue key */
86 84
 	unsigned long data_qkey;
85
+	/** Attached to multicast group
86
+	 *
87
+	 * This flag indicates whether or not we have attached our
88
+	 * data queue pair to the broadcast multicast GID.
89
+	 */
90
+	int broadcast_attached;
87 91
 };
88 92
 
89 93
 /**
@@ -272,6 +276,10 @@ static int ipoib_create_qset ( struct ipoib_device *ipoib,
272 276
 	struct ib_device *ibdev = ipoib->ibdev;
273 277
 	int rc;
274 278
 
279
+	/* Sanity check */
280
+	assert ( qset->cq == NULL );
281
+	assert ( qset->qp == NULL );
282
+
275 283
 	/* Store queue parameters */
276 284
 	qset->recv_max_fill = num_recv_wqes;
277 285
 
@@ -617,14 +625,24 @@ static void ipoib_recv_path_record ( struct ipoib_device *ipoib __unused,
617 625
  */
618 626
 static void ipoib_recv_mc_member_record ( struct ipoib_device *ipoib,
619 627
 			  struct ib_mad_mc_member_record *mc_member_record ) {
628
+	int joined;
629
+	int rc;
630
+
620 631
 	/* Record parameters */
621
-	ipoib->broadcast_joined =
622
-		( mc_member_record->scope__join_state & 0x0f );
632
+	joined = ( mc_member_record->scope__join_state & 0x0f );
623 633
 	ipoib->data_qkey = ntohl ( mc_member_record->qkey );
624 634
 	ipoib->broadcast_lid = ntohs ( mc_member_record->mlid );
625 635
 	DBGC ( ipoib, "IPoIB %p %s broadcast group: qkey %lx mlid %x\n",
626
-	       ipoib, ( ipoib->broadcast_joined ? "joined" : "left" ),
627
-	       ipoib->data_qkey, ipoib->broadcast_lid );
636
+	       ipoib, ( joined ? "joined" : "left" ), ipoib->data_qkey,
637
+	       ipoib->broadcast_lid );
638
+
639
+	/* Update data queue pair qkey */
640
+	if ( ( rc = ib_modify_qp ( ipoib->ibdev, ipoib->data.qp,
641
+				   IB_MODIFY_QKEY, ipoib->data_qkey ) ) != 0 ){
642
+		DBGC ( ipoib, "IPoIB %p could not update data qkey: %s\n",
643
+		       ipoib, strerror ( rc ) );
644
+		return;
645
+	}
628 646
 }
629 647
 
630 648
 /**
@@ -741,6 +759,56 @@ static void ipoib_irq ( struct net_device *netdev __unused,
741 759
 	/* No implementation */
742 760
 }
743 761
 
762
+/**
763
+ * Join IPv4 broadcast multicast group
764
+ *
765
+ * @v ipoib		IPoIB device
766
+ * @ret rc		Return status code
767
+ */
768
+static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
769
+	int rc;
770
+
771
+	/* Sanity check */
772
+	if ( ! ipoib->data.qp )
773
+		return 0;
774
+
775
+	/* Attach data queue to broadcast multicast GID */
776
+	assert ( ipoib->broadcast_attached == 0 );
777
+	if ( ( rc = ib_mcast_attach ( ipoib->ibdev, ipoib->data.qp,
778
+				      &ipoib->broadcast_gid ) ) != 0 ){
779
+		DBGC ( ipoib, "IPoIB %p could not attach to broadcast GID: "
780
+		       "%s\n", ipoib, strerror ( rc ) );
781
+		return rc;
782
+	}
783
+	ipoib->broadcast_attached = 1;
784
+
785
+	/* Initiate broadcast group join */
786
+	if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast_gid,
787
+					     1 ) ) != 0 ) {
788
+		DBGC ( ipoib, "IPoIB %p could not send broadcast join: %s\n",
789
+		       ipoib, strerror ( rc ) );
790
+		return rc;
791
+	}
792
+
793
+	return 0;
794
+}
795
+
796
+/**
797
+ * Leave IPv4 broadcast multicast group
798
+ *
799
+ * @v ipoib		IPoIB device
800
+ */
801
+static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
802
+
803
+	/* Detach data queue from broadcast multicast GID */
804
+	if ( ipoib->broadcast_attached ) {
805
+		assert ( ipoib->data.qp != NULL );
806
+		ib_mcast_detach ( ipoib->ibdev, ipoib->data.qp,
807
+				  &ipoib->broadcast_gid );
808
+		ipoib->broadcast_attached = 0;
809
+	}
810
+}
811
+
744 812
 /**
745 813
  * Open IPoIB network device
746 814
  *
@@ -749,22 +817,53 @@ static void ipoib_irq ( struct net_device *netdev __unused,
749 817
  */
750 818
 static int ipoib_open ( struct net_device *netdev ) {
751 819
 	struct ipoib_device *ipoib = netdev->priv;
752
-	struct ib_device *ibdev = ipoib->ibdev;
820
+	struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
753 821
 	int rc;
754 822
 
755
-	/* Attach to broadcast multicast GID */
756
-	if ( ( rc = ib_mcast_attach ( ibdev, ipoib->data.qp,
757
-				      &ipoib->broadcast_gid ) ) != 0 ) {
758
-		DBG ( "Could not attach to broadcast GID: %s\n",
759
-		      strerror ( rc ) );
760
-		return rc;
823
+	/* Allocate metadata queue set */
824
+	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
825
+					IPOIB_META_NUM_CQES,
826
+					IPOIB_META_NUM_SEND_WQES,
827
+					IPOIB_META_NUM_RECV_WQES,
828
+					IB_GLOBAL_QKEY ) ) != 0 ) {
829
+		DBGC ( ipoib, "IPoIB %p could not allocate metadata QP: %s\n",
830
+		       ipoib, strerror ( rc ) );
831
+		goto err_create_meta_qset;
761 832
 	}
762 833
 
834
+	/* Allocate data queue set */
835
+	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
836
+					IPOIB_DATA_NUM_CQES,
837
+					IPOIB_DATA_NUM_SEND_WQES,
838
+					IPOIB_DATA_NUM_RECV_WQES,
839
+					IB_GLOBAL_QKEY ) ) != 0 ) {
840
+		DBGC ( ipoib, "IPoIB %p could not allocate data QP: %s\n",
841
+		       ipoib, strerror ( rc ) );
842
+		goto err_create_data_qset;
843
+	}
844
+
845
+	/* Update MAC address with data QPN */
846
+	mac->qpn = htonl ( ipoib->data.qp->qpn );
847
+
763 848
 	/* Fill receive rings */
764 849
 	ipoib_refill_recv ( ipoib, &ipoib->meta );
765 850
 	ipoib_refill_recv ( ipoib, &ipoib->data );
766 851
 
852
+	/* Join broadcast group */
853
+	if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) {
854
+		DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
855
+		       ipoib, strerror ( rc ) );
856
+		goto err_join_broadcast;
857
+	}
858
+
767 859
 	return 0;
860
+
861
+ err_join_broadcast:
862
+	ipoib_destroy_qset ( ipoib, &ipoib->data );
863
+ err_create_data_qset:
864
+	ipoib_destroy_qset ( ipoib, &ipoib->meta );
865
+ err_create_meta_qset:
866
+	return rc;
768 867
 }
769 868
 
770 869
 /**
@@ -774,12 +873,17 @@ static int ipoib_open ( struct net_device *netdev ) {
774 873
  */
775 874
 static void ipoib_close ( struct net_device *netdev ) {
776 875
 	struct ipoib_device *ipoib = netdev->priv;
777
-	struct ib_device *ibdev = ipoib->ibdev;
876
+	struct ipoib_mac *mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
778 877
 
779
-	/* Detach from broadcast multicast GID */
780
-	ib_mcast_detach ( ibdev, ipoib->data.qp, &ipoib->broadcast_gid );
878
+	/* Leave broadcast group */
879
+	ipoib_leave_broadcast_group ( ipoib );
781 880
 
782
-	/* FIXME: should probably flush the receive ring */
881
+	/* Remove data QPN from MAC address */
882
+	mac->qpn = 0;
883
+
884
+	/* Tear down the queues */
885
+	ipoib_destroy_qset ( ipoib, &ipoib->data );
886
+	ipoib_destroy_qset ( ipoib, &ipoib->meta );
783 887
 }
784 888
 
785 889
 /** IPoIB network device operations */
@@ -792,44 +896,53 @@ static struct net_device_operations ipoib_operations = {
792 896
 };
793 897
 
794 898
 /**
795
- * Join IPoIB broadcast group
899
+ * Update IPoIB dynamic Infiniband parameters
796 900
  *
797 901
  * @v ipoib		IPoIB device
798
- * @ret rc		Return status code
902
+ *
903
+ * The Infiniband port GID and partition key will change at runtime,
904
+ * when the link is established (or lost).  The MAC address is based
905
+ * on the port GID, and the broadcast GID is based on the partition
906
+ * key.  This function recalculates these IPoIB device parameters.
799 907
  */
800
-static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
908
+static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) {
801 909
 	struct ib_device *ibdev = ipoib->ibdev;
802
-	unsigned int delay_ms;
803
-	int rc;
910
+	struct ipoib_mac *mac;
804 911
 
805
-	/* Make sure we have some receive descriptors */
806
-	ipoib_refill_recv ( ipoib, &ipoib->meta );
912
+	/* Calculate GID portion of MAC address based on port GID */
913
+	mac = ( ( struct ipoib_mac * ) ipoib->netdev->ll_addr );
914
+	memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) );
807 915
 
808
-	/* Send join request */
809
-	if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast_gid,
810
-					     1 ) ) != 0 ) {
811
-		DBGC ( ipoib, "IPoIB %p could not send broadcast join: %s\n",
812
-		       ipoib, strerror ( rc ) );
813
-		return rc;
814
-	}
916
+	/* Calculate broadcast GID based on partition key */
917
+	memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid,
918
+		 sizeof ( ipoib->broadcast_gid ) );
919
+	ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
920
+}
921
+
922
+/**
923
+ * Handle link status change
924
+ *
925
+ * @v ibdev		Infiniband device
926
+ */
927
+void ipoib_link_state_changed ( struct ib_device *ibdev ) {
928
+	struct net_device *netdev = ib_get_ownerdata ( ibdev );
929
+	struct ipoib_device *ipoib = netdev->priv;
930
+	int rc;
815 931
 
816
-	/* Wait for join to complete.  Ideally we wouldn't delay for
817
-	 * this long, but we need the queue key before we can set up
818
-	 * the data queue pair, which we need before we can know the
819
-	 * MAC address.
932
+	/* Leave existing broadcast group */
933
+	ipoib_leave_broadcast_group ( ipoib );
934
+
935
+	/* Update MAC address and broadcast GID based on new port GID
936
+	 * and partition key.
820 937
 	 */
821
-	for ( delay_ms = IPOIB_JOIN_MAX_DELAY_MS ; delay_ms ; delay_ms-- ) {
822
-		mdelay ( 1 );
823
-		ib_poll_cq ( ibdev, ipoib->meta.cq, ipoib_meta_complete_send,
824
-			     ipoib_meta_complete_recv );
825
-		ipoib_refill_recv ( ipoib, &ipoib->meta );
826
-		if ( ipoib->broadcast_joined )
827
-			return 0;
828
-	}
829
-	DBGC ( ipoib, "IPoIB %p timed out waiting for broadcast join\n",
830
-	       ipoib );
938
+	ipoib_set_ib_params ( ipoib );
831 939
 
832
-	return -ETIMEDOUT;
940
+	/* Join new broadcast group */
941
+	if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) {
942
+		DBGC ( ipoib, "IPoIB %p could not rejoin broadcast group: "
943
+		       "%s\n", ipoib, strerror ( rc ) );
944
+		return;
945
+	}
833 946
 }
834 947
 
835 948
 /**
@@ -841,7 +954,6 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
841 954
 int ipoib_probe ( struct ib_device *ibdev ) {
842 955
 	struct net_device *netdev;
843 956
 	struct ipoib_device *ipoib;
844
-	struct ipoib_mac *mac;
845 957
 	int rc;
846 958
 
847 959
 	/* Allocate network device */
@@ -856,44 +968,11 @@ int ipoib_probe ( struct ib_device *ibdev ) {
856 968
 	ipoib->netdev = netdev;
857 969
 	ipoib->ibdev = ibdev;
858 970
 
859
-	/* Calculate broadcast GID */
860
-	memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid,
861
-		 sizeof ( ipoib->broadcast_gid ) );
862
-	ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
863
-
864
-	/* Allocate metadata queue set */
865
-	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
866
-					IPOIB_META_NUM_CQES,
867
-					IPOIB_META_NUM_SEND_WQES,
868
-					IPOIB_META_NUM_RECV_WQES,
869
-					IB_GLOBAL_QKEY ) ) != 0 ) {
870
-		DBGC ( ipoib, "IPoIB %p could not allocate metadata QP: %s\n",
871
-		       ipoib, strerror ( rc ) );
872
-		goto err_create_meta_qset;
873
-	}
874
-
875
-	/* Join broadcast group */
876
-	if ( ( rc = ipoib_join_broadcast_group ( ipoib ) ) != 0 ) {
877
-		DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
878
-		       ipoib, strerror ( rc ) );
879
-		goto err_join_broadcast_group;
880
-	}
881
-
882
-	/* Allocate data queue set */
883
-	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
884
-					IPOIB_DATA_NUM_CQES,
885
-					IPOIB_DATA_NUM_SEND_WQES,
886
-					IPOIB_DATA_NUM_RECV_WQES,
887
-					ipoib->data_qkey ) ) != 0 ) {
888
-		DBGC ( ipoib, "IPoIB %p could not allocate data QP: %s\n",
889
-		       ipoib, strerror ( rc ) );
890
-		goto err_create_data_qset;
891
-	}
892
-
893
-	/* Construct MAC address */
894
-	mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
895
-	mac->qpn = htonl ( ipoib->data.qp->qpn );
896
-	memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) );
971
+	/* Calculate as much of the broadcast GID and the MAC address
972
+	 * as we can.  We won't know either of these in full until we
973
+	 * have link-up.
974
+	 */
975
+	ipoib_set_ib_params ( ipoib );
897 976
 
898 977
 	/* Register network device */
899 978
 	if ( ( rc = register_netdev ( netdev ) ) != 0 )
@@ -902,11 +981,6 @@ int ipoib_probe ( struct ib_device *ibdev ) {
902 981
 	return 0;
903 982
 
904 983
  err_register_netdev:
905
-	ipoib_destroy_qset ( ipoib, &ipoib->data );
906
- err_join_broadcast_group:
907
- err_create_data_qset:
908
-	ipoib_destroy_qset ( ipoib, &ipoib->meta );
909
- err_create_meta_qset:
910 984
 	netdev_nullify ( netdev );
911 985
 	netdev_put ( netdev );
912 986
 	return rc;
@@ -919,11 +993,8 @@ int ipoib_probe ( struct ib_device *ibdev ) {
919 993
  */
920 994
 void ipoib_remove ( struct ib_device *ibdev ) {
921 995
 	struct net_device *netdev = ib_get_ownerdata ( ibdev );
922
-	struct ipoib_device *ipoib = netdev->priv;
923 996
 
924 997
 	unregister_netdev ( netdev );
925
-	ipoib_destroy_qset ( ipoib, &ipoib->data );
926
-	ipoib_destroy_qset ( ipoib, &ipoib->meta );
927 998
 	netdev_nullify ( netdev );
928 999
 	netdev_put ( netdev );
929 1000
 }

+ 20
- 0
src/include/gpxe/infiniband.h 查看文件

@@ -95,6 +95,11 @@ struct ib_queue_pair {
95 95
 	void *owner_priv;
96 96
 };
97 97
 
98
+/** Infiniband queue pair modification flags */
99
+enum ib_queue_pair_mods {
100
+	IB_MODIFY_QKEY = 0x0001,
101
+};
102
+
98 103
 /** An Infiniband Completion Queue */
99 104
 struct ib_completion_queue {
100 105
 	/** Completion queue number */
@@ -187,6 +192,16 @@ struct ib_device_operations {
187 192
 	 */
188 193
 	int ( * create_qp ) ( struct ib_device *ibdev,
189 194
 			      struct ib_queue_pair *qp );
195
+	/** Modify queue pair
196
+	 *
197
+	 * @v ibdev		Infiniband device
198
+	 * @v qp		Queue pair
199
+	 * @v mod_list		Modification list
200
+	 * @ret rc		Return status code
201
+	 */
202
+	int ( * modify_qp ) ( struct ib_device *ibdev,
203
+			      struct ib_queue_pair *qp,
204
+			      unsigned long mod_list );
190 205
 	/** Destroy queue pair
191 206
 	 *
192 207
 	 * @v ibdev		Infiniband device
@@ -291,6 +306,8 @@ struct ib_device {
291 306
 	struct ib_device_operations *op;
292 307
 	/** Port number */
293 308
 	unsigned int port;
309
+	/** Link state */
310
+	int link_up;
294 311
 	/** Port GID */
295 312
 	struct ib_gid port_gid;
296 313
 	/** Subnet manager LID */
@@ -311,6 +328,8 @@ extern struct ib_queue_pair *
311 328
 ib_create_qp ( struct ib_device *ibdev, unsigned int num_send_wqes,
312 329
 	       struct ib_completion_queue *send_cq, unsigned int num_recv_wqes,
313 330
 	       struct ib_completion_queue *recv_cq, unsigned long qkey );
331
+extern int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
332
+			  unsigned long mod_list, unsigned long qkey );
314 333
 extern void ib_destroy_qp ( struct ib_device *ibdev,
315 334
 			    struct ib_queue_pair *qp );
316 335
 extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
@@ -319,6 +338,7 @@ extern struct ib_device * alloc_ibdev ( size_t priv_size );
319 338
 extern int register_ibdev ( struct ib_device *ibdev );
320 339
 extern void unregister_ibdev ( struct ib_device *ibdev );
321 340
 extern void free_ibdev ( struct ib_device *ibdev );
341
+extern void ib_link_state_changed ( struct ib_device *ibdev );
322 342
 
323 343
 /**
324 344
  * Post send work queue entry

+ 1
- 0
src/include/gpxe/ipoib.h 查看文件

@@ -72,6 +72,7 @@ static inline struct net_device * alloc_ipoibdev ( size_t priv_size ) {
72 72
 	return netdev;
73 73
 }
74 74
 
75
+extern void ipoib_link_state_changed ( struct ib_device *ibdev );
75 76
 extern int ipoib_probe ( struct ib_device *ibdev );
76 77
 extern void ipoib_remove ( struct ib_device *ibdev );
77 78
 

+ 52
- 41
src/net/infiniband.c 查看文件

@@ -152,15 +152,41 @@ struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
152 152
 	return qp;
153 153
 }
154 154
 
155
+/**
156
+ * Modify queue pair
157
+ *
158
+ * @v ibdev		Infiniband device
159
+ * @v qp		Queue pair
160
+ * @v mod_list		Modification list
161
+ * @v qkey		New queue key, if applicable
162
+ * @ret rc		Return status code
163
+ */
164
+int ib_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp,
165
+		   unsigned long mod_list, unsigned long qkey ) {
166
+	int rc;
167
+
168
+	DBGC ( ibdev, "IBDEV %p modifying QPN %#lx\n", ibdev, qp->qpn );
169
+
170
+	if ( mod_list & IB_MODIFY_QKEY )
171
+		qp->qkey = qkey;
172
+
173
+	if ( ( rc = ibdev->op->modify_qp ( ibdev, qp, mod_list ) ) != 0 ) {
174
+		DBGC ( ibdev, "IBDEV %p could not modify QPN %#lx: %s\n",
175
+		       ibdev, qp->qpn, strerror ( rc ) );
176
+		return rc;
177
+	}
178
+
179
+	return 0;
180
+}
181
+
155 182
 /**
156 183
  * Destroy queue pair
157 184
  *
158 185
  * @v ibdev		Infiniband device
159 186
  * @v qp		Queue pair
160 187
  */
161
-void ib_destroy_qp ( struct ib_device *ibdev,
162
-		     struct ib_queue_pair *qp ) {
163
-	DBGC ( ibdev, "IBDEV %p destroying queue pair %#lx\n",
188
+void ib_destroy_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) {
189
+	DBGC ( ibdev, "IBDEV %p destroying QPN %#lx\n",
164 190
 	       ibdev, qp->qpn );
165 191
 	ibdev->op->destroy_qp ( ibdev, qp );
166 192
 	list_del ( &qp->send.list );
@@ -279,38 +305,6 @@ static int ib_get_pkey_table ( struct ib_device *ibdev,
279 305
 	return 0;
280 306
 }
281 307
 
282
-/**
283
- * Wait for link up
284
- *
285
- * @v ibdev		Infiniband device
286
- * @ret rc		Return status code
287
- *
288
- * This function shouldn't really exist.  Unfortunately, IB links take
289
- * a long time to come up, and we can't get various key parameters
290
- * e.g. our own IPoIB MAC address without information from the subnet
291
- * manager).  We should eventually make link-up an asynchronous event.
292
- */
293
-static int ib_wait_for_link ( struct ib_device *ibdev ) {
294
-	struct ib_mad_port_info port_info;
295
-	unsigned int retries;
296
-	int rc;
297
-
298
-	printf ( "Waiting for Infiniband link-up..." );
299
-	for ( retries = 20 ; retries ; retries-- ) {
300
-		if ( ( rc = ib_get_port_info ( ibdev, &port_info ) ) != 0 )
301
-			continue;
302
-		if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
303
-		     == 4 ) {
304
-			printf ( "ok\n" );
305
-			return 0;
306
-		}
307
-		printf ( "." );
308
-		sleep ( 1 );
309
-	}
310
-	printf ( "failed\n" );
311
-	return -ENODEV;
312
-};
313
-
314 308
 /**
315 309
  * Get MAD parameters
316 310
  *
@@ -326,9 +320,13 @@ static int ib_get_mad_params ( struct ib_device *ibdev ) {
326 320
 	} u;
327 321
 	int rc;
328 322
 
329
-	/* Port info gives us the first half of the port GID and the SM LID */
323
+	/* Port info gives us the link state, the first half of the
324
+	 * port GID and the SM LID.
325
+	 */
330 326
 	if ( ( rc = ib_get_port_info ( ibdev, &u.port_info ) ) != 0 )
331 327
 		return rc;
328
+	ibdev->link_up = ( ( u.port_info.port_state__link_speed_supported
329
+			     & 0xf ) == 4 );
332 330
 	memcpy ( &ibdev->port_gid.u.bytes[0], u.port_info.gid_prefix, 8 );
333 331
 	ibdev->sm_lid = ntohs ( u.port_info.mastersm_lid );
334 332
 
@@ -391,10 +389,6 @@ int register_ibdev ( struct ib_device *ibdev ) {
391 389
 	if ( ( rc = ib_open ( ibdev ) ) != 0 )
392 390
 		goto err_open;
393 391
 
394
-	/* Wait for link */
395
-	if ( ( rc = ib_wait_for_link ( ibdev ) ) != 0 )
396
-		goto err_wait_for_link;
397
-
398 392
 	/* Get MAD parameters */
399 393
 	if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 )
400 394
 		goto err_get_mad_params;
@@ -410,7 +404,6 @@ int register_ibdev ( struct ib_device *ibdev ) {
410 404
 
411 405
  err_ipoib_probe:
412 406
  err_get_mad_params:
413
- err_wait_for_link:
414 407
 	ib_close ( ibdev );
415 408
  err_open:
416 409
 	return rc;
@@ -435,3 +428,21 @@ void free_ibdev ( struct ib_device *ibdev ) {
435 428
 	free ( ibdev );
436 429
 }
437 430
 
431
+/**
432
+ * Handle Infiniband link state change
433
+ *
434
+ * @v ibdev		Infiniband device
435
+ */
436
+void ib_link_state_changed ( struct ib_device *ibdev ) {
437
+	int rc;
438
+
439
+	/* Update MAD parameters */
440
+	if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) {
441
+		DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n",
442
+		       ibdev, strerror ( rc ) );
443
+		return;
444
+	}
445
+
446
+	/* Notify IPoIB of link state change */
447
+	ipoib_link_state_changed ( ibdev );
448
+}

Loading…
取消
儲存