Browse Source

[Infiniband] Move event-queue process from driver to Infiniband core

tags/v0.9.4
Michael Brown 16 years ago
parent
commit
35a5836677

+ 44
- 2
src/drivers/infiniband/arbel.c View File

@@ -835,6 +835,27 @@ static int arbel_create_qp ( struct ib_device *ibdev,
835 835
 	return rc;
836 836
 }
837 837
 
838
+/**
839
+ * Modify queue pair
840
+ *
841
+ * @v ibdev		Infiniband device
842
+ * @v qp		Queue pair
843
+ * @v mod_list		Modification list
844
+ * @ret rc		Return status code
845
+ */
846
+static int arbel_modify_qp ( struct ib_device *ibdev,
847
+			     struct ib_queue_pair *qp,
848
+			     unsigned long mod_list ) {
849
+	struct arbel *arbel = ib_get_drvdata ( ibdev );
850
+
851
+	/* TODO */
852
+	( void ) arbel;
853
+	( void ) qp;
854
+	( void ) mod_list;
855
+
856
+	return -ENOTSUP;
857
+}
858
+
838 859
 /**
839 860
  * Destroy queue pair
840 861
  *
@@ -1202,6 +1223,25 @@ static void arbel_poll_cq ( struct ib_device *ibdev,
1202 1223
 	}
1203 1224
 }
1204 1225
 
1226
+/***************************************************************************
1227
+ *
1228
+ * Event queues
1229
+ *
1230
+ ***************************************************************************
1231
+ */
1232
+
1233
+/**
1234
+ * Poll event queue
1235
+ *
1236
+ * @v ibdev		Infiniband device
1237
+ */
1238
+static void arbel_poll_eq ( struct ib_device *ibdev ) {
1239
+	struct arbel *arbel = ib_get_drvdata ( ibdev );
1240
+
1241
+	/* TODO */
1242
+	( void ) arbel;
1243
+}
1244
+
1205 1245
 /***************************************************************************
1206 1246
  *
1207 1247
  * Infiniband link-layer operations
@@ -1399,10 +1439,12 @@ static struct ib_device_operations arbel_ib_operations = {
1399 1439
 	.create_cq	= arbel_create_cq,
1400 1440
 	.destroy_cq	= arbel_destroy_cq,
1401 1441
 	.create_qp	= arbel_create_qp,
1442
+	.modify_qp	= arbel_modify_qp,
1402 1443
 	.destroy_qp	= arbel_destroy_qp,
1403 1444
 	.post_send	= arbel_post_send,
1404 1445
 	.post_recv	= arbel_post_recv,
1405 1446
 	.poll_cq	= arbel_poll_cq,
1447
+	.poll_eq	= arbel_poll_eq,
1406 1448
 	.open		= arbel_open,
1407 1449
 	.close		= arbel_close,
1408 1450
 	.mcast_attach	= arbel_mcast_attach,
@@ -1938,7 +1980,7 @@ static int arbel_probe ( struct pci_device *pci,
1938 1980
 	i = ( ARBEL_NUM_PORTS - 1 );
1939 1981
  err_alloc_ibdev:
1940 1982
 	for ( ; i >= 0 ; i-- )
1941
-		free_ibdev ( arbel->ibdev[i] );
1983
+		ibdev_put ( arbel->ibdev[i] );
1942 1984
 	free ( arbel );
1943 1985
  err_alloc_arbel:
1944 1986
 	return rc;
@@ -1962,7 +2004,7 @@ static void arbel_remove ( struct pci_device *pci ) {
1962 2004
 	free_dma ( arbel->mailbox_out, ARBEL_MBOX_SIZE );
1963 2005
 	free_dma ( arbel->mailbox_in, ARBEL_MBOX_SIZE );
1964 2006
 	for ( i = ( ARBEL_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
1965
-		free_ibdev ( arbel->ibdev[i] );
2007
+		ibdev_put ( arbel->ibdev[i] );
1966 2008
 	free ( arbel );
1967 2009
 }
1968 2010
 

+ 209
- 223
src/drivers/infiniband/hermon.c View File

@@ -30,7 +30,6 @@
30 30
 #include <gpxe/umalloc.h>
31 31
 #include <gpxe/iobuf.h>
32 32
 #include <gpxe/netdevice.h>
33
-#include <gpxe/process.h>
34 33
 #include <gpxe/infiniband.h>
35 34
 #include "hermon.h"
36 35
 
@@ -1226,215 +1225,6 @@ static void hermon_poll_cq ( struct ib_device *ibdev,
1226 1225
 	}
1227 1226
 }
1228 1227
 
1229
-/***************************************************************************
1230
- *
1231
- * Infiniband link-layer operations
1232
- *
1233
- ***************************************************************************
1234
- */
1235
-
1236
-/**
1237
- * Initialise Infiniband link
1238
- *
1239
- * @v ibdev		Infiniband device
1240
- * @ret rc		Return status code
1241
- */
1242
-static int hermon_open ( struct ib_device *ibdev ) {
1243
-	struct hermon *hermon = ib_get_drvdata ( ibdev );
1244
-	struct hermonprm_init_port init_port;
1245
-	int rc;
1246
-
1247
-	memset ( &init_port, 0, sizeof ( init_port ) );
1248
-	MLX_FILL_2 ( &init_port, 0,
1249
-		     port_width_cap, 3,
1250
-		     vl_cap, 1 );
1251
-	MLX_FILL_2 ( &init_port, 1,
1252
-		     mtu, HERMON_MTU_2048,
1253
-		     max_gid, 1 );
1254
-	MLX_FILL_1 ( &init_port, 2, max_pkey, 64 );
1255
-	if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port,
1256
-					   &init_port ) ) != 0 ) {
1257
-		DBGC ( hermon, "Hermon %p could not intialise port: %s\n",
1258
-		       hermon, strerror ( rc ) );
1259
-		return rc;
1260
-	}
1261
-
1262
-	return 0;
1263
-}
1264
-
1265
-/**
1266
- * Close Infiniband link
1267
- *
1268
- * @v ibdev		Infiniband device
1269
- */
1270
-static void hermon_close ( struct ib_device *ibdev ) {
1271
-	struct hermon *hermon = ib_get_drvdata ( ibdev );
1272
-	int rc;
1273
-
1274
-	if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
1275
-		DBGC ( hermon, "Hermon %p could not close port: %s\n",
1276
-		       hermon, strerror ( rc ) );
1277
-		/* Nothing we can do about this */
1278
-	}
1279
-}
1280
-
1281
-/***************************************************************************
1282
- *
1283
- * Multicast group operations
1284
- *
1285
- ***************************************************************************
1286
- */
1287
-
1288
-/**
1289
- * Attach to multicast group
1290
- *
1291
- * @v ibdev		Infiniband device
1292
- * @v qp		Queue pair
1293
- * @v gid		Multicast GID
1294
- * @ret rc		Return status code
1295
- */
1296
-static int hermon_mcast_attach ( struct ib_device *ibdev,
1297
-				 struct ib_queue_pair *qp,
1298
-				 struct ib_gid *gid ) {
1299
-	struct hermon *hermon = ib_get_drvdata ( ibdev );
1300
-	struct hermonprm_mgm_hash hash;
1301
-	struct hermonprm_mcg_entry mcg;
1302
-	unsigned int index;
1303
-	int rc;
1304
-
1305
-	/* Generate hash table index */
1306
-	if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
1307
-		DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
1308
-		       hermon, strerror ( rc ) );
1309
-		return rc;
1310
-	}
1311
-	index = MLX_GET ( &hash, hash );
1312
-
1313
-	/* Check for existing hash table entry */
1314
-	if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) {
1315
-		DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n",
1316
-		       hermon, index, strerror ( rc ) );
1317
-		return rc;
1318
-	}
1319
-	if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) {
1320
-		/* FIXME: this implementation allows only a single QP
1321
-		 * per multicast group, and doesn't handle hash
1322
-		 * collisions.  Sufficient for IPoIB but may need to
1323
-		 * be extended in future.
1324
-		 */
1325
-		DBGC ( hermon, "Hermon %p MGID index %#x already in use\n",
1326
-		       hermon, index );
1327
-		return -EBUSY;
1328
-	}
1329
-
1330
-	/* Update hash table entry */
1331
-	MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 );
1332
-	MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn );
1333
-	memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) );
1334
-	if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
1335
-		DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
1336
-		       hermon, index, strerror ( rc ) );
1337
-		return rc;
1338
-	}
1339
-
1340
-	return 0;
1341
-}
1342
-
1343
-/**
1344
- * Detach from multicast group
1345
- *
1346
- * @v ibdev		Infiniband device
1347
- * @v qp		Queue pair
1348
- * @v gid		Multicast GID
1349
- */
1350
-static void hermon_mcast_detach ( struct ib_device *ibdev,
1351
-				  struct ib_queue_pair *qp __unused,
1352
-				  struct ib_gid *gid ) {
1353
-	struct hermon *hermon = ib_get_drvdata ( ibdev );
1354
-	struct hermonprm_mgm_hash hash;
1355
-	struct hermonprm_mcg_entry mcg;
1356
-	unsigned int index;
1357
-	int rc;
1358
-
1359
-	/* Generate hash table index */
1360
-	if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
1361
-		DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
1362
-		       hermon, strerror ( rc ) );
1363
-		return;
1364
-	}
1365
-	index = MLX_GET ( &hash, hash );
1366
-
1367
-	/* Clear hash table entry */
1368
-	memset ( &mcg, 0, sizeof ( mcg ) );
1369
-	if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
1370
-		DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
1371
-		       hermon, index, strerror ( rc ) );
1372
-		return;
1373
-	}
1374
-}
1375
-
1376
-/***************************************************************************
1377
- *
1378
- * MAD operations
1379
- *
1380
- ***************************************************************************
1381
- */
1382
-
1383
-/**
1384
- * Issue management datagram
1385
- *
1386
- * @v ibdev		Infiniband device
1387
- * @v mad		Management datagram
1388
- * @v len		Length of management datagram
1389
- * @ret rc		Return status code
1390
- */
1391
-static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
1392
-			size_t len ) {
1393
-	struct hermon *hermon = ib_get_drvdata ( ibdev );
1394
-	union hermonprm_mad mad_ifc;
1395
-	int rc;
1396
-
1397
-	/* Copy in request packet */
1398
-	memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
1399
-	assert ( len <= sizeof ( mad_ifc.mad ) );
1400
-	memcpy ( &mad_ifc.mad, mad, len );
1401
-
1402
-	/* Issue MAD */
1403
-	if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
1404
-					 &mad_ifc ) ) != 0 ) {
1405
-		DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
1406
-		       hermon, strerror ( rc ) );
1407
-		return rc;
1408
-	}
1409
-
1410
-	/* Copy out reply packet */
1411
-	memcpy ( mad, &mad_ifc.mad, len );
1412
-
1413
-	if ( mad->status != 0 ) {
1414
-		DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
1415
-		       hermon, ntohs ( mad->status ) );
1416
-		return -EIO;
1417
-	}
1418
-	return 0;
1419
-}
1420
-
1421
-/** Hermon Infiniband operations */
1422
-static struct ib_device_operations hermon_ib_operations = {
1423
-	.create_cq	= hermon_create_cq,
1424
-	.destroy_cq	= hermon_destroy_cq,
1425
-	.create_qp	= hermon_create_qp,
1426
-	.modify_qp	= hermon_modify_qp,
1427
-	.destroy_qp	= hermon_destroy_qp,
1428
-	.post_send	= hermon_post_send,
1429
-	.post_recv	= hermon_post_recv,
1430
-	.poll_cq	= hermon_poll_cq,
1431
-	.open		= hermon_open,
1432
-	.close		= hermon_close,
1433
-	.mcast_attach	= hermon_mcast_attach,
1434
-	.mcast_detach	= hermon_mcast_detach,
1435
-	.mad		= hermon_mad,
1436
-};
1437
-
1438 1228
 /***************************************************************************
1439 1229
  *
1440 1230
  * Event queues
@@ -1597,9 +1387,10 @@ static void hermon_event_port_state_change ( struct hermon *hermon,
1597 1387
 /**
1598 1388
  * Poll event queue
1599 1389
  *
1600
- * @v hermon		Hermon device
1390
+ * @v ibdev		Infiniband device
1601 1391
  */
1602
-static void hermon_poll_eq ( struct hermon *hermon ) {
1392
+static void hermon_poll_eq ( struct ib_device *ibdev ) {
1393
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
1603 1394
 	struct hermon_event_queue *hermon_eq = &hermon->eq;
1604 1395
 	union hermonprm_event_entry *eqe;
1605 1396
 	union hermonprm_doorbell_register db_reg;
@@ -1644,18 +1435,216 @@ static void hermon_poll_eq ( struct hermon *hermon ) {
1644 1435
 	}
1645 1436
 }
1646 1437
 
1438
+/***************************************************************************
1439
+ *
1440
+ * Infiniband link-layer operations
1441
+ *
1442
+ ***************************************************************************
1443
+ */
1444
+
1445
+/**
1446
+ * Initialise Infiniband link
1447
+ *
1448
+ * @v ibdev		Infiniband device
1449
+ * @ret rc		Return status code
1450
+ */
1451
+static int hermon_open ( struct ib_device *ibdev ) {
1452
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
1453
+	struct hermonprm_init_port init_port;
1454
+	int rc;
1455
+
1456
+	memset ( &init_port, 0, sizeof ( init_port ) );
1457
+	MLX_FILL_2 ( &init_port, 0,
1458
+		     port_width_cap, 3,
1459
+		     vl_cap, 1 );
1460
+	MLX_FILL_2 ( &init_port, 1,
1461
+		     mtu, HERMON_MTU_2048,
1462
+		     max_gid, 1 );
1463
+	MLX_FILL_1 ( &init_port, 2, max_pkey, 64 );
1464
+	if ( ( rc = hermon_cmd_init_port ( hermon, ibdev->port,
1465
+					   &init_port ) ) != 0 ) {
1466
+		DBGC ( hermon, "Hermon %p could not intialise port: %s\n",
1467
+		       hermon, strerror ( rc ) );
1468
+		return rc;
1469
+	}
1470
+
1471
+	return 0;
1472
+}
1473
+
1647 1474
 /**
1648
- * Event queue poll processor
1475
+ * Close Infiniband link
1649 1476
  *
1650
- * @v process		Hermon event queue process
1477
+ * @v ibdev		Infiniband device
1651 1478
  */
1652
-static void hermon_step ( struct process *process ) {
1653
-	struct hermon *hermon =
1654
-		container_of ( process, struct hermon, event_process );
1479
+static void hermon_close ( struct ib_device *ibdev ) {
1480
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
1481
+	int rc;
1655 1482
 
1656
-	hermon_poll_eq ( hermon );
1483
+	if ( ( rc = hermon_cmd_close_port ( hermon, ibdev->port ) ) != 0 ) {
1484
+		DBGC ( hermon, "Hermon %p could not close port: %s\n",
1485
+		       hermon, strerror ( rc ) );
1486
+		/* Nothing we can do about this */
1487
+	}
1657 1488
 }
1658 1489
 
1490
+/***************************************************************************
1491
+ *
1492
+ * Multicast group operations
1493
+ *
1494
+ ***************************************************************************
1495
+ */
1496
+
1497
+/**
1498
+ * Attach to multicast group
1499
+ *
1500
+ * @v ibdev		Infiniband device
1501
+ * @v qp		Queue pair
1502
+ * @v gid		Multicast GID
1503
+ * @ret rc		Return status code
1504
+ */
1505
+static int hermon_mcast_attach ( struct ib_device *ibdev,
1506
+				 struct ib_queue_pair *qp,
1507
+				 struct ib_gid *gid ) {
1508
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
1509
+	struct hermonprm_mgm_hash hash;
1510
+	struct hermonprm_mcg_entry mcg;
1511
+	unsigned int index;
1512
+	int rc;
1513
+
1514
+	/* Generate hash table index */
1515
+	if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
1516
+		DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
1517
+		       hermon, strerror ( rc ) );
1518
+		return rc;
1519
+	}
1520
+	index = MLX_GET ( &hash, hash );
1521
+
1522
+	/* Check for existing hash table entry */
1523
+	if ( ( rc = hermon_cmd_read_mcg ( hermon, index, &mcg ) ) != 0 ) {
1524
+		DBGC ( hermon, "Hermon %p could not read MCG %#x: %s\n",
1525
+		       hermon, index, strerror ( rc ) );
1526
+		return rc;
1527
+	}
1528
+	if ( MLX_GET ( &mcg, hdr.members_count ) != 0 ) {
1529
+		/* FIXME: this implementation allows only a single QP
1530
+		 * per multicast group, and doesn't handle hash
1531
+		 * collisions.  Sufficient for IPoIB but may need to
1532
+		 * be extended in future.
1533
+		 */
1534
+		DBGC ( hermon, "Hermon %p MGID index %#x already in use\n",
1535
+		       hermon, index );
1536
+		return -EBUSY;
1537
+	}
1538
+
1539
+	/* Update hash table entry */
1540
+	MLX_FILL_1 ( &mcg, 1, hdr.members_count, 1 );
1541
+	MLX_FILL_1 ( &mcg, 8, qp[0].qpn, qp->qpn );
1542
+	memcpy ( &mcg.u.dwords[4], gid, sizeof ( *gid ) );
1543
+	if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
1544
+		DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
1545
+		       hermon, index, strerror ( rc ) );
1546
+		return rc;
1547
+	}
1548
+
1549
+	return 0;
1550
+}
1551
+
1552
+/**
1553
+ * Detach from multicast group
1554
+ *
1555
+ * @v ibdev		Infiniband device
1556
+ * @v qp		Queue pair
1557
+ * @v gid		Multicast GID
1558
+ */
1559
+static void hermon_mcast_detach ( struct ib_device *ibdev,
1560
+				  struct ib_queue_pair *qp __unused,
1561
+				  struct ib_gid *gid ) {
1562
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
1563
+	struct hermonprm_mgm_hash hash;
1564
+	struct hermonprm_mcg_entry mcg;
1565
+	unsigned int index;
1566
+	int rc;
1567
+
1568
+	/* Generate hash table index */
1569
+	if ( ( rc = hermon_cmd_mgid_hash ( hermon, gid, &hash ) ) != 0 ) {
1570
+		DBGC ( hermon, "Hermon %p could not hash GID: %s\n",
1571
+		       hermon, strerror ( rc ) );
1572
+		return;
1573
+	}
1574
+	index = MLX_GET ( &hash, hash );
1575
+
1576
+	/* Clear hash table entry */
1577
+	memset ( &mcg, 0, sizeof ( mcg ) );
1578
+	if ( ( rc = hermon_cmd_write_mcg ( hermon, index, &mcg ) ) != 0 ) {
1579
+		DBGC ( hermon, "Hermon %p could not write MCG %#x: %s\n",
1580
+		       hermon, index, strerror ( rc ) );
1581
+		return;
1582
+	}
1583
+}
1584
+
1585
+/***************************************************************************
1586
+ *
1587
+ * MAD operations
1588
+ *
1589
+ ***************************************************************************
1590
+ */
1591
+
1592
+/**
1593
+ * Issue management datagram
1594
+ *
1595
+ * @v ibdev		Infiniband device
1596
+ * @v mad		Management datagram
1597
+ * @v len		Length of management datagram
1598
+ * @ret rc		Return status code
1599
+ */
1600
+static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
1601
+			size_t len ) {
1602
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
1603
+	union hermonprm_mad mad_ifc;
1604
+	int rc;
1605
+
1606
+	/* Copy in request packet */
1607
+	memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
1608
+	assert ( len <= sizeof ( mad_ifc.mad ) );
1609
+	memcpy ( &mad_ifc.mad, mad, len );
1610
+
1611
+	/* Issue MAD */
1612
+	if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
1613
+					 &mad_ifc ) ) != 0 ) {
1614
+		DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
1615
+		       hermon, strerror ( rc ) );
1616
+		return rc;
1617
+	}
1618
+
1619
+	/* Copy out reply packet */
1620
+	memcpy ( mad, &mad_ifc.mad, len );
1621
+
1622
+	if ( mad->status != 0 ) {
1623
+		DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
1624
+		       hermon, ntohs ( mad->status ) );
1625
+		return -EIO;
1626
+	}
1627
+	return 0;
1628
+}
1629
+
1630
+/** Hermon Infiniband operations */
1631
+static struct ib_device_operations hermon_ib_operations = {
1632
+	.create_cq	= hermon_create_cq,
1633
+	.destroy_cq	= hermon_destroy_cq,
1634
+	.create_qp	= hermon_create_qp,
1635
+	.modify_qp	= hermon_modify_qp,
1636
+	.destroy_qp	= hermon_destroy_qp,
1637
+	.post_send	= hermon_post_send,
1638
+	.post_recv	= hermon_post_recv,
1639
+	.poll_cq	= hermon_poll_cq,
1640
+	.poll_eq	= hermon_poll_eq,
1641
+	.open		= hermon_open,
1642
+	.close		= hermon_close,
1643
+	.mcast_attach	= hermon_mcast_attach,
1644
+	.mcast_detach	= hermon_mcast_detach,
1645
+	.mad		= hermon_mad,
1646
+};
1647
+
1659 1648
 /***************************************************************************
1660 1649
  *
1661 1650
  * Firmware control
@@ -2168,7 +2157,6 @@ static int hermon_probe ( struct pci_device *pci,
2168 2157
 		goto err_alloc_hermon;
2169 2158
 	}
2170 2159
 	pci_set_drvdata ( pci, hermon );
2171
-	process_init ( &hermon->event_process, hermon_step, NULL );
2172 2160
 
2173 2161
 	/* Allocate Infiniband devices */
2174 2162
 	for ( i = 0 ; i < HERMON_NUM_PORTS ; i++ ) {
@@ -2270,8 +2258,7 @@ static int hermon_probe ( struct pci_device *pci,
2270 2258
 	i = ( HERMON_NUM_PORTS - 1 );
2271 2259
  err_alloc_ibdev:
2272 2260
 	for ( ; i >= 0 ; i-- )
2273
-		free_ibdev ( hermon->ibdev[i] );
2274
-	process_del ( &hermon->event_process );
2261
+		ibdev_put ( hermon->ibdev[i] );
2275 2262
 	free ( hermon );
2276 2263
  err_alloc_hermon:
2277 2264
 	return rc;
@@ -2296,8 +2283,7 @@ static void hermon_remove ( struct pci_device *pci ) {
2296 2283
 	free_dma ( hermon->mailbox_out, HERMON_MBOX_SIZE );
2297 2284
 	free_dma ( hermon->mailbox_in, HERMON_MBOX_SIZE );
2298 2285
 	for ( i = ( HERMON_NUM_PORTS - 1 ) ; i >= 0 ; i-- )
2299
-		free_ibdev ( hermon->ibdev[i] );
2300
-	process_del ( &hermon->event_process );
2286
+		ibdev_put ( hermon->ibdev[i] );
2301 2287
 	free ( hermon );
2302 2288
 }
2303 2289
 

+ 0
- 3
src/drivers/infiniband/hermon.h View File

@@ -9,7 +9,6 @@
9 9
 
10 10
 #include <stdint.h>
11 11
 #include <gpxe/uaccess.h>
12
-#include <gpxe/process.h>
13 12
 #include "mlx_bitops.h"
14 13
 #include "MT25408_PRM.h"
15 14
 
@@ -465,8 +464,6 @@ struct hermon {
465 464
 
466 465
 	/** Event queue */
467 466
 	struct hermon_event_queue eq;
468
-	/** Event queue process */
469
-	struct process event_process;
470 467
 
471 468
 	/** Completion queue in-use bitmask */
472 469
 	hermon_bitmask_t cq_inuse[ HERMON_BITMASK_SIZE ( HERMON_MAX_CQS ) ];

+ 33
- 1
src/include/gpxe/infiniband.h View File

@@ -8,6 +8,7 @@
8 8
  */
9 9
 
10 10
 #include <stdint.h>
11
+#include <gpxe/refcnt.h>
11 12
 #include <gpxe/device.h>
12 13
 
13 14
 /** Subnet administrator QPN */
@@ -254,6 +255,12 @@ struct ib_device_operations {
254 255
 			     struct ib_completion_queue *cq,
255 256
 			     ib_completer_t complete_send,
256 257
 			     ib_completer_t complete_recv );
258
+	/**
259
+	 * Poll event queue
260
+	 *
261
+	 * @v ibdev		Infiniband device
262
+	 */
263
+	void ( * poll_eq ) ( struct ib_device *ibdev );
257 264
 	/**
258 265
 	 * Open port
259 266
 	 *
@@ -300,6 +307,10 @@ struct ib_device_operations {
300 307
 
301 308
 /** An Infiniband device */
302 309
 struct ib_device {
310
+	/** Reference counter */
311
+	struct refcnt refcnt;
312
+	/** List of Infiniband devices */
313
+	struct list_head list;
303 314
 	/** Underlying device */
304 315
 	struct device *dev;
305 316
 	/** Infiniband operations */
@@ -337,7 +348,6 @@ extern struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
337 348
 extern struct ib_device * alloc_ibdev ( size_t priv_size );
338 349
 extern int register_ibdev ( struct ib_device *ibdev );
339 350
 extern void unregister_ibdev ( struct ib_device *ibdev );
340
-extern void free_ibdev ( struct ib_device *ibdev );
341 351
 extern void ib_link_state_changed ( struct ib_device *ibdev );
342 352
 
343 353
 /**
@@ -444,6 +454,28 @@ ib_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad, size_t len ) {
444 454
 	return ibdev->op->mad ( ibdev, mad, len );
445 455
 }
446 456
 
457
+/**
458
+ * Get reference to Infiniband device
459
+ *
460
+ * @v ibdev		Infiniband device
461
+ * @ret ibdev		Infiniband device
462
+ */
463
+static inline __attribute__ (( always_inline )) struct ib_device *
464
+ibdev_get ( struct ib_device *ibdev ) {
465
+	ref_get ( &ibdev->refcnt );
466
+	return ibdev;
467
+}
468
+
469
+/**
470
+ * Drop reference to Infiniband device
471
+ *
472
+ * @v ibdev		Infiniband device
473
+ */
474
+static inline __attribute__ (( always_inline )) void
475
+ibdev_put ( struct ib_device *ibdev ) {
476
+	ref_put ( &ibdev->refcnt );
477
+}
478
+
447 479
 /**
448 480
  * Set Infiniband work queue driver-private data
449 481
  *

+ 62
- 27
src/net/infiniband.c View File

@@ -29,6 +29,7 @@
29 29
 #include <gpxe/netdevice.h>
30 30
 #include <gpxe/iobuf.h>
31 31
 #include <gpxe/ipoib.h>
32
+#include <gpxe/process.h>
32 33
 #include <gpxe/infiniband.h>
33 34
 
34 35
 /** @file
@@ -37,6 +38,9 @@
37 38
  *
38 39
  */
39 40
 
41
+/** List of Infiniband devices */
42
+struct list_head ib_devices = LIST_HEAD_INIT ( ib_devices );
43
+
40 44
 /**
41 45
  * Create completion queue
42 46
  *
@@ -349,6 +353,50 @@ static int ib_get_mad_params ( struct ib_device *ibdev ) {
349 353
 	return 0;
350 354
 }
351 355
 
356
+/***************************************************************************
357
+ *
358
+ * Event queues
359
+ *
360
+ ***************************************************************************
361
+ */
362
+
363
+/**
364
+ * Handle Infiniband link state change
365
+ *
366
+ * @v ibdev		Infiniband device
367
+ */
368
+void ib_link_state_changed ( struct ib_device *ibdev ) {
369
+	int rc;
370
+
371
+	/* Update MAD parameters */
372
+	if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) {
373
+		DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n",
374
+		       ibdev, strerror ( rc ) );
375
+		return;
376
+	}
377
+
378
+	/* Notify IPoIB of link state change */
379
+	ipoib_link_state_changed ( ibdev );
380
+}
381
+
382
+/**
383
+ * Single-step the Infiniband event queue
384
+ *
385
+ * @v process		Infiniband event queue process
386
+ */
387
+static void ib_step ( struct process *process __unused ) {
388
+	struct ib_device *ibdev;
389
+
390
+	list_for_each_entry ( ibdev, &ib_devices, list ) {
391
+		ibdev->op->poll_eq ( ibdev );
392
+	}
393
+}
394
+
395
+/** Infiniband event queue process */
396
+struct process ib_process __permanent_process = {
397
+	.step = ib_step,
398
+};
399
+
352 400
 /***************************************************************************
353 401
  *
354 402
  * Infiniband device creation/destruction
@@ -385,6 +433,10 @@ struct ib_device * alloc_ibdev ( size_t priv_size ) {
385 433
 int register_ibdev ( struct ib_device *ibdev ) {
386 434
 	int rc;
387 435
 
436
+	/* Add to device list */
437
+	ibdev_get ( ibdev );
438
+	list_add_tail ( &ibdev->list, &ib_devices );
439
+
388 440
 	/* Open link */
389 441
 	if ( ( rc = ib_open ( ibdev ) ) != 0 )
390 442
 		goto err_open;
@@ -400,12 +452,16 @@ int register_ibdev ( struct ib_device *ibdev ) {
400 452
 		goto err_ipoib_probe;
401 453
 	}
402 454
 
455
+	DBGC ( ibdev, "IBDEV %p registered (phys %s)\n", ibdev,
456
+	       ibdev->dev->name );
403 457
 	return 0;
404 458
 
405 459
  err_ipoib_probe:
406 460
  err_get_mad_params:
407 461
 	ib_close ( ibdev );
408 462
  err_open:
463
+	list_del ( &ibdev->list );
464
+	ibdev_put ( ibdev );
409 465
 	return rc;
410 466
 }
411 467
 
@@ -415,34 +471,13 @@ int register_ibdev ( struct ib_device *ibdev ) {
415 471
  * @v ibdev		Infiniband device
416 472
  */
417 473
 void unregister_ibdev ( struct ib_device *ibdev ) {
474
+
475
+	/* Close device */
418 476
 	ipoib_remove ( ibdev );
419 477
 	ib_close ( ibdev );
420
-}
421
-
422
-/**
423
- * Free Infiniband device
424
- *
425
- * @v ibdev		Infiniband device
426
- */
427
-void free_ibdev ( struct ib_device *ibdev ) {
428
-	free ( ibdev );
429
-}
430
-
431
-/**
432
- * Handle Infiniband link state change
433
- *
434
- * @v ibdev		Infiniband device
435
- */
436
-void ib_link_state_changed ( struct ib_device *ibdev ) {
437
-	int rc;
438 478
 
439
-	/* Update MAD parameters */
440
-	if ( ( rc = ib_get_mad_params ( ibdev ) ) != 0 ) {
441
-		DBGC ( ibdev, "IBDEV %p could not update MAD parameters: %s\n",
442
-		       ibdev, strerror ( rc ) );
443
-		return;
444
-	}
445
-
446
-	/* Notify IPoIB of link state change */
447
-	ipoib_link_state_changed ( ibdev );
479
+	/* Remove from device list */
480
+	list_del ( &ibdev->list );
481
+	ibdev_put ( ibdev );
482
+	DBGC ( ibdev, "IBDEV %p unregistered\n", ibdev );
448 483
 }

Loading…
Cancel
Save