Browse Source

Broadcast GID is now calculated by IPoIB layer.

tags/v0.9.3
Michael Brown 17 years ago
parent
commit
2ed1acb9e9
3 changed files with 130 additions and 25 deletions
  1. 100
    3
      src/drivers/net/ipoib.c
  2. 7
    20
      src/drivers/net/mlx_ipoib/mt25218.c
  3. 23
    2
      src/include/gpxe/infiniband.h

+ 100
- 3
src/drivers/net/ipoib.c View File

84
 	struct ipoib_queue_set data;
84
 	struct ipoib_queue_set data;
85
 	/** Data queue set */
85
 	/** Data queue set */
86
 	struct ipoib_queue_set meta;
86
 	struct ipoib_queue_set meta;
87
+	/** Broadcast GID */
88
+	struct ib_gid broadcast_gid;
87
 };
89
 };
88
 
90
 
89
 /**
91
 /**
112
 /** Oldest IPoIB path cache entry index */
114
 /** Oldest IPoIB path cache entry index */
113
 static unsigned int ipoib_path_cache_idx = 0;
115
 static unsigned int ipoib_path_cache_idx = 0;
114
 
116
 
117
+/** IPoIB metadata TID */
118
+static uint32_t ipoib_meta_tid = 0;
119
+
120
+/** IPv4 broadcast GID */
121
+static const struct ib_gid ipv4_broadcast_gid = {
122
+	{ { 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
123
+	    0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff } }
124
+};
125
+
115
 /****************************************************************************
126
 /****************************************************************************
116
  *
127
  *
117
  * IPoIB link layer
128
  * IPoIB link layer
319
 	struct io_buffer *iobuf;
330
 	struct io_buffer *iobuf;
320
 	struct ib_mad_path_record *path_record;
331
 	struct ib_mad_path_record *path_record;
321
 	struct ib_address_vector av;
332
 	struct ib_address_vector av;
322
- 	static uint32_t tid = 0;
323
 	int rc;
333
 	int rc;
324
 
334
 
325
 	/* Allocate I/O buffer */
335
 	/* Allocate I/O buffer */
336
 	path_record->mad_hdr.class_version = 2;
346
 	path_record->mad_hdr.class_version = 2;
337
 	path_record->mad_hdr.method = IB_MGMT_METHOD_GET;
347
 	path_record->mad_hdr.method = IB_MGMT_METHOD_GET;
338
 	path_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
348
 	path_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_PATH_REC );
339
-	path_record->mad_hdr.tid = tid++;
349
+	path_record->mad_hdr.tid = ipoib_meta_tid++;
340
 	path_record->sa_hdr.comp_mask[1] =
350
 	path_record->sa_hdr.comp_mask[1] =
341
 		htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
351
 		htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
342
 	memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) );
352
 	memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) );
361
 	return 0;
371
 	return 0;
362
 }
372
 }
363
 
373
 
374
+/**
375
+ * Transmit multicast group membership request
376
+ *
377
+ * @v ipoib		IPoIB device
378
+ * @v gid		Multicast GID
379
+ * @v join		Join (rather than leave) group
380
+ * @ret rc		Return status code
381
+ */
382
+static int ipoib_mc_member_record ( struct ipoib_device *ipoib,
383
+				    struct ib_gid *gid, int join ) {
384
+	struct ib_device *ibdev = ipoib->ibdev;
385
+	struct io_buffer *iobuf;
386
+	struct ib_mad_mc_member_record *mc_member_record;
387
+	struct ib_address_vector av;
388
+	int rc;
389
+
390
+	/* Allocate I/O buffer */
391
+	iobuf = alloc_iob ( sizeof ( *mc_member_record ) );
392
+	if ( ! iobuf )
393
+		return -ENOMEM;
394
+	iob_put ( iobuf, sizeof ( *mc_member_record ) );
395
+	mc_member_record = iobuf->data;
396
+	memset ( mc_member_record, 0, sizeof ( *mc_member_record ) );
397
+
398
+	/* Construct path record request */
399
+	mc_member_record->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
400
+	mc_member_record->mad_hdr.mgmt_class = IB_MGMT_CLASS_SUBN_ADM;
401
+	mc_member_record->mad_hdr.class_version = 2;
402
+	mc_member_record->mad_hdr.method = 
403
+		( join ? IB_MGMT_METHOD_SET : IB_MGMT_METHOD_DELETE );
404
+	mc_member_record->mad_hdr.attr_id = htons ( IB_SA_ATTR_MC_MEMBER_REC );
405
+	mc_member_record->mad_hdr.tid = ipoib_meta_tid++;
406
+	mc_member_record->sa_hdr.comp_mask[1] =
407
+		htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
408
+			IB_SA_MCMEMBER_REC_JOIN_STATE );
409
+	mc_member_record->scope__join_state = 1;
410
+	memcpy ( &mc_member_record->mgid, gid,
411
+		 sizeof ( mc_member_record->mgid ) );
412
+	memcpy ( &mc_member_record->port_gid, &ibdev->port_gid,
413
+		 sizeof ( mc_member_record->port_gid ) );
414
+
415
+	/* Construct address vector */
416
+	memset ( &av, 0, sizeof ( av ) );
417
+	av.dlid = ibdev->sm_lid;
418
+	av.dest_qp = IB_SA_QPN;
419
+	av.qkey = IB_GLOBAL_QKEY;
420
+
421
+	/* Post send request */
422
+	if ( ( rc = ib_post_send ( ibdev, ipoib->meta.qp, &av,
423
+				   iobuf ) ) != 0 ) {
424
+		DBGC ( ipoib, "IPoIB %p could not send get path record: %s\n",
425
+		       ipoib, strerror ( rc ) );
426
+		free_iob ( iobuf );
427
+		return rc;
428
+	}
429
+
430
+	return 0;
431
+}
432
+
364
 /**
433
 /**
365
  * Transmit packet via IPoIB network device
434
  * Transmit packet via IPoIB network device
366
  *
435
  *
591
 
660
 
592
 	/* Attach to broadcast multicast GID */
661
 	/* Attach to broadcast multicast GID */
593
 	if ( ( rc = ib_mcast_attach ( ibdev, ipoib->data.qp,
662
 	if ( ( rc = ib_mcast_attach ( ibdev, ipoib->data.qp,
594
-				      &ibdev->broadcast_gid ) ) != 0 ) {
663
+				      &ipoib->broadcast_gid ) ) != 0 ) {
595
 		DBG ( "Could not attach to broadcast GID: %s\n",
664
 		DBG ( "Could not attach to broadcast GID: %s\n",
596
 		      strerror ( rc ) );
665
 		      strerror ( rc ) );
597
 		return rc;
666
 		return rc;
628
 	.irq		= ipoib_irq,
697
 	.irq		= ipoib_irq,
629
 };
698
 };
630
 
699
 
700
+/**
701
+ * Join IPoIB broadcast group
702
+ *
703
+ * @v ipoib		IPoIB device
704
+ * @ret rc		Return status code
705
+ */
706
+int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
707
+	int rc;
708
+
709
+	/* Send join request */
710
+	if ( ( rc = ipoib_mc_member_record ( ipoib, &ipoib->broadcast_gid,
711
+					     1 ) ) != 0 ) {
712
+		DBGC ( ipoib, "IPoIB %p could not send broadcast join: %s\n",
713
+		       ipoib, strerror ( rc ) );
714
+		return rc;
715
+	}
716
+
717
+
718
+	return 0;
719
+}
720
+
631
 /**
721
 /**
632
  * Probe IPoIB device
722
  * Probe IPoIB device
633
  *
723
  *
652
 	ipoib->netdev = netdev;
742
 	ipoib->netdev = netdev;
653
 	ipoib->ibdev = ibdev;
743
 	ipoib->ibdev = ibdev;
654
 
744
 
745
+	/* Calculate broadcast GID */
746
+	memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid,
747
+		 sizeof ( ipoib->broadcast_gid ) );
748
+	ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
749
+
655
 	/* Allocate metadata queue set */
750
 	/* Allocate metadata queue set */
656
 	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
751
 	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->meta,
657
 					IPOIB_META_NUM_CQES,
752
 					IPOIB_META_NUM_CQES,
663
 		goto err_create_meta_qset;
758
 		goto err_create_meta_qset;
664
 	}
759
 	}
665
 
760
 
761
+
762
+
666
 	/* Allocate data queue set */
763
 	/* Allocate data queue set */
667
 	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
764
 	if ( ( rc = ipoib_create_qset ( ipoib, &ipoib->data,
668
 					IPOIB_DATA_NUM_CQES,
765
 					IPOIB_DATA_NUM_CQES,

+ 7
- 20
src/drivers/net/mlx_ipoib/mt25218.c View File

1251
 	return 0;
1251
 	return 0;
1252
 }
1252
 }
1253
 
1253
 
1254
-static int arbel_get_broadcast_gid ( struct arbel *arbel,
1255
-				     struct ib_gid *broadcast_gid ) {
1256
-	static const struct ib_gid ipv4_broadcast_gid = {
1257
-		{ { 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
1258
-		    0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff } }
1259
-	};
1254
+static int arbel_get_pkey ( struct arbel *arbel, unsigned long *pkey ) {
1260
 	struct ib_mad_pkey_table pkey_table;
1255
 	struct ib_mad_pkey_table pkey_table;
1261
 	int rc;
1256
 	int rc;
1262
 
1257
 
1263
-	/* Start with the IPv4 broadcast GID */
1264
-	memcpy ( broadcast_gid, &ipv4_broadcast_gid,
1265
-		 sizeof ( *broadcast_gid ) );
1266
-
1267
-	/* Add partition key */
1268
 	if ( ( rc = arbel_get_pkey_table ( arbel, &pkey_table ) ) != 0 )
1258
 	if ( ( rc = arbel_get_pkey_table ( arbel, &pkey_table ) ) != 0 )
1269
 		return rc;
1259
 		return rc;
1270
-	memcpy ( &broadcast_gid->u.bytes[4], &pkey_table.pkey[0][0],
1271
-		 sizeof ( pkey_table.pkey[0][0] ) );
1272
-
1260
+	*pkey = ntohs ( pkey_table.pkey[0][0] );
1273
 	return 0;
1261
 	return 0;
1274
 }
1262
 }
1275
 
1263
 
1340
 		goto err_get_port_gid;
1328
 		goto err_get_port_gid;
1341
 	}
1329
 	}
1342
 
1330
 
1343
-	/* Get broadcast GID */
1344
-	if ( ( rc = arbel_get_broadcast_gid ( arbel,
1345
-					      &ibdev->broadcast_gid ) ) != 0 ){
1346
-		DBGC ( arbel, "Arbel %p could not determine broadcast GID: "
1331
+	/* Get partition key */
1332
+	if ( ( rc = arbel_get_pkey ( arbel, &ibdev->pkey ) ) != 0 ) {
1333
+		DBGC ( arbel, "Arbel %p could not determine partition key: "
1347
 		       "%s\n", arbel, strerror ( rc ) );
1334
 		       "%s\n", arbel, strerror ( rc ) );
1348
-		goto err_get_broadcast_gid;
1335
+		goto err_get_pkey;
1349
 	}
1336
 	}
1350
 
1337
 
1351
 	struct ud_av_st *bcast_av = ib_data.bcast_av;
1338
 	struct ud_av_st *bcast_av = ib_data.bcast_av;
1370
 	return 0;
1357
 	return 0;
1371
 
1358
 
1372
  err_ipoib_probe:
1359
  err_ipoib_probe:
1373
- err_get_broadcast_gid:
1360
+ err_get_pkey:
1374
  err_get_port_gid:
1361
  err_get_port_gid:
1375
  err_get_sm_lid:
1362
  err_get_sm_lid:
1376
  err_query_dev_lim:
1363
  err_query_dev_lim:

+ 23
- 2
src/include/gpxe/infiniband.h View File

20
 struct ib_gid {
20
 struct ib_gid {
21
 	union {
21
 	union {
22
 		uint8_t bytes[16];
22
 		uint8_t bytes[16];
23
+		uint16_t words[8];
23
 		uint32_t dwords[4];
24
 		uint32_t dwords[4];
24
 	} u;
25
 	} u;
25
 };
26
 };
258
 struct ib_device {
259
 struct ib_device {
259
 	/** Port GID */
260
 	/** Port GID */
260
 	struct ib_gid port_gid;
261
 	struct ib_gid port_gid;
261
-	/** Broadcast GID */
262
-	struct ib_gid broadcast_gid;
263
 	/** Subnet manager LID */
262
 	/** Subnet manager LID */
264
 	unsigned long sm_lid;
263
 	unsigned long sm_lid;
264
+	/** Partition key */
265
+	unsigned int pkey;
265
 	/** Underlying device */
266
 	/** Underlying device */
266
 	struct device *dev;
267
 	struct device *dev;
267
 	/** Infiniband operations */
268
 	/** Infiniband operations */
541
 	uint32_t reserved2[35];
542
 	uint32_t reserved2[35];
542
 } __attribute__ (( packed ));
543
 } __attribute__ (( packed ));
543
 
544
 
545
+struct ib_mad_mc_member_record {
546
+	struct ib_mad_hdr mad_hdr;
547
+	struct ib_rmpp_hdr rmpp_hdr;
548
+	struct ib_sa_hdr sa_hdr;
549
+	struct ib_gid mgid;
550
+	struct ib_gid port_gid;
551
+	uint32_t qkey;
552
+	uint16_t mlid;
553
+	uint8_t mtu_selector__mtu;
554
+	uint8_t tclass;
555
+	uint16_t pkey;
556
+	uint8_t rate_selector__rate;
557
+	uint8_t packet_lifetime_selector__packet_lifetime;
558
+	uint32_t sl__flow_label__hop_limit;
559
+	uint8_t scope__join_state;
560
+	uint8_t proxy_join__reserved;
561
+	uint16_t reserved0;
562
+	uint32_t reserved1[37];
563
+} __attribute__ (( packed ));
564
+
544
 union ib_mad {
565
 union ib_mad {
545
 	struct ib_mad_hdr mad_hdr;
566
 	struct ib_mad_hdr mad_hdr;
546
 	struct ib_mad_data data;
567
 	struct ib_mad_data data;

Loading…
Cancel
Save