[infiniband] Parse MLID, rate, and SL from multicast membership record

Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/drivers/net/ipoib.c

@@ -73,6 +73,16 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 /** Number of IPoIB completion entries */
 #define IPOIB_NUM_CQES 8
 
+/** An IPoIB broadcast address */
+struct ipoib_broadcast {
+	/** MAC address */
+	struct ipoib_mac mac;
+	/** Address vector */
+	struct ib_address_vector av;
+	/** Multicast group membership */
+	struct ib_mc_membership membership;
+};
+
 /** An IPoIB device */
 struct ipoib_device {
 	/** Network device */
@@ -87,10 +97,8 @@ struct ipoib_device {
 	struct ib_queue_pair *qp;
 	/** Local MAC */
 	struct ipoib_mac mac;
-	/** Broadcast MAC */
-	struct ipoib_mac broadcast;
-	/** IPv4 broadcast multicast group membership */
-	struct ib_mc_membership membership;
+	/** Broadcast address */
+	struct ipoib_broadcast broadcast;
 	/** REMAC cache */
 	struct list_head peers;
 };
@@ -149,7 +157,7 @@ static struct ipoib_mac * ipoib_find_remac ( struct ipoib_device *ipoib,
 	 * multicasts as broadcasts for simplicity.
 	 */
 	if ( is_multicast_ether_addr ( remac ) )
-		return &ipoib->broadcast;
+		return &ipoib->broadcast.mac;
 
 	/* Try to find via REMAC cache */
 	list_for_each_entry ( peer, &ipoib->peers, list ) {
@@ -559,6 +567,7 @@ static int ipoib_transmit ( struct net_device *netdev,
 	/* Construct address vector */
 	memset ( &dest, 0, sizeof ( dest ) );
 	dest.qpn = ( ntohl ( mac->flags__qpn ) & IB_QPN_MASK );
+	dest.qkey = ipoib->broadcast.av.qkey;
 	dest.gid_present = 1;
 	memcpy ( &dest.gid, &mac->gid, sizeof ( dest.gid ) );
 	if ( ( rc = ib_resolve_path ( ibdev, &dest ) ) != 0 ) {
@@ -650,8 +659,9 @@ static void ipoib_complete_recv ( struct ib_device *ibdev __unused,
 	ethhdr->h_protocol = net_proto;
 
 	/* Construct destination address */
-	if ( dest->gid_present && ( memcmp ( &dest->gid, &ipoib->broadcast.gid,
-					     sizeof ( dest->gid ) ) == 0 ) ) {
+	if ( dest->gid_present &&
+	     ( memcmp ( &dest->gid, &ipoib->broadcast.mac.gid,
+			sizeof ( dest->gid ) ) == 0 ) ) {
 		/* Broadcast GID; use the Ethernet broadcast address */
 		memcpy ( &ethhdr->h_dest, eth_broadcast,
 			 sizeof ( ethhdr->h_dest ) );
@@ -726,18 +736,13 @@ static void ipoib_poll ( struct net_device *netdev ) {
 /**
  * Handle IPv4 broadcast multicast group join completion
  *
- * @v ibdev		Infiniband device
- * @v qp		Queue pair
  * @v membership	Multicast group membership
  * @v rc		Status code
- * @v mad		Response MAD (or NULL on error)
  */
-void ipoib_join_complete ( struct ib_device *ibdev __unused,
-			   struct ib_queue_pair *qp __unused,
-			   struct ib_mc_membership *membership, int rc,
-			   union ib_mad *mad __unused ) {
-	struct ipoib_device *ipoib =
-		container_of ( membership, struct ipoib_device, membership );
+void ipoib_join_complete ( struct ib_mc_membership *membership, int rc ) {
+	struct ipoib_device *ipoib = container_of ( membership,
+						    struct ipoib_device,
+						    broadcast.membership );
 
 	/* Record join status as link status */
 	netdev_link_err ( ipoib->netdev, rc );
@@ -752,8 +757,10 @@ void ipoib_join_complete ( struct ib_device *ibdev __unused,
 static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
 	int rc;
 
+	/* Join multicast group */
 	if ( ( rc = ib_mcast_join ( ipoib->ibdev, ipoib->qp,
-				    &ipoib->membership, &ipoib->broadcast.gid,
+				    &ipoib->broadcast.membership,
+				    &ipoib->broadcast.av,
 				    ipoib_join_complete ) ) != 0 ) {
 		DBGC ( ipoib, "IPoIB %p could not join broadcast group: %s\n",
 		       ipoib, strerror ( rc ) );
@@ -770,7 +777,9 @@ static int ipoib_join_broadcast_group ( struct ipoib_device *ipoib ) {
  */
 static void ipoib_leave_broadcast_group ( struct ipoib_device *ipoib ) {
 
-	ib_mcast_leave ( ipoib->ibdev, ipoib->qp, &ipoib->membership );
+	/* Leave multicast group */
+	ib_mcast_leave ( ipoib->ibdev, ipoib->qp,
+			 &ipoib->broadcast.membership );
 }
 
 /**
@@ -791,10 +800,17 @@ static void ipoib_link_state_changed ( struct ipoib_device *ipoib ) {
 	memcpy ( &ipoib->mac.gid.s.prefix, &ibdev->gid.s.prefix,
 		 sizeof ( ipoib->mac.gid.s.prefix ) );
 
-	/* Update broadcast GID based on potentially-new partition key */
-	ipoib->broadcast.gid.words[2] =
+	/* Update broadcast MAC GID based on potentially-new partition key */
+	ipoib->broadcast.mac.gid.words[2] =
 		htons ( ibdev->pkey | IB_PKEY_FULL );
 
+	/* Construct broadcast address vector from broadcast MAC address */
+	memset ( &ipoib->broadcast.av, 0, sizeof ( ipoib->broadcast.av ) );
+	ipoib->broadcast.av.qpn = IB_QPN_BROADCAST;
+	ipoib->broadcast.av.gid_present = 1;
+	memcpy ( &ipoib->broadcast.av.gid, &ipoib->broadcast.mac.gid,
+		 sizeof ( ipoib->broadcast.av.gid ) );
+
 	/* Set net device link state to reflect Infiniband link state */
 	rc = ib_link_rc ( ibdev );
 	netdev_link_err ( netdev, ( rc ? rc : -EINPROGRESS_JOINING ) );
@@ -936,8 +952,8 @@ static int ipoib_probe ( struct ib_device *ibdev ) {
 		 sizeof ( ipoib->mac.gid.s.guid ) );
 
 	/* Set default broadcast MAC address */
-	memcpy ( &ipoib->broadcast, &ipoib_broadcast,
-		 sizeof ( ipoib->broadcast ) );
+	memcpy ( &ipoib->broadcast.mac, &ipoib_broadcast,
+		 sizeof ( ipoib->broadcast.mac ) );
 
 	/* Add to list of IPoIB devices */
 	list_add_tail ( &ipoib->list, &ipoib_devices );

src/include/ipxe/ib_mcast.h

@@ -17,32 +17,25 @@ struct ib_mad_transaction;
 struct ib_mc_membership {
 	/** Queue pair */
 	struct ib_queue_pair *qp;
-	/** Multicast GID */
-	union ib_gid gid;
+	/** Address vector */
+	struct ib_address_vector *av;
 	/** Attached to multicast GID */
 	int attached;
 	/** Multicast group join transaction */
 	struct ib_mad_transaction *madx;
 	/** Handle join success/failure
 	 *
-	 * @v ibdev		Infiniband device
-	 * @v qp		Queue pair
 	 * @v membership	Multicast group membership
 	 * @v rc		Status code
-	 * @v mad		Response MAD (or NULL on error)
 	 */
-	void ( * complete ) ( struct ib_device *ibdev, struct ib_queue_pair *qp,
-			      struct ib_mc_membership *membership, int rc,
-			      union ib_mad *mad );
+	void ( * complete ) ( struct ib_mc_membership *membership, int rc );
 };
 
 extern int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 			   struct ib_mc_membership *membership,
-			   union ib_gid *gid,
-			   void ( * joined ) ( struct ib_device *ibdev,
-					       struct ib_queue_pair *qp,
-					       struct ib_mc_membership *memb,
-					       int rc, union ib_mad *mad ) );
+			   struct ib_address_vector *av,
+			   void ( * joined ) ( struct ib_mc_membership *memb,
+					       int rc ) );
 
 extern void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 			     struct ib_mc_membership *membership );

src/net/infiniband/ib_mcast.c

@@ -42,11 +42,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * Generate multicast membership MAD
  *
  * @v ibdev		Infiniband device
- * @v gid		Multicast GID
+ * @v av		Address vector
  * @v join		Join (rather than leave) group
  * @v mad		MAD to fill in
  */
-static void ib_mcast_mad ( struct ib_device *ibdev, union ib_gid *gid,
+static void ib_mcast_mad ( struct ib_device *ibdev,
+			   struct ib_address_vector *av,
 			   int join, union ib_mad *mad ) {
 	struct ib_mad_sa *sa = &mad->sa;
 
@@ -61,7 +62,7 @@ static void ib_mcast_mad ( struct ib_device *ibdev, union ib_gid *gid,
 		htonl ( IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
 			IB_SA_MCMEMBER_REC_JOIN_STATE );
 	sa->sa_data.mc_member_record.scope__join_state = 1;
-	memcpy ( &sa->sa_data.mc_member_record.mgid, gid,
+	memcpy ( &sa->sa_data.mc_member_record.mgid, &av->gid,
 		 sizeof ( sa->sa_data.mc_member_record.mgid ) );
 	memcpy ( &sa->sa_data.mc_member_record.port_gid, &ibdev->gid,
 		 sizeof ( sa->sa_data.mc_member_record.port_gid ) );
@@ -75,20 +76,19 @@ static void ib_mcast_mad ( struct ib_device *ibdev, union ib_gid *gid,
  * @v madx		Management transaction
  * @v rc		Status code
  * @v mad		Received MAD (or NULL on error)
- * @v av		Source address vector (or NULL on error)
+ * @v src		Source address vector (or NULL on error)
  */
 static void ib_mcast_complete ( struct ib_device *ibdev,
 				struct ib_mad_interface *mi __unused,
 				struct ib_mad_transaction *madx,
 				int rc, union ib_mad *mad,
-				struct ib_address_vector *av __unused ) {
+				struct ib_address_vector *src __unused ) {
 	struct ib_mc_membership *membership = ib_madx_get_ownerdata ( madx );
 	struct ib_queue_pair *qp = membership->qp;
-	union ib_gid *gid = &membership->gid;
+	struct ib_address_vector *av = membership->av;
 	struct ib_mc_member_record *mc_member_record =
 		&mad->sa.sa_data.mc_member_record;
 	int joined;
-	unsigned long qkey;
 
 	/* Report failures */
 	if ( ( rc == 0 ) && ( mad->hdr.status != htons ( IB_MGMT_STATUS_OK ) ))
@@ -101,13 +101,17 @@ static void ib_mcast_complete ( struct ib_device *ibdev,
 
 	/* Extract values from MAD */
 	joined = ( mad->hdr.method == IB_MGMT_METHOD_GET_RESP );
-	qkey = ntohl ( mc_member_record->qkey );
-	DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %lx\n",
+	av->qkey = ntohl ( mc_member_record->qkey );
+	av->lid = ntohs ( mc_member_record->mlid );
+	av->rate = ( mc_member_record->rate_selector__rate & 0x3f );
+	av->sl = ( ( ntohl ( mc_member_record->sl__flow_label__hop_limit )
+		     >> 28 ) & 0x0f );
+	DBGC ( ibdev, "IBDEV %s QPN %#lx %s " IB_GID_FMT " qkey %#lx\n",
 	       ibdev->name, qp->qpn, ( joined ? "joined" : "left" ),
-	       IB_GID_ARGS ( gid ), qkey );
+	       IB_GID_ARGS ( &av->gid ), av->qkey );
 
 	/* Set queue key */
-	qp->qkey = qkey;
+	qp->qkey = av->qkey;
 	if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not modify qkey: %s\n",
 		       ibdev->name, qp->qpn, strerror ( rc ) );
@@ -120,7 +124,7 @@ static void ib_mcast_complete ( struct ib_device *ibdev,
 	membership->madx = NULL;
 
 	/* Hand off to upper completion handler */
-	membership->complete ( ibdev, qp, membership, rc, mad );
+	membership->complete ( membership, rc );
 }
 
 /** Multicast membership management transaction completion operations */
@@ -134,21 +138,20 @@ static struct ib_mad_transaction_operations ib_mcast_op = {
  * @v ibdev		Infiniband device
  * @v qp		Queue pair
  * @v membership	Multicast group membership
- * @v gid		Multicast GID to join
+ * @v av		Address vector to fill in
  * @v joined		Join completion handler
  * @ret rc		Return status code
  */
 int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
-		    struct ib_mc_membership *membership, union ib_gid *gid,
-		    void ( * complete ) ( struct ib_device *ibdev,
-					  struct ib_queue_pair *qp,
-					  struct ib_mc_membership *membership,
-					  int rc, union ib_mad *mad ) ) {
+		    struct ib_mc_membership *membership,
+		    struct ib_address_vector *av,
+		    void ( * complete ) ( struct ib_mc_membership *membership,
+					  int rc ) ) {
 	union ib_mad mad;
 	int rc;
 
 	DBGC ( ibdev, "IBDEV %s QPN %#lx joining " IB_GID_FMT "\n",
-	       ibdev->name, qp->qpn, IB_GID_ARGS ( gid ) );
+	       ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
 
 	/* Sanity checks */
 	assert ( qp != NULL );
@@ -156,11 +159,11 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 
 	/* Initialise structure */
 	membership->qp = qp;
-	memcpy ( &membership->gid, gid, sizeof ( membership->gid ) );
+	membership->av = av;
 	membership->complete = complete;
 
 	/* Attach queue pair to multicast GID */
-	if ( ( rc = ib_mcast_attach ( ibdev, qp, gid ) ) != 0 ) {
+	if ( ( rc = ib_mcast_attach ( ibdev, qp, &av->gid ) ) != 0 ) {
 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not attach: %s\n",
 		       ibdev->name, qp->qpn, strerror ( rc ) );
 		goto err_mcast_attach;
@@ -168,7 +171,7 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	membership->attached = 1;
 
 	/* Initiate multicast membership join */
-	ib_mcast_mad ( ibdev, gid, 1, &mad );
+	ib_mcast_mad ( ibdev, av, 1, &mad );
 	membership->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
 					    &ib_mcast_op );
 	if ( ! membership->madx ) {
@@ -183,7 +186,7 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 
 	ib_destroy_madx ( ibdev, ibdev->gsi, membership->madx );
  err_create_madx:
-	ib_mcast_detach ( ibdev, qp, gid );
+	ib_mcast_detach ( ibdev, qp, &av->gid );
 	membership->attached = 0;
  err_mcast_attach:
 	return rc;
@@ -198,7 +201,7 @@ int ib_mcast_join ( struct ib_device *ibdev, struct ib_queue_pair *qp,
  */
 void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 		      struct ib_mc_membership *membership ) {
-	union ib_gid *gid = &membership->gid;
+	struct ib_address_vector *av = membership->av;
 	union ib_mad mad;
 	int rc;
 
@@ -207,13 +210,13 @@ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 		return;
 
 	DBGC ( ibdev, "IBDEV %s QPN %#lx leaving " IB_GID_FMT "\n",
-	       ibdev->name, qp->qpn, IB_GID_ARGS ( gid ) );
+	       ibdev->name, qp->qpn, IB_GID_ARGS ( &av->gid ) );
 
 	/* Sanity check */
 	assert ( qp != NULL );
 
 	/* Detach from multicast GID */
-	ib_mcast_detach ( ibdev, qp, &membership->gid );
+	ib_mcast_detach ( ibdev, qp, &av->gid );
 	membership->attached = 0;
 
 	/* Cancel multicast membership join, if applicable */
@@ -223,7 +226,7 @@ void ib_mcast_leave ( struct ib_device *ibdev, struct ib_queue_pair *qp,
 	}
 
 	/* Send a single group leave MAD */
-	ib_mcast_mad ( ibdev, &membership->gid, 0, &mad );
+	ib_mcast_mad ( ibdev, av, 0, &mad );
 	if ( ( rc = ib_mi_send ( ibdev, ibdev->gsi, &mad, NULL ) ) != 0 ) {
 		DBGC ( ibdev, "IBDEV %s QPN %#lx could not send leave request: "
 		       "%s\n", ibdev->name, qp->qpn, strerror ( rc ) );