Browse Source

[infiniband] Add node GUID as distinct from the first port GUID

iPXE currently uses the first port's port GUID as the node GUID,
rather than using the (possibly distinct) real node GUID.  This can
confuse opensm during the handover to a loaded OS: it thinks the port
already belongs to a different node and so discards our port
information with a warning message about duplicate ports.  Everything
is picked up correctly on the second subnet sweep, after opensm has
established that the "old" node no longer exists, but this can delay
link-up unnecessarily by several seconds.

Fix by using the real node GUID.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
tags/v1.20.1
Michael Brown 14 years ago
parent
commit
42cf4a720c

+ 3
- 1
src/drivers/infiniband/linda.c View File

2364
 		goto err_init_i2c;
2364
 		goto err_init_i2c;
2365
 
2365
 
2366
 	/* Read EEPROM parameters */
2366
 	/* Read EEPROM parameters */
2367
-	if ( ( rc = linda_read_eeprom ( linda, &ibdev->gid.s.guid ) ) != 0 )
2367
+	if ( ( rc = linda_read_eeprom ( linda, &ibdev->node_guid ) ) != 0 )
2368
 		goto err_read_eeprom;
2368
 		goto err_read_eeprom;
2369
+	memcpy ( &ibdev->gid.s.guid, &ibdev->node_guid,
2370
+		 sizeof ( ibdev->gid.s.guid ) );
2369
 
2371
 
2370
 	/* Initialise send datapath */
2372
 	/* Initialise send datapath */
2371
 	if ( ( rc = linda_init_send ( linda ) ) != 0 )
2373
 	if ( ( rc = linda_init_send ( linda ) ) != 0 )

+ 2
- 0
src/drivers/infiniband/qib7322.c View File

2357
 			IB_LINK_WIDTH_4X; /* 1x does not work */
2357
 			IB_LINK_WIDTH_4X; /* 1x does not work */
2358
 		ibdev->link_speed_enabled = ibdev->link_speed_supported =
2358
 		ibdev->link_speed_enabled = ibdev->link_speed_supported =
2359
 			IB_LINK_SPEED_SDR; /* to avoid need for link tuning */
2359
 			IB_LINK_SPEED_SDR; /* to avoid need for link tuning */
2360
+		memcpy ( &ibdev->node_guid, &qib7322->guid,
2361
+			 sizeof ( ibdev->node_guid ) );
2360
 		memcpy ( &ibdev->gid.s.guid, &qib7322->guid,
2362
 		memcpy ( &ibdev->gid.s.guid, &qib7322->guid,
2361
 			 sizeof ( ibdev->gid.s.guid ) );
2363
 			 sizeof ( ibdev->gid.s.guid ) );
2362
 		assert ( ( ibdev->gid.s.guid.bytes[7] & i ) == 0 );
2364
 		assert ( ( ibdev->gid.s.guid.bytes[7] & i ) == 0 );

+ 4
- 3
src/include/ipxe/infiniband.h View File

404
 	uint8_t link_speed_enabled;
404
 	uint8_t link_speed_enabled;
405
 	/** Link speed active */
405
 	/** Link speed active */
406
 	uint8_t link_speed_active;
406
 	uint8_t link_speed_active;
407
-	/** Port GID */
407
+	/** Node GUID */
408
+	union ib_guid node_guid;
409
+	/** Port GID (comprising GID prefix and port GUID) */
408
 	union ib_gid gid;
410
 	union ib_gid gid;
409
 	/** Port LID */
411
 	/** Port LID */
410
 	uint16_t lid;
412
 	uint16_t lid;
503
 			     union ib_gid *gid );
505
 			     union ib_gid *gid );
504
 extern void ib_mcast_detach ( struct ib_device *ibdev,
506
 extern void ib_mcast_detach ( struct ib_device *ibdev,
505
 			      struct ib_queue_pair *qp, union ib_gid *gid );
507
 			      struct ib_queue_pair *qp, union ib_gid *gid );
506
-extern int ib_get_hca_info ( struct ib_device *ibdev,
507
-			     union ib_guid *hca_guid );
508
+extern int ib_count_ports ( struct ib_device *ibdev );
508
 extern int ib_set_port_info ( struct ib_device *ibdev, union ib_mad *mad );
509
 extern int ib_set_port_info ( struct ib_device *ibdev, union ib_mad *mad );
509
 extern int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad );
510
 extern int ib_set_pkey_table ( struct ib_device *ibdev, union ib_mad *mad );
510
 extern struct ib_device * alloc_ibdev ( size_t priv_size );
511
 extern struct ib_device * alloc_ibdev ( size_t priv_size );

+ 5
- 11
src/net/infiniband.c View File

761
  */
761
  */
762
 
762
 
763
 /**
763
 /**
764
- * Get Infiniband HCA information
764
+ * Count Infiniband HCA ports
765
  *
765
  *
766
  * @v ibdev		Infiniband device
766
  * @v ibdev		Infiniband device
767
- * @ret hca_guid	HCA GUID
768
  * @ret num_ports	Number of ports
767
  * @ret num_ports	Number of ports
769
  */
768
  */
770
-int ib_get_hca_info ( struct ib_device *ibdev, union ib_guid *hca_guid ) {
769
+int ib_count_ports ( struct ib_device *ibdev ) {
771
 	struct ib_device *tmp;
770
 	struct ib_device *tmp;
772
 	int num_ports = 0;
771
 	int num_ports = 0;
773
 
772
 
774
 	/* Search for IB devices with the same physical device to
773
 	/* Search for IB devices with the same physical device to
775
-	 * identify port count and a suitable Node GUID.
774
+	 * identify port count.
776
 	 */
775
 	 */
777
 	for_each_ibdev ( tmp ) {
776
 	for_each_ibdev ( tmp ) {
778
-		if ( tmp->dev != ibdev->dev )
779
-			continue;
780
-		if ( num_ports == 0 ) {
781
-			memcpy ( hca_guid, &tmp->gid.s.guid,
782
-				 sizeof ( *hca_guid ) );
783
-		}
784
-		num_ports++;
777
+		if ( tmp->dev == ibdev->dev )
778
+			num_ports++;
785
 	}
779
 	}
786
 	return num_ports;
780
 	return num_ports;
787
 }
781
 }

+ 1
- 1
src/net/infiniband/ib_cm.c View File

364
 	req->local_id = htonl ( conn->local_id );
364
 	req->local_id = htonl ( conn->local_id );
365
 	memcpy ( &req->service_id, &conn->service_id,
365
 	memcpy ( &req->service_id, &conn->service_id,
366
 		 sizeof ( req->service_id ) );
366
 		 sizeof ( req->service_id ) );
367
-	ib_get_hca_info ( ibdev, &req->local_ca );
367
+	memcpy ( &req->local_ca, &ibdev->node_guid, sizeof ( req->local_ca ) );
368
 	req->local_qpn__responder_resources = htonl ( ( qp->qpn << 8 ) | 1 );
368
 	req->local_qpn__responder_resources = htonl ( ( qp->qpn << 8 ) | 1 );
369
 	req->local_eecn__initiator_depth = htonl ( ( 0 << 8 ) | 1 );
369
 	req->local_eecn__initiator_depth = htonl ( ( 0 << 8 ) | 1 );
370
 	req->remote_eecn__remote_timeout__service_type__ee_flow_ctrl =
370
 	req->remote_eecn__remote_timeout__service_type__ee_flow_ctrl =

+ 8
- 7
src/net/infiniband/ib_sma.c View File

58
 	node_info->base_version = IB_MGMT_BASE_VERSION;
58
 	node_info->base_version = IB_MGMT_BASE_VERSION;
59
 	node_info->class_version = IB_SMP_CLASS_VERSION;
59
 	node_info->class_version = IB_SMP_CLASS_VERSION;
60
 	node_info->node_type = IB_NODE_TYPE_HCA;
60
 	node_info->node_type = IB_NODE_TYPE_HCA;
61
-	node_info->num_ports = ib_get_hca_info ( ibdev, &node_info->sys_guid );
62
-	memcpy ( &node_info->node_guid, &node_info->sys_guid,
61
+	node_info->num_ports = ib_count_ports ( ibdev );
62
+	memcpy ( &node_info->sys_guid, &ibdev->node_guid,
63
+		 sizeof ( node_info->sys_guid ) );
64
+	memcpy ( &node_info->node_guid, &ibdev->node_guid,
63
 		 sizeof ( node_info->node_guid ) );
65
 		 sizeof ( node_info->node_guid ) );
64
 	memcpy ( &node_info->port_guid, &ibdev->gid.s.guid,
66
 	memcpy ( &node_info->port_guid, &ibdev->gid.s.guid,
65
 		 sizeof ( node_info->port_guid ) );
67
 		 sizeof ( node_info->port_guid ) );
88
 			       union ib_mad *mad,
90
 			       union ib_mad *mad,
89
 			       struct ib_address_vector *av ) {
91
 			       struct ib_address_vector *av ) {
90
 	struct ib_node_desc *node_desc = &mad->smp.smp_data.node_desc;
92
 	struct ib_node_desc *node_desc = &mad->smp.smp_data.node_desc;
91
-	union ib_guid guid;
93
+	union ib_guid *guid = &ibdev->node_guid;
92
 	char hostname[ sizeof ( node_desc->node_string ) ];
94
 	char hostname[ sizeof ( node_desc->node_string ) ];
93
 	int hostname_len;
95
 	int hostname_len;
94
 	int rc;
96
 	int rc;
95
 
97
 
96
 	/* Fill in information */
98
 	/* Fill in information */
97
 	memset ( node_desc, 0, sizeof ( *node_desc ) );
99
 	memset ( node_desc, 0, sizeof ( *node_desc ) );
98
-	ib_get_hca_info ( ibdev, &guid );
99
 	hostname_len = fetch_string_setting ( NULL, &hostname_setting,
100
 	hostname_len = fetch_string_setting ( NULL, &hostname_setting,
100
 					      hostname, sizeof ( hostname ) );
101
 					      hostname, sizeof ( hostname ) );
101
 	snprintf ( node_desc->node_string, sizeof ( node_desc->node_string ),
102
 	snprintf ( node_desc->node_string, sizeof ( node_desc->node_string ),
102
 		   "iPXE %s%s%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x (%s)",
103
 		   "iPXE %s%s%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x (%s)",
103
 		   hostname, ( ( hostname_len >= 0 ) ? " " : "" ),
104
 		   hostname, ( ( hostname_len >= 0 ) ? " " : "" ),
104
-		   guid.bytes[0], guid.bytes[1], guid.bytes[2], guid.bytes[3],
105
-		   guid.bytes[4], guid.bytes[5], guid.bytes[6], guid.bytes[7],
106
-		   ibdev->dev->name );
105
+		   guid->bytes[0], guid->bytes[1], guid->bytes[2],
106
+		   guid->bytes[3], guid->bytes[4], guid->bytes[5],
107
+		   guid->bytes[6], guid->bytes[7], ibdev->dev->name );
107
 
108
 
108
 	/* Send GetResponse */
109
 	/* Send GetResponse */
109
 	mad->hdr.method = IB_MGMT_METHOD_GET_RESP;
110
 	mad->hdr.method = IB_MGMT_METHOD_GET_RESP;

+ 67
- 25
src/net/infiniband/ib_smc.c View File

35
  */
35
  */
36
 
36
 
37
 /**
37
 /**
38
- * Get port information
38
+ * Issue local MAD
39
  *
39
  *
40
  * @v ibdev		Infiniband device
40
  * @v ibdev		Infiniband device
41
+ * @v attr_id		Attribute ID, in network byte order
42
+ * @v attr_mod		Attribute modifier, in network byte order
41
  * @v local_mad		Method for issuing local MADs
43
  * @v local_mad		Method for issuing local MADs
42
  * @v mad		Management datagram to fill in
44
  * @v mad		Management datagram to fill in
43
  * @ret rc		Return status code
45
  * @ret rc		Return status code
44
  */
46
  */
45
-static int ib_smc_get_port_info ( struct ib_device *ibdev,
46
-				  ib_local_mad_t local_mad,
47
-				  union ib_mad *mad ) {
47
+static int ib_smc_mad ( struct ib_device *ibdev, uint16_t attr_id,
48
+			uint32_t attr_mod, ib_local_mad_t local_mad,
49
+			union ib_mad *mad ) {
48
 	int rc;
50
 	int rc;
49
 
51
 
50
 	/* Construct MAD */
52
 	/* Construct MAD */
53
 	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
55
 	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
54
 	mad->hdr.class_version = 1;
56
 	mad->hdr.class_version = 1;
55
 	mad->hdr.method = IB_MGMT_METHOD_GET;
57
 	mad->hdr.method = IB_MGMT_METHOD_GET;
56
-	mad->hdr.attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
57
-	mad->hdr.attr_mod = htonl ( ibdev->port );
58
+	mad->hdr.attr_id = attr_id;
59
+	mad->hdr.attr_mod = attr_mod;
60
+
61
+	/* Issue MAD */
62
+	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 )
63
+		return rc;
64
+
65
+	return 0;
66
+}
67
+
68
+/**
69
+ * Get node information
70
+ *
71
+ * @v ibdev		Infiniband device
72
+ * @v local_mad		Method for issuing local MADs
73
+ * @v mad		Management datagram to fill in
74
+ * @ret rc		Return status code
75
+ */
76
+static int ib_smc_get_node_info ( struct ib_device *ibdev,
77
+				  ib_local_mad_t local_mad,
78
+				  union ib_mad *mad ) {
79
+	int rc;
80
+
81
+	/* Issue MAD */
82
+	if ( ( rc = ib_smc_mad ( ibdev, htons ( IB_SMP_ATTR_NODE_INFO ), 0,
83
+				 local_mad, mad ) ) != 0 ) {
84
+		DBGC ( ibdev, "IBDEV %p could not get node info: %s\n",
85
+		       ibdev, strerror ( rc ) );
86
+		return rc;
87
+	}
88
+	return 0;
89
+}
58
 
90
 
59
-	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
91
+/**
92
+ * Get port information
93
+ *
94
+ * @v ibdev		Infiniband device
95
+ * @v local_mad		Method for issuing local MADs
96
+ * @v mad		Management datagram to fill in
97
+ * @ret rc		Return status code
98
+ */
99
+static int ib_smc_get_port_info ( struct ib_device *ibdev,
100
+				  ib_local_mad_t local_mad,
101
+				  union ib_mad *mad ) {
102
+	int rc;
103
+
104
+	/* Issue MAD */
105
+	if ( ( rc = ib_smc_mad ( ibdev, htons ( IB_SMP_ATTR_PORT_INFO ),
106
+				 htonl ( ibdev->port ), local_mad, mad )) !=0){
60
 		DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
107
 		DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
61
 		       ibdev, strerror ( rc ) );
108
 		       ibdev, strerror ( rc ) );
62
 		return rc;
109
 		return rc;
77
 				  union ib_mad *mad ) {
124
 				  union ib_mad *mad ) {
78
 	int rc;
125
 	int rc;
79
 
126
 
80
-	/* Construct MAD */
81
-	memset ( mad, 0, sizeof ( *mad ) );
82
-	mad->hdr.base_version = IB_MGMT_BASE_VERSION;
83
-	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
84
-	mad->hdr.class_version = 1;
85
-	mad->hdr.method = IB_MGMT_METHOD_GET;
86
-	mad->hdr.attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
87
-
88
-	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
127
+	/* Issue MAD */
128
+	if ( ( rc = ib_smc_mad ( ibdev, htons ( IB_SMP_ATTR_GUID_INFO ), 0,
129
+				 local_mad, mad ) ) != 0 ) {
89
 		DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
130
 		DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
90
 		       ibdev, strerror ( rc ) );
131
 		       ibdev, strerror ( rc ) );
91
 		return rc;
132
 		return rc;
106
 				   union ib_mad *mad ) {
147
 				   union ib_mad *mad ) {
107
 	int rc;
148
 	int rc;
108
 
149
 
109
-	/* Construct MAD */
110
-	memset ( mad, 0, sizeof ( *mad ) );
111
-	mad->hdr.base_version = IB_MGMT_BASE_VERSION;
112
-	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
113
-	mad->hdr.class_version = 1;
114
-	mad->hdr.method = IB_MGMT_METHOD_GET;
115
-	mad->hdr.attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
116
-
117
-	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
150
+	/* Issue MAD */
151
+	if ( ( rc = ib_smc_mad ( ibdev, htons ( IB_SMP_ATTR_PKEY_TABLE ), 0,
152
+				 local_mad, mad ) ) != 0 ) {
118
 		DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
153
 		DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
119
 		       ibdev, strerror ( rc ) );
154
 		       ibdev, strerror ( rc ) );
120
 		return rc;
155
 		return rc;
131
  */
166
  */
132
 static int ib_smc_get ( struct ib_device *ibdev, ib_local_mad_t local_mad ) {
167
 static int ib_smc_get ( struct ib_device *ibdev, ib_local_mad_t local_mad ) {
133
 	union ib_mad mad;
168
 	union ib_mad mad;
169
+	struct ib_node_info *node_info = &mad.smp.smp_data.node_info;
134
 	struct ib_port_info *port_info = &mad.smp.smp_data.port_info;
170
 	struct ib_port_info *port_info = &mad.smp.smp_data.port_info;
135
 	struct ib_guid_info *guid_info = &mad.smp.smp_data.guid_info;
171
 	struct ib_guid_info *guid_info = &mad.smp.smp_data.guid_info;
136
 	struct ib_pkey_table *pkey_table = &mad.smp.smp_data.pkey_table;
172
 	struct ib_pkey_table *pkey_table = &mad.smp.smp_data.pkey_table;
137
 	int rc;
173
 	int rc;
138
 
174
 
175
+	/* Node info gives us the node GUID */
176
+	if ( ( rc = ib_smc_get_node_info ( ibdev, local_mad, &mad ) ) != 0 )
177
+		return rc;
178
+	memcpy ( &ibdev->node_guid, &node_info->node_guid,
179
+		 sizeof ( ibdev->node_guid ) );
180
+
139
 	/* Port info gives us the link state, the first half of the
181
 	/* Port info gives us the link state, the first half of the
140
 	 * port GID and the SM LID.
182
 	 * port GID and the SM LID.
141
 	 */
183
 	 */

Loading…
Cancel
Save