Browse Source

Gets most of the way through initialisation.

tags/v0.9.3
Michael Brown 17 years ago
parent
commit
8423b0b109

+ 9
- 2
src/drivers/net/mlx_ipoib/arbel.h View File

40
 #define ARBEL_HCR_QUERY_FW		0x0004
40
 #define ARBEL_HCR_QUERY_FW		0x0004
41
 #define ARBEL_HCR_INIT_HCA		0x0007
41
 #define ARBEL_HCR_INIT_HCA		0x0007
42
 #define ARBEL_HCR_CLOSE_HCA		0x0008
42
 #define ARBEL_HCR_CLOSE_HCA		0x0008
43
+#define ARBEL_HCR_INIT_IB		0x0009
44
+#define ARBEL_HCR_CLOSE_IB		0x000a
43
 #define ARBEL_HCR_SW2HW_MPT		0x000d
45
 #define ARBEL_HCR_SW2HW_MPT		0x000d
46
+#define ARBEL_HCR_MAP_EQ		0x0012
47
+#define ARBEL_HCR_SW2HW_EQ		0x0013
48
+#define ARBEL_HCR_HW2SW_EQ		0x0014
44
 #define ARBEL_HCR_SW2HW_CQ		0x0016
49
 #define ARBEL_HCR_SW2HW_CQ		0x0016
45
 #define ARBEL_HCR_HW2SW_CQ		0x0017
50
 #define ARBEL_HCR_HW2SW_CQ		0x0017
46
 #define ARBEL_HCR_RST2INIT_QPEE		0x0019
51
 #define ARBEL_HCR_RST2INIT_QPEE		0x0019
68
 /* MTUs */
73
 /* MTUs */
69
 #define ARBEL_MTU_2048			0x04
74
 #define ARBEL_MTU_2048			0x04
70
 
75
 
76
+#define ARBEL_NO_EQ			64
77
+
71
 #define ARBEL_INVALID_LKEY		0x00000100UL
78
 #define ARBEL_INVALID_LKEY		0x00000100UL
72
 
79
 
73
 /*
80
 /*
98
 struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
105
 struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
99
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record );
106
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_arm_db_record );
100
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
107
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
108
+struct MLX_DECLARE_STRUCT ( arbelprm_eqc );
101
 struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
109
 struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
102
 struct MLX_DECLARE_STRUCT ( arbelprm_init_hca );
110
 struct MLX_DECLARE_STRUCT ( arbelprm_init_hca );
111
+struct MLX_DECLARE_STRUCT ( arbelprm_init_ib );
103
 struct MLX_DECLARE_STRUCT ( arbelprm_mad_ifc );
112
 struct MLX_DECLARE_STRUCT ( arbelprm_mad_ifc );
104
 struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry );
113
 struct MLX_DECLARE_STRUCT ( arbelprm_mgm_entry );
105
 struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash );
114
 struct MLX_DECLARE_STRUCT ( arbelprm_mgm_hash );
321
 	 * Used to get unrestricted memory access.
330
 	 * Used to get unrestricted memory access.
322
 	 */
331
 	 */
323
 	unsigned long reserved_lkey;
332
 	unsigned long reserved_lkey;
324
-	/** Event queue number */
325
-	unsigned long eqn;
326
 
333
 
327
 	/** Completion queue in-use bitmask */
334
 	/** Completion queue in-use bitmask */
328
 	arbel_bitmask_t cq_inuse[ ARBEL_BITMASK_SIZE ( ARBEL_MAX_CQS ) ];
335
 	arbel_bitmask_t cq_inuse[ ARBEL_BITMASK_SIZE ( ARBEL_MAX_CQS ) ];

+ 1
- 1
src/drivers/net/mlx_ipoib/ib_mt25218.c View File

811
 	DBG ( "mpt_base_addr_l = %lx\n", icm_start );
811
 	DBG ( "mpt_base_addr_l = %lx\n", icm_start );
812
 	init_hca.mpt_base_addr_l = icm_start;
812
 	init_hca.mpt_base_addr_l = icm_start;
813
 	init_hca.log_mpt_sz = log2_entries;
813
 	init_hca.log_mpt_sz = log2_entries;
814
-	DBG ( "log2_entries for mpt = %d\n", log2_entries );
814
+	DBG ( "log2_entries for mpt = %ld\n", log2_entries );
815
 	icm_start += (tmp << 12);
815
 	icm_start += (tmp << 12);
816
 	icm_size += (tmp << 12);
816
 	icm_size += (tmp << 12);
817
 
817
 

+ 146
- 18
src/drivers/net/mlx_ipoib/mt25218.c View File

30
 #include "arbel.h"
30
 #include "arbel.h"
31
 
31
 
32
 
32
 
33
-
34
-
35
-
33
+/* Port to use */
34
+#define PXE_IB_PORT 1
36
 
35
 
37
 /***************************************************************************
36
 /***************************************************************************
38
  *
37
  *
242
 			   0, NULL, 0, NULL );
241
 			   0, NULL, 0, NULL );
243
 }
242
 }
244
 
243
 
244
+static inline int
245
+arbel_cmd_init_ib ( struct arbel *arbel, unsigned int port,
246
+		    const struct arbelprm_init_ib *init_ib ) {
247
+	return arbel_cmd ( arbel,
248
+			   ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT_IB,
249
+					      1, sizeof ( *init_ib ) ),
250
+			   0, init_ib, port, NULL );
251
+}
252
+
253
+static inline int
254
+arbel_cmd_close_ib ( struct arbel *arbel, unsigned int port ) {
255
+	return arbel_cmd ( arbel,
256
+			   ARBEL_HCR_VOID_CMD ( ARBEL_HCR_CLOSE_IB ),
257
+			   0, NULL, port, NULL );
258
+}
259
+
245
 static inline int
260
 static inline int
246
 arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index,
261
 arbel_cmd_sw2hw_mpt ( struct arbel *arbel, unsigned int index,
247
 		      const struct arbelprm_mpt *mpt ) {
262
 		      const struct arbelprm_mpt *mpt ) {
251
 			   0, mpt, index, NULL );
266
 			   0, mpt, index, NULL );
252
 }
267
 }
253
 
268
 
269
+static inline int
270
+arbel_cmd_sw2hw_eq ( struct arbel *arbel, unsigned int index,
271
+		     const struct arbelprm_eqc *eqc ) {
272
+	return arbel_cmd ( arbel,
273
+			   ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_EQ,
274
+					      1, sizeof ( *eqc ) ),
275
+			   0, eqc, index, NULL );
276
+}
277
+
278
+static inline int
279
+arbel_cmd_hw2sw_eq ( struct arbel *arbel, unsigned int index ) {
280
+	return arbel_cmd ( arbel,
281
+			   ARBEL_HCR_VOID_CMD ( ARBEL_HCR_HW2SW_EQ ),
282
+			   1, NULL, index, NULL );
283
+}
284
+
254
 static inline int
285
 static inline int
255
 arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
286
 arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
256
 		     const struct arbelprm_completion_queue_context *cqctx ) {
287
 		     const struct arbelprm_completion_queue_context *cqctx ) {
421
 			   0, map, 1, NULL );
452
 			   0, map, 1, NULL );
422
 }
453
 }
423
 
454
 
424
-/***************************************************************************
425
- *
426
- * Event queue operations
427
- *
428
- ***************************************************************************
429
- */
430
-
431
-static int arbel_create_eq ( struct arbel *arbel ) {
432
-}
433
-
434
 /***************************************************************************
455
 /***************************************************************************
435
  *
456
  *
436
  * Completion queue operations
457
  * Completion queue operations
508
 	MLX_FILL_2 ( &cqctx, 3,
529
 	MLX_FILL_2 ( &cqctx, 3,
509
 		     usr_page, arbel->limits.reserved_uars,
530
 		     usr_page, arbel->limits.reserved_uars,
510
 		     log_cq_size, fls ( cq->num_cqes - 1 ) );
531
 		     log_cq_size, fls ( cq->num_cqes - 1 ) );
511
-	MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eqn );
532
+	MLX_FILL_1 ( &cqctx, 5, c_eqn, ARBEL_NO_EQ );
512
 	MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
533
 	MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
513
 	MLX_FILL_1 ( &cqctx, 7, l_key, arbel->reserved_lkey );
534
 	MLX_FILL_1 ( &cqctx, 7, l_key, arbel->reserved_lkey );
514
 	MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn );
535
 	MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn );
1423
 	return 0;
1444
 	return 0;
1424
 }
1445
 }
1425
 
1446
 
1447
+/**
1448
+ * Wait for link up
1449
+ *
1450
+ * @v arbel		Arbel device
1451
+ * @ret rc		Return status code
1452
+ *
1453
+ * This function shouldn't really exist.  Unfortunately, IB links take
1454
+ * a long time to come up, and we can't get various key parameters
1455
+ * e.g. our own IPoIB MAC address without information from the subnet
1456
+ * manager).  We should eventually make link-up an asynchronous event.
1457
+ */
1458
+static int arbel_wait_for_link ( struct arbel *arbel ) {
1459
+	struct ib_mad_port_info port_info;
1460
+	unsigned int retries;
1461
+	int rc;
1462
+
1463
+	printf ( "Waiting for Infiniband link-up..." );
1464
+	for ( retries = 20 ; retries ; retries-- ) {
1465
+		if ( ( rc = arbel_get_port_info ( arbel, &port_info ) ) != 0 )
1466
+			continue;
1467
+		if ( ( ( port_info.port_state__link_speed_supported ) & 0xf )
1468
+		     == 4 ) {
1469
+			printf ( "ok\n" );
1470
+			return 0;
1471
+		}
1472
+		printf ( "." );
1473
+		sleep ( 1 );
1474
+	}
1475
+	printf ( "failed\n" );
1476
+	return -ENODEV;
1477
+};
1478
+
1426
 /**
1479
 /**
1427
  * Get MAD parameters
1480
  * Get MAD parameters
1428
  *
1481
  *
1819
 	arbel->icm = UNULL;
1872
 	arbel->icm = UNULL;
1820
 }
1873
 }
1821
 
1874
 
1875
+/***************************************************************************
1876
+ *
1877
+ * Infiniband link-layer operations
1878
+ *
1879
+ ***************************************************************************
1880
+ */
1881
+
1882
+/**
1883
+ * Initialise Infiniband link
1884
+ *
1885
+ * @v arbel		Arbel device
1886
+ * @ret rc		Return status code
1887
+ */
1888
+static int arbel_init_ib ( struct arbel *arbel ) {
1889
+	struct arbelprm_init_ib init_ib;
1890
+	int rc;
1891
+
1892
+	memset ( &init_ib, 0, sizeof ( init_ib ) );
1893
+	MLX_FILL_3 ( &init_ib, 0,
1894
+		     mtu_cap, ARBEL_MTU_2048,
1895
+		     port_width_cap, 3,
1896
+		     vl_cap, 1 );
1897
+	MLX_FILL_1 ( &init_ib, 2, max_pkey, 64 );
1898
+	if ( ( rc = arbel_cmd_init_ib ( arbel, PXE_IB_PORT,
1899
+					&init_ib ) ) != 0 ) {
1900
+		DBGC ( arbel, "Arbel %p could not intialise IB: %s\n",
1901
+		       arbel, strerror ( rc ) );
1902
+		return rc;
1903
+	}
1904
+
1905
+	return 0;
1906
+}
1907
+
1908
+/**
1909
+ * Close Infiniband link
1910
+ *
1911
+ * @v arbel		Arbel device
1912
+ */
1913
+static void arbel_close_ib ( struct arbel *arbel ) {
1914
+	int rc;
1915
+
1916
+	if ( ( rc = arbel_cmd_close_ib ( arbel, PXE_IB_PORT ) ) != 0 ) {
1917
+		DBGC ( arbel, "Arbel %p could not close IB: %s\n",
1918
+		       arbel, strerror ( rc ) );
1919
+		/* Nothing we can do about this */
1920
+	}
1921
+}
1922
+
1822
 /***************************************************************************
1923
 /***************************************************************************
1823
  *
1924
  *
1824
  * PCI interface
1925
  * PCI interface
1933
 	if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
2034
 	if ( ( rc = arbel_alloc_icm ( arbel, &init_hca ) ) != 0 )
1934
 		goto err_alloc_icm;
2035
 		goto err_alloc_icm;
1935
 
2036
 
2037
+	
2038
+	unsigned long uar_offset = ( arbel->limits.reserved_uars * 4096 );
2039
+	arbel->db_rec = phys_to_virt ( user_to_phys ( arbel->icm,
2040
+						      uar_offset ) );
2041
+	memset ( arbel->db_rec, 0, 4096 );
2042
+	union arbelprm_doorbell_record *db_rec;
2043
+	db_rec = &arbel->db_rec[ARBEL_GROUP_SEPARATOR_DOORBELL];
2044
+	MLX_FILL_1 ( &db_rec->qp, 1, res, ARBEL_UAR_RES_GROUP_SEP );
2045
+
2046
+
1936
 	/* Initialise HCA */
2047
 	/* Initialise HCA */
1937
 	MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 1 );
2048
 	MLX_FILL_1 ( &init_hca, 74, uar_parameters.log_max_uars, 1 );
1938
 	if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
2049
 	if ( ( rc = arbel_cmd_init_hca ( arbel, &init_hca ) ) != 0 ) {
1945
 	if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
2056
 	if ( ( rc = arbel_setup_mpt ( arbel ) ) != 0 )
1946
 		goto err_setup_mpt;
2057
 		goto err_setup_mpt;
1947
 
2058
 
1948
-		     
2059
+	/* Bring up IB layer */
2060
+	if ( ( rc = arbel_init_ib ( arbel ) ) != 0 )
2061
+		goto err_init_ib;
2062
+
2063
+	/* Wait for link */
2064
+	if ( ( rc = arbel_wait_for_link ( arbel ) ) != 0 )
2065
+		goto err_wait_for_link;
2066
+	
1949
 #endif
2067
 #endif
1950
 
2068
 
1951
 
2069
 
1957
 	arbel->mailbox_in = dev_buffers_p->inprm_buf;
2075
 	arbel->mailbox_in = dev_buffers_p->inprm_buf;
1958
 	arbel->mailbox_out = dev_buffers_p->outprm_buf;
2076
 	arbel->mailbox_out = dev_buffers_p->outprm_buf;
1959
 #endif
2077
 #endif
1960
-#if ! SELF_INIT
2078
+#if SELF_INIT
2079
+#else
1961
 	arbel->reserved_lkey = dev_ib_data.mkey;
2080
 	arbel->reserved_lkey = dev_ib_data.mkey;
1962
-#endif
1963
 	arbel->db_rec = dev_ib_data.uar_context_base;
2081
 	arbel->db_rec = dev_ib_data.uar_context_base;
1964
-	arbel->eqn = dev_ib_data.eq.eqn;
2082
+#endif
2083
+	//	arbel->eqn = dev_ib_data.eq.eqn;
1965
 
2084
 
1966
 
2085
 
1967
 	/* Get MAD parameters */
2086
 	/* Get MAD parameters */
1968
 	if ( ( rc = arbel_get_mad_params ( ibdev ) ) != 0 )
2087
 	if ( ( rc = arbel_get_mad_params ( ibdev ) ) != 0 )
1969
 		goto err_get_mad_params;
2088
 		goto err_get_mad_params;
1970
 
2089
 
2090
+	DBGC ( arbel, "Arbel %p port GID is %08lx:%08lx:%08lx:%08lx\n", arbel,
2091
+	       htonl ( ibdev->port_gid.u.dwords[0] ),
2092
+	       htonl ( ibdev->port_gid.u.dwords[1] ),
2093
+	       htonl ( ibdev->port_gid.u.dwords[2] ),
2094
+	       htonl ( ibdev->port_gid.u.dwords[3] ) );
2095
+
1971
 	/* Add IPoIB device */
2096
 	/* Add IPoIB device */
1972
 	if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
2097
 	if ( ( rc = ipoib_probe ( ibdev ) ) != 0 ) {
1973
 		DBGC ( arbel, "Arbel %p could not add IPoIB device: %s\n",
2098
 		DBGC ( arbel, "Arbel %p could not add IPoIB device: %s\n",
1982
 	ib_driver_close ( 0 );
2107
 	ib_driver_close ( 0 );
1983
  err_ib_driver_init:
2108
  err_ib_driver_init:
1984
 
2109
 
2110
+ err_wait_for_link:
2111
+	arbel_close_ib ( arbel );
2112
+ err_init_ib:
1985
  err_setup_mpt:
2113
  err_setup_mpt:
1986
 	arbel_cmd_close_hca ( arbel );
2114
 	arbel_cmd_close_hca ( arbel );
1987
  err_init_hca:
2115
  err_init_hca:

Loading…
Cancel
Save