Você não pode selecionar mais de 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.

mt25218.c 40KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422
  1. /**************************************************************************
  2. Etherboot - BOOTP/TFTP Bootstrap Program
  3. Skeleton NIC driver for Etherboot
  4. ***************************************************************************/
  5. /*
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public License as
  8. * published by the Free Software Foundation; either version 2, or (at
  9. * your option) any later version.
  10. */
  11. #include <errno.h>
  12. #include <gpxe/pci.h>
  13. #include <gpxe/malloc.h>
  14. #include <gpxe/iobuf.h>
  15. #include <gpxe/netdevice.h>
  16. #include <gpxe/infiniband.h>
  17. /* to get some global routines like printf */
  18. #include "etherboot.h"
  19. /* to get the interface to the body of the program */
  20. #include "nic.h"
  21. #define CREATE_OWN 1
  22. #include "mt25218_imp.c"
  23. #include "arbel.h"
  24. static const struct ib_gid arbel_no_gid = {
  25. { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }
  26. };
  27. #define MLX_RX_MAX_FILL NUM_IPOIB_RCV_WQES
  28. struct mlx_nic {
  29. #if ! CREATE_OWN
  30. /** Queue pair handle */
  31. udqp_t ipoib_qph;
  32. /** Send completion queue */
  33. cq_t snd_cqh;
  34. /** Receive completion queue */
  35. cq_t rcv_cqh;
  36. #endif
  37. /** Broadcast Address Vector */
  38. ud_av_t bcast_av;
  39. /** RX fill level */
  40. unsigned int rx_fill;
  41. #if CREATE_OWN
  42. struct ib_completion_queue *own_send_cq;
  43. struct ib_completion_queue *own_recv_cq;
  44. struct ib_queue_pair *own_qp;
  45. #endif
  46. };
  47. static struct io_buffer *static_ipoib_tx_ring[NUM_IPOIB_SND_WQES];
  48. static struct io_buffer *static_ipoib_rx_ring[NUM_IPOIB_RCV_WQES];
  49. static struct arbel static_arbel;
  50. #if ! CREATE_OWN
  51. static struct arbel_completion_queue static_arbel_ipoib_send_cq = {
  52. .ci_doorbell_idx = IPOIB_SND_CQ_CI_DB_IDX,
  53. };
  54. static struct ib_completion_queue static_ipoib_send_cq = {
  55. .cqn = 1234, /* Only used for debug messages */
  56. .num_cqes = NUM_IPOIB_SND_CQES,
  57. .work_queues = LIST_HEAD_INIT ( static_ipoib_send_cq.work_queues ),
  58. .dev_priv = &static_arbel_ipoib_send_cq,
  59. };
  60. static struct arbel_completion_queue static_arbel_ipoib_recv_cq = {
  61. .ci_doorbell_idx = IPOIB_RCV_CQ_CI_DB_IDX,
  62. };
  63. static struct ib_completion_queue static_ipoib_recv_cq = {
  64. .cqn = 2345, /* Only used for debug messages */
  65. .num_cqes = NUM_IPOIB_RCV_CQES,
  66. .work_queues = LIST_HEAD_INIT ( static_ipoib_recv_cq.work_queues ),
  67. .dev_priv = &static_arbel_ipoib_recv_cq,
  68. };
  69. static struct arbel_queue_pair static_arbel_ipoib_qp = {
  70. .send = {
  71. .doorbell_idx = IPOIB_SND_QP_DB_IDX,
  72. },
  73. .recv = {
  74. .doorbell_idx = IPOIB_RCV_QP_DB_IDX,
  75. },
  76. };
  77. static struct ib_queue_pair static_ipoib_qp = {
  78. .send = {
  79. .qp = &static_ipoib_qp,
  80. .is_send = 1,
  81. .cq = &static_ipoib_send_cq,
  82. .num_wqes = NUM_IPOIB_SND_WQES,
  83. .iobufs = static_ipoib_tx_ring,
  84. .list = LIST_HEAD_INIT (static_ipoib_qp.send.list),
  85. .dev_priv = &static_arbel_ipoib_qp.send,
  86. },
  87. .recv = {
  88. .qp = &static_ipoib_qp,
  89. .is_send = 0,
  90. .cq = &static_ipoib_recv_cq,
  91. .num_wqes = NUM_IPOIB_RCV_WQES,
  92. .iobufs = static_ipoib_rx_ring,
  93. .list = LIST_HEAD_INIT (static_ipoib_qp.recv.list),
  94. .dev_priv = &static_arbel_ipoib_qp.recv,
  95. },
  96. .dev_priv = &static_arbel_ipoib_qp,
  97. };
  98. #endif
  99. static struct ib_device static_ibdev = {
  100. .dev_priv = &static_arbel,
  101. };
  102. /**
  103. * Open network device
  104. *
  105. * @v netdev Network device
  106. * @ret rc Return status code
  107. */
  108. static int mlx_open ( struct net_device *netdev ) {
  109. ( void ) netdev;
  110. return 0;
  111. }
  112. /**
  113. * Close network device
  114. *
  115. * @v netdev Network device
  116. */
  117. static void mlx_close ( struct net_device *netdev ) {
  118. ( void ) netdev;
  119. }
  120. static int arbel_post_send ( struct ib_device *ibdev,
  121. struct ib_queue_pair *qp,
  122. struct ib_address_vector *av,
  123. struct io_buffer *iobuf );
  124. static int mlx_transmit_direct ( struct net_device *netdev,
  125. struct io_buffer *iobuf ) {
  126. struct mlx_nic *mlx = netdev->priv;
  127. int rc;
  128. struct ud_av_st *bcast_av = mlx->bcast_av;
  129. struct arbelprm_ud_address_vector *bav =
  130. ( struct arbelprm_ud_address_vector * ) &bcast_av->av;
  131. struct ib_address_vector av = {
  132. .dest_qp = bcast_av->dest_qp,
  133. .qkey = bcast_av->qkey,
  134. .dlid = MLX_GET ( bav, rlid ),
  135. .rate = ( MLX_GET ( bav, max_stat_rate ) ? 1 : 4 ),
  136. .sl = MLX_GET ( bav, sl ),
  137. .gid_present = 1,
  138. };
  139. memcpy ( &av.gid, ( ( void * ) bav ) + 16, 16 );
  140. rc = arbel_post_send ( &static_ibdev,
  141. #if CREATE_OWN
  142. mlx->own_qp,
  143. #else
  144. &static_ipoib_qp,
  145. #endif
  146. &av, iobuf );
  147. return rc;
  148. }
  149. static void arbel_poll_cq ( struct ib_device *ibdev,
  150. struct ib_completion_queue *cq,
  151. ib_completer_t complete_send,
  152. ib_completer_t complete_recv );
  153. static void temp_complete_send ( struct ib_device *ibdev __unused,
  154. struct ib_queue_pair *qp,
  155. struct ib_completion *completion,
  156. struct io_buffer *iobuf ) {
  157. struct net_device *netdev = qp->owner_priv;
  158. DBG ( "Wahey! TX completion\n" );
  159. netdev_tx_complete_err ( netdev, iobuf,
  160. ( completion->syndrome ? -EIO : 0 ) );
  161. }
  162. static void temp_complete_recv ( struct ib_device *ibdev __unused,
  163. struct ib_queue_pair *qp,
  164. struct ib_completion *completion,
  165. struct io_buffer *iobuf ) {
  166. struct net_device *netdev = qp->owner_priv;
  167. struct mlx_nic *mlx = netdev->priv;
  168. DBG ( "Yay! RX completion on %p len %zx:\n", iobuf, completion->len );
  169. if ( completion->syndrome ) {
  170. netdev_rx_err ( netdev, iobuf, -EIO );
  171. } else {
  172. iob_put ( iobuf, completion->len );
  173. iob_pull ( iobuf, sizeof ( struct ib_global_route_header ) );
  174. netdev_rx ( netdev, iobuf );
  175. }
  176. mlx->rx_fill--;
  177. }
  178. static int arbel_post_recv ( struct ib_device *ibdev,
  179. struct ib_queue_pair *qp,
  180. struct io_buffer *iobuf );
  181. static void mlx_refill_rx ( struct net_device *netdev ) {
  182. struct mlx_nic *mlx = netdev->priv;
  183. struct io_buffer *iobuf;
  184. int rc;
  185. while ( mlx->rx_fill < MLX_RX_MAX_FILL ) {
  186. iobuf = alloc_iob ( 2048 );
  187. if ( ! iobuf )
  188. break;
  189. DBG ( "Posting RX buffer %p:\n", iobuf );
  190. if ( ( rc = arbel_post_recv ( &static_ibdev,
  191. #if CREATE_OWN
  192. mlx->own_qp,
  193. #else
  194. &static_ipoib_qp,
  195. #endif
  196. iobuf ) ) != 0 ) {
  197. free_iob ( iobuf );
  198. break;
  199. }
  200. mlx->rx_fill++;
  201. }
  202. }
  203. /**
  204. * Poll for completed and received packets
  205. *
  206. * @v netdev Network device
  207. */
  208. static void mlx_poll ( struct net_device *netdev ) {
  209. struct mlx_nic *mlx = netdev->priv;
  210. int rc;
  211. if ( ( rc = poll_error_buf() ) != 0 ) {
  212. DBG ( "poll_error_buf() failed: %s\n", strerror ( rc ) );
  213. return;
  214. }
  215. /* Drain event queue. We can ignore events, since we're going
  216. * to just poll all completion queues anyway.
  217. */
  218. if ( ( rc = drain_eq() ) != 0 ) {
  219. DBG ( "drain_eq() failed: %s\n", strerror ( rc ) );
  220. return;
  221. }
  222. /* Poll completion queues */
  223. arbel_poll_cq ( &static_ibdev,
  224. #if CREATE_OWN
  225. mlx->own_send_cq,
  226. #else
  227. &static_ipoib_send_cq,
  228. #endif
  229. temp_complete_send, temp_complete_recv );
  230. arbel_poll_cq ( &static_ibdev,
  231. #if CREATE_OWN
  232. mlx->own_recv_cq,
  233. #else
  234. &static_ipoib_recv_cq,
  235. #endif
  236. temp_complete_send, temp_complete_recv );
  237. mlx_refill_rx ( netdev );
  238. }
  239. /**
  240. * Enable or disable interrupts
  241. *
  242. * @v netdev Network device
  243. * @v enable Interrupts should be enabled
  244. */
  245. static void mlx_irq ( struct net_device *netdev, int enable ) {
  246. ( void ) netdev;
  247. ( void ) enable;
  248. }
  249. static struct net_device_operations mlx_operations = {
  250. .open = mlx_open,
  251. .close = mlx_close,
  252. .transmit = mlx_transmit_direct,
  253. .poll = mlx_poll,
  254. .irq = mlx_irq,
  255. };
  256. /***************************************************************************
  257. *
  258. * Queue number allocation
  259. *
  260. ***************************************************************************
  261. */
  262. /**
  263. * Allocate queue number
  264. *
  265. * @v q_inuse Queue usage bitmask
  266. * @v max_inuse Maximum number of in-use queues
  267. * @ret qn_offset Free queue number offset, or negative error
  268. */
  269. static int arbel_alloc_qn_offset ( arbel_bitmask_t *q_inuse,
  270. unsigned int max_inuse ) {
  271. unsigned int qn_offset = 0;
  272. arbel_bitmask_t mask = 1;
  273. while ( qn_offset < max_inuse ) {
  274. if ( ( mask & *q_inuse ) == 0 ) {
  275. *q_inuse |= mask;
  276. return qn_offset;
  277. }
  278. qn_offset++;
  279. mask <<= 1;
  280. if ( ! mask ) {
  281. mask = 1;
  282. q_inuse++;
  283. }
  284. }
  285. return -ENFILE;
  286. }
  287. /**
  288. * Free queue number
  289. *
  290. * @v q_inuse Queue usage bitmask
  291. * @v qn_offset Queue number offset
  292. */
  293. static void arbel_free_qn_offset ( arbel_bitmask_t *q_inuse, int qn_offset ) {
  294. arbel_bitmask_t mask;
  295. mask = ( 1 << ( qn_offset % ( 8 * sizeof ( mask ) ) ) );
  296. q_inuse += ( qn_offset / ( 8 * sizeof ( mask ) ) );
  297. *q_inuse &= ~mask;
  298. }
  299. /***************************************************************************
  300. *
  301. * HCA commands
  302. *
  303. ***************************************************************************
  304. */
  305. /**
  306. * Wait for Arbel command completion
  307. *
  308. * @v arbel Arbel device
  309. * @ret rc Return status code
  310. */
  311. static int arbel_cmd_wait ( struct arbel *arbel,
  312. struct arbelprm_hca_command_register *hcr ) {
  313. unsigned int wait;
  314. for ( wait = ARBEL_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
  315. hcr->u.dwords[6] =
  316. readl ( arbel->config + ARBEL_HCR_REG ( 6 ) );
  317. if ( MLX_GET ( hcr, go ) == 0 )
  318. return 0;
  319. mdelay ( 1 );
  320. }
  321. return -EBUSY;
  322. }
  323. /**
  324. * Issue HCA command
  325. *
  326. * @v arbel Arbel device
  327. * @v command Command opcode, flags and input/output lengths
  328. * @v op_mod Opcode modifier (0 if no modifier applicable)
  329. * @v in Input parameters
  330. * @v in_mod Input modifier (0 if no modifier applicable)
  331. * @v out Output parameters
  332. * @ret rc Return status code
  333. */
  334. static int arbel_cmd ( struct arbel *arbel, unsigned long command,
  335. unsigned int op_mod, const void *in,
  336. unsigned int in_mod, void *out ) {
  337. struct arbelprm_hca_command_register hcr;
  338. unsigned int opcode = ARBEL_HCR_OPCODE ( command );
  339. size_t in_len = ARBEL_HCR_IN_LEN ( command );
  340. size_t out_len = ARBEL_HCR_OUT_LEN ( command );
  341. void *in_buffer;
  342. void *out_buffer;
  343. unsigned int status;
  344. unsigned int i;
  345. int rc;
  346. DBGC ( arbel, "Arbel %p command %02x in %zx%s out %zx%s\n",
  347. arbel, opcode, in_len,
  348. ( ( command & ARBEL_HCR_IN_MBOX ) ? "(mbox)" : "" ), out_len,
  349. ( ( command & ARBEL_HCR_OUT_MBOX ) ? "(mbox)" : "" ) );
  350. /* Check that HCR is free */
  351. if ( ( rc = arbel_cmd_wait ( arbel, &hcr ) ) != 0 ) {
  352. DBGC ( arbel, "Arbel %p command interface locked\n", arbel );
  353. return rc;
  354. }
  355. /* Prepare HCR */
  356. memset ( &hcr, 0, sizeof ( hcr ) );
  357. in_buffer = &hcr.u.dwords[0];
  358. if ( in_len && ( command & ARBEL_HCR_IN_MBOX ) ) {
  359. in_buffer = arbel->mailbox_in;
  360. MLX_FILL_1 ( &hcr, 1, in_param_l, virt_to_bus ( in_buffer ) );
  361. }
  362. memcpy ( in_buffer, in, in_len );
  363. MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
  364. out_buffer = &hcr.u.dwords[3];
  365. if ( out_len && ( command & ARBEL_HCR_OUT_MBOX ) ) {
  366. out_buffer = arbel->mailbox_out;
  367. MLX_FILL_1 ( &hcr, 4, out_param_l,
  368. virt_to_bus ( out_buffer ) );
  369. }
  370. MLX_FILL_3 ( &hcr, 6,
  371. opcode, opcode,
  372. opcode_modifier, op_mod,
  373. go, 1 );
  374. DBG_HD ( &hcr, sizeof ( hcr ) );
  375. if ( in_len ) {
  376. size_t dump_len = in_len;
  377. if ( dump_len > 256 )
  378. dump_len = 256;
  379. DBG ( "Input:\n" );
  380. DBG_HD ( in, dump_len );
  381. }
  382. /* Issue command */
  383. for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
  384. i++ ) {
  385. writel ( hcr.u.dwords[i],
  386. arbel->config + ARBEL_HCR_REG ( i ) );
  387. barrier();
  388. }
  389. /* Wait for command completion */
  390. if ( ( rc = arbel_cmd_wait ( arbel, &hcr ) ) != 0 ) {
  391. DBGC ( arbel, "Arbel %p timed out waiting for command:\n",
  392. arbel );
  393. DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
  394. return rc;
  395. }
  396. /* Check command status */
  397. status = MLX_GET ( &hcr, status );
  398. if ( status != 0 ) {
  399. DBGC ( arbel, "Arbel %p command failed with status %02x:\n",
  400. arbel, status );
  401. DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
  402. return -EIO;
  403. }
  404. /* Read output parameters, if any */
  405. hcr.u.dwords[3] = readl ( arbel->config + ARBEL_HCR_REG ( 3 ) );
  406. hcr.u.dwords[4] = readl ( arbel->config + ARBEL_HCR_REG ( 4 ) );
  407. memcpy ( out, out_buffer, out_len );
  408. if ( out_len ) {
  409. size_t dump_len = out_len;
  410. if ( dump_len > 256 )
  411. dump_len = 256;
  412. DBG ( "Output:\n" );
  413. DBG_HD ( out, dump_len );
  414. }
  415. return 0;
  416. }
  417. static inline int
  418. arbel_cmd_query_dev_lim ( struct arbel *arbel,
  419. struct arbelprm_query_dev_lim *dev_lim ) {
  420. return arbel_cmd ( arbel,
  421. ARBEL_HCR_OUT_CMD ( ARBEL_HCR_QUERY_DEV_LIM,
  422. 1, sizeof ( *dev_lim ) ),
  423. 0, NULL, 0, dev_lim );
  424. }
  425. static inline int
  426. arbel_cmd_sw2hw_cq ( struct arbel *arbel, unsigned long cqn,
  427. const struct arbelprm_completion_queue_context *cqctx ) {
  428. return arbel_cmd ( arbel,
  429. ARBEL_HCR_IN_CMD ( ARBEL_HCR_SW2HW_CQ,
  430. 1, sizeof ( *cqctx ) ),
  431. 0, cqctx, cqn, NULL );
  432. }
  433. static inline int
  434. arbel_cmd_hw2sw_cq ( struct arbel *arbel, unsigned long cqn ) {
  435. return arbel_cmd ( arbel,
  436. ARBEL_HCR_VOID_CMD ( ARBEL_HCR_HW2SW_CQ ),
  437. 1, NULL, cqn, NULL );
  438. }
  439. static inline int
  440. arbel_cmd_rst2init_qpee ( struct arbel *arbel, unsigned long qpn,
  441. const struct arbelprm_qp_ee_state_transitions *ctx ){
  442. return arbel_cmd ( arbel,
  443. ARBEL_HCR_IN_CMD ( ARBEL_HCR_RST2INIT_QPEE,
  444. 1, sizeof ( *ctx ) ),
  445. 0, ctx, qpn, NULL );
  446. }
  447. static inline int
  448. arbel_cmd_init2rtr_qpee ( struct arbel *arbel, unsigned long qpn,
  449. const struct arbelprm_qp_ee_state_transitions *ctx ){
  450. return arbel_cmd ( arbel,
  451. ARBEL_HCR_IN_CMD ( ARBEL_HCR_INIT2RTR_QPEE,
  452. 1, sizeof ( *ctx ) ),
  453. 0, ctx, qpn, NULL );
  454. }
  455. static inline int
  456. arbel_cmd_rtr2rts_qpee ( struct arbel *arbel, unsigned long qpn,
  457. const struct arbelprm_qp_ee_state_transitions *ctx ) {
  458. return arbel_cmd ( arbel,
  459. ARBEL_HCR_IN_CMD ( ARBEL_HCR_RTR2RTS_QPEE,
  460. 1, sizeof ( *ctx ) ),
  461. 0, ctx, qpn, NULL );
  462. }
  463. static inline int
  464. arbel_cmd_2rst_qpee ( struct arbel *arbel, unsigned long qpn ) {
  465. return arbel_cmd ( arbel,
  466. ARBEL_HCR_VOID_CMD ( ARBEL_HCR_2RST_QPEE ),
  467. 0x03, NULL, qpn, NULL );
  468. }
  469. /***************************************************************************
  470. *
  471. * Completion queue operations
  472. *
  473. ***************************************************************************
  474. */
  475. /**
  476. * Create completion queue
  477. *
  478. * @v ibdev Infiniband device
  479. * @v cq Completion queue
  480. * @ret rc Return status code
  481. */
  482. static int arbel_create_cq ( struct ib_device *ibdev,
  483. struct ib_completion_queue *cq ) {
  484. struct arbel *arbel = ibdev->dev_priv;
  485. struct arbel_completion_queue *arbel_cq;
  486. struct arbelprm_completion_queue_context cqctx;
  487. struct arbelprm_cq_ci_db_record *ci_db_rec;
  488. struct arbelprm_cq_arm_db_record *arm_db_rec;
  489. int cqn_offset;
  490. unsigned int i;
  491. int rc;
  492. /* Find a free completion queue number */
  493. cqn_offset = arbel_alloc_qn_offset ( arbel->cq_inuse, ARBEL_MAX_CQS );
  494. if ( cqn_offset < 0 ) {
  495. DBGC ( arbel, "Arbel %p out of completion queues\n", arbel );
  496. rc = cqn_offset;
  497. goto err_cqn_offset;
  498. }
  499. cq->cqn = ( arbel->limits.reserved_cqs + cqn_offset );
  500. /* Allocate control structures */
  501. arbel_cq = zalloc ( sizeof ( *arbel_cq ) );
  502. if ( ! arbel_cq ) {
  503. rc = -ENOMEM;
  504. goto err_arbel_cq;
  505. }
  506. arbel_cq->ci_doorbell_idx = arbel_cq_ci_doorbell_idx ( cqn_offset );
  507. arbel_cq->arm_doorbell_idx = arbel_cq_arm_doorbell_idx ( cqn_offset );
  508. /* Allocate completion queue itself */
  509. arbel_cq->cqe_size = ( cq->num_cqes * sizeof ( arbel_cq->cqe[0] ) );
  510. arbel_cq->cqe = malloc_dma ( arbel_cq->cqe_size,
  511. sizeof ( arbel_cq->cqe[0] ) );
  512. if ( ! arbel_cq->cqe ) {
  513. rc = -ENOMEM;
  514. goto err_cqe;
  515. }
  516. memset ( arbel_cq->cqe, 0, arbel_cq->cqe_size );
  517. for ( i = 0 ; i < cq->num_cqes ; i++ ) {
  518. MLX_FILL_1 ( &arbel_cq->cqe[i].normal, 7, owner, 1 );
  519. }
  520. barrier();
  521. /* Initialise doorbell records */
  522. ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
  523. MLX_FILL_1 ( ci_db_rec, 0, counter, 0 );
  524. MLX_FILL_2 ( ci_db_rec, 1,
  525. res, ARBEL_UAR_RES_CQ_CI,
  526. cq_number, cq->cqn );
  527. arm_db_rec = &arbel->db_rec[arbel_cq->arm_doorbell_idx].cq_arm;
  528. MLX_FILL_1 ( arm_db_rec, 0, counter, 0 );
  529. MLX_FILL_2 ( arm_db_rec, 1,
  530. res, ARBEL_UAR_RES_CQ_ARM,
  531. cq_number, cq->cqn );
  532. /* Hand queue over to hardware */
  533. memset ( &cqctx, 0, sizeof ( cqctx ) );
  534. MLX_FILL_1 ( &cqctx, 0, st, 0xa /* "Event fired" */ );
  535. MLX_FILL_1 ( &cqctx, 2, start_address_l,
  536. virt_to_bus ( arbel_cq->cqe ) );
  537. MLX_FILL_2 ( &cqctx, 3,
  538. usr_page, arbel->limits.reserved_uars,
  539. log_cq_size, fls ( cq->num_cqes - 1 ) );
  540. MLX_FILL_1 ( &cqctx, 5, c_eqn, arbel->eqn );
  541. MLX_FILL_1 ( &cqctx, 6, pd, ARBEL_GLOBAL_PD );
  542. MLX_FILL_1 ( &cqctx, 7, l_key, arbel->reserved_lkey );
  543. MLX_FILL_1 ( &cqctx, 12, cqn, cq->cqn );
  544. MLX_FILL_1 ( &cqctx, 13,
  545. cq_ci_db_record, arbel_cq->ci_doorbell_idx );
  546. MLX_FILL_1 ( &cqctx, 14,
  547. cq_state_db_record, arbel_cq->arm_doorbell_idx );
  548. if ( ( rc = arbel_cmd_sw2hw_cq ( arbel, cq->cqn, &cqctx ) ) != 0 ) {
  549. DBGC ( arbel, "Arbel %p SW2HW_CQ failed: %s\n",
  550. arbel, strerror ( rc ) );
  551. goto err_sw2hw_cq;
  552. }
  553. cq->dev_priv = arbel_cq;
  554. return 0;
  555. err_sw2hw_cq:
  556. MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  557. MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  558. free_dma ( arbel_cq->cqe, arbel_cq->cqe_size );
  559. err_cqe:
  560. free ( arbel_cq );
  561. err_arbel_cq:
  562. arbel_free_qn_offset ( arbel->cq_inuse, cqn_offset );
  563. err_cqn_offset:
  564. return rc;
  565. }
  566. /**
  567. * Destroy completion queue
  568. *
  569. * @v ibdev Infiniband device
  570. * @v cq Completion queue
  571. */
  572. static void arbel_destroy_cq ( struct ib_device *ibdev,
  573. struct ib_completion_queue *cq ) {
  574. struct arbel *arbel = ibdev->dev_priv;
  575. struct arbel_completion_queue *arbel_cq = cq->dev_priv;
  576. struct arbelprm_cq_ci_db_record *ci_db_rec;
  577. struct arbelprm_cq_arm_db_record *arm_db_rec;
  578. int cqn_offset;
  579. int rc;
  580. /* Take ownership back from hardware */
  581. if ( ( rc = arbel_cmd_hw2sw_cq ( arbel, cq->cqn ) ) != 0 ) {
  582. DBGC ( arbel, "Arbel %p FATAL HW2SW_CQ failed on CQN %#lx: "
  583. "%s\n", arbel, cq->cqn, strerror ( rc ) );
  584. /* Leak memory and return; at least we avoid corruption */
  585. return;
  586. }
  587. /* Clear doorbell records */
  588. ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
  589. arm_db_rec = &arbel->db_rec[arbel_cq->arm_doorbell_idx].cq_arm;
  590. MLX_FILL_1 ( ci_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  591. MLX_FILL_1 ( arm_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  592. /* Free memory */
  593. free_dma ( arbel_cq->cqe, arbel_cq->cqe_size );
  594. free ( arbel_cq );
  595. /* Mark queue number as free */
  596. cqn_offset = ( cq->cqn - arbel->limits.reserved_cqs );
  597. arbel_free_qn_offset ( arbel->cq_inuse, cqn_offset );
  598. cq->dev_priv = NULL;
  599. }
  600. /***************************************************************************
  601. *
  602. * Queue pair operations
  603. *
  604. ***************************************************************************
  605. */
  606. /**
  607. * Create send work queue
  608. *
  609. * @v arbel_send_wq Send work queue
  610. * @v num_wqes Number of work queue entries
  611. * @ret rc Return status code
  612. */
  613. static int arbel_create_send_wq ( struct arbel_send_work_queue *arbel_send_wq,
  614. unsigned int num_wqes ) {
  615. struct arbelprm_ud_send_wqe *wqe;
  616. struct arbelprm_ud_send_wqe *next_wqe;
  617. unsigned int wqe_idx_mask;
  618. unsigned int i;
  619. /* Allocate work queue */
  620. arbel_send_wq->wqe_size = ( num_wqes *
  621. sizeof ( arbel_send_wq->wqe[0] ) );
  622. arbel_send_wq->wqe = malloc_dma ( arbel_send_wq->wqe_size,
  623. sizeof ( arbel_send_wq->wqe[0] ) );
  624. if ( ! arbel_send_wq->wqe )
  625. return -ENOMEM;
  626. memset ( arbel_send_wq->wqe, 0, arbel_send_wq->wqe_size );
  627. /* Link work queue entries */
  628. wqe_idx_mask = ( num_wqes - 1 );
  629. for ( i = 0 ; i < num_wqes ; i++ ) {
  630. wqe = &arbel_send_wq->wqe[i].ud;
  631. next_wqe = &arbel_send_wq->wqe[ ( i + 1 ) & wqe_idx_mask ].ud;
  632. MLX_FILL_1 ( &wqe->next, 0, nda_31_6,
  633. ( virt_to_bus ( next_wqe ) >> 6 ) );
  634. }
  635. return 0;
  636. }
  637. /**
  638. * Create receive work queue
  639. *
  640. * @v arbel_recv_wq Receive work queue
  641. * @v num_wqes Number of work queue entries
  642. * @ret rc Return status code
  643. */
  644. static int arbel_create_recv_wq ( struct arbel_recv_work_queue *arbel_recv_wq,
  645. unsigned int num_wqes ) {
  646. struct arbelprm_recv_wqe *wqe;
  647. struct arbelprm_recv_wqe *next_wqe;
  648. unsigned int wqe_idx_mask;
  649. size_t nds;
  650. unsigned int i;
  651. unsigned int j;
  652. /* Allocate work queue */
  653. arbel_recv_wq->wqe_size = ( num_wqes *
  654. sizeof ( arbel_recv_wq->wqe[0] ) );
  655. arbel_recv_wq->wqe = malloc_dma ( arbel_recv_wq->wqe_size,
  656. sizeof ( arbel_recv_wq->wqe[0] ) );
  657. if ( ! arbel_recv_wq->wqe )
  658. return -ENOMEM;
  659. memset ( arbel_recv_wq->wqe, 0, arbel_recv_wq->wqe_size );
  660. /* Link work queue entries */
  661. wqe_idx_mask = ( num_wqes - 1 );
  662. nds = ( ( offsetof ( typeof ( *wqe ), data ) +
  663. sizeof ( wqe->data[0] ) ) >> 4 );
  664. for ( i = 0 ; i < num_wqes ; i++ ) {
  665. wqe = &arbel_recv_wq->wqe[i].recv;
  666. next_wqe = &arbel_recv_wq->wqe[( i + 1 ) & wqe_idx_mask].recv;
  667. MLX_FILL_1 ( &wqe->next, 0, nda_31_6,
  668. ( virt_to_bus ( next_wqe ) >> 6 ) );
  669. MLX_FILL_1 ( &wqe->next, 1, nds, ( sizeof ( *wqe ) / 16 ) );
  670. for ( j = 0 ; ( ( ( void * ) &wqe->data[j] ) <
  671. ( ( void * ) ( wqe + 1 ) ) ) ; j++ ) {
  672. MLX_FILL_1 ( &wqe->data[j], 1,
  673. l_key, ARBEL_INVALID_LKEY );
  674. }
  675. }
  676. return 0;
  677. }
  678. /**
  679. * Create queue pair
  680. *
  681. * @v ibdev Infiniband device
  682. * @v qp Queue pair
  683. * @ret rc Return status code
  684. */
  685. static int arbel_create_qp ( struct ib_device *ibdev,
  686. struct ib_queue_pair *qp ) {
  687. struct arbel *arbel = ibdev->dev_priv;
  688. struct arbel_queue_pair *arbel_qp;
  689. struct arbelprm_qp_ee_state_transitions qpctx;
  690. struct arbelprm_qp_db_record *send_db_rec;
  691. struct arbelprm_qp_db_record *recv_db_rec;
  692. int qpn_offset;
  693. int rc;
  694. /* Find a free queue pair number */
  695. qpn_offset = arbel_alloc_qn_offset ( arbel->qp_inuse, ARBEL_MAX_QPS );
  696. if ( qpn_offset < 0 ) {
  697. DBGC ( arbel, "Arbel %p out of queue pairs\n", arbel );
  698. rc = qpn_offset;
  699. goto err_qpn_offset;
  700. }
  701. qp->qpn = ( ARBEL_QPN_BASE + arbel->limits.reserved_qps + qpn_offset );
  702. /* Allocate control structures */
  703. arbel_qp = zalloc ( sizeof ( *arbel_qp ) );
  704. if ( ! arbel_qp ) {
  705. rc = -ENOMEM;
  706. goto err_arbel_qp;
  707. }
  708. arbel_qp->send.doorbell_idx = arbel_send_doorbell_idx ( qpn_offset );
  709. arbel_qp->recv.doorbell_idx = arbel_recv_doorbell_idx ( qpn_offset );
  710. /* Create send and receive work queues */
  711. if ( ( rc = arbel_create_send_wq ( &arbel_qp->send,
  712. qp->send.num_wqes ) ) != 0 )
  713. goto err_create_send_wq;
  714. if ( ( rc = arbel_create_recv_wq ( &arbel_qp->recv,
  715. qp->recv.num_wqes ) ) != 0 )
  716. goto err_create_recv_wq;
  717. /* Initialise doorbell records */
  718. send_db_rec = &arbel->db_rec[arbel_qp->send.doorbell_idx].qp;
  719. MLX_FILL_1 ( send_db_rec, 0, counter, 0 );
  720. MLX_FILL_2 ( send_db_rec, 1,
  721. res, ARBEL_UAR_RES_SQ,
  722. qp_number, qp->qpn );
  723. recv_db_rec = &arbel->db_rec[arbel_qp->recv.doorbell_idx].qp;
  724. MLX_FILL_1 ( recv_db_rec, 0, counter, 0 );
  725. MLX_FILL_2 ( recv_db_rec, 1,
  726. res, ARBEL_UAR_RES_RQ,
  727. qp_number, qp->qpn );
  728. /* Hand queue over to hardware */
  729. memset ( &qpctx, 0, sizeof ( qpctx ) );
  730. MLX_FILL_3 ( &qpctx, 2,
  731. qpc_eec_data.de, 1,
  732. qpc_eec_data.pm_state, 0x03 /* Always 0x03 for UD */,
  733. qpc_eec_data.st, ARBEL_ST_UD );
  734. MLX_FILL_6 ( &qpctx, 4,
  735. qpc_eec_data.mtu, ARBEL_MTU_2048,
  736. qpc_eec_data.msg_max, 11 /* 2^11 = 2048 */,
  737. qpc_eec_data.log_rq_size, fls ( qp->recv.num_wqes - 1 ),
  738. qpc_eec_data.log_rq_stride,
  739. ( fls ( sizeof ( arbel_qp->recv.wqe[0] ) - 1 ) - 4 ),
  740. qpc_eec_data.log_sq_size, fls ( qp->send.num_wqes - 1 ),
  741. qpc_eec_data.log_sq_stride,
  742. ( fls ( sizeof ( arbel_qp->send.wqe[0] ) - 1 ) - 4 ) );
  743. MLX_FILL_1 ( &qpctx, 5,
  744. qpc_eec_data.usr_page, arbel->limits.reserved_uars );
  745. MLX_FILL_1 ( &qpctx, 10, qpc_eec_data.primary_address_path.port_number,
  746. PXE_IB_PORT );
  747. MLX_FILL_1 ( &qpctx, 27, qpc_eec_data.pd, ARBEL_GLOBAL_PD );
  748. MLX_FILL_1 ( &qpctx, 29, qpc_eec_data.wqe_lkey, arbel->reserved_lkey );
  749. MLX_FILL_1 ( &qpctx, 30, qpc_eec_data.ssc, 1 );
  750. MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
  751. MLX_FILL_1 ( &qpctx, 34, qpc_eec_data.snd_wqe_base_adr_l,
  752. ( virt_to_bus ( arbel_qp->send.wqe ) >> 6 ) );
  753. MLX_FILL_1 ( &qpctx, 35, qpc_eec_data.snd_db_record_index,
  754. arbel_qp->send.doorbell_idx );
  755. MLX_FILL_1 ( &qpctx, 38, qpc_eec_data.rsc, 1 );
  756. MLX_FILL_1 ( &qpctx, 41, qpc_eec_data.cqn_rcv, qp->recv.cq->cqn );
  757. MLX_FILL_1 ( &qpctx, 42, qpc_eec_data.rcv_wqe_base_adr_l,
  758. ( virt_to_bus ( arbel_qp->recv.wqe ) >> 6 ) );
  759. MLX_FILL_1 ( &qpctx, 43, qpc_eec_data.rcv_db_record_index,
  760. arbel_qp->recv.doorbell_idx );
  761. MLX_FILL_1 ( &qpctx, 44, qpc_eec_data.q_key, qp->qkey );
  762. if ( ( rc = arbel_cmd_rst2init_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
  763. DBGC ( arbel, "Arbel %p RST2INIT_QPEE failed: %s\n",
  764. arbel, strerror ( rc ) );
  765. goto err_rst2init_qpee;
  766. }
  767. memset ( &qpctx, 0, sizeof ( qpctx ) );
  768. MLX_FILL_2 ( &qpctx, 4,
  769. qpc_eec_data.mtu, ARBEL_MTU_2048,
  770. qpc_eec_data.msg_max, 11 /* 2^11 = 2048 */ );
  771. if ( ( rc = arbel_cmd_init2rtr_qpee ( arbel, qp->qpn, &qpctx )) != 0 ){
  772. DBGC ( arbel, "Arbel %p INIT2RTR_QPEE failed: %s\n",
  773. arbel, strerror ( rc ) );
  774. goto err_init2rtr_qpee;
  775. }
  776. memset ( &qpctx, 0, sizeof ( qpctx ) );
  777. if ( ( rc = arbel_cmd_rtr2rts_qpee ( arbel, qp->qpn, &qpctx ) ) != 0 ){
  778. DBGC ( arbel, "Arbel %p RTR2RTS_QPEE failed: %s\n",
  779. arbel, strerror ( rc ) );
  780. goto err_rtr2rts_qpee;
  781. }
  782. qp->dev_priv = arbel_qp;
  783. return 0;
  784. err_rtr2rts_qpee:
  785. err_init2rtr_qpee:
  786. arbel_cmd_2rst_qpee ( arbel, qp->qpn );
  787. err_rst2init_qpee:
  788. MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  789. MLX_FILL_1 ( recv_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  790. free_dma ( arbel_qp->recv.wqe, arbel_qp->recv.wqe_size );
  791. err_create_recv_wq:
  792. free_dma ( arbel_qp->send.wqe, arbel_qp->send.wqe_size );
  793. err_create_send_wq:
  794. free ( arbel_qp );
  795. err_arbel_qp:
  796. arbel_free_qn_offset ( arbel->qp_inuse, qpn_offset );
  797. err_qpn_offset:
  798. return rc;
  799. }
  800. /**
  801. * Destroy queue pair
  802. *
  803. * @v ibdev Infiniband device
  804. * @v qp Queue pair
  805. */
  806. static void arbel_destroy_qp ( struct ib_device *ibdev,
  807. struct ib_queue_pair *qp ) {
  808. struct arbel *arbel = ibdev->dev_priv;
  809. struct arbel_queue_pair *arbel_qp = qp->dev_priv;
  810. struct arbelprm_qp_db_record *send_db_rec;
  811. struct arbelprm_qp_db_record *recv_db_rec;
  812. int qpn_offset;
  813. int rc;
  814. /* Take ownership back from hardware */
  815. if ( ( rc = arbel_cmd_2rst_qpee ( arbel, qp->qpn ) ) != 0 ) {
  816. DBGC ( arbel, "Arbel %p FATAL 2RST_QPEE failed on QPN %#lx: "
  817. "%s\n", arbel, qp->qpn, strerror ( rc ) );
  818. /* Leak memory and return; at least we avoid corruption */
  819. return;
  820. }
  821. /* Clear doorbell records */
  822. send_db_rec = &arbel->db_rec[arbel_qp->send.doorbell_idx].qp;
  823. recv_db_rec = &arbel->db_rec[arbel_qp->recv.doorbell_idx].qp;
  824. MLX_FILL_1 ( send_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  825. MLX_FILL_1 ( recv_db_rec, 1, res, ARBEL_UAR_RES_NONE );
  826. /* Free memory */
  827. free_dma ( arbel_qp->send.wqe, arbel_qp->send.wqe_size );
  828. free_dma ( arbel_qp->recv.wqe, arbel_qp->recv.wqe_size );
  829. free ( arbel_qp );
  830. /* Mark queue number as free */
  831. qpn_offset = ( qp->qpn - ARBEL_QPN_BASE - arbel->limits.reserved_qps );
  832. arbel_free_qn_offset ( arbel->qp_inuse, qpn_offset );
  833. qp->dev_priv = NULL;
  834. }
  835. /***************************************************************************
  836. *
  837. * Work request operations
  838. *
  839. ***************************************************************************
  840. */
  841. /**
  842. * Ring doorbell register in UAR
  843. *
  844. * @v arbel Arbel device
  845. * @v db_reg Doorbell register structure
  846. * @v offset Address of doorbell
  847. */
  848. static void arbel_ring_doorbell ( struct arbel *arbel,
  849. union arbelprm_doorbell_register *db_reg,
  850. unsigned int offset ) {
  851. DBG ( "arbel_ring_doorbell %08lx:%08lx to %lx\n",
  852. db_reg->dword[0], db_reg->dword[1],
  853. virt_to_phys ( arbel->uar + offset ) );
  854. barrier();
  855. writel ( db_reg->dword[0], ( arbel->uar + offset + 0 ) );
  856. barrier();
  857. writel ( db_reg->dword[1], ( arbel->uar + offset + 4 ) );
  858. }
  859. /**
  860. * Post send work queue entry
  861. *
  862. * @v ibdev Infiniband device
  863. * @v qp Queue pair
  864. * @v av Address vector
  865. * @v iobuf I/O buffer
  866. * @ret rc Return status code
  867. */
  868. static int arbel_post_send ( struct ib_device *ibdev,
  869. struct ib_queue_pair *qp,
  870. struct ib_address_vector *av,
  871. struct io_buffer *iobuf ) {
  872. struct arbel *arbel = ibdev->dev_priv;
  873. struct arbel_queue_pair *arbel_qp = qp->dev_priv;
  874. struct ib_work_queue *wq = &qp->send;
  875. struct arbel_send_work_queue *arbel_send_wq = &arbel_qp->send;
  876. struct arbelprm_ud_send_wqe *prev_wqe;
  877. struct arbelprm_ud_send_wqe *wqe;
  878. struct arbelprm_qp_db_record *qp_db_rec;
  879. union arbelprm_doorbell_register db_reg;
  880. const struct ib_gid *gid;
  881. unsigned int wqe_idx_mask;
  882. size_t nds;
  883. /* Allocate work queue entry */
  884. wqe_idx_mask = ( wq->num_wqes - 1 );
  885. if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
  886. DBGC ( arbel, "Arbel %p send queue full", arbel );
  887. return -ENOBUFS;
  888. }
  889. wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
  890. prev_wqe = &arbel_send_wq->wqe[(wq->next_idx - 1) & wqe_idx_mask].ud;
  891. wqe = &arbel_send_wq->wqe[wq->next_idx & wqe_idx_mask].ud;
  892. /* Construct work queue entry */
  893. MLX_FILL_1 ( &wqe->next, 1, always1, 1 );
  894. memset ( &wqe->ctrl, 0, sizeof ( wqe->ctrl ) );
  895. MLX_FILL_1 ( &wqe->ctrl, 0, always1, 1 );
  896. memset ( &wqe->ud, 0, sizeof ( wqe->ud ) );
  897. MLX_FILL_2 ( &wqe->ud, 0,
  898. ud_address_vector.pd, ARBEL_GLOBAL_PD,
  899. ud_address_vector.port_number, PXE_IB_PORT );
  900. MLX_FILL_2 ( &wqe->ud, 1,
  901. ud_address_vector.rlid, av->dlid,
  902. ud_address_vector.g, av->gid_present );
  903. MLX_FILL_2 ( &wqe->ud, 2,
  904. ud_address_vector.max_stat_rate,
  905. ( ( av->rate >= 3 ) ? 0 : 1 ),
  906. ud_address_vector.msg, 3 );
  907. MLX_FILL_1 ( &wqe->ud, 3, ud_address_vector.sl, av->sl );
  908. gid = ( av->gid_present ? &av->gid : &arbel_no_gid );
  909. memcpy ( &wqe->ud.u.dwords[4], gid, sizeof ( *gid ) );
  910. MLX_FILL_1 ( &wqe->ud, 8, destination_qp, av->dest_qp );
  911. MLX_FILL_1 ( &wqe->ud, 9, q_key, av->qkey );
  912. MLX_FILL_1 ( &wqe->data[0], 0, byte_count, iob_len ( iobuf ) );
  913. MLX_FILL_1 ( &wqe->data[0], 1, l_key, arbel->reserved_lkey );
  914. MLX_FILL_1 ( &wqe->data[0], 3,
  915. local_address_l, virt_to_bus ( iobuf->data ) );
  916. /* Update previous work queue entry's "next" field */
  917. nds = ( ( offsetof ( typeof ( *wqe ), data ) +
  918. sizeof ( wqe->data[0] ) ) >> 4 );
  919. MLX_SET ( &prev_wqe->next, nopcode, ARBEL_OPCODE_SEND );
  920. MLX_FILL_3 ( &prev_wqe->next, 1,
  921. nds, nds,
  922. f, 1,
  923. always1, 1 );
  924. /* Update doorbell record */
  925. barrier();
  926. qp_db_rec = &arbel->db_rec[arbel_send_wq->doorbell_idx].qp;
  927. MLX_FILL_1 ( qp_db_rec, 0,
  928. counter, ( ( wq->next_idx + 1 ) & 0xffff ) );
  929. /* Ring doorbell register */
  930. MLX_FILL_4 ( &db_reg.send, 0,
  931. nopcode, ARBEL_OPCODE_SEND,
  932. f, 1,
  933. wqe_counter, ( wq->next_idx & 0xffff ),
  934. wqe_cnt, 1 );
  935. MLX_FILL_2 ( &db_reg.send, 1,
  936. nds, nds,
  937. qpn, qp->qpn );
  938. arbel_ring_doorbell ( arbel, &db_reg, POST_SND_OFFSET );
  939. /* Update work queue's index */
  940. wq->next_idx++;
  941. return 0;
  942. }
  943. /**
  944. * Post receive work queue entry
  945. *
  946. * @v ibdev Infiniband device
  947. * @v qp Queue pair
  948. * @v iobuf I/O buffer
  949. * @ret rc Return status code
  950. */
  951. static int arbel_post_recv ( struct ib_device *ibdev,
  952. struct ib_queue_pair *qp,
  953. struct io_buffer *iobuf ) {
  954. struct arbel *arbel = ibdev->dev_priv;
  955. struct arbel_queue_pair *arbel_qp = qp->dev_priv;
  956. struct ib_work_queue *wq = &qp->recv;
  957. struct arbel_recv_work_queue *arbel_recv_wq = &arbel_qp->recv;
  958. struct arbelprm_recv_wqe *wqe;
  959. union arbelprm_doorbell_record *db_rec;
  960. unsigned int wqe_idx_mask;
  961. /* Allocate work queue entry */
  962. wqe_idx_mask = ( wq->num_wqes - 1 );
  963. if ( wq->iobufs[wq->next_idx & wqe_idx_mask] ) {
  964. DBGC ( arbel, "Arbel %p receive queue full", arbel );
  965. return -ENOBUFS;
  966. }
  967. wq->iobufs[wq->next_idx & wqe_idx_mask] = iobuf;
  968. wqe = &arbel_recv_wq->wqe[wq->next_idx & wqe_idx_mask].recv;
  969. /* Construct work queue entry */
  970. MLX_FILL_1 ( &wqe->data[0], 0, byte_count, iob_tailroom ( iobuf ) );
  971. MLX_FILL_1 ( &wqe->data[0], 1, l_key, arbel->reserved_lkey );
  972. MLX_FILL_1 ( &wqe->data[0], 3,
  973. local_address_l, virt_to_bus ( iobuf->data ) );
  974. /* Update doorbell record */
  975. barrier();
  976. db_rec = &arbel->db_rec[arbel_recv_wq->doorbell_idx];
  977. MLX_FILL_1 ( &db_rec->qp, 0,
  978. counter, ( ( wq->next_idx + 1 ) & 0xffff ) );
  979. /* Update work queue's index */
  980. wq->next_idx++;
  981. return 0;
  982. }
  983. /**
  984. * Handle completion
  985. *
  986. * @v ibdev Infiniband device
  987. * @v cq Completion queue
  988. * @v cqe Hardware completion queue entry
  989. * @v complete_send Send completion handler
  990. * @v complete_recv Receive completion handler
  991. * @ret rc Return status code
  992. */
  993. static int arbel_complete ( struct ib_device *ibdev,
  994. struct ib_completion_queue *cq,
  995. union arbelprm_completion_entry *cqe,
  996. ib_completer_t complete_send,
  997. ib_completer_t complete_recv ) {
  998. struct arbel *arbel = ibdev->dev_priv;
  999. struct ib_completion completion;
  1000. struct ib_work_queue *wq;
  1001. struct ib_queue_pair *qp;
  1002. struct arbel_queue_pair *arbel_qp;
  1003. struct arbel_send_work_queue *arbel_send_wq;
  1004. struct arbel_recv_work_queue *arbel_recv_wq;
  1005. struct io_buffer *iobuf;
  1006. ib_completer_t complete;
  1007. unsigned int opcode;
  1008. unsigned long qpn;
  1009. int is_send;
  1010. unsigned long wqe_adr;
  1011. unsigned int wqe_idx;
  1012. int rc = 0;
  1013. /* Parse completion */
  1014. memset ( &completion, 0, sizeof ( completion ) );
  1015. completion.len = MLX_GET ( &cqe->normal, byte_cnt );
  1016. qpn = MLX_GET ( &cqe->normal, my_qpn );
  1017. is_send = MLX_GET ( &cqe->normal, s );
  1018. wqe_adr = ( MLX_GET ( &cqe->normal, wqe_adr ) << 6 );
  1019. opcode = MLX_GET ( &cqe->normal, opcode );
  1020. if ( opcode >= ARBEL_OPCODE_RECV_ERROR ) {
  1021. /* "s" field is not valid for error opcodes */
  1022. is_send = ( opcode == ARBEL_OPCODE_SEND_ERROR );
  1023. completion.syndrome = MLX_GET ( &cqe->error, syndrome );
  1024. DBGC ( arbel, "Arbel %p CPN %lx syndrome %x vendor %lx\n",
  1025. arbel, cq->cqn, completion.syndrome,
  1026. MLX_GET ( &cqe->error, vendor_code ) );
  1027. rc = -EIO;
  1028. /* Don't return immediately; propagate error to completer */
  1029. }
  1030. /* Identify work queue */
  1031. wq = ib_find_wq ( cq, qpn, is_send );
  1032. if ( ! wq ) {
  1033. DBGC ( arbel, "Arbel %p CQN %lx unknown %s QPN %lx\n",
  1034. arbel, cq->cqn, ( is_send ? "send" : "recv" ), qpn );
  1035. return -EIO;
  1036. }
  1037. qp = wq->qp;
  1038. arbel_qp = qp->dev_priv;
  1039. /* Identify work queue entry index */
  1040. if ( is_send ) {
  1041. arbel_send_wq = &arbel_qp->send;
  1042. wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_send_wq->wqe ) ) /
  1043. sizeof ( arbel_send_wq->wqe[0] ) );
  1044. } else {
  1045. arbel_recv_wq = &arbel_qp->recv;
  1046. wqe_idx = ( ( wqe_adr - virt_to_bus ( arbel_recv_wq->wqe ) ) /
  1047. sizeof ( arbel_recv_wq->wqe[0] ) );
  1048. }
  1049. /* Identify I/O buffer */
  1050. iobuf = wq->iobufs[wqe_idx];
  1051. if ( ! iobuf ) {
  1052. DBGC ( arbel, "Arbel %p CQN %lx QPN %lx empty WQE %x\n",
  1053. arbel, cq->cqn, qpn, wqe_idx );
  1054. return -EIO;
  1055. }
  1056. wq->iobufs[wqe_idx] = NULL;
  1057. /* Pass off to caller's completion handler */
  1058. complete = ( is_send ? complete_send : complete_recv );
  1059. complete ( ibdev, qp, &completion, iobuf );
  1060. return rc;
  1061. }
  1062. /**
  1063. * Poll completion queue
  1064. *
  1065. * @v ibdev Infiniband device
  1066. * @v cq Completion queue
  1067. * @v complete_send Send completion handler
  1068. * @v complete_recv Receive completion handler
  1069. */
  1070. static void arbel_poll_cq ( struct ib_device *ibdev,
  1071. struct ib_completion_queue *cq,
  1072. ib_completer_t complete_send,
  1073. ib_completer_t complete_recv ) {
  1074. struct arbel *arbel = ibdev->dev_priv;
  1075. struct arbel_completion_queue *arbel_cq = cq->dev_priv;
  1076. struct arbelprm_cq_ci_db_record *ci_db_rec;
  1077. union arbelprm_completion_entry *cqe;
  1078. unsigned int cqe_idx_mask;
  1079. int rc;
  1080. while ( 1 ) {
  1081. /* Look for completion entry */
  1082. cqe_idx_mask = ( cq->num_cqes - 1 );
  1083. cqe = &arbel_cq->cqe[cq->next_idx & cqe_idx_mask];
  1084. if ( MLX_GET ( &cqe->normal, owner ) != 0 ) {
  1085. /* Entry still owned by hardware; end of poll */
  1086. break;
  1087. }
  1088. /* Handle completion */
  1089. if ( ( rc = arbel_complete ( ibdev, cq, cqe, complete_send,
  1090. complete_recv ) ) != 0 ) {
  1091. DBGC ( arbel, "Arbel %p failed to complete: %s\n",
  1092. arbel, strerror ( rc ) );
  1093. DBGC_HD ( arbel, cqe, sizeof ( *cqe ) );
  1094. }
  1095. /* Return ownership to hardware */
  1096. MLX_FILL_1 ( &cqe->normal, 7, owner, 1 );
  1097. barrier();
  1098. /* Update completion queue's index */
  1099. cq->next_idx++;
  1100. /* Update doorbell record */
  1101. ci_db_rec = &arbel->db_rec[arbel_cq->ci_doorbell_idx].cq_ci;
  1102. MLX_FILL_1 ( ci_db_rec, 0,
  1103. counter, ( cq->next_idx & 0xffffffffUL ) );
  1104. }
  1105. }
  1106. /** Arbel Infiniband operations */
  1107. static struct ib_device_operations arbel_ib_operations = {
  1108. .create_cq = arbel_create_cq,
  1109. .destroy_cq = arbel_destroy_cq,
  1110. .create_qp = arbel_create_qp,
  1111. .destroy_qp = arbel_destroy_qp,
  1112. .post_send = arbel_post_send,
  1113. .post_recv = arbel_post_recv,
  1114. .poll_cq = arbel_poll_cq,
  1115. };
  1116. /**
  1117. * Remove PCI device
  1118. *
  1119. * @v pci PCI device
  1120. */
  1121. static void arbel_remove ( struct pci_device *pci ) {
  1122. struct net_device *netdev = pci_get_drvdata ( pci );
  1123. unregister_netdev ( netdev );
  1124. ib_driver_close ( 0 );
  1125. netdev_nullify ( netdev );
  1126. netdev_put ( netdev );
  1127. }
  1128. /**
  1129. * Probe PCI device
  1130. *
  1131. * @v pci PCI device
  1132. * @v id PCI ID
  1133. * @ret rc Return status code
  1134. */
  1135. static int arbel_probe ( struct pci_device *pci,
  1136. const struct pci_device_id *id __unused ) {
  1137. struct net_device *netdev;
  1138. struct arbelprm_query_dev_lim dev_lim;
  1139. struct arbel *arbel = &static_arbel;
  1140. struct mlx_nic *mlx;
  1141. struct ib_mac *mac;
  1142. udqp_t qph;
  1143. int rc;
  1144. /* Allocate net device */
  1145. netdev = alloc_ibdev ( sizeof ( *mlx ) );
  1146. if ( ! netdev )
  1147. return -ENOMEM;
  1148. netdev_init ( netdev, &mlx_operations );
  1149. mlx = netdev->priv;
  1150. pci_set_drvdata ( pci, netdev );
  1151. netdev->dev = &pci->dev;
  1152. memset ( mlx, 0, sizeof ( *mlx ) );
  1153. /* Fix up PCI device */
  1154. adjust_pci_device ( pci );
  1155. /* Initialise hardware */
  1156. if ( ( rc = ib_driver_init ( pci, &qph ) ) != 0 )
  1157. goto err_ipoib_init;
  1158. #if ! CREATE_OWN
  1159. mlx->ipoib_qph = qph;
  1160. mlx->bcast_av = ib_data.bcast_av;
  1161. mlx->snd_cqh = ib_data.ipoib_snd_cq;
  1162. mlx->rcv_cqh = ib_data.ipoib_rcv_cq;
  1163. mac = ( ( struct ib_mac * ) netdev->ll_addr );
  1164. mac->qpn = htonl ( ib_get_qpn ( mlx->ipoib_qph ) );
  1165. memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) );
  1166. #endif
  1167. /* Hack up IB structures */
  1168. arbel->config = memfree_pci_dev.cr_space;
  1169. arbel->mailbox_in = dev_buffers_p->inprm_buf;
  1170. arbel->mailbox_out = dev_buffers_p->outprm_buf;
  1171. arbel->uar = memfree_pci_dev.uar;
  1172. arbel->db_rec = dev_ib_data.uar_context_base;
  1173. arbel->reserved_lkey = dev_ib_data.mkey;
  1174. arbel->eqn = dev_ib_data.eq.eqn;
  1175. #if ! CREATE_OWN
  1176. static_arbel_ipoib_qp.send.wqe =
  1177. ( ( struct udqp_st * ) qph )->snd_wq;
  1178. static_arbel_ipoib_qp.recv.wqe =
  1179. ( ( struct udqp_st * ) qph )->rcv_wq;
  1180. static_arbel_ipoib_send_cq.cqe =
  1181. ( ( struct cq_st * ) ib_data.ipoib_snd_cq )->cq_buf;
  1182. static_arbel_ipoib_recv_cq.cqe =
  1183. ( ( struct cq_st * ) ib_data.ipoib_rcv_cq )->cq_buf;
  1184. static_ipoib_qp.qpn = ib_get_qpn ( qph );
  1185. static_ipoib_qp.owner_priv = netdev;
  1186. list_add ( &static_ipoib_qp.send.list,
  1187. &static_ipoib_send_cq.work_queues );
  1188. list_add ( &static_ipoib_qp.recv.list,
  1189. &static_ipoib_recv_cq.work_queues );
  1190. #endif
  1191. static_ibdev.op = &arbel_ib_operations;
  1192. /* Get device limits */
  1193. if ( ( rc = arbel_cmd_query_dev_lim ( arbel, &dev_lim ) ) != 0 ) {
  1194. DBGC ( arbel, "Arbel %p could not get device limits: %s\n",
  1195. arbel, strerror ( rc ) );
  1196. goto err_query_dev_lim;
  1197. }
  1198. arbel->limits.reserved_uars = MLX_GET ( &dev_lim, num_rsvd_uars );
  1199. arbel->limits.reserved_cqs =
  1200. ( 1 << MLX_GET ( &dev_lim, log2_rsvd_cqs ) );
  1201. arbel->limits.reserved_qps =
  1202. ( 1 << MLX_GET ( &dev_lim, log2_rsvd_qps ) );
  1203. #if CREATE_OWN
  1204. struct ib_device *ibdev = &static_ibdev;
  1205. mlx->own_send_cq = ib_create_cq ( ibdev, 32 );
  1206. if ( ! mlx->own_send_cq ) {
  1207. DBG ( "Could not create send CQ\n" );
  1208. return -EIO;
  1209. }
  1210. mlx->own_recv_cq = ib_create_cq ( ibdev, 32 );
  1211. if ( ! mlx->own_recv_cq ) {
  1212. DBG ( "Could not create send CQ\n" );
  1213. return -EIO;
  1214. }
  1215. mlx->own_qp = ib_create_qp ( ibdev, NUM_IPOIB_SND_WQES,
  1216. mlx->own_send_cq, NUM_IPOIB_RCV_WQES,
  1217. mlx->own_recv_cq, ipoib_qkey );
  1218. if ( ! mlx->own_qp ) {
  1219. DBG ( "Could not create QP\n" );
  1220. return -EIO;
  1221. }
  1222. mlx->own_qp->owner_priv = netdev;
  1223. mac = ( ( struct ib_mac * ) netdev->ll_addr );
  1224. mac->qpn = htonl ( mlx->own_qp->qpn );
  1225. memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) );
  1226. #endif
  1227. #if 0
  1228. DBG ( "MADS SND CQN = %#lx\n", dev_ib_data.mads_qp.snd_cq.cqn );
  1229. struct ib_completion_queue *test_cq;
  1230. test_cq = ib_create_cq ( &static_ibdev, 32 );
  1231. if ( test_cq ) {
  1232. DBG ( "Woot: create_cq() passed!\n" );
  1233. }
  1234. #endif
  1235. /* Register network device */
  1236. if ( ( rc = register_netdev ( netdev ) ) != 0 )
  1237. goto err_register_netdev;
  1238. return 0;
  1239. err_query_dev_lim:
  1240. err_register_netdev:
  1241. err_ipoib_init:
  1242. ib_driver_close ( 0 );
  1243. netdev_nullify ( netdev );
  1244. netdev_put ( netdev );
  1245. return rc;
  1246. }
  1247. static struct pci_device_id arbel_nics[] = {
  1248. PCI_ROM ( 0x15b3, 0x6282, "MT25218", "MT25218 HCA driver" ),
  1249. PCI_ROM ( 0x15b3, 0x6274, "MT25204", "MT25204 HCA driver" ),
  1250. };
  1251. struct pci_driver arbel_driver __pci_driver = {
  1252. .ids = arbel_nics,
  1253. .id_count = ( sizeof ( arbel_nics ) / sizeof ( arbel_nics[0] ) ),
  1254. .probe = arbel_probe,
  1255. .remove = arbel_remove,
  1256. };