[Hermon] Fix event queue doorbells.

Event queue doorbells must use UAR pages 0-127 depending on event queue
number; other doorbells must use pages 128+ (and we choose to use page
128).

src/drivers/infiniband/hermon.c

@@ -344,6 +344,15 @@ hermon_cmd_hw2sw_eq ( struct hermon *hermon, unsigned int index,
 			    1, NULL, index, eqctx );
 }
 
+static inline int
+hermon_cmd_query_eq ( struct hermon *hermon, unsigned int index,
+		      struct hermonprm_eqc *eqctx ) {
+	return hermon_cmd ( hermon,
+			    HERMON_HCR_OUT_CMD ( HERMON_HCR_QUERY_EQ,
+						 1, sizeof ( *eqctx ) ),
+			    0, NULL, index, eqctx );
+}
+
 static inline int
 hermon_cmd_sw2hw_cq ( struct hermon *hermon, unsigned long cqn,
 		      const struct hermonprm_completion_queue_context *cqctx ){
@@ -667,7 +676,7 @@ static int hermon_create_cq ( struct ib_device *ibdev,
 	MLX_FILL_1 ( &cqctx, 2,
 		     page_offset, ( hermon_cq->mtt.page_offset >> 5 ) );
 	MLX_FILL_2 ( &cqctx, 3,
-		     usr_page, HERMON_UAR_PAGE,
+		     usr_page, HERMON_UAR_NON_EQ_PAGE,
 		     log_cq_size, fls ( cq->num_cqes - 1 ) );
 	MLX_FILL_1 ( &cqctx, 7, mtt_base_addr_l,
 		     ( hermon_cq->mtt.mtt_base_addr >> 3 ) );
@@ -773,6 +782,11 @@ static int hermon_create_qp ( struct ib_device *ibdev,
 		goto err_hermon_qp;
 	}
 
+	/* Calculate doorbell address */
+	hermon_qp->send.doorbell =
+		( hermon->uar + HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE +
+		  HERMON_DB_POST_SND_OFFSET );
+
 	/* Allocate work queue buffer */
 	hermon_qp->send.num_wqes = ( qp->send.num_wqes /* headroom */ + 1 +
 				( 2048 / sizeof ( hermon_qp->send.wqe[0] ) ) );
@@ -817,7 +831,7 @@ static int hermon_create_qp ( struct ib_device *ibdev,
 		     qpc_eec_data.log_sq_stride,
 		     ( fls ( sizeof ( hermon_qp->send.wqe[0] ) - 1 ) - 4 ) );
 	MLX_FILL_1 ( &qpctx, 5,
-		     qpc_eec_data.usr_page, HERMON_UAR_PAGE );
+		     qpc_eec_data.usr_page, HERMON_UAR_NON_EQ_PAGE );
 	MLX_FILL_1 ( &qpctx, 33, qpc_eec_data.cqn_snd, qp->send.cq->cqn );
 	MLX_FILL_1 ( &qpctx, 38, qpc_eec_data.page_offset,
 		     ( hermon_qp->mtt.page_offset >> 6 ) );
@@ -1029,9 +1043,8 @@ static int hermon_post_send ( struct ib_device *ibdev,
 	/* Ring doorbell register */
 	MLX_FILL_1 ( &db_reg.send, 0, qn, qp->qpn );
 	DBGCP ( hermon, "Ringing doorbell %08lx with %08lx\n",
-		virt_to_phys ( hermon->uar + HERMON_DB_POST_SND_OFFSET ),
-		db_reg.dword[0] );
-	writel ( db_reg.dword[0], ( hermon->uar + HERMON_DB_POST_SND_OFFSET ));
+		virt_to_phys ( hermon_send_wq->doorbell ), db_reg.dword[0] );
+	writel ( db_reg.dword[0], ( hermon_send_wq->doorbell ) );
 
 	/* Update work queue's index */
 	wq->next_idx++;
@@ -1209,7 +1222,7 @@ static void hermon_poll_cq ( struct ib_device *ibdev,
 
 		/* Update doorbell record */
 		MLX_FILL_1 ( &hermon_cq->doorbell, 0, update_ci,
-			     ( cq->next_idx & 0xffffffUL ) );
+			     ( cq->next_idx & 0x00ffffffUL ) );
 	}
 }
 
@@ -1442,6 +1455,15 @@ static int hermon_create_eq ( struct hermon *hermon ) {
 	unsigned int i;
 	int rc;
 
+	/* Select event queue number */
+	hermon_eq->eqn = ( 4 * hermon->cap.reserved_uars );
+	if ( hermon_eq->eqn < hermon->cap.reserved_eqs )
+		hermon_eq->eqn = hermon->cap.reserved_eqs;
+
+	/* Calculate doorbell address */
+	hermon_eq->doorbell =
+		( hermon->uar + HERMON_DB_EQ_OFFSET ( hermon_eq->eqn ) );
+
 	/* Allocate event queue itself */
 	hermon_eq->eqe_size =
 		( HERMON_NUM_EQES * sizeof ( hermon_eq->eqe[0] ) );
@@ -1471,7 +1493,8 @@ static int hermon_create_eq ( struct hermon *hermon ) {
 	MLX_FILL_1 ( &eqctx, 3, log_eq_size, fls ( HERMON_NUM_EQES - 1 ) );
 	MLX_FILL_1 ( &eqctx, 7, mtt_base_addr_l,
 		     ( hermon_eq->mtt.mtt_base_addr >> 3 ) );
-	if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, 0, &eqctx ) ) != 0 ) {
+	if ( ( rc = hermon_cmd_sw2hw_eq ( hermon, hermon_eq->eqn,
+					  &eqctx ) ) != 0 ) {
 		DBGC ( hermon, "Hermon %p SW2HW_EQ failed: %s\n",
 		       hermon, strerror ( rc ) );
 		goto err_sw2hw_eq;
@@ -1480,17 +1503,21 @@ static int hermon_create_eq ( struct hermon *hermon ) {
 	/* Map events to this event queue */
 	memset ( &mask, 0, sizeof ( mask ) );
 	MLX_FILL_1 ( &mask, 1, port_state_change, 1 );
-	if ( ( rc = hermon_cmd_map_eq ( hermon, ( HERMON_MAP_EQ_MAP | 0 ),
+	if ( ( rc = hermon_cmd_map_eq ( hermon,
+					( HERMON_MAP_EQ | hermon_eq->eqn ),
 					&mask ) ) != 0 ) {
 		DBGC ( hermon, "Hermon %p MAP_EQ failed: %s\n",
 		       hermon, strerror ( rc )  );
 		goto err_map_eq;
 	}
 
+	DBGC ( hermon, "Hermon %p EQN %#lx ring at [%p,%p])\n",
+	       hermon, hermon_eq->eqn, hermon_eq->eqe,
+	       ( ( ( void * ) hermon_eq->eqe ) + hermon_eq->eqe_size ) );
 	return 0;
 
  err_map_eq:
-	hermon_cmd_hw2sw_eq ( hermon, 0, &eqctx );
+	hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn, &eqctx );
  err_sw2hw_eq:
 	hermon_free_mtt ( hermon, &hermon_eq->mtt );
  err_alloc_mtt:
@@ -1514,7 +1541,8 @@ static void hermon_destroy_eq ( struct hermon *hermon ) {
 	/* Unmap events from event queue */
 	memset ( &mask, 0, sizeof ( mask ) );
 	MLX_FILL_1 ( &mask, 1, port_state_change, 1 );
-	if ( ( rc = hermon_cmd_map_eq ( hermon, ( HERMON_MAP_EQ_UNMAP | 0 ),
+	if ( ( rc = hermon_cmd_map_eq ( hermon,
+					( HERMON_UNMAP_EQ | hermon_eq->eqn ),
 					&mask ) ) != 0 ) {
 		DBGC ( hermon, "Hermon %p FATAL MAP_EQ failed to unmap: %s\n",
 		       hermon, strerror ( rc ) );
@@ -1522,7 +1550,8 @@ static void hermon_destroy_eq ( struct hermon *hermon ) {
 	}
 
 	/* Take ownership back from hardware */
-	if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, 0, &eqctx ) ) != 0 ) {
+	if ( ( rc = hermon_cmd_hw2sw_eq ( hermon, hermon_eq->eqn,
+					  &eqctx ) ) != 0 ) {
 		DBGC ( hermon, "Hermon %p FATAL HW2SW_EQ failed: %s\n",
 		       hermon, strerror ( rc ) );
 		/* Leak memory and return; at least we avoid corruption */
@@ -1578,6 +1607,7 @@ static void hermon_poll_eq ( struct hermon *hermon ) {
 	unsigned int event_type;
 
 	while ( 1 ) {
+		/* Look for event entry */
 		eqe_idx_mask = ( HERMON_NUM_EQES - 1 );
 		eqe = &hermon_eq->eqe[hermon_eq->next_idx & eqe_idx_mask];
 		if ( MLX_GET ( &eqe->generic, owner ) ^
@@ -1605,13 +1635,12 @@ static void hermon_poll_eq ( struct hermon *hermon ) {
 		hermon_eq->next_idx++;
 
 		/* Ring doorbell */
-		memset ( &db_reg, 0, sizeof ( db_reg ) );
-		MLX_FILL_1 ( &db_reg.event, 0, ci, hermon_eq->next_idx );
+		MLX_FILL_1 ( &db_reg.event, 0,
+			     ci, ( hermon_eq->next_idx & 0x00ffffffUL ) );
 		DBGCP ( hermon, "Ringing doorbell %08lx with %08lx\n",
-			virt_to_phys ( hermon->uar + HERMON_DB_EQ0_OFFSET ),
+			virt_to_phys ( hermon_eq->doorbell ),
 			db_reg.dword[0] );
-		writel ( db_reg.dword[0],
-			 ( hermon->uar + HERMON_DB_EQ0_OFFSET ) );
+		writel ( db_reg.dword[0], hermon_eq->doorbell );
 	}
 }
 
@@ -2161,9 +2190,8 @@ static int hermon_probe ( struct pci_device *pci,
 	/* Get PCI BARs */
 	hermon->config = ioremap ( pci_bar_start ( pci, HERMON_PCI_CONFIG_BAR),
 				   HERMON_PCI_CONFIG_BAR_SIZE );
-	hermon->uar = ioremap ( ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ) +
-				  HERMON_UAR_PAGE * HERMON_PAGE_SIZE ),
-				HERMON_PAGE_SIZE );
+	hermon->uar = ioremap ( pci_bar_start ( pci, HERMON_PCI_UAR_BAR ),
+				HERMON_UAR_NON_EQ_PAGE * HERMON_PAGE_SIZE );
 
 	/* Allocate space for mailboxes */
 	hermon->mailbox_in = malloc_dma ( HERMON_MBOX_SIZE,

src/drivers/infiniband/hermon.h

@@ -44,6 +44,7 @@
 #define HERMON_HCR_MAP_EQ		0x0012
 #define HERMON_HCR_SW2HW_EQ		0x0013
 #define HERMON_HCR_HW2SW_EQ		0x0014
+#define HERMON_HCR_QUERY_EQ		0x0015
 #define HERMON_HCR_SW2HW_CQ		0x0016
 #define HERMON_HCR_HW2SW_CQ		0x0017
 #define HERMON_HCR_RST2INIT_QP		0x0019
@@ -77,12 +78,13 @@
 #define HERMON_PAGE_SIZE		4096
 
 #define HERMON_DB_POST_SND_OFFSET	0x14
-#define HERMON_DB_EQ0_OFFSET		0x800
+#define HERMON_DB_EQ_OFFSET(_eqn)	\
+	( 0x800 + HERMON_PAGE_SIZE * ( (_eqn) / 4 ) + 0x08 * ( (_eqn) % 4 ) )
 
 #define HERMON_QP_OPT_PARAM_QKEY	0x00000020UL
 
-#define HERMON_MAP_EQ_MAP		( 0UL << 31 )
-#define HERMON_MAP_EQ_UNMAP		( 1UL << 31 )
+#define HERMON_MAP_EQ			( 0UL << 31 )
+#define HERMON_UNMAP_EQ			( 1UL << 31 )
 
 #define HERMON_EV_PORT_STATE_CHANGE	0x09
 
@@ -292,7 +294,7 @@ enum hermon_icm_map_regions {
  * Pages 0-127 are reserved for event queue doorbells only, so we use
  * page 128.
  */
-#define HERMON_UAR_PAGE		128
+#define HERMON_UAR_NON_EQ_PAGE	128
 
 /** Maximum number of allocatable MTT entries
  *
@@ -334,6 +336,8 @@ struct hermon_send_work_queue {
 	union hermon_send_wqe *wqe;
 	/** Size of work queue */
 	size_t wqe_size;
+	/** Doorbell register */
+	void *doorbell;
 };
 
 /** Alignment of Hermon receive work queue entries */
@@ -400,7 +404,7 @@ struct hermon_completion_queue {
  *
  * This is a policy decision, not a device limit.
  */
-#define HERMON_MAX_EQS		4
+#define HERMON_MAX_EQS		8
 
 /** A Hermon event queue */
 struct hermon_event_queue {
@@ -410,8 +414,12 @@ struct hermon_event_queue {
 	size_t eqe_size;
 	/** MTT descriptor */
 	struct hermon_mtt mtt;
+	/** Event queue number */
+	unsigned long eqn;
 	/** Next event queue entry index */
 	unsigned long next_idx;
+	/** Doorbell register */
+	void *doorbell;
 };
 
 /** Number of event queue entries