Started added poll_cq() verb.

Started reworking MLX_EXTRACT(), MLX_POPULATE() etc. to automatically
determine type information.

src/drivers/net/mlx_ipoib/bit_ops.h

@@ -137,112 +137,139 @@ struct addr_64_st {
 
 /* Remaining code Copyright Fen Systems Ltd. 2007 */
 
+/**
+ * Wrapper structure for pseudo_bit_t structures
+ *
+ * This structure provides a wrapper around the autogenerated
+ * pseudo_bit_t structures.  It has the correct size, and also
+ * encapsulates type information about the underlying pseudo_bit_t
+ * structure, which allows the MLX_POPULATE etc. macros to work
+ * without requiring explicit type information.
+ */
+#define MLX_DECLARE_STRUCT( _structure )				     \
+	_structure {							     \
+	    union {							     \
+		uint8_t bytes[ sizeof ( struct _structure ## _st ) / 8 ];    \
+		uint32_t dwords[ sizeof ( struct _structure ## _st ) / 32 ]; \
+		struct _structure ## _st *dummy[0];			     \
+	    } u;							     \
+	}
+
+/** Get pseudo_bit_t structure type from wrapper structure pointer */
+#define MLX_PSEUDO_STRUCT( _ptr )					     \
+	typeof ( *((_ptr)->u.dummy[0]) )
+
 /** Bit offset of a field within a pseudo_bit_t structure */
-#define MLX_BIT_OFFSET( _structure, _field )				   \
-	offsetof ( struct _structure, _field )
+#define MLX_BIT_OFFSET( _structure_st, _field )				     \
+	offsetof ( _structure_st, _field )
 
 /** Dword offset of a field within a pseudo_bit_t structure */
-#define MLX_DWORD_OFFSET( _structure, _field )				   \
-	( MLX_BIT_OFFSET ( _structure, _field ) / 32 )
+#define MLX_DWORD_OFFSET( _structure_st, _field )			     \
+	( MLX_BIT_OFFSET ( _structure_st, _field ) / 32 )
 
 /** Dword bit offset of a field within a pseudo_bit_t structure
  *
  * Yes, using mod-32 would work, but would lose the check for the
  * error of specifying a mismatched field name and dword index.
  */
-#define MLX_DWORD_BIT_OFFSET( _structure, _index, _field )		   \
-	( MLX_BIT_OFFSET ( _structure, _field ) - ( 32 * (_index) ) )
+#define MLX_DWORD_BIT_OFFSET( _structure_st, _index, _field )		     \
+	( MLX_BIT_OFFSET ( _structure_st, _field ) - ( 32 * (_index) ) )
 
 /** Bit width of a field within a pseudo_bit_t structure */
-#define MLX_BIT_WIDTH( _structure, _field )				   \
-	sizeof ( ( ( struct _structure * ) NULL )->_field )
+#define MLX_BIT_WIDTH( _structure_st, _field )				     \
+	sizeof ( ( ( _structure_st * ) NULL )->_field )
 
 /** Bit mask for a field within a pseudo_bit_t structure */
-#define MLX_BIT_MASK( _structure, _field )				   \
-	( ( 1 << MLX_BIT_WIDTH ( _structure, _field ) ) - 1 )
+#define MLX_BIT_MASK( _structure_st, _field )				     \
+	( ( 1 << MLX_BIT_WIDTH ( _structure_st, _field ) ) - 1 )
 
 /*
  * Assemble native-endian dword from named fields and values
  *
  */
 
-#define MLX_ASSEMBLE_1( _structure, _index, _field, _value )		   \
-	( (_value) << MLX_DWORD_BIT_OFFSET ( _structure, _index, _field ) )
+#define MLX_ASSEMBLE_1( _structure_st, _index, _field, _value )		     \
+	( (_value) << MLX_DWORD_BIT_OFFSET ( _structure_st, _index, _field ) )
 
-#define MLX_ASSEMBLE_2( _structure, _index, _field, _value, ... )	   \
-	( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) |	   \
-	  MLX_ASSEMBLE_1 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_ASSEMBLE_2( _structure_st, _index, _field, _value, ... )	     \
+	( MLX_ASSEMBLE_1 ( _structure_st, _index, _field, _value ) |	     \
+	  MLX_ASSEMBLE_1 ( _structure_st, _index, __VA_ARGS__ ) )
 
-#define MLX_ASSEMBLE_3( _structure, _index, _field, _value, ... )	   \
-	( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) |	   \
-	  MLX_ASSEMBLE_2 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_ASSEMBLE_3( _structure_st, _index, _field, _value, ... )	     \
+	( MLX_ASSEMBLE_1 ( _structure_st, _index, _field, _value ) |	     \
+	  MLX_ASSEMBLE_2 ( _structure_st, _index, __VA_ARGS__ ) )
 
-#define MLX_ASSEMBLE_4( _structure, _index, _field, _value, ... )	   \
-	( MLX_ASSEMBLE_1 ( _structure, _index, _field, _value ) |	   \
-	  MLX_ASSEMBLE_3 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_ASSEMBLE_4( _structure_st, _index, _field, _value, ... )	     \
+	( MLX_ASSEMBLE_1 ( _structure_st, _index, _field, _value ) |	     \
+	  MLX_ASSEMBLE_3 ( _structure_st, _index, __VA_ARGS__ ) )
 
 /*
  * Build native-endian (positive) dword bitmasks from named fields
  *
  */
 
-#define MLX_MASK_1( _structure, _index, _field )			   \
-	( MLX_BIT_MASK ( _structure, _field ) <<			   \
-	  MLX_DWORD_BIT_OFFSET ( _structure, _index, _field ) )
+#define MLX_MASK_1( _structure_st, _index, _field )			     \
+	( MLX_BIT_MASK ( _structure_st, _field ) <<			     \
+	  MLX_DWORD_BIT_OFFSET ( _structure_st, _index, _field ) )
 
-#define MLX_MASK_2( _structure, _index, _field, ... )			   \
-	( MLX_MASK_1 ( _structure, _index, _field ) |			   \
-	  MLX_MASK_1 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_MASK_2( _structure_st, _index, _field, ... )		     \
+	( MLX_MASK_1 ( _structure_st, _index, _field ) |		     \
+	  MLX_MASK_1 ( _structure_st, _index, __VA_ARGS__ ) )
 
-#define MLX_MASK_3( _structure, _index, _field, ... )			   \
-	( MLX_MASK_1 ( _structure, _index, _field ) |			   \
-	  MLX_MASK_2 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_MASK_3( _structure_st, _index, _field, ... )		     \
+	( MLX_MASK_1 ( _structure_st, _index, _field ) |		     \
+	  MLX_MASK_2 ( _structure_st, _index, __VA_ARGS__ ) )
 
-#define MLX_MASK_4( _structure, _index, _field, ... )			   \
-	( MLX_MASK_1 ( _structure, _index, _field ) |			   \
-	  MLX_MASK_3 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_MASK_4( _structure_st, _index, _field, ... )		     \
+	( MLX_MASK_1 ( _structure_st, _index, _field ) |		     \
+	  MLX_MASK_3 ( _structure_st, _index, __VA_ARGS__ ) )
 
 /*
  * Populate big-endian dwords from named fields and values
  *
  */
 
-#define MLX_POPULATE( _base, _index, _assembled )			   \
-	do {								   \
-		uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + (_index) ); \
-		uint32_t __assembled = (_assembled);			   \
-		*__ptr = cpu_to_be32 ( __assembled );			   \
+#define MLX_POPULATE( _ptr, _index, _assembled )			     \
+	do {								     \
+		uint32_t *__ptr = &(_ptr)->u.dwords[(_index)];		     \
+		uint32_t __assembled = (_assembled);			     \
+		*__ptr = cpu_to_be32 ( __assembled );			     \
 	} while ( 0 )
 
-#define MLX_POPULATE_1( _base, _structure, _index, ... )		   \
-	MLX_POPULATE ( _base, _index,					   \
-		       MLX_ASSEMBLE_1 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_POPULATE_1( _ptr, _index, ... )				     \
+	MLX_POPULATE ( _ptr, _index,					     \
+		       MLX_ASSEMBLE_1 ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					_index, __VA_ARGS__ ) )
 
-#define MLX_POPULATE_2( _base, _structure, _index, ... )		   \
-	MLX_POPULATE ( _base, _index,					   \
-		       MLX_ASSEMBLE_2 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_POPULATE_2( _ptr, _index, ... )				     \
+	MLX_POPULATE ( _ptr, _index,					     \
+		       MLX_ASSEMBLE_2 ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					_index, __VA_ARGS__ ) )
 
-#define MLX_POPULATE_3( _base, _structure, _index, ... )		   \
-	MLX_POPULATE ( _base, _index,					   \
-		       MLX_ASSEMBLE_3 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_POPULATE_3( _ptr, _index, ... )				     \
+	MLX_POPULATE ( _ptr, _index,					     \
+		       MLX_ASSEMBLE_3 ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					_index, __VA_ARGS__ ) )
 
-#define MLX_POPULATE_4( _base, _structure, _index, ... )		   \
-	MLX_POPULATE ( _base, _index,					   \
-		       MLX_ASSEMBLE_4 ( _structure, _index, __VA_ARGS__ ) )
+#define MLX_POPULATE_4( _ptr, _index, ... )				     \
+	MLX_POPULATE ( _ptr, _index,					     \
+		       MLX_ASSEMBLE_4 ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					_index, __VA_ARGS__ ) )
 
 /*
  * Modify big-endian dword using named field and value
  *
  */
 
-#define MLX_MODIFY( _base, _structure, _index, _field, _value )		   \
-	do {								   \
-		uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + (_index) ); \
-		uint32_t __value = be32_to_cpu ( *__ptr );		   \
-		__value &= ~( MLX_MASK_1 ( _structure, _index, _field ) ); \
-		__value |= MLX_ASSEMBLE_1 ( _structure, _index,		   \
-					    _field, _value );		   \
-		*__ptr = cpu_to_be32 ( __value );			   \
+#define MLX_MODIFY( _ptr, _index, _field, _value )			     \
+	do {								     \
+		uint32_t *__ptr = &(_ptr)->u.dwords[(_index)];		     \
+		uint32_t __value = be32_to_cpu ( *__ptr );		     \
+		__value &= ~( MLX_MASK_1 ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					   _index, _field ) );		     \
+		__value |= MLX_ASSEMBLE_1 ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					    _index, _field, _value );	     \
+		*__ptr = cpu_to_be32 ( __value );			     \
 	} while ( 0 )
 
 /*
@@ -250,16 +277,18 @@ struct addr_64_st {
  *
  */
 
-#define MLX_EXTRACT( _base, _structure, _field )			   \
-	( {								   \
-		unsigned int __index = 					   \
-			MLX_DWORD_OFFSET ( _structure, _field );	   \
-		uint32_t *__ptr = ( ( (uint32_t *) (_base) ) + __index );  \
-		uint32_t __value = be32_to_cpu ( *__ptr );		   \
-		__value >>= MLX_DWORD_BIT_OFFSET ( _structure, __index,	   \
-						   _field );		   \
-		__value &= MLX_BIT_MASK ( _structure, _field );		   \
-		__value;						   \
+#define MLX_EXTRACT( _ptr, _field )					     \
+	( {								     \
+		unsigned int __index = 					     \
+		    MLX_DWORD_OFFSET ( MLX_PSEUDO_STRUCT ( _ptr ), _field ); \
+		uint32_t *__ptr = &(_ptr)->u.dwords[__index];		     \
+		uint32_t __value = be32_to_cpu ( *__ptr );		     \
+		__value >>=						     \
+		    MLX_DWORD_BIT_OFFSET ( MLX_PSEUDO_STRUCT ( _ptr ),	     \
+					    __index, _field );		     \
+		__value &=						     \
+		    MLX_BIT_MASK ( MLX_PSEUDO_STRUCT ( _ptr ), _field );     \
+		__value;						     \
 	} )
 
 #endif				/* __bit_ops_h__ */

src/drivers/net/mlx_ipoib/mt25218.c

@@ -23,14 +23,23 @@ Skeleton NIC driver for Etherboot
 
 #include "mt25218_imp.c"
 
+#include "arbel.h"
+
 struct arbel_send_work_queue {
-	/** Doorbell number */
+	/** Doorbell record number */
 	unsigned int doorbell_idx;
 	/** Work queue entries */
 	//	struct ud_send_wqe_st *wqe;
 	union ud_send_wqe_u *wqe_u;
 };
 
+struct arbel_completion_queue {
+	/** Doorbell record number */
+	unsigned int doorbell_idx;
+	/** Completion queue entries */
+	union cqe_st *cqe;
+};
+
 struct arbel {
 	/** User Access Region */
 	void *uar;
@@ -143,13 +152,14 @@ static int mlx_transmit_direct ( struct net_device *netdev,
 		},
 	};
 	struct ud_av_st *bcast_av = mlx->bcast_av;
-	struct address_vector_st *bav = &bcast_av->av;
+	struct arbelprm_ud_address_vector *bav =
+		( struct arbelprm_ud_address_vector * ) &bcast_av->av;
 	struct ib_address_vector av = {
 		.dest_qp = bcast_av->dest_qp,
 		.qkey = bcast_av->qkey,
-		.dlid = MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, rlid ),
-		.rate = ( MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, max_stat_rate ) ? 1 : 4 ),
-		.sl = MLX_EXTRACT ( bav, arbelprm_ud_address_vector_st, sl ),
+		.dlid = MLX_EXTRACT ( bav, rlid ),
+		.rate = ( MLX_EXTRACT ( bav, max_stat_rate ) ? 1 : 4 ),
+		.sl = MLX_EXTRACT ( bav, sl ),
 		.gid_present = 1,
 	};
 	memcpy ( &av.gid, ( ( void * ) bav ) + 16, 16 );
@@ -301,6 +311,13 @@ static struct ib_gid arbel_no_gid = {
 	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }
 };
 
+/**
+ * Ring doorbell register in UAR
+ *
+ * @v arbel		Arbel device
+ * @v db_reg		Doorbell register structure
+ * @v offset		Address of doorbell
+ */
 static void arbel_ring_doorbell ( struct arbel *arbel, void *db_reg,
 				  unsigned int offset ) {
 	uint32_t *db_reg_dword = db_reg;
@@ -315,6 +332,15 @@ static void arbel_ring_doorbell ( struct arbel *arbel, void *db_reg,
 	writel ( db_reg_dword[1], ( arbel->uar + offset + 4 ) );
 }
 
+/**
+ * Post send work queue entry
+ *
+ * @v ibdev		Infiniband device
+ * @v iobuf		I/O buffer
+ * @v av		Address vector
+ * @v qp		Queue pair
+ * @ret rc		Return status code
+ */
 static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf,
 			     struct ib_address_vector *av,
 			     struct ib_queue_pair *qp ) {
@@ -365,14 +391,8 @@ static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf,
 			 destination_qp, av->dest_qp );
 	MLX_POPULATE_1 ( &wqe->udseg, arbelprm_wqe_segment_ud_st, 9,
 			 q_key, av->qkey );
-
 	wqe->mpointer[0].local_addr_l =
 		cpu_to_be32 ( virt_to_bus ( iobuf->data ) );
-
-	//	memcpy ( bus_to_virt ( be32_to_cpu ( wqe->mpointer[0].local_addr_l ) ),
-	//		 iobuf->data, iob_len ( iobuf ) );
-
-
 	wqe->mpointer[0].byte_count = cpu_to_be32 ( iob_len ( iobuf ) );
 
 	DBG ( "Work queue entry:\n" );
@@ -416,8 +436,69 @@ static int arbel_post_send ( struct ib_device *ibdev, struct io_buffer *iobuf,
 	return 0;
 }
 
+static void arbel_parse_completion ( struct arbel *arbel,
+				     union cqe_st *cqe,
+				     struct ib_completion *completion ) {
+	memset ( completion, 0, sizeof ( *completion ) );
+	is_send = MLX_EXTRACT ( cqe, arbelprm_completion_queue_entry_st, s );					
+	completion->len =
+		MLX_EXTRACT ( cqe, arbelprm_completion_queue_entry_st,
+			      byte_cnt );}
+
+/**
+ * Poll completion queue
+ *
+ * @v ibdev		Infiniband device
+ * @v cq		Completion queue
+ * @v complete		Completion handler
+ */
+static void arbel_poll_cq ( struct ib_device *ibdev,
+			    struct ib_completion_queue *cq,
+			    ib_completer_t complete_send,
+			    ib_completer_t complete_recv ) {
+	struct arbel *arbel = ibdev->priv;
+	struct arbel_completion_queue *arbel_cq = cq->priv;
+	unsigned int cqe_idx_mask = ( cq->num_cqes - 1 );
+	union db_record_st *db_rec = &arbel->db_rec[arbel_cq->doorbell_idx];
+	union cqe_st *cqe;
+	struct ib_completion completion;
+	struct io_buffer *iobuf;
+	int is_send;
+
+	while ( 1 ) {
+		/* Look for completion entry */
+		cqe = &arbel_cq->cqe[cq->next_idx & cqe_idx_mask];
+		if ( MLX_EXTRACT ( cqe, arbelprm_completion_queue_entry_st,
+				   owner ) != 0 ) {
+			/* Entry still owned by hardware; end of poll */
+			break;
+		}
+
+		/* Parse completion */
+
+		
+		
+		/* Handle completion */
+		( is_send ? complete_send : complete_recv ) ( ibdev,
+							      &completion,
+							      iobuf );
+
+		/* Return ownership to hardware */
+		MLX_POPULATE_1 ( cqe, arbelprm_completion_queue_entry_st, 7,
+				 owner, 1 );
+		barrier();
+		/* Update completion queue's index */
+		cq->next_idx++;
+		/* Update doorbell record */
+		MLX_POPULATE_1 ( db_rec, arbelprm_cq_ci_db_record_st, 0,
+				 counter, ( cq->next_idx & 0xffffffffUL ) );
+	}
+}
+
+/** Arbel Infiniband operations */
 static struct ib_device_operations arbel_ib_operations = {
 	.post_send	= arbel_post_send,
+	.poll_cq	= arbel_poll_cq,
 };
 
 /**

src/include/gpxe/infiniband.h

@@ -63,6 +63,7 @@ struct ibhdr {
 
 
 
+struct ib_device;
 
 /** An Infiniband Work Queue */
 struct ib_work_queue {
@@ -71,9 +72,11 @@ struct ib_work_queue {
 	/** Next work queue entry index
 	 *
 	 * This is the index of the next entry to be filled (i.e. the
-	 * first empty entry).
+	 * first empty entry).  This value is not bounded by num_wqes;
+	 * users must logical-AND with (num_wqes-1) to generate an
+	 * array index.
 	 */
-	unsigned int next_idx;
+	unsigned long next_idx;
 	/** I/O buffers assigned to work queue */
 	struct io_buffer **iobufs;
 	/** Driver private data */
@@ -92,6 +95,38 @@ struct ib_queue_pair {
 	void *priv;
 };
 
+/** An Infiniband Completion Queue */
+struct ib_completion_queue {
+	/** Number of completion queue entries */
+	unsigned int num_cqes;
+	/** Next completion queue entry index
+	 *
+	 * This is the index of the next entry to be filled (i.e. the
+	 * first empty entry).  This value is not bounded by num_wqes;
+	 * users must logical-AND with (num_wqes-1) to generate an
+	 * array index.
+	 */
+	unsigned long next_idx;
+	/** Driver private data */
+	void *priv;
+};
+
+/** An Infiniband completion */
+struct ib_completion {
+	/** Length */
+	size_t len;
+};
+
+/** An Infiniband completion handler
+ *
+ * @v ibdev		Infiniband device
+ * @v completion	Completion
+ * @v iobuf		I/O buffer
+ */
+typedef void ( * ib_completer_t ) ( struct ib_device *ibdev,
+				    struct ib_completion *completion,
+				    struct io_buffer *iobuf );
+
 /** An Infiniband Address Vector */
 struct ib_address_vector {
 	/** Destination Queue Pair */
@@ -110,15 +145,13 @@ struct ib_address_vector {
 	struct ib_gid gid;
 };
 
-struct ib_device;
-
 /**
  * Infiniband device operations
  *
  * These represent a subset of the Infiniband Verbs.
  */
 struct ib_device_operations {
-	/** Post Send work queue entry
+	/** Post send work queue entry
 	 *
 	 * @v ibdev		Infiniband device
 	 * @v iobuf		I/O buffer
@@ -135,6 +168,19 @@ struct ib_device_operations {
 			      struct io_buffer *iobuf,
 			      struct ib_address_vector *av,
 			      struct ib_queue_pair *qp );
+	/** Poll completion queue
+	 *
+	 * @v ibdev		Infiniband device
+	 * @v cq		Completion queue
+	 * @v complete_send	Send completion handler
+	 * @v complete_recv	Receive completion handler
+	 *
+	 * The completion handler takes ownership of the I/O buffer.
+	 */
+	void ( * poll_cq ) ( struct ib_device *ibdev,
+			     struct ib_completion_queue *cq,
+			     ib_completer_t complete_send,
+			     ib_completer_t complete_recv );
 };
 
 /** An Infiniband device */