First (working) draft of command interface.

src/drivers/net/mlx_ipoib/arbel.h

@@ -16,14 +16,34 @@
 #define ARBEL_OPCODE_RECV_ERROR		0xfe
 #define ARBEL_OPCODE_SEND_ERROR		0xff
 
+/*
+ * HCA commands
+ *
+ */
+
+#define ARBEL_HCR_BASE			0x80680
+#define ARBEL_HCR_REG(x)		( ARBEL_HCR_BASE + 4 * (x) )
+#define ARBEL_HCR_MAX_WAIT_MS		2000
+
+#define ARBEL_HCR_OPCODE_MASK		0x0000ffffUL
+#define ARBEL_HCR_IN_IMMEDIATE		0x00010000UL
+#define ARBEL_HCR_IN_MAILBOX		0x00020000UL
+#define ARBEL_HCR_OUT_IMMEDIATE		0x00040000UL
+#define ARBEL_HCR_OUT_MAILBOX		0x00080000UL
+
+#define ARBEL_HCR_OP_SW2HW_CQ		( 0x0016 | ARBEL_HCR_IN_MAILBOX )
+#define ARBEL_HCR_OP_NOP		( 0x0031 )
+
 /*
  * Wrapper structures for hardware datatypes
  *
  */
 
+struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_context );
 struct MLX_DECLARE_STRUCT ( arbelprm_completion_queue_entry );
 struct MLX_DECLARE_STRUCT ( arbelprm_completion_with_error );
 struct MLX_DECLARE_STRUCT ( arbelprm_cq_ci_db_record );
+struct MLX_DECLARE_STRUCT ( arbelprm_hca_command_register );
 struct MLX_DECLARE_STRUCT ( arbelprm_qp_db_record );
 struct MLX_DECLARE_STRUCT ( arbelprm_recv_wqe_segment_next );
 struct MLX_DECLARE_STRUCT ( arbelprm_send_doorbell );
@@ -126,6 +146,13 @@ struct arbel_completion_queue {
 
 /** An Arbel device */
 struct arbel {
+	/** Configuration registers */
+	void *config;
+	/** Command input mailbox */
+	void *mailbox_in;
+	/** Command output mailbox */
+	void *mailbox_out;
+
 	/** User Access Region */
 	void *uar;
 	/** Doorbell records */

src/drivers/net/mlx_ipoib/mt25218.c

@@ -26,6 +26,11 @@ Skeleton NIC driver for Etherboot
 #include "arbel.h"
 
 
+static const struct ib_gid arbel_no_gid = {
+	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }
+};
+
+
 #define MLX_RX_MAX_FILL NUM_IPOIB_RCV_WQES
 
 struct mlx_nic {
@@ -263,11 +268,121 @@ static struct net_device_operations mlx_operations = {
 	.irq		= mlx_irq,
 };
 
+/**
+ * Wait for Arbel command completion
+ *
+ * @v arbel		Arbel device
+ * @ret rc		Return status code
+ */
+static int arbel_command_wait ( struct arbel *arbel,
+				struct arbelprm_hca_command_register *hcr ) {
+	unsigned int wait;
+
+	for ( wait = ARBEL_HCR_MAX_WAIT_MS ; wait ; wait-- ) {
+		hcr->u.dwords[6] =
+			readl ( arbel->config + ARBEL_HCR_REG ( 6 ) );
+		if ( MLX_GET ( hcr, go ) == 0 )
+			return 0;
+		mdelay ( 1 );
+	}
+	return -EBUSY;
+}
 
+/**
+ * Issue HCA command
+ *
+ * @v arbel		Arbel device
+ * @v op_fl		Opcode (plus implied flags)
+ * @v op_mod		Opcode modifier (0 if no modifier applicable)
+ * @v in_param		Input parameter
+ * @v in_param_len	Input parameter length
+ * @v in_mod		Input modifier (0 if no modifier applicable)
+ * @v out_param		Output parameter
+ * @ret rc		Return status code
+ */
+static int arbel_command ( struct arbel *arbel, unsigned int op_fl,
+			   unsigned int op_mod, const void *in_param,
+			   size_t in_param_len, unsigned int in_mod,
+			   void *out_param, size_t out_param_len ) {
+	struct arbelprm_hca_command_register hcr;
+	unsigned int status;
+	unsigned int i;
+	int rc;
+
+	/* Check that HCR is free */
+	if ( ( rc = arbel_command_wait ( arbel, &hcr ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p command interface locked\n", arbel );
+		return rc;
+	}
+
+	/* Prepare HCR */
+	memset ( &hcr, 0, sizeof ( hcr ) );
+	if ( op_fl & ARBEL_HCR_IN_IMMEDIATE ) {
+		memcpy ( &hcr.u.dwords[0], in_param, 8 );
+	} else if ( op_fl & ARBEL_HCR_IN_MAILBOX ) {
+		memcpy ( arbel->mailbox_in, in_param, in_param_len );
+		MLX_FILL_1 ( &hcr, 1, in_param_l,
+			     virt_to_bus ( arbel->mailbox_in ) );
+	}
+	MLX_FILL_1 ( &hcr, 2, input_modifier, in_mod );
+	if ( op_fl & ARBEL_HCR_OUT_MAILBOX ) {
+		MLX_FILL_1 ( &hcr, 4, out_param_l,
+			     virt_to_bus ( arbel->mailbox_out ) );
+	}
+	MLX_FILL_3 ( &hcr, 6,
+		     opcode, ( op_fl & ARBEL_HCR_OPCODE_MASK ),
+		     opcode_modifier, op_mod,
+		     go, 1 );
+
+	/* Issue command */
+	for ( i = 0 ; i < ( sizeof ( hcr ) / sizeof ( hcr.u.dwords[0] ) ) ;
+	      i++ ) {
+		writel ( hcr.u.dwords[i],
+			 arbel->config + ARBEL_HCR_REG ( i ) );
+		barrier();
+	}
+
+	/* Wait for command completion */
+	if ( ( rc = arbel_command_wait ( arbel, &hcr ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p timed out waiting for command:\n",
+		       arbel );
+		DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
+		return rc;
+	}
+
+	/* Check command status */
+	status = MLX_GET ( &hcr, status );
+	if ( status != 0 ) {
+		DBGC ( arbel, "Arbel %p command failed with status %02x:\n",
+		       arbel, status );
+		DBGC_HD ( arbel, &hcr, sizeof ( hcr ) );
+		return -EIO;
+	}
+
+	/* Read output parameters, if any */
+	hcr.u.dwords[3] = readl ( arbel->config + ARBEL_HCR_REG ( 3 ) );
+	hcr.u.dwords[4] = readl ( arbel->config + ARBEL_HCR_REG ( 4 ) );
+	if ( op_fl & ARBEL_HCR_OUT_IMMEDIATE ) {
+		memcpy ( out_param, &hcr.u.dwords[3], 8 );
+	} else if ( op_fl & ARBEL_HCR_OUT_MAILBOX ) {
+		memcpy ( out_param, arbel->mailbox_out, out_param_len );
+	}
+
+	return 0;
+}
+
+/**
+ * Create completion queue
+ *
+ * @v ibdev		Infiniband device
+ * @v 
+ */
+static int arbel_create_cq ( struct ib_device *ibdev ) {
+	struct arbelprm_completion_queue_context *cqctx;
+
+
+}
 
-static struct ib_gid arbel_no_gid = {
-	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2 }
-};
 
 /**
  * Ring doorbell register in UAR
@@ -310,7 +425,7 @@ static int arbel_post_send ( struct ib_device *ibdev,
 	struct arbelprm_ud_send_wqe *wqe;
 	union arbelprm_doorbell_record *db_rec;
 	union arbelprm_doorbell_register db_reg;
-	struct ib_gid *gid;
+	const struct ib_gid *gid;
 	unsigned int wqe_idx_mask;
 	size_t nds;
 
@@ -616,6 +731,9 @@ static int arbel_probe ( struct pci_device *pci,
 	memcpy ( &mac->gid, ib_data.port_gid.raw, sizeof ( mac->gid ) );
 
 	/* Hack up IB structures */
+	static_arbel.config = memfree_pci_dev.cr_space;
+	static_arbel.mailbox_in = dev_buffers_p->inprm_buf;
+	static_arbel.mailbox_out = dev_buffers_p->outprm_buf;
 	static_arbel.uar = memfree_pci_dev.uar;
 	static_arbel.db_rec = dev_ib_data.uar_context_base;
 	static_arbel.reserved_lkey = dev_ib_data.mkey;
@@ -634,6 +752,16 @@ static int arbel_probe ( struct pci_device *pci,
 	list_add ( &static_ipoib_qp.recv.list,
 		   &static_ipoib_recv_cq.work_queues );
 
+	uint8_t buf[512];
+	memset ( buf, 0xaa, sizeof ( buf ) );
+	if ( ( rc = arbel_command ( &static_arbel,
+				    ( 0x03 | ARBEL_HCR_OUT_MAILBOX ), 0,
+				    NULL, 0, 0, buf, 256 ) ) != 0 ) {
+		DBG ( "QUERY_DEV_LIM failed: %s\n", strerror ( rc ) );
+	}
+	DBG ( "Device limits:\n ");
+	DBG_HD ( &buf[0], sizeof ( buf ) );
+
 	/* Register network device */
 	if ( ( rc = register_netdev ( netdev ) ) != 0 )
 		goto err_register_netdev;