[mtnic] Add multiport support and some minor fixes

Signed-off-by: Michael Brown <mcb30@etherboot.org>

src/drivers/net/mtnic.h

 /*
 * Device setup
 */
-
-/*
-	Note port number can be changed under mtnic.c !
-*/
 #define MTNIC_MAX_PORTS		2
+#define MTNIC_PORT1		0
+#define MTNIC_PORT2		1
 #define NUM_TX_RINGS		1
 #define NUM_RX_RINGS		1
 #define NUM_CQS 		(NUM_RX_RINGS + NUM_TX_RINGS)
 #define GO_BIT_TIMEOUT		6000
 #define TBIT_RETRIES		100
 #define UNITS_BUFFER_SIZE 	8 /* can be configured to 4/8/16 */
-#define MAX_GAP_PROD_CONS 	(UNITS_BUFFER_SIZE/4)
-#define DEF_MTU 		1600
-#define DEF_IOBUF_SIZE 		1600
+#define MAX_GAP_PROD_CONS 	( UNITS_BUFFER_SIZE / 4 )
+#define ETH_DEF_LEN		1540          /* 40 bytes used by the card */
+#define ETH_FCS_LEN		14
+#define DEF_MTU 		ETH_DEF_LEN + ETH_FCS_LEN
+#define DEF_IOBUF_SIZE 		ETH_DEF_LEN
+
 #define MAC_ADDRESS_SIZE 	6
 #define NUM_EQES 		16
 #define ROUND_TO_CHECK		0x400
 
+#define DELAY_LINK_CHECK	300
+#define CHECK_LINK_TIMES	7
+
 
 #define XNOR(x,y)		(!(x) == !(y))
 #define dma_addr_t 		unsigned long
 	MTNIC_IF_CMD_CONFIG_RX = 0x005, /* general receive configuration */
 	MTNIC_IF_CMD_CONFIG_TX = 0x006, /* general transmit configuration */
 	MTNIC_IF_CMD_CONFIG_INT_FREQ = 0x007, /* interrupt timers freq limits */
-	MTNIC_IF_CMD_HEART_BEAT	= 0x008, /* NOP command testing liveliness */
+	MTNIC_IF_CMD_HEART_BEAT = 0x008, /* NOP command testing liveliness */
 	MTNIC_IF_CMD_CLOSE_NIC = 0x009, /* release memory and stop the NIC */
 
 	/* Port commands: */
 	MTNIC_IF_CMD_CONFIG_PORT_VLAN_FILTER   = 0x14, /* configure VLAN filter */
 	MTNIC_IF_CMD_CONFIG_PORT_MCAST_FILTER  = 0x15, /* configure mcast filter */
 	MTNIC_IF_CMD_ENABLE_PORT_MCAST_FILTER  = 0x16, /* enable/disable */
-	MTNIC_IF_CMD_SET_PORT_MTU	       = 0x17, /* set port MTU */
+	MTNIC_IF_CMD_SET_PORT_MTU              = 0x17, /* set port MTU */
 	MTNIC_IF_CMD_SET_PORT_PROMISCUOUS_MODE = 0x18, /* enable/disable promisc */
 	MTNIC_IF_CMD_SET_PORT_DEFAULT_RING     = 0x19, /* set the default ring */
-	MTNIC_IF_CMD_SET_PORT_STATE	       = 0x1a, /* set link up/down */
-	MTNIC_IF_CMD_DUMP_STAT		       = 0x1b, /* dump statistics */
+	MTNIC_IF_CMD_SET_PORT_STATE            = 0x1a, /* set link up/down */
+	MTNIC_IF_CMD_DUMP_STAT                 = 0x1b, /* dump statistics */
 	MTNIC_IF_CMD_ARM_PORT_STATE_EVENT      = 0x1c, /* arm the port state event */
 
 	/* Ring / Completion queue commands: */
-	MTNIC_IF_CMD_CONFIG_CQ		  = 0x20,  /* set up completion queue */
-	MTNIC_IF_CMD_CONFIG_RX_RING	  = 0x21,  /* setup Rx ring */
-	MTNIC_IF_CMD_SET_RX_RING_ADDR	  = 0x22,  /* set Rx ring filter by address */
+	MTNIC_IF_CMD_CONFIG_CQ            = 0x20,  /* set up completion queue */
+	MTNIC_IF_CMD_CONFIG_RX_RING       = 0x21,  /* setup Rx ring */
+	MTNIC_IF_CMD_SET_RX_RING_ADDR     = 0x22,  /* set Rx ring filter by address */
 	MTNIC_IF_CMD_SET_RX_RING_MCAST    = 0x23,  /* set Rx ring mcast filter */
-	MTNIC_IF_CMD_ARM_RX_RING_WM	  = 0x24,  /* one-time low-watermark INT */
-	MTNIC_IF_CMD_CONFIG_TX_RING	  = 0x25,  /* set up Tx ring */
+	MTNIC_IF_CMD_ARM_RX_RING_WM       = 0x24,  /* one-time low-watermark INT */
+	MTNIC_IF_CMD_CONFIG_TX_RING       = 0x25,  /* set up Tx ring */
 	MTNIC_IF_CMD_ENFORCE_TX_RING_ADDR = 0x26,  /* setup anti spoofing */
-	MTNIC_IF_CMD_CONFIG_EQ		  = 0x27,  /* config EQ ring */
+	MTNIC_IF_CMD_CONFIG_EQ            = 0x27,  /* config EQ ring */
 	MTNIC_IF_CMD_RELEASE_RESOURCE     = 0x28,  /* release internal ref to resource */
 }
 mtnic_if_cmd_t;
 typedef enum mtnic_if_caps {
 	MTNIC_IF_CAP_MAX_TX_RING_PER_PORT = 0x0,
 	MTNIC_IF_CAP_MAX_RX_RING_PER_PORT = 0x1,
-	MTNIC_IF_CAP_MAX_CQ_PER_PORT	  = 0x2,
-	MTNIC_IF_CAP_NUM_PORTS		  = 0x3,
-	MTNIC_IF_CAP_MAX_TX_DESC	  = 0x4,
-	MTNIC_IF_CAP_MAX_RX_DESC	  = 0x5,
-	MTNIC_IF_CAP_MAX_CQES		  = 0x6,
-	MTNIC_IF_CAP_MAX_TX_SG_ENTRIES	  = 0x7,
-	MTNIC_IF_CAP_MAX_RX_SG_ENTRIES	  = 0x8,
-	MTNIC_IF_CAP_MEM_KEY  		  = 0x9, /* key to mem (after map_pages) */
-	MTNIC_IF_CAP_RSS_HASH_TYPE	  = 0xa, /* one of mtnic_if_rss_types_t */
+	MTNIC_IF_CAP_MAX_CQ_PER_PORT      = 0x2,
+	MTNIC_IF_CAP_NUM_PORTS            = 0x3,
+	MTNIC_IF_CAP_MAX_TX_DESC          = 0x4,
+	MTNIC_IF_CAP_MAX_RX_DESC          = 0x5,
+	MTNIC_IF_CAP_MAX_CQES             = 0x6,
+	MTNIC_IF_CAP_MAX_TX_SG_ENTRIES    = 0x7,
+	MTNIC_IF_CAP_MAX_RX_SG_ENTRIES    = 0x8,
+	MTNIC_IF_CAP_MEM_KEY              = 0x9, /* key to mem (after map_pages) */
+	MTNIC_IF_CAP_RSS_HASH_TYPE        = 0xa, /* one of mtnic_if_rss_types_t */
 	MTNIC_IF_CAP_MAX_PORT_UCAST_ADDR  = 0xc,
 	MTNIC_IF_CAP_MAX_RING_UCAST_ADDR  = 0xd, /* only for ADDR steer */
 	MTNIC_IF_CAP_MAX_PORT_MCAST_ADDR  = 0xe,
 	MTNIC_IF_CAP_EQ_DB_OFFSET         = 0x14, /* offset in bytes for EQ doorbell record */
 
 	/* These are per port - using port number from cap modifier field */
-	MTNIC_IF_CAP_SPEED		  = 0x20,
-	MTNIC_IF_CAP_DEFAULT_MAC	  = 0x21,
-	MTNIC_IF_CAP_EQ_OFFSET		  = 0x22,
-	MTNIC_IF_CAP_CQ_OFFSET		  = 0x23,
+	MTNIC_IF_CAP_SPEED                = 0x20,
+	MTNIC_IF_CAP_DEFAULT_MAC          = 0x21,
+	MTNIC_IF_CAP_EQ_OFFSET            = 0x22,
+	MTNIC_IF_CAP_CQ_OFFSET            = 0x23,
 	MTNIC_IF_CAP_TX_OFFSET            = 0x24,
 	MTNIC_IF_CAP_RX_OFFSET            = 0x25,
 
 } mtnic_if_caps_t;
 
 typedef enum mtnic_if_steer_types {
-        MTNIC_IF_STEER_NONE     = 0,
-        MTNIC_IF_STEER_PRIORITY = 1,
-        MTNIC_IF_STEER_RSS      = 2,
-        MTNIC_IF_STEER_ADDRESS  = 3,
+	MTNIC_IF_STEER_NONE     = 0,
+	MTNIC_IF_STEER_PRIORITY = 1,
+	MTNIC_IF_STEER_RSS      = 2,
+	MTNIC_IF_STEER_ADDRESS  = 3,
 } mtnic_if_steer_types_t;
 
 /** types of memory access modes */
 
 
 enum {
-	MTNIC_HCR_BASE		= 0x1f000,
-	MTNIC_HCR_SIZE		= 0x0001c,
-	MTNIC_CLR_INT_SIZE	= 0x00008,
+	MTNIC_HCR_BASE          = 0x1f000,
+	MTNIC_HCR_SIZE          = 0x0001c,
+	MTNIC_CLR_INT_SIZE      = 0x00008,
 };
 
-#define MELLANOX_VENDOR_ID	0x15b3
-#define MTNIC_DEVICE_ID 	0x00a00190
 #define MTNIC_RESET_OFFSET 	0xF0010
-#define MTNIC_DEVICE_ID_OFFSET 	0xF0014
-
-
-
-
 
 
 
 
 	/* Buffers */
 	u32 buf_size; /* ring buffer size in bytes */
-        dma_addr_t dma;
+	dma_addr_t dma;
 	void *buf;
 	struct io_buffer *iobuf[UNITS_BUFFER_SIZE];
 
 	u32 db_offset;
 
 	/* Rx ring only */
-        dma_addr_t iobuf_dma;
+	dma_addr_t iobuf_dma;
 	struct mtnic_rx_db_record *db;
 	dma_addr_t db_dma;
 };
 
 struct mtnic_eq {
 	u32 size; /* number of EQEs in ring */
-        u32 buf_size; /* EQ size in bytes */
+	u32 buf_size; /* EQ size in bytes */
 	void *buf;
 	dma_addr_t dma;
 };
 
 enum mtnic_state {
 	CARD_DOWN,
-        CARD_INITIALIZED,
-        CARD_UP
+	CARD_INITIALIZED,
+	CARD_UP,
+	CARD_LINK_DOWN,
 };
 
 /* FW */
 
 
 struct mtnic_cmd {
-	void                     *buf;
-	unsigned long             mapping;
-	u32	 	      	  tbit;
+	void                    *buf;
+	unsigned long           mapping;
+	u32                     tbit;
 };
 
 
  * Device private data
  *
  */
-struct mtnic_priv {
-	struct net_device *dev;
-	struct pci_device *pdev;
-	u8 port;
+struct mtnic {
+	struct net_device               *netdev[MTNIC_MAX_PORTS];
+	struct mtnic_if_cmd_reg         *hcr;
+	struct mtnic_cmd                cmd;
+	struct pci_device               *pdev;
 
-	enum mtnic_state		state;
-        /* Firmware and board info */
-	u64              		fw_ver;
+	struct mtnic_eq                 eq;
+	u32                             *eq_db;
+
+	/* Firmware and board info */
+	u64                             fw_ver;
 	struct {
-		struct mtnic_pages	fw_pages;
-		struct mtnic_pages	extra_pages;
-		struct mtnic_err_buf 	err_buf;
-		u16			ifc_rev;
-		u8			num_ports;
-                u64			mac[MTNIC_MAX_PORTS];
-		u16			cq_offset;
-		u16			tx_offset[MTNIC_MAX_PORTS];
-		u16			rx_offset[MTNIC_MAX_PORTS];
-                u32			mem_type_snoop_be;
-                u32			txcq_db_offset;
-		u32			eq_db_offset;
-        } fw;
-
-
-	struct mtnic_if_cmd_reg 	*hcr;
-        struct mtnic_cmd		cmd;
+		struct mtnic_pages      fw_pages;
+		struct mtnic_pages      extra_pages;
+		struct mtnic_err_buf    err_buf;
+		u16                     ifc_rev;
+		u8                      num_ports;
+		u64                     mac[MTNIC_MAX_PORTS];
+		u16                     cq_offset;
+		u16                     tx_offset[MTNIC_MAX_PORTS];
+		u16                     rx_offset[MTNIC_MAX_PORTS];
+		u32                     mem_type_snoop_be;
+		u32                     txcq_db_offset;
+		u32                     eq_db_offset;
+	} fw;
+};
+
+
+
+
+
+struct mtnic_port {
+
+	struct mtnic                    *mtnic;
+	u8                              port;
+
+	enum mtnic_state                state;
 
 	/* TX, RX, CQs, EQ */
-        struct mtnic_ring tx_ring;
-	struct mtnic_ring rx_ring;
-	struct mtnic_cq cq[NUM_CQS];
-	struct mtnic_eq			eq;
-	u32 				*eq_db;
-	u32				poll_counter;
+	struct mtnic_ring               tx_ring;
+	struct mtnic_ring               rx_ring;
+	struct mtnic_cq                 cq[NUM_CQS];
+	u32                             poll_counter;
+	struct net_device               *netdev;
+
+
 };
 
 
 /* CMD MTNIC_IF_CMD_QUERY_CAP */
 struct mtnic_if_query_cap_in_imm {
 	u16 reserved1;
-	u8		 cap_modifier;	 /* a modifier for the particular capability */
-	u8		 cap_index;	 /* the index of the capability queried */
+	u8               cap_modifier;   /* a modifier for the particular capability */
+	u8               cap_index;      /* the index of the capability queried */
 	u32 reserved2;
 };
 
 /* CMD OPEN_NIC */
 struct mtnic_if_open_nic_in_mbox {
-    u16 reserved1;
-    u16 mkey; /* number of mem keys for all chip*/
-    u32 mkey_entry; /* mem key entries for each key*/
-    u8 log_rx_p1; /* log2 rx rings for port1 */
-    u8 log_cq_p1; /* log2 cq for port1 */
-    u8 log_tx_p1; /* log2 tx rings for port1 */
-    u8 steer_p1;  /* port 1 steering mode */
-    u16 reserved2;
-    u8 log_vlan_p1; /* log2 vlan per rx port1 */
-    u8 log_mac_p1;  /* log2 mac per rx port1 */
-
-    u8 log_rx_p2; /* log2 rx rings for port1 */
-    u8 log_cq_p2; /* log2 cq for port1 */
-    u8 log_tx_p2; /* log2 tx rings for port1 */
-    u8 steer_p2;  /* port 1 steering mode */
-    u16 reserved3;
-    u8 log_vlan_p2; /* log2 vlan per rx port1 */
-    u8 log_mac_p2;  /* log2 mac per rx port1 */
+	u16 reserved1;
+	u16 mkey; /* number of mem keys for all chip*/
+	u32 mkey_entry; /* mem key entries for each key*/
+	u8 log_rx_p1; /* log2 rx rings for port1 */
+	u8 log_cq_p1; /* log2 cq for port1 */
+	u8 log_tx_p1; /* log2 tx rings for port1 */
+	u8 steer_p1;  /* port 1 steering mode */
+	u16 reserved2;
+	u8 log_vlan_p1; /* log2 vlan per rx port1 */
+	u8 log_mac_p1;  /* log2 mac per rx port1 */
+
+	u8 log_rx_p2; /* log2 rx rings for port1 */
+	u8 log_cq_p2; /* log2 cq for port1 */
+	u8 log_tx_p2; /* log2 tx rings for port1 */
+	u8 steer_p2;  /* port 1 steering mode */
+	u16 reserved3;
+	u8 log_vlan_p2; /* log2 vlan per rx port1 */
+	u8 log_mac_p2;  /* log2 mac per rx port1 */
 };
 
+
 /* CMD CONFIG_RX */
 struct mtnic_if_config_rx_in_imm {
 	u16 spkt_size; /* size of small packets interrupts enabled on CQ */
 
 /* CMD HEART_BEAT */
 struct mtnic_if_heart_beat_out_imm {
-    u32 flags; /* several flags */
+	u32 flags; /* several flags */
 #define MTNIC_MASK_HEAR_BEAT_INT_ERROR  MTNIC_BC(31,1)
-    u32 reserved;
+	u32 reserved;
 };
 
 
 /* CMD CONFIG_PORT_VLAN_FILTER */
 /* in mbox is a 4K bits mask - bit per VLAN */
 struct mtnic_if_config_port_vlan_filter_in_mbox {
-    u64 filter[64]; /* vlans[63:0] sit in filter[0], vlans[127:64] sit in filter[1] ..  */
+	u64 filter[64]; /* vlans[63:0] sit in filter[0], vlans[127:64] sit in filter[1] ..  */
 };
 
 
 /* CMD SET_PORT_MTU */
 struct mtnic_if_set_port_mtu_in_imm {
 	u16 reserved1;
-	u16 mtu;			/* The MTU of the port in bytes */
+	u16 mtu;                        /* The MTU of the port in bytes */
 	u32 reserved2;
 };
 
 
 /* CMD CONFIG_CQ */
 struct mtnic_if_config_cq_in_mbox {
-	u8	     reserved1;
-	u8	     cq;
-	u8	     size;	  /* Num CQs is 2^size (size <= 22) */
-	u8	     offset; /* start address of CQE in first page (11:6) */
-	u16  tlast;	 /* interrupt moderation timer from last completion usec */
+	u8           reserved1;
+	u8           cq;
+	u8           size;        /* Num CQs is 2^size (size <= 22) */
+	u8           offset; /* start address of CQE in first page (11:6) */
+	u16  tlast;      /* interrupt moderation timer from last completion usec */
 	u8      flags;  /* flags */
-	u8	    int_vector; /* MSI index if MSI is enabled, otherwise reserved */
+	u8          int_vector; /* MSI index if MSI is enabled, otherwise reserved */
 	u16 reserved2;
 	u16 max_cnt;    /* interrupt moderation counter */
-	u8	    page_size;	 /* each mapped page is 2^(12+page_size) bytes */
-	u8	 reserved4[3];
+	u8          page_size;   /* each mapped page is 2^(12+page_size) bytes */
+	u8       reserved4[3];
 	u32 db_record_addr_h;  /*physical address of CQ doorbell record */
 	u32 db_record_addr_l;  /*physical address of CQ doorbell record */
 	u32 page_address[0]; /* 64 bit page addresses of CQ buffer */
 
 /* CMD CONFIG_RX_RING */
 struct mtnic_if_config_rx_ring_in_mbox {
-	u8	 reserved1;
-	u8	 ring;				/* The ring index (with offset) */
-	u8	 stride_size;		/* stride and size */
+	u8       reserved1;
+	u8       ring;                          /* The ring index (with offset) */
+	u8       stride_size;           /* stride and size */
 	/* Entry size = 16* (2^stride) bytes */
 #define MTNIC_MASK_CONFIG_RX_RING_STRIDE     MTNIC_BC(4,3)
 	/* Rx ring size is 2^size entries */
 #define MTNIC_MASK_CONFIG_RX_RING_SIZE	      MTNIC_BC(0,4)
-	u8	 flags;				/* Bit0 - header separation */
-	u8	 page_size;			  /* Each mapped page is 2^(12+page_size) bytes */
-	u8	 reserved2[2];
-	u8	 cq;					  /* CQ associated with this ring */
-	u32	 db_record_addr_h;
-	u32	 db_record_addr_l;
-	u32 	 page_address[0];/* Array of 2^size 64b page descriptor addresses */
-								  /* Must hold all Rx descriptors + doorbell record. */
+	u8       flags;                         /* Bit0 - header separation */
+	u8       page_size;                       /* Each mapped page is 2^(12+page_size) bytes */
+	u8       reserved2[2];
+	u8       cq;                                      /* CQ associated with this ring */
+	u32      db_record_addr_h;
+	u32      db_record_addr_l;
+	u32      page_address[0];/* Array of 2^size 64b page descriptor addresses */
+	/* Must hold all Rx descriptors + doorbell record. */
 };
 
 /* The modifier for SET_RX_RING_ADDR */
 
 /* CMD SET_RX_RING_ADDR */
 struct mtnic_if_set_rx_ring_addr_in_imm {
-	u16 mac_47_32;		 /* UCAST MAC Address bits 47:32 */
+	u16 mac_47_32;           /* UCAST MAC Address bits 47:32 */
 	u16 flags_vlan_id; /* MAC/VLAN flags and vlan id */
 #define MTNIC_MASK_SET_RX_RING_ADDR_VLAN_ID MTNIC_BC(0,12)
 #define MTNIC_MASK_SET_RX_RING_ADDR_BY_MAC  MTNIC_BC(12,1)
 #define MTNIC_MASK_SET_RX_RING_ADDR_BY_VLAN MTNIC_BC(13,1)
-	u32 mac_31_0;	/* UCAST MAC Address bits 31:0 */
+	u32 mac_31_0;   /* UCAST MAC Address bits 31:0 */
 };
 
 /* CMD CONFIG_TX_RING */
 struct mtnic_if_config_send_ring_in_mbox {
-	u16 ring;			/* The ring index (with offset) */
+	u16 ring;                       /* The ring index (with offset) */
 #define MTNIC_MASK_CONFIG_TX_RING_INDEX  MTNIC_BC(0,8)
-	u8	 size;				/* Tx ring size is 32*2^size bytes */
+	u8       size;                          /* Tx ring size is 32*2^size bytes */
 #define MTNIC_MASK_CONFIG_TX_RING_SIZE	  MTNIC_BC(0,4)
-	u8	 reserved;
-	u8	 page_size;			/* Each mapped page is 2^(12+page_size) bytes */
-	u8	 qos_class;			/* The COS used for this Tx */
-	u16 cq;				/* CQ associated with this ring */
+	u8       reserved;
+	u8       page_size;                     /* Each mapped page is 2^(12+page_size) bytes */
+	u8       qos_class;                     /* The COS used for this Tx */
+	u16 cq;                         /* CQ associated with this ring */
 #define MTNIC_MASK_CONFIG_TX_CQ_INDEX	  MTNIC_BC(0,8)
 	u32 page_address[0]; /* 64 bit page addresses of descriptor buffer. */
-			/* The buffer must accommodate all Tx descriptors */
+	/* The buffer must accommodate all Tx descriptors */
 };
 
 /* CMD CONFIG_EQ */
 	u8 reserved1;
 	u8 int_vector; /* MSI index if MSI enabled; otherwise reserved */
 #define MTNIC_MASK_CONFIG_EQ_INT_VEC MTNIC_BC(0,6)
-	u8 size;			/* Num CQs is 2^size entries (size <= 22) */
+	u8 size;                        /* Num CQs is 2^size entries (size <= 22) */
 #define MTNIC_MASK_CONFIG_EQ_SIZE	 MTNIC_BC(0,5)
-	u8 offset;		/* Start address of CQE in first page (11:6) */
+	u8 offset;              /* Start address of CQE in first page (11:6) */
 #define MTNIC_MASK_CONFIG_EQ_OFFSET	 MTNIC_BC(0,6)
 	u8 page_size; /* Each mapped page is 2^(12+page_size) bytes*/
 	u8 reserved[3];