[golan] Add Connect-IB, ConnectX-4 and ConnectX-4 Lx (Infiniband) support

Signed-off-by: Wissam Shoukair <wissams@mellanox.com>
Signed-off-by: Michael Brown <mcb30@ipxe.org>

src/drivers/infiniband/mlx_nodnic/include/mlx_cmd.h

@@ -0,0 +1,43 @@
+#ifndef NODNIC_CMD_H_
+#define NODNIC_CMD_H_
+
+/*
+ * Copyright (C) 2015 Mellanox Technologies Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include "mlx_nodnic_data_structures.h"
+#include "../../mlx_utils/include/public/mlx_utils.h"
+#include "../../mlx_utils/include/public/mlx_pci_gw.h"
+
+mlx_status
+nodnic_cmd_read(
+				IN nodnic_device_priv *device_priv,
+				IN mlx_uint32 address,
+				OUT mlx_pci_gw_buffer *buffer
+				);
+
+mlx_status
+nodnic_cmd_write(
+				IN nodnic_device_priv *device_priv,
+				IN mlx_uint32 address,
+				IN mlx_pci_gw_buffer buffer
+				);
+
+#endif /* STUB_NODNIC_CMD_H_ */

src/drivers/infiniband/mlx_nodnic/include/mlx_device.h

@@ -0,0 +1,80 @@
+#ifndef NODNIC_DEVICE_H_
+#define NODNIC_DEVICE_H_
+
+/*
+ * Copyright (C) 2015 Mellanox Technologies Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include "mlx_nodnic_data_structures.h"
+
+#define NODIC_SUPPORTED_REVISION 1
+//Initialization segment
+#define NODNIC_CMDQ_PHY_ADDR_HIGH_OFFSET 0x10
+#define NODNIC_CMDQ_PHY_ADDR_LOW_OFFSET 0x14
+#define NODNIC_NIC_INTERFACE_OFFSET 0x14
+#define NODNIC_INITIALIZING_OFFSET 0x1fc
+#define NODNIC_NIC_INTERFACE_SUPPORTED_OFFSET 0x1fc
+#define NODNIC_LOCATION_OFFSET 0x240
+
+#define NODNIC_CMDQ_PHY_ADDR_LOW_MASK 0xFFFFE000
+#define NODNIC_NIC_INTERFACE_SUPPORTED_MASK 0x4000000
+
+#define NODNIC_NIC_INTERFACE_BIT 9
+#define NODNIC_DISABLE_INTERFACE_BIT 8
+#define NODNIC_NIC_INTERFACE_SUPPORTED_BIT 26
+#define NODNIC_INITIALIZING_BIT 31
+
+#define NODNIC_NIC_DISABLE_INT_OFFSET	0x100c
+
+//nodnic segment
+#define NODNIC_REVISION_OFFSET 0x0
+#define NODNIC_HARDWARE_FORMAT_OFFSET 0x0
+
+
+
+mlx_status
+nodnic_device_init(
+				IN nodnic_device_priv *device_priv
+				);
+
+mlx_status
+nodnic_device_teardown(
+				IN nodnic_device_priv *device_priv
+				);
+
+
+mlx_status
+nodnic_device_get_cap(
+				IN nodnic_device_priv *device_priv
+				);
+
+mlx_status
+nodnic_device_clear_int (
+				IN nodnic_device_priv *device_priv
+				);
+
+mlx_status
+nodnic_device_get_fw_version(
+				IN nodnic_device_priv *device_priv,
+				OUT mlx_uint16		*fw_ver_minor,
+				OUT mlx_uint16  	*fw_ver_sub_minor,
+				OUT mlx_uint16  	*fw_ver_major
+				);
+#endif /* STUB_NODNIC_DEVICE_H_ */

src/drivers/infiniband/mlx_nodnic/include/mlx_nodnic_data_structures.h

@@ -0,0 +1,201 @@
+#ifndef NODNIC_NODNICDATASTRUCTURES_H_
+#define NODNIC_NODNICDATASTRUCTURES_H_
+
+/*
+ * Copyright (C) 2015 Mellanox Technologies Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include "../../mlx_utils/include/public/mlx_utils.h"
+
+/* todo: fix coding convention */
+#define NODNIC_MEMORY_ALIGN		0x1000
+
+#define NODNIC_MAX_MAC_FILTERS 5
+#define NODNIC_MAX_MGID_FILTERS 4
+
+typedef struct _nodnic_device_priv 				nodnic_device_priv;
+typedef struct _nodnic_port_priv 				nodnic_port_priv;
+typedef struct _nodnic_device_capabilites 		nodnic_device_capabilites;
+typedef struct _nodnic_qp 						nodnic_qp;
+typedef struct _nodnic_cq 						nodnic_cq;
+typedef struct _nodnic_eq						nodnic_eq;
+
+/* NODNIC Port states
+ * Bit 0 - port open/close
+ * Bit 1 - port is [not] in disabling DMA
+ * 0 - closed and not disabling DMA
+ * 1 - opened and not disabling DMA
+ * 3 - opened and disabling DMA
+ */
+#define NODNIC_PORT_OPENED			0b00000001
+#define NODNIC_PORT_DISABLING_DMA	0b00000010
+
+typedef enum {
+	ConnectX3 = 0,
+	Connectx4
+}nodnic_hardware_format;
+
+
+typedef enum {
+	NODNIC_QPT_SMI,
+	NODNIC_QPT_GSI,
+	NODNIC_QPT_UD,
+	NODNIC_QPT_RC,
+	NODNIC_QPT_ETH,
+}nodnic_queue_pair_type;
+typedef enum {
+	NODNIC_PORT_TYPE_IB = 0,
+	NODNIC_PORT_TYPE_ETH,
+	NODNIC_PORT_TYPE_UNKNOWN,
+}nodnic_port_type;
+
+
+#define RECV_WQE_SIZE 16
+#define NODNIC_WQBB_SIZE 64
+/** A nodnic send wqbb */
+struct nodnic_send_wqbb {
+	mlx_uint8 force_align[NODNIC_WQBB_SIZE];
+};
+struct nodnic_ring {
+	mlx_uint32 offset;
+	/** Work queue entries */
+	/* TODO: add to memory entity */
+	mlx_physical_address wqe_physical;
+	mlx_void *map;
+	/** Size of work queue */
+	mlx_size wq_size;
+	/** Next work queue entry index
+	 *
+	 * This is the index of the next entry to be filled (i.e. the
+	 * first empty entry).  This value is not bounded by num_wqes;
+	 * users must logical-AND with (num_wqes-1) to generate an
+	 * array index.
+	 */
+	mlx_uint32 num_wqes;
+	mlx_uint32 qpn;
+	mlx_uint32 next_idx;
+	mlx_uint32	ring_pi;
+};
+
+struct nodnic_send_ring{
+	struct nodnic_ring nodnic_ring;
+	struct nodnic_send_wqbb *wqe_virt;
+};
+
+
+struct nodnic_recv_ring{
+	struct nodnic_ring nodnic_ring;
+	void *wqe_virt;
+};
+struct _nodnic_qp{
+	nodnic_queue_pair_type	type;
+	struct nodnic_send_ring		send;
+	struct nodnic_recv_ring		receive;
+};
+
+struct _nodnic_cq{
+	/** cq entries */
+	mlx_void *cq_virt;
+	mlx_physical_address cq_physical;
+	mlx_void *map;
+	/** cq */
+	mlx_size cq_size;
+};
+
+struct _nodnic_eq{
+	mlx_void *eq_virt;
+	mlx_physical_address eq_physical;
+	mlx_void *map;
+	mlx_size eq_size;
+};
+struct _nodnic_device_capabilites{
+	mlx_boolean					support_mac_filters;
+	mlx_boolean					support_promisc_filter;
+	mlx_boolean					support_promisc_multicast_filter;
+	mlx_uint8					log_working_buffer_size;
+	mlx_uint8					log_pkey_table_size;
+	mlx_boolean					num_ports; // 0 - single port, 1 - dual port
+	mlx_uint8					log_max_ring_size;
+#ifdef DEVICE_CX3
+	mlx_uint8					crspace_doorbells;
+#endif
+};
+
+#ifdef DEVICE_CX3
+/* This is the structure of the data in the scratchpad
+ * Read/Write data from/to its field using PCI accesses only */
+typedef struct _nodnic_port_data_flow_gw nodnic_port_data_flow_gw;
+struct _nodnic_port_data_flow_gw {
+	mlx_uint32	send_doorbell;
+	mlx_uint32	recv_doorbell;
+	mlx_uint32	reserved2[2];
+	mlx_uint32	armcq_cq_ci_dword;
+	mlx_uint32	dma_en;
+} __attribute__ ((packed));
+#endif
+
+struct _nodnic_device_priv{
+	mlx_boolean					is_initiailzied;
+	mlx_utils					*utils;
+
+	//nodnic structure offset in init segment
+	mlx_uint32					device_offset;
+
+	nodnic_device_capabilites	device_cap;
+
+	mlx_uint8					nodnic_revision;
+	nodnic_hardware_format		hardware_format;
+	mlx_uint32					pd;
+	mlx_uint32					lkey;
+	mlx_uint64					device_guid;
+	nodnic_port_priv			*ports;
+#ifdef DEVICE_CX3
+	mlx_void					*crspace_clear_int;
+#endif
+};
+
+struct _nodnic_port_priv{
+	nodnic_device_priv		*device;
+	mlx_uint32				port_offset;
+	mlx_uint8				port_state;
+	mlx_boolean				network_state;
+	mlx_boolean				dma_state;
+	nodnic_port_type		port_type;
+	mlx_uint8				port_num;
+	nodnic_eq				eq;
+	mlx_mac_address			mac_filters[5];
+	mlx_status (*send_doorbell)(
+			IN nodnic_port_priv		*port_priv,
+			IN struct nodnic_ring	*ring,
+			IN mlx_uint16 index);
+	mlx_status (*recv_doorbell)(
+			IN nodnic_port_priv		*port_priv,
+			IN struct nodnic_ring	*ring,
+			IN mlx_uint16 index);
+	mlx_status (*set_dma)(
+			IN nodnic_port_priv		*port_priv,
+			IN mlx_boolean			value);
+#ifdef DEVICE_CX3
+	nodnic_port_data_flow_gw *data_flow_gw;
+#endif
+};
+
+
+#endif /* STUB_NODNIC_NODNICDATASTRUCTURES_H_ */

src/drivers/infiniband/mlx_nodnic/include/mlx_port.h

@@ -0,0 +1,229 @@
+#ifndef NODNIC_PORT_H_
+#define NODNIC_PORT_H_
+
+/*
+ * Copyright (C) 2015 Mellanox Technologies Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include "mlx_nodnic_data_structures.h"
+
+#define NODNIC_PORT_MAC_FILTERS_OFFSET 0x10
+
+typedef enum {
+	nodnic_port_option_link_type = 0,
+	nodnic_port_option_mac_low,
+	nodnic_port_option_mac_high,
+	nodnic_port_option_log_cq_size,
+	nodnic_port_option_reset_needed,
+	nodnic_port_option_mac_filters_en,
+	nodnic_port_option_port_state,
+	nodnic_port_option_network_en,
+	nodnic_port_option_dma_en,
+	nodnic_port_option_eq_addr_low,
+	nodnic_port_option_eq_addr_high,
+	nodnic_port_option_cq_addr_low,
+	nodnic_port_option_cq_addr_high,
+	nodnic_port_option_port_management_change_event,
+	nodnic_port_option_port_promisc_en,
+	nodnic_port_option_arm_cq,
+	nodnic_port_option_port_promisc_multicast_en,
+#ifdef DEVICE_CX3
+	nodnic_port_option_crspace_en,
+#endif
+}nodnic_port_option;
+
+struct nodnic_port_data_entry{
+	nodnic_port_option	option;
+	mlx_uint32			offset;
+	mlx_uint8			align;
+	mlx_uint32			mask;
+};
+
+struct nodnic_qp_data_entry{
+	nodnic_queue_pair_type	type;
+	mlx_uint32			send_offset;
+	mlx_uint32			recv_offset;
+};
+
+
+typedef enum {
+	nodnic_port_state_down = 0,
+	nodnic_port_state_initialize,
+	nodnic_port_state_armed,
+	nodnic_port_state_active,
+}nodnic_port_state;
+
+mlx_status
+nodnic_port_get_state(
+					IN  nodnic_port_priv	*port_priv,
+					OUT nodnic_port_state			*state
+					);
+
+mlx_status
+nodnic_port_get_type(
+					IN  nodnic_port_priv	*port_priv,
+					OUT nodnic_port_type	*type
+					);
+
+mlx_status
+nodnic_port_query(
+					IN  nodnic_port_priv	*port_priv,
+					IN  nodnic_port_option		option,
+					OUT	mlx_uint32				*out
+					);
+
+mlx_status
+nodnic_port_set(
+					IN  nodnic_port_priv	*port_priv,
+					IN  nodnic_port_option		option,
+					IN	mlx_uint32				in
+					);
+
+mlx_status
+nodnic_port_create_cq(
+					IN nodnic_port_priv	*port_priv,
+					IN mlx_size	cq_size,
+					OUT nodnic_cq	**cq
+					);
+
+mlx_status
+nodnic_port_destroy_cq(
+					IN nodnic_port_priv	*port_priv,
+					IN nodnic_cq	*cq
+					);
+
+mlx_status
+nodnic_port_create_qp(
+					IN nodnic_port_priv	*port_priv,
+					IN nodnic_queue_pair_type	type,
+					IN mlx_size	send_wq_size,
+					IN mlx_uint32 send_wqe_num,
+					IN mlx_size	receive_wq_size,
+					IN mlx_uint32 recv_wqe_num,
+					OUT nodnic_qp	**qp
+					);
+
+mlx_status
+nodnic_port_destroy_qp(
+					IN nodnic_port_priv	*port_priv,
+					IN nodnic_queue_pair_type	type,
+					IN nodnic_qp	*qp
+					);
+mlx_status
+nodnic_port_get_qpn(
+			IN nodnic_port_priv	*port_priv,
+			IN struct nodnic_ring *ring,
+			OUT mlx_uint32 *qpn
+			);
+mlx_status
+nodnic_port_update_ring_doorbell(
+					IN nodnic_port_priv	*port_priv,
+					IN struct nodnic_ring *ring,
+					IN mlx_uint16 index
+					);
+mlx_status
+nodnic_port_get_cq_size(
+		IN nodnic_port_priv	*port_priv,
+		OUT mlx_uint64 *cq_size
+		);
+
+mlx_status
+nodnic_port_allocate_eq(
+					IN  nodnic_port_priv	*port_priv,
+					IN  mlx_uint8			log_eq_size
+					);
+mlx_status
+nodnic_port_free_eq(
+					IN  nodnic_port_priv	*port_priv
+					);
+
+mlx_status
+nodnic_port_add_mac_filter(
+					IN  nodnic_port_priv	*port_priv,
+					IN  mlx_mac_address 	mac
+					);
+
+mlx_status
+nodnic_port_remove_mac_filter(
+					IN  nodnic_port_priv	*port_priv,
+					IN  mlx_mac_address 	mac
+					);
+mlx_status
+nodnic_port_add_mgid_filter(
+					IN  nodnic_port_priv	*port_priv,
+					IN  mlx_mac_address 	mac
+					);
+
+mlx_status
+nodnic_port_remove_mgid_filter(
+					IN  nodnic_port_priv	*port_priv,
+					IN  mlx_mac_address 	mac
+					);
+mlx_status
+nodnic_port_thin_init(
+		IN nodnic_device_priv	*device_priv,
+		IN nodnic_port_priv		*port_priv,
+		IN mlx_uint8			port_index
+		);
+
+mlx_status
+nodnic_port_set_promisc(
+		IN nodnic_port_priv		*port_priv,
+		IN mlx_boolean			value
+		);
+
+mlx_status
+nodnic_port_set_promisc_multicast(
+		IN nodnic_port_priv		*port_priv,
+		IN mlx_boolean			value
+		);
+
+mlx_status
+nodnic_port_init(
+		IN nodnic_port_priv		*port_priv
+		);
+
+mlx_status
+nodnic_port_close(
+		IN nodnic_port_priv		*port_priv
+		);
+
+mlx_status
+nodnic_port_enable_dma(
+		IN nodnic_port_priv		*port_priv
+		);
+
+mlx_status
+nodnic_port_disable_dma(
+		IN nodnic_port_priv		*port_priv
+		);
+
+mlx_status
+nodnic_port_read_reset_needed(
+						IN nodnic_port_priv		*port_priv,
+						OUT mlx_boolean			*reset_needed
+						);
+
+mlx_status
+nodnic_port_read_port_management_change_event(
+						IN nodnic_port_priv		*port_priv,
+						OUT mlx_boolean			*change_event
+						);
+#endif /* STUB_NODNIC_PORT_H_ */

src/drivers/infiniband/mlx_nodnic/src/mlx_cmd.c

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2015 Mellanox Technologies Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include "../include/mlx_cmd.h"
+#include "../../mlx_utils/include/public/mlx_pci_gw.h"
+#include "../../mlx_utils/include/public/mlx_bail.h"
+#include "../../mlx_utils/include/public/mlx_pci.h"
+#include "../../mlx_utils/include/public/mlx_logging.h"
+
+mlx_status
+nodnic_cmd_read(
+				IN nodnic_device_priv *device_priv,
+				IN mlx_uint32 address,
+				OUT mlx_pci_gw_buffer *buffer
+				)
+{
+	mlx_status 		status = MLX_SUCCESS;
+	mlx_utils 		*utils = NULL;
+
+	if ( device_priv == NULL || buffer == NULL ) {
+		status = MLX_INVALID_PARAMETER;
+		goto bad_param;
+	}
+
+	utils = device_priv->utils;
+
+	status = mlx_pci_gw_read(utils, PCI_GW_SPACE_NODNIC, address, buffer);
+	MLX_CHECK_STATUS(device_priv, status, read_error,"mlx_pci_gw_read failed");
+
+read_error:
+bad_param:
+	return status;
+}
+
+mlx_status
+nodnic_cmd_write(
+				IN nodnic_device_priv *device_priv,
+				IN mlx_uint32 address,
+				IN mlx_pci_gw_buffer buffer
+				)
+{
+	mlx_status 		status = MLX_SUCCESS;
+	mlx_utils 		*utils = NULL;
+
+
+	if ( device_priv == NULL ) {
+		status = MLX_INVALID_PARAMETER;
+		goto bad_param;
+	}
+
+	utils = device_priv->utils;
+
+
+	status = mlx_pci_gw_write(utils, PCI_GW_SPACE_NODNIC, address, buffer);
+	MLX_CHECK_STATUS(device_priv, status, write_error,"mlx_pci_gw_write failed");
+write_error:
+bad_param:
+	return status;
+}

src/drivers/infiniband/mlx_nodnic/src/mlx_device.c

@@ -0,0 +1,339 @@
+/*
+ * Copyright (C) 2015 Mellanox Technologies Ltd.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+FILE_LICENCE ( GPL2_OR_LATER );
+
+#include "../include/mlx_device.h"
+#include "../include/mlx_cmd.h"
+#include "../../mlx_utils/include/public/mlx_bail.h"
+#include "../../mlx_utils/include/public/mlx_pci.h"
+#include "../../mlx_utils/include/public/mlx_memory.h"
+#include "../../mlx_utils/include/public/mlx_logging.h"
+
+#define CHECK_BIT(field, offset)	(((field) & ((mlx_uint32)1 << (offset))) != 0)
+
+static
+mlx_status
+check_nodnic_interface_supported(
+							IN nodnic_device_priv* device_priv,
+							OUT mlx_boolean *out
+							)
+{
+	mlx_status status = MLX_SUCCESS;
+	mlx_uint32	output = 0;
+	status = nodnic_cmd_read(device_priv, NODNIC_NIC_INTERFACE_SUPPORTED_OFFSET,
+			&output);
+	MLX_FATAL_CHECK_STATUS(status, read_error, "failed to read nic_interface_supported");
+	*out = CHECK_BIT(output, NODNIC_NIC_INTERFACE_SUPPORTED_BIT);
+read_error:
+	return status;
+}
+
+static
+mlx_status
+wait_for_device_initialization(
+							IN nodnic_device_priv* device_priv
+							)
+{
+	mlx_status status = MLX_SUCCESS;
+	mlx_uint8 try = 0;
+	mlx_uint32			buffer = 0;
+
+#define CHECK_DEVICE_INIT_TRIES 10
+	for( ; try < CHECK_DEVICE_INIT_TRIES ; try++){
+		status = nodnic_cmd_read(device_priv, NODNIC_INITIALIZING_OFFSET, &buffer);
+		MLX_CHECK_STATUS(device_priv, status, read_error, "failed to read initializing");
+		if( !CHECK_BIT(buffer, NODNIC_INITIALIZING_BIT)){
+			goto init_done;
+		}
+		mlx_utils_delay_in_ms(100);
+	}
+	status = MLX_FAILED;
+read_error:
+init_done:
+	return status;
+}
+
+static
+mlx_status
+disable_nodnic_inteface(
+						IN nodnic_device_priv *device_priv
+						)
+{
+	mlx_status 			status = MLX_SUCCESS;
+	mlx_uint32			buffer = 0;
+
+	buffer = (1 << NODNIC_DISABLE_INTERFACE_BIT);
+	status = nodnic_cmd_write(device_priv, NODNIC_CMDQ_PHY_ADDR_LOW_OFFSET, buffer);
+	MLX_FATAL_CHECK_STATUS(status, write_err, "failed to write cmdq_phy_addr + nic_interface");
+
+	status = wait_for_device_initialization(device_priv);
+	MLX_FATAL_CHECK_STATUS(status, init_err, "failed to initialize device");
+init_err:
+write_err:
+	return status;
+}
+static
+mlx_status
+nodnic_device_start_nodnic(
+				IN nodnic_device_priv *device_priv
+				)
+{
+	mlx_status 			status = MLX_SUCCESS;
+	mlx_uint32			buffer = 0;
+	mlx_boolean			nodnic_supported = 0;
+
+	status = wait_for_device_initialization(device_priv);
+	MLX_FATAL_CHECK_STATUS(status, wait_for_fw_err, "failed to initialize device");
+
+	status = check_nodnic_interface_supported(device_priv, &nodnic_supported);
+	MLX_FATAL_CHECK_STATUS(status, read_err,"failed to check nic_interface_supported");
+
+	if(	nodnic_supported == 0 ){
+		status = MLX_UNSUPPORTED;
+		goto nodnic_unsupported;
+	}
+	buffer =  (1 << NODNIC_NIC_INTERFACE_BIT);
+	status = nodnic_cmd_write(device_priv, NODNIC_NIC_INTERFACE_OFFSET, buffer);
+	MLX_FATAL_CHECK_STATUS(status, write_err, "failed to write cmdq_phy_addr + nic_interface");
+
+	status = wait_for_device_initialization(device_priv);
+	MLX_FATAL_CHECK_STATUS(status, init_err, "failed to initialize device");
+init_err:
+read_err:
+write_err:
+nodnic_unsupported:
+wait_for_fw_err:
+	return status;
+}
+
+static
+mlx_status
+nodnic_device_get_nodnic_data(
+				IN nodnic_device_priv *device_priv
+				)
+{
+	mlx_status 			status = MLX_SUCCESS;
+	mlx_uint32			buffer = 0;
+
+	status = nodnic_cmd_read(device_priv, NODNIC_LOCATION_OFFSET, &device_priv->device_offset);
+	MLX_FATAL_CHECK_STATUS(status, nodnic_offset_read_err, "failed to read nodnic offset");
+
+	status = nodnic_cmd_read(device_priv,
+			device_priv->device_offset + NODNIC_REVISION_OFFSET, &buffer);
+	MLX_FATAL_CHECK_STATUS(status, nodnic_revision_read_err, "failed to read nodnic revision");
+
+	device_priv->nodnic_revision = (buffer >> 24) & 0xFF;
+	if( device_priv->nodnic_revision != NODIC_SUPPORTED_REVISION ){
+		MLX_DEBUG_ERROR(device_priv, "nodnic revision not supported\n");
+		status = MLX_UNSUPPORTED;
+		goto unsupported_revision;
+	}
+
+	status = nodnic_cmd_read(device_priv,
+			device_priv->device_offset + NODNIC_HARDWARE_FORMAT_OFFSET, &buffer);
+	MLX_FATAL_CHECK_STATUS(status, nodnic_hardware_format_read_err, "failed to read nodnic revision");
+	device_priv->hardware_format = (buffer >> 16) & 0xFF;
+
+	return status;
+
+unsupported_revision:
+nodnic_hardware_format_read_err:
+nodnic_offset_read_err:
+nodnic_revision_read_err:
+	disable_nodnic_inteface(device_priv);
+	return status;
+}
+
+mlx_status
+nodnic_device_clear_int (
+				IN nodnic_device_priv *device_priv
+				)
+{
+	mlx_status 			status = MLX_SUCCESS;
+	mlx_uint32			disable = 1;
+#ifndef DEVICE_CX3
+	status = nodnic_cmd_write(device_priv, NODNIC_NIC_DISABLE_INT_OFFSET, disable);
+	MLX_CHECK_STATUS(device_priv, status, clear_int_done, "failed writing to disable_bit");
+#else
+	mlx_utils *utils = device_priv->utils;
+	mlx_uint64 clear_int = (mlx_uint64)(device_priv->crspace_clear_int);
+	mlx_uint32 swapped = 0;
+
+	if (device_priv->device_cap.crspace_doorbells == 0) {
+		status = nodnic_cmd_write(device_priv, NODNIC_NIC_DISABLE_INT_OFFSET, disable);
+		MLX_CHECK_STATUS(device_priv, status, clear_int_done, "failed writing to disable_bit");
+	} else {
+		/* Write the new index and update FW that new data was submitted */
+		disable = 0x80000000;
+		mlx_memory_cpu_to_be32(utils, disable, &swapped);
+		mlx_pci_mem_write (utils, MlxPciWidthUint32, 0, clear_int, 1, &swapped);
+		mlx_pci_mem_read (utils, MlxPciWidthUint32, 0, clear_int, 1, &swapped);
+	}
+#endif
+clear_int_done:
+	return status;
+}
+
+mlx_status
+nodnic_device_init(
+				IN nodnic_device_priv *device_priv
+				)
+{
+	mlx_status 			status = MLX_SUCCESS;
+
+	if( device_priv == NULL ){
+		status = MLX_INVALID_PARAMETER;
+		goto parm_err;
+	}
+	status = nodnic_device_start_nodnic(device_priv);
+	MLX_FATAL_CHECK_STATUS(status, start_nodnic_err, "nodnic_device_start_nodnic failed");
+
+	status = nodnic_device_get_nodnic_data(device_priv);
+	MLX_FATAL_CHECK_STATUS(status, data_err, "nodnic_device_get_nodnic_data failed");
+	return status;
+data_err:
+start_nodnic_err:
+parm_err:
+	return status;
+}
+
+mlx_status
+nodnic_device_teardown(
+				IN nodnic_device_priv *device_priv
+				)
+{
+	mlx_status 			status = MLX_SUCCESS;
+	status = disable_nodnic_inteface(device_priv);
+	MLX_FATAL_CHECK_STATUS(status, disable_failed, "failed to disable nodnic interface");
+disable_failed:
+	return status;
+}
+
+mlx_status
+nodnic_device_get_cap(
+				IN nodnic_device_priv *device_priv
+				)
+{
+	mlx_status 					status = MLX_SUCCESS;
+	nodnic_device_capabilites 	*device_cap = NULL;
+	mlx_uint32					buffer = 0;
+	mlx_uint64					guid_l = 0;
+	mlx_uint64					guid_h = 0;
+	if( device_priv == NULL ){
+		status = MLX_INVALID_PARAMETER;
+		goto parm_err;
+	}
+
+	device_cap = &device_priv->device_cap;
+
+	//get device capabilities
+	status = nodnic_cmd_read(device_priv, device_priv->device_offset + 0x0, &buffer);
+	MLX_FATAL_CHECK_STATUS(status, read_err, "failed to read nodnic first dword");
+
+#define NODNIC_DEVICE_SUPPORT_MAC_FILTERS_OFFSET 15
+#define NODNIC_DEVICE_SUPPORT_PROMISC_FILTER_OFFSET 14
+#define NODNIC_DEVICE_SUPPORT_PROMISC_MULT_FILTER_OFFSET 13
+#define NODNIC_DEVICE_LOG_WORKING_BUFFER_SIZE_OFFSET 8
+#define NODNIC_DEVICE_LOG_WORKING_BUFFER_SIZE_MASK 0x7
+#define NODNIC_DEVICE_LOG_PKEY_TABLE_SIZE_OFFSET 4
+#define NODNIC_DEVICE_LOG_PKEY_TABLE_SIZE_MASK 0xF
+#define NODNIC_DEVICE_NUM_PORTS_OFFSET 0
+	device_cap->support_mac_filters = CHECK_BIT(buffer, NODNIC_DEVICE_SUPPORT_MAC_FILTERS_OFFSET);
+
+	device_cap->support_promisc_filter = CHECK_BIT(buffer, NODNIC_DEVICE_SUPPORT_PROMISC_FILTER_OFFSET);
+
+	device_cap->support_promisc_multicast_filter = CHECK_BIT(buffer, NODNIC_DEVICE_SUPPORT_PROMISC_MULT_FILTER_OFFSET);
+
+	device_cap->log_working_buffer_size =
+			(buffer >> NODNIC_DEVICE_LOG_WORKING_BUFFER_SIZE_OFFSET) & NODNIC_DEVICE_LOG_WORKING_BUFFER_SIZE_MASK;
+
+	device_cap->log_pkey_table_size =
+			(buffer >> NODNIC_DEVICE_LOG_PKEY_TABLE_SIZE_OFFSET) & NODNIC_DEVICE_LOG_PKEY_TABLE_SIZE_MASK;
+
+	device_cap->num_ports = CHECK_BIT(buffer, NODNIC_DEVICE_NUM_PORTS_OFFSET) + 1;
+
+#ifdef DEVICE_CX3
+#define NODNIC_DEVICE_CRSPACE_DB_OFFSET 12
+	device_cap->crspace_doorbells = CHECK_BIT(buffer, NODNIC_DEVICE_CRSPACE_DB_OFFSET);
+#endif
+
+	status = nodnic_cmd_read(device_priv, device_priv->device_offset + 0x4, &buffer);
+	MLX_FATAL_CHECK_STATUS(status, read_err, "failed to read nodnic second dword");
+
+#define NODNIC_DEVICE_LOG_MAX_RING_SIZE_OFFSET 24
+#define NODNIC_DEVICE_LOG_MAX_RING_SIZE_MASK 0x3F
+#define NODNIC_DEVICE_PD_MASK 0xFFFFFF
+	device_cap->log_max_ring_size =
+			(buffer >> NODNIC_DEVICE_LOG_MAX_RING_SIZE_OFFSET) & NODNIC_DEVICE_LOG_MAX_RING_SIZE_MASK;
+
+	//get device magic numbers
+	device_priv->pd = buffer & NODNIC_DEVICE_PD_MASK;
+
+	status = nodnic_cmd_read(device_priv, device_priv->device_offset + 0x8, &buffer);
+	MLX_FATAL_CHECK_STATUS(status, read_err, "failed to read nodnic third dword");
+	device_priv->lkey = buffer;
+
+#ifdef DEVICE_CX3
+	if ( device_cap->crspace_doorbells ) {
+		status = nodnic_cmd_read(device_priv, device_priv->device_offset + 0x18, &buffer);
+		MLX_FATAL_CHECK_STATUS(status, read_err, "failed to read nodnic_crspace_clear_int address");
+		device_priv->crspace_clear_int = device_priv->utils->config + buffer;
+	}
+#endif
+
+	status = nodnic_cmd_read(device_priv, device_priv->device_offset + 0x10, (mlx_uint32*)&guid_h);
+	MLX_FATAL_CHECK_STATUS(status, read_err, "failed to read nodnic guid_h");
+	status = nodnic_cmd_read(device_priv, device_priv->device_offset + 0x14, (mlx_uint32*)&guid_l);
+	MLX_FATAL_CHECK_STATUS(status, read_err, "failed to read nodnic guid_l");
+	device_priv->device_guid = guid_l | (guid_h << 32);
+read_err:
+parm_err:
+	return status;
+}
+
+mlx_status
+nodnic_device_get_fw_version(
+				IN nodnic_device_priv *device_priv,
+				OUT mlx_uint16		*fw_ver_minor,
+				OUT mlx_uint16  	*fw_ver_sub_minor,
+				OUT mlx_uint16  	*fw_ver_major
+				){
+	mlx_status 		status = MLX_SUCCESS;
+	mlx_uint32		buffer = 0;
+
+	if( device_priv == NULL ){
+		status = MLX_INVALID_PARAMETER;
+		goto parm_err;
+	}
+
+	status = nodnic_cmd_read(device_priv, 0x0, &buffer);
+	MLX_CHECK_STATUS(device_priv, status, read_err, "failed to read fw revision major and minor");
+
+	*fw_ver_minor = (mlx_uint16)(buffer >> 16);
+	*fw_ver_major = (mlx_uint16)buffer;
+
+	status = nodnic_cmd_read(device_priv, 0x4, &buffer);
+	MLX_CHECK_STATUS(device_priv, status, read_err, "failed to read fw revision sub minor");
+
+	*fw_ver_sub_minor = (mlx_uint16)buffer;
+read_err:
+parm_err:
+	return status;
+}