[infiniband] Split subnet management agent client out into ib_smc.c

Not all Infiniband cards have embedded subnet management agents.
Split out the code that communicates with such an embedded SMA into a
separate ib_smc.c file, and have drivers call ib_smc_update()
explicitly when they suspect that the answers given by the embedded
SMA may have changed.

src/drivers/infiniband/arbel.c

@@ -33,6 +33,7 @@
 #include <gpxe/iobuf.h>
 #include <gpxe/netdevice.h>
 #include <gpxe/infiniband.h>
+#include <gpxe/ib_smc.h>
 #include "arbel.h"
 
 /**
@@ -482,6 +483,50 @@ arbel_cmd_map_fa ( struct arbel *arbel,
 			   0, map, 1, NULL );
 }
 
+/***************************************************************************
+ *
+ * MAD operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Issue management datagram
+ *
+ * @v ibdev		Infiniband device
+ * @v mad		Management datagram
+ * @ret rc		Return status code
+ */
+static int arbel_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
+	struct arbel *arbel = ib_get_drvdata ( ibdev );
+	union arbelprm_mad mad_ifc;
+	int rc;
+
+	linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
+			mad_size_mismatch );
+
+	/* Copy in request packet */
+	memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
+
+	/* Issue MAD */
+	if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
+					&mad_ifc ) ) != 0 ) {
+		DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
+		       arbel, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Copy out reply packet */
+	memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
+
+	if ( mad->hdr.status != 0 ) {
+		DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
+		       arbel, ntohs ( mad->hdr.status ) );
+		return -EIO;
+	}
+	return 0;
+}
+
 /***************************************************************************
  *
  * Completion queue operations
@@ -1394,6 +1439,9 @@ static void arbel_event_port_state_change ( struct arbel *arbel,
 		return;
 	}
 
+	/* Update MAD parameters */
+	ib_smc_update ( arbel->ibdev[port], arbel_mad );
+
 	/* Notify Infiniband core of link state change */
 	ib_link_state_changed ( arbel->ibdev[port] );
 }
@@ -1483,6 +1531,9 @@ static int arbel_open ( struct ib_device *ibdev ) {
 		return rc;
 	}
 
+	/* Update MAD parameters */
+	ib_smc_update ( ibdev, arbel_mad );
+
 	return 0;
 }
 
@@ -1598,51 +1649,6 @@ static void arbel_mcast_detach ( struct ib_device *ibdev,
 	}
 }
 
-/***************************************************************************
- *
- * MAD operations
- *
- ***************************************************************************
- */
-
-/**
- * Issue management datagram
- *
- * @v ibdev		Infiniband device
- * @v mad		Management datagram
- * @v len		Length of management datagram
- * @ret rc		Return status code
- */
-static int arbel_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
-		       size_t len ) {
-	struct arbel *arbel = ib_get_drvdata ( ibdev );
-	union arbelprm_mad mad_ifc;
-	int rc;
-
-	/* Copy in request packet */
-	memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
-	assert ( len <= sizeof ( mad_ifc.mad ) );
-	memcpy ( &mad_ifc.mad, mad, len );
-
-	/* Issue MAD */
-	if ( ( rc = arbel_cmd_mad_ifc ( arbel, ibdev->port,
-					&mad_ifc ) ) != 0 ) {
-		DBGC ( arbel, "Arbel %p could not issue MAD IFC: %s\n",
-		       arbel, strerror ( rc ) );
-		return rc;
-	}
-
-	/* Copy out reply packet */
-	memcpy ( mad, &mad_ifc.mad, len );
-
-	if ( mad->status != 0 ) {
-		DBGC ( arbel, "Arbel %p MAD IFC status %04x\n",
-		       arbel, ntohs ( mad->status ) );
-		return -EIO;
-	}
-	return 0;
-}
-
 /** Arbel Infiniband operations */
 static struct ib_device_operations arbel_ib_operations = {
 	.create_cq	= arbel_create_cq,
@@ -1658,7 +1664,6 @@ static struct ib_device_operations arbel_ib_operations = {
 	.close		= arbel_close,
 	.mcast_attach	= arbel_mcast_attach,
 	.mcast_detach	= arbel_mcast_detach,
-	.mad		= arbel_mad,
 };
 
 /***************************************************************************

src/drivers/infiniband/hermon.c

@@ -31,6 +31,7 @@
 #include <gpxe/iobuf.h>
 #include <gpxe/netdevice.h>
 #include <gpxe/infiniband.h>
+#include <gpxe/ib_smc.h>
 #include "hermon.h"
 
 /**
@@ -608,6 +609,50 @@ static void hermon_free_mtt ( struct hermon *hermon,
 			      mtt->num_pages );
 }
 
+/***************************************************************************
+ *
+ * MAD operations
+ *
+ ***************************************************************************
+ */
+
+/**
+ * Issue management datagram
+ *
+ * @v ibdev		Infiniband device
+ * @v mad		Management datagram
+ * @ret rc		Return status code
+ */
+static int hermon_mad ( struct ib_device *ibdev, union ib_mad *mad ) {
+	struct hermon *hermon = ib_get_drvdata ( ibdev );
+	union hermonprm_mad mad_ifc;
+	int rc;
+
+	linker_assert ( sizeof ( *mad ) == sizeof ( mad_ifc.mad ),
+			mad_size_mismatch );
+
+	/* Copy in request packet */
+	memcpy ( &mad_ifc.mad, mad, sizeof ( mad_ifc.mad ) );
+
+	/* Issue MAD */
+	if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
+					 &mad_ifc ) ) != 0 ) {
+		DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
+		       hermon, strerror ( rc ) );
+		return rc;
+	}
+
+	/* Copy out reply packet */
+	memcpy ( mad, &mad_ifc.mad, sizeof ( *mad ) );
+
+	if ( mad->hdr.status != 0 ) {
+		DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
+		       hermon, ntohs ( mad->hdr.status ) );
+		return -EIO;
+	}
+	return 0;
+}
+
 /***************************************************************************
  *
  * Completion queue operations
@@ -1377,6 +1422,9 @@ static void hermon_event_port_state_change ( struct hermon *hermon,
 		return;
 	}
 
+	/* Update MAD parameters */
+	ib_smc_update ( hermon->ibdev[port], hermon_mad );
+
 	/* Notify Infiniband core of link state change */
 	ib_link_state_changed ( hermon->ibdev[port] );
 }
@@ -1465,6 +1513,9 @@ static int hermon_open ( struct ib_device *ibdev ) {
 		return rc;
 	}
 
+	/* Update MAD parameters */
+	ib_smc_update ( ibdev, hermon_mad );
+
 	return 0;
 }
 
@@ -1579,51 +1630,6 @@ static void hermon_mcast_detach ( struct ib_device *ibdev,
 	}
 }
 
-/***************************************************************************
- *
- * MAD operations
- *
- ***************************************************************************
- */
-
-/**
- * Issue management datagram
- *
- * @v ibdev		Infiniband device
- * @v mad		Management datagram
- * @v len		Length of management datagram
- * @ret rc		Return status code
- */
-static int hermon_mad ( struct ib_device *ibdev, struct ib_mad_hdr *mad,
-			size_t len ) {
-	struct hermon *hermon = ib_get_drvdata ( ibdev );
-	union hermonprm_mad mad_ifc;
-	int rc;
-
-	/* Copy in request packet */
-	memset ( &mad_ifc, 0, sizeof ( mad_ifc ) );
-	assert ( len <= sizeof ( mad_ifc.mad ) );
-	memcpy ( &mad_ifc.mad, mad, len );
-
-	/* Issue MAD */
-	if ( ( rc = hermon_cmd_mad_ifc ( hermon, ibdev->port,
-					 &mad_ifc ) ) != 0 ) {
-		DBGC ( hermon, "Hermon %p could not issue MAD IFC: %s\n",
-		       hermon, strerror ( rc ) );
-		return rc;
-	}
-
-	/* Copy out reply packet */
-	memcpy ( mad, &mad_ifc.mad, len );
-
-	if ( mad->status != 0 ) {
-		DBGC ( hermon, "Hermon %p MAD IFC status %04x\n",
-		       hermon, ntohs ( mad->status ) );
-		return -EIO;
-	}
-	return 0;
-}
-
 /** Hermon Infiniband operations */
 static struct ib_device_operations hermon_ib_operations = {
 	.create_cq	= hermon_create_cq,
@@ -1639,7 +1645,6 @@ static struct ib_device_operations hermon_ib_operations = {
 	.close		= hermon_close,
 	.mcast_attach	= hermon_mcast_attach,
 	.mcast_detach	= hermon_mcast_detach,
-	.mad		= hermon_mad,
 };
 
 /***************************************************************************

src/drivers/infiniband/ib_smc.c

@@ -0,0 +1,166 @@
+/*
+ * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <byteswap.h>
+#include <gpxe/infiniband.h>
+#include <gpxe/ib_smc.h>
+
+/**
+ * @file
+ *
+ * Infiniband Subnet Management Client
+ *
+ */
+
+/**
+ * Get port information
+ *
+ * @v ibdev		Infiniband device
+ * @v local_mad		Method for issuing local MADs
+ * @v mad		Management datagram to fill in
+ * @ret rc		Return status code
+ */
+static int ib_smc_get_port_info ( struct ib_device *ibdev,
+				  ib_local_mad_t local_mad,
+				  union ib_mad *mad ) {
+	int rc;
+
+	/* Construct MAD */
+	memset ( mad, 0, sizeof ( *mad ) );
+	mad->hdr.base_version = IB_MGMT_BASE_VERSION;
+	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+	mad->hdr.class_version = 1;
+	mad->hdr.method = IB_MGMT_METHOD_GET;
+	mad->hdr.attr_id = htons ( IB_SMP_ATTR_PORT_INFO );
+	mad->hdr.attr_mod = htonl ( ibdev->port );
+
+	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
+		DBGC ( ibdev, "IBDEV %p could not get port info: %s\n",
+		       ibdev, strerror ( rc ) );
+		return rc;
+	}
+	return 0;
+}
+
+/**
+ * Get GUID information
+ *
+ * @v ibdev		Infiniband device
+ * @v local_mad		Method for issuing local MADs
+ * @v mad		Management datagram to fill in
+ * @ret rc		Return status code
+ */
+static int ib_smc_get_guid_info ( struct ib_device *ibdev,
+				  ib_local_mad_t local_mad,
+				  union ib_mad *mad ) {
+	int rc;
+
+	/* Construct MAD */
+	memset ( mad, 0, sizeof ( *mad ) );
+	mad->hdr.base_version = IB_MGMT_BASE_VERSION;
+	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+	mad->hdr.class_version = 1;
+	mad->hdr.method = IB_MGMT_METHOD_GET;
+	mad->hdr.attr_id = htons ( IB_SMP_ATTR_GUID_INFO );
+
+	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
+		DBGC ( ibdev, "IBDEV %p could not get GUID info: %s\n",
+		       ibdev, strerror ( rc ) );
+		return rc;
+	}
+	return 0;
+}
+
+/**
+ * Get partition key table
+ *
+ * @v ibdev		Infiniband device
+ * @v local_mad		Method for issuing local MADs
+ * @v mad		Management datagram to fill in
+ * @ret rc		Return status code
+ */
+static int ib_smc_get_pkey_table ( struct ib_device *ibdev,
+				   ib_local_mad_t local_mad,
+				   union ib_mad *mad ) {
+	int rc;
+
+	/* Construct MAD */
+	memset ( mad, 0, sizeof ( *mad ) );
+	mad->hdr.base_version = IB_MGMT_BASE_VERSION;
+	mad->hdr.mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
+	mad->hdr.class_version = 1;
+	mad->hdr.method = IB_MGMT_METHOD_GET;
+	mad->hdr.attr_id = htons ( IB_SMP_ATTR_PKEY_TABLE );
+
+	if ( ( rc = local_mad ( ibdev, mad ) ) != 0 ) {
+		DBGC ( ibdev, "IBDEV %p could not get pkey table: %s\n",
+		       ibdev, strerror ( rc ) );
+		return rc;
+	}
+	return 0;
+}
+
+/**
+ * Get MAD parameters
+ *
+ * @v ibdev		Infiniband device
+ * @v local_mad		Method for issuing local MADs
+ * @ret rc		Return status code
+ */
+int ib_smc_update ( struct ib_device *ibdev, ib_local_mad_t local_mad ) {
+	union ib_mad mad;
+	union ib_smp_data *smp = &mad.smp.smp_data;
+	int rc;
+
+	/* Port info gives us the link state, the first half of the
+	 * port GID and the SM LID.
+	 */
+	if ( ( rc = ib_smc_get_port_info ( ibdev, local_mad, &mad ) ) != 0 )
+		return rc;
+	ibdev->port_state =
+		( smp->port_info.link_speed_supported__port_state & 0x0f );
+	memcpy ( &ibdev->gid.u.half[0], smp->port_info.gid_prefix,
+		 sizeof ( ibdev->gid.u.half[0] ) );
+	ibdev->lid = ntohs ( smp->port_info.lid );
+	ibdev->sm_lid = ntohs ( smp->port_info.mastersm_lid );
+	ibdev->sm_sl = ( smp->port_info.neighbour_mtu__mastersm_sl & 0xf );
+
+	/* GUID info gives us the second half of the port GID */
+	if ( ( rc = ib_smc_get_guid_info ( ibdev, local_mad, &mad ) ) != 0 )
+		return rc;
+	memcpy ( &ibdev->gid.u.half[1], smp->guid_info.guid[0],
+		 sizeof ( ibdev->gid.u.half[1] ) );
+
+	/* Get partition key */
+	if ( ( rc = ib_smc_get_pkey_table ( ibdev, local_mad, &mad ) ) != 0 )
+		return rc;
+	ibdev->pkey = ntohs ( smp->pkey_table.pkey[0] );
+
+	DBGC ( ibdev, "IBDEV %p port GID is %08lx:%08lx:%08lx:%08lx\n", ibdev,
+	       htonl ( ibdev->gid.u.dwords[0] ),
+	       htonl ( ibdev->gid.u.dwords[1] ),
+	       htonl ( ibdev->gid.u.dwords[2] ),
+	       htonl ( ibdev->gid.u.dwords[3] ) );
+
+	return 0;
+}

src/drivers/net/ipoib.c

@@ -383,12 +383,13 @@ static int ipoib_get_path_record ( struct ipoib_device *ipoib,
 	path_record->sa_hdr.comp_mask[1] =
 		htonl ( IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID );
 	memcpy ( &path_record->dgid, gid, sizeof ( path_record->dgid ) );
-	memcpy ( &path_record->sgid, &ibdev->port_gid,
+	memcpy ( &path_record->sgid, &ibdev->gid,
 		 sizeof ( path_record->sgid ) );
 
 	/* Construct address vector */
 	memset ( &av, 0, sizeof ( av ) );
 	av.lid = ibdev->sm_lid;
+	av.sl = ibdev->sm_sl;
 	av.qpn = IB_SA_QPN;
 	av.qkey = IB_GLOBAL_QKEY;
 
@@ -443,12 +444,13 @@ static int ipoib_mc_member_record ( struct ipoib_device *ipoib,
 	mc_member_record->scope__join_state = 1;
 	memcpy ( &mc_member_record->mgid, gid,
 		 sizeof ( mc_member_record->mgid ) );
-	memcpy ( &mc_member_record->port_gid, &ibdev->port_gid,
+	memcpy ( &mc_member_record->port_gid, &ibdev->gid,
 		 sizeof ( mc_member_record->port_gid ) );
 
 	/* Construct address vector */
 	memset ( &av, 0, sizeof ( av ) );
 	av.lid = ibdev->sm_lid;
+	av.sl = ibdev->sm_sl;
 	av.qpn = IB_SA_QPN;
 	av.qkey = IB_GLOBAL_QKEY;
 
@@ -491,7 +493,7 @@ static int ipoib_transmit ( struct net_device *netdev,
 	/* Attempting transmission while link is down will put the
 	 * queue pair into an error state, so don't try it.
 	 */
-	if ( ! ibdev->link_up )
+	if ( ! ib_link_ok ( ibdev ) )
 		return -ENETUNREACH;
 
 	/* Construct address vector */
@@ -691,13 +693,13 @@ ipoib_meta_complete_recv ( struct ib_device *ibdev __unused,
 	}
 	mad = iobuf->data;
 
-	if ( mad->mad_hdr.status != 0 ) {
+	if ( mad->hdr.status != 0 ) {
 		DBGC ( ipoib, "IPoIB %p metadata RX err status %04x\n",
-		       ipoib, ntohs ( mad->mad_hdr.status ) );
+		       ipoib, ntohs ( mad->hdr.status ) );
 		goto done;
 	}
 
-	switch ( mad->mad_hdr.tid[0] ) {
+	switch ( mad->hdr.tid[0] ) {
 	case IPOIB_TID_GET_PATH_REC:
 		ipoib_recv_path_record ( ipoib, &mad->path_record );
 		break;
@@ -928,7 +930,7 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) {
 
 	/* Calculate GID portion of MAC address based on port GID */
 	mac = ( ( struct ipoib_mac * ) netdev->ll_addr );
-	memcpy ( &mac->gid, &ibdev->port_gid, sizeof ( mac->gid ) );
+	memcpy ( &mac->gid, &ibdev->gid, sizeof ( mac->gid ) );
 
 	/* Calculate broadcast GID based on partition key */
 	memcpy ( &ipoib->broadcast_gid, &ipv4_broadcast_gid,
@@ -936,7 +938,7 @@ static void ipoib_set_ib_params ( struct ipoib_device *ipoib ) {
 	ipoib->broadcast_gid.u.words[2] = htons ( ibdev->pkey );
 
 	/* Set net device link state to reflect Infiniband link state */
-	if ( ibdev->link_up ) {
+	if ( ib_link_ok ( ibdev ) ) {
 		netdev_link_up ( netdev );
 	} else {
 		netdev_link_down ( netdev );