[infiniband] Update all other MAD users to use a management interface

This commit is contained in:
Michael Brown
2009-08-06 01:18:38 +01:00
parent 44251ebb9a
commit 34bfc04e4c
12 changed files with 752 additions and 997 deletions

View File

@@ -24,10 +24,8 @@ FILE_LICENCE ( GPL2_OR_LATER );
#include <byteswap.h>
#include <errno.h>
#include <assert.h>
#include <gpxe/list.h>
#include <gpxe/process.h>
#include <gpxe/infiniband.h>
#include <gpxe/ib_gma.h>
#include <gpxe/ib_mi.h>
#include <gpxe/ib_pathrec.h>
#include <gpxe/ib_cm.h>
@@ -38,64 +36,170 @@ FILE_LICENCE ( GPL2_OR_LATER );
*
*/
/** An outstanding connection request */
struct ib_cm_request {
/** List of all outstanding requests */
struct list_head list;
/** Local communication ID */
uint32_t local_id;
/** Remote communication ID */
uint32_t remote_id;
/** Queue pair */
struct ib_queue_pair *qp;
/** Target service ID */
struct ib_gid_half service_id;
/** Connection process */
struct process process;
/** Notification handler
*
* @v qp Queue pair
* @v rc Connection status code
* @v private_data Private data
* @v private_data_len Length of private data
*/
void ( * notify ) ( struct ib_queue_pair *qp, int rc,
void *private_data, size_t private_data_len );
/** Private data length */
size_t private_data_len;
/** Private data */
uint8_t private_data[0];
};
/** List of all outstanding connection requests */
static LIST_HEAD ( ib_cm_requests );
/**
* Send connection request
* Send "ready to use" response
*
* @v request Connection request
* @v ibdev Infiniband device
* @v mi Management interface
* @v conn Connection
* @v av Address vector
* @ret rc Return status code
*/
static int ib_cm_send_request ( struct ib_cm_request *request ) {
struct ib_queue_pair *qp = request->qp;
struct ib_device *ibdev = qp->ibdev;
struct ib_gma *gma = ibdev->gma;
static int ib_cm_send_rtu ( struct ib_device *ibdev,
struct ib_mad_interface *mi,
struct ib_connection *conn,
struct ib_address_vector *av ) {
union ib_mad mad;
struct ib_mad_cm *cm = &mad.cm;
struct ib_cm_connect_request *connect_req =
&cm->cm_data.connect_request;
size_t private_data_len;
struct ib_cm_ready_to_use *ready =
&mad.cm.cm_data.ready_to_use;
int rc;
/* Construct "ready to use" response */
memset ( &mad, 0, sizeof ( mad ) );
mad.hdr.mgmt_class = IB_MGMT_CLASS_CM;
mad.hdr.class_version = IB_CM_CLASS_VERSION;
mad.hdr.method = IB_MGMT_METHOD_SEND;
mad.hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE );
ready->local_id = htonl ( conn->local_id );
ready->remote_id = htonl ( conn->remote_id );
if ( ( rc = ib_mi_send ( ibdev, mi, &mad, av ) ) != 0 ){
DBGC ( conn, "CM %p could not send RTU: %s\n",
conn, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Handle connection request transaction completion
*
* @v ibdev Infiniband device
* @v mi Management interface
* @v madx Management transaction
* @v rc Status code
* @v mad Received MAD (or NULL on error)
* @v av Source address vector (or NULL on error)
*/
static void ib_cm_req_complete ( struct ib_device *ibdev,
struct ib_mad_interface *mi,
struct ib_mad_transaction *madx,
int rc, union ib_mad *mad,
struct ib_address_vector *av ) {
struct ib_connection *conn = ib_madx_get_ownerdata ( madx );
struct ib_queue_pair *qp = conn->qp;
struct ib_cm_common *common = &mad->cm.cm_data.common;
struct ib_cm_connect_reply *connect_rep =
&mad->cm.cm_data.connect_reply;
struct ib_cm_connect_reject *connect_rej =
&mad->cm.cm_data.connect_reject;
void *private_data = NULL;
size_t private_data_len = 0;
/* Report failures */
if ( rc != 0 ) {
DBGC ( conn, "CM %p connection request failed: %s\n",
conn, strerror ( rc ) );
goto out;
}
/* Record remote communication ID */
conn->remote_id = ntohl ( common->local_id );
/* Handle response */
switch ( mad->hdr.attr_id ) {
case htons ( IB_CM_ATTR_CONNECT_REPLY ) :
/* Extract fields */
qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 );
qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 );
private_data = &connect_rep->private_data;
private_data_len = sizeof ( connect_rep->private_data );
DBGC ( conn, "CM %p connected to QPN %lx PSN %x\n",
conn, qp->av.qpn, qp->send.psn );
/* Modify queue pair */
if ( ( rc = ib_modify_qp ( ibdev, qp ) ) != 0 ) {
DBGC ( conn, "CM %p could not modify queue pair: %s\n",
conn, strerror ( rc ) );
goto out;
}
/* Send "ready to use" reply */
if ( ( rc = ib_cm_send_rtu ( ibdev, mi, conn, av ) ) != 0 ) {
/* Treat as non-fatal */
rc = 0;
}
break;
case htons ( IB_CM_ATTR_CONNECT_REJECT ) :
/* Extract fields */
DBGC ( conn, "CM %p connection rejected (reason %d)\n",
conn, ntohs ( connect_rej->reason ) );
private_data = &connect_rej->private_data;
private_data_len = sizeof ( connect_rej->private_data );
rc = -ENOTCONN;
break;
default:
DBGC ( conn, "CM %p unexpected response (attribute %04x)\n",
conn, ntohs ( mad->hdr.attr_id ) );
rc = -EIO;
break;
}
out:
/* Destroy the completed transaction */
ib_destroy_madx ( ibdev, ibdev->gsi, madx );
conn->madx = NULL;
/* Hand off to the upper completion handler */
conn->op->changed ( ibdev, qp, conn, rc, private_data,
private_data_len );
}
/** Connection request operations */
static struct ib_mad_transaction_operations ib_cm_req_op = {
.complete = ib_cm_req_complete,
};
/**
* Handle connection path transaction completion
*
* @v ibdev Infiniband device
* @v path Path
* @v rc Status code
* @v av Address vector, or NULL on error
*/
static void ib_cm_path_complete ( struct ib_device *ibdev,
struct ib_path *path, int rc,
struct ib_address_vector *av ) {
struct ib_connection *conn = ib_path_get_ownerdata ( path );
struct ib_queue_pair *qp = conn->qp;
union ib_mad mad;
struct ib_cm_connect_request *connect_req =
&mad.cm.cm_data.connect_request;
size_t private_data_len;
/* Report failures */
if ( rc != 0 ) {
DBGC ( conn, "CM %p path lookup failed: %s\n",
conn, strerror ( rc ) );
conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 );
goto out;
}
/* Update queue pair peer path */
memcpy ( &qp->av, av, sizeof ( qp->av ) );
/* Construct connection request */
memset ( cm, 0, sizeof ( *cm ) );
cm->mad_hdr.base_version = IB_MGMT_BASE_VERSION;
cm->mad_hdr.mgmt_class = IB_MGMT_CLASS_CM;
cm->mad_hdr.class_version = IB_CM_CLASS_VERSION;
cm->mad_hdr.method = IB_MGMT_METHOD_SEND;
cm->mad_hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST );
connect_req->local_id = htonl ( request->local_id );
memcpy ( &connect_req->service_id, &request->service_id,
memset ( &mad, 0, sizeof ( mad ) );
mad.hdr.mgmt_class = IB_MGMT_CLASS_CM;
mad.hdr.class_version = IB_CM_CLASS_VERSION;
mad.hdr.method = IB_MGMT_METHOD_SEND;
mad.hdr.attr_id = htons ( IB_CM_ATTR_CONNECT_REQUEST );
connect_req->local_id = htonl ( conn->local_id );
memcpy ( &connect_req->service_id, &conn->service_id,
sizeof ( connect_req->service_id ) );
ib_get_hca_info ( ibdev, &connect_req->local_ca );
connect_req->local_qpn__responder_resources =
@@ -113,217 +217,116 @@ static int ib_cm_send_request ( struct ib_cm_request *request ) {
connect_req->max_cm_retries__srq =
( ( 0x0f << 4 ) | ( 0 << 3 ) );
connect_req->primary.local_lid = htons ( ibdev->lid );
connect_req->primary.remote_lid = htons ( request->qp->av.lid );
connect_req->primary.remote_lid = htons ( conn->qp->av.lid );
memcpy ( &connect_req->primary.local_gid, &ibdev->gid,
sizeof ( connect_req->primary.local_gid ) );
memcpy ( &connect_req->primary.remote_gid, &request->qp->av.gid,
memcpy ( &connect_req->primary.remote_gid, &conn->qp->av.gid,
sizeof ( connect_req->primary.remote_gid ) );
connect_req->primary.flow_label__rate =
htonl ( ( 0 << 12 ) | ( request->qp->av.rate << 0 ) );
htonl ( ( 0 << 12 ) | ( conn->qp->av.rate << 0 ) );
connect_req->primary.hop_limit = 0;
connect_req->primary.sl__subnet_local =
( ( request->qp->av.sl << 4 ) | ( 1 << 3 ) );
( ( conn->qp->av.sl << 4 ) | ( 1 << 3 ) );
connect_req->primary.local_ack_timeout = ( 0x13 << 3 );
private_data_len = request->private_data_len;
private_data_len = conn->private_data_len;
if ( private_data_len > sizeof ( connect_req->private_data ) )
private_data_len = sizeof ( connect_req->private_data );
memcpy ( &connect_req->private_data, &request->private_data,
memcpy ( &connect_req->private_data, &conn->private_data,
private_data_len );
/* Send request */
if ( ( rc = ib_gma_request ( gma, &mad, NULL, 1 ) ) != 0 ) {
DBGC ( gma, "GMA %p could not send connection request: %s\n",
gma, strerror ( rc ) );
return rc;
/* Create connection request */
conn->madx = ib_create_madx ( ibdev, ibdev->gsi, &mad, NULL,
&ib_cm_req_op );
if ( ! conn->madx ) {
DBGC ( conn, "CM %p could not create connection request\n",
conn );
conn->op->changed ( ibdev, qp, conn, rc, NULL, 0 );
goto out;
}
ib_madx_set_ownerdata ( conn->madx, conn );
return 0;
out:
/* Destroy the completed transaction */
ib_destroy_path ( ibdev, path );
conn->path = NULL;
}
/**
* Connection request process step
*
* @v process Connection request process
*/
static void ib_cm_step ( struct process *process ) {
struct ib_cm_request *request =
container_of ( process, struct ib_cm_request, process );
struct ib_queue_pair *qp = request->qp;
struct ib_device *ibdev = qp->ibdev;
int rc;
/* Wait until path can be resolved */
if ( ( rc = ib_resolve_path ( ibdev, &request->qp->av ) ) != 0 )
return;
/* Wait until request can be sent */
if ( ( rc = ib_cm_send_request ( request ) ) != 0 )
return;
/* Stop process */
process_del ( process );
}
/**
* Identify connection request by communication ID
*
* @v local_id Local communication ID
* @v remote_id Remote communication ID
* @ret request Connection request, or NULL
*/
static struct ib_cm_request * ib_cm_find_request ( uint32_t local_id,
uint32_t remote_id ) {
struct ib_cm_request *request;
list_for_each_entry ( request, &ib_cm_requests, list ) {
if ( request->local_id == local_id ) {
request->remote_id = remote_id;
return request;
}
}
return NULL;
}
/**
* Handle connection reply
*
* @v gma General management agent
* @v mad MAD
* @ret response MAD response
*/
static union ib_mad * ib_cm_connect_reply ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_cm_connect_reply *connect_rep =
&mad->cm.cm_data.connect_reply;
struct ib_cm_ready_to_use *ready =
&mad->cm.cm_data.ready_to_use;
struct ib_cm_request *request;
int rc;
/* Identify request */
request = ib_cm_find_request ( ntohl ( connect_rep->remote_id ),
ntohl ( connect_rep->local_id ) );
if ( ! request ) {
DBGC ( gma, "GMA %p received connection reply with unknown "
"ID %08x\n", gma, ntohl ( connect_rep->remote_id ) );
return NULL;
}
/* Extract fields */
request->qp->av.qpn = ( ntohl ( connect_rep->local_qpn ) >> 8 );
request->qp->send.psn = ( ntohl ( connect_rep->starting_psn ) >> 8 );
DBGC ( gma, "GMA %p QPN %lx connected to QPN %lx PSN %x\n", gma,
request->qp->qpn, request->qp->av.qpn, request->qp->send.psn );
/* Modify queue pair */
if ( ( rc = ib_modify_qp ( request->qp->ibdev, request->qp ) ) != 0 ) {
DBGC ( gma, "GMA %p QPN %lx could not modify queue pair: %s\n",
gma, request->qp->qpn, strerror ( rc ) );
return NULL;
}
/* Inform recipient that we are now connected */
request->notify ( request->qp, 0, &connect_rep->private_data,
sizeof ( connect_rep->private_data ) );
/* Construct ready to use reply */
mad->hdr.attr_id = htons ( IB_CM_ATTR_READY_TO_USE );
memset ( ready, 0, sizeof ( *ready ) );
ready->local_id = htonl ( request->local_id );
ready->remote_id = htonl ( request->remote_id );
return mad;
}
/**
* Handle connection rejection
*
* @v gma General management agent
* @v mad MAD
* @ret response MAD response
*/
static union ib_mad * ib_cm_connect_reject ( struct ib_gma *gma,
union ib_mad *mad ) {
struct ib_cm_connect_reject *connect_rej =
&mad->cm.cm_data.connect_reject;
struct ib_cm_request *request;
uint16_t reason;
/* Identify request */
request = ib_cm_find_request ( ntohl ( connect_rej->remote_id ),
ntohl ( connect_rej->local_id ) );
if ( ! request ) {
DBGC ( gma, "GMA %p received connection rejection with "
"unknown ID %08x\n", gma,
ntohl ( connect_rej->remote_id ) );
return NULL;
}
/* Extract fields */
reason = ntohs ( connect_rej->reason );
DBGC ( gma, "GMA %p QPN %lx connection rejected (reason %d)\n",
gma, request->qp->qpn, reason );
/* Inform recipient that we are now disconnected */
request->notify ( request->qp, -ENOTCONN, &connect_rej->private_data,
sizeof ( connect_rej->private_data ) );
return NULL;
}
/** Communication management MAD handlers */
struct ib_gma_handler ib_cm_handlers[] __ib_gma_handler = {
{
.mgmt_class = IB_MGMT_CLASS_CM,
.class_version = IB_CM_CLASS_VERSION,
.method = IB_MGMT_METHOD_SEND,
.attr_id = htons ( IB_CM_ATTR_CONNECT_REPLY ),
.handle = ib_cm_connect_reply,
},
{
.mgmt_class = IB_MGMT_CLASS_CM,
.class_version = IB_CM_CLASS_VERSION,
.method = IB_MGMT_METHOD_SEND,
.attr_id = htons ( IB_CM_ATTR_CONNECT_REJECT ),
.handle = ib_cm_connect_reject,
},
/** Connection path operations */
static struct ib_path_operations ib_cm_path_op = {
.complete = ib_cm_path_complete,
};
/**
* Connect to remote QP
* Create connection to remote QP
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v dgid Target GID
* @v service_id Target service ID
* @v private_data Private data
* @v private_data_len Length of private data
* @ret rc Return status code
* @v private_data Connection request private data
* @v private_data_len Length of connection request private data
* @v op Connection operations
* @ret conn Connection
*/
int ib_cm_connect ( struct ib_queue_pair *qp, struct ib_gid *dgid,
struct ib_gid_half *service_id,
void *private_data, size_t private_data_len,
void ( * notify ) ( struct ib_queue_pair *qp, int rc,
void *private_data,
size_t private_data_len ) ) {
struct ib_cm_request *request;
struct ib_connection *
ib_create_conn ( struct ib_device *ibdev, struct ib_queue_pair *qp,
struct ib_gid *dgid, struct ib_gid_half *service_id,
void *private_data, size_t private_data_len,
struct ib_connection_operations *op ) {
struct ib_connection *conn;
/* Allocate and initialise request */
request = zalloc ( sizeof ( *request ) + private_data_len );
if ( ! request )
return -ENOMEM;
list_add ( &request->list, &ib_cm_requests );
request->local_id = random();
request->qp = qp;
conn = zalloc ( sizeof ( *conn ) + private_data_len );
if ( ! conn )
goto err_alloc_conn;
conn->ibdev = ibdev;
conn->qp = qp;
memset ( &qp->av, 0, sizeof ( qp->av ) );
qp->av.gid_present = 1;
memcpy ( &qp->av.gid, dgid, sizeof ( qp->av.gid ) );
memcpy ( &request->service_id, service_id,
sizeof ( request->service_id ) );
request->notify = notify;
request->private_data_len = private_data_len;
memcpy ( &request->private_data, private_data, private_data_len );
process_init ( &request->process, ib_cm_step, NULL );
conn->local_id = random();
memcpy ( &conn->service_id, service_id, sizeof ( conn->service_id ) );
conn->op = op;
conn->private_data_len = private_data_len;
memcpy ( &conn->private_data, private_data, private_data_len );
return 0;
/* Create path */
conn->path = ib_create_path ( ibdev, &qp->av, &ib_cm_path_op );
if ( ! conn->path )
goto err_create_path;
ib_path_set_ownerdata ( conn->path, conn );
DBGC ( conn, "CM %p created for IBDEV %p QPN %lx\n",
conn, ibdev, qp->qpn );
DBGC ( conn, "CM %p connecting to %08x:%08x:%08x:%08x %08x:%08x\n",
conn, ntohl ( dgid->u.dwords[0] ), ntohl ( dgid->u.dwords[1] ),
ntohl ( dgid->u.dwords[2] ), ntohl ( dgid->u.dwords[3] ),
ntohl ( service_id->u.dwords[0] ),
ntohl ( service_id->u.dwords[1] ) );
return conn;
ib_destroy_path ( ibdev, conn->path );
err_create_path:
free ( conn );
err_alloc_conn:
return NULL;
}
/**
* Destroy connection to remote QP
*
* @v ibdev Infiniband device
* @v qp Queue pair
* @v conn Connection
*/
void ib_destroy_conn ( struct ib_device *ibdev,
struct ib_queue_pair *qp __unused,
struct ib_connection *conn ) {
if ( conn->madx )
ib_destroy_madx ( ibdev, ibdev->gsi, conn->madx );
if ( conn->path )
ib_destroy_path ( ibdev, conn->path );
free ( conn );
}